mirror of
https://github.com/warmbly/warmbly.git
synced 2026-11-17 08:18:51 +00:00
feat: deploy stable releases to AWS automatically with provenance checks, a migration gate, alarm-watched waves and reverse rollback, move worker and consumer hosts in canary rollouts with host-side health rollback and opt-in env refresh, rotate instance keys inside AWS through a rotator task and the rotate-keys workflow, plan and apply the AWS Terraform from CI with approval for replacements, and add the admin Keys tab and the matching runbooks
This commit is contained in:
+356
-69
@@ -1,8 +1,26 @@
|
||||
// Deploys one published release to the AWS control plane: render every task
|
||||
// definition from the revision Terraform last registered, swap in the release
|
||||
// digests, gate on a dependency check and the migration, roll the services in
|
||||
// waves with a bake, then hand the workers a canary rollout. It never builds,
|
||||
// never reads a secret value, and restores everything it changed on failure.
|
||||
const {
|
||||
ECSClient, DescribeTaskDefinitionCommand, DescribeServicesCommand,
|
||||
ECSClient, DescribeTaskDefinitionCommand, DescribeServicesCommand, RegisterTaskDefinitionCommand,
|
||||
RunTaskCommand, DescribeTasksCommand, UpdateServiceCommand,
|
||||
} = require("@aws-sdk/client-ecs");
|
||||
const { SecretsManagerClient, DescribeSecretCommand, ListSecretVersionIdsCommand } = require("@aws-sdk/client-secrets-manager");
|
||||
const { CloudWatchClient, DescribeAlarmsCommand } = require("@aws-sdk/client-cloudwatch");
|
||||
const { CloudWatchLogsClient, GetLogEventsCommand } = require("@aws-sdk/client-cloudwatch-logs");
|
||||
const { createHash } = require("node:crypto");
|
||||
const { readFileSync } = require("node:fs");
|
||||
|
||||
const SERVICES = ["backend", "consumer", "forms", "tracking", "realtime"];
|
||||
const WAVES = [["backend"], ["consumer"], ["forms", "tracking", "realtime"]];
|
||||
const MODES = ["deploy", "rollback", "redeploy", "refresh-secrets"];
|
||||
const RELEASE_TAG = "warmbly-release";
|
||||
const VARIANT_TAG = "warmbly-image-variant";
|
||||
const TAG_RE = /^v(\d+)\.(\d+)\.(\d+)$/;
|
||||
const DIGEST_RE = /^sha256:[a-f0-9]{64}$/;
|
||||
const VERSION_RE = /^[A-Za-z0-9-]{32,64}$/;
|
||||
|
||||
function configuration(env) {
|
||||
const cluster = env.AWS_ECS_CLUSTER;
|
||||
@@ -10,22 +28,38 @@ function configuration(env) {
|
||||
if (!match) throw new Error("AWS_ECS_CLUSTER must be a us-east-1 cluster ARN");
|
||||
const prefix = env.AWS_ECS_SERVICE_PREFIX || match[2];
|
||||
if (!/^[\w-]+$/.test(prefix)) throw new Error("Invalid service prefix");
|
||||
const definitions = {};
|
||||
const counts = {};
|
||||
for (const name of ["migration", "backend", "consumer"]) {
|
||||
const value = env[`AWS_${name.toUpperCase()}_TASK_DEFINITION`];
|
||||
const expected = `arn:aws:ecs:us-east-1:${match[1]}:task-definition/${prefix}-${name}:`;
|
||||
if (!value?.startsWith(expected) || !/^\d+$/.test(value.slice(expected.length))) {
|
||||
throw new Error(`Provide an explicit ${name} task definition revision`);
|
||||
}
|
||||
definitions[name] = value;
|
||||
if (name !== "migration") {
|
||||
const raw = env[`AWS_${name.toUpperCase()}_DESIRED_COUNT`] || "preserve";
|
||||
if (raw !== "preserve" && !/^[0-5]$/.test(raw)) throw new Error("Desired counts must be preserve or 0 through 5");
|
||||
counts[name] = raw === "preserve" ? undefined : Number(raw);
|
||||
const mode = env.DEPLOY_MODE || "deploy";
|
||||
if (!MODES.includes(mode)) throw new Error(`DEPLOY_MODE must be one of ${MODES.join(", ")}`);
|
||||
const tag = (env.DEPLOY_TAG || "").trim();
|
||||
if (tag && !TAG_RE.test(tag)) throw new Error("DEPLOY_TAG must be a stable vX.Y.Z tag; prereleases never deploy");
|
||||
if (!tag && (mode === "deploy" || mode === "rollback")) throw new Error(`${mode} needs DEPLOY_TAG`);
|
||||
const raw = (env.DEPLOY_SERVICES || "all").trim();
|
||||
const services = raw === "all" ? [...SERVICES] : raw.split(",").map((s) => s.trim()).filter(Boolean);
|
||||
if (!services.length || services.some((s) => !SERVICES.includes(s))) throw new Error(`DEPLOY_SERVICES must be all or a list of ${SERVICES.join(", ")}`);
|
||||
let secretVersions = {};
|
||||
if (env.DEPLOY_SECRET_VERSIONS) {
|
||||
secretVersions = JSON.parse(env.DEPLOY_SECRET_VERSIONS);
|
||||
for (const [k, v] of Object.entries(secretVersions)) {
|
||||
if (!SERVICES.includes(k) || !VERSION_RE.test(v)) throw new Error("DEPLOY_SECRET_VERSIONS maps a service to a secret version id");
|
||||
}
|
||||
if (mode !== "refresh-secrets") throw new Error("DEPLOY_SECRET_VERSIONS is only accepted with refresh-secrets");
|
||||
}
|
||||
return { cluster, prefix, definitions, counts, run: `${env.GITHUB_RUN_ID || "operator"}/${env.GITHUB_RUN_ATTEMPT || "1"}` };
|
||||
const bake = Number(env.DEPLOY_BAKE_SECONDS ?? 300);
|
||||
if (!Number.isInteger(bake) || bake < 0 || bake > 3600) throw new Error("DEPLOY_BAKE_SECONDS must be 0 through 3600");
|
||||
return {
|
||||
cluster, prefix, mode, tag, services, secretVersions, bake,
|
||||
fleet: env.DEPLOY_FLEET !== "false",
|
||||
allowDroppedSecrets: env.DEPLOY_ALLOW_DROPPED_SECRETS === "true",
|
||||
images: env.DEPLOY_IMAGES_JSON ? JSON.parse(readFileSync(env.DEPLOY_IMAGES_JSON, "utf8")) : null,
|
||||
run: `${env.GITHUB_RUN_ID || "operator"}/${env.GITHUB_RUN_ATTEMPT || "1"}`,
|
||||
};
|
||||
}
|
||||
|
||||
function compareTags(a, b) {
|
||||
const x = TAG_RE.exec(a).slice(1).map(Number);
|
||||
const y = TAG_RE.exec(b).slice(1).map(Number);
|
||||
for (let i = 0; i < 3; i++) if (x[i] !== y[i]) return x[i] - y[i];
|
||||
return 0;
|
||||
}
|
||||
|
||||
function assertResponse(out) {
|
||||
@@ -33,6 +67,141 @@ function assertResponse(out) {
|
||||
return out;
|
||||
}
|
||||
|
||||
const tagValue = (tags, key) => tags?.find((t) => t.key === key)?.value;
|
||||
|
||||
// parseRef splits arn:aws:secretsmanager:<region>:<acct>:secret:<name>:<key>:<stage>:<version>.
|
||||
function parseRef(valueFrom) {
|
||||
const parts = (valueFrom || "").split(":");
|
||||
if (parts[2] !== "secretsmanager" || parts.length !== 10) return null;
|
||||
return { arn: parts.slice(0, 7).join(":"), key: parts[7], stage: parts[8], version: parts[9] };
|
||||
}
|
||||
|
||||
// secretService names the service a secret belongs to: <prefix>/<service>-XXXXXX.
|
||||
function secretService(arn, prefix) {
|
||||
const name = arn.split(":")[6] || "";
|
||||
const m = new RegExp(`^${prefix}/([a-z]+)-[A-Za-z0-9]{6}$`).exec(name);
|
||||
return m ? m[1] : null;
|
||||
}
|
||||
|
||||
const REGISTER_FIELDS = [
|
||||
"family", "taskRoleArn", "executionRoleArn", "networkMode", "containerDefinitions", "volumes",
|
||||
"placementConstraints", "requiresCompatibilities", "cpu", "memory", "pidMode", "ipcMode",
|
||||
"proxyConfiguration", "inferenceAccelerators", "ephemeralStorage", "runtimePlatform", "enableFaultInjection",
|
||||
];
|
||||
|
||||
async function describe(client, taskDefinition) {
|
||||
const out = await client.send(new DescribeTaskDefinitionCommand({ taskDefinition, include: ["TAGS"] }));
|
||||
if (!out.taskDefinition || out.taskDefinition.status !== "ACTIVE") throw new Error(`${taskDefinition} is not an active task definition`);
|
||||
return { definition: out.taskDefinition, tags: out.tags || [] };
|
||||
}
|
||||
|
||||
async function resolveVersions(clients, config, refs) {
|
||||
const pins = {};
|
||||
for (const [arn, seen] of Object.entries(refs)) {
|
||||
const service = secretService(arn, config.prefix);
|
||||
if (config.secretVersions[service]) {
|
||||
pins[arn] = config.secretVersions[service];
|
||||
continue;
|
||||
}
|
||||
// Explicit versions mean a targeted roll; everyone else keeps the newest pin.
|
||||
if (config.mode === "refresh-secrets" && !Object.keys(config.secretVersions).length) {
|
||||
const out = await clients.secrets.send(new DescribeSecretCommand({ SecretId: arn }));
|
||||
const current = Object.entries(out.VersionIdsToStages || {}).find(([, stages]) => stages.includes("AWSCURRENT"));
|
||||
if (!current) throw new Error(`${service || "a"} secret has no AWSCURRENT version`);
|
||||
pins[arn] = current[0];
|
||||
continue;
|
||||
}
|
||||
// Rotation pins newer versions than Terraform's, so the newest pin anyone uses wins.
|
||||
const out = await clients.secrets.send(new ListSecretVersionIdsCommand({ SecretId: arn, IncludeDeprecated: true, MaxResults: 100 }));
|
||||
const created = new Map((out.Versions || []).map((v) => [v.VersionId, new Date(v.CreatedDate).getTime()]));
|
||||
const known = [...seen].filter((v) => created.has(v));
|
||||
if (!known.length) throw new Error(`no pinned version of the ${service || ""} secret still exists`);
|
||||
pins[arn] = known.sort((a, b) => created.get(b) - created.get(a))[0];
|
||||
}
|
||||
return pins;
|
||||
}
|
||||
|
||||
function repin(container, pins) {
|
||||
for (const secret of container.secrets || []) {
|
||||
const ref = parseRef(secret.valueFrom);
|
||||
if (ref && ref.version && pins[ref.arn]) {
|
||||
secret.valueFrom = `${ref.arn}:${ref.key}:${ref.stage}:${pins[ref.arn]}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function same(a, b) {
|
||||
return JSON.stringify(REGISTER_FIELDS.map((f) => a[f] ?? null)) === JSON.stringify(REGISTER_FIELDS.map((f) => b[f] ?? null));
|
||||
}
|
||||
|
||||
async function register(client, source, rendered, tags, release, report) {
|
||||
const carried = tags.filter((t) => t.key !== RELEASE_TAG && !t.key.startsWith("aws:"));
|
||||
if (release) carried.push({ key: RELEASE_TAG, value: release });
|
||||
if (same(source.definition, rendered) && tagValue(source.tags, RELEASE_TAG) === release) return source.definition.taskDefinitionArn;
|
||||
const input = {};
|
||||
for (const f of REGISTER_FIELDS) if (rendered[f] !== undefined && rendered[f] !== null) input[f] = rendered[f];
|
||||
input.tags = carried;
|
||||
const out = await client.send(new RegisterTaskDefinitionCommand(input));
|
||||
const arn = out.taskDefinition?.taskDefinitionArn;
|
||||
if (!arn) throw new Error(`registering ${rendered.family} returned no revision`);
|
||||
report(`Registered ${arn.split("/").pop()}`);
|
||||
return arn;
|
||||
}
|
||||
|
||||
async function logTail(clients, definition, task, report, quiet) {
|
||||
const container = definition.containerDefinitions[0];
|
||||
const options = container.logConfiguration?.options || {};
|
||||
if (!options["awslogs-group"] || !options["awslogs-stream-prefix"]) return [];
|
||||
try {
|
||||
const out = await clients.logs.send(new GetLogEventsCommand({
|
||||
logGroupName: options["awslogs-group"],
|
||||
logStreamName: `${options["awslogs-stream-prefix"]}/${container.name}/${task.split("/").pop()}`,
|
||||
startFromHead: false, limit: 60,
|
||||
}));
|
||||
const lines = (out.events || []).map((e) => String(e.message).slice(0, 2000));
|
||||
if (!quiet) for (const line of lines) report(` | ${line}`);
|
||||
return lines;
|
||||
} catch {
|
||||
report(" | (no log events could be read for this task)");
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// runTask runs a one-off task to completion; capture may be a predicate the log tail must meet.
|
||||
async function runTask(clients, config, network, arn, definition, label, sleep, report, options = {}) {
|
||||
const { command, environment, capture } = Array.isArray(options) ? { command: options } : options;
|
||||
const token = createHash("sha256").update(`${config.run}/${arn}/${label}/${JSON.stringify(command || [])}/${JSON.stringify(environment || [])}`).digest("hex");
|
||||
const name = definition.containerDefinitions[0].name;
|
||||
const override = { name, ...(command ? { command } : {}), ...(environment ? { environment } : {}) };
|
||||
const started = assertResponse(await clients.ecs.send(new RunTaskCommand({
|
||||
cluster: config.cluster, taskDefinition: arn, launchType: "FARGATE", platformVersion: "1.4.0", count: 1,
|
||||
networkConfiguration: network, clientToken: token.slice(0, 64), startedBy: `${label.replace(/\W+/g, "-")}-${token.slice(0, 20)}`.slice(0, 128),
|
||||
...(command || environment ? { overrides: { containerOverrides: [override] } } : {}),
|
||||
})));
|
||||
const task = started.tasks?.[0]?.taskArn;
|
||||
if (!task) throw new Error(`${label} task was not created`);
|
||||
report(`${label} task: ${task.split("/").pop()}`);
|
||||
for (let attempt = 0; attempt < 240; attempt++) {
|
||||
const out = assertResponse(await clients.ecs.send(new DescribeTasksCommand({ cluster: config.cluster, tasks: [task] })));
|
||||
const status = out.tasks?.[0];
|
||||
if (!status) throw new Error(`${label} task is missing`);
|
||||
if (status.lastStatus === "STOPPED") {
|
||||
const container = status.containers?.find((c) => c.name === name);
|
||||
const ok = status.stopCode === "EssentialContainerExited" && container?.exitCode === 0;
|
||||
let lines = !ok || command || capture ? await logTail(clients, definition, task, report, ok && capture) : [];
|
||||
if (!ok) throw new Error(`${label} failed; its log tail is above`);
|
||||
// Log delivery can trail the task stopping by a few seconds.
|
||||
for (let retry = 0; typeof capture === "function" && !capture(lines) && retry < 6; retry++) {
|
||||
await sleep(5000);
|
||||
lines = await logTail(clients, definition, task, report, true);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
await sleep(5000);
|
||||
}
|
||||
throw new Error(`${label} is unresolved after 20 minutes; no services were updated. Inspect the task before retrying`);
|
||||
}
|
||||
|
||||
async function waitForService(client, config, name, definition, sleep) {
|
||||
for (let attempt = 0; attempt < 180; attempt++) {
|
||||
const out = assertResponse(await client.send(new DescribeServicesCommand({ cluster: config.cluster, services: [`${config.prefix}-${name}`] })));
|
||||
@@ -47,78 +216,196 @@ async function waitForService(client, config, name, definition, sleep) {
|
||||
throw new Error(`${name} deployment did not become healthy within 15 minutes`);
|
||||
}
|
||||
|
||||
async function deploy(client, config, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), report = console.log) {
|
||||
const definitions = {};
|
||||
for (const name of ["migration", "backend", "consumer"]) {
|
||||
const out = await client.send(new DescribeTaskDefinitionCommand({ taskDefinition: config.definitions[name] }));
|
||||
const definition = out.taskDefinition;
|
||||
const container = definition?.containerDefinitions?.find((c) => c.name === name);
|
||||
if (!container || definition.status !== "ACTIVE" || !/@sha256:[a-f0-9]{64}$/.test(container.image || "")) throw new Error(`Invalid or unpinned ${name} task definition`);
|
||||
definitions[name] = { definition, container };
|
||||
// bake watches a stable service and its alarms; ECS rolling back counts as failure.
|
||||
async function bake(clients, config, name, definition, sleep, report) {
|
||||
for (let waited = 0; waited < config.bake; waited += 15) {
|
||||
const alarms = await clients.cloudwatch.send(new DescribeAlarmsCommand({ AlarmNamePrefix: `${config.prefix}-${name}-`, StateValue: "ALARM" }));
|
||||
const firing = (alarms.MetricAlarms || []).map((a) => a.AlarmName);
|
||||
if (firing.length) throw new Error(`${name} raised ${firing.join(", ")} during the bake`);
|
||||
const out = assertResponse(await clients.ecs.send(new DescribeServicesCommand({ cluster: config.cluster, services: [`${config.prefix}-${name}`] })));
|
||||
if (out.services?.[0]?.taskDefinition !== definition) throw new Error(`${name} was rolled back during the bake`);
|
||||
await sleep(15000);
|
||||
}
|
||||
const migration = definitions.migration.container;
|
||||
if (migration.image !== definitions.backend.container.image || migration.entryPoint?.join(" ") !== "/app/migrate") throw new Error("Migration must use /app/migrate from the exact backend image");
|
||||
const migrationSecret = migration.secrets?.find((s) => s.name === "PRIMARY_DB");
|
||||
const directSecret = definitions.backend.container.secrets?.find((s) => s.name === "PRIMARY_DB_DIRECT");
|
||||
if (!directSecret || directSecret.valueFrom !== migrationSecret?.valueFrom) throw new Error("Migration and backend must reference the same direct database secret version");
|
||||
if (!definitions.backend.container.environment?.some((e) => e.name === "DB_MIGRATIONS_ON_START" && e.value === "false")) throw new Error("Backend startup migrations must be disabled for this deployment");
|
||||
const current = assertResponse(await client.send(new DescribeServicesCommand({ cluster: config.cluster, services: [`${config.prefix}-backend`, `${config.prefix}-consumer`] })));
|
||||
if (current.services?.length !== 2 || current.services.some((s) => s.status !== "ACTIVE")) throw new Error("Both ECS services must already exist");
|
||||
const backend = current.services.find((s) => s.serviceName === `${config.prefix}-backend`);
|
||||
if (config.bake) report(`${name} baked for ${config.bake}s with no alarm`);
|
||||
}
|
||||
|
||||
function imageFor(config, key, fallback) {
|
||||
if (!config.images) return fallback;
|
||||
const digest = config.images.images?.[key];
|
||||
if (!DIGEST_RE.test(digest || "")) throw new Error(`images.json for ${config.tag} has no ${key} digest`);
|
||||
return `${config.images.registry}/${key.replace(/-kafka$/, "")}@${digest}`;
|
||||
}
|
||||
|
||||
async function deploy(clients, config, sleep = (ms) => new Promise((r) => setTimeout(r, ms)), report = console.log) {
|
||||
const ecs = clients.ecs;
|
||||
const names = SERVICES.map((s) => `${config.prefix}-${s}`);
|
||||
const current = assertResponse(await ecs.send(new DescribeServicesCommand({ cluster: config.cluster, services: names })));
|
||||
const live = {};
|
||||
for (const s of current.services || []) live[s.serviceName.slice(config.prefix.length + 1)] = s;
|
||||
for (const name of config.services) {
|
||||
if (live[name]?.status !== "ACTIVE") throw new Error(`${name} must already exist as an ECS service`);
|
||||
}
|
||||
const backend = live.backend;
|
||||
if (!backend?.networkConfiguration?.awsvpcConfiguration) throw new Error("Backend task networking is missing");
|
||||
const token = createHash("sha256").update(`${config.run}/${config.definitions.migration}`).digest("hex");
|
||||
const started = assertResponse(await client.send(new RunTaskCommand({
|
||||
cluster: config.cluster, taskDefinition: config.definitions.migration,
|
||||
launchType: "FARGATE", platformVersion: "1.4.0", count: 1,
|
||||
networkConfiguration: backend.networkConfiguration, clientToken: token,
|
||||
startedBy: `migration-${token.slice(0, 24)}`,
|
||||
})));
|
||||
const task = started.tasks?.[0]?.taskArn;
|
||||
if (!task) throw new Error("Migration task was not created");
|
||||
report(`Migration task: ${task}`);
|
||||
let migrated = false;
|
||||
for (let attempt = 0; attempt < 240; attempt++) {
|
||||
const out = assertResponse(await client.send(new DescribeTasksCommand({ cluster: config.cluster, tasks: [task] })));
|
||||
const status = out.tasks?.[0];
|
||||
if (!status) throw new Error("Migration task is missing");
|
||||
if (status.lastStatus === "STOPPED") {
|
||||
const container = status.containers?.find((c) => c.name === "migration");
|
||||
if (status.stopCode !== "EssentialContainerExited" || container?.exitCode !== 0) throw new Error("Migration failed; inspect its CloudWatch logs before retrying");
|
||||
migrated = true;
|
||||
break;
|
||||
}
|
||||
await sleep(5000);
|
||||
const liveDefs = {};
|
||||
for (const [name, s] of Object.entries(live)) liveDefs[name] = await describe(ecs, s.taskDefinition);
|
||||
|
||||
const liveTag = tagValue(liveDefs.backend.tags, RELEASE_TAG);
|
||||
const tag = config.tag || liveTag;
|
||||
if (!tag) throw new Error("The live backend carries no release tag yet; pass DEPLOY_TAG");
|
||||
if (config.tag && liveTag && TAG_RE.test(liveTag) && compareTags(config.tag, liveTag) < 0 && config.mode !== "rollback") {
|
||||
throw new Error(`${config.tag} is older than the live ${liveTag}; use mode=rollback to go back`);
|
||||
}
|
||||
if (!migrated) throw new Error("Migration status is unresolved; no services were updated. Inspect the running task before retrying");
|
||||
report("Migration completed successfully");
|
||||
if (config.mode === "rollback" && liveTag && TAG_RE.test(liveTag) && compareTags(config.tag, liveTag) > 0) {
|
||||
throw new Error(`rollback target ${config.tag} is newer than the live ${liveTag}`);
|
||||
}
|
||||
if (config.images && config.images.tag !== tag) throw new Error(`images.json describes ${config.images.tag}, not ${tag}`);
|
||||
if (!config.images && config.mode !== "refresh-secrets") throw new Error("DEPLOY_IMAGES_JSON is required unless refreshing secrets");
|
||||
report(`Release ${tag} (${config.mode}); live backend reports ${liveTag || "no release"}`);
|
||||
|
||||
const deployBackend = config.services.includes("backend");
|
||||
const families = [...config.services];
|
||||
const extra = {};
|
||||
for (const name of ["migration", "validation", "ops"]) {
|
||||
if (name === "migration" && (!deployBackend || config.mode === "rollback")) continue;
|
||||
try {
|
||||
extra[name] = await describe(ecs, `${config.prefix}-${name}`);
|
||||
families.push(name);
|
||||
} catch (error) {
|
||||
if (name === "migration") throw error;
|
||||
report(`No ${name} task definition; skipping it`);
|
||||
}
|
||||
}
|
||||
const sources = {};
|
||||
for (const name of config.services) sources[name] = await describe(ecs, `${config.prefix}-${name}`);
|
||||
Object.assign(sources, extra);
|
||||
|
||||
const refs = {};
|
||||
const collect = (def) => {
|
||||
for (const c of def.containerDefinitions || []) for (const s of c.secrets || []) {
|
||||
const ref = parseRef(s.valueFrom);
|
||||
if (ref?.version) (refs[ref.arn] ||= new Set()).add(ref.version);
|
||||
}
|
||||
};
|
||||
for (const name of families) collect(sources[name].definition);
|
||||
for (const def of Object.values(liveDefs)) collect(def.definition);
|
||||
const pins = await resolveVersions(clients, config, refs);
|
||||
|
||||
const variant = (name) => {
|
||||
const v = tagValue(sources[name].tags, VARIANT_TAG);
|
||||
if (v !== "kafka" && v !== "default") throw new Error(`${config.prefix}-${name} has no ${VARIANT_TAG} tag; apply the runtime Terraform first`);
|
||||
return v;
|
||||
};
|
||||
const backendKey = deployBackend || families.includes("ops") ? `backend${variant(deployBackend ? "backend" : "ops") === "kafka" ? "-kafka" : ""}` : null;
|
||||
const rendered = {};
|
||||
for (const name of families) {
|
||||
const def = structuredClone(sources[name].definition);
|
||||
const container = def.containerDefinitions?.[0];
|
||||
if (!container || def.containerDefinitions.length !== 1) throw new Error(`${name} must have exactly one container`);
|
||||
repin(container, pins);
|
||||
if (SERVICES.includes(name)) {
|
||||
if (container.name !== name) throw new Error(`${name} container is named ${container.name}`);
|
||||
const key = `${name}${variant(name) === "kafka" ? "-kafka" : ""}`;
|
||||
if (config.images && !container.image.startsWith(`${config.images.registry}/${name}@`)) {
|
||||
throw new Error(`${name} does not run a digest from ${config.images.registry}; fix its image in Terraform first`);
|
||||
}
|
||||
container.image = imageFor(config, key, liveDefs[name].definition.containerDefinitions[0].image);
|
||||
} else if (name !== "validation") {
|
||||
container.image = imageFor(config, backendKey, liveDefs.backend.definition.containerDefinitions[0].image);
|
||||
}
|
||||
rendered[name] = def;
|
||||
}
|
||||
|
||||
for (const name of config.services) {
|
||||
const before = new Set((liveDefs[name].definition.containerDefinitions[0].secrets || []).map((s) => s.name));
|
||||
const after = new Set((rendered[name].containerDefinitions[0].secrets || []).map((s) => s.name));
|
||||
const dropped = [...before].filter((k) => !after.has(k));
|
||||
if (dropped.length && !config.allowDroppedSecrets) {
|
||||
throw new Error(`${name} would lose ${dropped.join(", ")}; restore them in Terraform or set DEPLOY_ALLOW_DROPPED_SECRETS=true`);
|
||||
}
|
||||
}
|
||||
if (rendered.migration) {
|
||||
const m = rendered.migration.containerDefinitions[0];
|
||||
const b = rendered.backend.containerDefinitions[0];
|
||||
if (m.image !== b.image || m.entryPoint?.join(" ") !== "/app/migrate") throw new Error("Migration must use /app/migrate from the exact backend image");
|
||||
const direct = b.secrets?.find((s) => s.name === "PRIMARY_DB_DIRECT");
|
||||
if (!direct || direct.valueFrom !== m.secrets?.find((s) => s.name === "PRIMARY_DB")?.valueFrom) throw new Error("Migration and backend must reference the same direct database secret version");
|
||||
if (!b.environment?.some((e) => e.name === "DB_MIGRATIONS_ON_START" && e.value === "false")) throw new Error("Backend startup migrations must be disabled for this deployment");
|
||||
}
|
||||
|
||||
const arns = {};
|
||||
for (const name of families) arns[name] = await register(ecs, sources[name], rendered[name], sources[name].tags, tag, report);
|
||||
|
||||
const network = backend.networkConfiguration;
|
||||
if (arns.validation) await runTask(clients, config, network, arns.validation, rendered.validation, "Dependency check", sleep, report);
|
||||
if (arns.migration) {
|
||||
await runTask(clients, config, network, arns.migration, rendered.migration, "Migration", sleep, report);
|
||||
report("Migration completed successfully");
|
||||
}
|
||||
|
||||
const updated = [];
|
||||
try {
|
||||
for (const name of ["backend", "consumer"]) {
|
||||
const previous = current.services.find((s) => s.serviceName === `${config.prefix}-${name}`);
|
||||
updated.push(previous);
|
||||
await client.send(new UpdateServiceCommand({ cluster: config.cluster, service: previous.serviceName, taskDefinition: config.definitions[name], ...(config.counts[name] === undefined ? {} : { desiredCount: config.counts[name] }) }));
|
||||
await waitForService(client, config, name, config.definitions[name], sleep);
|
||||
report(`${name} service is stable on the requested revision`);
|
||||
for (const wave of WAVES) {
|
||||
const group = wave.filter((n) => config.services.includes(n));
|
||||
// allSettled: a sibling still rolling must finish before anything is restored.
|
||||
const results = await Promise.allSettled(group.map(async (name) => {
|
||||
const previous = live[name];
|
||||
const unchanged = previous.taskDefinition === arns[name];
|
||||
if (unchanged && config.mode !== "redeploy") {
|
||||
report(`${name} already runs ${arns[name].split("/").pop()}`);
|
||||
return;
|
||||
}
|
||||
updated.push(previous);
|
||||
await ecs.send(new UpdateServiceCommand({ cluster: config.cluster, service: previous.serviceName, taskDefinition: arns[name], forceNewDeployment: unchanged }));
|
||||
await waitForService(ecs, config, name, arns[name], sleep);
|
||||
report(`${name} is stable on ${arns[name].split("/").pop()}`);
|
||||
await bake(clients, config, name, arns[name], sleep, report);
|
||||
}));
|
||||
const failed = results.find((r) => r.status === "rejected");
|
||||
if (failed) throw failed.reason;
|
||||
}
|
||||
} catch (error) {
|
||||
for (const previous of updated.reverse()) {
|
||||
try {
|
||||
await client.send(new UpdateServiceCommand({ cluster: config.cluster, service: previous.serviceName, taskDefinition: previous.taskDefinition, desiredCount: previous.desiredCount }));
|
||||
report(`Restored previous deployment configuration for ${previous.serviceName}; verify its health`);
|
||||
await ecs.send(new UpdateServiceCommand({ cluster: config.cluster, service: previous.serviceName, taskDefinition: previous.taskDefinition, desiredCount: previous.desiredCount }));
|
||||
report(`Restored ${previous.serviceName} to ${previous.taskDefinition.split("/").pop()}; verify its health`);
|
||||
} catch {
|
||||
report(`Rollback needs operator attention for ${previous.serviceName}`);
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (config.fleet && arns.ops && config.images && (config.mode === "deploy" || config.mode === "rollback")) {
|
||||
const digests = ["worker", "worker-kafka", "consumer", "consumer-kafka"]
|
||||
.filter((k) => DIGEST_RE.test(config.images.images?.[k] || ""))
|
||||
.map((k) => `${k}=${config.images.images[k]}`);
|
||||
const command = ["fleet", "rollout", "start", tag, "--canary", "1", "--waves", "100", "--bake", "30m"];
|
||||
if (digests.length) command.push("--digests", digests.join(","));
|
||||
await runTask(clients, config, network, arns.ops, rendered.ops, "Fleet rollout", sleep, report, { command });
|
||||
report(`Workers move to ${tag} in canary waves; follow with \`warmblyctl fleet rollout status\``);
|
||||
}
|
||||
return { tag, arns, pins: Object.fromEntries(Object.entries(pins).map(([arn, v]) => [secretService(arn, config.prefix) || arn, v])) };
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
const client = new ECSClient({ region: "us-east-1", maxAttempts: 2 });
|
||||
Promise.resolve().then(() => deploy(client, configuration(process.env))).catch((error) => {
|
||||
console.error(error?.$metadata ? `AWS request failed: ${error.name}` : error.message);
|
||||
const region = { region: "us-east-1", maxAttempts: 3 };
|
||||
const clients = { ecs: new ECSClient(region), secrets: new SecretsManagerClient(region), cloudwatch: new CloudWatchClient(region), logs: new CloudWatchLogsClient(region) };
|
||||
const lines = [];
|
||||
const report = (line) => { console.log(line); lines.push(line); };
|
||||
Promise.resolve().then(() => deploy(clients, configuration(process.env), undefined, report)).then((out) => {
|
||||
report(`Deployed ${out.tag}. Secret versions: ${Object.entries(out.pins).map(([k, v]) => `${k}=${v.slice(0, 8)}`).join(", ") || "none"}`);
|
||||
}).catch((error) => {
|
||||
const message = error?.$metadata ? `AWS request failed: ${error.name}` : error.message;
|
||||
console.error(message);
|
||||
lines.push(`**Failed:** ${message}`);
|
||||
process.exitCode = 1;
|
||||
}).finally(() => client.destroy());
|
||||
}).finally(() => {
|
||||
if (process.env.GITHUB_STEP_SUMMARY) {
|
||||
require("node:fs").appendFileSync(process.env.GITHUB_STEP_SUMMARY, `### AWS deploy\n\n\`\`\`\n${lines.join("\n")}\n\`\`\`\n`);
|
||||
}
|
||||
for (const c of Object.values(clients)) c.destroy();
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = { configuration, deploy };
|
||||
module.exports = { configuration, deploy, runTask, describe, compareTags, parseRef, secretService, SERVICES };
|
||||
|
||||
@@ -1,89 +1,305 @@
|
||||
const { test } = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
const { configuration, deploy } = require("./aws-deploy.cjs");
|
||||
const { mkdtempSync, writeFileSync } = require("node:fs");
|
||||
const { join } = require("node:path");
|
||||
const { tmpdir } = require("node:os");
|
||||
const { configuration, deploy, compareTags, parseRef } = require("./aws-deploy.cjs");
|
||||
|
||||
const root = "arn:aws:ecs:us-east-1:000000000000:";
|
||||
const digest = "sha256:" + "a".repeat(64);
|
||||
const secret = "arn:aws:secretsmanager:us-east-1:000000000000:secret:fixture:PRIMARY_DB_DIRECT::version";
|
||||
function config() {
|
||||
const registry = "ghcr.io/warmbly/warmbly";
|
||||
const d = (c) => "sha256:" + c.repeat(64);
|
||||
const KAFKA = ["backend", "consumer", "tracking", "migration", "ops"];
|
||||
const FAMILIES = ["backend", "consumer", "forms", "tracking", "realtime", "migration", "validation", "ops"];
|
||||
const secretArn = (svc) => `arn:aws:secretsmanager:us-east-1:000000000000:secret:fixture/${svc}-AbCdEf`;
|
||||
|
||||
function imagesFile(tag = "v1.3.0") {
|
||||
const dir = mkdtempSync(join(tmpdir(), "aws-deploy-"));
|
||||
const images = {};
|
||||
for (const svc of ["backend", "consumer", "worker", "forms", "tracking", "realtime"]) images[svc] = d("b");
|
||||
for (const svc of ["backend", "consumer", "worker", "tracking"]) images[`${svc}-kafka`] = d("c");
|
||||
writeFileSync(join(dir, "images.json"), JSON.stringify({ tag, registry, images }));
|
||||
return join(dir, "images.json");
|
||||
}
|
||||
|
||||
function config(env = {}) {
|
||||
return configuration({
|
||||
AWS_ECS_CLUSTER: root + "cluster/fixture",
|
||||
AWS_MIGRATION_TASK_DEFINITION: root + "task-definition/fixture-migration:2",
|
||||
AWS_BACKEND_TASK_DEFINITION: root + "task-definition/fixture-backend:2",
|
||||
AWS_CONSUMER_TASK_DEFINITION: root + "task-definition/fixture-consumer:2",
|
||||
AWS_ECS_CLUSTER: root + "cluster/fixture", DEPLOY_TAG: "v1.3.0", DEPLOY_IMAGES_JSON: imagesFile(env.DEPLOY_TAG || "v1.3.0"),
|
||||
DEPLOY_BAKE_SECONDS: "30", ...env,
|
||||
});
|
||||
}
|
||||
|
||||
function definition(family, revision, options) {
|
||||
const secretSvc = ["migration", "validation", "ops"].includes(family) ? "backend" : family;
|
||||
const version = family === "backend" && revision === 1 ? "rotated" : options.versions?.[family] || "terraform";
|
||||
const secrets = family === "migration"
|
||||
? [{ name: "PRIMARY_DB", valueFrom: `${secretArn("backend")}:PRIMARY_DB_DIRECT::${version}` }]
|
||||
: [
|
||||
{ name: "PRIMARY_DB_DIRECT", valueFrom: `${secretArn(secretSvc)}:PRIMARY_DB_DIRECT::${version}` },
|
||||
{ name: "REDIS_PASSWORD", valueFrom: `${secretArn(secretSvc)}:REDIS_PASSWORD::` },
|
||||
];
|
||||
if (family === "backend" && (revision === 1 || !options.dropReadKeys)) secrets.push({ name: "CREDENTIALS_ENCRYPTION_READ_KEYS", valueFrom: `${secretArn("backend")}:CREDENTIALS_ENCRYPTION_READ_KEYS::${version}` });
|
||||
const image = family === "validation" ? "node:22-alpine"
|
||||
: `${options.registry || registry}/${["migration", "ops"].includes(family) ? "backend" : family}@${d("a")}`;
|
||||
return {
|
||||
taskDefinitionArn: `${root}task-definition/fixture-${family}:${revision}`, family: `fixture-${family}`, revision, status: "ACTIVE",
|
||||
networkMode: "awsvpc", cpu: "256", memory: "512", requiresCompatibilities: ["FARGATE"],
|
||||
containerDefinitions: [{
|
||||
name: family, image, secrets,
|
||||
entryPoint: family === "migration" ? ["/app/migrate"] : undefined,
|
||||
environment: [{ name: "DB_MIGRATIONS_ON_START", value: "false" }],
|
||||
logConfiguration: { logDriver: "awslogs", options: { "awslogs-group": "/fixture", "awslogs-stream-prefix": family } },
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(options = {}) {
|
||||
const calls = [];
|
||||
const services = ["backend", "consumer"].map((name) => ({
|
||||
serviceName: `fixture-${name}`, status: "ACTIVE", desiredCount: 1, runningCount: 1, pendingCount: 0,
|
||||
taskDefinition: root + `task-definition/fixture-${name}:1`, deployments: [{ rolloutState: "COMPLETED" }],
|
||||
networkConfiguration: { awsvpcConfiguration: { subnets: ["subnet-fixture"], securityGroups: ["sg-fixture"], assignPublicIp: "ENABLED" } },
|
||||
}));
|
||||
const client = { async send(command) {
|
||||
const name = command.constructor.name;
|
||||
calls.push({ name, input: command.input });
|
||||
if (name === "DescribeTaskDefinitionCommand") {
|
||||
const service = /fixture-(\w+):/.exec(command.input.taskDefinition)[1];
|
||||
return { taskDefinition: { status: "ACTIVE", containerDefinitions: [{
|
||||
name: service, image: `registry/${service === "migration" ? "backend" : service}@${service === "migration" && options.imageMismatch ? "sha256:" + "b".repeat(64) : digest}`,
|
||||
entryPoint: service === "migration" ? ["/app/migrate"] : undefined,
|
||||
environment: [{ name: "DB_MIGRATIONS_ON_START", value: "false" }],
|
||||
secrets: [{ name: service === "migration" ? "PRIMARY_DB" : "PRIMARY_DB_DIRECT", valueFrom: secret }],
|
||||
}] } };
|
||||
const defs = {};
|
||||
const tagsOf = {};
|
||||
for (const family of FAMILIES) {
|
||||
for (const revision of [1, 2]) {
|
||||
const arn = `${root}task-definition/fixture-${family}:${revision}`;
|
||||
defs[arn] = definition(family, revision, options);
|
||||
const tags = [{ key: "Environment", value: "production" }];
|
||||
if (!(options.noVariant && family === "backend" && revision === 2)) tags.push({ key: "warmbly-image-variant", value: KAFKA.includes(family) ? "kafka" : "default" });
|
||||
if (revision === 1) tags.push({ key: "warmbly-release", value: options.liveTag || "v1.2.0" });
|
||||
tagsOf[arn] = tags;
|
||||
}
|
||||
if (name === "DescribeServicesCommand") return { services: services.filter((s) => command.input.services.includes(s.serviceName)).map((s) => structuredClone(s)) };
|
||||
if (name === "RunTaskCommand") return options.placementFailure ? { failures: [{ reason: "fixture" }] } : { tasks: [{ taskArn: root + "task/fixture/migration" }] };
|
||||
if (name === "DescribeTasksCommand") return { tasks: [{ lastStatus: "STOPPED", stopCode: "EssentialContainerExited", containers: [{ name: "migration", exitCode: options.migrationFailure ? 1 : 0 }] }] };
|
||||
if (name === "UpdateServiceCommand") {
|
||||
if (command.input.service === "fixture-consumer" && command.input.taskDefinition.endsWith(":2") && options.consumerFailure) throw new Error("consumer update failed");
|
||||
}
|
||||
const latest = (family) => Object.values(defs).filter((x) => x.family === `fixture-${family}`).sort((a, b) => b.revision - a.revision)[0];
|
||||
const services = ["backend", "consumer", "forms", "tracking", "realtime"].map((name) => ({
|
||||
serviceName: `fixture-${name}`, status: "ACTIVE", desiredCount: 2, runningCount: 2, pendingCount: 0,
|
||||
taskDefinition: `${root}task-definition/fixture-${name}:1`, deployments: [{ rolloutState: "COMPLETED" }],
|
||||
networkConfiguration: { awsvpcConfiguration: { subnets: ["subnet-fixture"], securityGroups: ["sg-fixture"], assignPublicIp: "DISABLED" } },
|
||||
}));
|
||||
const tasks = {};
|
||||
const record = (client) => ({ async send(command) {
|
||||
const name = command.constructor.name;
|
||||
calls.push({ client, name, input: structuredClone(command.input) });
|
||||
switch (name) {
|
||||
case "DescribeTaskDefinitionCommand": {
|
||||
const ref = command.input.taskDefinition;
|
||||
const def = ref.startsWith("arn:") ? defs[ref] : latest(ref.replace(/^fixture-/, ""));
|
||||
if (!def) throw new Error(`no task definition ${ref}`);
|
||||
return { taskDefinition: structuredClone(def), tags: structuredClone(tagsOf[def.taskDefinitionArn]) };
|
||||
}
|
||||
case "RegisterTaskDefinitionCommand": {
|
||||
const prev = latest(command.input.family.replace(/^fixture-/, ""));
|
||||
const arn = `${root}task-definition/${command.input.family}:${prev.revision + 1}`;
|
||||
defs[arn] = { ...structuredClone(command.input), revision: prev.revision + 1, taskDefinitionArn: arn, status: "ACTIVE" };
|
||||
delete defs[arn].tags;
|
||||
tagsOf[arn] = command.input.tags;
|
||||
return { taskDefinition: { taskDefinitionArn: arn } };
|
||||
}
|
||||
case "DescribeServicesCommand":
|
||||
return { services: services.filter((s) => command.input.services.includes(s.serviceName)).map((s) => structuredClone(s)) };
|
||||
case "UpdateServiceCommand": {
|
||||
const service = services.find((s) => s.serviceName === command.input.service);
|
||||
const svc = service.serviceName.replace("fixture-", "");
|
||||
if (options.updateFailure === svc && !command.input.taskDefinition.endsWith(":1")) throw new Error(`${svc} update failed`);
|
||||
service.taskDefinition = command.input.taskDefinition;
|
||||
if (command.input.desiredCount !== undefined) service.runningCount = service.desiredCount = command.input.desiredCount;
|
||||
if (options.circuitBreakerRollback && command.input.taskDefinition.endsWith(":2")) service.taskDefinition = root + "task-definition/fixture-backend:1";
|
||||
if (command.input.desiredCount !== undefined) service.desiredCount = command.input.desiredCount;
|
||||
service.deployments = [{ rolloutState: options.rolloutFailure === svc && !command.input.taskDefinition.endsWith(":1") ? "FAILED" : "COMPLETED" }];
|
||||
if (options.circuitBreaker === svc) service.taskDefinition = `${root}task-definition/fixture-${svc}:1`;
|
||||
return {};
|
||||
}
|
||||
throw new Error(`Unexpected command ${name}`);
|
||||
} };
|
||||
return { client, calls };
|
||||
case "RunTaskCommand": {
|
||||
if (options.placementFailure) return { failures: [{ reason: "fixture" }] };
|
||||
const family = /fixture-(\w+):/.exec(command.input.taskDefinition)[1];
|
||||
const arn = `${root}task/fixture/${family}-${Object.keys(tasks).length}`;
|
||||
tasks[arn] = family;
|
||||
return { tasks: [{ taskArn: arn }] };
|
||||
}
|
||||
case "DescribeTasksCommand": {
|
||||
const family = tasks[command.input.tasks[0]];
|
||||
const exitCode = options.taskFailure === family ? 1 : 0;
|
||||
return { tasks: [{ lastStatus: "STOPPED", stopCode: "EssentialContainerExited", containers: [{ name: family, exitCode }] }] };
|
||||
}
|
||||
case "GetLogEventsCommand":
|
||||
return { events: [{ message: `log line from ${command.input.logStreamName}` }] };
|
||||
case "DescribeAlarmsCommand": {
|
||||
const firing = options.alarm && command.input.AlarmNamePrefix === `fixture-${options.alarm}-`;
|
||||
return { MetricAlarms: firing ? [{ AlarmName: `fixture-${options.alarm}-5xx` }] : [] };
|
||||
}
|
||||
case "ListSecretVersionIdsCommand":
|
||||
return { Versions: [
|
||||
{ VersionId: "terraform", CreatedDate: new Date("2026-01-01") },
|
||||
{ VersionId: "rotated", CreatedDate: new Date("2026-02-01") },
|
||||
{ VersionId: "stale", CreatedDate: new Date("2025-01-01") },
|
||||
] };
|
||||
case "DescribeSecretCommand":
|
||||
return { VersionIdsToStages: { terraform: ["AWSPREVIOUS"], "current-version-id-000000000000000000": ["AWSCURRENT"] } };
|
||||
default:
|
||||
throw new Error(`Unexpected command ${name}`);
|
||||
}
|
||||
} });
|
||||
const clients = { ecs: record("ecs"), secrets: record("secrets"), cloudwatch: record("cloudwatch"), logs: record("logs") };
|
||||
return { clients, calls, defs, tagsOf, services };
|
||||
}
|
||||
|
||||
test("migration success gates both service updates and preserves desired counts", async () => {
|
||||
const { client, calls } = fixture();
|
||||
await deploy(client, config(), async () => {}, () => {});
|
||||
const migration = calls.findIndex((c) => c.name === "DescribeTasksCommand");
|
||||
const updates = calls.filter((c) => c.name === "UpdateServiceCommand");
|
||||
assert.equal(updates.length, 2);
|
||||
assert.ok(calls.indexOf(updates[0]) > migration);
|
||||
assert.ok(updates.every((c) => !Object.hasOwn(c.input, "desiredCount")));
|
||||
assert.equal(calls.filter((c) => c.name === "RunTaskCommand").length, 1);
|
||||
const run = (f, cfg, lines = []) => deploy(f.clients, cfg, async () => {}, (l) => lines.push(l));
|
||||
const registered = (f) => f.calls.filter((c) => c.name === "RegisterTaskDefinitionCommand").map((c) => c.input);
|
||||
const updates = (f) => f.calls.filter((c) => c.name === "UpdateServiceCommand").map((c) => c.input);
|
||||
const runs = (f) => f.calls.filter((c) => c.name === "RunTaskCommand").map((c) => c.input);
|
||||
const valueFrom = (def, key) => def.containerDefinitions[0].secrets.find((s) => s.name === key).valueFrom;
|
||||
|
||||
test("configuration refuses prereleases, bad modes and secret pins outside refresh-secrets", () => {
|
||||
assert.throws(() => config({ DEPLOY_TAG: "v1.3.0-rc.1" }), /prereleases never deploy/);
|
||||
assert.throws(() => config({ DEPLOY_MODE: "yolo" }), /DEPLOY_MODE/);
|
||||
assert.throws(() => config({ DEPLOY_SERVICES: "backend,worker" }), /DEPLOY_SERVICES/);
|
||||
assert.throws(() => config({ DEPLOY_SECRET_VERSIONS: JSON.stringify({ backend: "a".repeat(36) }) }), /refresh-secrets/);
|
||||
assert.throws(() => config({ DEPLOY_MODE: "refresh-secrets", DEPLOY_SECRET_VERSIONS: JSON.stringify({ backend: "x" }) }), /secret version id/);
|
||||
assert.ok(compareTags("v1.10.0", "v1.9.9") > 0);
|
||||
assert.deepEqual(parseRef(`${secretArn("backend")}:K::v`), { arn: secretArn("backend"), key: "K", stage: "", version: "v" });
|
||||
});
|
||||
|
||||
for (const failure of ["migrationFailure", "placementFailure", "imageMismatch"]) {
|
||||
test(`${failure} leaves existing services untouched`, async () => {
|
||||
const { client, calls } = fixture({ [failure]: true });
|
||||
await assert.rejects(deploy(client, config(), async () => {}, () => {}));
|
||||
assert.ok(calls.every((c) => c.name !== "UpdateServiceCommand"));
|
||||
test("a release renders from the latest revision, swaps variant digests and carries tags", async () => {
|
||||
const f = fixture();
|
||||
await run(f, config());
|
||||
const byFamily = Object.fromEntries(registered(f).map((r) => [r.family.replace("fixture-", ""), r]));
|
||||
assert.equal(byFamily.backend.containerDefinitions[0].image, `${registry}/backend@${d("c")}`);
|
||||
assert.equal(byFamily.forms.containerDefinitions[0].image, `${registry}/forms@${d("b")}`);
|
||||
assert.equal(byFamily.tracking.containerDefinitions[0].image, `${registry}/tracking@${d("c")}`);
|
||||
assert.equal(byFamily.migration.containerDefinitions[0].image, `${registry}/backend@${d("c")}`);
|
||||
assert.equal(byFamily.ops.containerDefinitions[0].image, `${registry}/backend@${d("c")}`);
|
||||
assert.equal(byFamily.validation.containerDefinitions[0].image, "node:22-alpine");
|
||||
for (const r of Object.values(byFamily)) {
|
||||
assert.ok(r.tags.some((t) => t.key === "Environment"));
|
||||
assert.deepEqual(r.tags.filter((t) => t.key === "warmbly-release"), [{ key: "warmbly-release", value: "v1.3.0" }]);
|
||||
assert.equal(r.cpu, "256");
|
||||
}
|
||||
});
|
||||
|
||||
test("the newest pinned secret version wins so a stale Terraform pin cannot undo a rotation", async () => {
|
||||
const f = fixture();
|
||||
await run(f, config());
|
||||
const backend = registered(f).find((r) => r.family === "fixture-backend");
|
||||
const migration = registered(f).find((r) => r.family === "fixture-migration");
|
||||
assert.ok(valueFrom(backend, "PRIMARY_DB_DIRECT").endsWith("::rotated"));
|
||||
assert.ok(valueFrom(migration, "PRIMARY_DB").endsWith("::rotated"));
|
||||
assert.ok(valueFrom(backend, "REDIS_PASSWORD").endsWith("REDIS_PASSWORD::"));
|
||||
const consumer = registered(f).find((r) => r.family === "fixture-consumer");
|
||||
assert.ok(valueFrom(consumer, "PRIMARY_DB_DIRECT").endsWith("::terraform"));
|
||||
});
|
||||
|
||||
test("refresh-secrets pins every secret to AWSCURRENT and keeps the live images", async () => {
|
||||
const f = fixture();
|
||||
await run(f, configuration({ AWS_ECS_CLUSTER: root + "cluster/fixture", DEPLOY_MODE: "refresh-secrets", DEPLOY_BAKE_SECONDS: "0" }));
|
||||
const backend = registered(f).find((r) => r.family === "fixture-backend");
|
||||
const consumer = registered(f).find((r) => r.family === "fixture-consumer");
|
||||
assert.ok(valueFrom(backend, "PRIMARY_DB_DIRECT").endsWith("::current-version-id-000000000000000000"));
|
||||
assert.ok(valueFrom(consumer, "PRIMARY_DB_DIRECT").endsWith("::current-version-id-000000000000000000"));
|
||||
assert.equal(backend.containerDefinitions[0].image, `${registry}/backend@${d("a")}`);
|
||||
assert.ok(backend.tags.some((t) => t.key === "warmbly-release" && t.value === "v1.2.0"));
|
||||
assert.ok(!runs(f).some((r) => r.overrides), "refresh-secrets starts no fleet rollout");
|
||||
});
|
||||
|
||||
test("explicit secret versions roll only what they name; other secrets keep their newest pin", async () => {
|
||||
const f = fixture();
|
||||
await run(f, configuration({ AWS_ECS_CLUSTER: root + "cluster/fixture", DEPLOY_MODE: "refresh-secrets", DEPLOY_BAKE_SECONDS: "0",
|
||||
DEPLOY_SERVICES: "consumer", DEPLOY_SECRET_VERSIONS: JSON.stringify({ consumer: "explicit-version-id-0000000000000000" }) }));
|
||||
const consumer = registered(f).find((r) => r.family === "fixture-consumer");
|
||||
const validation = registered(f).find((r) => r.family === "fixture-validation");
|
||||
assert.ok(valueFrom(consumer, "PRIMARY_DB_DIRECT").endsWith("::explicit-version-id-0000000000000000"));
|
||||
assert.ok(valueFrom(validation, "PRIMARY_DB_DIRECT").endsWith("::rotated"));
|
||||
assert.ok(!f.calls.some((c) => c.name === "DescribeSecretCommand"));
|
||||
assert.deepEqual(updates(f).map((u) => u.service), ["fixture-consumer"]);
|
||||
});
|
||||
|
||||
test("validation and migration gate every service update", async () => {
|
||||
const f = fixture();
|
||||
await run(f, config());
|
||||
const firstUpdate = f.calls.findIndex((c) => c.name === "UpdateServiceCommand");
|
||||
const taskRuns = f.calls.map((c, i) => [c, i]).filter(([c]) => c.name === "RunTaskCommand");
|
||||
assert.deepEqual(taskRuns.slice(0, 2).map(([c]) => /fixture-(\w+):/.exec(c.input.taskDefinition)[1]), ["validation", "migration"]);
|
||||
assert.ok(taskRuns[1][1] < firstUpdate);
|
||||
assert.deepEqual(updates(f).map((u) => u.service), ["fixture-backend", "fixture-consumer", "fixture-forms", "fixture-tracking", "fixture-realtime"]);
|
||||
assert.ok(updates(f).every((u) => !Object.hasOwn(u, "desiredCount")));
|
||||
});
|
||||
|
||||
for (const [failure, message] of [
|
||||
[{ taskFailure: "migration" }, /Migration failed/], [{ taskFailure: "validation" }, /Dependency check failed/],
|
||||
[{ placementFailure: true }, /task-placement failure/], [{ noVariant: true }, /no warmbly-image-variant tag/],
|
||||
[{ registry: "docker.io/someone" }, /does not run a digest from ghcr/],
|
||||
]) {
|
||||
test(`${JSON.stringify(failure)} leaves every service untouched`, async () => {
|
||||
const f = fixture(failure);
|
||||
await assert.rejects(run(f, config()), message);
|
||||
assert.equal(updates(f).length, 0);
|
||||
});
|
||||
}
|
||||
|
||||
test("consumer rollout failure restores previous service revisions without database rollback", async () => {
|
||||
const { client, calls } = fixture({ consumerFailure: true });
|
||||
await assert.rejects(deploy(client, config(), async () => {}, () => {}), /consumer update failed/);
|
||||
const updates = calls.filter((c) => c.name === "UpdateServiceCommand");
|
||||
assert.equal(updates.length, 4);
|
||||
assert.ok(updates.slice(2).every((c) => c.input.taskDefinition.endsWith(":1") && c.input.desiredCount === 1));
|
||||
assert.equal(calls.filter((c) => c.name === "RunTaskCommand").length, 1);
|
||||
test("a failed migration prints its log tail", async () => {
|
||||
const f = fixture({ taskFailure: "migration" });
|
||||
const lines = [];
|
||||
await assert.rejects(run(f, config(), lines), /Migration failed/);
|
||||
assert.ok(lines.some((l) => l.includes("log line from migration/migration/")));
|
||||
});
|
||||
|
||||
test("an older tag is refused unless the mode is rollback, and rollback skips the migration", async () => {
|
||||
await assert.rejects(run(fixture({ liveTag: "v1.4.0" }), config()), /older than the live v1.4.0/);
|
||||
await assert.rejects(run(fixture(), config({ DEPLOY_MODE: "rollback" })), /newer than the live/);
|
||||
const f = fixture({ liveTag: "v1.4.0" });
|
||||
await run(f, config({ DEPLOY_MODE: "rollback" }));
|
||||
assert.ok(!runs(f).some((r) => r.taskDefinition.includes("fixture-migration")));
|
||||
assert.ok(runs(f).some((r) => r.overrides), "rollback still moves the fleet");
|
||||
});
|
||||
|
||||
test("a revision that drops a secret the live service reads is refused", async () => {
|
||||
const f = fixture({ dropReadKeys: true });
|
||||
await assert.rejects(run(f, config()), /would lose CREDENTIALS_ENCRYPTION_READ_KEYS/);
|
||||
assert.equal(registered(f).length, 0);
|
||||
const g = fixture({ dropReadKeys: true });
|
||||
await run(g, config({ DEPLOY_ALLOW_DROPPED_SECRETS: "true" }));
|
||||
});
|
||||
|
||||
test("an alarm during the bake restores every updated service in reverse order", async () => {
|
||||
const f = fixture({ alarm: "consumer" });
|
||||
await assert.rejects(run(f, config()), /fixture-consumer-5xx/);
|
||||
const restores = updates(f).filter((u) => u.taskDefinition.endsWith(":1"));
|
||||
assert.deepEqual(restores.map((u) => u.service), ["fixture-consumer", "fixture-backend"]);
|
||||
assert.ok(restores.every((u) => u.desiredCount === 2));
|
||||
assert.ok(!updates(f).some((u) => u.service === "fixture-forms"));
|
||||
});
|
||||
|
||||
test("a failure in the parallel wave waits for its siblings and then restores all of them", async () => {
|
||||
const f = fixture({ rolloutFailure: "forms" });
|
||||
await assert.rejects(run(f, config()), /forms deployment failed/);
|
||||
const forward = updates(f).filter((u) => !u.taskDefinition.endsWith(":1")).map((u) => u.service);
|
||||
const restores = updates(f).filter((u) => u.taskDefinition.endsWith(":1")).map((u) => u.service);
|
||||
assert.equal(forward.length, 5);
|
||||
assert.deepEqual([...restores].sort(), [...forward].sort());
|
||||
assert.deepEqual(restores.slice(-2), ["fixture-consumer", "fixture-backend"]);
|
||||
});
|
||||
|
||||
test("a circuit breaker rollback cannot be reported as a successful deployment", async () => {
|
||||
const { client } = fixture({ circuitBreakerRollback: true });
|
||||
await assert.rejects(deploy(client, config(), async () => {}, () => {}), /rolled back/);
|
||||
const f = fixture({ circuitBreaker: "backend" });
|
||||
await assert.rejects(run(f, config()), /backend deployment rolled back/);
|
||||
});
|
||||
|
||||
test("deployment requires explicit revisions in the selected account and region", () => {
|
||||
assert.throws(() => configuration({ AWS_ECS_CLUSTER: root.replace("us-east-1", "eu-west-1") + "cluster/fixture" }));
|
||||
const c = config();
|
||||
assert.equal(c.counts.backend, undefined);
|
||||
test("the fleet rollout gets the worker and consumer digests per variant", async () => {
|
||||
const f = fixture();
|
||||
await run(f, config());
|
||||
const ops = runs(f).find((r) => r.overrides);
|
||||
const command = ops.overrides.containerOverrides[0].command;
|
||||
assert.deepEqual(command.slice(0, 4), ["fleet", "rollout", "start", "v1.3.0"]);
|
||||
assert.equal(command[command.indexOf("--digests") + 1], `worker=${d("b")},worker-kafka=${d("c")},consumer=${d("b")},consumer-kafka=${d("c")}`);
|
||||
assert.equal(ops.overrides.containerOverrides[0].name, "ops");
|
||||
});
|
||||
|
||||
test("an unchanged definition reuses its revision and redeploy forces a fresh deployment", async () => {
|
||||
const f = fixture();
|
||||
await run(f, config({ DEPLOY_SERVICES: "forms", DEPLOY_FLEET: "false" }));
|
||||
const formsArn = updates(f)[0].taskDefinition;
|
||||
f.services.find((s) => s.serviceName === "fixture-forms").taskDefinition = formsArn;
|
||||
const before = registered(f).length;
|
||||
await run(f, config({ DEPLOY_SERVICES: "forms", DEPLOY_MODE: "redeploy", DEPLOY_FLEET: "false" }));
|
||||
assert.equal(registered(f).length, before);
|
||||
assert.deepEqual(updates(f).at(-1), { cluster: root + "cluster/fixture", service: "fixture-forms", taskDefinition: formsArn, forceNewDeployment: true });
|
||||
});
|
||||
|
||||
test("no secret value or version-free reference is ever read", async () => {
|
||||
const f = fixture();
|
||||
await run(f, config());
|
||||
assert.ok(!f.calls.some((c) => c.name === "GetSecretValueCommand"));
|
||||
});
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
// Drives a key rotation from GitHub without ever holding a value: the rotator
|
||||
// task writes new secret versions inside AWS and reports their ids, the deploy
|
||||
// engine rolls services onto exactly those versions, and warmblyctl in the ops
|
||||
// task proves each step is safe before the old value is retired.
|
||||
const { DescribeServicesCommand } = require("@aws-sdk/client-ecs");
|
||||
const engine = require("./aws-deploy.cjs");
|
||||
|
||||
const { configuration, SERVICES } = engine;
|
||||
|
||||
const KINDS = {
|
||||
CREDENTIALS_ENCRYPTION_KEY: "seal",
|
||||
TWOFA_SECRET: "seal",
|
||||
UNSUBSCRIBE_SIGNING_SECRET: "sign",
|
||||
INTERNAL_API_TOKEN: "token",
|
||||
NODE_BROKER_TOKEN: "token",
|
||||
};
|
||||
const ACTIONS = ["status", "prepare", "rotate", "retire", "abort"];
|
||||
|
||||
function rotationConfig(env) {
|
||||
const action = env.ROTATE_ACTION;
|
||||
const key = env.ROTATE_KEY;
|
||||
if (!ACTIONS.includes(action)) throw new Error(`ROTATE_ACTION must be one of ${ACTIONS.join(", ")}`);
|
||||
if (!KINDS[key]) throw new Error(`ROTATE_KEY must be one of ${Object.keys(KINDS).join(", ")}`);
|
||||
const wait = Number(env.ROTATE_NODE_WAIT_MINUTES ?? 30);
|
||||
if (!Number.isInteger(wait) || wait < 0 || wait > 240) throw new Error("ROTATE_NODE_WAIT_MINUTES must be 0 through 240");
|
||||
const base = configuration({ ...env, DEPLOY_MODE: "refresh-secrets", DEPLOY_TAG: "", DEPLOY_SERVICES: "all", DEPLOY_SECRET_VERSIONS: "", DEPLOY_IMAGES_JSON: "", DEPLOY_FLEET: "false" });
|
||||
return { ...base, action, key, kind: KINDS[key], nodeWaitMs: wait * 60000 };
|
||||
}
|
||||
|
||||
async function rotateKeys(clients, config, sleep = (ms) => new Promise((r) => setTimeout(r, ms)), report = console.log, deps = engine) {
|
||||
const { deploy, runTask, describe } = deps;
|
||||
// ECS run tokens are idempotent, so every task and roll in one run needs its own.
|
||||
let seq = 0;
|
||||
const step = () => `${config.run}/${++seq}`;
|
||||
const services = await clients.ecs.send(new DescribeServicesCommand({ cluster: config.cluster, services: [`${config.prefix}-backend`] }));
|
||||
const network = services.services?.[0]?.networkConfiguration;
|
||||
if (!network?.awsvpcConfiguration) throw new Error("Backend task networking is missing");
|
||||
|
||||
const rotator = async (action) => {
|
||||
const { definition } = await describe(clients.ecs, `${config.prefix}-rotator`);
|
||||
const environment = [
|
||||
{ name: "ROTATE_ACTION", value: action }, { name: "ROTATE_KEY", value: config.key }, { name: "ROTATE_RUN", value: `${step()}/${action}` },
|
||||
];
|
||||
const lines = await runTask(clients, { ...config, run: step() }, network, definition.taskDefinitionArn, definition, `Rotator ${action}`, sleep, report, { environment, capture: (ls) => ls.some((l) => l.startsWith("ROTATION_RESULT ")) });
|
||||
const line = lines.filter((l) => l.startsWith("ROTATION_RESULT ")).pop();
|
||||
if (!line) throw new Error(`the rotator reported no result for ${action}`);
|
||||
const result = JSON.parse(line.slice("ROTATION_RESULT ".length));
|
||||
for (const l of lines) if (!l.startsWith("ROTATION_RESULT ")) report(` rotator: ${l}`);
|
||||
return result;
|
||||
};
|
||||
const ops = async (label, command) => {
|
||||
const { definition } = await describe(clients.ecs, `${config.prefix}-ops`);
|
||||
return runTask(clients, { ...config, run: step() }, network, definition.taskDefinitionArn, definition, label, sleep, report, { command });
|
||||
};
|
||||
const roll = async (versions) => {
|
||||
const names = Object.keys(versions || {}).filter((s) => SERVICES.includes(s));
|
||||
if (!names.length) return;
|
||||
report(`Rolling ${names.join(", ")} onto the new secret versions`);
|
||||
await deploy(clients, { ...config, run: step(), services: names, secretVersions: versions }, sleep, report);
|
||||
};
|
||||
const nodesCurrent = async () => {
|
||||
const checks = Math.floor(config.nodeWaitMs / 120000) + 1;
|
||||
for (let i = 1; ; i++) {
|
||||
try {
|
||||
await ops("Fleet env status", ["fleet", "env-status"]);
|
||||
report("Every live node has applied the new env");
|
||||
return true;
|
||||
} catch {
|
||||
if (i >= checks) return false;
|
||||
report("Some nodes have not applied the new env yet; checking again in two minutes");
|
||||
await sleep(120000);
|
||||
}
|
||||
}
|
||||
};
|
||||
const resealed = async () => {
|
||||
await ops("Reseal", ["keys", "reseal", "--apply"]);
|
||||
try {
|
||||
await ops("Key status", ["keys", "status", "--require-primary"]);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
const retire = async () => {
|
||||
const result = await rotator("retire");
|
||||
await roll(result.versions);
|
||||
report(`${config.key} rotated; the previous value is retired`);
|
||||
};
|
||||
// Nodes first: a node still on the old env would seal under the old key after the reseal.
|
||||
const gate = async () => {
|
||||
if ((config.kind === "token" || config.key === "CREDENTIALS_ENCRYPTION_KEY") && !(await nodesCurrent())) {
|
||||
return "some nodes have not applied the new env and still work on the previous value. Run retire once `warmblyctl fleet env-status` is clean";
|
||||
}
|
||||
if (config.kind === "seal" && !(await resealed())) return `some values still open only under the previous ${config.key}; run retire once \`warmblyctl keys status\` is clean`;
|
||||
return null;
|
||||
};
|
||||
|
||||
const status = await rotator("status");
|
||||
if (status.key && status.key !== config.key) throw new Error(`${status.key} is mid-rotation; finish or abort it first`);
|
||||
let phase = status.key === config.key ? status.phase : "none";
|
||||
report(`${config.key}: ${phase}`);
|
||||
// A run that died after writing resumes here: rolling versions already live is a no-op.
|
||||
const resume = () => roll(status.pinned);
|
||||
|
||||
switch (config.action) {
|
||||
case "status":
|
||||
if (config.kind === "seal") await ops("Key status", ["keys", "status"]).catch(() => report("keys status reports unreadable values"));
|
||||
return { phase };
|
||||
case "prepare": {
|
||||
const result = await rotator("prepare");
|
||||
await roll(result.versions);
|
||||
report(`Overlap keys exist for ${config.key}. Add them to secret_keys for ${result.holders.join(", ")} and apply the runtime Terraform before rotating.`);
|
||||
return { phase };
|
||||
}
|
||||
case "abort": {
|
||||
if (phase === "none") throw new Error(`no rotation of ${config.key} is in progress`);
|
||||
const result = await rotator("abort");
|
||||
await roll(result.versions);
|
||||
report(`${config.key} restored to its previous value`);
|
||||
return { phase: "none" };
|
||||
}
|
||||
case "retire": {
|
||||
if (phase !== "staged") throw new Error(`${config.key} has no staged rotation to retire`);
|
||||
await resume();
|
||||
const blocked = await gate();
|
||||
if (blocked) throw new Error(blocked);
|
||||
await retire();
|
||||
return { phase: "none" };
|
||||
}
|
||||
case "rotate": {
|
||||
if (phase !== "none") await resume();
|
||||
if (phase === "staged" && config.kind === "sign") {
|
||||
report(`${config.key} is staged; run retire after ${status.until}`);
|
||||
return { phase };
|
||||
}
|
||||
if (phase === "none") {
|
||||
const result = await rotator("stage");
|
||||
await roll(result.versions);
|
||||
phase = result.phase;
|
||||
}
|
||||
if (phase === "introduced") {
|
||||
if (!(await nodesCurrent())) {
|
||||
report("The new key is readable everywhere except nodes that have not refreshed; run rotate again to make it primary");
|
||||
return { phase };
|
||||
}
|
||||
const result = await rotator("stage");
|
||||
await roll(result.versions);
|
||||
phase = result.phase;
|
||||
}
|
||||
if (config.kind === "sign") {
|
||||
report(`${config.key} rotated. Links signed with the previous value keep working for a year, the life of a link; run retire after that`);
|
||||
return { phase };
|
||||
}
|
||||
const blocked = await gate();
|
||||
if (blocked) {
|
||||
report(`${config.key} is staged: ${blocked}`);
|
||||
return { phase };
|
||||
}
|
||||
await retire();
|
||||
return { phase: "none" };
|
||||
}
|
||||
}
|
||||
return { phase };
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
const { ECSClient } = require("@aws-sdk/client-ecs");
|
||||
const { SecretsManagerClient } = require("@aws-sdk/client-secrets-manager");
|
||||
const { CloudWatchClient } = require("@aws-sdk/client-cloudwatch");
|
||||
const { CloudWatchLogsClient } = require("@aws-sdk/client-cloudwatch-logs");
|
||||
const region = { region: "us-east-1", maxAttempts: 3 };
|
||||
const clients = { ecs: new ECSClient(region), secrets: new SecretsManagerClient(region), cloudwatch: new CloudWatchClient(region), logs: new CloudWatchLogsClient(region) };
|
||||
const lines = [];
|
||||
const report = (line) => { console.log(line); lines.push(line); };
|
||||
Promise.resolve().then(() => rotateKeys(clients, rotationConfig(process.env), undefined, report)).catch((error) => {
|
||||
const message = error?.$metadata ? `AWS request failed: ${error.name}` : error.message;
|
||||
console.error(message);
|
||||
lines.push(`**Failed:** ${message}`);
|
||||
process.exitCode = 1;
|
||||
}).finally(() => {
|
||||
if (process.env.GITHUB_STEP_SUMMARY) {
|
||||
require("node:fs").appendFileSync(process.env.GITHUB_STEP_SUMMARY, `### Key rotation\n\n\`\`\`\n${lines.join("\n")}\n\`\`\`\n`);
|
||||
}
|
||||
for (const c of Object.values(clients)) c.destroy();
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = { rotationConfig, rotateKeys };
|
||||
@@ -0,0 +1,153 @@
|
||||
const { test } = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
const { rotationConfig, rotateKeys } = require("./aws-rotate.cjs");
|
||||
|
||||
const CLUSTER = "arn:aws:ecs:us-east-1:000000000000:cluster/fixture";
|
||||
const V = (c) => c.repeat(32);
|
||||
|
||||
// world plays the rotator, the ops task and the deploy engine against a phase machine.
|
||||
function world({ key = "INTERNAL_API_TOKEN", action = "rotate", state = {}, envLagging = 0, unsealed = false, holders } = {}) {
|
||||
const config = rotationConfig({ AWS_ECS_CLUSTER: CLUSTER, ROTATE_ACTION: action, ROTATE_KEY: key, ROTATE_NODE_WAIT_MINUTES: "4", GITHUB_RUN_ID: "9", GITHUB_RUN_ATTEMPT: "1" });
|
||||
const log = [];
|
||||
const tokens = new Set();
|
||||
let lag = envLagging;
|
||||
const owners = holders || (key.startsWith("CREDENTIALS") ? ["backend", "consumer"] : ["backend", "tracking"]);
|
||||
const versions = () => Object.fromEntries(owners.map((s, i) => [s, V(String(i + 1))]));
|
||||
const rotator = (act) => {
|
||||
log.push(`rotator ${act}`);
|
||||
if (act === "status") return { phase: state.phase || "none", key: state.key || null, versions: {}, pinned: state.key ? versions() : {}, until: state.until || null };
|
||||
if (act === "stage") {
|
||||
state = key === "CREDENTIALS_ENCRYPTION_KEY" && !state.key ? { key, phase: "introduced" } : { key, phase: "staged" };
|
||||
return { phase: state.phase, versions: versions(), holders: owners };
|
||||
}
|
||||
state = {};
|
||||
return { phase: "none", versions: act === "prepare" ? { backend: V("p") } : versions(), holders: owners };
|
||||
};
|
||||
const deps = {
|
||||
async describe(_, family) {
|
||||
return { definition: { taskDefinitionArn: `${family}:1`, family, containerDefinitions: [{ name: family.split("-").pop() }] } };
|
||||
},
|
||||
async runTask(_, cfg, network, arn, definition, label, sleep, report, options) {
|
||||
assert.ok(network.awsvpcConfiguration);
|
||||
const token = `${cfg.run}/${label}`;
|
||||
assert.ok(!tokens.has(token), `run token reused for ${label}`);
|
||||
tokens.add(token);
|
||||
if (arn.startsWith("fixture-rotator")) {
|
||||
const act = options.environment.find((e) => e.name === "ROTATE_ACTION").value;
|
||||
assert.equal(options.environment.find((e) => e.name === "ROTATE_KEY").value, key);
|
||||
const lines = ["phase written", `ROTATION_RESULT ${JSON.stringify(rotator(act))}`];
|
||||
assert.ok(options.capture(lines) && !options.capture(["phase written"]));
|
||||
return lines;
|
||||
}
|
||||
const command = options.command.join(" ");
|
||||
log.push(`ops ${command}`);
|
||||
if (command === "fleet env-status" && lag-- > 0) throw new Error("not current");
|
||||
if (command === "keys status --require-primary" && unsealed) throw new Error("on read key");
|
||||
return [];
|
||||
},
|
||||
async deploy(_, cfg) {
|
||||
assert.equal(cfg.mode, "refresh-secrets");
|
||||
assert.equal(cfg.fleet, false);
|
||||
assert.equal(cfg.tag, "");
|
||||
log.push(`roll ${cfg.services.join(",")}`);
|
||||
return {};
|
||||
},
|
||||
};
|
||||
const clients = { ecs: { async send() {
|
||||
return { services: [{ networkConfiguration: { awsvpcConfiguration: { subnets: ["s"] } } }] };
|
||||
} } };
|
||||
const report = [];
|
||||
const run = () => rotateKeys(clients, config, async () => {}, (l) => report.push(l), deps);
|
||||
return { run, log, report, state: () => state };
|
||||
}
|
||||
|
||||
test("configuration refuses unknown keys, actions and waits", () => {
|
||||
const base = { AWS_ECS_CLUSTER: CLUSTER, ROTATE_ACTION: "rotate", ROTATE_KEY: "INTERNAL_API_TOKEN" };
|
||||
assert.throws(() => rotationConfig({ ...base, ROTATE_KEY: "AUTH_SECRET" }), /ROTATE_KEY/);
|
||||
assert.throws(() => rotationConfig({ ...base, ROTATE_ACTION: "delete" }), /ROTATE_ACTION/);
|
||||
assert.throws(() => rotationConfig({ ...base, ROTATE_NODE_WAIT_MINUTES: "999" }), /0 through 240/);
|
||||
assert.equal(rotationConfig(base).kind, "token");
|
||||
});
|
||||
|
||||
test("a token is staged receiver first, waits for nodes, then retires in one run", async () => {
|
||||
const w = world({ envLagging: 1 });
|
||||
assert.deepEqual(await w.run(), { phase: "none" });
|
||||
assert.deepEqual(w.log, [
|
||||
"rotator status", "rotator stage", "roll backend,tracking",
|
||||
"ops fleet env-status", "ops fleet env-status", "rotator retire", "roll backend,tracking",
|
||||
]);
|
||||
});
|
||||
|
||||
test("a token stays staged when nodes never catch up, and says how to finish", async () => {
|
||||
const w = world({ envLagging: 99 });
|
||||
assert.deepEqual(await w.run(), { phase: "staged" });
|
||||
assert.ok(!w.log.includes("rotator retire"));
|
||||
assert.equal(w.log.filter((l) => l === "ops fleet env-status").length, 3);
|
||||
assert.match(w.report.at(-1), /is staged: .*fleet env-status/);
|
||||
});
|
||||
|
||||
test("the credentials key is introduced, made primary once nodes read it, resealed, then retired", async () => {
|
||||
const w = world({ key: "CREDENTIALS_ENCRYPTION_KEY" });
|
||||
assert.deepEqual(await w.run(), { phase: "none" });
|
||||
assert.deepEqual(w.log, [
|
||||
"rotator status", "rotator stage", "roll backend,consumer",
|
||||
"ops fleet env-status", "rotator stage", "roll backend,consumer",
|
||||
"ops fleet env-status", "ops keys reseal --apply", "ops keys status --require-primary",
|
||||
"rotator retire", "roll backend,consumer",
|
||||
]);
|
||||
});
|
||||
|
||||
test("the credentials key waits at introduced when a node has not refreshed", async () => {
|
||||
const w = world({ key: "CREDENTIALS_ENCRYPTION_KEY", envLagging: 99 });
|
||||
assert.deepEqual(await w.run(), { phase: "introduced" });
|
||||
assert.equal(w.log.filter((l) => l === "rotator stage").length, 1);
|
||||
});
|
||||
|
||||
test("a seal key is not retired while values still need the old key", async () => {
|
||||
const w = world({ key: "TWOFA_SECRET", unsealed: true, holders: ["backend"] });
|
||||
assert.deepEqual(await w.run(), { phase: "staged" });
|
||||
assert.ok(!w.log.includes("ops fleet env-status"));
|
||||
assert.ok(!w.log.includes("rotator retire"));
|
||||
assert.match(w.report.at(-1), /keys status/);
|
||||
});
|
||||
|
||||
test("a signing secret stops at staged", async () => {
|
||||
const w = world({ key: "UNSUBSCRIBE_SIGNING_SECRET", holders: ["backend"] });
|
||||
assert.deepEqual(await w.run(), { phase: "staged" });
|
||||
assert.deepEqual(w.log, ["rotator status", "rotator stage", "roll backend"]);
|
||||
assert.match(w.report.at(-1), /for a year/);
|
||||
});
|
||||
|
||||
test("another key mid-rotation refuses before anything is written", async () => {
|
||||
const w = world({ state: { key: "NODE_BROKER_TOKEN", phase: "staged" } });
|
||||
await assert.rejects(w.run(), /NODE_BROKER_TOKEN is mid-rotation/);
|
||||
assert.deepEqual(w.log, ["rotator status"]);
|
||||
});
|
||||
|
||||
test("retire and abort refuse without a rotation and roll when there is one", async () => {
|
||||
await assert.rejects(world({ action: "retire" }).run(), /no staged rotation/);
|
||||
await assert.rejects(world({ action: "abort" }).run(), /no rotation/);
|
||||
const abort = world({ action: "abort", state: { key: "INTERNAL_API_TOKEN", phase: "staged" } });
|
||||
await abort.run();
|
||||
assert.deepEqual(abort.log, ["rotator status", "rotator abort", "roll backend,tracking"]);
|
||||
const retire = world({ action: "retire", state: { key: "INTERNAL_API_TOKEN", phase: "staged" } });
|
||||
await retire.run();
|
||||
assert.deepEqual(retire.log, ["rotator status", "roll backend,tracking", "ops fleet env-status", "rotator retire", "roll backend,tracking"]);
|
||||
});
|
||||
|
||||
test("prepare rolls only what changed and says to reference the keys in Terraform", async () => {
|
||||
const w = world({ action: "prepare" });
|
||||
await w.run();
|
||||
assert.deepEqual(w.log, ["rotator status", "rotator prepare", "roll backend"]);
|
||||
assert.match(w.report.at(-1), /secret_keys/);
|
||||
});
|
||||
|
||||
test("rotate resumes a run that stopped after staging instead of refusing", async () => {
|
||||
const w = world({ state: { key: "INTERNAL_API_TOKEN", phase: "staged" } });
|
||||
assert.deepEqual(await w.run(), { phase: "none" });
|
||||
assert.deepEqual(w.log, ["rotator status", "roll backend,tracking", "ops fleet env-status", "rotator retire", "roll backend,tracking"]);
|
||||
const sign = world({ key: "UNSUBSCRIBE_SIGNING_SECRET", holders: ["backend"], state: { key: "UNSUBSCRIBE_SIGNING_SECRET", phase: "staged", until: "2027-10-25T00:00:00.000Z" } });
|
||||
assert.deepEqual(await sign.run(), { phase: "staged" });
|
||||
assert.deepEqual(sign.log, ["rotator status", "roll backend"]);
|
||||
assert.match(sign.report.at(-1), /after 2027-10-25/);
|
||||
});
|
||||
Executable
+74
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env bash
|
||||
# Plans or applies one AWS stack without putting a value in a public log:
|
||||
# var files come from the private state bucket and are masked first, plan and
|
||||
# apply output goes to a private file, and only resource addresses and actions
|
||||
# are printed.
|
||||
#
|
||||
# terraform-stack.sh plan <foundation|runtime|workers>
|
||||
# terraform-stack.sh apply <stack> applies the plan saved by plan
|
||||
# terraform-stack.sh classify <plan.json>
|
||||
#
|
||||
# plan writes $WORK/<stack>.summary (one "action address" per line) and prints
|
||||
# safe, gated or none. Safe means only creates, in-place updates, and new
|
||||
# revisions of task definitions, which keep their old revision.
|
||||
set -euo pipefail
|
||||
|
||||
classify() {
|
||||
jq -r '
|
||||
[.resource_changes[]? | select(.change.actions != ["no-op"] and .change.actions != ["read"])] as $c |
|
||||
if ($c | length) == 0 then "none"
|
||||
elif any($c[]; (.change.actions | index("delete")) and
|
||||
((.type == "aws_ecs_task_definition" and (.change.actions | length) == 2) | not)) then "gated"
|
||||
else "safe" end' "$1"
|
||||
}
|
||||
|
||||
summary() {
|
||||
jq -r '.resource_changes[]? | select(.change.actions != ["no-op"] and .change.actions != ["read"]) |
|
||||
"\(.change.actions | join("/")) \(.address)"' "$1" | sort
|
||||
}
|
||||
|
||||
main() {
|
||||
local action=$1 stack=${2:-}
|
||||
if [ "$action" = classify ]; then classify "$stack"; return 0; fi
|
||||
case "$stack" in foundation|runtime|workers) ;; *) echo "unknown stack $stack" >&2; return 1 ;; esac
|
||||
: "${STATE_BUCKET:?}" "${WORK:?}"
|
||||
local dir="deploy/terraform/aws/$stack" vars="$WORK/$stack.tfvars.json" out="$WORK/$stack.log"
|
||||
mkdir -p "$WORK" && chmod 700 "$WORK"
|
||||
|
||||
if [ "$action" = plan ]; then
|
||||
aws s3 cp --only-show-errors "s3://$STATE_BUCKET/warmbly/production/vars/$stack.tfvars.json" "$vars"
|
||||
chmod 600 "$vars"
|
||||
# Mask every value before anything can echo one.
|
||||
jq -r '.. | strings | select(length >= 6)' "$vars" | while IFS= read -r line; do
|
||||
[ -n "$line" ] && echo "::add-mask::$line"
|
||||
done
|
||||
terraform -chdir="$dir" init -input=false -no-color -backend-config="bucket=$STATE_BUCKET" >"$out" 2>&1 || { tail -n 20 "$out" >&2; return 1; }
|
||||
local code=0
|
||||
terraform -chdir="$dir" plan -input=false -no-color -lock-timeout=5m -var-file="$vars" -var="state_bucket=$STATE_BUCKET" \
|
||||
-out="$WORK/$stack.tfplan" -detailed-exitcode >>"$out" 2>"$WORK/$stack.err" || code=$?
|
||||
if [ "$code" = 1 ]; then
|
||||
echo "terraform plan failed for $stack:" >&2
|
||||
grep -E '^(Error|│ Error| on |│ on )' "$WORK/$stack.err" | head -n 20 >&2 || true
|
||||
return 1
|
||||
fi
|
||||
terraform -chdir="$dir" show -json "$WORK/$stack.tfplan" >"$WORK/$stack.json"
|
||||
summary "$WORK/$stack.json" >"$WORK/$stack.summary"
|
||||
classify "$WORK/$stack.json"
|
||||
rm -f "$WORK/$stack.json"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ "$action" = apply ]; then
|
||||
terraform -chdir="$dir" apply -input=false -no-color -lock-timeout=5m "$WORK/$stack.tfplan" >"$out" 2>"$WORK/$stack.err" || {
|
||||
echo "terraform apply failed for $stack:" >&2
|
||||
grep -E '^(Error|│ Error| on |│ on )' "$WORK/$stack.err" | head -n 20 >&2 || true
|
||||
return 1
|
||||
}
|
||||
grep -E '^Apply complete' "$out" || true
|
||||
return 0
|
||||
fi
|
||||
echo "unknown action $action" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user