Files
warmbly/.github/scripts/aws-deploy.test.cjs
T

383 lines
22 KiB
JavaScript

const { test } = require("node:test");
const assert = require("node:assert/strict");
const { mkdtempSync, writeFileSync } = require("node:fs");
const { join } = require("node:path");
const { tmpdir } = require("node:os");
const { configuration, deploy, runTask, compareTags, parseRef } = require("./aws-deploy.cjs");
const root = "arn:aws:ecs:us-east-1:000000000000:";
const registry = "ghcr.io/warmbly/warmbly";
const d = (c) => "sha256:" + c.repeat(64);
const KAFKA = ["backend", "consumer", "tracking", "migration", "ops"];
const FAMILIES = ["backend", "consumer", "forms", "tracking", "realtime", "migration", "validation", "ops"];
const secretArn = (svc) => `arn:aws:secretsmanager:us-east-1:000000000000:secret:fixture/${svc}-AbCdEf`;
function imagesFile(tag = "v1.3.0") {
const dir = mkdtempSync(join(tmpdir(), "aws-deploy-"));
const images = {};
for (const svc of ["backend", "consumer", "worker", "forms", "tracking", "realtime"]) images[svc] = d("b");
for (const svc of ["backend", "consumer", "worker", "tracking"]) images[`${svc}-kafka`] = d("c");
writeFileSync(join(dir, "images.json"), JSON.stringify({ tag, registry, images }));
return join(dir, "images.json");
}
function config(env = {}) {
return configuration({
AWS_ECS_CLUSTER: root + "cluster/fixture", DEPLOY_TAG: "v1.3.0", DEPLOY_IMAGES_JSON: imagesFile(env.DEPLOY_TAG || "v1.3.0"),
DEPLOY_BAKE_SECONDS: "30", ...env,
});
}
function definition(family, revision, options) {
const secretSvc = ["migration", "validation", "ops"].includes(family) ? "backend" : family;
const version = family === "backend" && revision === 1 ? "rotated" : options.versions?.[family] || "terraform";
const secrets = family === "migration"
? [{ name: "PRIMARY_DB", valueFrom: `${secretArn("backend")}:PRIMARY_DB_DIRECT::${version}` }]
: [
{ name: "PRIMARY_DB_DIRECT", valueFrom: `${secretArn(secretSvc)}:PRIMARY_DB_DIRECT::${version}` },
{ name: "REDIS_PASSWORD", valueFrom: `${secretArn(secretSvc)}:REDIS_PASSWORD::` },
];
if (family === "backend" && (revision === 1 || !options.dropReadKeys)) secrets.push({ name: "CREDENTIALS_ENCRYPTION_READ_KEYS", valueFrom: `${secretArn("backend")}:CREDENTIALS_ENCRYPTION_READ_KEYS::${version}` });
const image = family === "validation" ? "node:22-alpine"
: `${options.registry || registry}/${["migration", "ops"].includes(family) ? "backend" : family}@${d("a")}`;
return {
taskDefinitionArn: `${root}task-definition/fixture-${family}:${revision}`, family: `fixture-${family}`, revision, status: "ACTIVE",
networkMode: "awsvpc", cpu: "256", memory: "512", requiresCompatibilities: ["FARGATE"],
containerDefinitions: [{
name: family, image, secrets,
entryPoint: family === "migration" ? ["/app/migrate"] : undefined,
environment: [{ name: "DB_MIGRATIONS_ON_START", value: "false" }],
logConfiguration: { logDriver: "awslogs", options: { "awslogs-group": "/fixture", "awslogs-stream-prefix": family } },
}],
};
}
function fixture(options = {}) {
const calls = [];
const defs = {};
const tagsOf = {};
for (const family of FAMILIES) {
for (const revision of [1, 2]) {
const arn = `${root}task-definition/fixture-${family}:${revision}`;
defs[arn] = definition(family, revision, options);
const tags = [{ key: "Environment", value: "production" }];
if (!(options.noVariant && family === "backend" && revision === 2)) tags.push({ key: "warmbly-image-variant", value: KAFKA.includes(family) ? "kafka" : "default" });
if (revision === 1) tags.push({ key: "warmbly-release", value: options.liveTag || "v1.2.0" });
tagsOf[arn] = tags;
}
}
const latest = (family) => Object.values(defs).filter((x) => x.family === `fixture-${family}`).sort((a, b) => b.revision - a.revision)[0];
const services = ["backend", "consumer", "forms", "tracking", "realtime"].filter((n) => !options.absent?.includes(n)).map((name) => ({
serviceName: `fixture-${name}`, status: "ACTIVE", desiredCount: 2, runningCount: 2, pendingCount: 0,
taskDefinition: `${root}task-definition/fixture-${name}:1`, deployments: [{ rolloutState: "COMPLETED" }],
networkConfiguration: { awsvpcConfiguration: { subnets: ["subnet-fixture"], securityGroups: ["sg-fixture"], assignPublicIp: "DISABLED" } },
}));
const tasks = {};
const record = (client) => ({ async send(command) {
const name = command.constructor.name;
calls.push({ client, name, input: structuredClone(command.input) });
switch (name) {
case "DescribeTaskDefinitionCommand": {
const ref = command.input.taskDefinition;
const def = ref.startsWith("arn:") ? defs[ref] : latest(ref.replace(/^fixture-/, ""));
if (!def) throw new Error(`no task definition ${ref}`);
return { taskDefinition: structuredClone(def), tags: structuredClone(tagsOf[def.taskDefinitionArn]) };
}
case "RegisterTaskDefinitionCommand": {
const prev = latest(command.input.family.replace(/^fixture-/, ""));
const arn = `${root}task-definition/${command.input.family}:${prev.revision + 1}`;
defs[arn] = { ...structuredClone(command.input), revision: prev.revision + 1, taskDefinitionArn: arn, status: "ACTIVE" };
delete defs[arn].tags;
tagsOf[arn] = command.input.tags;
return { taskDefinition: { taskDefinitionArn: arn } };
}
case "DescribeServicesCommand": {
const found = services.filter((s) => command.input.services.includes(s.serviceName));
const failures = command.input.services.filter((n) => !found.some((s) => s.serviceName === n)).map((arn) => ({ arn, reason: "MISSING" }));
return { services: found.map((s) => structuredClone(s)), failures };
}
case "UpdateServiceCommand": {
const service = services.find((s) => s.serviceName === command.input.service);
const svc = service.serviceName.replace("fixture-", "");
if (options.updateFailure === svc && !command.input.taskDefinition.endsWith(":1")) throw new Error(`${svc} update failed`);
service.taskDefinition = command.input.taskDefinition;
if (command.input.desiredCount !== undefined) service.desiredCount = command.input.desiredCount;
service.deployments = [{ rolloutState: options.rolloutFailure === svc && !command.input.taskDefinition.endsWith(":1") ? "FAILED" : "COMPLETED" }];
if (options.circuitBreaker === svc) service.taskDefinition = `${root}task-definition/fixture-${svc}:1`;
return {};
}
case "RunTaskCommand": {
if (options.placementFailure) return { failures: [{ reason: "fixture" }] };
const family = /fixture-(\w+):/.exec(command.input.taskDefinition)[1];
const arn = `${root}task/fixture/${family}-${Object.keys(tasks).length}`;
tasks[arn] = family;
return { tasks: [{ taskArn: arn }] };
}
case "DescribeTasksCommand": {
const family = tasks[command.input.tasks[0]];
const exitCode = options.taskFailure === family ? 1 : 0;
return { tasks: [{ lastStatus: "STOPPED", stopCode: "EssentialContainerExited", containers: [{ name: family, exitCode }] }] };
}
case "GetLogEventsCommand":
return { events: [{ message: `log line from ${command.input.logStreamName}` }] };
case "DescribeAlarmsCommand": {
const firing = options.alarm && command.input.AlarmNamePrefix === `fixture-${options.alarm}-`;
return { MetricAlarms: firing ? [{ AlarmName: `fixture-${options.alarm}-5xx` }] : [] };
}
case "ListSecretVersionIdsCommand":
return { Versions: [
{ VersionId: "terraform", CreatedDate: new Date("2026-01-01") },
{ VersionId: "rotated", CreatedDate: new Date("2026-02-01") },
{ VersionId: "stale", CreatedDate: new Date("2025-01-01") },
] };
case "DescribeSecretCommand":
return { VersionIdsToStages: { terraform: ["AWSPREVIOUS"], "current-version-id-000000000000000000": ["AWSCURRENT"] } };
default:
throw new Error(`Unexpected command ${name}`);
}
} });
const clients = { ecs: record("ecs"), secrets: record("secrets"), cloudwatch: record("cloudwatch"), logs: record("logs") };
return { clients, calls, defs, tagsOf, services };
}
const run = (f, cfg, lines = []) => deploy(f.clients, cfg, async () => {}, (l) => lines.push(l));
const registered = (f) => f.calls.filter((c) => c.name === "RegisterTaskDefinitionCommand").map((c) => c.input);
const updates = (f) => f.calls.filter((c) => c.name === "UpdateServiceCommand").map((c) => c.input);
const runs = (f) => f.calls.filter((c) => c.name === "RunTaskCommand").map((c) => c.input);
const valueFrom = (def, key) => def.containerDefinitions[0].secrets.find((s) => s.name === key).valueFrom;
test("configuration refuses prereleases, bad modes and secret pins outside refresh-secrets", () => {
assert.throws(() => config({ DEPLOY_TAG: "v1.3.0-rc.1" }), /prereleases never deploy/);
assert.throws(() => config({ DEPLOY_MODE: "yolo" }), /DEPLOY_MODE/);
for (const value of ["0", "299", "3601", "NaN", "300.5"]) {
assert.throws(() => config({ DEPLOY_FLEET_BAKE_SECONDS: value }), /DEPLOY_FLEET_BAKE_SECONDS/);
}
assert.equal(config().fleetBake, 300);
assert.equal(config({ DEPLOY_FLEET_BAKE_SECONDS: "1800" }).fleetBake, 1800);
assert.throws(() => config({ DEPLOY_SERVICES: "backend,worker" }), /DEPLOY_SERVICES/);
assert.throws(() => config({ DEPLOY_SECRET_VERSIONS: JSON.stringify({ backend: "a".repeat(36) }) }), /refresh-secrets/);
assert.throws(() => config({ DEPLOY_MODE: "refresh-secrets", DEPLOY_SECRET_VERSIONS: JSON.stringify({ backend: "x" }) }), /secret version id/);
assert.ok(compareTags("v1.10.0", "v1.9.9") > 0);
assert.deepEqual(parseRef(`${secretArn("backend")}:K::v`), { arn: secretArn("backend"), key: "K", stage: "", version: "v" });
});
test("a release renders from the latest revision, swaps variant digests and carries tags", async () => {
const f = fixture();
await run(f, config());
const byFamily = Object.fromEntries(registered(f).map((r) => [r.family.replace("fixture-", ""), r]));
assert.equal(byFamily.backend.containerDefinitions[0].image, `${registry}/backend@${d("c")}`);
assert.equal(byFamily.forms.containerDefinitions[0].image, `${registry}/forms@${d("b")}`);
assert.equal(byFamily.tracking.containerDefinitions[0].image, `${registry}/tracking@${d("c")}`);
assert.equal(byFamily.migration.containerDefinitions[0].image, `${registry}/backend@${d("c")}`);
assert.equal(byFamily.ops.containerDefinitions[0].image, `${registry}/backend@${d("c")}`);
assert.equal(byFamily.validation.containerDefinitions[0].image, "node:22-alpine");
for (const r of Object.values(byFamily)) {
assert.ok(r.tags.some((t) => t.key === "Environment"));
assert.deepEqual(r.tags.filter((t) => t.key === "warmbly-release"), [{ key: "warmbly-release", value: "v1.3.0" }]);
assert.equal(r.cpu, "256");
}
});
test("the newest pinned secret version wins so a stale Terraform pin cannot undo a rotation", async () => {
const f = fixture();
await run(f, config());
const backend = registered(f).find((r) => r.family === "fixture-backend");
const migration = registered(f).find((r) => r.family === "fixture-migration");
assert.ok(valueFrom(backend, "PRIMARY_DB_DIRECT").endsWith("::rotated"));
assert.ok(valueFrom(migration, "PRIMARY_DB").endsWith("::rotated"));
assert.ok(valueFrom(backend, "REDIS_PASSWORD").endsWith("REDIS_PASSWORD::"));
const consumer = registered(f).find((r) => r.family === "fixture-consumer");
assert.ok(valueFrom(consumer, "PRIMARY_DB_DIRECT").endsWith("::terraform"));
});
test("refresh-secrets pins every secret to AWSCURRENT and keeps the live images", async () => {
const f = fixture();
await run(f, configuration({ AWS_ECS_CLUSTER: root + "cluster/fixture", DEPLOY_MODE: "refresh-secrets", DEPLOY_BAKE_SECONDS: "0" }));
const backend = registered(f).find((r) => r.family === "fixture-backend");
const consumer = registered(f).find((r) => r.family === "fixture-consumer");
assert.ok(valueFrom(backend, "PRIMARY_DB_DIRECT").endsWith("::current-version-id-000000000000000000"));
assert.ok(valueFrom(consumer, "PRIMARY_DB_DIRECT").endsWith("::current-version-id-000000000000000000"));
assert.equal(backend.containerDefinitions[0].image, `${registry}/backend@${d("a")}`);
assert.ok(backend.tags.some((t) => t.key === "warmbly-release" && t.value === "v1.2.0"));
assert.ok(!runs(f).some((r) => r.overrides), "refresh-secrets starts no fleet rollout");
});
test("explicit secret versions roll only what they name; other secrets keep their newest pin", async () => {
const f = fixture();
await run(f, configuration({ AWS_ECS_CLUSTER: root + "cluster/fixture", DEPLOY_MODE: "refresh-secrets", DEPLOY_BAKE_SECONDS: "0",
DEPLOY_SERVICES: "consumer", DEPLOY_SECRET_VERSIONS: JSON.stringify({ consumer: "explicit-version-id-0000000000000000" }) }));
const consumer = registered(f).find((r) => r.family === "fixture-consumer");
const validation = registered(f).find((r) => r.family === "fixture-validation");
assert.ok(valueFrom(consumer, "PRIMARY_DB_DIRECT").endsWith("::explicit-version-id-0000000000000000"));
assert.ok(valueFrom(validation, "PRIMARY_DB_DIRECT").endsWith("::rotated"));
assert.ok(!f.calls.some((c) => c.name === "DescribeSecretCommand"));
assert.deepEqual(updates(f).map((u) => u.service), ["fixture-consumer"]);
});
test("validation and migration gate every service update", async () => {
const f = fixture();
await run(f, config());
const firstUpdate = f.calls.findIndex((c) => c.name === "UpdateServiceCommand");
const taskRuns = f.calls.map((c, i) => [c, i]).filter(([c]) => c.name === "RunTaskCommand");
assert.deepEqual(taskRuns.slice(0, 2).map(([c]) => /fixture-(\w+):/.exec(c.input.taskDefinition)[1]), ["validation", "migration"]);
assert.ok(taskRuns[1][1] < firstUpdate);
assert.deepEqual(updates(f).map((u) => u.service), ["fixture-backend", "fixture-consumer", "fixture-forms", "fixture-tracking", "fixture-realtime"]);
assert.ok(updates(f).every((u) => !Object.hasOwn(u, "desiredCount")));
});
for (const [failure, message] of [
[{ taskFailure: "migration" }, /Migration failed/], [{ taskFailure: "validation" }, /Dependency check failed/],
[{ placementFailure: true }, /task-placement failure/], [{ noVariant: true }, /no warmbly-image-variant tag/],
[{ registry: "docker.io/someone" }, /does not run a digest from ghcr/],
]) {
test(`${JSON.stringify(failure)} leaves every service untouched`, async () => {
const f = fixture(failure);
await assert.rejects(run(f, config()), message);
assert.equal(updates(f).length, 0);
});
}
test("a failed migration prints its log tail", async () => {
const f = fixture({ taskFailure: "migration" });
const lines = [];
await assert.rejects(run(f, config(), lines), /Migration failed/);
assert.ok(lines.some((l) => l.includes("log line from migration/migration/")));
});
test("an older tag is refused unless the mode is rollback, and rollback skips the migration", async () => {
await assert.rejects(run(fixture({ liveTag: "v1.4.0" }), config()), /older than the live v1.4.0/);
await assert.rejects(run(fixture(), config({ DEPLOY_MODE: "rollback" })), /newer than the live/);
const f = fixture({ liveTag: "v1.4.0" });
await run(f, config({ DEPLOY_MODE: "rollback" }));
assert.ok(!runs(f).some((r) => r.taskDefinition.includes("fixture-migration")));
assert.ok(runs(f).some((r) => r.overrides), "rollback still moves the fleet");
});
test("a revision that drops a secret the live service reads is refused", async () => {
const f = fixture({ dropReadKeys: true });
await assert.rejects(run(f, config()), /would lose CREDENTIALS_ENCRYPTION_READ_KEYS/);
assert.equal(registered(f).length, 0);
const g = fixture({ dropReadKeys: true });
await run(g, config({ DEPLOY_ALLOW_DROPPED_SECRETS: "true" }));
});
test("an alarm during the bake restores every updated service in reverse order", async () => {
const f = fixture({ alarm: "consumer" });
await assert.rejects(run(f, config()), /fixture-consumer-5xx/);
const restores = updates(f).filter((u) => u.taskDefinition.endsWith(":1"));
assert.deepEqual(restores.map((u) => u.service), ["fixture-consumer", "fixture-backend"]);
assert.ok(restores.every((u) => u.desiredCount === 2));
assert.ok(!updates(f).some((u) => u.service === "fixture-forms"));
});
test("a failure in the parallel wave waits for its siblings and then restores all of them", async () => {
const f = fixture({ rolloutFailure: "forms" });
await assert.rejects(run(f, config()), /forms deployment failed/);
const forward = updates(f).filter((u) => !u.taskDefinition.endsWith(":1")).map((u) => u.service);
const restores = updates(f).filter((u) => u.taskDefinition.endsWith(":1")).map((u) => u.service);
assert.equal(forward.length, 5);
assert.deepEqual([...restores].sort(), [...forward].sort());
assert.deepEqual(restores.slice(-2), ["fixture-consumer", "fixture-backend"]);
});
test("a circuit breaker rollback cannot be reported as a successful deployment", async () => {
const f = fixture({ circuitBreaker: "backend" });
await assert.rejects(run(f, config()), /backend deployment rolled back/);
});
test("the fleet rollout gets the worker and consumer digests per variant", async () => {
const f = fixture();
await run(f, config());
const ops = runs(f).find((r) => r.overrides);
const command = ops.overrides.containerOverrides[0].command;
assert.deepEqual(command.slice(0, 4), ["fleet", "rollout", "start", "v1.3.0"]);
assert.equal(command[command.indexOf("--bake") + 1], "300s");
assert.equal(command[command.indexOf("--digests") + 1], `worker=${d("b")},worker-kafka=${d("c")},consumer=${d("b")},consumer-kafka=${d("c")}`);
assert.equal(ops.overrides.containerOverrides[0].name, "ops");
});
test("hosted fleet rollout carries a configured longer bake", async () => {
const f = fixture();
await run(f, config({ DEPLOY_FLEET_BAKE_SECONDS: "1800" }));
const command = runs(f).find((r) => r.overrides).overrides.containerOverrides[0].command;
assert.equal(command[command.indexOf("--bake") + 1], "1800s");
});
test("fleet startup retries only confirmed image-pull failures before execution, with stable retry tokens", async () => {
const pullFailure = { lastStatus: "STOPPED", stopCode: "TaskFailedToStart", stoppedReason: "CannotPullContainerError: manifest not found", containers: [{ name: "ops", runtimeId: "allocated-before-pull" }] };
const success = { lastStatus: "STOPPED", stopCode: "EssentialContainerExited", containers: [{ name: "ops", exitCode: 0 }] };
async function attempt(statuses, options = {}) {
const requests = [], sleeps = [];
const clients = {
ecs: { async send(command) {
if (command.constructor.name === "RunTaskCommand") {
requests.push(command.input);
return { tasks: [{ taskArn: `${root}task/fixture/${requests.length}` }] };
}
return { tasks: [statuses[Math.min(requests.length - 1, statuses.length - 1)]] };
} },
logs: { async send() { return { events: [] }; } },
};
const execute = () => runTask(clients, config(), {}, "fixture-ops:3", definition("ops", 3, {}), "Fleet rollout", async (ms) => sleeps.push(ms), () => {}, { command: ["fleet", "rollout", "start", "v1.3.0"], imagePullRetries: 2, ...options });
return { execute, requests, sleeps };
}
const recovered = await attempt([pullFailure, success]);
await recovered.execute();
assert.equal(recovered.requests.length, 2);
assert.deepEqual(recovered.sleeps, [30000]);
assert.deepEqual(recovered.requests[0].overrides, recovered.requests[1].overrides);
assert.equal(recovered.requests[0].taskDefinition, recovered.requests[1].taskDefinition);
assert.notEqual(recovered.requests[0].clientToken, recovered.requests[1].clientToken);
const replay = await attempt([pullFailure, success]);
await replay.execute();
assert.deepEqual(replay.requests.map((r) => r.clientToken), recovered.requests.map((r) => r.clientToken));
const exhausted = await attempt([pullFailure]);
await assert.rejects(exhausted.execute(), /Fleet rollout failed/);
assert.equal(exhausted.requests.length, 3);
assert.deepEqual(exhausted.sleeps, [30000, 60000]);
for (const failure of [
{ ...success, containers: [{ name: "ops", exitCode: 1 }] },
{ ...pullFailure, startedAt: new Date() },
{ ...pullFailure, containers: [{ name: "ops", exitCode: 1 }] },
{ ...pullFailure, containers: [] },
{ ...pullFailure, stoppedReason: "ResourceInitializationError: unavailable" },
]) {
const failed = await attempt([failure]);
await assert.rejects(failed.execute(), /Fleet rollout failed/);
assert.equal(failed.requests.length, 1);
}
const migration = await attempt([pullFailure], { imagePullRetries: 0 });
await assert.rejects(migration.execute(), /failed/);
assert.equal(migration.requests.length, 1);
});
test("an unchanged definition reuses its revision and redeploy forces a fresh deployment", async () => {
const f = fixture();
await run(f, config({ DEPLOY_SERVICES: "forms", DEPLOY_FLEET: "false" }));
const formsArn = updates(f)[0].taskDefinition;
f.services.find((s) => s.serviceName === "fixture-forms").taskDefinition = formsArn;
const before = registered(f).length;
await run(f, config({ DEPLOY_SERVICES: "forms", DEPLOY_MODE: "redeploy", DEPLOY_FLEET: "false" }));
assert.equal(registered(f).length, before);
assert.deepEqual(updates(f).at(-1), { cluster: root + "cluster/fixture", service: "fixture-forms", taskDefinition: formsArn, forceNewDeployment: true });
});
test("no secret value or version-free reference is ever read", async () => {
const f = fixture();
await run(f, config());
assert.ok(!f.calls.some((c) => c.name === "GetSecretValueCommand"));
});
test("an empty bake setting means the default, not no bake", () => {
assert.equal(config({ DEPLOY_BAKE_SECONDS: "" }).bake, 300);
assert.equal(config({ DEPLOY_BAKE_SECONDS: " " }).bake, 300);
assert.equal(config({ DEPLOY_BAKE_SECONDS: "0" }).bake, 0);
});
test("all deploys the services this install runs, and a named missing service is refused", async () => {
const f = fixture({ absent: ["forms", "tracking", "realtime"] });
await run(f, config());
assert.deepEqual(updates(f).map((u) => u.service).sort(), ["fixture-backend", "fixture-consumer"]);
await assert.rejects(run(fixture({ absent: ["forms"] }), config({ DEPLOY_SERVICES: "backend,forms" })), /forms must already exist/);
});