mirror of
https://github.com/warmbly/warmbly.git
synced 2026-11-17 08:18:51 +00:00
322 lines
19 KiB
JavaScript
322 lines
19 KiB
JavaScript
const { test } = require("node:test");
|
|
const assert = require("node:assert/strict");
|
|
const { mkdtempSync, writeFileSync } = require("node:fs");
|
|
const { join } = require("node:path");
|
|
const { tmpdir } = require("node:os");
|
|
const { configuration, deploy, compareTags, parseRef } = require("./aws-deploy.cjs");
|
|
|
|
const root = "arn:aws:ecs:us-east-1:000000000000:";
|
|
const registry = "ghcr.io/warmbly/warmbly";
|
|
const d = (c) => "sha256:" + c.repeat(64);
|
|
const KAFKA = ["backend", "consumer", "tracking", "migration", "ops"];
|
|
const FAMILIES = ["backend", "consumer", "forms", "tracking", "realtime", "migration", "validation", "ops"];
|
|
const secretArn = (svc) => `arn:aws:secretsmanager:us-east-1:000000000000:secret:fixture/${svc}-AbCdEf`;
|
|
|
|
function imagesFile(tag = "v1.3.0") {
|
|
const dir = mkdtempSync(join(tmpdir(), "aws-deploy-"));
|
|
const images = {};
|
|
for (const svc of ["backend", "consumer", "worker", "forms", "tracking", "realtime"]) images[svc] = d("b");
|
|
for (const svc of ["backend", "consumer", "worker", "tracking"]) images[`${svc}-kafka`] = d("c");
|
|
writeFileSync(join(dir, "images.json"), JSON.stringify({ tag, registry, images }));
|
|
return join(dir, "images.json");
|
|
}
|
|
|
|
function config(env = {}) {
|
|
return configuration({
|
|
AWS_ECS_CLUSTER: root + "cluster/fixture", DEPLOY_TAG: "v1.3.0", DEPLOY_IMAGES_JSON: imagesFile(env.DEPLOY_TAG || "v1.3.0"),
|
|
DEPLOY_BAKE_SECONDS: "30", ...env,
|
|
});
|
|
}
|
|
|
|
function definition(family, revision, options) {
|
|
const secretSvc = ["migration", "validation", "ops"].includes(family) ? "backend" : family;
|
|
const version = family === "backend" && revision === 1 ? "rotated" : options.versions?.[family] || "terraform";
|
|
const secrets = family === "migration"
|
|
? [{ name: "PRIMARY_DB", valueFrom: `${secretArn("backend")}:PRIMARY_DB_DIRECT::${version}` }]
|
|
: [
|
|
{ name: "PRIMARY_DB_DIRECT", valueFrom: `${secretArn(secretSvc)}:PRIMARY_DB_DIRECT::${version}` },
|
|
{ name: "REDIS_PASSWORD", valueFrom: `${secretArn(secretSvc)}:REDIS_PASSWORD::` },
|
|
];
|
|
if (family === "backend" && (revision === 1 || !options.dropReadKeys)) secrets.push({ name: "CREDENTIALS_ENCRYPTION_READ_KEYS", valueFrom: `${secretArn("backend")}:CREDENTIALS_ENCRYPTION_READ_KEYS::${version}` });
|
|
const image = family === "validation" ? "node:22-alpine"
|
|
: `${options.registry || registry}/${["migration", "ops"].includes(family) ? "backend" : family}@${d("a")}`;
|
|
return {
|
|
taskDefinitionArn: `${root}task-definition/fixture-${family}:${revision}`, family: `fixture-${family}`, revision, status: "ACTIVE",
|
|
networkMode: "awsvpc", cpu: "256", memory: "512", requiresCompatibilities: ["FARGATE"],
|
|
containerDefinitions: [{
|
|
name: family, image, secrets,
|
|
entryPoint: family === "migration" ? ["/app/migrate"] : undefined,
|
|
environment: [{ name: "DB_MIGRATIONS_ON_START", value: "false" }],
|
|
logConfiguration: { logDriver: "awslogs", options: { "awslogs-group": "/fixture", "awslogs-stream-prefix": family } },
|
|
}],
|
|
};
|
|
}
|
|
|
|
function fixture(options = {}) {
|
|
const calls = [];
|
|
const defs = {};
|
|
const tagsOf = {};
|
|
for (const family of FAMILIES) {
|
|
for (const revision of [1, 2]) {
|
|
const arn = `${root}task-definition/fixture-${family}:${revision}`;
|
|
defs[arn] = definition(family, revision, options);
|
|
const tags = [{ key: "Environment", value: "production" }];
|
|
if (!(options.noVariant && family === "backend" && revision === 2)) tags.push({ key: "warmbly-image-variant", value: KAFKA.includes(family) ? "kafka" : "default" });
|
|
if (revision === 1) tags.push({ key: "warmbly-release", value: options.liveTag || "v1.2.0" });
|
|
tagsOf[arn] = tags;
|
|
}
|
|
}
|
|
const latest = (family) => Object.values(defs).filter((x) => x.family === `fixture-${family}`).sort((a, b) => b.revision - a.revision)[0];
|
|
const services = ["backend", "consumer", "forms", "tracking", "realtime"].filter((n) => !options.absent?.includes(n)).map((name) => ({
|
|
serviceName: `fixture-${name}`, status: "ACTIVE", desiredCount: 2, runningCount: 2, pendingCount: 0,
|
|
taskDefinition: `${root}task-definition/fixture-${name}:1`, deployments: [{ rolloutState: "COMPLETED" }],
|
|
networkConfiguration: { awsvpcConfiguration: { subnets: ["subnet-fixture"], securityGroups: ["sg-fixture"], assignPublicIp: "DISABLED" } },
|
|
}));
|
|
const tasks = {};
|
|
const record = (client) => ({ async send(command) {
|
|
const name = command.constructor.name;
|
|
calls.push({ client, name, input: structuredClone(command.input) });
|
|
switch (name) {
|
|
case "DescribeTaskDefinitionCommand": {
|
|
const ref = command.input.taskDefinition;
|
|
const def = ref.startsWith("arn:") ? defs[ref] : latest(ref.replace(/^fixture-/, ""));
|
|
if (!def) throw new Error(`no task definition ${ref}`);
|
|
return { taskDefinition: structuredClone(def), tags: structuredClone(tagsOf[def.taskDefinitionArn]) };
|
|
}
|
|
case "RegisterTaskDefinitionCommand": {
|
|
const prev = latest(command.input.family.replace(/^fixture-/, ""));
|
|
const arn = `${root}task-definition/${command.input.family}:${prev.revision + 1}`;
|
|
defs[arn] = { ...structuredClone(command.input), revision: prev.revision + 1, taskDefinitionArn: arn, status: "ACTIVE" };
|
|
delete defs[arn].tags;
|
|
tagsOf[arn] = command.input.tags;
|
|
return { taskDefinition: { taskDefinitionArn: arn } };
|
|
}
|
|
case "DescribeServicesCommand": {
|
|
const found = services.filter((s) => command.input.services.includes(s.serviceName));
|
|
const failures = command.input.services.filter((n) => !found.some((s) => s.serviceName === n)).map((arn) => ({ arn, reason: "MISSING" }));
|
|
return { services: found.map((s) => structuredClone(s)), failures };
|
|
}
|
|
case "UpdateServiceCommand": {
|
|
const service = services.find((s) => s.serviceName === command.input.service);
|
|
const svc = service.serviceName.replace("fixture-", "");
|
|
if (options.updateFailure === svc && !command.input.taskDefinition.endsWith(":1")) throw new Error(`${svc} update failed`);
|
|
service.taskDefinition = command.input.taskDefinition;
|
|
if (command.input.desiredCount !== undefined) service.desiredCount = command.input.desiredCount;
|
|
service.deployments = [{ rolloutState: options.rolloutFailure === svc && !command.input.taskDefinition.endsWith(":1") ? "FAILED" : "COMPLETED" }];
|
|
if (options.circuitBreaker === svc) service.taskDefinition = `${root}task-definition/fixture-${svc}:1`;
|
|
return {};
|
|
}
|
|
case "RunTaskCommand": {
|
|
if (options.placementFailure) return { failures: [{ reason: "fixture" }] };
|
|
const family = /fixture-(\w+):/.exec(command.input.taskDefinition)[1];
|
|
const arn = `${root}task/fixture/${family}-${Object.keys(tasks).length}`;
|
|
tasks[arn] = family;
|
|
return { tasks: [{ taskArn: arn }] };
|
|
}
|
|
case "DescribeTasksCommand": {
|
|
const family = tasks[command.input.tasks[0]];
|
|
const exitCode = options.taskFailure === family ? 1 : 0;
|
|
return { tasks: [{ lastStatus: "STOPPED", stopCode: "EssentialContainerExited", containers: [{ name: family, exitCode }] }] };
|
|
}
|
|
case "GetLogEventsCommand":
|
|
return { events: [{ message: `log line from ${command.input.logStreamName}` }] };
|
|
case "DescribeAlarmsCommand": {
|
|
const firing = options.alarm && command.input.AlarmNamePrefix === `fixture-${options.alarm}-`;
|
|
return { MetricAlarms: firing ? [{ AlarmName: `fixture-${options.alarm}-5xx` }] : [] };
|
|
}
|
|
case "ListSecretVersionIdsCommand":
|
|
return { Versions: [
|
|
{ VersionId: "terraform", CreatedDate: new Date("2026-01-01") },
|
|
{ VersionId: "rotated", CreatedDate: new Date("2026-02-01") },
|
|
{ VersionId: "stale", CreatedDate: new Date("2025-01-01") },
|
|
] };
|
|
case "DescribeSecretCommand":
|
|
return { VersionIdsToStages: { terraform: ["AWSPREVIOUS"], "current-version-id-000000000000000000": ["AWSCURRENT"] } };
|
|
default:
|
|
throw new Error(`Unexpected command ${name}`);
|
|
}
|
|
} });
|
|
const clients = { ecs: record("ecs"), secrets: record("secrets"), cloudwatch: record("cloudwatch"), logs: record("logs") };
|
|
return { clients, calls, defs, tagsOf, services };
|
|
}
|
|
|
|
const run = (f, cfg, lines = []) => deploy(f.clients, cfg, async () => {}, (l) => lines.push(l));
|
|
const registered = (f) => f.calls.filter((c) => c.name === "RegisterTaskDefinitionCommand").map((c) => c.input);
|
|
const updates = (f) => f.calls.filter((c) => c.name === "UpdateServiceCommand").map((c) => c.input);
|
|
const runs = (f) => f.calls.filter((c) => c.name === "RunTaskCommand").map((c) => c.input);
|
|
const valueFrom = (def, key) => def.containerDefinitions[0].secrets.find((s) => s.name === key).valueFrom;
|
|
|
|
test("configuration refuses prereleases, bad modes and secret pins outside refresh-secrets", () => {
|
|
assert.throws(() => config({ DEPLOY_TAG: "v1.3.0-rc.1" }), /prereleases never deploy/);
|
|
assert.throws(() => config({ DEPLOY_MODE: "yolo" }), /DEPLOY_MODE/);
|
|
assert.throws(() => config({ DEPLOY_SERVICES: "backend,worker" }), /DEPLOY_SERVICES/);
|
|
assert.throws(() => config({ DEPLOY_SECRET_VERSIONS: JSON.stringify({ backend: "a".repeat(36) }) }), /refresh-secrets/);
|
|
assert.throws(() => config({ DEPLOY_MODE: "refresh-secrets", DEPLOY_SECRET_VERSIONS: JSON.stringify({ backend: "x" }) }), /secret version id/);
|
|
assert.ok(compareTags("v1.10.0", "v1.9.9") > 0);
|
|
assert.deepEqual(parseRef(`${secretArn("backend")}:K::v`), { arn: secretArn("backend"), key: "K", stage: "", version: "v" });
|
|
});
|
|
|
|
test("a release renders from the latest revision, swaps variant digests and carries tags", async () => {
|
|
const f = fixture();
|
|
await run(f, config());
|
|
const byFamily = Object.fromEntries(registered(f).map((r) => [r.family.replace("fixture-", ""), r]));
|
|
assert.equal(byFamily.backend.containerDefinitions[0].image, `${registry}/backend@${d("c")}`);
|
|
assert.equal(byFamily.forms.containerDefinitions[0].image, `${registry}/forms@${d("b")}`);
|
|
assert.equal(byFamily.tracking.containerDefinitions[0].image, `${registry}/tracking@${d("c")}`);
|
|
assert.equal(byFamily.migration.containerDefinitions[0].image, `${registry}/backend@${d("c")}`);
|
|
assert.equal(byFamily.ops.containerDefinitions[0].image, `${registry}/backend@${d("c")}`);
|
|
assert.equal(byFamily.validation.containerDefinitions[0].image, "node:22-alpine");
|
|
for (const r of Object.values(byFamily)) {
|
|
assert.ok(r.tags.some((t) => t.key === "Environment"));
|
|
assert.deepEqual(r.tags.filter((t) => t.key === "warmbly-release"), [{ key: "warmbly-release", value: "v1.3.0" }]);
|
|
assert.equal(r.cpu, "256");
|
|
}
|
|
});
|
|
|
|
test("the newest pinned secret version wins so a stale Terraform pin cannot undo a rotation", async () => {
|
|
const f = fixture();
|
|
await run(f, config());
|
|
const backend = registered(f).find((r) => r.family === "fixture-backend");
|
|
const migration = registered(f).find((r) => r.family === "fixture-migration");
|
|
assert.ok(valueFrom(backend, "PRIMARY_DB_DIRECT").endsWith("::rotated"));
|
|
assert.ok(valueFrom(migration, "PRIMARY_DB").endsWith("::rotated"));
|
|
assert.ok(valueFrom(backend, "REDIS_PASSWORD").endsWith("REDIS_PASSWORD::"));
|
|
const consumer = registered(f).find((r) => r.family === "fixture-consumer");
|
|
assert.ok(valueFrom(consumer, "PRIMARY_DB_DIRECT").endsWith("::terraform"));
|
|
});
|
|
|
|
test("refresh-secrets pins every secret to AWSCURRENT and keeps the live images", async () => {
|
|
const f = fixture();
|
|
await run(f, configuration({ AWS_ECS_CLUSTER: root + "cluster/fixture", DEPLOY_MODE: "refresh-secrets", DEPLOY_BAKE_SECONDS: "0" }));
|
|
const backend = registered(f).find((r) => r.family === "fixture-backend");
|
|
const consumer = registered(f).find((r) => r.family === "fixture-consumer");
|
|
assert.ok(valueFrom(backend, "PRIMARY_DB_DIRECT").endsWith("::current-version-id-000000000000000000"));
|
|
assert.ok(valueFrom(consumer, "PRIMARY_DB_DIRECT").endsWith("::current-version-id-000000000000000000"));
|
|
assert.equal(backend.containerDefinitions[0].image, `${registry}/backend@${d("a")}`);
|
|
assert.ok(backend.tags.some((t) => t.key === "warmbly-release" && t.value === "v1.2.0"));
|
|
assert.ok(!runs(f).some((r) => r.overrides), "refresh-secrets starts no fleet rollout");
|
|
});
|
|
|
|
test("explicit secret versions roll only what they name; other secrets keep their newest pin", async () => {
|
|
const f = fixture();
|
|
await run(f, configuration({ AWS_ECS_CLUSTER: root + "cluster/fixture", DEPLOY_MODE: "refresh-secrets", DEPLOY_BAKE_SECONDS: "0",
|
|
DEPLOY_SERVICES: "consumer", DEPLOY_SECRET_VERSIONS: JSON.stringify({ consumer: "explicit-version-id-0000000000000000" }) }));
|
|
const consumer = registered(f).find((r) => r.family === "fixture-consumer");
|
|
const validation = registered(f).find((r) => r.family === "fixture-validation");
|
|
assert.ok(valueFrom(consumer, "PRIMARY_DB_DIRECT").endsWith("::explicit-version-id-0000000000000000"));
|
|
assert.ok(valueFrom(validation, "PRIMARY_DB_DIRECT").endsWith("::rotated"));
|
|
assert.ok(!f.calls.some((c) => c.name === "DescribeSecretCommand"));
|
|
assert.deepEqual(updates(f).map((u) => u.service), ["fixture-consumer"]);
|
|
});
|
|
|
|
test("validation and migration gate every service update", async () => {
|
|
const f = fixture();
|
|
await run(f, config());
|
|
const firstUpdate = f.calls.findIndex((c) => c.name === "UpdateServiceCommand");
|
|
const taskRuns = f.calls.map((c, i) => [c, i]).filter(([c]) => c.name === "RunTaskCommand");
|
|
assert.deepEqual(taskRuns.slice(0, 2).map(([c]) => /fixture-(\w+):/.exec(c.input.taskDefinition)[1]), ["validation", "migration"]);
|
|
assert.ok(taskRuns[1][1] < firstUpdate);
|
|
assert.deepEqual(updates(f).map((u) => u.service), ["fixture-backend", "fixture-consumer", "fixture-forms", "fixture-tracking", "fixture-realtime"]);
|
|
assert.ok(updates(f).every((u) => !Object.hasOwn(u, "desiredCount")));
|
|
});
|
|
|
|
for (const [failure, message] of [
|
|
[{ taskFailure: "migration" }, /Migration failed/], [{ taskFailure: "validation" }, /Dependency check failed/],
|
|
[{ placementFailure: true }, /task-placement failure/], [{ noVariant: true }, /no warmbly-image-variant tag/],
|
|
[{ registry: "docker.io/someone" }, /does not run a digest from ghcr/],
|
|
]) {
|
|
test(`${JSON.stringify(failure)} leaves every service untouched`, async () => {
|
|
const f = fixture(failure);
|
|
await assert.rejects(run(f, config()), message);
|
|
assert.equal(updates(f).length, 0);
|
|
});
|
|
}
|
|
|
|
test("a failed migration prints its log tail", async () => {
|
|
const f = fixture({ taskFailure: "migration" });
|
|
const lines = [];
|
|
await assert.rejects(run(f, config(), lines), /Migration failed/);
|
|
assert.ok(lines.some((l) => l.includes("log line from migration/migration/")));
|
|
});
|
|
|
|
test("an older tag is refused unless the mode is rollback, and rollback skips the migration", async () => {
|
|
await assert.rejects(run(fixture({ liveTag: "v1.4.0" }), config()), /older than the live v1.4.0/);
|
|
await assert.rejects(run(fixture(), config({ DEPLOY_MODE: "rollback" })), /newer than the live/);
|
|
const f = fixture({ liveTag: "v1.4.0" });
|
|
await run(f, config({ DEPLOY_MODE: "rollback" }));
|
|
assert.ok(!runs(f).some((r) => r.taskDefinition.includes("fixture-migration")));
|
|
assert.ok(runs(f).some((r) => r.overrides), "rollback still moves the fleet");
|
|
});
|
|
|
|
test("a revision that drops a secret the live service reads is refused", async () => {
|
|
const f = fixture({ dropReadKeys: true });
|
|
await assert.rejects(run(f, config()), /would lose CREDENTIALS_ENCRYPTION_READ_KEYS/);
|
|
assert.equal(registered(f).length, 0);
|
|
const g = fixture({ dropReadKeys: true });
|
|
await run(g, config({ DEPLOY_ALLOW_DROPPED_SECRETS: "true" }));
|
|
});
|
|
|
|
test("an alarm during the bake restores every updated service in reverse order", async () => {
|
|
const f = fixture({ alarm: "consumer" });
|
|
await assert.rejects(run(f, config()), /fixture-consumer-5xx/);
|
|
const restores = updates(f).filter((u) => u.taskDefinition.endsWith(":1"));
|
|
assert.deepEqual(restores.map((u) => u.service), ["fixture-consumer", "fixture-backend"]);
|
|
assert.ok(restores.every((u) => u.desiredCount === 2));
|
|
assert.ok(!updates(f).some((u) => u.service === "fixture-forms"));
|
|
});
|
|
|
|
test("a failure in the parallel wave waits for its siblings and then restores all of them", async () => {
|
|
const f = fixture({ rolloutFailure: "forms" });
|
|
await assert.rejects(run(f, config()), /forms deployment failed/);
|
|
const forward = updates(f).filter((u) => !u.taskDefinition.endsWith(":1")).map((u) => u.service);
|
|
const restores = updates(f).filter((u) => u.taskDefinition.endsWith(":1")).map((u) => u.service);
|
|
assert.equal(forward.length, 5);
|
|
assert.deepEqual([...restores].sort(), [...forward].sort());
|
|
assert.deepEqual(restores.slice(-2), ["fixture-consumer", "fixture-backend"]);
|
|
});
|
|
|
|
test("a circuit breaker rollback cannot be reported as a successful deployment", async () => {
|
|
const f = fixture({ circuitBreaker: "backend" });
|
|
await assert.rejects(run(f, config()), /backend deployment rolled back/);
|
|
});
|
|
|
|
test("the fleet rollout gets the worker and consumer digests per variant", async () => {
|
|
const f = fixture();
|
|
await run(f, config());
|
|
const ops = runs(f).find((r) => r.overrides);
|
|
const command = ops.overrides.containerOverrides[0].command;
|
|
assert.deepEqual(command.slice(0, 4), ["fleet", "rollout", "start", "v1.3.0"]);
|
|
assert.equal(command[command.indexOf("--digests") + 1], `worker=${d("b")},worker-kafka=${d("c")},consumer=${d("b")},consumer-kafka=${d("c")}`);
|
|
assert.equal(ops.overrides.containerOverrides[0].name, "ops");
|
|
});
|
|
|
|
test("an unchanged definition reuses its revision and redeploy forces a fresh deployment", async () => {
|
|
const f = fixture();
|
|
await run(f, config({ DEPLOY_SERVICES: "forms", DEPLOY_FLEET: "false" }));
|
|
const formsArn = updates(f)[0].taskDefinition;
|
|
f.services.find((s) => s.serviceName === "fixture-forms").taskDefinition = formsArn;
|
|
const before = registered(f).length;
|
|
await run(f, config({ DEPLOY_SERVICES: "forms", DEPLOY_MODE: "redeploy", DEPLOY_FLEET: "false" }));
|
|
assert.equal(registered(f).length, before);
|
|
assert.deepEqual(updates(f).at(-1), { cluster: root + "cluster/fixture", service: "fixture-forms", taskDefinition: formsArn, forceNewDeployment: true });
|
|
});
|
|
|
|
test("no secret value or version-free reference is ever read", async () => {
|
|
const f = fixture();
|
|
await run(f, config());
|
|
assert.ok(!f.calls.some((c) => c.name === "GetSecretValueCommand"));
|
|
});
|
|
|
|
test("an empty bake setting means the default, not no bake", () => {
|
|
assert.equal(config({ DEPLOY_BAKE_SECONDS: "" }).bake, 300);
|
|
assert.equal(config({ DEPLOY_BAKE_SECONDS: " " }).bake, 300);
|
|
assert.equal(config({ DEPLOY_BAKE_SECONDS: "0" }).bake, 0);
|
|
});
|
|
|
|
test("all deploys the services this install runs, and a named missing service is refused", async () => {
|
|
const f = fixture({ absent: ["forms", "tracking", "realtime"] });
|
|
await run(f, config());
|
|
assert.deepEqual(updates(f).map((u) => u.service).sort(), ["fixture-backend", "fixture-consumer"]);
|
|
await assert.rejects(run(fixture({ absent: ["forms"] }), config({ DEPLOY_SERVICES: "backend,forms" })), /forms must already exist/);
|
|
});
|