mirror of
https://github.com/warmbly/warmbly.git
synced 2026-11-17 16:19:13 +00:00
190 lines
8.9 KiB
JavaScript
190 lines
8.9 KiB
JavaScript
// Drives a key rotation from GitHub without ever holding a value: the rotator
|
|
// task writes new secret versions inside AWS and reports their ids, the deploy
|
|
// engine rolls services onto exactly those versions, and warmblyctl in the ops
|
|
// task proves each step is safe before the old value is retired.
|
|
const { DescribeServicesCommand } = require("@aws-sdk/client-ecs");
|
|
const engine = require("./aws-deploy.cjs");
|
|
|
|
const { configuration, SERVICES } = engine;
|
|
|
|
const KINDS = {
|
|
CREDENTIALS_ENCRYPTION_KEY: "seal",
|
|
TWOFA_SECRET: "seal",
|
|
UNSUBSCRIBE_SIGNING_SECRET: "sign",
|
|
INTERNAL_API_TOKEN: "token",
|
|
NODE_BROKER_TOKEN: "token",
|
|
};
|
|
const ACTIONS = ["status", "prepare", "rotate", "retire", "abort"];
|
|
|
|
function rotationConfig(env) {
|
|
const action = env.ROTATE_ACTION;
|
|
const key = env.ROTATE_KEY;
|
|
if (!ACTIONS.includes(action)) throw new Error(`ROTATE_ACTION must be one of ${ACTIONS.join(", ")}`);
|
|
if (!KINDS[key]) throw new Error(`ROTATE_KEY must be one of ${Object.keys(KINDS).join(", ")}`);
|
|
const wait = Number(env.ROTATE_NODE_WAIT_MINUTES ?? 30);
|
|
if (!Number.isInteger(wait) || wait < 0 || wait > 240) throw new Error("ROTATE_NODE_WAIT_MINUTES must be 0 through 240");
|
|
const base = configuration({ ...env, DEPLOY_MODE: "refresh-secrets", DEPLOY_TAG: "", DEPLOY_SERVICES: "all", DEPLOY_SECRET_VERSIONS: "", DEPLOY_IMAGES_JSON: "", DEPLOY_FLEET: "false" });
|
|
return { ...base, action, key, kind: KINDS[key], nodeWaitMs: wait * 60000 };
|
|
}
|
|
|
|
async function rotateKeys(clients, config, sleep = (ms) => new Promise((r) => setTimeout(r, ms)), report = console.log, deps = engine) {
|
|
const { deploy, runTask, describe } = deps;
|
|
// ECS run tokens are idempotent, so every task and roll in one run needs its own.
|
|
let seq = 0;
|
|
const step = () => `${config.run}/${++seq}`;
|
|
const services = await clients.ecs.send(new DescribeServicesCommand({ cluster: config.cluster, services: [`${config.prefix}-backend`] }));
|
|
const network = services.services?.[0]?.networkConfiguration;
|
|
if (!network?.awsvpcConfiguration) throw new Error("Backend task networking is missing");
|
|
|
|
const rotator = async (action) => {
|
|
const { definition } = await describe(clients.ecs, `${config.prefix}-rotator`);
|
|
const environment = [
|
|
{ name: "ROTATE_ACTION", value: action }, { name: "ROTATE_KEY", value: config.key }, { name: "ROTATE_RUN", value: `${step()}/${action}` },
|
|
];
|
|
const lines = await runTask(clients, { ...config, run: step() }, network, definition.taskDefinitionArn, definition, `Rotator ${action}`, sleep, report, { environment, capture: (ls) => ls.some((l) => l.startsWith("ROTATION_RESULT ")) });
|
|
const line = lines.filter((l) => l.startsWith("ROTATION_RESULT ")).pop();
|
|
if (!line) throw new Error(`the rotator reported no result for ${action}`);
|
|
const result = JSON.parse(line.slice("ROTATION_RESULT ".length));
|
|
for (const l of lines) if (!l.startsWith("ROTATION_RESULT ")) report(` rotator: ${l}`);
|
|
return result;
|
|
};
|
|
const ops = async (label, command) => {
|
|
const { definition } = await describe(clients.ecs, `${config.prefix}-ops`);
|
|
return runTask(clients, { ...config, run: step() }, network, definition.taskDefinitionArn, definition, label, sleep, report, { command });
|
|
};
|
|
const roll = async (versions) => {
|
|
const names = Object.keys(versions || {}).filter((s) => SERVICES.includes(s));
|
|
if (!names.length) return;
|
|
report(`Rolling ${names.join(", ")} onto the new secret versions`);
|
|
await deploy(clients, { ...config, run: step(), services: names, allServices: false, secretVersions: versions }, sleep, report);
|
|
};
|
|
const nodesCurrent = async () => {
|
|
const checks = Math.floor(config.nodeWaitMs / 120000) + 1;
|
|
for (let i = 1; ; i++) {
|
|
try {
|
|
await ops("Fleet env status", ["fleet", "env-status"]);
|
|
report("Every live node has applied the new env");
|
|
return true;
|
|
} catch {
|
|
if (i >= checks) return false;
|
|
report("Some nodes have not applied the new env yet; checking again in two minutes");
|
|
await sleep(120000);
|
|
}
|
|
}
|
|
};
|
|
const resealed = async () => {
|
|
await ops("Reseal", ["keys", "reseal", "--apply"]);
|
|
try {
|
|
await ops("Key status", ["keys", "status", "--require-primary"]);
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
};
|
|
const retire = async () => {
|
|
const result = await rotator("retire");
|
|
await roll(result.versions);
|
|
report(`${config.key} rotated; the previous value is retired`);
|
|
};
|
|
// Nodes first: a node still on the old env would seal under the old key after the reseal.
|
|
const gate = async () => {
|
|
if ((config.kind === "token" || config.key === "CREDENTIALS_ENCRYPTION_KEY") && !(await nodesCurrent())) {
|
|
return "some nodes have not applied the new env and still work on the previous value. Run retire once `warmblyctl fleet env-status` is clean";
|
|
}
|
|
if (config.kind === "seal" && !(await resealed())) return `some values still open only under the previous ${config.key}; run retire once \`warmblyctl keys status\` is clean`;
|
|
return null;
|
|
};
|
|
|
|
const status = await rotator("status");
|
|
if (status.key && status.key !== config.key) throw new Error(`${status.key} is mid-rotation; finish or abort it first`);
|
|
let phase = status.key === config.key ? status.phase : "none";
|
|
report(`${config.key}: ${phase}`);
|
|
// A run that died after writing resumes here: rolling versions already live is a no-op.
|
|
const resume = () => roll(status.pinned);
|
|
|
|
switch (config.action) {
|
|
case "status":
|
|
if (config.kind === "seal") await ops("Key status", ["keys", "status"]).catch(() => report("keys status reports unreadable values"));
|
|
return { phase };
|
|
case "prepare": {
|
|
const result = await rotator("prepare");
|
|
await roll(result.versions);
|
|
report(`Overlap keys exist for ${config.key}. Add them to secret_keys for ${result.holders.join(", ")} and apply the runtime Terraform before rotating.`);
|
|
return { phase };
|
|
}
|
|
case "abort": {
|
|
if (phase === "none") throw new Error(`no rotation of ${config.key} is in progress`);
|
|
const result = await rotator("abort");
|
|
await roll(result.versions);
|
|
report(`${config.key} restored to its previous value`);
|
|
return { phase: "none" };
|
|
}
|
|
case "retire": {
|
|
if (phase !== "staged") throw new Error(`${config.key} has no staged rotation to retire`);
|
|
await resume();
|
|
const blocked = await gate();
|
|
if (blocked) throw new Error(blocked);
|
|
await retire();
|
|
return { phase: "none" };
|
|
}
|
|
case "rotate": {
|
|
if (phase !== "none") await resume();
|
|
if (phase === "staged" && config.kind === "sign") {
|
|
report(`${config.key} is staged; run retire after ${status.until}`);
|
|
return { phase };
|
|
}
|
|
// introducing and staging are a stage that died part way; staging again finishes it with the saved values.
|
|
if (phase === "none" || phase === "introducing") {
|
|
const result = await rotator("stage");
|
|
await roll(result.versions);
|
|
phase = result.phase;
|
|
}
|
|
if (phase === "introduced" || phase === "staging") {
|
|
if (phase === "introduced" && !(await nodesCurrent())) {
|
|
report("The new key is readable everywhere except nodes that have not refreshed; run rotate again to make it primary");
|
|
return { phase };
|
|
}
|
|
const result = await rotator("stage");
|
|
await roll(result.versions);
|
|
phase = result.phase;
|
|
}
|
|
if (config.kind === "sign") {
|
|
report(`${config.key} rotated. Links signed with the previous value keep working for a year, the life of a link; run retire after that`);
|
|
return { phase };
|
|
}
|
|
const blocked = await gate();
|
|
if (blocked) {
|
|
report(`${config.key} is staged: ${blocked}`);
|
|
return { phase };
|
|
}
|
|
await retire();
|
|
return { phase: "none" };
|
|
}
|
|
}
|
|
return { phase };
|
|
}
|
|
|
|
if (require.main === module) {
|
|
const { ECSClient } = require("@aws-sdk/client-ecs");
|
|
const { SecretsManagerClient } = require("@aws-sdk/client-secrets-manager");
|
|
const { CloudWatchClient } = require("@aws-sdk/client-cloudwatch");
|
|
const { CloudWatchLogsClient } = require("@aws-sdk/client-cloudwatch-logs");
|
|
const region = { region: "us-east-1", maxAttempts: 3 };
|
|
const clients = { ecs: new ECSClient(region), secrets: new SecretsManagerClient(region), cloudwatch: new CloudWatchClient(region), logs: new CloudWatchLogsClient(region) };
|
|
const lines = [];
|
|
const report = (line) => { console.log(line); lines.push(line); };
|
|
Promise.resolve().then(() => rotateKeys(clients, rotationConfig(process.env), undefined, report)).catch((error) => {
|
|
const message = error?.$metadata ? `AWS request failed: ${error.name}` : error.message;
|
|
console.error(message);
|
|
lines.push(`**Failed:** ${message}`);
|
|
process.exitCode = 1;
|
|
}).finally(() => {
|
|
if (process.env.GITHUB_STEP_SUMMARY) {
|
|
require("node:fs").appendFileSync(process.env.GITHUB_STEP_SUMMARY, `### Key rotation\n\n\`\`\`\n${lines.join("\n")}\n\`\`\`\n`);
|
|
}
|
|
for (const c of Object.values(clients)) c.destroy();
|
|
});
|
|
}
|
|
|
|
module.exports = { rotationConfig, rotateKeys };
|