Files
warmbly/.github/scripts/aws-rotate.cjs
T

190 lines
8.9 KiB
JavaScript

// Drives a key rotation from GitHub without ever holding a value: the rotator
// task writes new secret versions inside AWS and reports their ids, the deploy
// engine rolls services onto exactly those versions, and warmblyctl in the ops
// task proves each step is safe before the old value is retired.
const { DescribeServicesCommand } = require("@aws-sdk/client-ecs");
const engine = require("./aws-deploy.cjs");
const { configuration, SERVICES } = engine;
const KINDS = {
CREDENTIALS_ENCRYPTION_KEY: "seal",
TWOFA_SECRET: "seal",
UNSUBSCRIBE_SIGNING_SECRET: "sign",
INTERNAL_API_TOKEN: "token",
NODE_BROKER_TOKEN: "token",
};
const ACTIONS = ["status", "prepare", "rotate", "retire", "abort"];
function rotationConfig(env) {
const action = env.ROTATE_ACTION;
const key = env.ROTATE_KEY;
if (!ACTIONS.includes(action)) throw new Error(`ROTATE_ACTION must be one of ${ACTIONS.join(", ")}`);
if (!KINDS[key]) throw new Error(`ROTATE_KEY must be one of ${Object.keys(KINDS).join(", ")}`);
const wait = Number(env.ROTATE_NODE_WAIT_MINUTES ?? 30);
if (!Number.isInteger(wait) || wait < 0 || wait > 240) throw new Error("ROTATE_NODE_WAIT_MINUTES must be 0 through 240");
const base = configuration({ ...env, DEPLOY_MODE: "refresh-secrets", DEPLOY_TAG: "", DEPLOY_SERVICES: "all", DEPLOY_SECRET_VERSIONS: "", DEPLOY_IMAGES_JSON: "", DEPLOY_FLEET: "false" });
return { ...base, action, key, kind: KINDS[key], nodeWaitMs: wait * 60000 };
}
async function rotateKeys(clients, config, sleep = (ms) => new Promise((r) => setTimeout(r, ms)), report = console.log, deps = engine) {
const { deploy, runTask, describe } = deps;
// ECS run tokens are idempotent, so every task and roll in one run needs its own.
let seq = 0;
const step = () => `${config.run}/${++seq}`;
const services = await clients.ecs.send(new DescribeServicesCommand({ cluster: config.cluster, services: [`${config.prefix}-backend`] }));
const network = services.services?.[0]?.networkConfiguration;
if (!network?.awsvpcConfiguration) throw new Error("Backend task networking is missing");
const rotator = async (action) => {
const { definition } = await describe(clients.ecs, `${config.prefix}-rotator`);
const environment = [
{ name: "ROTATE_ACTION", value: action }, { name: "ROTATE_KEY", value: config.key }, { name: "ROTATE_RUN", value: `${step()}/${action}` },
];
const lines = await runTask(clients, { ...config, run: step() }, network, definition.taskDefinitionArn, definition, `Rotator ${action}`, sleep, report, { environment, capture: (ls) => ls.some((l) => l.startsWith("ROTATION_RESULT ")) });
const line = lines.filter((l) => l.startsWith("ROTATION_RESULT ")).pop();
if (!line) throw new Error(`the rotator reported no result for ${action}`);
const result = JSON.parse(line.slice("ROTATION_RESULT ".length));
for (const l of lines) if (!l.startsWith("ROTATION_RESULT ")) report(` rotator: ${l}`);
return result;
};
const ops = async (label, command) => {
const { definition } = await describe(clients.ecs, `${config.prefix}-ops`);
return runTask(clients, { ...config, run: step() }, network, definition.taskDefinitionArn, definition, label, sleep, report, { command });
};
const roll = async (versions) => {
const names = Object.keys(versions || {}).filter((s) => SERVICES.includes(s));
if (!names.length) return;
report(`Rolling ${names.join(", ")} onto the new secret versions`);
await deploy(clients, { ...config, run: step(), services: names, allServices: false, secretVersions: versions }, sleep, report);
};
const nodesCurrent = async () => {
const checks = Math.floor(config.nodeWaitMs / 120000) + 1;
for (let i = 1; ; i++) {
try {
await ops("Fleet env status", ["fleet", "env-status"]);
report("Every live node has applied the new env");
return true;
} catch {
if (i >= checks) return false;
report("Some nodes have not applied the new env yet; checking again in two minutes");
await sleep(120000);
}
}
};
const resealed = async () => {
await ops("Reseal", ["keys", "reseal", "--apply"]);
try {
await ops("Key status", ["keys", "status", "--require-primary"]);
return true;
} catch {
return false;
}
};
const retire = async () => {
const result = await rotator("retire");
await roll(result.versions);
report(`${config.key} rotated; the previous value is retired`);
};
// Nodes first: a node still on the old env would seal under the old key after the reseal.
const gate = async () => {
if ((config.kind === "token" || config.key === "CREDENTIALS_ENCRYPTION_KEY") && !(await nodesCurrent())) {
return "some nodes have not applied the new env and still work on the previous value. Run retire once `warmblyctl fleet env-status` is clean";
}
if (config.kind === "seal" && !(await resealed())) return `some values still open only under the previous ${config.key}; run retire once \`warmblyctl keys status\` is clean`;
return null;
};
const status = await rotator("status");
if (status.key && status.key !== config.key) throw new Error(`${status.key} is mid-rotation; finish or abort it first`);
let phase = status.key === config.key ? status.phase : "none";
report(`${config.key}: ${phase}`);
// A run that died after writing resumes here: rolling versions already live is a no-op.
const resume = () => roll(status.pinned);
switch (config.action) {
case "status":
if (config.kind === "seal") await ops("Key status", ["keys", "status"]).catch(() => report("keys status reports unreadable values"));
return { phase };
case "prepare": {
const result = await rotator("prepare");
await roll(result.versions);
report(`Overlap keys exist for ${config.key}. Add them to secret_keys for ${result.holders.join(", ")} and apply the runtime Terraform before rotating.`);
return { phase };
}
case "abort": {
if (phase === "none") throw new Error(`no rotation of ${config.key} is in progress`);
const result = await rotator("abort");
await roll(result.versions);
report(`${config.key} restored to its previous value`);
return { phase: "none" };
}
case "retire": {
if (phase !== "staged") throw new Error(`${config.key} has no staged rotation to retire`);
await resume();
const blocked = await gate();
if (blocked) throw new Error(blocked);
await retire();
return { phase: "none" };
}
case "rotate": {
if (phase !== "none") await resume();
if (phase === "staged" && config.kind === "sign") {
report(`${config.key} is staged; run retire after ${status.until}`);
return { phase };
}
// introducing and staging are a stage that died part way; staging again finishes it with the saved values.
if (phase === "none" || phase === "introducing") {
const result = await rotator("stage");
await roll(result.versions);
phase = result.phase;
}
if (phase === "introduced" || phase === "staging") {
if (phase === "introduced" && !(await nodesCurrent())) {
report("The new key is readable everywhere except nodes that have not refreshed; run rotate again to make it primary");
return { phase };
}
const result = await rotator("stage");
await roll(result.versions);
phase = result.phase;
}
if (config.kind === "sign") {
report(`${config.key} rotated. Links signed with the previous value keep working for a year, the life of a link; run retire after that`);
return { phase };
}
const blocked = await gate();
if (blocked) {
report(`${config.key} is staged: ${blocked}`);
return { phase };
}
await retire();
return { phase: "none" };
}
}
return { phase };
}
if (require.main === module) {
const { ECSClient } = require("@aws-sdk/client-ecs");
const { SecretsManagerClient } = require("@aws-sdk/client-secrets-manager");
const { CloudWatchClient } = require("@aws-sdk/client-cloudwatch");
const { CloudWatchLogsClient } = require("@aws-sdk/client-cloudwatch-logs");
const region = { region: "us-east-1", maxAttempts: 3 };
const clients = { ecs: new ECSClient(region), secrets: new SecretsManagerClient(region), cloudwatch: new CloudWatchClient(region), logs: new CloudWatchLogsClient(region) };
const lines = [];
const report = (line) => { console.log(line); lines.push(line); };
Promise.resolve().then(() => rotateKeys(clients, rotationConfig(process.env), undefined, report)).catch((error) => {
const message = error?.$metadata ? `AWS request failed: ${error.name}` : error.message;
console.error(message);
lines.push(`**Failed:** ${message}`);
process.exitCode = 1;
}).finally(() => {
if (process.env.GITHUB_STEP_SUMMARY) {
require("node:fs").appendFileSync(process.env.GITHUB_STEP_SUMMARY, `### Key rotation\n\n\`\`\`\n${lines.join("\n")}\n\`\`\`\n`);
}
for (const c of Object.values(clients)) c.destroy();
});
}
module.exports = { rotationConfig, rotateKeys };