diff --git a/.github/scripts/aws-origin-tls-workflow.test.sh b/.github/scripts/aws-origin-tls-workflow.test.sh new file mode 100644 index 000000000..ddf18fb41 --- /dev/null +++ b/.github/scripts/aws-origin-tls-workflow.test.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/../.." +ruby -ryaml <<'RUBY' +workflow = YAML.load_file('.github/workflows/probe-origin-tls-aws.yml') +trigger = workflow['on'] || workflow[true] +raise 'only manual execution and offline checks are supported' unless trigger.keys.sort == ['pull_request', 'workflow_dispatch'] +raise 'no arbitrary execution inputs are supported' unless trigger['workflow_dispatch'].nil? +jobs = workflow.fetch('jobs') +raise 'only checks and a diagnostic belong here' unless jobs.keys.sort == ['check', 'probe'] +job = jobs.fetch('probe') +raise 'manual diagnostic must follow checks' unless job['needs'] == 'check' && job['if'] == "github.event_name == 'workflow_dispatch'" +raise 'ordinary environment required' unless job['environment'] == 'aws-infrastructure' +raise 'production serialization required' unless job.dig('concurrency', 'group') == 'aws-production' && job.dig('concurrency', 'cancel-in-progress') == false +raise 'bounded job required' unless job['timeout-minutes'] == 25 +raise 'only contents and OIDC permissions needed' unless job['permissions'] == {'contents'=>'read', 'id-token'=>'write'} +steps = job.fetch('steps') +credentials = steps.select { |s| s['uses'] == 'aws-actions/configure-aws-credentials@v4' } +raise 'existing role with restrictive session policy required' unless credentials.length == 1 && credentials[0]['with'] == { + 'role-to-assume'=>'${{ vars.AWS_TERRAFORM_APPLY_ROLE_ARN }}', 'aws-region'=>'us-east-1', + 'inline-session-policy'=>'${{ steps.scope.outputs.policy }}' +} +scope = steps.index { |s| s['run'] == 'node .github/scripts/aws-origin-tls.cjs session-policy' } +raise 'source gate must precede credentials' unless scope && scope < steps.index(credentials[0]) +probe = steps.find { |s| s['run'] == 'node .github/scripts/aws-origin-tls.cjs probe' } +raise 'probe timeout required' unless probe && probe['timeout-minutes'] == 10 +cleanup = steps.last +raise 'bounded cleanup must always follow credentials, including probe failure' unless cleanup['run'] == 'node .github/scripts/aws-origin-tls.cjs finish' && + cleanup['if'] == "always() && steps.credentials.outcome == 'success'" && cleanup['timeout-minutes'] == 8 +raise 'no secret inheritance, bypasses or public artifacts' if workflow.to_s.match?(/secrets|upload-artifact|AWS_TERRAFORM_ENABLED|AWS_TLS_RECOVERY|terraform apply|deploy-aws/) +jobs.each_value do |j| + raise 'failures must stop execution' if j['continue-on-error'] + j.fetch('steps').each do |s| + raise 'step failures must stop execution' if s['continue-on-error'] + raise 'checkout credentials must not persist' if s['uses'] == 'actions/checkout@v4' && s.dig('with', 'persist-credentials') != false + end +end +puts 'Origin TLS workflow source gate, ordinary environment, queue and unconditional bounded cleanup checks passed' +RUBY diff --git a/.github/scripts/aws-origin-tls.cjs b/.github/scripts/aws-origin-tls.cjs new file mode 100644 index 000000000..9fa56caeb --- /dev/null +++ b/.github/scripts/aws-origin-tls.cjs @@ -0,0 +1,258 @@ +const { spawnSync } = require("node:child_process"); +const { appendFileSync, mkdirSync, readFileSync, writeFileSync, existsSync } = require("node:fs"); +const { join } = require("node:path"); +const { ACCOUNT, BUCKET, ROLE, LB, LISTENER, DESIRED, hash, canonical } = require("./aws-origin-alb.cjs"); +const { CBC, GCM, TLS13, classify } = require("./origin-tls-client.cjs"); + +const REGION = "us-east-1"; +const ECS = `arn:aws:ecs:${REGION}:${ACCOUNT}`; +const CLUSTER = `${ECS}:cluster/warmbly-production`; +const SERVICE = `${ECS}:service/warmbly-production/warmbly-production-backend`; +const EXECUTION = `arn:aws:iam::${ACCOUNT}:role/warmbly-production-validation-execution`; +const VPC = "vpc-038e4f0058030800b"; +const SG = "sg-0ddd1eb3234dad9e7"; +const ALB_SG = "sg-03a4f6f8010daff8d"; +const LOG = "/ecs/warmbly-production/validation"; +const PREFIX = "warmbly/production/diagnostics/origin-tls"; +// Official Node 22 bookworm ARM64 manifest, including OpenSSL and Debian public CAs. +const IMAGE = "docker.io/library/node@sha256:a288699fc0f9676950742fae6efcd015180aa7ef2bf8a6477e4b0a8cc0ab1b3f"; +const requireThat = (ok, message) => { if (!ok) throw new Error(message); }; +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +function configuration(env) { + requireThat(env.GITHUB_ACTIONS === "true" && env.GITHUB_REPOSITORY === "warmbly/warmbly" && + env.GITHUB_REF === "refs/heads/main" && env.GITHUB_EVENT_NAME === "workflow_dispatch" && + env.GITHUB_WORKFLOW_REF === "warmbly/warmbly/.github/workflows/probe-origin-tls-aws.yml@refs/heads/main" && + env.GITHUB_RUN_ATTEMPT === "1" && /^[1-9]\d*$/.test(env.GITHUB_RUN_ID || "") && + /^[a-f0-9]{40}$/.test(env.GITHUB_SHA || ""), "A new main workflow dispatch is required; reruns are refused"); + requireThat(env.STATE_BUCKET === BUCKET && env.APPLY_ROLE === ROLE && env.WORK && + env.GITHUB_OUTPUT && env.GITHUB_STEP_SUMMARY, "Normal aws-infrastructure role, bucket and private paths required"); + return { run: env.GITHUB_RUN_ID, source: env.GITHUB_SHA, work: env.WORK, + family: `warmbly-production-origin-tls-${env.GITHUB_RUN_ID}`, startedBy: `origin-tls-${env.GITHUB_RUN_ID}` }; +} + +function sessionPolicy(config) { + return { Version: "2012-10-17", Statement: [ + { Effect: "Allow", Action: ["elasticloadbalancing:DescribeLoadBalancers", "elasticloadbalancing:DescribeListeners", + "ec2:DescribeNetworkInterfaces", "ec2:DescribeSubnets", "ecs:RegisterTaskDefinition", "ecs:ListTaskDefinitions", "ecs:ListTasks"], Resource: "*" }, + { Effect: "Allow", Action: "ecs:DescribeServices", Resource: SERVICE }, + { Effect: "Allow", Action: "ecs:RunTask", Resource: `${ECS}:task-definition/${config.family}:*`, + Condition: { ArnEquals: { "ecs:cluster": CLUSTER } } }, + { Effect: "Allow", Action: "ecs:DeregisterTaskDefinition", Resource: `${ECS}:task-definition/${config.family}:*` }, + { Effect: "Allow", Action: ["ecs:DescribeTasks", "ecs:StopTask"], Resource: `${ECS}:task/warmbly-production/*` }, + { Effect: "Allow", Action: "iam:PassRole", Resource: EXECUTION, + Condition: { StringEquals: { "iam:PassedToService": "ecs-tasks.amazonaws.com" } } }, + { Effect: "Allow", Action: "logs:GetLogEvents", Resource: `arn:aws:logs:${REGION}:${ACCOUNT}:log-group:${LOG}:log-stream:origin-tls-${config.run}/*` }, + { Effect: "Allow", Action: "s3:PutObject", Resource: `arn:aws:s3:::${BUCKET}/${PREFIX}/${config.run}/1/*`, + Condition: { StringEquals: { "s3:x-amz-server-side-encryption": "AES256", "s3:ResourceAccount": ACCOUNT }, + Bool: { "aws:SecureTransport": "true" }, Null: { "s3:if-none-match": "false" } } }, + ] }; +} + +function awsCLI(service, action, args = []) { + const result = spawnSync("aws", [service, action, ...args, "--region", REGION, "--output", "json", "--no-cli-pager", + "--cli-connect-timeout", "5", "--cli-read-timeout", "8"], { encoding: "utf8", timeout: 15000, + maxBuffer: 4 * 1024 * 1024, env: { ...process.env, AWS_PAGER: "", AWS_CLI_AUTO_PROMPT: "off", AWS_MAX_ATTEMPTS: "1" } }); + requireThat(!result.error && result.status === 0, `AWS ${service} ${action} failed; inspect normal access/configuration prerequisites`); + try { return JSON.parse(result.stdout); } catch { throw new Error("AWS diagnostic response was not valid JSON"); } +} + +async function discover(aws) { + const lb = (await aws("elbv2", "describe-load-balancers", ["--load-balancer-arns", LB])).LoadBalancers; + requireThat(lb?.length === 1 && lb[0].LoadBalancerArn === LB && lb[0].VpcId === VPC && + lb[0].State?.Code === "active" && lb[0].Type === "application" && lb[0].IpAddressType === "ipv4" && + canonical(lb[0].SecurityGroups) === canonical([ALB_SG]), "Exact active origin ALB required"); + const listener = (await aws("elbv2", "describe-listeners", ["--listener-arns", LISTENER])).Listeners; + requireThat(listener?.length === 1 && listener[0].ListenerArn === LISTENER && listener[0].LoadBalancerArn === LB && + listener[0].Protocol === "HTTPS" && listener[0].Port === 443 && listener[0].SslPolicy === DESIRED, + "Bound HTTPS listener must have the reviewed restricted policy"); + const enis = (await aws("ec2", "describe-network-interfaces", ["--filters", + "Name=description,Values=ELB app/warmbly-production-ingress/19d5c82c1651c871", `Name=vpc-id,Values=${VPC}`])).NetworkInterfaces; + requireThat(enis?.length === 2 && new Set(enis.map((e) => e.AvailabilityZone)).size === 2 && + new Set(enis.map((e) => e.PrivateIpAddress)).size === 2 && enis.every((e) => e.VpcId === VPC && e.Status === "in-use" && + e.RequesterManaged === true && e.OwnerId === ACCOUNT && e.Description === "ELB app/warmbly-production-ingress/19d5c82c1651c871" && + e.Groups?.length === 1 && e.Groups[0].GroupId === ALB_SG && + /^10\.42\.(0|1)\.(\d{1,3})$/.test(e.PrivateIpAddress) && + lb[0].AvailabilityZones?.some((z) => z.ZoneName === e.AvailabilityZone && z.SubnetId === e.SubnetId)), "Complete exact-ALB private ENIs required"); + const service = (await aws("ecs", "describe-services", ["--cluster", CLUSTER, "--services", SERVICE])).services; + const network = service?.[0]?.networkConfiguration?.awsvpcConfiguration; + requireThat(service?.length === 1 && service[0].serviceArn === SERVICE && service[0].status === "ACTIVE" && + network?.assignPublicIp === "ENABLED" && canonical(network.securityGroups) === canonical([SG]) && + network.subnets?.length === 2, "Existing application task network required"); + const subnets = (await aws("ec2", "describe-subnets", ["--subnet-ids", ...network.subnets])).Subnets; + requireThat(subnets?.length === 2 && subnets.every((s) => s.VpcId === VPC && s.State === "available" && + enis.some((e) => e.SubnetId === s.SubnetId)), "Existing application/ALB subnets required"); + return { at: new Date().toISOString(), lb: lb[0], listener: listener[0], enis, network }; +} + +function definition(config, targets) { + return { family: config.family, networkMode: "awsvpc", requiresCompatibilities: ["FARGATE"], cpu: "256", memory: "512", + executionRoleArn: EXECUTION, runtimePlatform: { cpuArchitecture: "ARM64", operatingSystemFamily: "LINUX" }, + containerDefinitions: [{ name: "probe", image: IMAGE, essential: true, user: "1000:1000", readonlyRootFilesystem: true, + linuxParameters: { capabilities: { drop: ["ALL"] } }, stopTimeout: 10, + entryPoint: ["timeout", "-s", "KILL", "210", "node"], command: ["-e", readFileSync(join(__dirname, "origin-tls-client.cjs"), "utf8")], + environment: [{ name: "ORIGIN_TLS_TASK", value: "1" }, { name: "ORIGIN_TLS_TARGETS", value: JSON.stringify(targets) }], + logConfiguration: { logDriver: "awslogs", options: { "awslogs-group": LOG, "awslogs-region": REGION, + "awslogs-stream-prefix": `origin-tls-${config.run}`, mode: "blocking" } } }] }; +} + +const save = (config, evidence) => writeFileSync(join(config.work, "evidence.json"), JSON.stringify(evidence), { mode: 0o600 }); +const load = (config) => JSON.parse(readFileSync(join(config.work, "evidence.json"))); +async function archive(aws, config, name, evidence) { + const bytes = Buffer.from(JSON.stringify(evidence)); + const path = join(config.work, name); + writeFileSync(path, bytes, { mode: 0o600 }); + const checksum = Buffer.from(hash(bytes), "hex").toString("base64"); + const result = await aws("s3api", "put-object", ["--bucket", BUCKET, "--key", `${PREFIX}/${config.run}/1/${name}`, + "--expected-bucket-owner", ACCOUNT, "--server-side-encryption", "AES256", "--if-none-match", "*", + "--checksum-algorithm", "SHA256", "--checksum-sha256", checksum, "--body", path]); + requireThat(result.ServerSideEncryption === "AES256" && result.ChecksumSHA256 === checksum && + typeof result.VersionId === "string" && result.VersionId.length > 0 && result.VersionId !== "null", "Private encrypted versioned evidence required"); + return { version: result.VersionId, sha256: hash(bytes) }; +} + +async function execute(aws, config, pause = sleep) { + mkdirSync(config.work, { recursive: true, mode: 0o700 }); + const evidence = { run: config.run, source: config.source, success: false, image: IMAGE }; + save(config, evidence); + try { + const identity = await aws("sts", "get-caller-identity"); + requireThat(identity.Account === ACCOUNT && /^arn:aws:sts::925323749693:assumed-role\/warmbly-production-terraform-apply\/[^/]+$/.test(identity.Arn || ""), "Existing infrastructure apply identity required"); + evidence.before = await discover(aws); + const targets = evidence.before.enis.map((e) => e.PrivateIpAddress).sort(); + evidence.definition = definition(config, targets); + await archive(aws, config, "intent.json", evidence); + evidence.registerAttempted = true; + save(config, evidence); + const registered = await aws("ecs", "register-task-definition", ["--cli-input-json", JSON.stringify(evidence.definition)]); + evidence.definitionArn = registered.taskDefinition?.taskDefinitionArn; + requireThat(evidence.definitionArn?.startsWith(`${ECS}:task-definition/${config.family}:`), "Dedicated diagnostic definition required"); + save(config, evidence); + // Refresh immediately before launch; a changed address requires a new dispatch. + evidence.prelaunch = await discover(aws); + requireThat(canonical(evidence.prelaunch.enis.map((e) => e.PrivateIpAddress).sort()) === canonical(targets) && + canonical(evidence.prelaunch.network) === canonical(evidence.before.network), "Origin/task network changed before launch"); + const request = { cluster: CLUSTER, taskDefinition: evidence.definitionArn, launchType: "FARGATE", platformVersion: "1.4.0", + count: 1, startedBy: config.startedBy, clientToken: config.startedBy, enableExecuteCommand: false, + networkConfiguration: { awsvpcConfiguration: evidence.before.network } }; + evidence.runAttempted = true; + save(config, evidence); + const launched = await aws("ecs", "run-task", ["--cli-input-json", JSON.stringify(request)]); + evidence.taskArn = launched.tasks?.[0]?.taskArn; + requireThat(!launched.failures?.length && launched.tasks?.length === 1 && + evidence.taskArn?.startsWith(`${ECS}:task/warmbly-production/`), "One disposable diagnostic task required"); + save(config, evidence); + for (let i = 0; i < 48; i++) { + const readback = await aws("ecs", "describe-tasks", ["--cluster", CLUSTER, "--tasks", evidence.taskArn]); + requireThat(!readback.failures?.length && readback.tasks?.length === 1, "Diagnostic task readback required"); + evidence.task = readback.tasks[0]; + requireThat(evidence.task.taskArn === evidence.taskArn && evidence.task.clusterArn === CLUSTER && + evidence.task.taskDefinitionArn === evidence.definitionArn && evidence.task.startedBy === config.startedBy && + evidence.task.launchType === "FARGATE" && evidence.task.enableExecuteCommand === false, "Exact diagnostic task identity required"); + save(config, evidence); + if (evidence.task.lastStatus === "STOPPED") break; + await pause(5000); + } + requireThat(evidence.task?.lastStatus === "STOPPED" && evidence.task.containers?.length === 1 && + evidence.task.containers[0].name === "probe" && evidence.task.containers[0].imageDigest === IMAGE.split("@")[1] && + evidence.task.containers[0].exitCode === 0, "Probe incomplete or inconclusive; inspect private evidence"); + evidence.after = await discover(aws); + requireThat(canonical(evidence.after.enis.map((e) => e.PrivateIpAddress).sort()) === canonical(targets), "Origin addresses changed during probe"); + evidence.success = true; + } catch (error) { + evidence.failure = error.message; + throw error; + } finally { save(config, evidence); } +} + +async function finish(aws, config, pause = sleep) { + if (!existsSync(join(config.work, "evidence.json"))) return; + const evidence = load(config); + evidence.cleanup = { errors: [], stopped: false, deregistered: false }; + const attempt = async (action) => { try { return await action(); } catch (error) { evidence.cleanup.errors.push(error.message); return null; } }; + // Recover identities even when a registration or launch response was lost. + let listed = { taskArns: [] }; + if (evidence.runAttempted) { + for (let i = 0; i < 3; i++) { + listed = await attempt(() => aws("ecs", "list-tasks", ["--cluster", CLUSTER, "--started-by", config.startedBy])); + if (listed?.taskArns?.length || evidence.taskArn) break; + await pause(5000); + } + } + const tasks = [...new Set([evidence.taskArn, ...(listed?.taskArns || [])].filter(Boolean))]; + evidence.cleanup.taskArns = tasks; + save(config, evidence); + for (const task of tasks) await attempt(() => aws("ecs", "stop-task", ["--cluster", CLUSTER, "--task", task, "--reason", "Bounded origin TLS diagnostic cleanup"])); + if (listed) { + for (let i = 0; i < 12; i++) { + const state = tasks.length ? await attempt(() => aws("ecs", "describe-tasks", ["--cluster", CLUSTER, "--tasks", ...tasks])) : { tasks: [] }; + evidence.cleanup.tasks = state; + if (state && !state.failures?.length && state.tasks?.length === tasks.length && state.tasks.every((t) => t.lastStatus === "STOPPED")) { + evidence.cleanup.stopped = true; + break; + } + await pause(5000); + } + } + const definitions = evidence.registerAttempted ? await attempt(() => aws("ecs", "list-task-definitions", ["--family-prefix", config.family, "--status", "ACTIVE"])) : { taskDefinitionArns: [] }; + const arns = [...new Set([evidence.definitionArn, ...(definitions?.taskDefinitionArns || [])].filter((arn) => + typeof arn === "string" && new RegExp(`^${ECS}:task-definition/${config.family}:\\d+$`).test(arn)))]; + evidence.cleanup.definitionArns = arns; + save(config, evidence); + for (const arn of arns) { + const result = await attempt(() => aws("ecs", "deregister-task-definition", ["--task-definition", arn])); + if (result?.taskDefinition?.taskDefinitionArn !== arn || result?.taskDefinition?.status !== "INACTIVE") evidence.cleanup.errors.push("Diagnostic definition deregistration unconfirmed"); + } + evidence.cleanup.deregistered = !!definitions && evidence.cleanup.errors.length === 0; + evidence.logs = []; + for (const task of tasks) { + let token; + for (let i = 0; i < 6; i++) { + const page = await attempt(() => aws("logs", "get-log-events", ["--log-group-name", LOG, + "--log-stream-name", `origin-tls-${config.run}/probe/${task.split("/").pop()}`, "--start-from-head", + ...(token ? ["--next-token", token] : [])])); + if (!page) break; + evidence.logs.push(...(page.events || [])); + if (page.nextForwardToken === token) { + if (verifiedLogs(evidence.logs, evidence.before?.enis?.map((e) => e.PrivateIpAddress) || [])) break; + if (i === 5) break; + await pause(5000); + evidence.logs = []; + token = undefined; + continue; + } + token = page.nextForwardToken; + } + } + const complete = verifiedLogs(evidence.logs, evidence.before?.enis?.map((e) => e.PrivateIpAddress) || []); + evidence.success = evidence.success && complete && evidence.cleanup.stopped && evidence.cleanup.deregistered && evidence.cleanup.errors.length === 0; + save(config, evidence); + const archived = await archive(aws, config, "result.json", evidence); + appendFileSync(process.env.GITHUB_STEP_SUMMARY, `Private origin TLS result: version ${archived.version}, SHA-256 ${archived.sha256}. Outcome: ${evidence.success ? "verified" : "inconclusive"}.\n`); + requireThat(evidence.success, "Diagnostic incomplete, inconclusive or cleanup failed; inspect the private result before any new dispatch"); +} + +function verifiedLogs(events, targets) { + const rows = events.map((e) => { try { return JSON.parse(e.message); } catch { return null; } }).filter(Boolean); + if (targets.length !== 2 || !rows.some((r) => r.kind === "complete" && r.success === true)) return false; + return targets.every((ip) => { + const handshakes = rows.filter((r) => r.kind === "handshake" && r.ip === ip); + return canonical(handshakes.map((r) => r.cipher)) === canonical([GCM[0], GCM[1], TLS13, ...CBC, GCM[0], TLS13]) && + handshakes.every((r) => classify(r, r.cipher === TLS13 ? "TLSv1.3" : "TLSv1.2", r.cipher, CBC.includes(r.cipher)) === + (CBC.includes(r.cipher) ? "refused" : "trusted")); + }); +} + +if (require.main === module) { + process.umask(0o077); + Promise.resolve().then(async () => { + const config = configuration(process.env); + const source = spawnSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }); + requireThat(source.status === 0 && source.stdout.trim() === config.source, "Checkout must match main workflow source"); + if (process.argv[2] === "session-policy") appendFileSync(process.env.GITHUB_OUTPUT, `policy=${canonical(sessionPolicy(config))}\n`); + else if (process.argv[2] === "probe") await execute(awsCLI, config); + else if (process.argv[2] === "finish") await finish(awsCLI, config); + else throw new Error("Unknown diagnostic command"); + }).catch((error) => { console.error(`Origin TLS diagnostic stopped: ${error.message}`); process.exitCode = 1; }); +} +module.exports = { configuration, sessionPolicy, discover, definition, archive, execute, finish, verifiedLogs, IMAGE, CLUSTER, SERVICE, EXECUTION, SG, ALB_SG, VPC, ECS, LOG }; diff --git a/.github/scripts/aws-origin-tls.test.cjs b/.github/scripts/aws-origin-tls.test.cjs new file mode 100644 index 000000000..90b66f1b5 --- /dev/null +++ b/.github/scripts/aws-origin-tls.test.cjs @@ -0,0 +1,247 @@ +const { test } = require("node:test"); +const assert = require("node:assert/strict"); +const { mkdtempSync, readFileSync, rmSync } = require("node:fs"); +const { tmpdir } = require("node:os"); +const { join } = require("node:path"); +const { ACCOUNT, BUCKET, ROLE, LB, LISTENER, DESIRED } = require("./aws-origin-alb.cjs"); +const { configuration, sessionPolicy, discover, definition, execute, finish, verifiedLogs, IMAGE, CLUSTER, SERVICE, EXECUTION, SG, ALB_SG, VPC, ECS } = require("./aws-origin-tls.cjs"); +const { probe, CBC, TLS13 } = require("./origin-tls-client.cjs"); + +function logFixture() { + const events = []; + probe(["10.42.0.85", "10.42.1.12"], (args) => { + if (args[0] === "version") return { status: 0, stdout: "OpenSSL fixture" }; + if (args[0] === "ciphers") return { status: 0, stdout: CBC.join(":") }; + const cipher = args.at(-1); + return { command: ["openssl", ...args], ...(CBC.includes(cipher) ? + { status: 1, stdout: "New, (NONE), Cipher is (NONE)", stderr: "sslv3 alert handshake failure" } : + { status: 0, stdout: `-----BEGIN CERTIFICATE-----\nfixture\nNew, ${cipher === TLS13 ? "TLSv1.3" : "TLSv1.2"}, Cipher is ${cipher}\nVerify return code: 0 (ok)`, stderr: "" }) }; + }, (row) => events.push({ message: JSON.stringify(row) })); + return events; +} + +const env = () => ({ GITHUB_ACTIONS: "true", GITHUB_REPOSITORY: "warmbly/warmbly", GITHUB_REF: "refs/heads/main", + GITHUB_EVENT_NAME: "workflow_dispatch", GITHUB_WORKFLOW_REF: "warmbly/warmbly/.github/workflows/probe-origin-tls-aws.yml@refs/heads/main", + GITHUB_RUN_ATTEMPT: "1", GITHUB_RUN_ID: "123456", GITHUB_SHA: "a".repeat(40), STATE_BUCKET: BUCKET, APPLY_ROLE: ROLE, + WORK: "/tmp/fixture", GITHUB_OUTPUT: "/tmp/output", GITHUB_STEP_SUMMARY: "/tmp/summary" }); + +function fixture(t, change = () => {}) { + const work = mkdtempSync(join(tmpdir(), "origin-tls-test-")); + t.after(() => rmSync(work, { recursive: true, force: true })); + const config = configuration({ ...env(), WORK: work }); + const arn = `${ECS}:task-definition/${config.family}:1`; + const task = `${ECS}:task/warmbly-production/${"1".repeat(32)}`; + const calls = []; + const zones = [{ ZoneName: "us-east-1a", SubnetId: "subnet-a" }, { ZoneName: "us-east-1b", SubnetId: "subnet-b" }]; + const network = { subnets: zones.map((z) => z.SubnetId), securityGroups: [SG], assignPublicIp: "ENABLED" }; + const state = { taskArn: task, clusterArn: CLUSTER, taskDefinitionArn: arn, startedBy: config.startedBy, + launchType: "FARGATE", enableExecuteCommand: false, lastStatus: "STOPPED", + containers: [{ name: "probe", exitCode: 0, imageDigest: IMAGE.split("@")[1] }] }; + const aws = async (service, action, args = []) => { + calls.push({ service, action, args }); + let result; + switch (action) { + case "get-caller-identity": result = { Account: ACCOUNT, Arn: `arn:aws:sts::${ACCOUNT}:assumed-role/warmbly-production-terraform-apply/fixture` }; break; + case "describe-load-balancers": result = { LoadBalancers: [{ LoadBalancerArn: LB, VpcId: VPC, + Type: "application", IpAddressType: "ipv4", State: { Code: "active" }, SecurityGroups: [ALB_SG], AvailabilityZones: zones }] }; break; + case "describe-listeners": result = { Listeners: [{ ListenerArn: LISTENER, LoadBalancerArn: LB, Protocol: "HTTPS", Port: 443, SslPolicy: DESIRED }] }; break; + case "describe-network-interfaces": result = { NetworkInterfaces: zones.map((z, i) => ({ Description: "ELB app/warmbly-production-ingress/19d5c82c1651c871", + AvailabilityZone: z.ZoneName, SubnetId: z.SubnetId, VpcId: VPC, Status: "in-use", RequesterManaged: true, OwnerId: ACCOUNT, + PrivateIpAddress: i ? "10.42.1.12" : "10.42.0.85", Groups: [{ GroupId: ALB_SG }] })) }; break; + case "describe-services": result = { services: [{ serviceArn: SERVICE, status: "ACTIVE", networkConfiguration: { awsvpcConfiguration: network } }] }; break; + case "describe-subnets": result = { Subnets: zones.map((z) => ({ SubnetId: z.SubnetId, VpcId: VPC, State: "available" })) }; break; + case "put-object": result = { ServerSideEncryption: "AES256", ChecksumSHA256: args[args.indexOf("--checksum-sha256") + 1], VersionId: "private-version" }; break; + case "register-task-definition": result = { taskDefinition: { taskDefinitionArn: arn } }; break; + case "run-task": result = { tasks: [state] }; break; + case "describe-tasks": result = { tasks: [state] }; break; + case "list-tasks": result = { taskArns: [task] }; break; + case "stop-task": result = { task: state }; break; + case "list-task-definitions": result = { taskDefinitionArns: [arn] }; break; + case "deregister-task-definition": result = { taskDefinition: { taskDefinitionArn: arn, status: "INACTIVE" } }; break; + case "get-log-events": result = { events: args.includes("--next-token") ? [] : logFixture(), nextForwardToken: "end" }; break; + default: throw new Error(`Unexpected ${service} ${action}`); + } + await change(action, result, calls); + return result; + }; + return { config, aws, calls, arn, task, state, evidence: () => JSON.parse(readFileSync(join(work, "evidence.json"))) }; +} + +test("only fresh main dispatches under normal infrastructure configuration are admitted", () => { + assert.equal(configuration(env()).run, "123456"); + for (const [key, value] of Object.entries({ GITHUB_ACTIONS: "false", GITHUB_REPOSITORY: "fork/warmbly", GITHUB_REF: "refs/heads/branch", + GITHUB_EVENT_NAME: "pull_request", GITHUB_WORKFLOW_REF: "other", GITHUB_RUN_ATTEMPT: "2", GITHUB_RUN_ID: "../1", + GITHUB_SHA: "bad", STATE_BUCKET: "other", APPLY_ROLE: "other", WORK: "", GITHUB_OUTPUT: "", GITHUB_STEP_SUMMARY: "" })) { + assert.throws(() => configuration({ ...env(), [key]: value }), key); + } +}); + +test("session policy admits only diagnostics, bound roles, cluster and encrypted conditional private archives", () => { + const config = configuration(env()); + const policy = sessionPolicy(config); + assert.ok(JSON.stringify(policy).length < 2048); + const all = policy.Statement.flatMap((s) => s.Action); + for (const forbidden of ["ecs:UpdateService", "ecs:ExecuteCommand", "secretsmanager:GetSecretValue", "ssm:GetParameters", + "iam:PutRolePolicy", "elasticloadbalancing:ModifyListener", "ec2:AuthorizeSecurityGroupIngress"]) assert.ok(!all.includes(forbidden)); + assert.deepEqual(policy.Statement.find((s) => s.Action === "iam:PassRole").Resource, EXECUTION); + assert.equal(policy.Statement.find((s) => s.Action === "ecs:RunTask").Condition.ArnEquals["ecs:cluster"], CLUSTER); + const put = policy.Statement.find((s) => s.Action === "s3:PutObject"); + assert.ok(put.Resource.endsWith(`/origin-tls/${config.run}/1/*`)); + assert.equal(put.Condition.StringEquals["s3:x-amz-server-side-encryption"], "AES256"); + assert.equal(put.Condition.Null["s3:if-none-match"], "false"); +}); + +test("fresh definition contains only the official digest, hard-bounded TLS client and existing execution/log facilities", () => { + const task = definition(configuration(env()), ["10.42.0.85", "10.42.1.12"]); + assert.deepEqual(Object.keys(task).sort(), ["family", "networkMode", "requiresCompatibilities", "cpu", "memory", "executionRoleArn", "runtimePlatform", "containerDefinitions"].sort()); + assert.equal(task.executionRoleArn, EXECUTION); + assert.equal(task.runtimePlatform.cpuArchitecture, "ARM64"); + const c = task.containerDefinitions[0]; + assert.match(c.image, /^docker\.io\/library\/node@sha256:[a-f0-9]{64}$/); + assert.equal(c.secrets, undefined); + assert.equal(c.repositoryCredentials, undefined); + assert.equal(c.portMappings, undefined); + assert.equal(c.mountPoints, undefined); + assert.equal(c.readonlyRootFilesystem, true); + assert.deepEqual(c.entryPoint, ["timeout", "-s", "KILL", "210", "node"]); + assert.deepEqual(c.environment.map((e) => e.name), ["ORIGIN_TLS_TASK", "ORIGIN_TLS_TARGETS"]); + assert.equal(c.logConfiguration.options["awslogs-create-group"], undefined); +}); + +test("only the exact active ALB, restricted listener, managed private ENIs and existing task network are accepted", async (t) => { + const changes = [ + ["describe-load-balancers", (r) => { r.LoadBalancers[0].VpcId = "vpc-other"; }], + ["describe-listeners", (r) => { r.Listeners[0].SslPolicy = "other"; }], + ["describe-network-interfaces", (r) => { r.NetworkInterfaces.pop(); }], + ["describe-network-interfaces", (r) => { r.NetworkInterfaces[0].Description = "another ALB"; }], + ["describe-network-interfaces", (r) => { r.NetworkInterfaces[0].Groups[0].GroupId = SG; }], + ["describe-network-interfaces", (r) => { r.NetworkInterfaces[0].RequesterManaged = false; }], + ["describe-services", (r) => { r.services[0].networkConfiguration.awsvpcConfiguration.securityGroups = ["sg-worker"]; }], + ["describe-subnets", (r) => { r.Subnets[0].VpcId = "vpc-other"; }], + ]; + for (const [target, mutate] of changes) { + const f = fixture(t, (action, result) => { if (action === target) mutate(result); }); + await assert.rejects(discover(f.aws)); + assert.ok(!f.calls.some((c) => c.action === "run-task")); + } +}); + +test("archive requires encryption, checksum and a real version before admitting a task", async (t) => { + for (const mutate of [(r) => { r.VersionId = "null"; }, (r) => { r.ChecksumSHA256 = "bad"; }, (r) => { r.ServerSideEncryption = ""; }]) { + const f = fixture(t, (action, result) => { if (action === "put-object") mutate(result); }); + await assert.rejects(execute(f.aws, f.config, async () => {}), /versioned/); + assert.ok(!f.calls.some((c) => c.action === "register-task-definition")); + } +}); + +test("successful lifecycle archives intent first, launches only one secretless task, stops/deregisters and retains private logs", async (t) => { + const f = fixture(t); + const oldSummary = process.env.GITHUB_STEP_SUMMARY; + process.env.GITHUB_STEP_SUMMARY = join(f.config.work, "summary"); + t.after(() => { if (oldSummary === undefined) delete process.env.GITHUB_STEP_SUMMARY; else process.env.GITHUB_STEP_SUMMARY = oldSummary; }); + await execute(f.aws, f.config, async () => {}); + await finish(f.aws, f.config, async () => {}); + assert.equal(f.evidence().success, true); + const actions = f.calls.map((c) => c.action); + assert.ok(actions.indexOf("put-object") < actions.indexOf("register-task-definition")); + assert.equal(actions.filter((a) => a === "run-task").length, 1); + const request = JSON.parse(f.calls.find((c) => c.action === "run-task").args[1]); + assert.equal(request.taskDefinition, f.arn); + assert.equal(request.enableExecuteCommand, false); + assert.equal(request.clientToken, f.config.startedBy); + assert.equal(request.overrides, undefined); + assert.deepEqual(request.networkConfiguration.awsvpcConfiguration.securityGroups, [SG]); + assert.ok(actions.indexOf("stop-task") < actions.indexOf("deregister-task-definition")); + assert.equal(f.evidence().cleanup.stopped, true); + assert.equal(f.evidence().cleanup.deregistered, true); + assert.ok(f.evidence().logs.length > 0); + assert.ok(!readFileSync(process.env.GITHUB_STEP_SUMMARY, "utf8").includes("10.42.")); +}); + +for (const failure of ["register-task-definition", "run-task", "describe-tasks"]) { + test(`lost ${failure} response still permits diagnostic-only recovery and private failure evidence`, async (t) => { + let lost = false; + const f = fixture(t, (action) => { if (action === failure && !lost) { lost = true; throw new Error("Fixture lost response"); } }); + const oldSummary = process.env.GITHUB_STEP_SUMMARY; + process.env.GITHUB_STEP_SUMMARY = join(f.config.work, "summary"); + t.after(() => { if (oldSummary === undefined) delete process.env.GITHUB_STEP_SUMMARY; else process.env.GITHUB_STEP_SUMMARY = oldSummary; }); + await assert.rejects(execute(f.aws, f.config, async () => {}), /lost response/); + await assert.rejects(finish(f.aws, f.config, async () => {}), /incomplete/); + assert.ok(f.calls.some((c) => c.action === "deregister-task-definition")); + if (failure !== "register-task-definition") assert.ok(f.calls.some((c) => c.action === "stop-task")); + assert.equal(f.evidence().success, false); + assert.equal(f.calls.filter((c) => c.action === "put-object").length, 2); + assert.equal(f.calls.filter((c) => c.action === "run-task").length, failure === "register-task-definition" ? 0 : 1); + }); +} + +test("network drift after registration prevents launch and remains eligible for cleanup", async (t) => { + let reads = 0; + const f = fixture(t, (action, result) => { if (action === "describe-network-interfaces" && ++reads > 1) result.NetworkInterfaces[0].PrivateIpAddress = "10.42.0.86"; }); + await assert.rejects(execute(f.aws, f.config, async () => {}), /changed before launch/); + assert.ok(!f.calls.some((c) => c.action === "run-task")); + assert.equal(f.evidence().registerAttempted, true); +}); + +test("bad image identity and nonzero exit never establish proof", async (t) => { + for (const mutate of [(r) => { r.tasks[0].containers[0].exitCode = 1; }, (r) => { r.tasks[0].containers[0].imageDigest = "sha256:other"; }]) { + const f = fixture(t, (action, result) => { if (action === "describe-tasks") mutate(result); }); + await assert.rejects(execute(f.aws, f.config, async () => {}), /inconclusive/); + assert.equal(f.evidence().success, false); + } +}); + +test("missing or truncated raw handshake logs cannot be rescued by a successful completion marker", () => { + const targets = ["10.42.0.85", "10.42.1.12"]; + assert.equal(verifiedLogs(logFixture(), targets), true); + assert.equal(verifiedLogs(logFixture().filter((e) => JSON.parse(e.message).kind !== "handshake"), targets), false); + assert.equal(verifiedLogs(logFixture().slice(0, -1), targets), false); + const logs = logFixture(); + logs[2].message = logs[2].message.replace("return code: 0 (ok)", "return code: 62 (hostname mismatch)"); + assert.equal(verifiedLogs(logs, targets), false); +}); + +test("delayed log delivery is retried within a fixed page budget before verification", async (t) => { + let reads = 0; + const f = fixture(t, (action, result) => { if (action === "get-log-events" && ++reads <= 2) result.events = []; }); + const oldSummary = process.env.GITHUB_STEP_SUMMARY; + process.env.GITHUB_STEP_SUMMARY = join(f.config.work, "summary"); + t.after(() => { if (oldSummary === undefined) delete process.env.GITHUB_STEP_SUMMARY; else process.env.GITHUB_STEP_SUMMARY = oldSummary; }); + await execute(f.aws, f.config, async () => {}); + await finish(f.aws, f.config, async () => {}); + assert.equal(f.evidence().success, true); + assert.equal(reads, 4); +}); + +test("a task that never stops and absent logs fail within bounded polling and preserve cleanup evidence", async (t) => { + let sleeps = 0; + const f = fixture(t, (action, result) => { + if (action === "describe-tasks") result.tasks[0].lastStatus = "RUNNING"; + if (action === "get-log-events") result.events = []; + }); + const oldSummary = process.env.GITHUB_STEP_SUMMARY; + process.env.GITHUB_STEP_SUMMARY = join(f.config.work, "summary"); + t.after(() => { if (oldSummary === undefined) delete process.env.GITHUB_STEP_SUMMARY; else process.env.GITHUB_STEP_SUMMARY = oldSummary; }); + await assert.rejects(execute(f.aws, f.config, async () => { sleeps++; }), /inconclusive/); + await assert.rejects(finish(f.aws, f.config, async () => { sleeps++; }), /cleanup failed/); + assert.equal(f.calls.filter((c) => c.action === "describe-tasks").length, 60); + assert.equal(f.calls.filter((c) => c.action === "get-log-events").length, 6); + assert.equal(sleeps, 62); + assert.equal(f.evidence().cleanup.stopped, false); + assert.equal(f.calls.at(-1).action, "put-object"); +}); + +test("cleanup failure stays bounded, still deregisters and archives, and never touches a neighboring family", async (t) => { + const f = fixture(t, (action, result) => { + if (action === "stop-task") throw new Error("Fixture stop denied"); + if (action === "list-task-definitions") result.taskDefinitionArns.push(`${ECS}:task-definition/${f.config.family}7:1`); + }); + const oldSummary = process.env.GITHUB_STEP_SUMMARY; + process.env.GITHUB_STEP_SUMMARY = join(f.config.work, "summary"); + t.after(() => { if (oldSummary === undefined) delete process.env.GITHUB_STEP_SUMMARY; else process.env.GITHUB_STEP_SUMMARY = oldSummary; }); + await execute(f.aws, f.config, async () => {}); + await assert.rejects(finish(f.aws, f.config, async () => {}), /cleanup failed/); + assert.deepEqual(f.calls.filter((c) => c.action === "deregister-task-definition").map((c) => c.args[1]), [f.arn]); + assert.ok(f.evidence().cleanup.errors.includes("Fixture stop denied")); + assert.equal(f.calls.at(-1).action, "put-object"); +}); diff --git a/.github/scripts/origin-tls-client.cjs b/.github/scripts/origin-tls-client.cjs new file mode 100644 index 000000000..3b5bd9cb6 --- /dev/null +++ b/.github/scripts/origin-tls-client.cjs @@ -0,0 +1,73 @@ +const { spawnSync } = require("node:child_process"); +const { existsSync } = require("node:fs"); + +const HOST = "api.warmbly.com"; +const CA = "/etc/ssl/certs/ca-certificates.crt"; +const CBC = ["ECDHE-RSA-AES128-SHA256", "ECDHE-RSA-AES256-SHA384"]; +const GCM = ["ECDHE-RSA-AES128-GCM-SHA256", "ECDHE-RSA-AES256-GCM-SHA384"]; +const TLS13 = "TLS_AES_128_GCM_SHA256"; + +function invoke(args, spawn = spawnSync) { + const result = spawn("openssl", args, { encoding: "utf8", input: "", timeout: 12000, + killSignal: "SIGKILL", maxBuffer: 128 * 1024 }); + return { at: new Date().toISOString(), command: ["openssl", ...args], status: result.status, + signal: result.signal || null, error: result.error?.code || null, + stdout: result.stdout || "", stderr: result.stderr || "" }; +} + +function classify(result, version, cipher, negative = false) { + if (result.error || result.signal) return "inconclusive"; + const output = result.stdout + result.stderr; + const negotiated = /New, (TLSv1\.[23]), Cipher is ([^\s]+)/.exec(output); + if (negative) { + if (/certificate verify failed|Verification error|verify error|no cipher match/i.test(output)) return "inconclusive"; + if (negotiated && negotiated[2] !== "(NONE)" && negotiated[2] !== "0000") return "accepted"; + return result.status !== 0 && /(?:sslv3|tlsv1) alert handshake failure|SSL alert number 40/i.test(output) && + /Cipher (?:is|:) (?:\(NONE\)|0000)/.test(output) ? "refused" : "inconclusive"; + } + return result.status === 0 && negotiated?.[1] === version && negotiated[2] === cipher && + /Verify return code: 0 \(ok\)/.test(output) && /-----BEGIN CERTIFICATE-----/.test(output) ? "trusted" : "inconclusive"; +} + +function probe(targets, run = invoke, emit = (row) => console.log(JSON.stringify(row))) { + if (!Array.isArray(targets) || targets.length !== 2 || new Set(targets).size !== 2 || + targets.some((ip) => !/^10\.42\.(?:0|1)\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4])$/.test(ip))) throw new Error("Two pinned origin IPv4 targets required"); + const version = run(["version", "-a"]); + emit({ kind: "client", ...version }); + if (version.status !== 0 || version.error || version.signal) throw new Error("OpenSSL unavailable"); + const support = run(["ciphers", "-s", "-tls1_2", CBC.join(":")]); + emit({ kind: "support", ...support }); + if (support.status !== 0 || support.error || support.signal || + support.stdout.trim().split(":").sort().join(":") !== [...CBC].sort().join(":")) throw new Error("Both CBC offers must be locally supported"); + let success = true; + for (const ip of targets) { + const handshake = (cipher, negative = false) => { + const tls13 = cipher === TLS13; + const result = run(["s_client", "-connect", `${ip}:443`, "-servername", HOST, "-verify_hostname", HOST, + "-verify_return_error", "-CAfile", CA, "-showcerts", tls13 ? "-tls1_3" : "-tls1_2", + tls13 ? "-ciphersuites" : "-cipher", cipher]); + const outcome = classify(result, tls13 ? "TLSv1.3" : "TLSv1.2", cipher, negative); + emit({ kind: "handshake", ip, cipher, negative, outcome, ...result }); + return outcome; + }; + const before = [handshake(GCM[0]), handshake(GCM[1]), handshake(TLS13)]; + const negatives = CBC.map((cipher) => handshake(cipher, true)); + const after = [handshake(GCM[0]), handshake(TLS13)]; + const passed = [...before, ...after].every((v) => v === "trusted") && negatives.every((v) => v === "refused"); + emit({ kind: "target", ip, outcome: passed ? "verified" : "inconclusive" }); + success = success && passed; + } + emit({ kind: "complete", success, at: new Date().toISOString() }); + return success; +} + +if (process.env.ORIGIN_TLS_TASK === "1") { + try { + if (!existsSync(CA)) throw new Error("System public CA bundle missing"); + process.exitCode = probe(JSON.parse(process.env.ORIGIN_TLS_TARGETS)) ? 0 : 1; + } catch (error) { + console.log(JSON.stringify({ kind: "inconclusive", message: error.message })); + process.exitCode = 1; + } +} +module.exports = { HOST, CA, CBC, GCM, TLS13, invoke, classify, probe }; diff --git a/.github/scripts/origin-tls-client.test.cjs b/.github/scripts/origin-tls-client.test.cjs new file mode 100644 index 000000000..d34f391bf --- /dev/null +++ b/.github/scripts/origin-tls-client.test.cjs @@ -0,0 +1,95 @@ +const { test } = require("node:test"); +const assert = require("node:assert/strict"); +const { CBC, GCM, TLS13, HOST, CA, invoke, classify, probe } = require("./origin-tls-client.cjs"); +const trusted = (cipher = GCM[0]) => ({ status: 0, stdout: `-----BEGIN CERTIFICATE-----\nfixture chain\n-----END CERTIFICATE-----\nNew, ${cipher === TLS13 ? "TLSv1.3" : "TLSv1.2"}, Cipher is ${cipher}\nVerify return code: 0 (ok)\n`, stderr: "" }); +const refused = () => ({ status: 1, stdout: "New, (NONE), Cipher is (NONE)\n", stderr: "error:0A000410:SSL routines:ssl3_read_bytes:sslv3 alert handshake failure: SSL alert number 40\n" }); + +test("positive proof requires exact negotiation, chain, verification and successful bounded execution", () => { + assert.equal(classify(trusted(), "TLSv1.2", GCM[0]), "trusted"); + for (const result of [ + { ...trusted(), status: 1 }, { ...trusted(), error: "ETIMEDOUT" }, { ...trusted(), signal: "SIGKILL" }, + { ...trusted(), stdout: trusted().stdout.replace("return code: 0 (ok)", "return code: 62 (hostname mismatch)") }, + { ...trusted(), stdout: trusted().stdout.replace("-----BEGIN CERTIFICATE-----", "") }, + trusted(GCM[1]), trusted(TLS13), + ]) assert.equal(classify(result, "TLSv1.2", GCM[0]), "inconclusive"); +}); + +test("only an explicit peer handshake refusal without negotiation establishes a negative", () => { + assert.equal(classify(refused(), "TLSv1.2", CBC[0], true), "refused"); + assert.equal(classify(trusted(CBC[0]), "TLSv1.2", CBC[0], true), "accepted"); + for (const result of [ + { ...refused(), signal: "SIGTERM" }, { ...refused(), error: "ETIMEDOUT" }, + { status: 1, stdout: "", stderr: "Connection reset by peer" }, + { status: 1, stdout: "", stderr: "connect:errno=110" }, + { status: 1, stdout: "", stderr: "no cipher match" }, + { status: 1, stdout: "", stderr: "certificate verify failed" }, + { status: 1, stdout: "", stderr: "tlsv1 alert protocol version:SSL alert number 70" }, + { ...refused(), stdout: "", status: 0 }, + ]) assert.equal(classify(result, "TLSv1.2", CBC[0], true), "inconclusive"); +}); + +function fixture(change = () => {}) { + const rows = []; + let handshakes = 0; + const run = (args) => { + if (args[0] === "version") return { status: 0, stdout: "OpenSSL 3.0 fixture", stderr: "" }; + if (args[0] === "ciphers") return { status: 0, stdout: CBC.join(":"), stderr: "" }; + handshakes++; + const cipher = args.at(-1); + const result = { ...(CBC.includes(cipher) ? refused() : trusted(cipher)), command: ["openssl", ...args] }; + change(result, handshakes); + return result; + }; + return { rows, run }; +} + +test("every private target has fresh trusted positives around both explicit CBC offers", () => { + const f = fixture(); + assert.equal(probe(["10.42.0.85", "10.42.1.12"], f.run, (row) => f.rows.push(row)), true); + const hs = f.rows.filter((r) => r.kind === "handshake"); + assert.equal(hs.length, 14); + for (const ip of ["10.42.0.85", "10.42.1.12"]) { + const own = hs.filter((r) => r.ip === ip); + assert.deepEqual(own.map((r) => r.cipher), [GCM[0], GCM[1], TLS13, ...CBC, GCM[0], TLS13]); + for (const row of own) { + const args = row.command; + for (const [flag, value] of [["-connect", `${ip}:443`], ["-servername", HOST], ["-verify_hostname", HOST], ["-CAfile", CA]]) { + assert.equal(args[args.indexOf(flag) + 1], value); + } + assert.ok(args.includes("-verify_return_error")); + assert.ok(args.includes("-showcerts")); + assert.ok(args.includes(row.cipher === TLS13 ? "-tls1_3" : "-tls1_2")); + } + } + assert.deepEqual(f.rows.at(-1).success, true); +}); + +test("timeout or broken trailing positive makes the whole run inconclusive", () => { + for (const index of [4, 6, 14]) { + const f = fixture((result, n) => { if (n === index) result.error = "ETIMEDOUT"; }); + assert.equal(probe(["10.42.0.85", "10.42.1.12"], f.run, (row) => f.rows.push(row)), false); + assert.equal(f.rows.at(-1).success, false); + } +}); + +test("local CBC support is checked before network traffic and arbitrary destinations are refused", () => { + const f = fixture(); + assert.throws(() => probe(["1.1.1.1", "10.42.1.12"], f.run), /pinned/); + assert.throws(() => probe(["10.42.0.85", "10.42.0.85"], f.run), /pinned/); + assert.throws(() => probe(["10.42.0.85", "10.42.1.12"], (args) => ({ status: 0, stdout: args[0] === "version" ? "OpenSSL" : CBC[0] }), () => {}), /Both CBC/); +}); + +test("OpenSSL processes get empty stdin, a hard timeout, bounded output and no shell", () => { + const result = invoke(["version"], (command, args, options) => { + assert.equal(command, "openssl"); + assert.deepEqual(args, ["version"]); + assert.equal(options.input, ""); + assert.equal(options.timeout, 12000); + assert.equal(options.killSignal, "SIGKILL"); + assert.equal(options.shell, undefined); + assert.equal(options.maxBuffer, 128 * 1024); + return { status: null, error: { code: "ETIMEDOUT" }, signal: "SIGKILL" }; + }); + assert.equal(result.error, "ETIMEDOUT"); + assert.equal(result.signal, "SIGKILL"); +}); diff --git a/.github/workflows/probe-origin-tls-aws.yml b/.github/workflows/probe-origin-tls-aws.yml new file mode 100644 index 000000000..269934a0d --- /dev/null +++ b/.github/workflows/probe-origin-tls-aws.yml @@ -0,0 +1,72 @@ +name: Probe AWS origin TLS + +on: + pull_request: + paths: + - '.github/workflows/probe-origin-tls-aws.yml' + - '.github/scripts/aws-origin-tls*' + - '.github/scripts/origin-tls-client*' + - '.github/scripts/aws-origin-alb.cjs' + workflow_dispatch: + +permissions: + contents: read + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-node@v4 + with: + node-version: "22" + - run: node --test .github/scripts/aws-origin-tls.test.cjs .github/scripts/origin-tls-client.test.cjs + - run: shellcheck .github/scripts/aws-origin-tls-workflow.test.sh + - run: bash .github/scripts/aws-origin-tls-workflow.test.sh + + probe: + needs: check + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 25 + environment: aws-infrastructure + concurrency: + group: aws-production + cancel-in-progress: false + permissions: + contents: read + id-token: write + env: + STATE_BUCKET: ${{ vars.AWS_TERRAFORM_STATE_BUCKET }} + APPLY_ROLE: ${{ vars.AWS_TERRAFORM_APPLY_ROLE_ARN }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-node@v4 + with: + node-version: "22" + - name: Check main and ordinary infrastructure configuration + id: scope + env: + WORK: ${{ runner.temp }}/origin-tls + run: node .github/scripts/aws-origin-tls.cjs session-policy + - uses: aws-actions/configure-aws-credentials@v4 + id: credentials + with: + role-to-assume: ${{ vars.AWS_TERRAFORM_APPLY_ROLE_ARN }} + aws-region: us-east-1 + inline-session-policy: ${{ steps.scope.outputs.policy }} + - name: Run disposable secretless TLS diagnostic + timeout-minutes: 10 + env: + WORK: ${{ runner.temp }}/origin-tls + run: node .github/scripts/aws-origin-tls.cjs probe + - name: Stop, deregister and privately archive diagnostic evidence + if: always() && steps.credentials.outcome == 'success' + timeout-minutes: 8 + env: + WORK: ${{ runner.temp }}/origin-tls + run: node .github/scripts/aws-origin-tls.cjs finish diff --git a/deploy/terraform/aws/runtime/README.md b/deploy/terraform/aws/runtime/README.md index 5b4871f41..2d4453b0c 100644 --- a/deploy/terraform/aws/runtime/README.md +++ b/deploy/terraform/aws/runtime/README.md @@ -214,3 +214,82 @@ background jobs immediately; its desired count is the activation control. Preserve compatible images, encryption material and shared coordination while the previous deployment drains. Verify target health and HTTPS routes before switching DNS. New injected values require replacing tasks. + +## Disposable origin TLS diagnostic + +`probe-origin-tls-aws.yml` verifies the hosted origin's TLS negotiation from the +existing application task security group. It uses `aws-infrastructure`, its main +branch restriction, `AWS_TERRAFORM_APPLY_ROLE_ARN`, +`AWS_TERRAFORM_STATE_BUCKET`, and the shared `aws-production` queue. It makes no +listener, service, security group, IAM or Terraform changes. Existing deployment +and self-host configuration remain compatible; no application release is needed. + +The dedicated Fargate definition has no task role, secrets, repository credentials, +mounts or ports. It uses the existing validation execution role and validation +log group, without running the dependency validation definition. The client is +the official Docker Library Node 22 bookworm ARM64 image at +`sha256:a288699fc0f9676950742fae6efcd015180aa7ef2bf8a6477e4b0a8cc0ab1b3f` +(official manifest from `node:22-bookworm`, source revision +`80f02a542da739e6a662451d1db66456a0dea253`). OpenSSL and the Debian public CA bundle +must be present. Nothing is installed at runtime. The source and image pin pass +through normal review; there is no image or command override input. + +Operator procedure: + +1. Merge the reviewed source normally. Complete the separate normal origin ALB + policy procedure first. The diagnostic requires the bound HTTPS listener's + `ELBSecurityPolicy-TLS13-1-2-Res-2021-06` readback. +2. Confirm the existing infrastructure OIDC role can register/list/deregister + this run's `warmbly-production-origin-tls-` task family; run one task + in `warmbly-production`; list/describe/stop its tasks; and pass only + `warmbly-production-validation-execution` to ECS. It also needs the ALB, + listener, network interface, subnet and backend service describe calls in + `.github/scripts/aws-origin-tls.cjs`, plus `logs:GetLogEvents` on + `/ecs/warmbly-production/validation` and conditional `s3:PutObject` on + `warmbly/production/diagnostics/origin-tls//1/*` in the private state + bucket. The workflow's session policy only restricts existing permissions. + Missing access requires the normal IAM process; the workflow never expands + a role or switches environments. Read-only TLS metadata does not require + access to secret values, SSM parameters or application task credentials. +3. The existing execution role must already support logging to that group. The + existing application subnets must support Fargate ARM64 platform `1.4.0`, + public image pulls and CloudWatch Logs, with the existing application task + security group reaching the ALB private interfaces on TCP 443. The workflow + reuses the backend's existing public-subnet network configuration. It does + not add a network path or enable ECS Exec. +4. Dispatch exactly once on merged `main`: + `gh workflow run probe-origin-tls-aws.yml --repo warmbly/warmbly --ref main`. + Apply the environment's normal protection/reviewer process. Reruns are + refused; investigate the private result before starting a new dispatch. +5. Read the run summary for the private result's exact S3 version ID and + SHA-256. Retrieve `intent.json` and `result.json` privately from + `warmbly/production/diagnostics/origin-tls//1/`. Both use conditional + creation, AES256 encryption, SHA-256 and a non-null version ID. Do not attach + raw output to a public issue, PR or Actions artifact. The result retains + UTC times, exact ALB/ENI/AZ identities, image and source identities, task + metadata, client version, arguments, exit status, stdout, stderr and peer + certificate chains. Success requires the complete raw handshake evidence, + exact task image digest and confirmed diagnostic cleanup. + +The transport targets are freshly described private interfaces of the exact +ALB, never the public API DNS address. The SNI and verified hostname stay +`api.warmbly.com`; the client trusts its system public CAs. Each interface gets +both TLS 1.2 RSA AES-GCM positives and a TLS 1.3 AES-GCM positive, then separate +CBC-only TLS 1.2 offers, then fresh TLS 1.2 and TLS 1.3 positives. Only explicit +peer handshake-failure alerts without a negotiated cipher, bracketed by those +trusted positives, establish refusal. Timeout, reset alone, unavailable local +ciphers, missing CA, certificate errors or incomplete evidence are inconclusive. +The task sends TLS handshakes only, with empty stdin and no HTTP or authentication. + +Each OpenSSL process has a 12-second kill bound, and the container command has a +210-second hard kill bound after it starts. The probe step is limited to ten +minutes. An `always()` final step has eight minutes to recover lost task/definition +responses, stop only this run's diagnostic tasks, confirm they stopped, +deregister only this run's diagnostic definitions, and privately archive logs and +cleanup results. Cleanup failure fails the workflow, even after successful +handshakes. Cancellation, runner loss or AWS API/access failures cannot guarantee +control-plane cleanup; the in-container limit still bounds a running client. +If the finalizer cannot complete, use the archived diagnostic ARNs (or list tasks +by `startedBy=origin-tls-` and definitions by the exact family) to stop, +confirm STOPPED and deregister only those resources through normal operator +access. Do not delete log groups, ENIs, security groups or application resources.