diff --git a/.github/scripts/runtime-tls-recovery-lineage.sh b/.github/scripts/runtime-tls-recovery-lineage.sh new file mode 100644 index 000000000..58276f7da --- /dev/null +++ b/.github/scripts/runtime-tls-recovery-lineage.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Preserve historical workflow identities when collecting recovery evidence. +set -euo pipefail + +check_contract() { + jq -e ' + .schemaVersion == 1 and .repository == "warmbly/warmbly" and + all(.release, .drain, .failedOn; + (.tag | type == "string" and test("^v[0-9]+\\.[0-9]+\\.[0-9]+$")) and + (.source | type == "string" and test("^[a-f0-9]{40}$")) and + (.runId | type == "number" and . > 0 and floor == .) and + (.attempt | type == "number" and . > 0 and floor == .)) and + ([.release.runId, .drain.runId, .failedOn.runId] | unique | length == 3) and + .drain.tag == .failedOn.tag and .release.tag != .drain.tag + ' "$1" >/dev/null || { echo 'Recovery requires explicit current and historical workflow identities' >&2; return 1; } +} + +check_lineage() { + local evidence=$1 contract=$2 repository=$3 tag source + check_contract "$contract" + jq -e --slurpfile contract "$contract" ' + $contract[0] as $expected | + def run_matches($identity; $path; $event; $branch; $conclusion): + .id == $identity.runId and .run_attempt == $identity.attempt and + .head_sha == $identity.source and .repository.full_name == $expected.repository and + .path == $path and .event == $event and .head_branch == $branch and + .status == "completed" and .conclusion == $conclusion; + (.runs.release | run_matches($expected.release; ".github/workflows/release.yml"; "push"; $expected.release.tag; "success")) and + (.runs.drain | run_matches($expected.drain; ".github/workflows/runtime-tls.yml"; "workflow_dispatch"; "main"; "success") and + .display_title == ("Runtime TLS drain " + $expected.drain.tag)) and + (.runs.failedOn | run_matches($expected.failedOn; ".github/workflows/runtime-tls.yml"; "workflow_dispatch"; "main"; "failure") and + .display_title == ("Runtime TLS on " + $expected.failedOn.tag)) and + (.failedOnJobs | type == "array" and length > 0) and + ([.failedOnJobs[].id] | unique | length) == (.failedOnJobs | length) and + all(.failedOnJobs[]; + .run_id == $expected.failedOn.runId and .run_attempt == $expected.failedOn.attempt and + .status == "completed") and + ([.failedOnJobs[] | select(.name == "apply")] | length == 1) and + any(.failedOnJobs[]; .name == "apply" and .conclusion == "success") and + any(.failedOnJobs[]; (.name | startswith("redeploy / ")) and .conclusion == "failure") + ' "$evidence" >/dev/null || { echo 'Recovery workflow evidence differs from the reviewed lineage' >&2; return 1; } + tag=$(jq -r '.release.tag' "$contract") + source=$(git -C "$repository" rev-parse --verify "refs/tags/$tag^{commit}") + [ "$source" = "$(jq -r '.release.source' "$contract")" ] || { echo 'Recovery release tag differs from its reviewed source' >&2; return 1; } + git -C "$repository" merge-base --is-ancestor "$(jq -r '.drain.source' "$contract")" "$(jq -r '.failedOn.source' "$contract")" || { + echo 'Recovery ON source does not descend from the reviewed drain source' >&2; return 1; + } + git -C "$repository" merge-base --is-ancestor "$(jq -r '.failedOn.source' "$contract")" "$source" || { + echo 'Recovery release does not descend from the reviewed ON source' >&2; return 1; + } +} + +collect_lineage() { + local contract=$1 work=$2 repository=$3 repo identity run attempt + check_contract "$contract" + umask 077 + mkdir "$work" + repo=$(jq -r '.repository' "$contract") + for identity in release drain failedOn; do + run=$(jq -r --arg identity "$identity" '.[$identity].runId' "$contract") + attempt=$(jq -r --arg identity "$identity" '.[$identity].attempt' "$contract") + gh api "repos/$repo/actions/runs/$run/attempts/$attempt" >"$work/$identity.json" + done + run=$(jq -r '.failedOn.runId' "$contract") + attempt=$(jq -r '.failedOn.attempt' "$contract") + gh api "repos/$repo/actions/runs/$run/attempts/$attempt/jobs?per_page=100" --paginate --slurp >"$work/job-pages.json" + jq -e '.[0].total_count == ([.[].jobs[]] | length)' "$work/job-pages.json" >/dev/null || { + echo 'Recovery requires the complete failed ON job inventory' >&2; return 1; + } + jq -n --slurpfile release "$work/release.json" --slurpfile drain "$work/drain.json" \ + --slurpfile failedOn "$work/failedOn.json" --slurpfile jobs "$work/job-pages.json" \ + '{runs:{release:$release[0],drain:$drain[0],failedOn:$failedOn[0]},failedOnJobs:[$jobs[0][].jobs[]]}' >"$work/candidate.json" + check_lineage "$work/candidate.json" "$contract" "$repository" + mv "$work/candidate.json" "$work/lineage.json" +} + +case "${1:-}" in + check) check_lineage "${2:?}" "${3:?}" "${4:?}" ;; + collect) collect_lineage "${2:?}" "${3:?}" "${4:?}" ;; + *) echo 'Only read-only recovery lineage check and collection are supported' >&2; exit 1 ;; +esac diff --git a/.github/scripts/runtime-tls-recovery-lineage.test.sh b/.github/scripts/runtime-tls-recovery-lineage.test.sh new file mode 100644 index 000000000..7974d4e0b --- /dev/null +++ b/.github/scripts/runtime-tls-recovery-lineage.test.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/../.." +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +guard=.github/scripts/runtime-tls-recovery-lineage.sh +mkdir "$work/repository" +git -C "$work/repository" init -q +git -C "$work/repository" config user.name 'Recovery fixture' +git -C "$work/repository" config user.email 'fixture@example.invalid' +git -C "$work/repository" commit -qm 'feat: establish drain fixture' --allow-empty +drain=$(git -C "$work/repository" rev-parse HEAD) +git -C "$work/repository" commit -qm 'feat: establish ON fixture' --allow-empty +on=$(git -C "$work/repository" rev-parse HEAD) +git -C "$work/repository" commit -qm 'feat: establish release fixture' --allow-empty +release=$(git -C "$work/repository" rev-parse HEAD) +git -C "$work/repository" tag v1.0.1 +jq -n --arg drain "$drain" --arg on "$on" --arg release "$release" ' + {schemaVersion:1,repository:"warmbly/warmbly", + release:{tag:"v1.0.1",source:$release,runId:101,attempt:1}, + drain:{tag:"v1.0.0",source:$drain,runId:102,attempt:1}, + failedOn:{tag:"v1.0.0",source:$on,runId:103,attempt:1}} +' >"$work/contract.json" +jq ' + def run($path; $event; $branch; $conclusion): + {id:.runId,run_attempt:.attempt,head_sha:.source,repository:{full_name:"warmbly/warmbly"}, + path:$path,event:$event,head_branch:$branch,status:"completed",conclusion:$conclusion}; + {runs:{ + release:(.release | run(".github/workflows/release.yml";"push";.tag;"success")), + drain:(.drain | run(".github/workflows/runtime-tls.yml";"workflow_dispatch";"main";"success") + {display_title:("Runtime TLS drain " + .tag)}), + failedOn:(.failedOn | run(".github/workflows/runtime-tls.yml";"workflow_dispatch";"main";"failure") + {display_title:("Runtime TLS on " + .tag)})}, + failedOnJobs:[{id:1,run_id:103,run_attempt:1,status:"completed",name:"apply",conclusion:"success"}, + {id:2,run_id:103,run_attempt:1,status:"completed",name:"redeploy / redeploy v1.0.0",conclusion:"failure"}]} +' "$work/contract.json" >"$work/evidence.json" +bash "$guard" check "$work/evidence.json" "$work/contract.json" "$work/repository" +for change in '.schemaVersion=2' '.repository="other/repo"' '.release.runId=.drain.runId' \ + '.drain.tag=.release.tag' '.failedOn.tag="v0.9.0"' '.release.tag="main"' \ + '.failedOn.attempt=0' '.drain.source="unknown"'; do + jq "$change" "$work/contract.json" >"$work/changed.json" + if bash "$guard" check "$work/evidence.json" "$work/changed.json" "$work/repository" 2>/dev/null; then + echo "Recovery accepted a different lineage contract: $change" >&2; exit 1 + fi +done +for change in '.runs.release.id=104' '.runs.release.head_sha=.runs.drain.head_sha' \ + '.runs.release.repository.full_name="other/repo"' '.runs.release.head_branch="main"' \ + '.runs.release.path=".github/workflows/deploy-aws.yml"' '.runs.release.conclusion="failure"' \ + '.runs.release.event="workflow_dispatch"' '.runs.release.status="in_progress"' \ + '.runs.drain.run_attempt=2' '.runs.drain.conclusion="failure"' \ + '.runs.drain.display_title="Runtime TLS drain v1.0.1"' \ + '.runs.failedOn.display_title="Runtime TLS on v1.0.1"' '.runs.failedOn.conclusion="success"' \ + '.failedOnJobs=[]' '.failedOnJobs += [.failedOnJobs[0]]' '.failedOnJobs[0].conclusion="failure"' \ + '.failedOnJobs[1].conclusion="success"' '.failedOnJobs[0].run_id=104' \ + '.failedOnJobs[0].run_attempt=2' '.failedOnJobs[1].status="in_progress"'; do + jq "$change" "$work/evidence.json" >"$work/changed.json" + if bash "$guard" check "$work/changed.json" "$work/contract.json" "$work/repository" 2>/dev/null; then + echo "Recovery accepted different workflow evidence: $change" >&2; exit 1 + fi +done +git -C "$work/repository" tag -f v1.0.1 "$drain" >/dev/null +if bash "$guard" check "$work/evidence.json" "$work/contract.json" "$work/repository" 2>/dev/null; then + echo 'Recovery accepted a release tag on a different source' >&2; exit 1 +fi +git -C "$work/repository" tag -f v1.0.1 "$release" >/dev/null +jq --arg source "$release" '.drain.source=$source' "$work/contract.json" >"$work/reversed-contract.json" +jq --arg source "$release" '.runs.drain.head_sha=$source' "$work/evidence.json" >"$work/reversed-evidence.json" +if bash "$guard" check "$work/reversed-evidence.json" "$work/reversed-contract.json" "$work/repository" 2>/dev/null; then + echo 'Recovery accepted reversed drain and ON ancestry' >&2; exit 1 +fi +jq --arg source "$drain" '.release.source=$source' "$work/contract.json" >"$work/reversed-contract.json" +jq --arg source "$drain" '.runs.release.head_sha=$source' "$work/evidence.json" >"$work/reversed-evidence.json" +git -C "$work/repository" tag -f v1.0.1 "$drain" >/dev/null +if bash "$guard" check "$work/reversed-evidence.json" "$work/reversed-contract.json" "$work/repository" 2>/dev/null; then + echo 'Recovery accepted a release that predates the ON source' >&2; exit 1 +fi +git -C "$work/repository" tag -f v1.0.1 "$release" >/dev/null +mkdir "$work/bin" +cat >"$work/bin/gh" <<'SH' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >>"$FIXTURES/requests.log" +[ "${API_FAIL:-false}" = false ] || exit 1 +case "$2" in + */runs/101/attempts/1) jq '.runs.release' "$FIXTURES/evidence.json" ;; + */runs/102/attempts/1) jq '.runs.drain' "$FIXTURES/evidence.json" ;; + */runs/103/attempts/1) jq '.runs.failedOn' "$FIXTURES/evidence.json" ;; + */runs/103/attempts/1/jobs\?per_page=100) + [ "$3 $4" = '--paginate --slurp' ] + jq --arg mode "${JOB_MODE:-complete}" '[{total_count:2,jobs:(if $mode == "partial" then .failedOnJobs[:1] else .failedOnJobs end)}]' "$FIXTURES/evidence.json" ;; + *) exit 1 ;; +esac +SH +chmod +x "$work/bin/gh" +PATH="$work/bin:$PATH" FIXTURES="$work" bash "$guard" collect "$work/contract.json" "$work/collected" "$work/repository" +cmp "$work/evidence.json" "$work/collected/lineage.json" +if PATH="$work/bin:$PATH" FIXTURES="$work" JOB_MODE=partial bash "$guard" collect "$work/contract.json" "$work/partial" "$work/repository" 2>/dev/null; then + echo 'Recovery accepted an incomplete job collection' >&2; exit 1 +fi +[ ! -e "$work/partial/lineage.json" ] +if PATH="$work/bin:$PATH" FIXTURES="$work" API_FAIL=true bash "$guard" collect "$work/contract.json" "$work/api-failure" "$work/repository" 2>/dev/null; then + echo 'Recovery accepted a failed GitHub API read' >&2; exit 1 +fi +[ ! -e "$work/api-failure/lineage.json" ] +if PATH="$work/bin:$PATH" FIXTURES="$work" bash "$guard" collect "$work/contract.json" "$work/collected" "$work/repository" 2>/dev/null; then + echo 'Recovery overwrote an existing lineage collection' >&2; exit 1 +fi +if bash "$guard" apply 2>/dev/null; then + echo 'Recovery lineage collector accepted an execution action' >&2; exit 1 +fi +echo 'Read-only recovery workflow lineage collection and checks passed' diff --git a/.github/workflows/runtime-tls.yml b/.github/workflows/runtime-tls.yml index 0606b67ac..9289782f8 100644 --- a/.github/workflows/runtime-tls.yml +++ b/.github/workflows/runtime-tls.yml @@ -46,6 +46,8 @@ jobs: - run: bash .github/scripts/runtime-tls.test.sh - run: shellcheck .github/scripts/runtime-tls-recovery.sh .github/scripts/runtime-tls-recovery.test.sh - run: bash .github/scripts/runtime-tls-recovery.test.sh + - run: shellcheck .github/scripts/runtime-tls-recovery-lineage.sh .github/scripts/runtime-tls-recovery-lineage.test.sh + - run: bash .github/scripts/runtime-tls-recovery-lineage.test.sh plan: needs: check diff --git a/docs/content/docs/development/aws-fargate.mdx b/docs/content/docs/development/aws-fargate.mdx index fac16624e..967d1dac9 100644 --- a/docs/content/docs/development/aws-fargate.mdx +++ b/docs/content/docs/development/aws-fargate.mdx @@ -447,6 +447,38 @@ HTTP references and registrations have drained. Review and approve each plan separately. Persist `target_tls=on` only after it succeeds, then verify the effective loopback listeners and HTTPS routes. +### Validate recovery evidence + +The read-only recovery validators keep a recovery contract separate from the +normal route, drain and on transition checks. They do not provide a production +recovery execution workflow. The baseline identifies four drained HTTP groups +and their reviewed state projection. `check-state` verifies a supplied raw +Terraform state file against the baseline SHA-256, lineage, serial and exact +group identities. `check-plan` permits only the four corresponding deletions +without refresh drift. The baseline and state version must come from a trusted, +reviewed collection process; matching a file hash alone does not establish that +provenance. + +`runtime-tls-recovery-lineage.sh` collects and checks the GitHub portion of a +recovery contract. Its version 1 contract names `warmbly/warmbly` and three +identities, `release`, `drain` and `failedOn`, each with a stable release `tag`, +40-character commit `source`, numeric `runId` and numeric `attempt`. The current +release is distinct from the historical release retained by both TLS runs. +Collection reads the exact run attempts and the complete failed ON job inventory. +It requires the current release and historical drain to have succeeded, the ON +apply to have succeeded before a failed redeploy, and each workflow path, branch, +title and source to match its declared role. The local checkout must include the +release tag and full commit history so the checker can verify the tag and +drain-to-ON-to-release ancestry. Collection writes a new private directory and +refuses to overwrite a prior collection. + +These checks neither relabel an earlier release's drain nor change the normal +workflow's predecessor requirements. A complete recovery workflow must also +collect current service, TLS, route and target health, bind the immutable vars +snapshot and state version, and preserve separate protected approval, production +serialization, plan digest comparison, attested redeployment and final effective +verification. Do not use these validators as an alternative to those gates. + ## One-time GitHub setup 1. On the `aws-production` environment, allow the `v*.*.*` tag pattern next to