From daaed572cce6dec7eeabc1df3f0ce4083a158da2 Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Wed, 4 Nov 2026 16:32:27 +0100 Subject: [PATCH] feat: power off a fleet worker or NAT instance whose boot script fails so the Auto Scaling group replaces it, stop an in-flight node update before the lifecycle watcher stops a terminating worker, cap fleet_worker_capacity at the 99999999 the worker accepts with a plan test, bound each CloudWatch fleet metrics publish to one interval, and document FLEET_METRICS_CLOUDWATCH as a fleet enable step and join token renewal every three weeks --- .../aws/workers/fleet-bootstrap.sh.tftpl | 18 +++++++++++++----- deploy/terraform/aws/workers/fleet.tf | 6 +++--- .../aws/workers/nat-bootstrap.sh.tftpl | 10 ++++++++++ .../aws/workers/tests/fleet.tftest.hcl | 10 ++++++++++ docs/content/docs/development/aws-workers.mdx | 11 ++++++++--- internal/app/fleetmetrics/fleetmetrics.go | 8 +++++++- 6 files changed, 51 insertions(+), 12 deletions(-) diff --git a/deploy/terraform/aws/workers/fleet-bootstrap.sh.tftpl b/deploy/terraform/aws/workers/fleet-bootstrap.sh.tftpl index 372ea5001..155ed4603 100644 --- a/deploy/terraform/aws/workers/fleet-bootstrap.sh.tftpl +++ b/deploy/terraform/aws/workers/fleet-bootstrap.sh.tftpl @@ -72,7 +72,8 @@ imds() { main() { while :; do if [ "$(imds meta-data/autoscaling/target-lifecycle-state 2>/dev/null || true)" = "Terminated" ]; then - systemctl stop warmbly-node-update.timer || true + # Stopping the service too ends an update in flight, whose rollback would restart the worker. + systemctl stop warmbly-node-update.timer warmbly-node-update.service || true systemctl stop warmbly-worker || true printf '%s\n' 'warmbly-lifecycle-watch: instance is terminating; worker stopped' return 0 @@ -120,6 +121,16 @@ write_local_env() { fi } +# Until the join succeeds, a failed step powers off, and the group replaces the instance. +on_exit() { + rc=$? + rm -f /run/warmbly-join.sh + if [ "$rc" -ne 0 ]; then + log "boot failed with status $rc; powering off" + systemctl poweroff || true + fi +} + join_once() { token=$(aws ssm get-parameter --region "$REGION" --name "$TOKEN_PARAMETER" --with-decryption --query Parameter.Value --output text) || return 1 [ -n "$token" ] || return 1 @@ -132,6 +143,7 @@ join_once() { } main() { + trap on_exit EXIT instance_id=$(retry imds meta-data/instance-id) NODE_NAME="$NAME_PREFIX-$instance_id" hostnamectl set-hostname "$NODE_NAME" @@ -149,15 +161,11 @@ main() { until join_once; do attempts=$((attempts + 1)) if [ "$attempts" -ge 20 ]; then - # Terminating lets the group launch a replacement instead of carrying a worker that never joined. log "join failed $attempts times; check the join token in $TOKEN_PARAMETER and $INSTANCE_URL" - rm -f /run/warmbly-join.sh - systemctl poweroff return 1 fi sleep 30 done - rm -f /run/warmbly-join.sh log "joined as $NODE_NAME" return 0 } diff --git a/deploy/terraform/aws/workers/fleet.tf b/deploy/terraform/aws/workers/fleet.tf index 64450755c..2f3c93ea8 100644 --- a/deploy/terraform/aws/workers/fleet.tf +++ b/deploy/terraform/aws/workers/fleet.tf @@ -2,7 +2,7 @@ # main.tf, which keep running until an operator retires them. variable "fleet_enabled" { - description = "Create the autoscaling worker fleet and its network" + description = "Create the autoscaling worker fleet and its network; the backend also needs FLEET_METRICS_CLOUDWATCH set to name, or the memory metrics it scales on are never published" type = bool default = false } @@ -79,8 +79,8 @@ variable "fleet_worker_capacity" { type = string default = "auto" validation { - condition = var.fleet_worker_capacity == "auto" || can(regex("^[1-9][0-9]*$", var.fleet_worker_capacity)) - error_message = "Use auto or a positive whole number." + condition = var.fleet_worker_capacity == "auto" || can(regex("^[1-9][0-9]{0,7}$", var.fleet_worker_capacity)) + error_message = "Use auto or a whole number from 1 to 99999999, the largest the worker accepts." } } diff --git a/deploy/terraform/aws/workers/nat-bootstrap.sh.tftpl b/deploy/terraform/aws/workers/nat-bootstrap.sh.tftpl index 1906f74a5..bc5194995 100644 --- a/deploy/terraform/aws/workers/nat-bootstrap.sh.tftpl +++ b/deploy/terraform/aws/workers/nat-bootstrap.sh.tftpl @@ -89,7 +89,17 @@ replace_route() { aws ec2 create-route --region "$REGION" --route-table-id "$ROUTE_TABLE_ID" --destination-cidr-block 0.0.0.0/0 --network-interface-id "$1" >/dev/null } +# A boot that fails leaves the zone without egress, so power off and let the group replace the instance. +on_exit() { + rc=$? + if [ "$rc" -ne 0 ]; then + log "boot failed with status $rc; powering off" + shutdown -h now || true + fi +} + main() { + trap on_exit EXIT retry dnf -y --setopt=retries=20 install iptables-nft write_kernel_settings write_rules_unit diff --git a/deploy/terraform/aws/workers/tests/fleet.tftest.hcl b/deploy/terraform/aws/workers/tests/fleet.tftest.hcl index 4850a3c5a..e11dd207a 100644 --- a/deploy/terraform/aws/workers/tests/fleet.tftest.hcl +++ b/deploy/terraform/aws/workers/tests/fleet.tftest.hcl @@ -169,6 +169,16 @@ run "fleet_worker_capacity_rejects_garbage" { expect_failures = [var.fleet_worker_capacity] } +run "fleet_worker_capacity_rejects_what_the_worker_discards" { + command = plan + variables { + fleet_enabled = true + fleet_instance_url = "https://app.example.com" + fleet_worker_capacity = "100000000" + } + expect_failures = [var.fleet_worker_capacity] +} + run "fleet_scale_in_sits_under_scale_out" { command = plan variables { diff --git a/docs/content/docs/development/aws-workers.mdx b/docs/content/docs/development/aws-workers.mdx index 59f213b3a..6a9cc61d7 100644 --- a/docs/content/docs/development/aws-workers.mdx +++ b/docs/content/docs/development/aws-workers.mdx @@ -78,8 +78,8 @@ The parameter name defaults to `//fleet/join-token` and can be changed with `fleet_join_token_parameter`. A join token works for at most 30 days, and issuing a new one revokes the previous one. Joined workers are unaffected, but a worker launched with an expired token cannot join. Issue and store a new token -before the old one expires, at least monthly, or scale-out stops producing -working workers. +before the old one expires, for example every three weeks, or scale-out stops +producing working workers. ## Enable the fleet @@ -100,6 +100,11 @@ working workers. `nats_security_group_id` is the NATS root's broker security group. Leave it unset only when workers reach the event bus some other way. +4. Set `FLEET_METRICS_CLOUDWATCH` on the backend to the root's `name` and give + its role `cloudwatch:PutMetricData` (see the [metric contract](#metric-contract)). + Without it nothing publishes `WorkerMemoryPercent`, so the fleet scales out + on CPU only and never scales in. + The `fleet` output names the group, its subnets and security group, the join token parameter, the metric contract and `nat_ipv4_addresses`. Workers appear in `warmblyctl fleet list` within a few minutes of launching. @@ -118,7 +123,7 @@ token parameter, the metric contract and `nat_ipv4_addresses`. Workers appear in | `fleet_min_size` | `2` | Smallest fleet | | `fleet_max_size` | `10` | Largest fleet | | `fleet_desired_capacity` | `null` | Starting size; scaling owns it afterwards. `null` starts at the minimum | -| `fleet_worker_capacity` | `auto` | `WARMBLY_WORKER_CAPACITY` on every fleet worker: `auto`, or a positive whole number of mailboxes | +| `fleet_worker_capacity` | `auto` | `WARMBLY_WORKER_CAPACITY` on every fleet worker: `auto`, or a whole number of mailboxes from 1 to 99999999 | | `fleet_region_label` | empty | Optional `--region` label for placement | | `fleet_image_repository` | `ghcr.io/warmbly/warmbly` | Repository `join.sh` pulls the worker image from | | `fleet_target_memory` | `65` | Target for `WorkerMemoryPercent`, the fleet's memory in use | diff --git a/internal/app/fleetmetrics/fleetmetrics.go b/internal/app/fleetmetrics/fleetmetrics.go index a9c8ae141..c5480f56e 100644 --- a/internal/app/fleetmetrics/fleetmetrics.go +++ b/internal/app/fleetmetrics/fleetmetrics.go @@ -87,7 +87,10 @@ func Run(ctx context.Context, d *db.DB, fleet string) error { } ticker := time.NewTicker(interval) defer ticker.Stop() - for { + publish := func() { + // A stalled read or request must not hold back the next minute's sample. + ctx, cancel := context.WithTimeout(ctx, interval) + defer cancel() if u, err := Read(ctx, d); err != nil { log.Warn().Err(err).Msg("fleet usage not published") } else if !u.Known { @@ -102,6 +105,9 @@ func Run(ctx context.Context, d *db.DB, fleet string) error { }); err != nil { log.Warn().Err(err).Msg("fleet usage not published") } + } + for { + publish() select { case <-ctx.Done(): return nil