feat: power off a fleet worker or NAT instance whose boot script fails so the Auto Scaling group replaces it, stop an in-flight node update before the lifecycle watcher stops a terminating worker, cap fleet_worker_capacity at the 99999999 the worker accepts with a plan test, bound each CloudWatch fleet metrics publish to one interval, and document FLEET_METRICS_CLOUDWATCH as a fleet enable step and join token renewal every three weeks

This commit is contained in:
Matthew Meszaros
2026-11-04 16:32:27 +01:00
parent a7721887a8
commit daaed572cc
6 changed files with 51 additions and 12 deletions
@@ -72,7 +72,8 @@ imds() {
main() {
while :; do
if [ "$(imds meta-data/autoscaling/target-lifecycle-state 2>/dev/null || true)" = "Terminated" ]; then
systemctl stop warmbly-node-update.timer || true
# Stopping the service too ends an update in flight, whose rollback would restart the worker.
systemctl stop warmbly-node-update.timer warmbly-node-update.service || true
systemctl stop warmbly-worker || true
printf '%s\n' 'warmbly-lifecycle-watch: instance is terminating; worker stopped'
return 0
@@ -120,6 +121,16 @@ write_local_env() {
fi
}
# Until the join succeeds, a failed step powers off, and the group replaces the instance.
on_exit() {
rc=$?
rm -f /run/warmbly-join.sh
if [ "$rc" -ne 0 ]; then
log "boot failed with status $rc; powering off"
systemctl poweroff || true
fi
}
join_once() {
token=$(aws ssm get-parameter --region "$REGION" --name "$TOKEN_PARAMETER" --with-decryption --query Parameter.Value --output text) || return 1
[ -n "$token" ] || return 1
@@ -132,6 +143,7 @@ join_once() {
}
main() {
trap on_exit EXIT
instance_id=$(retry imds meta-data/instance-id)
NODE_NAME="$NAME_PREFIX-$instance_id"
hostnamectl set-hostname "$NODE_NAME"
@@ -149,15 +161,11 @@ main() {
until join_once; do
attempts=$((attempts + 1))
if [ "$attempts" -ge 20 ]; then
# Terminating lets the group launch a replacement instead of carrying a worker that never joined.
log "join failed $attempts times; check the join token in $TOKEN_PARAMETER and $INSTANCE_URL"
rm -f /run/warmbly-join.sh
systemctl poweroff
return 1
fi
sleep 30
done
rm -f /run/warmbly-join.sh
log "joined as $NODE_NAME"
return 0
}
+3 -3
View File
@@ -2,7 +2,7 @@
# main.tf, which keep running until an operator retires them.
variable "fleet_enabled" {
description = "Create the autoscaling worker fleet and its network"
description = "Create the autoscaling worker fleet and its network; the backend also needs FLEET_METRICS_CLOUDWATCH set to name, or the memory metrics it scales on are never published"
type = bool
default = false
}
@@ -79,8 +79,8 @@ variable "fleet_worker_capacity" {
type = string
default = "auto"
validation {
condition = var.fleet_worker_capacity == "auto" || can(regex("^[1-9][0-9]*$", var.fleet_worker_capacity))
error_message = "Use auto or a positive whole number."
condition = var.fleet_worker_capacity == "auto" || can(regex("^[1-9][0-9]{0,7}$", var.fleet_worker_capacity))
error_message = "Use auto or a whole number from 1 to 99999999, the largest the worker accepts."
}
}
@@ -89,7 +89,17 @@ replace_route() {
aws ec2 create-route --region "$REGION" --route-table-id "$ROUTE_TABLE_ID" --destination-cidr-block 0.0.0.0/0 --network-interface-id "$1" >/dev/null
}
# A boot that fails leaves the zone without egress, so power off and let the group replace the instance.
on_exit() {
rc=$?
if [ "$rc" -ne 0 ]; then
log "boot failed with status $rc; powering off"
shutdown -h now || true
fi
}
main() {
trap on_exit EXIT
retry dnf -y --setopt=retries=20 install iptables-nft
write_kernel_settings
write_rules_unit
@@ -169,6 +169,16 @@ run "fleet_worker_capacity_rejects_garbage" {
expect_failures = [var.fleet_worker_capacity]
}
run "fleet_worker_capacity_rejects_what_the_worker_discards" {
command = plan
variables {
fleet_enabled = true
fleet_instance_url = "https://app.example.com"
fleet_worker_capacity = "100000000"
}
expect_failures = [var.fleet_worker_capacity]
}
run "fleet_scale_in_sits_under_scale_out" {
command = plan
variables {
@@ -78,8 +78,8 @@ The parameter name defaults to `/<name>/fleet/join-token` and can be changed
with `fleet_join_token_parameter`. A join token works for at most 30 days, and
issuing a new one revokes the previous one. Joined workers are unaffected, but a
worker launched with an expired token cannot join. Issue and store a new token
before the old one expires, at least monthly, or scale-out stops producing
working workers.
before the old one expires, for example every three weeks, or scale-out stops
producing working workers.
## Enable the fleet
@@ -100,6 +100,11 @@ working workers.
`nats_security_group_id` is the NATS root's broker security group. Leave it
unset only when workers reach the event bus some other way.
4. Set `FLEET_METRICS_CLOUDWATCH` on the backend to the root's `name` and give
its role `cloudwatch:PutMetricData` (see the [metric contract](#metric-contract)).
Without it nothing publishes `WorkerMemoryPercent`, so the fleet scales out
on CPU only and never scales in.
The `fleet` output names the group, its subnets and security group, the join
token parameter, the metric contract and `nat_ipv4_addresses`. Workers appear in
`warmblyctl fleet list` within a few minutes of launching.
@@ -118,7 +123,7 @@ token parameter, the metric contract and `nat_ipv4_addresses`. Workers appear in
| `fleet_min_size` | `2` | Smallest fleet |
| `fleet_max_size` | `10` | Largest fleet |
| `fleet_desired_capacity` | `null` | Starting size; scaling owns it afterwards. `null` starts at the minimum |
| `fleet_worker_capacity` | `auto` | `WARMBLY_WORKER_CAPACITY` on every fleet worker: `auto`, or a positive whole number of mailboxes |
| `fleet_worker_capacity` | `auto` | `WARMBLY_WORKER_CAPACITY` on every fleet worker: `auto`, or a whole number of mailboxes from 1 to 99999999 |
| `fleet_region_label` | empty | Optional `--region` label for placement |
| `fleet_image_repository` | `ghcr.io/warmbly/warmbly` | Repository `join.sh` pulls the worker image from |
| `fleet_target_memory` | `65` | Target for `WorkerMemoryPercent`, the fleet's memory in use |
+7 -1
View File
@@ -87,7 +87,10 @@ func Run(ctx context.Context, d *db.DB, fleet string) error {
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
publish := func() {
// A stalled read or request must not hold back the next minute's sample.
ctx, cancel := context.WithTimeout(ctx, interval)
defer cancel()
if u, err := Read(ctx, d); err != nil {
log.Warn().Err(err).Msg("fleet usage not published")
} else if !u.Known {
@@ -102,6 +105,9 @@ func Run(ctx context.Context, d *db.DB, fleet string) error {
}); err != nil {
log.Warn().Err(err).Msg("fleet usage not published")
}
}
for {
publish()
select {
case <-ctx.Done():
return nil