mirror of
https://github.com/warmbly/warmbly.git
synced 2026-11-17 08:18:51 +00:00
176 lines
5.7 KiB
Go
176 lines
5.7 KiB
Go
// Package fleetscale reads the worker Auto Scaling Group's size and moves its
|
|
// cap. Optional: a deployment that does not set FLEET_AUTOSCALING_GROUP runs
|
|
// none of it.
|
|
package fleetscale
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"math"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/aws/aws-sdk-go-v2/aws"
|
|
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
|
"github.com/aws/aws-sdk-go-v2/service/autoscaling"
|
|
astypes "github.com/aws/aws-sdk-go-v2/service/autoscaling/types"
|
|
"github.com/rs/zerolog/log"
|
|
)
|
|
|
|
// CeilingTag is the group tag holding the highest cap an operator may set.
|
|
const CeilingTag = "warmbly:max-ceiling"
|
|
|
|
// WarnShare is the share of the cap at which the fleet is reported as close to it.
|
|
const WarnShare = 0.8
|
|
|
|
const (
|
|
checkEvery = 5 * time.Minute
|
|
warnEvery = time.Hour
|
|
)
|
|
|
|
// GroupName is FLEET_AUTOSCALING_GROUP, the worker Auto Scaling Group's name.
|
|
func GroupName() string {
|
|
return strings.TrimSpace(os.Getenv("FLEET_AUTOSCALING_GROUP"))
|
|
}
|
|
|
|
var (
|
|
// ErrOutOfRange refuses a cap below the group's minimum or current size, or above its ceiling.
|
|
ErrOutOfRange = errors.New("fleet cap out of range")
|
|
// ErrNoCeiling refuses any change to a group without a readable ceiling tag.
|
|
ErrNoCeiling = errors.New("fleet group has no " + CeilingTag + " tag")
|
|
)
|
|
|
|
// Status is the group's size as the admin panel shows it.
|
|
type Status struct {
|
|
Group string `json:"group"`
|
|
Min int `json:"min"`
|
|
Max int `json:"max"`
|
|
Desired int `json:"desired"`
|
|
InService int `json:"in_service"`
|
|
Pending int `json:"pending"`
|
|
// Ceiling is the highest cap allowed; 0 when the tag is missing, which refuses every change.
|
|
Ceiling int `json:"ceiling"`
|
|
}
|
|
|
|
// AtCap reports a fleet that cannot add another worker.
|
|
func (s Status) AtCap() bool { return s.Desired >= s.Max }
|
|
|
|
// NearCap reports a fleet within WarnShare of its cap.
|
|
func (s Status) NearCap() bool { return float64(s.Desired) >= WarnShare*float64(s.Max) }
|
|
|
|
// API is the part of the Auto Scaling client the service uses.
|
|
type API interface {
|
|
DescribeAutoScalingGroups(ctx context.Context, in *autoscaling.DescribeAutoScalingGroupsInput, opts ...func(*autoscaling.Options)) (*autoscaling.DescribeAutoScalingGroupsOutput, error)
|
|
UpdateAutoScalingGroup(ctx context.Context, in *autoscaling.UpdateAutoScalingGroupInput, opts ...func(*autoscaling.Options)) (*autoscaling.UpdateAutoScalingGroupOutput, error)
|
|
}
|
|
|
|
// Service reads and moves one group's cap.
|
|
type Service struct {
|
|
api API
|
|
group string
|
|
}
|
|
|
|
// New uses the AWS default credential chain.
|
|
func New(ctx context.Context, group string) (*Service, error) {
|
|
cfg, err := awsconfig.LoadDefaultConfig(ctx)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("fleet autoscaling: aws config: %w", err)
|
|
}
|
|
return NewWithAPI(autoscaling.NewFromConfig(cfg), group), nil
|
|
}
|
|
|
|
// NewWithAPI builds a Service over any client, for tests.
|
|
func NewWithAPI(api API, group string) *Service { return &Service{api: api, group: group} }
|
|
|
|
// Status reads the group's current size, cap and ceiling.
|
|
func (s *Service) Status(ctx context.Context) (Status, error) {
|
|
out, err := s.api.DescribeAutoScalingGroups(ctx, &autoscaling.DescribeAutoScalingGroupsInput{AutoScalingGroupNames: []string{s.group}})
|
|
if err != nil {
|
|
return Status{}, err
|
|
}
|
|
if len(out.AutoScalingGroups) == 0 {
|
|
return Status{}, fmt.Errorf("fleet autoscaling group %q not found", s.group)
|
|
}
|
|
g := out.AutoScalingGroups[0]
|
|
st := Status{
|
|
Group: s.group,
|
|
Min: int(aws.ToInt32(g.MinSize)),
|
|
Max: int(aws.ToInt32(g.MaxSize)),
|
|
Desired: int(aws.ToInt32(g.DesiredCapacity)),
|
|
}
|
|
for _, inst := range g.Instances {
|
|
switch {
|
|
case inst.LifecycleState == astypes.LifecycleStateInService:
|
|
st.InService++
|
|
case strings.HasPrefix(string(inst.LifecycleState), "Pending"):
|
|
st.Pending++
|
|
}
|
|
}
|
|
for _, tag := range g.Tags {
|
|
if aws.ToString(tag.Key) == CeilingTag {
|
|
// A ceiling outside int32 cannot be a group size, so it reads as missing.
|
|
if n, err := strconv.ParseInt(strings.TrimSpace(aws.ToString(tag.Value)), 10, 32); err == nil && n > 0 {
|
|
st.Ceiling = int(n)
|
|
}
|
|
}
|
|
}
|
|
return st, nil
|
|
}
|
|
|
|
// SetMax moves the cap, never below the group's minimum or its current size and
|
|
// never above the ceiling. A scale-out landing between the read and the update
|
|
// loses the workers above the new cap at once, picked as any scale-in, through the drain hook.
|
|
func (s *Service) SetMax(ctx context.Context, limit int) (Status, error) {
|
|
st, err := s.Status(ctx)
|
|
if err != nil {
|
|
return Status{}, err
|
|
}
|
|
if err := st.allows(limit); err != nil {
|
|
return st, err
|
|
}
|
|
if _, err := s.api.UpdateAutoScalingGroup(ctx, &autoscaling.UpdateAutoScalingGroupInput{
|
|
AutoScalingGroupName: aws.String(s.group),
|
|
MaxSize: aws.Int32(int32(limit)),
|
|
}); err != nil {
|
|
return st, err
|
|
}
|
|
st.Max = limit
|
|
return st, nil
|
|
}
|
|
|
|
func (s Status) allows(limit int) error {
|
|
if s.Ceiling == 0 {
|
|
return ErrNoCeiling
|
|
}
|
|
floor := max(1, s.Min, s.Desired)
|
|
if limit < floor || limit > s.Ceiling || limit > math.MaxInt32 {
|
|
return fmt.Errorf("%w: choose %d to %d", ErrOutOfRange, floor, s.Ceiling)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Watch logs a warning at most once an hour while the fleet sits at its cap,
|
|
// until ctx ends. Run it on one backend at a time.
|
|
func Watch(ctx context.Context, s *Service) {
|
|
var warned time.Time
|
|
for {
|
|
st, err := s.Status(ctx)
|
|
switch {
|
|
case err != nil:
|
|
log.Warn().Err(err).Str("group", s.group).Msg("fleet size not read")
|
|
case st.AtCap() && time.Since(warned) >= warnEvery:
|
|
log.Warn().Str("group", s.group).Int("desired", st.Desired).Int("max", st.Max).Int("ceiling", st.Ceiling).
|
|
Msg("worker fleet is at its size cap and cannot grow; raise the cap in the admin panel")
|
|
warned = time.Now()
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-time.After(checkEvery):
|
|
}
|
|
}
|
|
}
|