mirror of
https://github.com/warmbly/warmbly.git
synced 2026-11-17 08:18:51 +00:00
301 lines
8.9 KiB
Go
301 lines
8.9 KiB
Go
package fleetnode
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/warmbly/warmbly/internal/models"
|
|
)
|
|
|
|
// ErrRolloutConflict means another backend changed the release state first.
|
|
var ErrRolloutConflict = errors.New("fleet release state changed concurrently; retry")
|
|
|
|
// ErrNoRollout means there is no rollout to act on.
|
|
var ErrNoRollout = errors.New("no fleet rollout is in progress")
|
|
|
|
func (s *Service) clock() time.Time {
|
|
if s.now != nil {
|
|
return s.now()
|
|
}
|
|
return time.Now()
|
|
}
|
|
|
|
// StartRollout moves the fleet to target in stages: a canary first, then the
|
|
// policy's waves, each admitted only after the previous one has baked.
|
|
// Starting the rollout already running for target is a no-op.
|
|
func (s *Service) StartRollout(ctx context.Context, target string, digests map[string]string, policy models.FleetRolloutPolicy) (*models.FleetReleaseState, error) {
|
|
target = strings.TrimSpace(target)
|
|
if target == "" {
|
|
return nil, errors.New("tag required")
|
|
}
|
|
if s.variant != "" {
|
|
target = strings.TrimSuffix(target, s.variant)
|
|
}
|
|
if err := s.CheckTarget(target); err != nil {
|
|
return nil, err
|
|
}
|
|
prev, err := s.settings.GetRelease(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if prev != nil && prev.Rollout.Running() && prev.Rollout.Target == target {
|
|
return prev, nil
|
|
}
|
|
now := s.clock()
|
|
next := &models.FleetReleaseState{Channel: models.FleetChannelPinned, ResolvedAt: now, Source: "rollout"}
|
|
if prev != nil {
|
|
next.Tag = prev.Tag
|
|
next.ResolvedAt = prev.ResolvedAt
|
|
next.Source = prev.Source
|
|
}
|
|
if next.Tag == target {
|
|
next.Rollout = nil
|
|
return s.commit(ctx, prev, next)
|
|
}
|
|
clean := map[string]string{}
|
|
for k, v := range digests {
|
|
if d := validDigest(strings.TrimSpace(v)); d != "" {
|
|
clean[strings.TrimSpace(k)] = d
|
|
}
|
|
}
|
|
r := &models.FleetRollout{
|
|
Target: target,
|
|
From: next.Tag,
|
|
Digests: clean,
|
|
Policy: policy.Normalize(),
|
|
Stage: models.FleetRolloutCanary,
|
|
StartedAt: now,
|
|
StageStartedAt: now,
|
|
}
|
|
candidates, err := s.rolloutCandidates(ctx, target)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(candidates) == 0 {
|
|
return s.commit(ctx, prev, finish(next, r, now))
|
|
}
|
|
n := r.Policy.Canary
|
|
if n > len(candidates) {
|
|
n = len(candidates)
|
|
}
|
|
for _, c := range candidates[:n] {
|
|
r.Admitted = append(r.Admitted, c.ID)
|
|
}
|
|
next.Rollout = r
|
|
return s.commit(ctx, prev, next)
|
|
}
|
|
|
|
// AdvanceRollout evaluates the running rollout once: it halts on a failed or
|
|
// silent node, admits the next stage once the current one has baked, and
|
|
// finishes when every node is admitted. force skips the bake, never the checks.
|
|
func (s *Service) AdvanceRollout(ctx context.Context, force bool) (*models.FleetReleaseState, error) {
|
|
prev, err := s.settings.GetRelease(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if prev == nil || !prev.Rollout.Running() {
|
|
return prev, nil
|
|
}
|
|
nodes, err := s.nodes.List(ctx, "")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
now := s.clock()
|
|
r := *prev.Rollout
|
|
r.Admitted = append([]uuid.UUID(nil), prev.Rollout.Admitted...)
|
|
next := *prev
|
|
next.Rollout = &r
|
|
|
|
if reason, done := s.stageVerdict(&r, nodes, now); reason != "" {
|
|
r.Stage = models.FleetRolloutHalted
|
|
r.HaltReason = reason
|
|
log.Printf("fleet: rollout to %s halted: %s", r.Target, reason)
|
|
return s.commit(ctx, prev, &next)
|
|
} else if !done {
|
|
return prev, nil
|
|
} else if !force && now.Sub(r.StageStartedAt) < time.Duration(r.Policy.BakeSeconds)*time.Second {
|
|
return prev, nil
|
|
}
|
|
|
|
candidates, err := s.rolloutCandidates(ctx, r.Target)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
remaining := make([]models.FleetNode, 0, len(candidates))
|
|
for _, c := range candidates {
|
|
if !r.IsAdmitted(c.ID) {
|
|
remaining = append(remaining, c)
|
|
}
|
|
}
|
|
wave := 0
|
|
if r.Stage == models.FleetRolloutWave {
|
|
wave = r.Wave + 1
|
|
}
|
|
total := len(r.Admitted) + len(remaining)
|
|
for ; wave < len(r.Policy.Waves) && len(remaining) > 0; wave++ {
|
|
want := (total*r.Policy.Waves[wave] + 99) / 100
|
|
added := 0
|
|
for len(r.Admitted) < want && len(remaining) > 0 {
|
|
r.Admitted = append(r.Admitted, remaining[0].ID)
|
|
remaining = remaining[1:]
|
|
added++
|
|
}
|
|
if added > 0 {
|
|
r.Stage = models.FleetRolloutWave
|
|
r.Wave = wave
|
|
r.StageStartedAt = now
|
|
log.Printf("fleet: rollout to %s admitted wave %d (%d of %d nodes)", r.Target, wave+1, len(r.Admitted), total)
|
|
return s.commit(ctx, prev, &next)
|
|
}
|
|
}
|
|
log.Printf("fleet: rollout to %s finished on %d nodes", r.Target, len(r.Admitted))
|
|
return s.commit(ctx, prev, finish(&next, &r, now))
|
|
}
|
|
|
|
// HaltRollout stops admitting nodes and sends admitted ones back to From.
|
|
func (s *Service) HaltRollout(ctx context.Context, reason string) (*models.FleetReleaseState, error) {
|
|
prev, err := s.settings.GetRelease(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if prev == nil || !prev.Rollout.Running() {
|
|
return nil, ErrNoRollout
|
|
}
|
|
r := *prev.Rollout
|
|
r.Stage = models.FleetRolloutHalted
|
|
if reason = strings.TrimSpace(reason); reason == "" {
|
|
reason = "halted by an operator"
|
|
}
|
|
r.HaltReason = reason
|
|
next := *prev
|
|
next.Rollout = &r
|
|
return s.commit(ctx, prev, &next)
|
|
}
|
|
|
|
// AbortRollout discards the rollout, running or halted, leaving Tag at From.
|
|
func (s *Service) AbortRollout(ctx context.Context) (*models.FleetReleaseState, error) {
|
|
prev, err := s.settings.GetRelease(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if prev == nil || prev.Rollout == nil {
|
|
return nil, ErrNoRollout
|
|
}
|
|
next := *prev
|
|
next.Rollout = nil
|
|
return s.commit(ctx, prev, &next)
|
|
}
|
|
|
|
// RunRolloutLoop advances any running rollout every interval until ctx ends.
|
|
func (s *Service) RunRolloutLoop(ctx context.Context, interval time.Duration) {
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
if _, err := s.AdvanceRollout(ctx, false); err != nil && !errors.Is(err, ErrRolloutConflict) {
|
|
log.Printf("fleet: rollout advance: %v", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// stageVerdict returns a halt reason, or whether every admitted node has
|
|
// reached the target and stayed up for the bake.
|
|
func (s *Service) stageVerdict(r *models.FleetRollout, nodes []models.FleetNode, now time.Time) (string, bool) {
|
|
byID := make(map[uuid.UUID]*models.FleetNode, len(nodes))
|
|
for i := range nodes {
|
|
byID[nodes[i].ID] = &nodes[i]
|
|
}
|
|
target := s.withVariant(r.Target)
|
|
bake := time.Duration(r.Policy.BakeSeconds) * time.Second
|
|
deadline := time.Duration(r.Policy.DeadlineSeconds) * time.Second
|
|
elapsed := now.Sub(r.StageStartedAt)
|
|
done := true
|
|
for _, id := range r.Admitted {
|
|
n := byID[id]
|
|
switch {
|
|
case n == nil:
|
|
return fmt.Sprintf("node %s was removed during the rollout", id), false
|
|
case n.PinnedVersion != "":
|
|
continue
|
|
case n.UpdateFailed != "" && n.UpdateFailed == target:
|
|
return fmt.Sprintf("node %s (%s) rolled back from %s on the host", n.Name, id, target), false
|
|
case !n.LiveAt(now):
|
|
// An updating node sends an inactive farewell before its replacement boots.
|
|
if n.Version != target && elapsed <= deadline && n.LastSeenAt != nil && !n.LastSeenAt.Before(r.StageStartedAt.Add(-models.NodeLivenessWindow)) {
|
|
done = false
|
|
continue
|
|
}
|
|
return fmt.Sprintf("node %s (%s) stopped heartbeating", n.Name, id), false
|
|
case n.Version != target:
|
|
if elapsed > deadline {
|
|
return fmt.Sprintf("node %s (%s) did not reach %s within %s", n.Name, id, target, deadline), false
|
|
}
|
|
done = false
|
|
case n.Usage.UptimeSeconds == nil || time.Duration(*n.Usage.UptimeSeconds)*time.Second < bake:
|
|
if elapsed > deadline+2*bake {
|
|
return fmt.Sprintf("node %s (%s) kept restarting on %s", n.Name, id, target), false
|
|
}
|
|
done = false
|
|
}
|
|
}
|
|
return "", done
|
|
}
|
|
|
|
// rolloutCandidates are live, unpinned nodes in admission order: workers
|
|
// first, then a stable shuffle per target so no machine always goes first.
|
|
func (s *Service) rolloutCandidates(ctx context.Context, target string) ([]models.FleetNode, error) {
|
|
nodes, err := s.nodes.List(ctx, "")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
now := s.clock()
|
|
out := make([]models.FleetNode, 0, len(nodes))
|
|
for _, n := range nodes {
|
|
if n.LiveAt(now) && n.PinnedVersion == "" {
|
|
out = append(out, n)
|
|
}
|
|
}
|
|
rank := func(n models.FleetNode) string {
|
|
h := sha256.Sum256([]byte(target + "/" + n.ID.String()))
|
|
return fmt.Sprintf("%x", h[:8])
|
|
}
|
|
sort.SliceStable(out, func(i, j int) bool {
|
|
if (out[i].Role == models.NodeRoleWorker) != (out[j].Role == models.NodeRoleWorker) {
|
|
return out[i].Role == models.NodeRoleWorker
|
|
}
|
|
return rank(out[i]) < rank(out[j])
|
|
})
|
|
return out, nil
|
|
}
|
|
|
|
func finish(next *models.FleetReleaseState, r *models.FleetRollout, now time.Time) *models.FleetReleaseState {
|
|
next.Tag = r.Target
|
|
next.Channel = models.FleetChannelPinned
|
|
next.ResolvedAt = now
|
|
next.Source = "rollout"
|
|
next.Rollout = nil
|
|
return next
|
|
}
|
|
|
|
func (s *Service) commit(ctx context.Context, prev, next *models.FleetReleaseState) (*models.FleetReleaseState, error) {
|
|
ok, err := s.settings.CompareAndSetRelease(ctx, prev, next)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !ok {
|
|
return nil, ErrRolloutConflict
|
|
}
|
|
return next, nil
|
|
}
|