// Package nodeagent is the node half of the pull-based fleet. // // Every Warmbly process that runs on a machine you own embeds it. The agent // heartbeats to the control plane, reports what it is running and what it is // using, and reads back the version it should be running. It never applies an // update itself: replacing a running container from inside that container is // how you get a process that cannot finish the job. Instead it writes the // target where the host-side updater (a systemd timer installed by `warmbly // join`) can see it, and that restarts the service. package nodeagent import ( "bytes" "context" "encoding/json" "log" "net/http" "os" "path/filepath" "runtime" "strings" "sync/atomic" "time" "github.com/google/uuid" "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/pkg/nodeauth" ) // DefaultInterval is used until the control plane says otherwise. The first // reply carries the server's liveness window and the agent re-paces itself to // a third of it, so the two can never drift apart. const DefaultInterval = 90 * time.Second type Config struct { LogToken string WarmupSendProtocol int NodeID uuid.UUID Role models.NodeRole Name string Region string Address string // CapacityTarget is meaningful for workers only. Zero leaves the control // plane's stored/default target unchanged. CapacityTarget float64 // Version is what this build reports. Empty means "unknown", which the // control plane must not read as "needs updating". Version string BaseURL string Token string // Credential is the node credential join issued to NodeID; empty sends the fleet token alone. Credential string // TargetVersionPath is where the resolved target is written for the // host-side updater to read. Empty disables that, which is what you want // in dev where nothing supervises the process. TargetVersionPath string // Condition is a standing problem reported on every beat while it lasts, // unlike ReportError, which is reported once. Nil or empty reports none. Condition func() string HTTPClient *http.Client } type Agent struct { logs *logCapture cfg Config http *http.Client started time.Time usage usageSampler address string addressChecked time.Time // lastErr is reported on the next beat and then cleared, so the dashboard // shows what went wrong without it sticking forever. lastErr atomic.Pointer[string] reloadRequests atomic.Uint64 } func New(cfg Config) *Agent { if cfg.HTTPClient == nil { cfg.HTTPClient = &http.Client{Timeout: 10 * time.Second} } a := &Agent{cfg: cfg, http: cfg.HTTPClient, started: time.Now()} a.prepareLogs() return a } // ReportError attaches a message to the next heartbeat. Safe from any // goroutine; the newest message wins. func (a *Agent) ReportError(msg string) { a.lastErr.Store(&msg) } // RequestReload coalesces missing-mailbox recovery into the next acknowledged heartbeat. func (a *Agent) RequestReload() { a.reloadRequests.Add(1) } // Run beats until ctx is cancelled, then sends one farewell beat so the node // goes inactive immediately rather than staying selectable until it ages out. // // A failed beat is logged and retried on the next tick. It is never fatal: a // node that cannot reach the control plane should keep doing the work it // already has, not stop. func (a *Agent) Run(ctx context.Context) { if a.cfg.BaseURL == "" || (a.cfg.Token == "" && a.cfg.Credential == "") { log.Println("nodeagent: no backend URL or token; heartbeats disabled") return } go a.runLogs(ctx) interval := DefaultInterval bootPending := true acknowledgedReload := a.reloadRequests.Load() if reply := a.beat(ctx, bootPending, false); reply != nil { bootPending = false interval = paceFrom(reply.LivenessSeconds) } ticker := time.NewTicker(interval) defer ticker.Stop() for { select { case <-ctx.Done(): // Fresh context: ctx is already cancelled, and without the // farewell the row stays selectable until the beat ages out, so // the control plane keeps handing work to a process that has exited. byeCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) a.beat(byeCtx, false, true) cancel() return case <-ticker.C: reload := a.reloadRequests.Load() if reply := a.beat(ctx, bootPending || reload != acknowledgedReload, false); reply != nil { bootPending = false acknowledgedReload = reload if next := paceFrom(reply.LivenessSeconds); next != interval { interval = next ticker.Reset(interval) } } } } } // paceFrom beats three times per liveness window, so two lost beats in a row // still do not look like a dead machine. func paceFrom(livenessSeconds int) time.Duration { if livenessSeconds <= 0 { return DefaultInterval } d := time.Duration(livenessSeconds) * time.Second / 3 if d < 15*time.Second { return 15 * time.Second } return d } func (a *Agent) beat(ctx context.Context, booted, stopping bool) *models.NodeHeartbeatReply { beat := models.NodeHeartbeat{ WarmupSendProtocol: a.cfg.WarmupSendProtocol, NodeID: a.cfg.NodeID, Role: a.cfg.Role, Name: a.cfg.Name, Region: a.cfg.Region, Address: a.address, CapacityTarget: a.cfg.CapacityTarget, Version: a.cfg.Version, Usage: a.sampleUsage(), Booted: booted, Stopping: stopping, } if !stopping { a.readUpdaterState(&beat) } if !stopping { beat.Address = a.publicAddress(ctx) } if p := a.lastErr.Swap(nil); p != nil { beat.LastError = *p } else if a.cfg.Condition != nil { beat.LastError = a.cfg.Condition() } body, err := json.Marshal(beat) if err != nil { return nil } url := strings.TrimRight(a.cfg.BaseURL, "/") + "/api/v1/internal/fleet/heartbeat" req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) if err != nil { log.Println("nodeagent: build heartbeat:", err) return nil } a.authorize(req) req.Header.Set("Content-Type", "application/json") resp, err := a.http.Do(req) if err != nil { log.Println("nodeagent: heartbeat failed:", err) return nil } defer resp.Body.Close() if resp.StatusCode >= 300 { log.Println("nodeagent: heartbeat returned status", resp.StatusCode) return nil } var reply *models.NodeHeartbeatReply if err := json.NewDecoder(resp.Body).Decode(&reply); err != nil || reply == nil { return nil } a.applyTarget(reply.DesiredVersion) a.applyDigest(reply.DesiredVersion, reply.DesiredDigest) a.applyEnvRevision(reply.EnvRevision) return reply } // authorize presents the fleet token and, when join issued one, this node's own credential. func (a *Agent) authorize(req *http.Request) { if a.cfg.Token != "" { req.Header.Set("Authorization", "Bearer "+a.cfg.Token) } if a.cfg.Credential != "" { req.Header.Set(nodeauth.IDHeader, a.cfg.NodeID.String()) req.Header.Set(nodeauth.CredentialHeader, a.cfg.Credential) } } // The host updater shares the target directory: it writes update-failed and // env-revision there, and reads target-digest and target-env-revision. func (a *Agent) stateFile(name string) string { if a.cfg.TargetVersionPath == "" { return "" } return filepath.Join(filepath.Dir(a.cfg.TargetVersionPath), name) } func (a *Agent) readUpdaterState(beat *models.NodeHeartbeat) { if a.cfg.TargetVersionPath == "" { return } beat.UpdateFailed = readToken(a.stateFile("update-failed")) beat.EnvRefresh = os.Getenv("WARMBLY_ENV_REFRESH") == "true" if beat.EnvRefresh { beat.EnvRevision = readToken(a.stateFile("env-revision")) } } // readToken returns the first line of a small state file, or "" when it is // absent or holds anything but a plain token. func readToken(path string) string { raw, err := os.ReadFile(path) if err != nil || len(raw) > 256 { return "" } v := strings.TrimSpace(strings.SplitN(string(raw), "\n", 2)[0]) for _, c := range v { if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || strings.ContainsRune(".-_+:", c)) { return "" } } return v } // applyDigest records the digest a staged rollout pins the target to. It is // written as " " so the updater uses it only for that tag. func (a *Agent) applyDigest(version, digest string) { path := a.stateFile("target-digest") if path == "" { return } if version == "" || digest == "" { _ = os.Remove(path) return } writeAtomic(path, version+" "+digest+"\n") } func (a *Agent) applyEnvRevision(rev string) { path := a.stateFile("target-env-revision") if path == "" || rev == "" || os.Getenv("WARMBLY_ENV_REFRESH") != "true" { return } if readToken(path) == rev { return } writeAtomic(path, rev+"\n") } func writeAtomic(path, body string) { tmp := path + ".tmp" if err := os.WriteFile(tmp, []byte(body), 0o644); err != nil { log.Println("nodeagent: cannot write updater state:", err) return } if err := os.Rename(tmp, path); err != nil { log.Println("nodeagent: cannot publish updater state:", err) } } // applyTarget records the version the control plane wants. Writing a file the // host-side updater polls keeps the decision (control plane) separate from the // mechanism (systemd timer), which is what lets a node replace itself without // asking a process to kill and restart its own container. // // An empty target is ignored rather than written: "no opinion" must never // become "downgrade to nothing". func (a *Agent) applyTarget(version string) { if version == "" || a.cfg.TargetVersionPath == "" || version == a.cfg.Version { return } dir := filepath.Dir(a.cfg.TargetVersionPath) if err := os.MkdirAll(dir, 0o755); err != nil { log.Println("nodeagent: cannot create target dir:", err) return } tmp := a.cfg.TargetVersionPath + ".tmp" if err := os.WriteFile(tmp, []byte(version+"\n"), 0o644); err != nil { log.Println("nodeagent: cannot write target version:", err) return } // Rename so the updater never reads a half-written file. if err := os.Rename(tmp, a.cfg.TargetVersionPath); err != nil { log.Println("nodeagent: cannot publish target version:", err) return } log.Printf("nodeagent: control plane wants %s (running %s); the host updater will apply it", version, a.cfg.Version) } func (a *Agent) sampleUsage() models.NodeUsage { var m runtime.MemStats runtime.ReadMemStats(&m) mem := int(m.Sys / 1024 / 1024) goroutines := runtime.NumGoroutine() uptime := int64(time.Since(a.started).Seconds()) usage := models.NodeUsage{ MemoryMB: &mem, Goroutines: &goroutines, UptimeSeconds: &uptime, } a.usage.sample(&usage) return usage }