mirror of
https://github.com/warmbly/warmbly.git
synced 2026-09-06 16:01:28 +00:00
Matthew Meszaros
f96bdfd2c1
Use subject-specific humanization so short warmup subjects are not mangled by body-level opener stripping. Add rDNS verification logging after provisioning and remove the unused public campaign breaker interface method.
414 lines
14 KiB
Go
414 lines
14 KiB
Go
// Package provisioning is the state machine that drives a provisioning_jobs
|
|
// row from "pending" through the lifecycle of creating a server, attaching
|
|
// IPs, setting rDNS, installing the worker binary, and verifying that the
|
|
// expected workers report in.
|
|
//
|
|
// The state machine is idempotent: each Run call resumes from the row's
|
|
// current state, so a backend crash mid-provision is recoverable. Each step
|
|
// records its progress to the database before attempting the next step.
|
|
//
|
|
// On failure at any step, state transitions to rolling_back and the inverse
|
|
// operations run (delete primary IPs, delete server). Cleanly failed jobs
|
|
// leave no orphaned resources at the cloud provider.
|
|
package provisioning
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/rs/zerolog/log"
|
|
"github.com/warmbly/warmbly/internal/infrastructure/cloudprovider"
|
|
"github.com/warmbly/warmbly/internal/models"
|
|
"github.com/warmbly/warmbly/internal/repository"
|
|
)
|
|
|
|
// UUIDv5 URL namespace, kept in sync with cmd/worker and the installer.
|
|
var uuidNamespaceURL = uuid.MustParse("6ba7b811-9dad-11d1-80b4-00c04fd430c8")
|
|
|
|
// WorkerIDForIP returns the deterministic UUIDv5 that a worker process will
|
|
// adopt when it boots with the given WORKER_BIND_IP.
|
|
func WorkerIDForIP(ip string) uuid.UUID {
|
|
return uuid.NewSHA1(uuidNamespaceURL, []byte(ip))
|
|
}
|
|
|
|
// JobConfig is the in-row snapshot of what an admin (or the scale loop)
|
|
// asked for. Mirrors the relevant subset of provisioning_templates plus the
|
|
// rendered worker env that the installer needs.
|
|
type JobConfig struct {
|
|
Provider string `json:"provider"`
|
|
Location string `json:"location"`
|
|
Datacenter string `json:"datacenter,omitempty"`
|
|
ServerType string `json:"server_type"`
|
|
Image string `json:"image"`
|
|
ServerCount int `json:"server_count"`
|
|
IPv4PerServer int `json:"ipv4_per_server"`
|
|
IPv6PerServer int `json:"ipv6_per_server"`
|
|
Tier string `json:"tier"`
|
|
EgressKind string `json:"egress_kind"`
|
|
Labels map[string]string `json:"labels,omitempty"`
|
|
PlacementGroup string `json:"placement_group,omitempty"`
|
|
PrivateNetwork string `json:"private_network,omitempty"`
|
|
Firewall string `json:"firewall,omitempty"`
|
|
ImageTag string `json:"image_tag"`
|
|
WorkerEnv string `json:"worker_env"`
|
|
SSHKeyID string `json:"ssh_key_id"` // pre-uploaded to provider
|
|
SSHPrivKeyPEM []byte `json:"ssh_priv_key_pem"` // encrypted at rest in config blob
|
|
SSHPort int `json:"ssh_port,omitempty"`
|
|
SSHUser string `json:"ssh_user,omitempty"`
|
|
RDNSPattern string `json:"rdns_pattern,omitempty"` // e.g. "w-{{ip}}.workers.example.com"
|
|
}
|
|
|
|
// Service is the orchestrator.
|
|
type Service struct {
|
|
Jobs repository.ProvisioningJobRepository
|
|
Providers map[string]cloudprovider.Provider // keyed by provider name
|
|
// ProviderResolver, when set, resolves the provider client for a job
|
|
// (e.g. building a Hetzner client from the job's stored credential).
|
|
// Takes precedence over the static Providers map; the map is the
|
|
// test/fallback path.
|
|
ProviderResolver func(ctx context.Context, job *repository.ProvisioningJob) (cloudprovider.Provider, error)
|
|
Installer Installer
|
|
// VerifyTimeout is how long Run waits for expected workers to heartbeat
|
|
// before failing the job. Default 5 min.
|
|
VerifyTimeout time.Duration
|
|
// VerifyAllReady is called repeatedly during the verify step to check
|
|
// which of the expected workers have registered via heartbeat. Returns
|
|
// the subset that are alive.
|
|
VerifyAllReady func(ctx context.Context, expected []uuid.UUID) ([]uuid.UUID, error)
|
|
}
|
|
|
|
// Run drives the given job to terminal state (completed or failed). Safe to
|
|
// call concurrently with itself for different jobs; not safe for the same job.
|
|
func (s *Service) Run(ctx context.Context, jobID uuid.UUID) error {
|
|
job, err := s.Jobs.Get(ctx, jobID)
|
|
if err != nil {
|
|
return fmt.Errorf("provisioning: load job: %w", err)
|
|
}
|
|
if job == nil {
|
|
return fmt.Errorf("provisioning: job %s not found", jobID)
|
|
}
|
|
|
|
provider, err := s.resolveProvider(ctx, job)
|
|
if err != nil {
|
|
return s.fail(ctx, jobID, err)
|
|
}
|
|
|
|
var cfg JobConfig
|
|
if err := json.Unmarshal(job.Config, &cfg); err != nil {
|
|
return s.fail(ctx, jobID, fmt.Errorf("decode config: %w", err))
|
|
}
|
|
if cfg.SSHPort == 0 {
|
|
cfg.SSHPort = 22
|
|
}
|
|
if cfg.SSHUser == "" {
|
|
cfg.SSHUser = "root"
|
|
}
|
|
if cfg.ServerCount <= 0 {
|
|
cfg.ServerCount = 1
|
|
}
|
|
if cfg.IPv4PerServer <= 0 {
|
|
cfg.IPv4PerServer = 1
|
|
}
|
|
|
|
// One server per Run call. server_count > 1 means the admin / scale loop
|
|
// must enqueue multiple jobs, one per server. Keeps each job's failure
|
|
// domain small.
|
|
|
|
for state := job.State; ; {
|
|
switch state {
|
|
case models.ProvJobPending:
|
|
state = models.ProvJobCreatingServer
|
|
|
|
case models.ProvJobCreatingServer:
|
|
if err := s.Jobs.UpdateState(ctx, jobID, state); err != nil {
|
|
return err
|
|
}
|
|
server, err := s.createServer(ctx, provider, cfg, jobID)
|
|
if err != nil {
|
|
return s.rollback(ctx, jobID, provider, fmt.Errorf("create_server: %w", err))
|
|
}
|
|
if err := s.Jobs.RecordServer(ctx, jobID, server.ID); err != nil {
|
|
return s.rollback(ctx, jobID, provider, err)
|
|
}
|
|
if err := s.Jobs.AppendIPs(ctx, jobID, []string{}, []string{server.PublicIPv4}); err != nil {
|
|
return s.rollback(ctx, jobID, provider, err)
|
|
}
|
|
job.ProviderServerID = &server.ID
|
|
job.IPs = append(job.IPs, server.PublicIPv4)
|
|
state = models.ProvJobCreatingIPs
|
|
|
|
case models.ProvJobCreatingIPs:
|
|
if err := s.Jobs.UpdateState(ctx, jobID, state); err != nil {
|
|
return err
|
|
}
|
|
// IPv4PerServer=1 means the server's default IP is the only IP.
|
|
// IPv4PerServer>1 means create (n-1) extra Primary IPs.
|
|
extra := cfg.IPv4PerServer - 1
|
|
if extra > 0 {
|
|
ipIDs := make([]string, 0, extra)
|
|
ips := make([]string, 0, extra)
|
|
for i := 0; i < extra; i++ {
|
|
ip, err := provider.CreatePrimaryIP(ctx, cloudprovider.CreatePrimaryIPRequest{
|
|
Type: "ipv4",
|
|
Name: fmt.Sprintf("warmbly-%s-%d", jobID.String()[:8], i),
|
|
Datacenter: cfg.Datacenter,
|
|
Labels: cfg.Labels,
|
|
})
|
|
if err != nil {
|
|
return s.rollback(ctx, jobID, provider, fmt.Errorf("create_ip %d: %w", i, err))
|
|
}
|
|
ipIDs = append(ipIDs, ip.ID)
|
|
ips = append(ips, ip.IP)
|
|
}
|
|
if err := s.Jobs.AppendIPs(ctx, jobID, ipIDs, ips); err != nil {
|
|
return s.rollback(ctx, jobID, provider, err)
|
|
}
|
|
job.ProviderIPIDs = append(job.ProviderIPIDs, ipIDs...)
|
|
job.IPs = append(job.IPs, ips...)
|
|
}
|
|
state = models.ProvJobAssigningIPs
|
|
|
|
case models.ProvJobAssigningIPs:
|
|
if err := s.Jobs.UpdateState(ctx, jobID, state); err != nil {
|
|
return err
|
|
}
|
|
if job.ProviderServerID != nil {
|
|
for _, ipID := range job.ProviderIPIDs {
|
|
if err := provider.AssignPrimaryIP(ctx, ipID, *job.ProviderServerID); err != nil {
|
|
return s.rollback(ctx, jobID, provider, fmt.Errorf("assign_ip %s: %w", ipID, err))
|
|
}
|
|
}
|
|
}
|
|
state = models.ProvJobSettingRDNS
|
|
|
|
case models.ProvJobSettingRDNS:
|
|
if err := s.Jobs.UpdateState(ctx, jobID, state); err != nil {
|
|
return err
|
|
}
|
|
if cfg.RDNSPattern != "" {
|
|
for i, ipID := range job.ProviderIPIDs {
|
|
ip := ""
|
|
if i+1 < len(job.IPs) {
|
|
ip = job.IPs[i+1] // index 0 is the server's default IP
|
|
}
|
|
hostname := strings.ReplaceAll(cfg.RDNSPattern, "{{ip}}", strings.ReplaceAll(ip, ".", "-"))
|
|
if err := provider.SetReverseDNS(ctx, ipID, hostname); err != nil {
|
|
// rDNS failure is non-fatal — log and continue.
|
|
log.Warn().Err(err).Str("ip", ip).Str("hostname", hostname).Msg("provisioning: SetReverseDNS failed")
|
|
continue
|
|
}
|
|
// Verify the PTR actually took and forward-confirms (FCrDNS).
|
|
// A missing/mismatched PTR is a classic mailbox-provider
|
|
// rejection cause, so surface it; non-fatal (DNS may still be
|
|
// propagating). Best-effort — only verify when we know the IP.
|
|
if ip != "" {
|
|
ptr, ok, vErr := VerifyReverseDNS(ctx, ip)
|
|
switch {
|
|
case vErr != nil:
|
|
log.Warn().Err(vErr).Str("ip", ip).Msg("provisioning: rDNS verification lookup failed")
|
|
case !ok:
|
|
log.Warn().Str("ip", ip).Str("ptr", ptr).Str("expected", hostname).Msg("provisioning: rDNS not yet forward-confirmed (FCrDNS)")
|
|
default:
|
|
log.Info().Str("ip", ip).Str("ptr", ptr).Msg("provisioning: rDNS forward-confirmed")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
state = models.ProvJobInstalling
|
|
|
|
case models.ProvJobInstalling:
|
|
if err := s.Jobs.UpdateState(ctx, jobID, state); err != nil {
|
|
return err
|
|
}
|
|
if len(job.IPs) == 0 {
|
|
return s.rollback(ctx, jobID, provider, fmt.Errorf("no IPs to install on"))
|
|
}
|
|
expected := make([]uuid.UUID, 0, len(job.IPs))
|
|
for _, ip := range job.IPs {
|
|
expected = append(expected, WorkerIDForIP(ip))
|
|
}
|
|
res, err := s.Installer.Install(ctx, InstallRequest{
|
|
Host: job.IPs[0], // use server's default IP for SSH
|
|
SSHPort: cfg.SSHPort,
|
|
SSHUser: cfg.SSHUser,
|
|
SSHKeyPEM: cfg.SSHPrivKeyPEM,
|
|
IPs: job.IPs,
|
|
WorkerEnv: cfg.WorkerEnv,
|
|
ImageTag: cfg.ImageTag,
|
|
ExpectedIDs: expected,
|
|
})
|
|
if err != nil {
|
|
return s.rollback(ctx, jobID, provider, fmt.Errorf("install: %w", err))
|
|
}
|
|
if len(res.InstalledWorkerIDs) > 0 {
|
|
if err := s.Jobs.AppendWorkerIDs(ctx, jobID, res.InstalledWorkerIDs); err != nil {
|
|
return s.rollback(ctx, jobID, provider, err)
|
|
}
|
|
}
|
|
state = models.ProvJobVerifying
|
|
|
|
case models.ProvJobVerifying:
|
|
if err := s.Jobs.UpdateState(ctx, jobID, state); err != nil {
|
|
return err
|
|
}
|
|
timeout := s.VerifyTimeout
|
|
if timeout == 0 {
|
|
timeout = 5 * time.Minute
|
|
}
|
|
vctx, cancel := context.WithTimeout(ctx, timeout)
|
|
defer cancel()
|
|
expected := make([]uuid.UUID, 0, len(job.IPs))
|
|
for _, ip := range job.IPs {
|
|
expected = append(expected, WorkerIDForIP(ip))
|
|
}
|
|
ok, err := s.waitForHeartbeats(vctx, expected)
|
|
if err != nil {
|
|
return s.rollback(ctx, jobID, provider, fmt.Errorf("verify: %w", err))
|
|
}
|
|
if !ok {
|
|
return s.rollback(ctx, jobID, provider, fmt.Errorf("verify: timeout waiting for workers to register"))
|
|
}
|
|
state = models.ProvJobCompleted
|
|
|
|
case models.ProvJobCompleted:
|
|
return s.Jobs.MarkCompleted(ctx, jobID)
|
|
|
|
case models.ProvJobFailed, models.ProvJobRollingBack:
|
|
return fmt.Errorf("provisioning job %s already terminal: %s", jobID, state)
|
|
|
|
default:
|
|
return s.fail(ctx, jobID, fmt.Errorf("unknown state %q", state))
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *Service) resolveProvider(ctx context.Context, job *repository.ProvisioningJob) (cloudprovider.Provider, error) {
|
|
if s.ProviderResolver != nil {
|
|
return s.ProviderResolver(ctx, job)
|
|
}
|
|
if p, ok := s.Providers[job.Provider]; ok {
|
|
return p, nil
|
|
}
|
|
return nil, fmt.Errorf("no provider client for %q", job.Provider)
|
|
}
|
|
|
|
func (s *Service) createServer(ctx context.Context, p cloudprovider.Provider, cfg JobConfig, jobID uuid.UUID) (*cloudprovider.Server, error) {
|
|
sshKeys := []string{}
|
|
if cfg.SSHKeyID != "" {
|
|
sshKeys = []string{cfg.SSHKeyID}
|
|
}
|
|
return p.CreateServer(ctx, cloudprovider.CreateServerRequest{
|
|
Name: fmt.Sprintf("warmbly-%s-%s", cfg.Location, jobID.String()[:8]),
|
|
ServerType: cfg.ServerType,
|
|
Image: cfg.Image,
|
|
Location: cfg.Location,
|
|
Datacenter: cfg.Datacenter,
|
|
SSHKeyIDs: sshKeys,
|
|
UserData: renderCloudInit(cfg),
|
|
Labels: cfg.Labels,
|
|
PlacementGroup: cfg.PlacementGroup,
|
|
PrivateNetwork: cfg.PrivateNetwork,
|
|
Firewall: cfg.Firewall,
|
|
StartAfterCreate: true,
|
|
})
|
|
}
|
|
|
|
func renderCloudInit(cfg JobConfig) string {
|
|
// Minimal cloud-init: ensure Docker is present so install-worker.sh
|
|
// doesn't have to fetch it. The installer handles the rest.
|
|
return `#cloud-config
|
|
package_update: true
|
|
packages:
|
|
- curl
|
|
- ca-certificates
|
|
- docker.io
|
|
runcmd:
|
|
- [systemctl, enable, --now, docker]
|
|
`
|
|
}
|
|
|
|
func (s *Service) waitForHeartbeats(ctx context.Context, expected []uuid.UUID) (bool, error) {
|
|
if s.VerifyAllReady == nil {
|
|
// No verify hook wired — assume installer-side check is sufficient.
|
|
return true, nil
|
|
}
|
|
wantSet := map[uuid.UUID]struct{}{}
|
|
for _, id := range expected {
|
|
wantSet[id] = struct{}{}
|
|
}
|
|
tick := time.NewTicker(5 * time.Second)
|
|
defer tick.Stop()
|
|
for {
|
|
alive, err := s.VerifyAllReady(ctx, expected)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
for _, id := range alive {
|
|
delete(wantSet, id)
|
|
}
|
|
if len(wantSet) == 0 {
|
|
return true, nil
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return false, nil
|
|
case <-tick.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
// rollback transitions the job to rolling_back, undoes provider-side
|
|
// resources we created, then marks the job failed.
|
|
func (s *Service) rollback(ctx context.Context, jobID uuid.UUID, p cloudprovider.Provider, rootCause error) error {
|
|
_ = s.Jobs.UpdateState(ctx, jobID, models.ProvJobRollingBack)
|
|
job, _ := s.Jobs.Get(ctx, jobID)
|
|
if job != nil {
|
|
for _, ipID := range job.ProviderIPIDs {
|
|
_ = p.UnassignPrimaryIP(ctx, ipID)
|
|
_ = p.DeletePrimaryIP(ctx, ipID)
|
|
}
|
|
if job.ProviderServerID != nil {
|
|
_ = p.DeleteServer(ctx, *job.ProviderServerID)
|
|
}
|
|
}
|
|
return s.fail(ctx, jobID, rootCause)
|
|
}
|
|
|
|
func (s *Service) fail(ctx context.Context, jobID uuid.UUID, err error) error {
|
|
_ = s.Jobs.MarkFailed(ctx, jobID, err.Error())
|
|
return err
|
|
}
|
|
|
|
// SubmitConfigJSON marshals a JobConfig to the raw JSON the row stores.
|
|
func SubmitConfigJSON(cfg JobConfig) (json.RawMessage, error) {
|
|
return json.Marshal(cfg)
|
|
}
|
|
|
|
// randomToken is a small helper for default name suffixes.
|
|
func randomToken(n int) string {
|
|
b := make([]byte, n)
|
|
if _, err := rand.Read(b); err != nil {
|
|
return ""
|
|
}
|
|
return hex.EncodeToString(b)
|
|
}
|
|
|
|
// ParseIPs is a small helper for callers that need to round-trip the
|
|
// INET[]→string→net.IP conversion when reading job rows.
|
|
func ParseIPs(raw []string) []net.IP {
|
|
out := make([]net.IP, 0, len(raw))
|
|
for _, s := range raw {
|
|
if ip := net.ParseIP(s); ip != nil {
|
|
out = append(out, ip)
|
|
}
|
|
}
|
|
return out
|
|
}
|