feat: store a renewed fleet join token in Parameter Store before making it the join token, so a failed write leaves the previous token joining, and encrypt it with FLEET_JOIN_TOKEN_KMS_KEY when set

This commit is contained in:
Matthew Meszaros
2026-11-13 07:06:07 +01:00
parent f6087afc51
commit 2bc7538d56
6 changed files with 94 additions and 44 deletions
+1 -1
View File
@@ -2311,7 +2311,7 @@ func main() {
// The worker Auto Scaling Group joins with the token in this parameter; unset on every other deployment.
if parameter := fleetjoin.Parameter(); parameter != "" && fleetNodeService != nil {
elector.Go("fleet_join_token", func(ctx context.Context) {
if err := fleetjoin.Run(ctx, fleetNodeService, parameter); err != nil {
if err := fleetjoin.Run(ctx, fleetNodeService, parameter, fleetjoin.KMSKey()); err != nil {
log.Printf("Warning: fleet join token renewal disabled: %v", err)
}
})
@@ -69,7 +69,10 @@ table's default route.
Set [`FLEET_JOIN_TOKEN_PARAMETER`](/development/configuration/#workers) on the
backend to the parameter name, `/<name>/fleet/join-token` unless
`fleet_join_token_parameter` changes it, and give the backend's role
`ssm:GetParameter` and `ssm:PutParameter` on it. Within a minute of starting,
`ssm:GetParameter` and `ssm:PutParameter` on it. When the parameter uses a
customer managed key (`fleet_join_token_kms_key_arn`), also set
[`FLEET_JOIN_TOKEN_KMS_KEY`](/development/configuration/#workers) to that key and
give the backend's role `kms:Encrypt` and `kms:Decrypt` on it. Within a minute of starting,
the backend issues a 30-day join token and stores it there as a SecureString.
It checks every hour and issues a new one when the stored token expires within
10 days or no longer joins, so launches never meet an expired token. Workers
@@ -586,7 +586,8 @@ Keep the copies private. GeoLite2 may not be redistributed, which is also why no
| `WARMBLY_ENV_REFRESH` | Read by the host updater, not the node. `true` lets it replace `node.env` with what the control plane renders now whenever the heartbeat names a different env revision, then restart the node. `node.local.env` is never touched. Written by the join script on a new join and by `--adopt --env-refresh`; a node without it keeps its env until it rejoins | unset | no (read by the updater) |
| `APP_INTERNAL_URL` | Read by the backend: the address a joining node is told to call back on when `ENCRYPTED_KEYS_BACKEND_URL` is unset on the backend. Instance health checks whichever of the two is set | unset | no (read per join) |
| `FLEET_METRICS_CLOUDWATCH` | Read by the backend: the fleet name it publishes `Warmbly/Fleet` metrics under (dimension `Fleet`) every minute from the workers' heartbeats: `WorkerMemoryPercent` and `WorkerMemoryMaxPercent`, which the [AWS worker fleet](/development/aws-workers/#metric-contract) scales on, and `WorkerUtilizationPercent` for reading only. Set it to the workers root's `name`. Uses the AWS default credential chain, so the backend's role needs `cloudwatch:PutMetricData`. Leave it unset on any other deployment | unset | yes |
| `FLEET_JOIN_TOKEN_PARAMETER` | Read by the backend: the Parameter Store SecureString the [AWS worker fleet](/development/aws-workers/#prepare-the-join-token) joins with. The backend checks it every hour and, when it is missing, no longer joins, or expires within 10 days, issues a 30-day join token and stores it there. Issuing revokes the previous token, so once this is set, read the token from the parameter rather than issuing one with `warmblyctl fleet join-token`. The parameter uses the `aws/ssm` key. Uses the AWS default credential chain, so the backend's role needs `ssm:GetParameter` and `ssm:PutParameter` on that parameter. Leave it unset on any other deployment | unset | yes |
| `FLEET_JOIN_TOKEN_PARAMETER` | Read by the backend: the Parameter Store SecureString the [AWS worker fleet](/development/aws-workers/#prepare-the-join-token) joins with. The backend checks it every hour and, when it is missing, no longer joins, or expires within 10 days, issues a 30-day join token and stores it there. Issuing revokes the previous token, so once this is set, read the token from the parameter rather than issuing one with `warmblyctl fleet join-token`. The new token is stored first and only then made the join token, so a failed write leaves the previous one working. Uses the AWS default credential chain, so the backend's role needs `ssm:GetParameter` and `ssm:PutParameter` on that parameter. Leave it unset on any other deployment | unset | yes |
| `FLEET_JOIN_TOKEN_KMS_KEY` | Read by the backend with `FLEET_JOIN_TOKEN_PARAMETER`: the KMS key ID or ARN the parameter is encrypted with, the workers root's `fleet_join_token_kms_key_arn`. The backend's role then also needs `kms:Encrypt` and `kms:Decrypt` on that key. Unset stores the parameter with `aws/ssm` | unset | yes |
<Callout type="warn" title="An unset key URL is silent">
An empty `ENCRYPTED_KEYS_BACKEND_URL` or `ENCRYPTED_KEYS_WORKER_TOKEN` lets the worker start, subscribe and never register. There is no log line. The `no_worker_heartbeat` check on [Instance health](/development/instance-health/#no_worker_heartbeat) is what surfaces it.
+23 -9
View File
@@ -31,6 +31,11 @@ func Parameter() string {
return strings.TrimSpace(os.Getenv("FLEET_JOIN_TOKEN_PARAMETER"))
}
// KMSKey is FLEET_JOIN_TOKEN_KMS_KEY, the key the parameter is encrypted with; empty means aws/ssm.
func KMSKey() string {
return strings.TrimSpace(os.Getenv("FLEET_JOIN_TOKEN_KMS_KEY"))
}
// Store holds the plaintext token; Get answers "" for a parameter that does not exist.
type Store interface {
Get(ctx context.Context) (string, error)
@@ -41,12 +46,13 @@ type Store interface {
type Issuer interface {
VerifyJoinToken(ctx context.Context, token string) error
JoinTokenExpiresAt(ctx context.Context) (*time.Time, error)
IssueJoinToken(ctx context.Context, ttl time.Duration) (string, time.Time, error)
ActivateJoinToken(ctx context.Context, token string, expiresAt time.Time) error
}
// Sync leaves a stored token that still joins for more than RenewBefore, and
// otherwise issues a new one and stores it. Issuing revokes the previous token,
// so a failed store is retried by the next Sync rather than undone.
// otherwise stores a new one and then activates it. A failed store leaves the
// previous token stored and joining; a failed activation is repaired by the
// next Sync, which finds a stored token that does not join.
func Sync(ctx context.Context, store Store, issuer Issuer, now time.Time) (bool, error) {
current, err := store.Get(ctx)
if err != nil {
@@ -61,24 +67,27 @@ func Sync(ctx context.Context, store Store, issuer Issuer, now time.Time) (bool,
return false, nil
}
}
token, _, err := issuer.IssueJoinToken(ctx, fleetnode.MaxJoinTokenTTL)
token, expiresAt, err := fleetnode.NewJoinToken(fleetnode.MaxJoinTokenTTL)
if err != nil {
return false, err
}
if err := store.Put(ctx, token); err != nil {
return true, fmt.Errorf("store renewed join token: %w", err)
return false, fmt.Errorf("store renewed join token: %w", err)
}
if err := issuer.ActivateJoinToken(ctx, token, expiresAt); err != nil {
return false, fmt.Errorf("activate stored join token: %w", err)
}
return true, nil
}
// Run keeps the parameter current until ctx ends. Run it on one backend at a
// time: two replicas issuing at once would each revoke the other's token.
func Run(ctx context.Context, issuer Issuer, parameter string) error {
func Run(ctx context.Context, issuer Issuer, parameter, kmsKey string) error {
cfg, err := awsconfig.LoadDefaultConfig(ctx)
if err != nil {
return fmt.Errorf("fleet join token: aws config: %w", err)
}
store := &parameterStore{client: ssm.NewFromConfig(cfg), name: parameter}
store := &parameterStore{client: ssm.NewFromConfig(cfg), name: parameter, kmsKey: kmsKey}
for {
wait := checkEvery
renewed, err := Sync(ctx, store, issuer, time.Now())
@@ -100,6 +109,7 @@ func Run(ctx context.Context, issuer Issuer, parameter string) error {
type parameterStore struct {
client *ssm.Client
name string
kmsKey string
}
func (p *parameterStore) Get(ctx context.Context) (string, error) {
@@ -115,11 +125,15 @@ func (p *parameterStore) Get(ctx context.Context) (string, error) {
}
func (p *parameterStore) Put(ctx context.Context, token string) error {
_, err := p.client.PutParameter(ctx, &ssm.PutParameterInput{
in := &ssm.PutParameterInput{
Name: aws.String(p.name),
Value: aws.String(token),
Type: ssmtypes.ParameterTypeSecureString,
Overwrite: aws.Bool(true),
})
}
if p.kmsKey != "" {
in.KeyId = aws.String(p.kmsKey)
}
_, err := p.client.PutParameter(ctx, in)
return err
}
+46 -26
View File
@@ -25,58 +25,78 @@ func (s *fakeStore) Put(_ context.Context, token string) error {
}
type fakeIssuer struct {
token string
expiresAt *time.Time
issued int
token string
expiresAt *time.Time
activated int
activateErr error
}
func (i *fakeIssuer) VerifyJoinToken(_ context.Context, token string) error {
if token != i.token || i.token == "" {
return fleetnode.ErrBadToken
}
if i.expiresAt == nil || !time.Now().Before(*i.expiresAt) {
return fleetnode.ErrJoinTokenExpired
}
return nil
}
func (i *fakeIssuer) JoinTokenExpiresAt(context.Context) (*time.Time, error) { return i.expiresAt, nil }
func (i *fakeIssuer) IssueJoinToken(_ context.Context, ttl time.Duration) (string, time.Time, error) {
i.issued++
i.token = "token-" + string(rune('a'+i.issued))
at := time.Now().Add(ttl)
i.expiresAt = &at
return i.token, at, nil
func (i *fakeIssuer) ActivateJoinToken(_ context.Context, token string, expiresAt time.Time) error {
if i.activateErr != nil {
return i.activateErr
}
i.activated++
i.token, i.expiresAt = token, &expiresAt
return nil
}
func TestSyncKeepsTheStoredTokenJoinable(t *testing.T) {
now := time.Now()
in := func(d time.Duration) *time.Time { at := now.Add(d); return &at }
storeFail := errors.New("store unavailable")
fail := errors.New("unavailable")
cases := []struct {
name string
stored string
issuer fakeIssuer
getErr error
putErr error
wantRenewed bool
wantErr bool
wantIssued int
name string
stored string
issuer fakeIssuer
getErr error
putErr error
wantRenewed bool
wantErr bool
wantActivated int
}{
{name: "nothing stored", wantRenewed: true, wantIssued: 1},
{name: "nothing stored", wantRenewed: true, wantActivated: 1},
{name: "valid for weeks", stored: "live", issuer: fakeIssuer{token: "live", expiresAt: in(20 * 24 * time.Hour)}},
{name: "expiring soon", stored: "live", issuer: fakeIssuer{token: "live", expiresAt: in(5 * 24 * time.Hour)}, wantRenewed: true, wantIssued: 1},
{name: "stored token was revoked", stored: "old", issuer: fakeIssuer{token: "live", expiresAt: in(20 * 24 * time.Hour)}, wantRenewed: true, wantIssued: 1},
{name: "unreadable parameter revokes nothing", stored: "live", issuer: fakeIssuer{token: "live", expiresAt: in(time.Hour)}, getErr: storeFail, wantErr: true},
{name: "failed store is reported", putErr: storeFail, wantRenewed: true, wantErr: true, wantIssued: 1},
{name: "expiring soon", stored: "live", issuer: fakeIssuer{token: "live", expiresAt: in(5 * 24 * time.Hour)}, wantRenewed: true, wantActivated: 1},
{name: "stored token was revoked", stored: "old", issuer: fakeIssuer{token: "live", expiresAt: in(20 * 24 * time.Hour)}, wantRenewed: true, wantActivated: 1},
{name: "unreadable parameter changes nothing", stored: "live", issuer: fakeIssuer{token: "live", expiresAt: in(time.Hour)}, getErr: fail, wantErr: true},
{name: "failed store keeps the stored token joining", stored: "live", issuer: fakeIssuer{token: "live", expiresAt: in(5 * 24 * time.Hour)}, putErr: fail, wantErr: true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
store := &fakeStore{value: tc.stored, getErr: tc.getErr, putErr: tc.putErr}
issuer := tc.issuer
renewed, err := Sync(context.Background(), store, &issuer, now)
if renewed != tc.wantRenewed || (err != nil) != tc.wantErr || issuer.issued != tc.wantIssued {
t.Fatalf("renewed=%v err=%v issued=%d, want renewed=%v err=%v issued=%d", renewed, err, issuer.issued, tc.wantRenewed, tc.wantErr, tc.wantIssued)
if renewed != tc.wantRenewed || (err != nil) != tc.wantErr || issuer.activated != tc.wantActivated {
t.Fatalf("renewed=%v err=%v activated=%d, want renewed=%v err=%v activated=%d", renewed, err, issuer.activated, tc.wantRenewed, tc.wantErr, tc.wantActivated)
}
if err == nil && issuer.VerifyJoinToken(context.Background(), store.value) != nil {
if tc.getErr == nil && issuer.VerifyJoinToken(context.Background(), store.value) != nil {
t.Fatalf("stored token %q does not join", store.value)
}
})
}
}
func TestSyncRepairsAFailedActivation(t *testing.T) {
store := &fakeStore{}
issuer := &fakeIssuer{activateErr: errors.New("database unavailable")}
if _, err := Sync(context.Background(), store, issuer, time.Now()); err == nil {
t.Fatal("a failed activation must be reported")
}
issuer.activateErr = nil
if renewed, err := Sync(context.Background(), store, issuer, time.Now()); err != nil || !renewed {
t.Fatalf("renewed=%v err=%v, want the next sync to store and activate a token", renewed, err)
}
if err := issuer.VerifyJoinToken(context.Background(), store.value); err != nil {
t.Fatalf("stored token does not join after repair: %v", err)
}
}
+18 -6
View File
@@ -119,6 +119,18 @@ const (
// any previous token, which is also how you revoke one. Existing nodes are
// unaffected: they are already enrolled, and the token only gates joining.
func (s *Service) IssueJoinToken(ctx context.Context, ttl time.Duration) (string, time.Time, error) {
token, expiresAt, err := NewJoinToken(ttl)
if err != nil {
return "", time.Time{}, err
}
if err := s.ActivateJoinToken(ctx, token, expiresAt); err != nil {
return "", time.Time{}, err
}
return token, expiresAt, nil
}
// NewJoinToken mints a token that joins nothing until ActivateJoinToken stores it.
func NewJoinToken(ttl time.Duration) (string, time.Time, error) {
if ttl <= 0 {
ttl = DefaultJoinTokenTTL
}
@@ -129,12 +141,12 @@ func (s *Service) IssueJoinToken(ctx context.Context, ttl time.Duration) (string
if _, err := rand.Read(raw); err != nil {
return "", time.Time{}, err
}
token := base64.RawURLEncoding.EncodeToString(raw)
expiresAt := time.Now().Add(ttl).UTC().Truncate(time.Second)
if err := s.settings.SetJoinToken(ctx, crypt.SHA256(token), expiresAt); err != nil {
return "", time.Time{}, err
}
return token, expiresAt, nil
return base64.RawURLEncoding.EncodeToString(raw), time.Now().Add(ttl).UTC().Truncate(time.Second), nil
}
// ActivateJoinToken makes token the instance's join token until expiresAt, revoking the previous one.
func (s *Service) ActivateJoinToken(ctx context.Context, token string, expiresAt time.Time) error {
return s.settings.SetJoinToken(ctx, crypt.SHA256(token), expiresAt)
}
// VerifyJoinToken checks a presented token in constant time, then its expiry.