From 2bc7538d563ccb1f8a8071b9e39245a22e342ffa Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Fri, 13 Nov 2026 07:06:07 +0100 Subject: [PATCH] feat: store a renewed fleet join token in Parameter Store before making it the join token, so a failed write leaves the previous token joining, and encrypt it with FLEET_JOIN_TOKEN_KMS_KEY when set --- cmd/backend/main.go | 2 +- docs/content/docs/development/aws-workers.mdx | 5 +- .../docs/development/configuration.mdx | 3 +- internal/app/fleetjoin/fleetjoin.go | 32 ++++++--- internal/app/fleetjoin/fleetjoin_test.go | 72 ++++++++++++------- internal/app/fleetnode/service.go | 24 +++++-- 6 files changed, 94 insertions(+), 44 deletions(-) diff --git a/cmd/backend/main.go b/cmd/backend/main.go index ee8a7e7b0..ffaf856d6 100644 --- a/cmd/backend/main.go +++ b/cmd/backend/main.go @@ -2311,7 +2311,7 @@ func main() { // The worker Auto Scaling Group joins with the token in this parameter; unset on every other deployment. if parameter := fleetjoin.Parameter(); parameter != "" && fleetNodeService != nil { elector.Go("fleet_join_token", func(ctx context.Context) { - if err := fleetjoin.Run(ctx, fleetNodeService, parameter); err != nil { + if err := fleetjoin.Run(ctx, fleetNodeService, parameter, fleetjoin.KMSKey()); err != nil { log.Printf("Warning: fleet join token renewal disabled: %v", err) } }) diff --git a/docs/content/docs/development/aws-workers.mdx b/docs/content/docs/development/aws-workers.mdx index 0f2606d30..a9f8a750d 100644 --- a/docs/content/docs/development/aws-workers.mdx +++ b/docs/content/docs/development/aws-workers.mdx @@ -69,7 +69,10 @@ table's default route. Set [`FLEET_JOIN_TOKEN_PARAMETER`](/development/configuration/#workers) on the backend to the parameter name, `//fleet/join-token` unless `fleet_join_token_parameter` changes it, and give the backend's role -`ssm:GetParameter` and `ssm:PutParameter` on it. Within a minute of starting, +`ssm:GetParameter` and `ssm:PutParameter` on it. When the parameter uses a +customer managed key (`fleet_join_token_kms_key_arn`), also set +[`FLEET_JOIN_TOKEN_KMS_KEY`](/development/configuration/#workers) to that key and +give the backend's role `kms:Encrypt` and `kms:Decrypt` on it. Within a minute of starting, the backend issues a 30-day join token and stores it there as a SecureString. It checks every hour and issues a new one when the stored token expires within 10 days or no longer joins, so launches never meet an expired token. Workers diff --git a/docs/content/docs/development/configuration.mdx b/docs/content/docs/development/configuration.mdx index 45755c678..e2a3fe233 100644 --- a/docs/content/docs/development/configuration.mdx +++ b/docs/content/docs/development/configuration.mdx @@ -586,7 +586,8 @@ Keep the copies private. GeoLite2 may not be redistributed, which is also why no | `WARMBLY_ENV_REFRESH` | Read by the host updater, not the node. `true` lets it replace `node.env` with what the control plane renders now whenever the heartbeat names a different env revision, then restart the node. `node.local.env` is never touched. Written by the join script on a new join and by `--adopt --env-refresh`; a node without it keeps its env until it rejoins | unset | no (read by the updater) | | `APP_INTERNAL_URL` | Read by the backend: the address a joining node is told to call back on when `ENCRYPTED_KEYS_BACKEND_URL` is unset on the backend. Instance health checks whichever of the two is set | unset | no (read per join) | | `FLEET_METRICS_CLOUDWATCH` | Read by the backend: the fleet name it publishes `Warmbly/Fleet` metrics under (dimension `Fleet`) every minute from the workers' heartbeats: `WorkerMemoryPercent` and `WorkerMemoryMaxPercent`, which the [AWS worker fleet](/development/aws-workers/#metric-contract) scales on, and `WorkerUtilizationPercent` for reading only. Set it to the workers root's `name`. Uses the AWS default credential chain, so the backend's role needs `cloudwatch:PutMetricData`. Leave it unset on any other deployment | unset | yes | -| `FLEET_JOIN_TOKEN_PARAMETER` | Read by the backend: the Parameter Store SecureString the [AWS worker fleet](/development/aws-workers/#prepare-the-join-token) joins with. The backend checks it every hour and, when it is missing, no longer joins, or expires within 10 days, issues a 30-day join token and stores it there. Issuing revokes the previous token, so once this is set, read the token from the parameter rather than issuing one with `warmblyctl fleet join-token`. The parameter uses the `aws/ssm` key. Uses the AWS default credential chain, so the backend's role needs `ssm:GetParameter` and `ssm:PutParameter` on that parameter. Leave it unset on any other deployment | unset | yes | +| `FLEET_JOIN_TOKEN_PARAMETER` | Read by the backend: the Parameter Store SecureString the [AWS worker fleet](/development/aws-workers/#prepare-the-join-token) joins with. The backend checks it every hour and, when it is missing, no longer joins, or expires within 10 days, issues a 30-day join token and stores it there. Issuing revokes the previous token, so once this is set, read the token from the parameter rather than issuing one with `warmblyctl fleet join-token`. The new token is stored first and only then made the join token, so a failed write leaves the previous one working. Uses the AWS default credential chain, so the backend's role needs `ssm:GetParameter` and `ssm:PutParameter` on that parameter. Leave it unset on any other deployment | unset | yes | +| `FLEET_JOIN_TOKEN_KMS_KEY` | Read by the backend with `FLEET_JOIN_TOKEN_PARAMETER`: the KMS key ID or ARN the parameter is encrypted with, the workers root's `fleet_join_token_kms_key_arn`. The backend's role then also needs `kms:Encrypt` and `kms:Decrypt` on that key. Unset stores the parameter with `aws/ssm` | unset | yes | An empty `ENCRYPTED_KEYS_BACKEND_URL` or `ENCRYPTED_KEYS_WORKER_TOKEN` lets the worker start, subscribe and never register. There is no log line. The `no_worker_heartbeat` check on [Instance health](/development/instance-health/#no_worker_heartbeat) is what surfaces it. diff --git a/internal/app/fleetjoin/fleetjoin.go b/internal/app/fleetjoin/fleetjoin.go index 0a9cc3256..452d99cbd 100644 --- a/internal/app/fleetjoin/fleetjoin.go +++ b/internal/app/fleetjoin/fleetjoin.go @@ -31,6 +31,11 @@ func Parameter() string { return strings.TrimSpace(os.Getenv("FLEET_JOIN_TOKEN_PARAMETER")) } +// KMSKey is FLEET_JOIN_TOKEN_KMS_KEY, the key the parameter is encrypted with; empty means aws/ssm. +func KMSKey() string { + return strings.TrimSpace(os.Getenv("FLEET_JOIN_TOKEN_KMS_KEY")) +} + // Store holds the plaintext token; Get answers "" for a parameter that does not exist. type Store interface { Get(ctx context.Context) (string, error) @@ -41,12 +46,13 @@ type Store interface { type Issuer interface { VerifyJoinToken(ctx context.Context, token string) error JoinTokenExpiresAt(ctx context.Context) (*time.Time, error) - IssueJoinToken(ctx context.Context, ttl time.Duration) (string, time.Time, error) + ActivateJoinToken(ctx context.Context, token string, expiresAt time.Time) error } // Sync leaves a stored token that still joins for more than RenewBefore, and -// otherwise issues a new one and stores it. Issuing revokes the previous token, -// so a failed store is retried by the next Sync rather than undone. +// otherwise stores a new one and then activates it. A failed store leaves the +// previous token stored and joining; a failed activation is repaired by the +// next Sync, which finds a stored token that does not join. func Sync(ctx context.Context, store Store, issuer Issuer, now time.Time) (bool, error) { current, err := store.Get(ctx) if err != nil { @@ -61,24 +67,27 @@ func Sync(ctx context.Context, store Store, issuer Issuer, now time.Time) (bool, return false, nil } } - token, _, err := issuer.IssueJoinToken(ctx, fleetnode.MaxJoinTokenTTL) + token, expiresAt, err := fleetnode.NewJoinToken(fleetnode.MaxJoinTokenTTL) if err != nil { return false, err } if err := store.Put(ctx, token); err != nil { - return true, fmt.Errorf("store renewed join token: %w", err) + return false, fmt.Errorf("store renewed join token: %w", err) + } + if err := issuer.ActivateJoinToken(ctx, token, expiresAt); err != nil { + return false, fmt.Errorf("activate stored join token: %w", err) } return true, nil } // Run keeps the parameter current until ctx ends. Run it on one backend at a // time: two replicas issuing at once would each revoke the other's token. -func Run(ctx context.Context, issuer Issuer, parameter string) error { +func Run(ctx context.Context, issuer Issuer, parameter, kmsKey string) error { cfg, err := awsconfig.LoadDefaultConfig(ctx) if err != nil { return fmt.Errorf("fleet join token: aws config: %w", err) } - store := ¶meterStore{client: ssm.NewFromConfig(cfg), name: parameter} + store := ¶meterStore{client: ssm.NewFromConfig(cfg), name: parameter, kmsKey: kmsKey} for { wait := checkEvery renewed, err := Sync(ctx, store, issuer, time.Now()) @@ -100,6 +109,7 @@ func Run(ctx context.Context, issuer Issuer, parameter string) error { type parameterStore struct { client *ssm.Client name string + kmsKey string } func (p *parameterStore) Get(ctx context.Context) (string, error) { @@ -115,11 +125,15 @@ func (p *parameterStore) Get(ctx context.Context) (string, error) { } func (p *parameterStore) Put(ctx context.Context, token string) error { - _, err := p.client.PutParameter(ctx, &ssm.PutParameterInput{ + in := &ssm.PutParameterInput{ Name: aws.String(p.name), Value: aws.String(token), Type: ssmtypes.ParameterTypeSecureString, Overwrite: aws.Bool(true), - }) + } + if p.kmsKey != "" { + in.KeyId = aws.String(p.kmsKey) + } + _, err := p.client.PutParameter(ctx, in) return err } diff --git a/internal/app/fleetjoin/fleetjoin_test.go b/internal/app/fleetjoin/fleetjoin_test.go index 786898461..e56e12110 100644 --- a/internal/app/fleetjoin/fleetjoin_test.go +++ b/internal/app/fleetjoin/fleetjoin_test.go @@ -25,58 +25,78 @@ func (s *fakeStore) Put(_ context.Context, token string) error { } type fakeIssuer struct { - token string - expiresAt *time.Time - issued int + token string + expiresAt *time.Time + activated int + activateErr error } func (i *fakeIssuer) VerifyJoinToken(_ context.Context, token string) error { if token != i.token || i.token == "" { return fleetnode.ErrBadToken } + if i.expiresAt == nil || !time.Now().Before(*i.expiresAt) { + return fleetnode.ErrJoinTokenExpired + } return nil } func (i *fakeIssuer) JoinTokenExpiresAt(context.Context) (*time.Time, error) { return i.expiresAt, nil } -func (i *fakeIssuer) IssueJoinToken(_ context.Context, ttl time.Duration) (string, time.Time, error) { - i.issued++ - i.token = "token-" + string(rune('a'+i.issued)) - at := time.Now().Add(ttl) - i.expiresAt = &at - return i.token, at, nil +func (i *fakeIssuer) ActivateJoinToken(_ context.Context, token string, expiresAt time.Time) error { + if i.activateErr != nil { + return i.activateErr + } + i.activated++ + i.token, i.expiresAt = token, &expiresAt + return nil } func TestSyncKeepsTheStoredTokenJoinable(t *testing.T) { now := time.Now() in := func(d time.Duration) *time.Time { at := now.Add(d); return &at } - storeFail := errors.New("store unavailable") + fail := errors.New("unavailable") cases := []struct { - name string - stored string - issuer fakeIssuer - getErr error - putErr error - wantRenewed bool - wantErr bool - wantIssued int + name string + stored string + issuer fakeIssuer + getErr error + putErr error + wantRenewed bool + wantErr bool + wantActivated int }{ - {name: "nothing stored", wantRenewed: true, wantIssued: 1}, + {name: "nothing stored", wantRenewed: true, wantActivated: 1}, {name: "valid for weeks", stored: "live", issuer: fakeIssuer{token: "live", expiresAt: in(20 * 24 * time.Hour)}}, - {name: "expiring soon", stored: "live", issuer: fakeIssuer{token: "live", expiresAt: in(5 * 24 * time.Hour)}, wantRenewed: true, wantIssued: 1}, - {name: "stored token was revoked", stored: "old", issuer: fakeIssuer{token: "live", expiresAt: in(20 * 24 * time.Hour)}, wantRenewed: true, wantIssued: 1}, - {name: "unreadable parameter revokes nothing", stored: "live", issuer: fakeIssuer{token: "live", expiresAt: in(time.Hour)}, getErr: storeFail, wantErr: true}, - {name: "failed store is reported", putErr: storeFail, wantRenewed: true, wantErr: true, wantIssued: 1}, + {name: "expiring soon", stored: "live", issuer: fakeIssuer{token: "live", expiresAt: in(5 * 24 * time.Hour)}, wantRenewed: true, wantActivated: 1}, + {name: "stored token was revoked", stored: "old", issuer: fakeIssuer{token: "live", expiresAt: in(20 * 24 * time.Hour)}, wantRenewed: true, wantActivated: 1}, + {name: "unreadable parameter changes nothing", stored: "live", issuer: fakeIssuer{token: "live", expiresAt: in(time.Hour)}, getErr: fail, wantErr: true}, + {name: "failed store keeps the stored token joining", stored: "live", issuer: fakeIssuer{token: "live", expiresAt: in(5 * 24 * time.Hour)}, putErr: fail, wantErr: true}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { store := &fakeStore{value: tc.stored, getErr: tc.getErr, putErr: tc.putErr} issuer := tc.issuer renewed, err := Sync(context.Background(), store, &issuer, now) - if renewed != tc.wantRenewed || (err != nil) != tc.wantErr || issuer.issued != tc.wantIssued { - t.Fatalf("renewed=%v err=%v issued=%d, want renewed=%v err=%v issued=%d", renewed, err, issuer.issued, tc.wantRenewed, tc.wantErr, tc.wantIssued) + if renewed != tc.wantRenewed || (err != nil) != tc.wantErr || issuer.activated != tc.wantActivated { + t.Fatalf("renewed=%v err=%v activated=%d, want renewed=%v err=%v activated=%d", renewed, err, issuer.activated, tc.wantRenewed, tc.wantErr, tc.wantActivated) } - if err == nil && issuer.VerifyJoinToken(context.Background(), store.value) != nil { + if tc.getErr == nil && issuer.VerifyJoinToken(context.Background(), store.value) != nil { t.Fatalf("stored token %q does not join", store.value) } }) } } + +func TestSyncRepairsAFailedActivation(t *testing.T) { + store := &fakeStore{} + issuer := &fakeIssuer{activateErr: errors.New("database unavailable")} + if _, err := Sync(context.Background(), store, issuer, time.Now()); err == nil { + t.Fatal("a failed activation must be reported") + } + issuer.activateErr = nil + if renewed, err := Sync(context.Background(), store, issuer, time.Now()); err != nil || !renewed { + t.Fatalf("renewed=%v err=%v, want the next sync to store and activate a token", renewed, err) + } + if err := issuer.VerifyJoinToken(context.Background(), store.value); err != nil { + t.Fatalf("stored token does not join after repair: %v", err) + } +} diff --git a/internal/app/fleetnode/service.go b/internal/app/fleetnode/service.go index 51ab76906..fcfbed953 100644 --- a/internal/app/fleetnode/service.go +++ b/internal/app/fleetnode/service.go @@ -119,6 +119,18 @@ const ( // any previous token, which is also how you revoke one. Existing nodes are // unaffected: they are already enrolled, and the token only gates joining. func (s *Service) IssueJoinToken(ctx context.Context, ttl time.Duration) (string, time.Time, error) { + token, expiresAt, err := NewJoinToken(ttl) + if err != nil { + return "", time.Time{}, err + } + if err := s.ActivateJoinToken(ctx, token, expiresAt); err != nil { + return "", time.Time{}, err + } + return token, expiresAt, nil +} + +// NewJoinToken mints a token that joins nothing until ActivateJoinToken stores it. +func NewJoinToken(ttl time.Duration) (string, time.Time, error) { if ttl <= 0 { ttl = DefaultJoinTokenTTL } @@ -129,12 +141,12 @@ func (s *Service) IssueJoinToken(ctx context.Context, ttl time.Duration) (string if _, err := rand.Read(raw); err != nil { return "", time.Time{}, err } - token := base64.RawURLEncoding.EncodeToString(raw) - expiresAt := time.Now().Add(ttl).UTC().Truncate(time.Second) - if err := s.settings.SetJoinToken(ctx, crypt.SHA256(token), expiresAt); err != nil { - return "", time.Time{}, err - } - return token, expiresAt, nil + return base64.RawURLEncoding.EncodeToString(raw), time.Now().Add(ttl).UTC().Truncate(time.Second), nil +} + +// ActivateJoinToken makes token the instance's join token until expiresAt, revoking the previous one. +func (s *Service) ActivateJoinToken(ctx context.Context, token string, expiresAt time.Time) error { + return s.settings.SetJoinToken(ctx, crypt.SHA256(token), expiresAt) } // VerifyJoinToken checks a presented token in constant time, then its expiry.