mirror of
https://github.com/warmbly/warmbly.git
synced 2026-11-17 08:18:51 +00:00
feat: store a renewed fleet join token in Parameter Store before making it the join token, so a failed write leaves the previous token joining, and encrypt it with FLEET_JOIN_TOKEN_KMS_KEY when set
This commit is contained in:
+1
-1
@@ -2311,7 +2311,7 @@ func main() {
|
||||
// The worker Auto Scaling Group joins with the token in this parameter; unset on every other deployment.
|
||||
if parameter := fleetjoin.Parameter(); parameter != "" && fleetNodeService != nil {
|
||||
elector.Go("fleet_join_token", func(ctx context.Context) {
|
||||
if err := fleetjoin.Run(ctx, fleetNodeService, parameter); err != nil {
|
||||
if err := fleetjoin.Run(ctx, fleetNodeService, parameter, fleetjoin.KMSKey()); err != nil {
|
||||
log.Printf("Warning: fleet join token renewal disabled: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -69,7 +69,10 @@ table's default route.
|
||||
Set [`FLEET_JOIN_TOKEN_PARAMETER`](/development/configuration/#workers) on the
|
||||
backend to the parameter name, `/<name>/fleet/join-token` unless
|
||||
`fleet_join_token_parameter` changes it, and give the backend's role
|
||||
`ssm:GetParameter` and `ssm:PutParameter` on it. Within a minute of starting,
|
||||
`ssm:GetParameter` and `ssm:PutParameter` on it. When the parameter uses a
|
||||
customer managed key (`fleet_join_token_kms_key_arn`), also set
|
||||
[`FLEET_JOIN_TOKEN_KMS_KEY`](/development/configuration/#workers) to that key and
|
||||
give the backend's role `kms:Encrypt` and `kms:Decrypt` on it. Within a minute of starting,
|
||||
the backend issues a 30-day join token and stores it there as a SecureString.
|
||||
It checks every hour and issues a new one when the stored token expires within
|
||||
10 days or no longer joins, so launches never meet an expired token. Workers
|
||||
|
||||
@@ -586,7 +586,8 @@ Keep the copies private. GeoLite2 may not be redistributed, which is also why no
|
||||
| `WARMBLY_ENV_REFRESH` | Read by the host updater, not the node. `true` lets it replace `node.env` with what the control plane renders now whenever the heartbeat names a different env revision, then restart the node. `node.local.env` is never touched. Written by the join script on a new join and by `--adopt --env-refresh`; a node without it keeps its env until it rejoins | unset | no (read by the updater) |
|
||||
| `APP_INTERNAL_URL` | Read by the backend: the address a joining node is told to call back on when `ENCRYPTED_KEYS_BACKEND_URL` is unset on the backend. Instance health checks whichever of the two is set | unset | no (read per join) |
|
||||
| `FLEET_METRICS_CLOUDWATCH` | Read by the backend: the fleet name it publishes `Warmbly/Fleet` metrics under (dimension `Fleet`) every minute from the workers' heartbeats: `WorkerMemoryPercent` and `WorkerMemoryMaxPercent`, which the [AWS worker fleet](/development/aws-workers/#metric-contract) scales on, and `WorkerUtilizationPercent` for reading only. Set it to the workers root's `name`. Uses the AWS default credential chain, so the backend's role needs `cloudwatch:PutMetricData`. Leave it unset on any other deployment | unset | yes |
|
||||
| `FLEET_JOIN_TOKEN_PARAMETER` | Read by the backend: the Parameter Store SecureString the [AWS worker fleet](/development/aws-workers/#prepare-the-join-token) joins with. The backend checks it every hour and, when it is missing, no longer joins, or expires within 10 days, issues a 30-day join token and stores it there. Issuing revokes the previous token, so once this is set, read the token from the parameter rather than issuing one with `warmblyctl fleet join-token`. The parameter uses the `aws/ssm` key. Uses the AWS default credential chain, so the backend's role needs `ssm:GetParameter` and `ssm:PutParameter` on that parameter. Leave it unset on any other deployment | unset | yes |
|
||||
| `FLEET_JOIN_TOKEN_PARAMETER` | Read by the backend: the Parameter Store SecureString the [AWS worker fleet](/development/aws-workers/#prepare-the-join-token) joins with. The backend checks it every hour and, when it is missing, no longer joins, or expires within 10 days, issues a 30-day join token and stores it there. Issuing revokes the previous token, so once this is set, read the token from the parameter rather than issuing one with `warmblyctl fleet join-token`. The new token is stored first and only then made the join token, so a failed write leaves the previous one working. Uses the AWS default credential chain, so the backend's role needs `ssm:GetParameter` and `ssm:PutParameter` on that parameter. Leave it unset on any other deployment | unset | yes |
|
||||
| `FLEET_JOIN_TOKEN_KMS_KEY` | Read by the backend with `FLEET_JOIN_TOKEN_PARAMETER`: the KMS key ID or ARN the parameter is encrypted with, the workers root's `fleet_join_token_kms_key_arn`. The backend's role then also needs `kms:Encrypt` and `kms:Decrypt` on that key. Unset stores the parameter with `aws/ssm` | unset | yes |
|
||||
|
||||
<Callout type="warn" title="An unset key URL is silent">
|
||||
An empty `ENCRYPTED_KEYS_BACKEND_URL` or `ENCRYPTED_KEYS_WORKER_TOKEN` lets the worker start, subscribe and never register. There is no log line. The `no_worker_heartbeat` check on [Instance health](/development/instance-health/#no_worker_heartbeat) is what surfaces it.
|
||||
|
||||
@@ -31,6 +31,11 @@ func Parameter() string {
|
||||
return strings.TrimSpace(os.Getenv("FLEET_JOIN_TOKEN_PARAMETER"))
|
||||
}
|
||||
|
||||
// KMSKey is FLEET_JOIN_TOKEN_KMS_KEY, the key the parameter is encrypted with; empty means aws/ssm.
|
||||
func KMSKey() string {
|
||||
return strings.TrimSpace(os.Getenv("FLEET_JOIN_TOKEN_KMS_KEY"))
|
||||
}
|
||||
|
||||
// Store holds the plaintext token; Get answers "" for a parameter that does not exist.
|
||||
type Store interface {
|
||||
Get(ctx context.Context) (string, error)
|
||||
@@ -41,12 +46,13 @@ type Store interface {
|
||||
type Issuer interface {
|
||||
VerifyJoinToken(ctx context.Context, token string) error
|
||||
JoinTokenExpiresAt(ctx context.Context) (*time.Time, error)
|
||||
IssueJoinToken(ctx context.Context, ttl time.Duration) (string, time.Time, error)
|
||||
ActivateJoinToken(ctx context.Context, token string, expiresAt time.Time) error
|
||||
}
|
||||
|
||||
// Sync leaves a stored token that still joins for more than RenewBefore, and
|
||||
// otherwise issues a new one and stores it. Issuing revokes the previous token,
|
||||
// so a failed store is retried by the next Sync rather than undone.
|
||||
// otherwise stores a new one and then activates it. A failed store leaves the
|
||||
// previous token stored and joining; a failed activation is repaired by the
|
||||
// next Sync, which finds a stored token that does not join.
|
||||
func Sync(ctx context.Context, store Store, issuer Issuer, now time.Time) (bool, error) {
|
||||
current, err := store.Get(ctx)
|
||||
if err != nil {
|
||||
@@ -61,24 +67,27 @@ func Sync(ctx context.Context, store Store, issuer Issuer, now time.Time) (bool,
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
token, _, err := issuer.IssueJoinToken(ctx, fleetnode.MaxJoinTokenTTL)
|
||||
token, expiresAt, err := fleetnode.NewJoinToken(fleetnode.MaxJoinTokenTTL)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := store.Put(ctx, token); err != nil {
|
||||
return true, fmt.Errorf("store renewed join token: %w", err)
|
||||
return false, fmt.Errorf("store renewed join token: %w", err)
|
||||
}
|
||||
if err := issuer.ActivateJoinToken(ctx, token, expiresAt); err != nil {
|
||||
return false, fmt.Errorf("activate stored join token: %w", err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Run keeps the parameter current until ctx ends. Run it on one backend at a
|
||||
// time: two replicas issuing at once would each revoke the other's token.
|
||||
func Run(ctx context.Context, issuer Issuer, parameter string) error {
|
||||
func Run(ctx context.Context, issuer Issuer, parameter, kmsKey string) error {
|
||||
cfg, err := awsconfig.LoadDefaultConfig(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fleet join token: aws config: %w", err)
|
||||
}
|
||||
store := ¶meterStore{client: ssm.NewFromConfig(cfg), name: parameter}
|
||||
store := ¶meterStore{client: ssm.NewFromConfig(cfg), name: parameter, kmsKey: kmsKey}
|
||||
for {
|
||||
wait := checkEvery
|
||||
renewed, err := Sync(ctx, store, issuer, time.Now())
|
||||
@@ -100,6 +109,7 @@ func Run(ctx context.Context, issuer Issuer, parameter string) error {
|
||||
type parameterStore struct {
|
||||
client *ssm.Client
|
||||
name string
|
||||
kmsKey string
|
||||
}
|
||||
|
||||
func (p *parameterStore) Get(ctx context.Context) (string, error) {
|
||||
@@ -115,11 +125,15 @@ func (p *parameterStore) Get(ctx context.Context) (string, error) {
|
||||
}
|
||||
|
||||
func (p *parameterStore) Put(ctx context.Context, token string) error {
|
||||
_, err := p.client.PutParameter(ctx, &ssm.PutParameterInput{
|
||||
in := &ssm.PutParameterInput{
|
||||
Name: aws.String(p.name),
|
||||
Value: aws.String(token),
|
||||
Type: ssmtypes.ParameterTypeSecureString,
|
||||
Overwrite: aws.Bool(true),
|
||||
})
|
||||
}
|
||||
if p.kmsKey != "" {
|
||||
in.KeyId = aws.String(p.kmsKey)
|
||||
}
|
||||
_, err := p.client.PutParameter(ctx, in)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -25,58 +25,78 @@ func (s *fakeStore) Put(_ context.Context, token string) error {
|
||||
}
|
||||
|
||||
type fakeIssuer struct {
|
||||
token string
|
||||
expiresAt *time.Time
|
||||
issued int
|
||||
token string
|
||||
expiresAt *time.Time
|
||||
activated int
|
||||
activateErr error
|
||||
}
|
||||
|
||||
func (i *fakeIssuer) VerifyJoinToken(_ context.Context, token string) error {
|
||||
if token != i.token || i.token == "" {
|
||||
return fleetnode.ErrBadToken
|
||||
}
|
||||
if i.expiresAt == nil || !time.Now().Before(*i.expiresAt) {
|
||||
return fleetnode.ErrJoinTokenExpired
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (i *fakeIssuer) JoinTokenExpiresAt(context.Context) (*time.Time, error) { return i.expiresAt, nil }
|
||||
func (i *fakeIssuer) IssueJoinToken(_ context.Context, ttl time.Duration) (string, time.Time, error) {
|
||||
i.issued++
|
||||
i.token = "token-" + string(rune('a'+i.issued))
|
||||
at := time.Now().Add(ttl)
|
||||
i.expiresAt = &at
|
||||
return i.token, at, nil
|
||||
func (i *fakeIssuer) ActivateJoinToken(_ context.Context, token string, expiresAt time.Time) error {
|
||||
if i.activateErr != nil {
|
||||
return i.activateErr
|
||||
}
|
||||
i.activated++
|
||||
i.token, i.expiresAt = token, &expiresAt
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestSyncKeepsTheStoredTokenJoinable(t *testing.T) {
|
||||
now := time.Now()
|
||||
in := func(d time.Duration) *time.Time { at := now.Add(d); return &at }
|
||||
storeFail := errors.New("store unavailable")
|
||||
fail := errors.New("unavailable")
|
||||
cases := []struct {
|
||||
name string
|
||||
stored string
|
||||
issuer fakeIssuer
|
||||
getErr error
|
||||
putErr error
|
||||
wantRenewed bool
|
||||
wantErr bool
|
||||
wantIssued int
|
||||
name string
|
||||
stored string
|
||||
issuer fakeIssuer
|
||||
getErr error
|
||||
putErr error
|
||||
wantRenewed bool
|
||||
wantErr bool
|
||||
wantActivated int
|
||||
}{
|
||||
{name: "nothing stored", wantRenewed: true, wantIssued: 1},
|
||||
{name: "nothing stored", wantRenewed: true, wantActivated: 1},
|
||||
{name: "valid for weeks", stored: "live", issuer: fakeIssuer{token: "live", expiresAt: in(20 * 24 * time.Hour)}},
|
||||
{name: "expiring soon", stored: "live", issuer: fakeIssuer{token: "live", expiresAt: in(5 * 24 * time.Hour)}, wantRenewed: true, wantIssued: 1},
|
||||
{name: "stored token was revoked", stored: "old", issuer: fakeIssuer{token: "live", expiresAt: in(20 * 24 * time.Hour)}, wantRenewed: true, wantIssued: 1},
|
||||
{name: "unreadable parameter revokes nothing", stored: "live", issuer: fakeIssuer{token: "live", expiresAt: in(time.Hour)}, getErr: storeFail, wantErr: true},
|
||||
{name: "failed store is reported", putErr: storeFail, wantRenewed: true, wantErr: true, wantIssued: 1},
|
||||
{name: "expiring soon", stored: "live", issuer: fakeIssuer{token: "live", expiresAt: in(5 * 24 * time.Hour)}, wantRenewed: true, wantActivated: 1},
|
||||
{name: "stored token was revoked", stored: "old", issuer: fakeIssuer{token: "live", expiresAt: in(20 * 24 * time.Hour)}, wantRenewed: true, wantActivated: 1},
|
||||
{name: "unreadable parameter changes nothing", stored: "live", issuer: fakeIssuer{token: "live", expiresAt: in(time.Hour)}, getErr: fail, wantErr: true},
|
||||
{name: "failed store keeps the stored token joining", stored: "live", issuer: fakeIssuer{token: "live", expiresAt: in(5 * 24 * time.Hour)}, putErr: fail, wantErr: true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
store := &fakeStore{value: tc.stored, getErr: tc.getErr, putErr: tc.putErr}
|
||||
issuer := tc.issuer
|
||||
renewed, err := Sync(context.Background(), store, &issuer, now)
|
||||
if renewed != tc.wantRenewed || (err != nil) != tc.wantErr || issuer.issued != tc.wantIssued {
|
||||
t.Fatalf("renewed=%v err=%v issued=%d, want renewed=%v err=%v issued=%d", renewed, err, issuer.issued, tc.wantRenewed, tc.wantErr, tc.wantIssued)
|
||||
if renewed != tc.wantRenewed || (err != nil) != tc.wantErr || issuer.activated != tc.wantActivated {
|
||||
t.Fatalf("renewed=%v err=%v activated=%d, want renewed=%v err=%v activated=%d", renewed, err, issuer.activated, tc.wantRenewed, tc.wantErr, tc.wantActivated)
|
||||
}
|
||||
if err == nil && issuer.VerifyJoinToken(context.Background(), store.value) != nil {
|
||||
if tc.getErr == nil && issuer.VerifyJoinToken(context.Background(), store.value) != nil {
|
||||
t.Fatalf("stored token %q does not join", store.value)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncRepairsAFailedActivation(t *testing.T) {
|
||||
store := &fakeStore{}
|
||||
issuer := &fakeIssuer{activateErr: errors.New("database unavailable")}
|
||||
if _, err := Sync(context.Background(), store, issuer, time.Now()); err == nil {
|
||||
t.Fatal("a failed activation must be reported")
|
||||
}
|
||||
issuer.activateErr = nil
|
||||
if renewed, err := Sync(context.Background(), store, issuer, time.Now()); err != nil || !renewed {
|
||||
t.Fatalf("renewed=%v err=%v, want the next sync to store and activate a token", renewed, err)
|
||||
}
|
||||
if err := issuer.VerifyJoinToken(context.Background(), store.value); err != nil {
|
||||
t.Fatalf("stored token does not join after repair: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,6 +119,18 @@ const (
|
||||
// any previous token, which is also how you revoke one. Existing nodes are
|
||||
// unaffected: they are already enrolled, and the token only gates joining.
|
||||
func (s *Service) IssueJoinToken(ctx context.Context, ttl time.Duration) (string, time.Time, error) {
|
||||
token, expiresAt, err := NewJoinToken(ttl)
|
||||
if err != nil {
|
||||
return "", time.Time{}, err
|
||||
}
|
||||
if err := s.ActivateJoinToken(ctx, token, expiresAt); err != nil {
|
||||
return "", time.Time{}, err
|
||||
}
|
||||
return token, expiresAt, nil
|
||||
}
|
||||
|
||||
// NewJoinToken mints a token that joins nothing until ActivateJoinToken stores it.
|
||||
func NewJoinToken(ttl time.Duration) (string, time.Time, error) {
|
||||
if ttl <= 0 {
|
||||
ttl = DefaultJoinTokenTTL
|
||||
}
|
||||
@@ -129,12 +141,12 @@ func (s *Service) IssueJoinToken(ctx context.Context, ttl time.Duration) (string
|
||||
if _, err := rand.Read(raw); err != nil {
|
||||
return "", time.Time{}, err
|
||||
}
|
||||
token := base64.RawURLEncoding.EncodeToString(raw)
|
||||
expiresAt := time.Now().Add(ttl).UTC().Truncate(time.Second)
|
||||
if err := s.settings.SetJoinToken(ctx, crypt.SHA256(token), expiresAt); err != nil {
|
||||
return "", time.Time{}, err
|
||||
}
|
||||
return token, expiresAt, nil
|
||||
return base64.RawURLEncoding.EncodeToString(raw), time.Now().Add(ttl).UTC().Truncate(time.Second), nil
|
||||
}
|
||||
|
||||
// ActivateJoinToken makes token the instance's join token until expiresAt, revoking the previous one.
|
||||
func (s *Service) ActivateJoinToken(ctx context.Context, token string, expiresAt time.Time) error {
|
||||
return s.settings.SetJoinToken(ctx, crypt.SHA256(token), expiresAt)
|
||||
}
|
||||
|
||||
// VerifyJoinToken checks a presented token in constant time, then its expiry.
|
||||
|
||||
Reference in New Issue
Block a user