From c178dadf81f0f31c035e90b760079bddea705192 Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Sun, 4 Oct 2026 05:18:01 -0700 Subject: [PATCH] feat: keep the key and blob brokers on NODE_BROKER_TOKEN only, and let an operator admit INTERNAL_API_TOKEN on the other node routes with NODE_ACCEPT_INTERNAL_TOKEN=true while nodes joined before the split upgrade --- .../docs/development/configuration.mdx | 1 + .../docs/development/split-deployment.mdx | 2 ++ internal/api/middleware/internal_auth.go | 24 +++++++++++++++++-- internal/api/routes.go | 10 ++++---- 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/docs/content/docs/development/configuration.mdx b/docs/content/docs/development/configuration.mdx index cdf7220c9..a19a44322 100644 --- a/docs/content/docs/development/configuration.mdx +++ b/docs/content/docs/development/configuration.mdx @@ -58,6 +58,7 @@ Five values protect the whole instance. Compose ships a working default for each | `AUTH_SECRET` | 32 characters or more, enforced at boot | JWT and session signing. The realtime service reads the same value as `JWT_SECRET` and applies the same floor. Both refuse to start below it: a shorter HS256 key can be recovered offline from any token the service has issued. It also keys the `warmbly-verify=` values workspaces publish in `_warmbly` `TXT` records to prove a domain, so changing it changes every one of them: [root redirects](/guides/sending-domains/#root-redirects) stop at their next check until the new value is published | yes | | `INTERNAL_API_TOKEN` | any random string | The backend's `/api/v1/internal/` routes the tracking and forms services call (click tickets, root redirects, page views, hosted forms). Fleet nodes authenticate with it too when `NODE_BROKER_TOKEN` is unset | yes | | `NODE_BROKER_TOKEN` | any random string | Every internal route only a fleet node calls: organization data keys (read, store and open), blob signing, provider access tokens for a mailbox Warmbly Cloud manages, the message map and sync lookups, worker config and the node heartbeat. Optional, and falls back to `INTERNAL_API_TOKEN`. The same value has to be set on the control plane and on every node, or those routes answer 401; a node joined after it is set receives it in `node.env` in place of `INTERNAL_API_TOKEN`, and a node joined before has to join again. The installer generates one for a new install; an existing install keeps the shared token until you set one | yes | +| `NODE_ACCEPT_INTERNAL_TOKEN` | `true` or unset | While `true`, the node-only routes other than the key and blob brokers also accept `INTERNAL_API_TOKEN`, so nodes joined before `NODE_BROKER_TOKEN` covered those routes can keep heartbeating and self-update. Set it for the upgrade, re-join or update every node, then remove it: while it is on, the tracking and forms services' token reaches those routes | no | | `SECRET_KEY_BASE` | 64 characters or more | Phoenix session signing in the realtime service | yes | | `KMS_LOCAL_MASTER_KEY` | base64, exactly 32 bytes | The root key that seals every per-organization data key | yes | | `CREDENTIALS_ENCRYPTION_KEY` | exactly 64 hex characters | Mailbox credentials at rest: SMTP and IMAP passwords, and Gmail and Outlook OAuth access and refresh tokens | yes | diff --git a/docs/content/docs/development/split-deployment.mdx b/docs/content/docs/development/split-deployment.mdx index 95d1fec89..761d36c24 100644 --- a/docs/content/docs/development/split-deployment.mdx +++ b/docs/content/docs/development/split-deployment.mdx @@ -77,6 +77,8 @@ Two limits are worth knowing. A signed URL covers one verb on one key, and only That credential is `NODE_BROKER_TOKEN`, and it covers every internal route only a node calls: the key and blob brokers, the provider access token for a mailbox Warmbly Cloud manages, data keys, the message map, sync lookups, worker config and the heartbeat. It falls back to `INTERNAL_API_TOKEN`, but set it to its own value here: the tracking and forms services are internet-facing and carry `INTERNAL_API_TOKEN`, and with a separate node token what they hold reaches none of those routes. A node joined after it is set receives it in place of the shared token. +Upgrading a fleet that already sets `NODE_BROKER_TOKEN` to a release where it covers every node route: nodes joined earlier still send `INTERNAL_API_TOKEN` for the heartbeat, data keys, the message map, sync lookups and worker config. Set `NODE_ACCEPT_INTERNAL_TOKEN=true` on the control plane before upgrading it, let every node take the new version from its heartbeat (or join it again), then remove the setting. + ## Building it diff --git a/internal/api/middleware/internal_auth.go b/internal/api/middleware/internal_auth.go index e6bed5349..2980ea2b7 100644 --- a/internal/api/middleware/internal_auth.go +++ b/internal/api/middleware/internal_auth.go @@ -73,7 +73,23 @@ func loadBrokerToken() { func nodeBrokerAuth(c *gin.Context) { brokerTokenOnce.Do(loadBrokerToken) - if len(brokerToken) == 0 { + checkNodeToken(c, brokerToken, nil) +} + +// NodeAuthMiddleware guards node-only routes on NODE_BROKER_TOKEN; NODE_ACCEPT_INTERNAL_TOKEN=true also admits INTERNAL_API_TOKEN while older nodes upgrade. +func (h *Handler) NodeAuthMiddleware() gin.HandlerFunc { + return func(c *gin.Context) { + brokerTokenOnce.Do(loadBrokerToken) + var legacy []byte + if os.Getenv("NODE_ACCEPT_INTERNAL_TOKEN") == "true" { + legacy = []byte(os.Getenv("INTERNAL_API_TOKEN")) + } + checkNodeToken(c, brokerToken, legacy) + } +} + +func checkNodeToken(c *gin.Context, token, legacy []byte) { + if len(token) == 0 { c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "internal auth not configured"}) return } @@ -83,7 +99,11 @@ func nodeBrokerAuth(c *gin.Context) { return } provided := []byte(strings.TrimPrefix(header, "Bearer ")) - if subtle.ConstantTimeCompare(provided, brokerToken) != 1 { + ok := subtle.ConstantTimeCompare(provided, token) == 1 + if !ok && len(legacy) > 0 { + ok = subtle.ConstantTimeCompare(provided, legacy) == 1 + } + if !ok { c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "invalid bearer token"}) return } diff --git a/internal/api/routes.go b/internal/api/routes.go index 465cf19ac..3e9282a54 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -155,22 +155,24 @@ func Run( // calls. It takes NODE_BROKER_TOKEN, which falls back to // INTERNAL_API_TOKEN, so the internet-facing tracking and forms services // never need a credential that opens a key or enrols a node. + broker := r.Group("/api/v1/internal") + broker.Use(m.NodeBrokerAuthMiddleware()) node := r.Group("/api/v1/internal") - node.Use(m.NodeBrokerAuthMiddleware()) + node.Use(m.NodeAuthMiddleware()) { // Opens a sealed data key for a node running KMS_PROVIDER=brokered, so // a machine you own needs no cloud credential of its own. - node.POST("/dek/decrypt", h.InternalDecryptDEK) + broker.POST("/dek/decrypt", h.InternalDecryptDEK) // Signs one blob operation for a node running BLOB_PROVIDER=brokered. // The node then transfers directly against the object store, so bodies // and attachments never pass through here. - node.POST("/blobs/presign", h.InternalPresignBlob) + broker.POST("/blobs/presign", h.InternalPresignBlob) // Mints a live provider access token for a mailbox Warmbly Cloud // manages, which is worth more than any record the rest of the // internal API moves. - node.GET("/cloud-link/token/:id", h.InternalCloudLinkToken) + broker.GET("/cloud-link/token/:id", h.InternalCloudLinkToken) node.GET("/dek/:orgID", h.InternalGetDEK) node.PUT("/dek/:orgID", h.InternalPutDEK)