feat: keep the key and blob brokers on NODE_BROKER_TOKEN only, and let an operator admit INTERNAL_API_TOKEN on the other node routes with NODE_ACCEPT_INTERNAL_TOKEN=true while nodes joined before the split upgrade

docs/content/docs/development/configuration.mdx

@@ -58,6 +58,7 @@ Five values protect the whole instance. Compose ships a working default for each
 | `AUTH_SECRET` | 32 characters or more, enforced at boot | JWT and session signing. The realtime service reads the same value as `JWT_SECRET` and applies the same floor. Both refuse to start below it: a shorter HS256 key can be recovered offline from any token the service has issued. It also keys the `warmbly-verify=` values workspaces publish in `_warmbly` `TXT` records to prove a domain, so changing it changes every one of them: [root redirects](/guides/sending-domains/#root-redirects) stop at their next check until the new value is published | yes |
 | `INTERNAL_API_TOKEN` | any random string | The backend's `/api/v1/internal/` routes the tracking and forms services call (click tickets, root redirects, page views, hosted forms). Fleet nodes authenticate with it too when `NODE_BROKER_TOKEN` is unset | yes |
 | `NODE_BROKER_TOKEN` | any random string | Every internal route only a fleet node calls: organization data keys (read, store and open), blob signing, provider access tokens for a mailbox Warmbly Cloud manages, the message map and sync lookups, worker config and the node heartbeat. Optional, and falls back to `INTERNAL_API_TOKEN`. The same value has to be set on the control plane and on every node, or those routes answer 401; a node joined after it is set receives it in `node.env` in place of `INTERNAL_API_TOKEN`, and a node joined before has to join again. The installer generates one for a new install; an existing install keeps the shared token until you set one | yes |
+| `NODE_ACCEPT_INTERNAL_TOKEN` | `true` or unset | While `true`, the node-only routes other than the key and blob brokers also accept `INTERNAL_API_TOKEN`, so nodes joined before `NODE_BROKER_TOKEN` covered those routes can keep heartbeating and self-update. Set it for the upgrade, re-join or update every node, then remove it: while it is on, the tracking and forms services' token reaches those routes | no |
 | `SECRET_KEY_BASE` | 64 characters or more | Phoenix session signing in the realtime service | yes |
 | `KMS_LOCAL_MASTER_KEY` | base64, exactly 32 bytes | The root key that seals every per-organization data key | yes |
 | `CREDENTIALS_ENCRYPTION_KEY` | exactly 64 hex characters | Mailbox credentials at rest: SMTP and IMAP passwords, and Gmail and Outlook OAuth access and refresh tokens | yes |

docs/content/docs/development/split-deployment.mdx

@@ -77,6 +77,8 @@ Two limits are worth knowing. A signed URL covers one verb on one key, and only
 
 That credential is `NODE_BROKER_TOKEN`, and it covers every internal route only a node calls: the key and blob brokers, the provider access token for a mailbox Warmbly Cloud manages, data keys, the message map, sync lookups, worker config and the heartbeat. It falls back to `INTERNAL_API_TOKEN`, but set it to its own value here: the tracking and forms services are internet-facing and carry `INTERNAL_API_TOKEN`, and with a separate node token what they hold reaches none of those routes. A node joined after it is set receives it in place of the shared token.
 
+Upgrading a fleet that already sets `NODE_BROKER_TOKEN` to a release where it covers every node route: nodes joined earlier still send `INTERNAL_API_TOKEN` for the heartbeat, data keys, the message map, sync lookups and worker config. Set `NODE_ACCEPT_INTERNAL_TOKEN=true` on the control plane before upgrading it, let every node take the new version from its heartbeat (or join it again), then remove the setting.
+
 ## Building it
 
 <Steps>

internal/api/middleware/internal_auth.go

@@ -73,7 +73,23 @@ func loadBrokerToken() {
 
 func nodeBrokerAuth(c *gin.Context) {
 	brokerTokenOnce.Do(loadBrokerToken)
-	if len(brokerToken) == 0 {
+	checkNodeToken(c, brokerToken, nil)
+}
+
+// NodeAuthMiddleware guards node-only routes on NODE_BROKER_TOKEN; NODE_ACCEPT_INTERNAL_TOKEN=true also admits INTERNAL_API_TOKEN while older nodes upgrade.
+func (h *Handler) NodeAuthMiddleware() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		brokerTokenOnce.Do(loadBrokerToken)
+		var legacy []byte
+		if os.Getenv("NODE_ACCEPT_INTERNAL_TOKEN") == "true" {
+			legacy = []byte(os.Getenv("INTERNAL_API_TOKEN"))
+		}
+		checkNodeToken(c, brokerToken, legacy)
+	}
+}
+
+func checkNodeToken(c *gin.Context, token, legacy []byte) {
+	if len(token) == 0 {
 		c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "internal auth not configured"})
 		return
 	}
@@ -83,7 +99,11 @@ func nodeBrokerAuth(c *gin.Context) {
 		return
 	}
 	provided := []byte(strings.TrimPrefix(header, "Bearer "))
-	if subtle.ConstantTimeCompare(provided, brokerToken) != 1 {
+	ok := subtle.ConstantTimeCompare(provided, token) == 1
+	if !ok && len(legacy) > 0 {
+		ok = subtle.ConstantTimeCompare(provided, legacy) == 1
+	}
+	if !ok {
 		c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "invalid bearer token"})
 		return
 	}

internal/api/routes.go

@@ -155,22 +155,24 @@ func Run(
 	// calls. It takes NODE_BROKER_TOKEN, which falls back to
 	// INTERNAL_API_TOKEN, so the internet-facing tracking and forms services
 	// never need a credential that opens a key or enrols a node.
+	broker := r.Group("/api/v1/internal")
+	broker.Use(m.NodeBrokerAuthMiddleware())
 	node := r.Group("/api/v1/internal")
-	node.Use(m.NodeBrokerAuthMiddleware())
+	node.Use(m.NodeAuthMiddleware())
 	{
 		// Opens a sealed data key for a node running KMS_PROVIDER=brokered, so
 		// a machine you own needs no cloud credential of its own.
-		node.POST("/dek/decrypt", h.InternalDecryptDEK)
+		broker.POST("/dek/decrypt", h.InternalDecryptDEK)
 
 		// Signs one blob operation for a node running BLOB_PROVIDER=brokered.
 		// The node then transfers directly against the object store, so bodies
 		// and attachments never pass through here.
-		node.POST("/blobs/presign", h.InternalPresignBlob)
+		broker.POST("/blobs/presign", h.InternalPresignBlob)
 
 		// Mints a live provider access token for a mailbox Warmbly Cloud
 		// manages, which is worth more than any record the rest of the
 		// internal API moves.
-		node.GET("/cloud-link/token/:id", h.InternalCloudLinkToken)
+		broker.GET("/cloud-link/token/:id", h.InternalCloudLinkToken)
 
 		node.GET("/dek/:orgID", h.InternalGetDEK)
 		node.PUT("/dek/:orgID", h.InternalPutDEK)