Files
warmbly/internal/app/fleetnode/nodeenv.go
T

216 lines
8.0 KiB
Go

package fleetnode
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"os"
"strings"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/config"
"github.com/warmbly/warmbly/internal/models"
)
// NodeEnvKeys are the settings a node needs to do its job, in the order they
// are written. Every one is read from the backend's own environment, so a node
// is configured with exactly the infrastructure the control plane uses and
// there is nothing to keep in sync by hand.
//
// Every name here must be one the node's own code reads. Two of them were not
// (S3_BUCKET and KMS_KEY_ID, against a storage layer reading BLOB_BUCKET and a
// KMS factory reading KMS_AWS_KEY_ID), which sent an AWS-backed node to the
// default bucket and the default key alias with nothing logged.
//
// KMS_PROVIDER and BLOB_PROVIDER are absent because they are not copied but
// translated; see nodeProviders. The AWS-shaped settings above stay, so an
// operator who deliberately overrides a node back to a direct provider in
// node.local.env only has to add the credential.
//
// Deliberately excluded: PRIMARY_DB for a worker, and anything else that would
// give one direct database access. A worker reaches relational data through
// the internal API and nothing else. RenderEnv sends the DSN to a consumer,
// which is control plane and updates relational state itself.
var NodeEnvKeys = []string{
"APP_ENV",
"EVENTBUS_PROVIDER",
"NATS_URL",
// Base64, not a path: a node has no file to point at, and the env file
// docker reads cannot hold a multi-line value.
"NATS_CREDS_B64",
"NATS_MAX_BYTES",
"KAFKA_BOOTSTRAP_SERVERS",
"KAFKA_SASL_USERNAME",
"KAFKA_SASL_PASSWORD",
"SCHEMA_REGISTRY_URL",
"SCHEMA_REGISTRY_KEY",
"SCHEMA_REGISTRY_SECRET",
"CODEC_PROVIDER",
"REDIS",
"KMS_LOCAL_MASTER_KEY",
"KMS_AWS_KEY_ID",
"CREDENTIALS_ENCRYPTION_KEY",
// Earlier keys stay readable on nodes while a rotation reseals.
"CREDENTIALS_ENCRYPTION_READ_KEYS",
"BLOB_FS_ROOT",
"BLOB_BUCKET",
"AWS_REGION",
"AWS_ENDPOINT_URL_S3",
"BOX_GOOGLE_CLIENT_ID",
"BOX_GOOGLE_CLIENT_SECRET",
"BOX_OUTLOOK_CLIENT_ID",
"BOX_OUTLOOK_CLIENT_SECRET",
"POSTHOG_KEY",
"POSTHOG_HOST",
"POSTHOG_ERROR_TRACKING",
"SENTRY_DSN",
}
// consumerEnvKeys reach a consumer and never a worker. A consumer runs the
// integration actions, refreshes their OAuth tokens and drains the CRM outbox,
// so it reads the integration client credentials (integration.NewOAuthManager),
// APP_URL (the "Open in Warmbly" link written to the CRM) and the backend's
// public URL (where it registers Pipedrive's webhooks).
var consumerEnvKeys = []string{
"APP_URL",
"BACKEND_PUBLIC_URL",
"API_PUBLIC_URL",
"HUBSPOT_OAUTH_CLIENT_ID",
"HUBSPOT_OAUTH_CLIENT_SECRET",
"SLACK_OAUTH_CLIENT_ID",
"SLACK_OAUTH_CLIENT_SECRET",
"GOOGLE_SHEETS_OAUTH_CLIENT_ID",
"GOOGLE_SHEETS_OAUTH_CLIENT_SECRET",
"PIPEDRIVE_OAUTH_CLIENT_ID",
"PIPEDRIVE_OAUTH_CLIENT_SECRET",
"SALESFORCE_OAUTH_CLIENT_ID",
"SALESFORCE_OAUTH_CLIENT_SECRET",
}
// nodeProviders translates the control plane's own crypto and blob providers
// into the ones a node should run.
//
// A node has no cloud credentials and no way to be handed any: node.env is
// regenerated from this instance's environment on every join, and shipping an
// IAM key to every machine in the fleet is exactly the thing worth avoiding. So
// a provider that needs a credential becomes its brokered form, which
// authenticates with the internal API token the node already holds and asks
// this instance to do the one privileged operation. A provider that needs no
// credential (local KMS, filesystem blobs) passes through unchanged.
//
// Brokering costs one HTTPS call to the control plane per DEK open (Redis
// caches the result) and per blob operation. The bytes still go straight
// between the node and the object store.
func nodeProviders() (kmsProvider, blobProvider string) {
kmsProvider = config.KMSProvider()
if kmsProvider == "aws" || kmsProvider == "aws-kms" {
kmsProvider = "brokered"
}
blobProvider = config.BlobProvider()
if blobProvider == "s3" {
blobProvider = "brokered"
}
return kmsProvider, blobProvider
}
// RenderEnv builds the env file a node writes to disk on join.
func RenderEnv(nodeID uuid.UUID, role models.NodeRole, region string) string {
var b strings.Builder
b.WriteString("# Written by `warmbly join`. Regenerate by joining again.\n")
fmt.Fprintf(&b, "WARMBLY_NODE_ID=%s\n", nodeID)
fmt.Fprintf(&b, "WARMBLY_NODE_ROLE=%s\n", role)
if region != "" {
fmt.Fprintf(&b, "WARMBLY_NODE_REGION=%s\n", region)
}
// A worker resolves its own identity from WORKER_ID; keeping the two equal
// means the placement row and the node row are the same machine.
if role == models.NodeRoleWorker {
fmt.Fprintf(&b, "WORKER_ID=%s\n", nodeID)
}
backend := strings.TrimRight(os.Getenv("ENCRYPTED_KEYS_BACKEND_URL"), "/")
if backend == "" {
backend = strings.TrimRight(os.Getenv("APP_INTERNAL_URL"), "/")
}
// A consumer is control plane: it opens Postgres itself, so it gets the DSN
// a worker is deliberately never given, and reads keys straight from the
// table rather than back through the API it sits behind. An instance whose
// DSN lives in SSM rather than the environment has nothing to send, so the
// node falls back to the worker's HTTP key path and join.sh tells the
// operator to supply PRIMARY_DB in node.local.env.
dsn := os.Getenv("PRIMARY_DB")
keysProvider := "http"
if role == models.NodeRoleConsumer && dsn != "" {
keysProvider = "postgres"
fmt.Fprintf(&b, "PRIMARY_DB=%s\n", dsn)
}
fmt.Fprintf(&b, "WARMBLY_BACKEND_URL=%s\n", backend)
fmt.Fprintf(&b, "ENCRYPTED_KEYS_PROVIDER=%s\n", keysProvider)
fmt.Fprintf(&b, "ENCRYPTED_KEYS_BACKEND_URL=%s\n", backend)
// A node calls only the node-only internal routes. With a separate
// NODE_BROKER_TOKEN it is sent that alone and never the edge services' token.
if v := os.Getenv("NODE_BROKER_TOKEN"); v != "" {
fmt.Fprintf(&b, "ENCRYPTED_KEYS_WORKER_TOKEN=%s\n", v)
fmt.Fprintf(&b, "NODE_BROKER_TOKEN=%s\n", v)
} else {
fmt.Fprintf(&b, "ENCRYPTED_KEYS_WORKER_TOKEN=%s\n", os.Getenv("INTERNAL_API_TOKEN"))
fmt.Fprintf(&b, "INTERNAL_API_TOKEN=%s\n", os.Getenv("INTERNAL_API_TOKEN"))
}
kmsProvider, blobProvider := nodeProviders()
fmt.Fprintf(&b, "KMS_PROVIDER=%s\n", kmsProvider)
fmt.Fprintf(&b, "BLOB_PROVIDER=%s\n", blobProvider)
// Resolved here: a node cannot tell a hosted instance from a self-host on its own.
fmt.Fprintf(&b, "MAIL_ALLOW_PRIVATE_HOSTS=%t\n", config.PrivateMailHostsAllowed())
fmt.Fprintf(&b, "MAIL_TLS_INSECURE=%t\n", config.MailTLSInsecureAllowed())
keys := NodeEnvKeys
if role == models.NodeRoleConsumer {
keys = append(append([]string{}, NodeEnvKeys...), consumerEnvKeys...)
}
for _, k := range keys {
if v := os.Getenv(k); v != "" {
fmt.Fprintf(&b, "%s=%s\n", k, v)
}
}
return b.String()
}
// EnvRevision names one rendered env, so a node can tell its file is stale.
func EnvRevision(env string) string {
sum := sha256.Sum256([]byte(env))
return hex.EncodeToString(sum[:16])
}
// NodeEnvState is whether one live node runs the env the backend renders now.
type NodeEnvState struct {
ID string `json:"id"`
Name string `json:"name"`
Role string `json:"role"`
Applied string `json:"applied,omitempty"`
Tracked bool `json:"tracked"`
Current bool `json:"current"`
}
// EnvStatus is the fleet's answer to "has every node picked up the current env".
type EnvStatus struct {
AllCurrent bool `json:"all_current"`
Nodes []NodeEnvState `json:"nodes"`
}
// EnvStatusOf compares each live node's applied revision with the env this process renders for it.
func EnvStatusOf(nodes []models.FleetNode) EnvStatus {
out := EnvStatus{AllCurrent: true, Nodes: []NodeEnvState{}}
for _, n := range nodes {
if !n.Live() {
continue
}
want := EnvRevision(RenderEnv(n.ID, n.Role, n.Region))
row := NodeEnvState{ID: n.ID.String(), Name: n.Name, Role: string(n.Role), Applied: n.EnvRevision, Tracked: n.EnvRevision != "", Current: n.EnvRevision == want}
out.AllCurrent = out.AllCurrent && row.Current
out.Nodes = append(out.Nodes, row)
}
return out
}