Files
warmbly/internal/repository/pg_credentials.go
T
Matthew Meszaros 261cc439ad feat(admin): manage worker fleet from dashboard with encrypted credentials and GitHub release auto-update
Workers are no longer curl|sh-only. Admins add and manage them from the
dashboard over SSH, with all runtime config (Kafka, Schema Registry,
Redis, AWS keys) stored encrypted via the existing KMS-envelope cipher
service.

Worker lifecycle:
  1. Admin POSTs host/port/user. Backend generates an ed25519 keypair,
     encrypts the private key under uuid.Nil (platform identity), and
     stores the row in 'pending' state.
  2. Admin pastes the returned public key into the VPS's authorized_keys.
  3. Test connection — runs `true` over SSH, pins the host SHA256
     fingerprint on first success (TOFU).
  4. Install — backend scp's install-worker.sh + a per-worker env file
     and runs it. State moves pending → provisioning → installed.
  5. From then on: restart, update image, apply config, uninstall,
     rotate keys, tail logs, live status, OS package update, reboot —
     all dashboard buttons backed by SSH operations.

Credentials are reusable entities:
  - aws_credentials: named keypair, secret encrypted at rest
  - worker_profiles: bundles Kafka + Schema Registry + Redis + image +
    release channel, references one AWS credentials row
  - workers.profile_id links a worker to a profile; many workers can
    share one profile

Saving a profile doesn't restart anything. The dashboard compares
profile.updated_at to each worker's config_applied_at and shows a
"stale config" badge; Apply rewrites /etc/warmbly/worker.env over SSH
and restarts the unit.

Auto-update on GitHub release:
  - profile.release_channel ∈ {pinned, stable, dev}
  - profile.auto_update toggles automatic rollout
  - Trigger model is push, not poll: one check on backend boot, then
    the /webhooks/github/releases endpoint (HMAC-validated with
    RELEASES_WEBHOOK_SECRET) on every release event. Manual "Check now"
    button as fallback.
  - When a new tag resolves, the orchestrator SSHes into each assigned
    worker, runs install-worker.sh --update --image <new>, which now
    rewrites the systemd unit (not just `docker pull`) so the image
    actually changes. workers.image_version captures the running tag
    for the UI's "v1.2.3 → v1.2.4" diff.

Self-hostable: every release knob is env-driven —
RELEASES_GITHUB_REPO, RELEASES_WORKER_IMAGE_REPO,
RELEASES_WEBHOOK_SECRET, RELEASES_GITHUB_TOKEN, RELEASES_ENABLED. Set
RELEASES_ENABLED=false to disable the feature entirely.

OS-level updates and reboot are also exposed: detect apt / dnf / yum /
pacman / apk, run the right upgrade noninteractively, return the full
output and a reboot-required flag. Reboots are never automatic.

Migrations:
  000028_worker_ssh        — ssh fields, install_state enum, last_seen,
                              host fingerprint
  000029_worker_credentials — aws_credentials + worker_profiles +
                              workers.profile_id + workers.config_applied_at
  000030_worker_releases   — release_channel enum, auto_update,
                              resolved_image_tag, workers.image_version

Endpoints added:
  POST   /admin/workers                        (create + keypair)
  GET    /admin/workers/managed
  GET    /admin/workers/:id/managed
  POST   /admin/workers/:id/{test,install,restart,upgrade,uninstall,rotate-keys,apply,system-update,reboot}
  PUT    /admin/workers/:id/profile
  GET    /admin/workers/:id/{live-status,logs}
  DELETE /admin/workers/:id
  GET    /admin/aws-credentials                CRUD
  GET    /admin/worker-profiles                CRUD + /workers + /apply + /release
  GET    /admin/releases/state
  POST   /admin/releases/check
  POST   /webhooks/github/releases             public, HMAC-validated

Admin UI:
  /app/admin/workers           list with status + version columns
  /app/admin/workers/new       add form with profile dropdown
  /app/admin/workers/:id       detail with all actions + logs + system update
  /app/admin/credentials       tabs: AWS credentials + worker profiles,
                                Releases panel, channel selector +
                                auto-update toggle in profile form
2026-05-18 13:09:11 +00:00

353 lines
12 KiB
Go

package repository
import (
"context"
"errors"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/warmbly/warmbly/internal/models"
)
// AWS credentials
type CredentialsRepository interface {
// AWS credentials
CreateAWSCreds(ctx context.Context, in CreateAWSCredsInput) (uuid.UUID, error)
UpdateAWSCreds(ctx context.Context, id uuid.UUID, in UpdateAWSCredsInput) error
GetAWSCreds(ctx context.Context, id uuid.UUID) (*models.AWSCredentials, error)
ListAWSCreds(ctx context.Context) ([]models.AWSCredentials, error)
DeleteAWSCreds(ctx context.Context, id uuid.UUID) error
// Plain decrypted helpers for the orchestrator — never expose over API.
GetAWSCredsEncryptedSecret(ctx context.Context, id uuid.UUID) (string, error)
// Worker profiles
CreateProfile(ctx context.Context, in CreateProfileInput) (uuid.UUID, error)
UpdateProfile(ctx context.Context, id uuid.UUID, in UpdateProfileInput) error
GetProfile(ctx context.Context, id uuid.UUID) (*models.WorkerProfile, error)
GetProfileEncrypted(ctx context.Context, id uuid.UUID) (*ProfileEncrypted, error)
ListProfiles(ctx context.Context) ([]models.WorkerProfile, error)
DeleteProfile(ctx context.Context, id uuid.UUID) error
// Release channel management
UpdateProfileRelease(ctx context.Context, id uuid.UUID, channel models.ReleaseChannel, autoUpdate bool) error
ListProfilesByChannel(ctx context.Context, channel models.ReleaseChannel) ([]models.WorkerProfile, error)
RecordResolvedTag(ctx context.Context, id uuid.UUID, image, tag string) error
}
type CreateAWSCredsInput struct {
Name string
Description string
Region string
AccessKeyID string
SecretAccessKeyEncrypted string
}
type UpdateAWSCredsInput struct {
Name string
Description string
Region string
AccessKeyID string
// When empty, the existing encrypted secret is kept; otherwise replaced.
SecretAccessKeyEncrypted string
}
type CreateProfileInput struct {
Name string
Description string
AppEnv string
WorkerImage string
KafkaBootstrap string
KafkaSASLUsername string
KafkaSASLPasswordEncrypted string
SchemaRegistryURL string
SchemaRegistryKey string
SchemaRegistrySecretEncrypted string
RedisURLEncrypted string
AWSCredentialID *uuid.UUID
}
// UpdateProfileInput mirrors create. Empty-string encrypted fields mean
// "leave the stored value as-is" (so the UI can submit a partial update
// without re-typing all secrets).
type UpdateProfileInput = CreateProfileInput
// ProfileEncrypted exposes the still-encrypted secret material to the
// orchestrator, which is the only caller that has cipher access.
type ProfileEncrypted struct {
Profile *models.WorkerProfile
KafkaSASLPasswordEncrypted string
SchemaRegistrySecretEncrypted string
RedisURLEncrypted string
}
type credentialsRepository struct {
db *pgxpool.Pool
}
func NewCredentialsRepository(db *pgxpool.Pool) CredentialsRepository {
return &credentialsRepository{db: db}
}
// AWS creds impl
func (r *credentialsRepository) CreateAWSCreds(ctx context.Context, in CreateAWSCredsInput) (uuid.UUID, error) {
id := uuid.New()
_, err := r.db.Exec(ctx, `
INSERT INTO aws_credentials (id, name, description, region, access_key_id, secret_access_key_encrypted)
VALUES ($1, $2, $3, $4, $5, $6)
`, id, in.Name, in.Description, in.Region, in.AccessKeyID, in.SecretAccessKeyEncrypted)
return id, err
}
func (r *credentialsRepository) UpdateAWSCreds(ctx context.Context, id uuid.UUID, in UpdateAWSCredsInput) error {
if in.SecretAccessKeyEncrypted == "" {
_, err := r.db.Exec(ctx, `
UPDATE aws_credentials
SET name=$2, description=$3, region=$4, access_key_id=$5, updated_at=NOW()
WHERE id=$1
`, id, in.Name, in.Description, in.Region, in.AccessKeyID)
return err
}
_, err := r.db.Exec(ctx, `
UPDATE aws_credentials
SET name=$2, description=$3, region=$4, access_key_id=$5,
secret_access_key_encrypted=$6, updated_at=NOW()
WHERE id=$1
`, id, in.Name, in.Description, in.Region, in.AccessKeyID, in.SecretAccessKeyEncrypted)
return err
}
func (r *credentialsRepository) GetAWSCreds(ctx context.Context, id uuid.UUID) (*models.AWSCredentials, error) {
var c models.AWSCredentials
var secret string
err := r.db.QueryRow(ctx, `
SELECT id, name, description, region, access_key_id, secret_access_key_encrypted, created_at, updated_at
FROM aws_credentials WHERE id=$1
`, id).Scan(&c.ID, &c.Name, &c.Description, &c.Region, &c.AccessKeyID, &secret, &c.CreatedAt, &c.UpdatedAt)
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil
}
if err != nil {
return nil, err
}
c.SecretAccessKeyEncrypted = secret
c.HasSecret = secret != ""
return &c, nil
}
func (r *credentialsRepository) ListAWSCreds(ctx context.Context) ([]models.AWSCredentials, error) {
rows, err := r.db.Query(ctx, `
SELECT id, name, description, region, access_key_id, secret_access_key_encrypted, created_at, updated_at
FROM aws_credentials ORDER BY name
`)
if err != nil {
return nil, err
}
defer rows.Close()
out := make([]models.AWSCredentials, 0)
for rows.Next() {
var c models.AWSCredentials
var secret string
if err := rows.Scan(&c.ID, &c.Name, &c.Description, &c.Region, &c.AccessKeyID, &secret, &c.CreatedAt, &c.UpdatedAt); err != nil {
return nil, err
}
c.HasSecret = secret != ""
// Strip ciphertext from list responses to avoid accidentally serializing.
c.SecretAccessKeyEncrypted = ""
out = append(out, c)
}
return out, rows.Err()
}
func (r *credentialsRepository) DeleteAWSCreds(ctx context.Context, id uuid.UUID) error {
_, err := r.db.Exec(ctx, `DELETE FROM aws_credentials WHERE id=$1`, id)
return err
}
func (r *credentialsRepository) GetAWSCredsEncryptedSecret(ctx context.Context, id uuid.UUID) (string, error) {
var s string
err := r.db.QueryRow(ctx, `SELECT secret_access_key_encrypted FROM aws_credentials WHERE id=$1`, id).Scan(&s)
if errors.Is(err, pgx.ErrNoRows) {
return "", nil
}
return s, err
}
// worker profiles impl
func (r *credentialsRepository) CreateProfile(ctx context.Context, in CreateProfileInput) (uuid.UUID, error) {
id := uuid.New()
_, err := r.db.Exec(ctx, `
INSERT INTO worker_profiles (
id, name, description,
app_env, worker_image,
kafka_bootstrap_servers, kafka_sasl_username, kafka_sasl_password_encrypted,
schema_registry_url, schema_registry_key, schema_registry_secret_encrypted,
redis_url_encrypted, aws_credential_id
) VALUES (
$1, $2, $3,
$4, $5,
$6, $7, $8,
$9, $10, $11,
$12, $13
)
`,
id, in.Name, in.Description,
in.AppEnv, in.WorkerImage,
in.KafkaBootstrap, in.KafkaSASLUsername, in.KafkaSASLPasswordEncrypted,
in.SchemaRegistryURL, in.SchemaRegistryKey, in.SchemaRegistrySecretEncrypted,
in.RedisURLEncrypted, in.AWSCredentialID,
)
return id, err
}
func (r *credentialsRepository) UpdateProfile(ctx context.Context, id uuid.UUID, in UpdateProfileInput) error {
// COALESCE-style: empty encrypted fields keep existing values.
_, err := r.db.Exec(ctx, `
UPDATE worker_profiles SET
name=$2, description=$3,
app_env=$4, worker_image=$5,
kafka_bootstrap_servers=$6, kafka_sasl_username=$7,
kafka_sasl_password_encrypted = CASE WHEN $8 = '' THEN kafka_sasl_password_encrypted ELSE $8 END,
schema_registry_url=$9, schema_registry_key=$10,
schema_registry_secret_encrypted = CASE WHEN $11 = '' THEN schema_registry_secret_encrypted ELSE $11 END,
redis_url_encrypted = CASE WHEN $12 = '' THEN redis_url_encrypted ELSE $12 END,
aws_credential_id=$13,
updated_at=NOW()
WHERE id=$1
`,
id, in.Name, in.Description,
in.AppEnv, in.WorkerImage,
in.KafkaBootstrap, in.KafkaSASLUsername, in.KafkaSASLPasswordEncrypted,
in.SchemaRegistryURL, in.SchemaRegistryKey, in.SchemaRegistrySecretEncrypted,
in.RedisURLEncrypted, in.AWSCredentialID,
)
return err
}
func (r *credentialsRepository) scanProfile(row pgx.Row) (*models.WorkerProfile, string, string, string, error) {
var p models.WorkerProfile
var kafkaSecret, schemaSecret, redisURL string
err := row.Scan(
&p.ID, &p.Name, &p.Description,
&p.AppEnv, &p.WorkerImage,
&p.KafkaBootstrapServers, &p.KafkaSASLUsername, &kafkaSecret,
&p.SchemaRegistryURL, &p.SchemaRegistryKey, &schemaSecret,
&redisURL, &p.AWSCredentialID,
&p.ReleaseChannel, &p.AutoUpdate, &p.ResolvedImageTag, &p.LastReleaseCheckAt,
&p.CreatedAt, &p.UpdatedAt,
)
if err != nil {
return nil, "", "", "", err
}
p.HasKafkaPassword = kafkaSecret != ""
p.HasSchemaSecret = schemaSecret != ""
p.HasRedisURL = redisURL != ""
return &p, kafkaSecret, schemaSecret, redisURL, nil
}
const profileColumns = `
id, name, description,
app_env, worker_image,
kafka_bootstrap_servers, kafka_sasl_username, kafka_sasl_password_encrypted,
schema_registry_url, schema_registry_key, schema_registry_secret_encrypted,
redis_url_encrypted, aws_credential_id,
release_channel, auto_update, resolved_image_tag, last_release_check_at,
created_at, updated_at
`
func (r *credentialsRepository) GetProfile(ctx context.Context, id uuid.UUID) (*models.WorkerProfile, error) {
p, _, _, _, err := r.scanProfile(r.db.QueryRow(ctx, `SELECT `+profileColumns+` FROM worker_profiles WHERE id=$1`, id))
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil
}
return p, err
}
func (r *credentialsRepository) GetProfileEncrypted(ctx context.Context, id uuid.UUID) (*ProfileEncrypted, error) {
p, kafkaEnc, schemaEnc, redisEnc, err := r.scanProfile(
r.db.QueryRow(ctx, `SELECT `+profileColumns+` FROM worker_profiles WHERE id=$1`, id),
)
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil
}
if err != nil {
return nil, err
}
return &ProfileEncrypted{
Profile: p,
KafkaSASLPasswordEncrypted: kafkaEnc,
SchemaRegistrySecretEncrypted: schemaEnc,
RedisURLEncrypted: redisEnc,
}, nil
}
func (r *credentialsRepository) ListProfiles(ctx context.Context) ([]models.WorkerProfile, error) {
rows, err := r.db.Query(ctx, `SELECT `+profileColumns+` FROM worker_profiles ORDER BY name`)
if err != nil {
return nil, err
}
defer rows.Close()
out := make([]models.WorkerProfile, 0)
for rows.Next() {
p, _, _, _, err := r.scanProfile(rows)
if err != nil {
return nil, err
}
out = append(out, *p)
}
return out, rows.Err()
}
func (r *credentialsRepository) DeleteProfile(ctx context.Context, id uuid.UUID) error {
_, err := r.db.Exec(ctx, `DELETE FROM worker_profiles WHERE id=$1`, id)
return err
}
func (r *credentialsRepository) UpdateProfileRelease(ctx context.Context, id uuid.UUID, channel models.ReleaseChannel, autoUpdate bool) error {
_, err := r.db.Exec(ctx, `
UPDATE worker_profiles
SET release_channel = $2, auto_update = $3, updated_at = NOW()
WHERE id = $1
`, id, channel, autoUpdate)
return err
}
func (r *credentialsRepository) ListProfilesByChannel(ctx context.Context, channel models.ReleaseChannel) ([]models.WorkerProfile, error) {
rows, err := r.db.Query(ctx, `SELECT `+profileColumns+` FROM worker_profiles WHERE release_channel = $1`, channel)
if err != nil {
return nil, err
}
defer rows.Close()
out := make([]models.WorkerProfile, 0)
for rows.Next() {
p, _, _, _, err := r.scanProfile(rows)
if err != nil {
return nil, err
}
out = append(out, *p)
}
return out, rows.Err()
}
// RecordResolvedTag updates the profile's worker_image and resolved_image_tag
// after the release poller resolves a channel to a concrete tag. Bumps
// updated_at so the UI's "stale config" indicator fires for assigned workers
// that haven't rolled yet.
func (r *credentialsRepository) RecordResolvedTag(ctx context.Context, id uuid.UUID, image, tag string) error {
_, err := r.db.Exec(ctx, `
UPDATE worker_profiles
SET worker_image = $2,
resolved_image_tag = $3,
last_release_check_at = NOW(),
updated_at = NOW()
WHERE id = $1
`, id, image, tag)
return err
}