From c26300ae5e351d703db05b3dca9d5b274d88c7dd Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Mon, 7 Sep 2026 21:40:38 -0700 Subject: [PATCH 1/2] feat: backend half of the admin panel upgrade: delete the unrouted provisioning, releases, plan, discount and enterprise-inquiry admin handlers with their service and repository methods, retire the six admin permission bits nothing gated as reserved placeholders so live bit positions and existing masks are unchanged and IsSuperAdmin checks the live set, add forty admin endpoints for mailbox sync governor state with clear-throttle and restart-backfill, in-flight send reservations, cross-workspace dead letters with replay, task failures, webhook delivery health with reclaim, fleet capacity, the control loops decision log, dedicated bindings with release and the routed convert-to-dedicated, operator-driven workspace export and import, per-organization API keys and webhooks, warmup invalid-token abuse and action history, and signups by acquisition channel, and add a scheduled_job_runs table (migration 000135) with a jobrun package that every backend and consumer loop now records through and a run-now request the owning process picks up within fifteen seconds --- cmd/backend/main.go | 45 +- cmd/consumer/main.go | 5 + internal/api/handler/admin.go | 205 ----- internal/api/handler/admin_credentials.go | 375 +------- internal/api/handler/admin_fleet.go | 178 ++++ internal/api/handler/admin_insight.go | 162 ++++ internal/api/handler/admin_jobs.go | 57 ++ internal/api/handler/admin_provisioning.go | 814 ------------------ internal/api/handler/admin_releases.go | 100 --- internal/api/handler/admin_sends.go | 186 ++++ internal/api/handler/admin_sync.go | 121 +++ internal/api/handler/admin_transfer.go | 260 ++++++ internal/api/handler/handler.go | 14 +- internal/api/middleware/admin.go | 4 +- internal/api/routes.go | 55 +- internal/app/admin/service.go | 240 ------ internal/app/advisor/runner.go | 17 +- internal/app/consumer/auth_check_sweep.go | 21 +- internal/app/consumer/dead_worker.go | 21 +- internal/app/consumer/dlq_retry.go | 32 +- internal/app/consumer/lifecycle_rebalancer.go | 15 +- internal/app/consumer/risk_rebalancer.go | 31 +- internal/app/consumer/stuck_send_reclaimer.go | 19 +- .../app/consumer/warmup_engagement_poller.go | 17 +- internal/app/consumer/warmup_health_sweep.go | 39 +- internal/app/consumer/worker_heartbeat.go | 21 +- internal/app/correlate/correlate.go | 15 +- internal/app/fleet/quarantine.go | 14 +- internal/app/fleet/rebalance.go | 14 +- internal/app/fleet/scale.go | 18 +- internal/app/orgrisk/service.go | 15 +- .../000135_scheduled_job_runs.down.sql | 1 + .../000135_scheduled_job_runs.up.sql | 25 + internal/jobrun/jobrun.go | 152 ++++ internal/jobs/audit_retention.go | 23 +- internal/jobs/danger_zone.go | 22 +- internal/jobs/email_verification.go | 42 +- internal/jobs/form_events_retention.go | 13 +- internal/jobs/forms_domain_sweep.go | 13 +- internal/jobs/guardrail.go | 22 +- internal/jobs/org_transfer.go | 22 +- internal/jobs/placement_poller.go | 19 +- internal/jobs/schedule.go | 17 + internal/jobs/trial_expiration.go | 24 +- internal/jobs/verification_evidence.go | 19 +- internal/jobs/warmup_batch_poller.go | 19 +- internal/jobs/warmup_generation.go | 19 +- internal/jobs/website_tracking_retention.go | 13 +- internal/models/admin.go | 161 ---- internal/models/admin_ops.go | 357 ++++++++ internal/models/admin_permission.go | 58 +- internal/repository/admin_schema_live_test.go | 98 --- internal/repository/pg_admin.go | 645 -------------- internal/repository/pg_admin_fleet.go | 171 ++++ internal/repository/pg_admin_insight.go | 293 +++++++ internal/repository/pg_admin_sends.go | 326 +++++++ internal/repository/pg_admin_sync.go | 211 +++++ internal/repository/pg_job_runs.go | 120 +++ 58 files changed, 2950 insertions(+), 3085 deletions(-) create mode 100644 internal/api/handler/admin_fleet.go create mode 100644 internal/api/handler/admin_insight.go create mode 100644 internal/api/handler/admin_jobs.go delete mode 100644 internal/api/handler/admin_provisioning.go delete mode 100644 internal/api/handler/admin_releases.go create mode 100644 internal/api/handler/admin_sends.go create mode 100644 internal/api/handler/admin_sync.go create mode 100644 internal/api/handler/admin_transfer.go create mode 100644 internal/infrastructure/db/migrations/000135_scheduled_job_runs.down.sql create mode 100644 internal/infrastructure/db/migrations/000135_scheduled_job_runs.up.sql create mode 100644 internal/jobrun/jobrun.go create mode 100644 internal/jobs/schedule.go create mode 100644 internal/models/admin_ops.go create mode 100644 internal/repository/pg_admin_fleet.go create mode 100644 internal/repository/pg_admin_insight.go create mode 100644 internal/repository/pg_admin_sends.go create mode 100644 internal/repository/pg_admin_sync.go create mode 100644 internal/repository/pg_job_runs.go diff --git a/cmd/backend/main.go b/cmd/backend/main.go index 6c34e7f8..c1670f7d 100644 --- a/cmd/backend/main.go +++ b/cmd/backend/main.go @@ -122,6 +122,7 @@ import ( "github.com/warmbly/warmbly/internal/infrastructure/kms" "github.com/warmbly/warmbly/internal/infrastructure/pubsub" "github.com/warmbly/warmbly/internal/infrastructure/storage" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/jobs" "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/notify" @@ -273,6 +274,16 @@ func main() { // Workspace archives (export/import between instances) var orgTransferService orgtransfer.Service + // Admin operations pages and the scheduled job registry. + var ( + adminSyncRepo repository.AdminSyncRepository + adminSendsRepo repository.AdminSendsRepository + adminFleetRepo repository.AdminFleetRepository + adminInsightRepo repository.AdminInsightRepository + jobRunRepo repository.JobRunRepository + webhookRepoForHandler repository.WebhookRepository + ) + // Organization-wide audit trail var auditService audit.AuditService @@ -690,6 +701,16 @@ func main() { webhookRepository := repository.NewWebhookRepository(primaryDB.Pool) webhookService := webhook.NewService(webhookRepository) webhookServiceForHandler = webhookService + webhookRepoForHandler = webhookRepository + + // Every background loop in this process records to scheduled_job_runs + // from here on, and the admin panel can ask any of them to run now. + jobRunRepo = repository.NewJobRunRepository(primaryDB) + jobrun.Configure(jobRunRepo, "backend") + adminSyncRepo = repository.NewAdminSyncRepository(primaryDB) + adminSendsRepo = repository.NewAdminSendsRepository(primaryDB) + adminFleetRepo = repository.NewAdminFleetRepository(primaryDB) + adminInsightRepo = repository.NewAdminInsightRepository(primaryDB) integrationRepository := repository.NewIntegrationRepository(primaryDB.Pool) // OAuth 2.1 authorization server (third-party app registration + token flow). @@ -1032,20 +1053,7 @@ func main() { // rebalance + scale + quarantine evaluators see fresh rolling // metrics. The materialized view is what aggregates the 1h windows // across all workers. - go func() { - tick := time.NewTicker(time.Minute) - defer tick.Stop() - for { - select { - case <-ctx.Done(): - return - case <-tick.C: - if err := workerRepository.RefreshWorkerCapacityView(ctx); err != nil { - log.Printf("worker_capacity_view refresh: %v", err) - } - } - } - }() + go jobrun.Loop(ctx, "worker_capacity_refresh", time.Minute, false, workerRepository.RefreshWorkerCapacityView) go (&fleet.Rebalancer{ WorkerRepo: workerRepository, @@ -2021,7 +2029,6 @@ func main() { WorkerOrchestrator: workerOrchestrator, WorkerRepo: workerRepoForHandler, CredentialsRepo: credentialsRepository, - ReleasesService: releasesService, UpdatesService: updatesService, // Notifications @@ -2100,6 +2107,14 @@ func main() { // Admin System Status probes SystemChecker: systemChecker, + // Admin operations pages. + AdminSyncRepo: adminSyncRepo, + AdminSendsRepo: adminSendsRepo, + AdminFleetRepo: adminFleetRepo, + AdminInsightRepo: adminInsightRepo, + JobRuns: jobRunRepo, + WebhookRepo: webhookRepoForHandler, + // Organization-wide audit trail, backed by Postgres. The no-op // fallback (audit.NewNoOpService) remains for entrypoints without // a database. diff --git a/cmd/consumer/main.go b/cmd/consumer/main.go index 9b8dfff8..e9ee7fd0 100644 --- a/cmd/consumer/main.go +++ b/cmd/consumer/main.go @@ -44,6 +44,7 @@ import ( "github.com/warmbly/warmbly/internal/infrastructure/kms" "github.com/warmbly/warmbly/internal/infrastructure/pubsub" "github.com/warmbly/warmbly/internal/infrastructure/storage" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/notify" "github.com/warmbly/warmbly/internal/observability" @@ -400,6 +401,10 @@ func main() { eventsPublisher := events.NewPublisher(consumerBus, s3Client, consumerCodec, cipherService) + // Every consumer loop records to scheduled_job_runs, so the admin panel + // lists it next to the backend's and can ask it to run now. + jobrun.Configure(repository.NewJobRunRepository(primaryDB), "consumer") + // JobsService jobsService := &jobs.JobsService{ Bus: consumerBus, diff --git a/internal/api/handler/admin.go b/internal/api/handler/admin.go index b6536a30..a122bd05 100644 --- a/internal/api/handler/admin.go +++ b/internal/api/handler/admin.go @@ -682,28 +682,6 @@ func (h *Handler) AdminGetHourlyEmailStats(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"stats": stats}) } -// AdminGetWorkerLoadStats gets worker load statistics -func (h *Handler) AdminGetWorkerLoadStats(c *gin.Context) { - stats, xerr := h.AdminService.GetWorkerLoadStats(c.Request.Context()) - if xerr != nil { - errx.JSON(c, xerr) - return - } - - c.JSON(http.StatusOK, gin.H{"stats": stats}) -} - -// AdminGetEmailDistribution gets email distribution across workers -func (h *Handler) AdminGetEmailDistribution(c *gin.Context) { - dist, xerr := h.AdminService.GetEmailDistribution(c.Request.Context()) - if xerr != nil { - errx.JSON(c, xerr) - return - } - - c.JSON(http.StatusOK, gin.H{"distribution": dist}) -} - // AdminGetUserGrowthStats gets user growth statistics func (h *Handler) AdminGetUserGrowthStats(c *gin.Context) { startDate, endDate := parseDateRange(c) @@ -717,189 +695,6 @@ func (h *Handler) AdminGetUserGrowthStats(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"stats": stats}) } -// Plan Management Handlers - -// AdminListPlans lists the plan catalog with the shared faceted query params; -// returns the standard {data, pagination} envelope. -func (h *Handler) AdminListPlans(c *gin.Context) { - var search models.AdminPlanSearch - if err := c.ShouldBindQuery(&search); err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid query parameters")) - return - } - // duration is durations.title (free text, no DB enum) — validate app-side. - if search.Duration != "" && search.Duration != "month" && search.Duration != "year" { - errx.JSON(c, errx.New(errx.BadRequest, "invalid duration")) - return - } - - result, xerr := h.AdminService.SearchPlansForAdmin(c.Request.Context(), &search) - if xerr != nil { - errx.JSON(c, xerr) - return - } - - c.JSON(http.StatusOK, result) -} - -// AdminCreatePlan creates a new plan -func (h *Handler) AdminCreatePlan(c *gin.Context) { - adminID := middleware.GetAdminUserID(c) - if adminID == nil { - errx.JSON(c, errx.ErrUnauthorized) - return - } - - var req models.CreatePlanRequest - if err := c.ShouldBindJSON(&req); err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid request body")) - return - } - - plan, xerr := h.AdminService.CreatePlan(c.Request.Context(), *adminID, &req, c.ClientIP(), c.GetHeader("User-Agent")) - if xerr != nil { - errx.JSON(c, xerr) - return - } - - c.JSON(http.StatusCreated, plan) -} - -// AdminGetPlan gets a plan by ID -func (h *Handler) AdminGetPlan(c *gin.Context) { - planID, err := uuid.Parse(c.Param("id")) - if err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid plan ID")) - return - } - - plan, xerr := h.AdminService.GetPlan(c.Request.Context(), planID) - if xerr != nil { - errx.JSON(c, xerr) - return - } - - c.JSON(http.StatusOK, plan) -} - -// AdminUpdatePlan updates a plan -func (h *Handler) AdminUpdatePlan(c *gin.Context) { - adminID := middleware.GetAdminUserID(c) - if adminID == nil { - errx.JSON(c, errx.ErrUnauthorized) - return - } - - planID, err := uuid.Parse(c.Param("id")) - if err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid plan ID")) - return - } - - var req models.UpdatePlanRequest - if err := c.ShouldBindJSON(&req); err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid request body")) - return - } - - plan, xerr := h.AdminService.UpdatePlan(c.Request.Context(), *adminID, planID, &req, c.ClientIP(), c.GetHeader("User-Agent")) - if xerr != nil { - errx.JSON(c, xerr) - return - } - - c.JSON(http.StatusOK, plan) -} - -// AdminDeletePlan deletes a plan -func (h *Handler) AdminDeletePlan(c *gin.Context) { - adminID := middleware.GetAdminUserID(c) - if adminID == nil { - errx.JSON(c, errx.ErrUnauthorized) - return - } - - planID, err := uuid.Parse(c.Param("id")) - if err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid plan ID")) - return - } - - xerr := h.AdminService.DeletePlan(c.Request.Context(), *adminID, planID, c.ClientIP(), c.GetHeader("User-Agent")) - if xerr != nil { - errx.JSON(c, xerr) - return - } - - c.JSON(http.StatusOK, gin.H{"message": "plan deleted successfully"}) -} - -// Enterprise Inquiry Handlers - -// AdminListEnterpriseInquiries lists enterprise inquiries with the shared -// faceted query params; returns the standard {data, pagination} envelope. -func (h *Handler) AdminListEnterpriseInquiries(c *gin.Context) { - var search models.AdminEnterpriseInquirySearch - if err := c.ShouldBindQuery(&search); err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid query parameters")) - return - } - - result, xerr := h.AdminService.ListEnterpriseInquiries(c.Request.Context(), &search) - if xerr != nil { - errx.JSON(c, xerr) - return - } - - c.JSON(http.StatusOK, result) -} - -// AdminGetEnterpriseInquiry gets a specific enterprise inquiry -func (h *Handler) AdminGetEnterpriseInquiry(c *gin.Context) { - inquiryID, err := uuid.Parse(c.Param("id")) - if err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid inquiry ID")) - return - } - - inquiry, xerr := h.AdminService.GetEnterpriseInquiry(c.Request.Context(), inquiryID) - if xerr != nil { - errx.JSON(c, xerr) - return - } - - c.JSON(http.StatusOK, inquiry) -} - -// AdminUpdateEnterpriseInquiry updates an enterprise inquiry -func (h *Handler) AdminUpdateEnterpriseInquiry(c *gin.Context) { - adminID := middleware.GetAdminUserID(c) - if adminID == nil { - errx.JSON(c, errx.ErrUnauthorized) - return - } - - inquiryID, err := uuid.Parse(c.Param("id")) - if err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid inquiry ID")) - return - } - - var req models.UpdateEnterpriseInquiryRequest - if err := c.ShouldBindJSON(&req); err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid request body")) - return - } - - xerr := h.AdminService.UpdateEnterpriseInquiry(c.Request.Context(), *adminID, inquiryID, &req, c.ClientIP(), c.GetHeader("User-Agent")) - if xerr != nil { - errx.JSON(c, xerr) - return - } - - c.JSON(http.StatusOK, gin.H{"message": "inquiry updated successfully"}) -} - // Admin Management Handlers // AdminListAdmins lists all admin users diff --git a/internal/api/handler/admin_credentials.go b/internal/api/handler/admin_credentials.go index 4c23a464..5c26ad87 100644 --- a/internal/api/handler/admin_credentials.go +++ b/internal/api/handler/admin_credentials.go @@ -1,324 +1,19 @@ -// Admin endpoints for reusable worker credentials. +// Admin endpoints binding a worker to a reusable worker profile. // -// /admin/aws-credentials list / create -// /admin/aws-credentials/:id get / update / delete -// /admin/worker-profiles list / create -// /admin/worker-profiles/:id get / update / delete -// /admin/worker-profiles/:id/workers list workers using this profile -// /admin/worker-profiles/:id/apply re-write env + restart on every assigned worker -// /admin/workers/:id/profile assign / unassign a profile to a worker -// /admin/workers/:id/apply re-write env + restart for a single worker -// -// Secret material is never returned over the API. Update bodies use empty -// strings to mean "keep the stored value as-is". The dashboard renders set -// secrets as "••••••". +// /admin/workers/:id/profile assign / unassign a profile to a worker +// /admin/workers/:id/apply re-write env + restart for a single worker package handler import ( - "fmt" "net/http" "github.com/gin-gonic/gin" "github.com/google/uuid" "github.com/warmbly/warmbly/internal/errx" "github.com/warmbly/warmbly/internal/models" - "github.com/warmbly/warmbly/internal/repository" ) -// AWS credentials - -type awsCredsBody struct { - Name string `json:"name" binding:"required"` - Description string `json:"description"` - Region string `json:"region" binding:"required"` - AccessKeyID string `json:"access_key_id" binding:"required"` - SecretAccessKey string `json:"secret_access_key"` // empty on update = keep -} - -func (h *Handler) AdminListAWSCreds(c *gin.Context) { - if h.CredentialsRepo == nil { - errx.JSON(c, errx.New(errx.Internal, "credentials repo not configured")) - return - } - creds, err := h.CredentialsRepo.ListAWSCreds(c.Request.Context()) - if err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - c.JSON(http.StatusOK, gin.H{"data": creds}) -} - -func (h *Handler) AdminCreateAWSCreds(c *gin.Context) { - var body awsCredsBody - if err := c.ShouldBindJSON(&body); err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid request body")) - return - } - if body.SecretAccessKey == "" { - errx.JSON(c, errx.New(errx.BadRequest, "secret_access_key is required on create")) - return - } - enc, err := h.WorkerOrchestrator.EncryptSecret(c.Request.Context(), body.SecretAccessKey) - if err != nil { - errx.JSON(c, errx.New(errx.Internal, "encrypt: "+err.Error())) - return - } - id, err := h.CredentialsRepo.CreateAWSCreds(c.Request.Context(), repository.CreateAWSCredsInput{ - Name: body.Name, - Description: body.Description, - Region: body.Region, - AccessKeyID: body.AccessKeyID, - SecretAccessKeyEncrypted: enc, - }) - if err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - h.audit(c, models.AuditActionCreate, models.AuditEntityAWSCredentials, &id, map[string]string{ - "name": body.Name, - "region": body.Region, - }) - c.JSON(http.StatusCreated, gin.H{"id": id}) -} - -func (h *Handler) AdminGetAWSCreds(c *gin.Context) { - id, ok := parseUUID(c, "id") - if !ok { - return - } - creds, err := h.CredentialsRepo.GetAWSCreds(c.Request.Context(), id) - if err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - if creds == nil { - errx.JSON(c, errx.New(errx.NotFound, "credentials not found")) - return - } - // Don't leak ciphertext. - creds.SecretAccessKeyEncrypted = "" - c.JSON(http.StatusOK, creds) -} - -func (h *Handler) AdminUpdateAWSCreds(c *gin.Context) { - id, ok := parseUUID(c, "id") - if !ok { - return - } - var body awsCredsBody - if err := c.ShouldBindJSON(&body); err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid request body")) - return - } - enc := "" - if body.SecretAccessKey != "" { - var err error - enc, err = h.WorkerOrchestrator.EncryptSecret(c.Request.Context(), body.SecretAccessKey) - if err != nil { - errx.JSON(c, errx.New(errx.Internal, "encrypt: "+err.Error())) - return - } - } - if err := h.CredentialsRepo.UpdateAWSCreds(c.Request.Context(), id, repository.UpdateAWSCredsInput{ - Name: body.Name, - Description: body.Description, - Region: body.Region, - AccessKeyID: body.AccessKeyID, - SecretAccessKeyEncrypted: enc, - }); err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - changes := map[string]string{} - if enc != "" { - changes["secret_access_key"] = "rotated" - } - h.audit(c, models.AuditActionUpdate, models.AuditEntityAWSCredentials, &id, changes) - c.JSON(http.StatusOK, gin.H{"ok": true}) -} - -func (h *Handler) AdminDeleteAWSCreds(c *gin.Context) { - id, ok := parseUUID(c, "id") - if !ok { - return - } - if err := h.CredentialsRepo.DeleteAWSCreds(c.Request.Context(), id); err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - h.audit(c, models.AuditActionDelete, models.AuditEntityAWSCredentials, &id, nil) - c.JSON(http.StatusOK, gin.H{"ok": true}) -} - -// worker profiles - -type profileBody struct { - Name string `json:"name" binding:"required"` - Description string `json:"description"` - AppEnv string `json:"app_env"` - WorkerImage string `json:"worker_image"` - KafkaBootstrap string `json:"kafka_bootstrap_servers"` - KafkaSASLUsername string `json:"kafka_sasl_username"` - KafkaSASLPassword string `json:"kafka_sasl_password"` - SchemaRegistryURL string `json:"schema_registry_url"` - SchemaRegistryKey string `json:"schema_registry_key"` - SchemaRegistrySecret string `json:"schema_registry_secret"` - RedisURL string `json:"redis_url"` - AWSCredentialID *string `json:"aws_credential_id"` -} - -func (h *Handler) AdminListProfiles(c *gin.Context) { - if h.CredentialsRepo == nil { - errx.JSON(c, errx.New(errx.Internal, "credentials repo not configured")) - return - } - profiles, err := h.CredentialsRepo.ListProfiles(c.Request.Context()) - if err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - c.JSON(http.StatusOK, gin.H{"data": profiles}) -} - -func (h *Handler) AdminCreateProfile(c *gin.Context) { - var body profileBody - if err := c.ShouldBindJSON(&body); err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid request body")) - return - } - in, xerr := h.profileBodyToInput(c, body) - if xerr { - return - } - id, err := h.CredentialsRepo.CreateProfile(c.Request.Context(), in) - if err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - h.audit(c, models.AuditActionCreate, models.AuditEntityWorkerProfile, &id, map[string]string{ - "name": body.Name, - "app_env": body.AppEnv, - }) - c.JSON(http.StatusCreated, gin.H{"id": id}) -} - -func (h *Handler) AdminGetProfile(c *gin.Context) { - id, ok := parseUUID(c, "id") - if !ok { - return - } - p, err := h.CredentialsRepo.GetProfile(c.Request.Context(), id) - if err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - if p == nil { - errx.JSON(c, errx.New(errx.NotFound, "profile not found")) - return - } - c.JSON(http.StatusOK, p) -} - -func (h *Handler) AdminUpdateProfile(c *gin.Context) { - id, ok := parseUUID(c, "id") - if !ok { - return - } - var body profileBody - if err := c.ShouldBindJSON(&body); err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid request body")) - return - } - in, xerr := h.profileBodyToInput(c, body) - if xerr { - return - } - if err := h.CredentialsRepo.UpdateProfile(c.Request.Context(), id, in); err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - // Record which secret fields were rotated (we don't log the values, only that they changed). - changes := map[string]string{} - if body.KafkaSASLPassword != "" { - changes["kafka_sasl_password"] = "rotated" - } - if body.SchemaRegistrySecret != "" { - changes["schema_registry_secret"] = "rotated" - } - if body.RedisURL != "" { - changes["redis_url"] = "rotated" - } - h.audit(c, models.AuditActionUpdate, models.AuditEntityWorkerProfile, &id, changes) - c.JSON(http.StatusOK, gin.H{"ok": true}) -} - -func (h *Handler) AdminDeleteProfile(c *gin.Context) { - id, ok := parseUUID(c, "id") - if !ok { - return - } - if err := h.CredentialsRepo.DeleteProfile(c.Request.Context(), id); err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - h.audit(c, models.AuditActionDelete, models.AuditEntityWorkerProfile, &id, nil) - c.JSON(http.StatusOK, gin.H{"ok": true}) -} - -func (h *Handler) AdminListProfileWorkers(c *gin.Context) { - id, ok := parseUUID(c, "id") - if !ok { - return - } - workers, err := h.WorkerRepo.ListWorkersByProfile(c.Request.Context(), id) - if err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - c.JSON(http.StatusOK, gin.H{"data": workers}) -} - -// AdminApplyProfile re-applies the profile to every assigned worker by -// re-writing /etc/warmbly/worker.env and restarting the service. -// -// Reports a per-worker outcome map so the UI can show what succeeded. -func (h *Handler) AdminApplyProfile(c *gin.Context) { - id, ok := parseUUID(c, "id") - if !ok { - return - } - workers, err := h.WorkerRepo.ListWorkersByProfile(c.Request.Context(), id) - if err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - results := make([]gin.H, 0, len(workers)) - for _, w := range workers { - // Skip workers that aren't installed yet — apply is restart-based. - if w.InstallState != models.WorkerInstallStateInstalled { - results = append(results, gin.H{"worker_id": w.ID, "ok": false, "skipped": "not installed"}) - continue - } - applyErr := h.WorkerOrchestrator.ApplyConfig(c.Request.Context(), w.ID) - r := gin.H{"worker_id": w.ID, "ok": applyErr == nil} - if applyErr != nil { - r["error"] = applyErr.Error() - } - results = append(results, r) - } - okCount := 0 - for _, r := range results { - if v, _ := r["ok"].(bool); v { - okCount++ - } - } - h.audit(c, models.AuditActionApply, models.AuditEntityWorkerProfile, &id, map[string]string{ - "workers_applied": fmt.Sprintf("%d/%d", okCount, len(results)), - }) - c.JSON(http.StatusOK, gin.H{"results": results}) -} - // worker → profile binding type assignProfileBody struct { @@ -374,70 +69,6 @@ func (h *Handler) AdminApplyWorkerConfig(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"ok": true}) } -// helpers - -func (h *Handler) profileBodyToInput(c *gin.Context, body profileBody) (repository.CreateProfileInput, bool) { - ctx := c.Request.Context() - encrypt := func(s string) (string, bool) { - if s == "" { - return "", true - } - enc, err := h.WorkerOrchestrator.EncryptSecret(ctx, s) - if err != nil { - errx.JSON(c, errx.New(errx.Internal, "encrypt: "+err.Error())) - return "", false - } - return enc, true - } - - kafkaEnc, ok := encrypt(body.KafkaSASLPassword) - if !ok { - return repository.CreateProfileInput{}, true - } - schemaEnc, ok := encrypt(body.SchemaRegistrySecret) - if !ok { - return repository.CreateProfileInput{}, true - } - redisEnc, ok := encrypt(body.RedisURL) - if !ok { - return repository.CreateProfileInput{}, true - } - - var awsID *uuid.UUID - if body.AWSCredentialID != nil && *body.AWSCredentialID != "" { - parsed, err := uuid.Parse(*body.AWSCredentialID) - if err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid aws_credential_id")) - return repository.CreateProfileInput{}, true - } - awsID = &parsed - } - - appEnv := body.AppEnv - if appEnv == "" { - appEnv = "prod" - } - image := body.WorkerImage - if image == "" { - image = "ghcr.io/warmbly/worker:latest" - } - - return repository.CreateProfileInput{ - Name: body.Name, - Description: body.Description, - AppEnv: appEnv, - WorkerImage: image, - KafkaBootstrap: body.KafkaBootstrap, - KafkaSASLUsername: body.KafkaSASLUsername, - KafkaSASLPasswordEncrypted: kafkaEnc, - SchemaRegistryURL: body.SchemaRegistryURL, - SchemaRegistryKey: body.SchemaRegistryKey, - SchemaRegistrySecretEncrypted: schemaEnc, - RedisURLEncrypted: redisEnc, - AWSCredentialID: awsID, - }, false -} - func parseUUID(c *gin.Context, param string) (uuid.UUID, bool) { id, err := uuid.Parse(c.Param(param)) if err != nil { diff --git a/internal/api/handler/admin_fleet.go b/internal/api/handler/admin_fleet.go new file mode 100644 index 00000000..06ab38db --- /dev/null +++ b/internal/api/handler/admin_fleet.go @@ -0,0 +1,178 @@ +package handler + +import ( + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" +) + +// Fleet placement as the operator sees it: every worker against its capacity +// row, the decision log the control loops write, and the dedicated bindings. + +// AdminFleetCapacity lists every worker with its capacity-view row, hottest first. +// +// GET /admin/fleet/capacity +func (h *Handler) AdminFleetCapacity(c *gin.Context) { + if h.AdminFleetRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "fleet view is not available on this instance")) + return + } + rows, err := h.AdminFleetRepo.Capacity(c.Request.Context()) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + c.JSON(http.StatusOK, gin.H{"data": rows}) +} + +// AdminFleetDecisions lists decision_log newest first. +// +// GET /admin/fleet/decisions?kind=&worker_id=&limit= +func (h *Handler) AdminFleetDecisions(c *gin.Context) { + if h.AdminFleetRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "fleet view is not available on this instance")) + return + } + var workerID *uuid.UUID + if raw := c.Query("worker_id"); raw != "" { + id, err := uuid.Parse(raw) + if err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid worker_id")) + return + } + workerID = &id + } + limit := 0 + if raw := c.Query("limit"); raw != "" { + n, err := strconv.Atoi(raw) + if err != nil || n <= 0 { + errx.JSON(c, errx.New(errx.BadRequest, "invalid limit")) + return + } + limit = n + } + rows, err := h.AdminFleetRepo.Decisions(c.Request.Context(), c.Query("kind"), workerID, limit) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + c.JSON(http.StatusOK, gin.H{"data": rows}) +} + +// AdminFleetDedicated lists active worker-to-organization bindings. +// +// GET /admin/fleet/dedicated +func (h *Handler) AdminFleetDedicated(c *gin.Context) { + if h.AdminFleetRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "fleet view is not available on this instance")) + return + } + rows, err := h.AdminFleetRepo.DedicatedAssignments(c.Request.Context()) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + c.JSON(http.StatusOK, gin.H{"data": rows}) +} + +// AdminFleetReleaseDedicated is the inverse of AdminConvertWorkerToDedicated: +// the org's mailboxes go back to shared premium workers, the binding is +// released, and the worker re-enters the shared pool once nothing binds it. +// +// POST /admin/fleet/dedicated/:orgId/release +func (h *Handler) AdminFleetReleaseDedicated(c *gin.Context) { + orgID, ok := parseUUIDParam(c, "orgId") + if !ok { + return + } + if h.WorkerAssignmentService == nil || h.WorkerRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "worker placement is not available on this instance")) + return + } + ctx := c.Request.Context() + + assignment, err := h.WorkerRepo.GetActiveDedicatedAssignment(ctx, orgID) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + if assignment == nil { + errx.JSON(c, errx.New(errx.NotFound, "organization has no active dedicated worker")) + return + } + workerID := assignment.WorkerID + + before, err := h.WorkerRepo.GetEmailAccountsByWorkerID(ctx, workerID) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, "list accounts: "+err.Error())) + return + } + + // Moves every org mailbox onto a live shared premium worker and releases + // the binding; a mailbox with no shared target stays put rather than failing. + if err := h.WorkerAssignmentService.MigrateOrgToShared(ctx, orgID); err != nil { + errx.JSON(c, errx.New(errx.Internal, "migrate to shared: "+err.Error())) + return + } + // MigrateOrgToShared swallows its own release error, so release again; the + // UPDATE is a no-op when the row is already released. + if err := h.WorkerAssignmentService.ReleaseDedicatedWorker(ctx, orgID); err != nil { + errx.JSON(c, errx.New(errx.Internal, "release assignment: "+err.Error())) + return + } + + after, err := h.WorkerRepo.GetEmailAccountsByWorkerID(ctx, workerID) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, "list accounts: "+err.Error())) + return + } + moved := len(before) - len(after) + if moved < 0 { + moved = 0 + } + + // The worker only returns to the shared pool when no other org binds it. + stillBound := false + if h.AdminFleetRepo != nil { + active, err := h.AdminFleetRepo.DedicatedAssignments(ctx) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, "list assignments: "+err.Error())) + return + } + for _, a := range active { + if a.WorkerID == workerID { + stillBound = true + break + } + } + } + returnedToShared := false + if !stillBound { + if err := h.WorkerRepo.SetWorkerType(ctx, workerID, models.WorkerTypeShared); err != nil { + errx.JSON(c, errx.New(errx.Internal, "set type: "+err.Error())) + return + } + returnedToShared = true + } + + h.audit(c, "release_dedicated", models.AuditEntityWorker, &workerID, map[string]string{ + "organization_id": orgID.String(), + "subscription_id": assignment.SubscriptionID.String(), + "assignment_id": assignment.ID.String(), + "accounts_moved": itoa(moved), + "accounts_remaining": itoa(len(after)), + "returned_to_shared": boolStr(returnedToShared), + }) + c.JSON(http.StatusOK, gin.H{ + "ok": true, + "worker_id": workerID, + "accounts_moved": moved, + "accounts_remaining": len(after), + "returned_to_shared": returnedToShared, + }) +} diff --git a/internal/api/handler/admin_insight.go b/internal/api/handler/admin_insight.go new file mode 100644 index 00000000..0d1da919 --- /dev/null +++ b/internal/api/handler/admin_insight.go @@ -0,0 +1,162 @@ +package handler + +import ( + "net/http" + "strconv" + "time" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" +) + +const adminInsightUnavailable = "this insight is not available on this instance" + +// queryInt reads an integer query parameter, clamping it to [min, max]. +func queryInt(c *gin.Context, key string, def, min, max int) int { + v := def + if raw := c.Query(key); raw != "" { + if n, err := strconv.Atoi(raw); err == nil { + v = n + } + } + if v < min { + v = min + } + if v > max { + v = max + } + return v +} + +// AdminGetAcquisition is GET /admin/analytics/acquisition?days=30. +func (h *Handler) AdminGetAcquisition(c *gin.Context) { + if h.AdminInsightRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, adminInsightUnavailable)) + return + } + days := queryInt(c, "days", 30, 1, 365) + out, err := h.AdminInsightRepo.Acquisition(c.Request.Context(), days) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, "failed to load acquisition")) + return + } + c.JSON(http.StatusOK, out) +} + +// AdminWarmupAbuse is GET /admin/warmup/abuse?window=24h|7d|30d&limit=50. +func (h *Handler) AdminWarmupAbuse(c *gin.Context) { + if h.AdminInsightRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, adminInsightUnavailable)) + return + } + var window time.Duration + switch c.DefaultQuery("window", "7d") { + case "24h": + window = 24 * time.Hour + case "7d": + window = 7 * 24 * time.Hour + case "30d": + window = 30 * 24 * time.Hour + default: + errx.JSON(c, errx.New(errx.BadRequest, "window must be 24h, 7d or 30d")) + return + } + limit := queryInt(c, "limit", 50, 1, 200) + rows, err := h.AdminInsightRepo.WarmupAbuse(c.Request.Context(), time.Now().Add(-window), limit) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, "failed to load warmup abuse signals")) + return + } + c.JSON(http.StatusOK, gin.H{"data": rows}) +} + +// AdminWarmupActions is GET /admin/warmup/actions?limit=100. +func (h *Handler) AdminWarmupActions(c *gin.Context) { + if h.AdminInsightRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, adminInsightUnavailable)) + return + } + limit := queryInt(c, "limit", 100, 1, 500) + rows, err := h.AdminInsightRepo.WarmupActions(c.Request.Context(), limit) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, "failed to load warmup actions")) + return + } + c.JSON(http.StatusOK, gin.H{"data": rows}) +} + +// AdminListOrgAPIKeys is GET /admin/organizations/:id/api-keys. +func (h *Handler) AdminListOrgAPIKeys(c *gin.Context) { + if h.AdminInsightRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, adminInsightUnavailable)) + return + } + orgID, err := uuid.Parse(c.Param("id")) + if err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid organization ID")) + return + } + rows, err := h.AdminInsightRepo.OrgAPIKeys(c.Request.Context(), orgID) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, "failed to load API keys")) + return + } + c.JSON(http.StatusOK, gin.H{"data": rows}) +} + +// AdminRevokeOrgAPIKey is DELETE /admin/organizations/:id/api-keys/:keyId. +func (h *Handler) AdminRevokeOrgAPIKey(c *gin.Context) { + if h.APIKeyService == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "API keys are not available on this instance")) + return + } + orgID, err := uuid.Parse(c.Param("id")) + if err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid organization ID")) + return + } + keyID, err := uuid.Parse(c.Param("keyId")) + if err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid API key ID")) + return + } + var body struct { + Reason string `json:"reason"` + } + if c.Request.ContentLength != 0 { + if err := c.ShouldBindJSON(&body); err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid request body")) + return + } + } + reason := body.Reason + if reason == "" { + reason = "revoked by platform admin" + } + if xerr := h.APIKeyService.Revoke(c.Request.Context(), orgID, keyID, reason); xerr != nil { + errx.JSON(c, xerr) + return + } + h.audit(c, models.AuditActionRevoke, models.AuditEntityAPIKey, &keyID, map[string]string{ + "organization_id": orgID.String(), + "reason": reason, + }) + c.JSON(http.StatusOK, gin.H{"id": keyID, "organization_id": orgID, "status": models.APIKeyStatusRevoked}) +} + +// AdminListTransfers is GET /admin/transfers?limit=100. +func (h *Handler) AdminListTransfers(c *gin.Context) { + if h.AdminInsightRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, adminInsightUnavailable)) + return + } + limit := queryInt(c, "limit", 100, 1, 500) + rows, err := h.AdminInsightRepo.ListTransfers(c.Request.Context(), limit) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, "failed to load transfers")) + return + } + c.JSON(http.StatusOK, gin.H{"data": rows}) +} diff --git a/internal/api/handler/admin_jobs.go b/internal/api/handler/admin_jobs.go new file mode 100644 index 00000000..b74665c4 --- /dev/null +++ b/internal/api/handler/admin_jobs.go @@ -0,0 +1,57 @@ +// Admin endpoints for the scheduled jobs registry: every background loop the +// backend and consumer run, with its last run, and a "run now" request the +// owning loop picks up on its next poll. + +package handler + +import ( + "net/http" + + "github.com/gin-gonic/gin" + + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" +) + +const scheduledJobEntity models.AuditEntityType = "scheduled_job" + +// AdminListJobs lists every registered background loop. +func (h *Handler) AdminListJobs(c *gin.Context) { + if h.JobRuns == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "job registry is not available on this instance")) + return + } + jobs, err := h.JobRuns.List(c.Request.Context()) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + if jobs == nil { + jobs = []models.ScheduledJobRun{} + } + c.JSON(http.StatusOK, gin.H{"data": jobs}) +} + +// AdminRunJob asks the loop that owns the job to run at its next poll. +func (h *Handler) AdminRunJob(c *gin.Context) { + if h.JobRuns == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "job registry is not available on this instance")) + return + } + name := c.Param("name") + if name == "" { + errx.JSON(c, errx.New(errx.BadRequest, "job name is required")) + return + } + found, err := h.JobRuns.RequestRun(c.Request.Context(), name) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + if !found { + errx.JSON(c, errx.New(errx.NotFound, "job not found")) + return + } + h.audit(c, models.AuditActionStart, scheduledJobEntity, nil, map[string]string{"job": name}) + c.JSON(http.StatusOK, gin.H{"requested": true}) +} diff --git a/internal/api/handler/admin_provisioning.go b/internal/api/handler/admin_provisioning.go deleted file mode 100644 index 0aa2353c..00000000 --- a/internal/api/handler/admin_provisioning.go +++ /dev/null @@ -1,814 +0,0 @@ -package handler - -import ( - "context" - "encoding/json" - "errors" - "net/http" - "strconv" - "time" - - "github.com/gin-gonic/gin" - "github.com/google/uuid" - "github.com/warmbly/warmbly/internal/infrastructure/cloudprovider" - "github.com/warmbly/warmbly/internal/infrastructure/cloudprovider/hetzner" - "github.com/warmbly/warmbly/internal/models" - "github.com/warmbly/warmbly/internal/repository" -) - -// Admin endpoints under /admin for autonomous fleet management: -// -// /admin/cloud-credentials CRUD + test -// /admin/cloud-providers/:provider/... discovery (locations, server_types, images) -// /admin/provisioning-templates CRUD -// /admin/provisioning-jobs list + create + detail -// /admin/provisioning-policy per-provider budget caps -// -// All gated by AdminPermManageSettings via the route registration. - -// --------------------------------------------------------------------------- -// Cloud credentials -// --------------------------------------------------------------------------- - -type CloudCredentialResponse struct { - ID uuid.UUID `json:"id"` - Provider string `json:"provider"` - Name string `json:"name"` - TokenRedacted string `json:"token_redacted"` - LastUsedAt *time.Time `json:"last_used_at,omitempty"` - LastTestAt *time.Time `json:"last_test_at,omitempty"` - LastTestOK *bool `json:"last_test_ok,omitempty"` - LastTestError *string `json:"last_test_error,omitempty"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` -} - -func toCredResponse(c *repository.CloudCredential) CloudCredentialResponse { - return CloudCredentialResponse{ - ID: c.ID, - Provider: c.Provider, - Name: c.Name, - TokenRedacted: maskToken(c.EncryptedToken), - LastUsedAt: c.LastUsedAt, - LastTestAt: c.LastTestAt, - LastTestOK: c.LastTestOK, - LastTestError: c.LastTestError, - CreatedAt: c.CreatedAt, - UpdatedAt: c.UpdatedAt, - } -} - -func maskToken(t string) string { - if len(t) <= 8 { - return "***" - } - return t[:4] + "***" + t[len(t)-4:] -} - -func (h *Handler) AdminListCloudCredentials(c *gin.Context) { - if h.CloudCredentialRepo == nil { - c.JSON(http.StatusOK, gin.H{"data": []any{}}) - return - } - rows, err := h.CloudCredentialRepo.List(c.Request.Context()) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - out := make([]CloudCredentialResponse, 0, len(rows)) - for _, r := range rows { - out = append(out, toCredResponse(&r)) - } - c.JSON(http.StatusOK, gin.H{"data": out}) -} - -type CreateCloudCredentialRequest struct { - Provider string `json:"provider"` - Name string `json:"name"` - Token string `json:"token"` -} - -func (h *Handler) AdminCreateCloudCredential(c *gin.Context) { - if h.CloudCredentialRepo == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "cloud credentials repo not configured"}) - return - } - var req CreateCloudCredentialRequest - if err := c.ShouldBindJSON(&req); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid body"}) - return - } - if req.Provider == "" || req.Token == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "provider and token required"}) - return - } - if req.Name == "" { - req.Name = req.Provider + "-default" - } - - // TODO: cipher-encrypt the token via h.CipherService. For now we store - // as-is so the wiring works end-to-end; flag this in audit log. - row := &repository.CloudCredential{ - Provider: req.Provider, - Name: req.Name, - EncryptedToken: req.Token, - } - if err := h.CloudCredentialRepo.Create(c.Request.Context(), row); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - c.JSON(http.StatusCreated, toCredResponse(row)) -} - -func (h *Handler) AdminDeleteCloudCredential(c *gin.Context) { - if h.CloudCredentialRepo == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "cloud credentials repo not configured"}) - return - } - id, err := uuid.Parse(c.Param("id")) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"}) - return - } - if err := h.CloudCredentialRepo.Delete(c.Request.Context(), id); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - c.Status(http.StatusNoContent) -} - -func (h *Handler) AdminTestCloudCredential(c *gin.Context) { - if h.CloudCredentialRepo == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "cloud credentials repo not configured"}) - return - } - id, err := uuid.Parse(c.Param("id")) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"}) - return - } - cred, err := h.CloudCredentialRepo.Get(c.Request.Context(), id) - if err != nil || cred == nil { - c.JSON(http.StatusNotFound, gin.H{"error": "credential not found"}) - return - } - provider, err := buildProvider(cred) - if err != nil { - _ = h.CloudCredentialRepo.UpdateTestResult(c.Request.Context(), id, false, err.Error()) - c.JSON(http.StatusOK, gin.H{"ok": false, "error": err.Error()}) - return - } - ctx, cancel := context.WithTimeout(c.Request.Context(), 15*time.Second) - defer cancel() - if err := provider.Verify(ctx); err != nil { - _ = h.CloudCredentialRepo.UpdateTestResult(ctx, id, false, err.Error()) - c.JSON(http.StatusOK, gin.H{"ok": false, "error": err.Error()}) - return - } - _ = h.CloudCredentialRepo.UpdateTestResult(ctx, id, true, "") - c.JSON(http.StatusOK, gin.H{"ok": true}) -} - -// buildProvider picks the right cloudprovider.Provider implementation for a -// credential row. Today only Hetzner; adding OVH/Vultr means another case. -func buildProvider(c *repository.CloudCredential) (cloudprovider.Provider, error) { - switch c.Provider { - case "hetzner": - return hetzner.New(c.EncryptedToken) - default: - return nil, errors.New("unsupported provider: " + c.Provider) - } -} - -// --------------------------------------------------------------------------- -// Provider catalog discovery (used by admin UI dropdowns) -// --------------------------------------------------------------------------- - -func (h *Handler) AdminListProviderLocations(c *gin.Context) { - provider, err := h.providerByName(c) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - return - } - ctx, cancel := context.WithTimeout(c.Request.Context(), 15*time.Second) - defer cancel() - locs, err := provider.Locations(ctx) - if err != nil { - c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) - return - } - c.JSON(http.StatusOK, gin.H{"data": locs}) -} - -func (h *Handler) AdminListProviderServerTypes(c *gin.Context) { - provider, err := h.providerByName(c) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - return - } - ctx, cancel := context.WithTimeout(c.Request.Context(), 15*time.Second) - defer cancel() - types, err := provider.ServerTypes(ctx) - if err != nil { - c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) - return - } - c.JSON(http.StatusOK, gin.H{"data": types}) -} - -func (h *Handler) AdminListProviderImages(c *gin.Context) { - provider, err := h.providerByName(c) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - return - } - ctx, cancel := context.WithTimeout(c.Request.Context(), 15*time.Second) - defer cancel() - images, err := provider.Images(ctx) - if err != nil { - c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) - return - } - c.JSON(http.StatusOK, gin.H{"data": images}) -} - -// providerByName resolves a provider name to a Provider client, looking up -// the most recent credential row for that provider. -func (h *Handler) providerByName(c *gin.Context) (cloudprovider.Provider, error) { - if h.CloudCredentialRepo == nil { - return nil, errors.New("cloud credentials not configured") - } - name := c.Param("provider") - if name == "" { - return nil, errors.New("provider path param required") - } - cred, err := h.CloudCredentialRepo.GetByProvider(c.Request.Context(), name) - if err != nil { - return nil, err - } - if cred == nil { - return nil, errors.New("no credential configured for provider " + name) - } - return buildProvider(cred) -} - -// --------------------------------------------------------------------------- -// Provisioning templates -// -// The admin UI works in a nested {name, description, config:{...}, -// auto_provision_tier, is_draft} shape; the repository row is flat. The DTO -// helpers below translate between the two so the two sides agree on the wire -// format (they did not previously, which silently 400'd every template save). -// --------------------------------------------------------------------------- - -type provLabelDTO struct { - Key string `json:"key"` - Value string `json:"value"` -} - -type provTemplateConfigDTO struct { - Provider string `json:"provider"` - CredentialID *uuid.UUID `json:"credential_id,omitempty"` - Location string `json:"location"` - ServerType string `json:"server_type"` - ServerCount int `json:"server_count"` - IPv4PerServer int `json:"ipv4_per_server"` - IPv6PerServer int `json:"ipv6_per_server"` - WorkerTier string `json:"worker_tier"` - WorkerProfileID *uuid.UUID `json:"worker_profile_id,omitempty"` - EgressKind string `json:"egress_kind"` - Image string `json:"image"` - Datacenter string `json:"datacenter,omitempty"` - PlacementGroup string `json:"placement_group,omitempty"` - PrivateNetwork string `json:"private_network,omitempty"` - Firewall string `json:"firewall"` - Labels []provLabelDTO `json:"labels"` -} - -type provTemplateDTO struct { - ID uuid.UUID `json:"id,omitempty"` - Name string `json:"name"` - Description string `json:"description,omitempty"` - Config provTemplateConfigDTO `json:"config"` - AutoProvisionTier string `json:"auto_provision_tier,omitempty"` - IsDraft bool `json:"is_draft"` - CreatedAt time.Time `json:"created_at,omitempty"` - UpdatedAt time.Time `json:"updated_at,omitempty"` -} - -func labelsToDTO(m map[string]string) []provLabelDTO { - out := make([]provLabelDTO, 0, len(m)) - for k, v := range m { - out = append(out, provLabelDTO{Key: k, Value: v}) - } - return out -} - -func labelsFromDTO(in []provLabelDTO) map[string]string { - m := map[string]string{} - for _, l := range in { - if l.Key != "" { - m[l.Key] = l.Value - } - } - return m -} - -func toTemplateDTO(t *repository.ProvisioningTemplate) provTemplateDTO { - auto := "" - if t.IsAutoTemplate { - auto = t.Tier - } - return provTemplateDTO{ - ID: t.ID, - Name: t.Name, - Description: t.Description, - Config: provTemplateConfigDTO{ - Provider: t.Provider, - Location: t.Location, - ServerType: t.ServerType, - ServerCount: t.ServerCount, - IPv4PerServer: t.IPv4PerServer, - IPv6PerServer: t.IPv6PerServer, - WorkerTier: t.Tier, - WorkerProfileID: t.WorkerProfileID, - EgressKind: t.EgressKind, - Image: t.Image, - Datacenter: t.Datacenter, - PlacementGroup: t.PlacementGroup, - PrivateNetwork: t.PrivateNetwork, - Firewall: t.Firewall, - Labels: labelsToDTO(t.Labels), - }, - AutoProvisionTier: auto, - IsDraft: t.IsDraft, - CreatedAt: t.CreatedAt, - UpdatedAt: t.UpdatedAt, - } -} - -// fromTemplateDTO maps the UI shape onto the flat repo model and applies the -// field defaults the create path used to apply inline. -func fromTemplateDTO(d *provTemplateDTO) *repository.ProvisioningTemplate { - cfg := d.Config - t := &repository.ProvisioningTemplate{ - ID: d.ID, - Name: d.Name, - Description: d.Description, - Provider: cfg.Provider, - Location: cfg.Location, - Datacenter: cfg.Datacenter, - ServerType: cfg.ServerType, - Image: cfg.Image, - ServerCount: cfg.ServerCount, - IPv4PerServer: cfg.IPv4PerServer, - IPv6PerServer: cfg.IPv6PerServer, - WorkerProfileID: cfg.WorkerProfileID, - Tier: cfg.WorkerTier, - EgressKind: cfg.EgressKind, - Labels: labelsFromDTO(cfg.Labels), - PlacementGroup: cfg.PlacementGroup, - PrivateNetwork: cfg.PrivateNetwork, - Firewall: cfg.Firewall, - IsDraft: d.IsDraft, - // A draft is never eligible as the tier's auto-provision template. - IsAutoTemplate: !d.IsDraft && d.AutoProvisionTier != "", - } - if t.Image == "" { - t.Image = "ubuntu-22.04" - } - if t.ServerCount == 0 { - t.ServerCount = 1 - } - if t.IPv4PerServer == 0 { - t.IPv4PerServer = 1 - } - if t.IPv6PerServer == 0 { - t.IPv6PerServer = 1 - } - if t.EgressKind == "" { - t.EgressKind = "cold_smtp" - } - return t -} - -func (h *Handler) AdminListProvisioningTemplates(c *gin.Context) { - if h.ProvisioningTemplateRepo == nil { - c.JSON(http.StatusOK, gin.H{"data": []any{}}) - return - } - rows, err := h.ProvisioningTemplateRepo.List(c.Request.Context()) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - out := make([]provTemplateDTO, 0, len(rows)) - for i := range rows { - out = append(out, toTemplateDTO(&rows[i])) - } - c.JSON(http.StatusOK, gin.H{"data": out}) -} - -func (h *Handler) AdminGetProvisioningTemplate(c *gin.Context) { - id, err := uuid.Parse(c.Param("id")) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"}) - return - } - row, err := h.ProvisioningTemplateRepo.Get(c.Request.Context(), id) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - if row == nil { - c.Status(http.StatusNotFound) - return - } - c.JSON(http.StatusOK, toTemplateDTO(row)) -} - -func (h *Handler) AdminCreateProvisioningTemplate(c *gin.Context) { - if h.ProvisioningTemplateRepo == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "template repo not configured"}) - return - } - var d provTemplateDTO - if err := c.ShouldBindJSON(&d); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid body: " + err.Error()}) - return - } - t := fromTemplateDTO(&d) - if t.Name == "" || t.Provider == "" || t.Location == "" || t.ServerType == "" || t.Tier == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "name, provider, location, server_type and worker_tier are required"}) - return - } - if !repository.IsClientRequestableTier(t.Tier) { - c.JSON(http.StatusBadRequest, gin.H{ - "error": "dedicated tier templates cannot be created; dedicated workers are allocated automatically by the control plane", - "code": "tier_not_allowed", - }) - return - } - if err := h.ProvisioningTemplateRepo.Create(c.Request.Context(), t); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - c.JSON(http.StatusCreated, toTemplateDTO(t)) -} - -func (h *Handler) AdminUpdateProvisioningTemplate(c *gin.Context) { - id, err := uuid.Parse(c.Param("id")) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"}) - return - } - var d provTemplateDTO - if err := c.ShouldBindJSON(&d); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid body"}) - return - } - t := fromTemplateDTO(&d) - t.ID = id - if t.Name == "" || t.Provider == "" || t.Location == "" || t.ServerType == "" || t.Tier == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "name, provider, location, server_type and worker_tier are required"}) - return - } - if !repository.IsClientRequestableTier(t.Tier) { - c.JSON(http.StatusBadRequest, gin.H{ - "error": "templates cannot be set to the dedicated tier; dedicated workers are allocated automatically by the control plane", - "code": "tier_not_allowed", - }) - return - } - if err := h.ProvisioningTemplateRepo.Update(c.Request.Context(), t); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - c.JSON(http.StatusOK, toTemplateDTO(t)) -} - -func (h *Handler) AdminDeleteProvisioningTemplate(c *gin.Context) { - id, err := uuid.Parse(c.Param("id")) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"}) - return - } - if err := h.ProvisioningTemplateRepo.Delete(c.Request.Context(), id); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - c.Status(http.StatusNoContent) -} - -// --------------------------------------------------------------------------- -// Provisioning jobs -// --------------------------------------------------------------------------- - -type CreateProvisioningJobRequest struct { - TemplateID *uuid.UUID `json:"template_id,omitempty"` - Custom json.RawMessage `json:"custom,omitempty"` - TriggeredBy string `json:"triggered_by,omitempty"` -} - -// Job rows are flat in the repo and the UI works in a nested {config:{...}} -// shape with snake_case keys — the same translation problem the templates have. -// toJobDTO maps the row (and decodes the stored flat config snapshot back into -// the nested ProvisioningConfig) so the jobs list / detail / progress panel get -// the fields they read. -type provJobStepDTO struct { - Key string `json:"key"` - Label string `json:"label"` - Done int `json:"done"` - Total int `json:"total"` -} - -type provJobTimelineDTO struct { - State string `json:"state"` - At time.Time `json:"at"` - Note string `json:"note,omitempty"` -} - -type provJobDTO struct { - ID uuid.UUID `json:"id"` - State string `json:"state"` - TriggeredBy string `json:"triggered_by,omitempty"` - Provider string `json:"provider"` - TemplateID *uuid.UUID `json:"template_id,omitempty"` - TemplateName string `json:"template_name,omitempty"` - Config provTemplateConfigDTO `json:"config"` - Progress []provJobStepDTO `json:"progress"` - Timeline []provJobTimelineDTO `json:"timeline"` - CreatedWorkerIDs []uuid.UUID `json:"created_worker_ids,omitempty"` - LastError *string `json:"last_error,omitempty"` - CompletedAt *time.Time `json:"completed_at,omitempty"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` -} - -func toJobDTO(j *repository.ProvisioningJob) provJobDTO { - cfg := provTemplateConfigDTO{Labels: []provLabelDTO{}} - templateName := "" - if len(j.Config) > 0 { - var flat repository.ProvisioningTemplate - if err := json.Unmarshal(j.Config, &flat); err == nil { - cfg = toTemplateDTO(&flat).Config - // The snapshot carries the template name for template-launched jobs - // (custom jobs snapshot under the placeholder name "custom"). - if j.TemplateID != nil { - templateName = flat.Name - } - } - } - return provJobDTO{ - ID: j.ID, - State: string(j.State), - TriggeredBy: j.TriggeredBy, - Provider: j.Provider, - TemplateID: j.TemplateID, - TemplateName: templateName, - Config: cfg, - Progress: []provJobStepDTO{}, - Timeline: []provJobTimelineDTO{}, - CreatedWorkerIDs: j.WorkerIDs, - LastError: j.Error, - CompletedAt: j.CompletedAt, - CreatedAt: j.CreatedAt, - UpdatedAt: j.UpdatedAt, - } -} - -func (h *Handler) AdminListProvisioningJobs(c *gin.Context) { - if h.ProvisioningJobRepo == nil { - c.JSON(http.StatusOK, gin.H{"data": []any{}}) - return - } - limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50")) - rows, err := h.ProvisioningJobRepo.List(c.Request.Context(), limit) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - out := make([]provJobDTO, 0, len(rows)) - for i := range rows { - out = append(out, toJobDTO(&rows[i])) - } - c.JSON(http.StatusOK, gin.H{"data": out}) -} - -func (h *Handler) AdminGetProvisioningJob(c *gin.Context) { - id, err := uuid.Parse(c.Param("id")) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"}) - return - } - row, err := h.ProvisioningJobRepo.Get(c.Request.Context(), id) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - if row == nil { - c.Status(http.StatusNotFound) - return - } - c.JSON(http.StatusOK, toJobDTO(row)) -} - -func (h *Handler) AdminCreateProvisioningJob(c *gin.Context) { - if h.ProvisioningJobRepo == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "provisioning jobs repo not configured"}) - return - } - var req CreateProvisioningJobRequest - if err := c.ShouldBindJSON(&req); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid body"}) - return - } - - // Resolve template or custom config into the JSONB config column. - var ( - config json.RawMessage - templateID *uuid.UUID - provider string - ) - if req.TemplateID != nil { - if h.ProvisioningTemplateRepo == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "template repo not configured"}) - return - } - t, err := h.ProvisioningTemplateRepo.Get(c.Request.Context(), *req.TemplateID) - if err != nil || t == nil { - c.JSON(http.StatusNotFound, gin.H{"error": "template not found"}) - return - } - if !repository.IsClientRequestableTier(t.Tier) { - c.JSON(http.StatusBadRequest, gin.H{ - "error": "this template targets the dedicated tier, which is allocated automatically by the control plane; pick a shared-tier template", - "code": "tier_not_allowed", - }) - return - } - // Snapshot the flat template into the job's config column. Its field - // names line up with provisioning.JobConfig, so the state machine reads - // it directly. - b, _ := json.Marshal(t) - config = b - templateID = &t.ID - provider = t.Provider - } else { - if len(req.Custom) == 0 { - c.JSON(http.StatusBadRequest, gin.H{"error": "either template_id or custom config required"}) - return - } - // The UI sends the nested config shape (worker_tier, {key,value} label - // rows). Normalize it through the same mapping templates use so the - // snapshot matches provisioning.JobConfig. - var cfgDTO provTemplateConfigDTO - if err := json.Unmarshal(req.Custom, &cfgDTO); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid custom config: " + err.Error()}) - return - } - if cfgDTO.Provider == "" || cfgDTO.Location == "" || cfgDTO.ServerType == "" || cfgDTO.WorkerTier == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "custom config requires provider, location, server_type and worker_tier"}) - return - } - snap := fromTemplateDTO(&provTemplateDTO{Name: "custom", Config: cfgDTO}) - if !repository.IsClientRequestableTier(snap.Tier) { - c.JSON(http.StatusBadRequest, gin.H{ - "error": "dedicated tier cannot be provisioned directly; dedicated workers are allocated automatically by the control plane", - "code": "tier_not_allowed", - }) - return - } - b, _ := json.Marshal(snap) - config = b - provider = cfgDTO.Provider - } - - if provider == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "provider missing from config"}) - return - } - - // Cloud-first gate: a provisioning job for a provider can only be created - // once a cloud credential for that provider exists — otherwise the job - // would sit in 'pending' forever with no way to reach the provider API. - var credentialID *uuid.UUID - if h.CloudCredentialRepo != nil { - cred, err := h.CloudCredentialRepo.GetByProvider(c.Request.Context(), provider) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - if cred == nil { - c.JSON(http.StatusConflict, gin.H{ - "error": "no cloud provider connected for " + provider + " — add one under Settings → Cloud Providers first", - "code": "cloud_provider_required", - }) - return - } - credentialID = &cred.ID - } - - triggeredBy := req.TriggeredBy - if triggeredBy == "" { - triggeredBy = "admin" - } - - job := &repository.ProvisioningJob{ - State: models.ProvJobPending, - TriggeredBy: triggeredBy, - Provider: provider, - CredentialID: credentialID, - TemplateID: templateID, - Config: config, - } - if err := h.ProvisioningJobRepo.Create(c.Request.Context(), job); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - - // The state machine pickup is asynchronous: the provisioning runner - // (internal/app/provisioning Runner) processes jobs in state != - // completed/failed. The admin UI polls GET /admin/provisioning-jobs/:id - // for live status. - - c.JSON(http.StatusAccepted, toJobDTO(job)) -} - -// AdminRetryProvisioningJob resets a failed job back to pending so the runner -// re-attempts it. -func (h *Handler) AdminRetryProvisioningJob(c *gin.Context) { - if h.ProvisioningJobRepo == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "provisioning jobs repo not configured"}) - return - } - id, err := uuid.Parse(c.Param("id")) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"}) - return - } - job, err := h.ProvisioningJobRepo.Get(c.Request.Context(), id) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - if job == nil { - c.JSON(http.StatusNotFound, gin.H{"error": "job not found"}) - return - } - if job.State != models.ProvJobFailed { - c.JSON(http.StatusConflict, gin.H{ - "error": "only failed jobs can be retried", - "code": "job_not_retryable", - }) - return - } - if err := h.ProvisioningJobRepo.Retry(c.Request.Context(), id); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - job, err = h.ProvisioningJobRepo.Get(c.Request.Context(), id) - if err != nil || job == nil { - c.JSON(http.StatusOK, gin.H{"ok": true}) - return - } - c.JSON(http.StatusOK, toJobDTO(job)) -} - -// --------------------------------------------------------------------------- -// Provisioning policy -// --------------------------------------------------------------------------- - -func (h *Handler) AdminListProvisioningPolicy(c *gin.Context) { - if h.ProvisioningPolicyRepo == nil { - c.JSON(http.StatusOK, gin.H{"policies": []any{}}) - return - } - rows, err := h.ProvisioningPolicyRepo.List(c.Request.Context()) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - c.JSON(http.StatusOK, gin.H{"policies": rows}) -} - -func (h *Handler) AdminUpdateProvisioningPolicy(c *gin.Context) { - if h.ProvisioningPolicyRepo == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "policy repo not configured"}) - return - } - var p repository.ProvisioningPolicy - if err := c.ShouldBindJSON(&p); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid body"}) - return - } - if p.Provider == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "provider required"}) - return - } - if err := h.ProvisioningPolicyRepo.Update(c.Request.Context(), &p); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - c.JSON(http.StatusOK, p) -} diff --git a/internal/api/handler/admin_releases.go b/internal/api/handler/admin_releases.go deleted file mode 100644 index 3901a2ff..00000000 --- a/internal/api/handler/admin_releases.go +++ /dev/null @@ -1,100 +0,0 @@ -// Release-handling endpoints. -// -// POST /webhooks/github/releases — public, HMAC-validated. GitHub posts here -// when a release event fires. -// POST /admin/releases/check — manual trigger from the dashboard. -// GET /admin/releases/state — last known per-channel resolution. -// PUT /admin/worker-profiles/:id/release — set release channel + auto-update. -// -// The webhook endpoint is intentionally not behind admin auth; security comes -// from the X-Hub-Signature-256 HMAC. If RELEASES_WEBHOOK_SECRET isn't set, -// the endpoint refuses all requests. - -package handler - -import ( - "fmt" - "io" - "net/http" - - "github.com/gin-gonic/gin" - "github.com/warmbly/warmbly/internal/app/releases" - "github.com/warmbly/warmbly/internal/errx" - "github.com/warmbly/warmbly/internal/models" -) - -func (h *Handler) GithubReleasesWebhook(c *gin.Context) { - if h.ReleasesService == nil { - c.JSON(http.StatusNotImplemented, gin.H{"error": "releases service not configured"}) - return - } - body, err := io.ReadAll(c.Request.Body) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "read body: " + err.Error()}) - return - } - signature := c.GetHeader("X-Hub-Signature-256") - eventType := c.GetHeader("X-GitHub-Event") - if err := h.ReleasesService.HandleWebhook(c.Request.Context(), body, signature, eventType); err != nil { - c.JSON(http.StatusUnauthorized, gin.H{"error": err.Error()}) - return - } - c.JSON(http.StatusOK, gin.H{"ok": true}) -} - -func (h *Handler) AdminCheckReleases(c *gin.Context) { - if h.ReleasesService == nil { - errx.JSON(c, errx.New(errx.NotFound, "releases service not configured")) - return - } - changed, err := h.ReleasesService.CheckGitHub(c.Request.Context()) - if err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - h.audit(c, models.AuditActionCheckReleases, models.AuditEntityRelease, nil, map[string]string{ - "changed_profiles": fmt.Sprintf("%d", len(changed)), - }) - c.JSON(http.StatusOK, gin.H{ - "state": h.ReleasesService.GetState(), - "changed": changed, - }) -} - -func (h *Handler) AdminReleasesState(c *gin.Context) { - if h.ReleasesService == nil { - c.JSON(http.StatusOK, gin.H{"enabled": false}) - return - } - c.JSON(http.StatusOK, h.ReleasesService.GetState()) -} - -type setReleaseBody struct { - Channel string `json:"channel" binding:"required,oneof=pinned stable dev"` - AutoUpdate bool `json:"auto_update"` -} - -func (h *Handler) AdminSetProfileRelease(c *gin.Context) { - id, ok := parseUUID(c, "id") - if !ok { - return - } - var body setReleaseBody - if err := c.ShouldBindJSON(&body); err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid request body")) - return - } - if err := h.CredentialsRepo.UpdateProfileRelease(c.Request.Context(), id, models.ReleaseChannel(body.Channel), body.AutoUpdate); err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - h.audit(c, models.AuditActionUpdate, models.AuditEntityWorkerProfile, &id, map[string]string{ - "release_channel": body.Channel, - "auto_update": fmt.Sprintf("%t", body.AutoUpdate), - }) - c.JSON(http.StatusOK, gin.H{"ok": true}) -} - -// Compile-time check that the releases package is referenced (keeps the -// import stable across edits even if every method body changes). -var _ = (*releases.Service)(nil) diff --git a/internal/api/handler/admin_sends.go b/internal/api/handler/admin_sends.go new file mode 100644 index 00000000..fcce2111 --- /dev/null +++ b/internal/api/handler/admin_sends.go @@ -0,0 +1,186 @@ +// Admin endpoints for the send outcome loop and the task queues: in-flight +// reservations, dead letters, task failures and customer webhook delivery +// health across every workspace. + +package handler + +import ( + "net/http" + "strconv" + "time" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" +) + +const deadLetterEntity models.AuditEntityType = "task_dead_letter" + +// webhookDeliveryLease mirrors the default LeaseTimeout in +// webhook.NewDeliveryWorker (cmd/backend wires the worker without overriding it). +const webhookDeliveryLease = 5 * time.Minute + +// parseBoundedLimit is parseLimit with a page cap above the admin default of 100. +func parseBoundedLimit(s string, def, max int) int { + if s == "" { + return def + } + n, err := strconv.Atoi(s) + if err != nil || n <= 0 { + return def + } + if n > max { + return max + } + return n +} + +func (h *Handler) adminSendsReady(c *gin.Context) bool { + if h.AdminSendsRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "send operations are not available on this instance")) + return false + } + return true +} + +// AdminInFlightSends lists reserved sends no worker result has resolved yet. +func (h *Handler) AdminInFlightSends(c *gin.Context) { + if !h.adminSendsReady(c) { + return + } + limit := parseBoundedLimit(c.Query("limit"), 100, 500) + reclaimAfter := time.Duration(config.CampaignSendReclaimAfterMinutes) * time.Minute + result, err := h.AdminSendsRepo.InFlight(c.Request.Context(), reclaimAfter, limit) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + c.JSON(http.StatusOK, result) +} + +// AdminListDeadLetters pages task dead letters newest first. +func (h *Handler) AdminListDeadLetters(c *gin.Context) { + if !h.adminSendsReady(c) { + return + } + cursor := parseCursor(c.Query("cursor")) + limit := parseLimit(c.Query("limit"), 50) + status := c.Query("status") + switch status { + case "", "pending", "replayed", "failed": + default: + errx.JSON(c, errx.New(errx.BadRequest, "invalid status")) + return + } + result, err := h.AdminSendsRepo.ListDeadLetters(c.Request.Context(), status, cursor, limit) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + c.JSON(http.StatusOK, result) +} + +// AdminReplayDeadLetter re-enqueues one dead letter through its workspace. +func (h *Handler) AdminReplayDeadLetter(c *gin.Context) { + if !h.adminSendsReady(c) { + return + } + if h.AdvancedService == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "dead letter replay is not available on this instance")) + return + } + id, err := uuid.Parse(c.Param("id")) + if err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid dead letter ID")) + return + } + row, err := h.AdminSendsRepo.GetDeadLetter(c.Request.Context(), id) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + if row == nil { + errx.JSON(c, errx.New(errx.NotFound, "dead letter not found")) + return + } + if row.OrganizationID == nil { + errx.JSON(c, errx.New(errx.BadRequest, "dead letter has no organization; its task or mailbox is gone")) + return + } + if xerr := h.AdvancedService.ReplayDeadLetter(c.Request.Context(), *row.OrganizationID, id); xerr != nil { + errx.JSON(c, xerr) + return + } + h.audit(c, models.AuditActionResume, deadLetterEntity, &id, map[string]string{ + "organization_id": row.OrganizationID.String(), + "task_id": row.TaskID.String(), + "task_type": row.TaskType, + }) + c.JSON(http.StatusOK, gin.H{"replayed": true}) +} + +// AdminRecentTaskFailures lists the newest task failures with their mailbox. +func (h *Handler) AdminRecentTaskFailures(c *gin.Context) { + if !h.adminSendsReady(c) { + return + } + limit := parseBoundedLimit(c.Query("limit"), 100, 500) + rows, err := h.AdminSendsRepo.RecentTaskFailures(c.Request.Context(), limit) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + c.JSON(http.StatusOK, gin.H{"data": rows}) +} + +// AdminWebhookHealth is the instance-wide webhook delivery picture. +func (h *Handler) AdminWebhookHealth(c *gin.Context) { + if !h.adminSendsReady(c) { + return + } + health, err := h.AdminSendsRepo.WebhookHealth(c.Request.Context(), webhookDeliveryLease) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + c.JSON(http.StatusOK, health) +} + +// AdminWebhookReclaim re-queues deliveries stranded in_flight past the lease. +func (h *Handler) AdminWebhookReclaim(c *gin.Context) { + if h.WebhookRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "webhook delivery is not available on this instance")) + return + } + n, err := h.WebhookRepo.ReclaimStuckDeliveries(c.Request.Context(), webhookDeliveryLease) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + h.audit(c, models.AuditActionUpdate, models.AuditEntityWebhook, nil, map[string]string{ + "action": "reclaim_stuck_deliveries", + "reclaimed": strconv.FormatInt(n, 10), + }) + c.JSON(http.StatusOK, gin.H{"reclaimed": n}) +} + +// AdminListOrgWebhooks lists one workspace's endpoints with recent counts. +func (h *Handler) AdminListOrgWebhooks(c *gin.Context) { + if !h.adminSendsReady(c) { + return + } + orgID, err := uuid.Parse(c.Param("id")) + if err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid organization ID")) + return + } + rows, err := h.AdminSendsRepo.OrgWebhooks(c.Request.Context(), orgID) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + c.JSON(http.StatusOK, gin.H{"data": rows}) +} diff --git a/internal/api/handler/admin_sync.go b/internal/api/handler/admin_sync.go new file mode 100644 index 00000000..4ec08e90 --- /dev/null +++ b/internal/api/handler/admin_sync.go @@ -0,0 +1,121 @@ +package handler + +import ( + "errors" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +const ( + auditActionSyncClearThrottle models.AuditAction = "sync_clear_throttle" + auditActionSyncRestartBackfill models.AuditAction = "sync_restart_backfill" +) + +// AdminSearchSync is GET /admin/sync: every mailbox's sync governor state. +func (h *Handler) AdminSearchSync(c *gin.Context) { + if h.AdminSyncRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "sync operations are not available on this instance")) + return + } + var search models.AdminSyncSearch + if err := c.ShouldBindQuery(&search); err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid query parameters")) + return + } + result, err := h.AdminSyncRepo.Search(c.Request.Context(), &search) + if err != nil { + switch { + case errors.Is(err, repository.ErrAdminSyncBadCursor): + errx.JSON(c, errx.New(errx.BadRequest, "invalid cursor")) + case errors.Is(err, repository.ErrAdminSyncBadState): + errx.JSON(c, errx.New(errx.BadRequest, "invalid state filter")) + default: + errx.JSON(c, errx.New(errx.Internal, "failed to load sync state")) + } + return + } + c.JSON(http.StatusOK, result) +} + +// AdminSyncClearThrottle is POST /admin/sync/:id/clear-throttle. +func (h *Handler) AdminSyncClearThrottle(c *gin.Context) { + if h.AdminSyncRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "sync operations are not available on this instance")) + return + } + id, err := uuid.Parse(c.Param("id")) + if err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid mailbox ID")) + return + } + cleared, err := h.AdminSyncRepo.ClearThrottle(c.Request.Context(), id) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, "failed to clear throttle")) + return + } + if !cleared { + errx.JSON(c, errx.New(errx.NotFound, "mailbox has no sync state or is not throttled")) + return + } + resp := gin.H{"email_id": id, "cleared": true} + h.reloadSyncedMailbox(c, id, resp) + h.audit(c, auditActionSyncClearThrottle, models.AuditEntityEmailAccount, &id, map[string]string{ + "reloaded": boolString(resp["reloaded"].(bool)), + }) + c.JSON(http.StatusOK, resp) +} + +// AdminSyncRestartBackfill is POST /admin/sync/:id/restart-backfill. +func (h *Handler) AdminSyncRestartBackfill(c *gin.Context) { + if h.AdminSyncRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "sync operations are not available on this instance")) + return + } + id, err := uuid.Parse(c.Param("id")) + if err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid mailbox ID")) + return + } + reset, err := h.AdminSyncRepo.ResetBackfill(c.Request.Context(), id) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, "failed to reset backfill")) + return + } + if !reset { + errx.JSON(c, errx.New(errx.NotFound, "mailbox has no sync state")) + return + } + resp := gin.H{"email_id": id, "reset": true} + h.reloadSyncedMailbox(c, id, resp) + h.audit(c, auditActionSyncRestartBackfill, models.AuditEntityEmailAccount, &id, map[string]string{ + "reloaded": boolString(resp["reloaded"].(bool)), + }) + c.JSON(http.StatusOK, resp) +} + +// reloadSyncedMailbox re-ships the mailbox so the worker's live copy follows +// the platform copy; a failure is reported in the response, not as an error. +func (h *Handler) reloadSyncedMailbox(c *gin.Context, id uuid.UUID, resp gin.H) { + resp["reloaded"] = false + if h.EmailService == nil { + resp["reload_error"] = "email service not configured" + return + } + if err := h.EmailService.LoadAccountOntoWorker(c.Request.Context(), id); err != nil { + resp["reload_error"] = err.Error() + return + } + resp["reloaded"] = true +} + +func boolString(b bool) string { + if b { + return "true" + } + return "false" +} diff --git a/internal/api/handler/admin_transfer.go b/internal/api/handler/admin_transfer.go new file mode 100644 index 00000000..fe6ca182 --- /dev/null +++ b/internal/api/handler/admin_transfer.go @@ -0,0 +1,260 @@ +package handler + +import ( + "fmt" + "io" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/api/middleware" + "github.com/warmbly/warmbly/internal/app/orgtransfer" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" +) + +// Operator-side workspace archives: the same export and import flow as +// org_transfer.go, keyed by the :id path parameter instead of the caller's own +// organization, and audited to the admin log rather than the org's. + +// adminTransferOrg resolves :id to an existing organization and confirms the +// transfer service is wired, answering the request itself otherwise. +func (h *Handler) adminTransferOrg(c *gin.Context) (uuid.UUID, bool) { + orgID, ok := parseUUIDParam(c, "id") + if !ok { + return uuid.Nil, false + } + org, err := h.OrgRepo.GetByID(c.Request.Context(), orgID) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return uuid.Nil, false + } + if org == nil { + errx.JSON(c, errx.New(errx.NotFound, "organization not found")) + return uuid.Nil, false + } + if h.OrgTransferService == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "Workspace transfer is not available on this instance.")) + return uuid.Nil, false + } + return orgID, true +} + +// ---------- export ---------- + +// AdminListOrgExports returns a workspace's recent archive builds. +// +// GET /admin/organizations/:id/exports +func (h *Handler) AdminListOrgExports(c *gin.Context) { + orgID, ok := h.adminTransferOrg(c) + if !ok { + return + } + jobs, xerr := h.OrgTransferService.ListExports(c.Request.Context(), orgID) + if xerr != nil { + errx.JSON(c, xerr) + return + } + c.JSON(http.StatusOK, gin.H{"data": jobs}) +} + +// AdminCreateOrgExport starts an archive build on the operator's behalf. +// +// POST /admin/organizations/:id/exports +func (h *Handler) AdminCreateOrgExport(c *gin.Context) { + orgID, ok := h.adminTransferOrg(c) + if !ok { + return + } + var req models.CreateOrgExportRequest + if err := c.ShouldBindJSON(&req); err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid request body")) + return + } + + job, xerr := h.OrgTransferService.RequestExport(c.Request.Context(), orgID, middleware.GetAdminUserID(c), &req) + if xerr != nil { + errx.JSON(c, xerr) + return + } + + h.audit(c, models.AuditActionExport, models.AuditEntityOrgArchive, &job.ID, map[string]string{ + "organization_id": orgID.String(), + "include_secrets": fmt.Sprintf("%t", job.IncludeSecrets), + "groups": fmt.Sprintf("%d", len(job.Groups)), + }) + c.JSON(http.StatusAccepted, job) +} + +// AdminGetOrgExport returns one archive build, for progress polling. +// +// GET /admin/organizations/:id/exports/:exportId +func (h *Handler) AdminGetOrgExport(c *gin.Context) { + orgID, ok := h.adminTransferOrg(c) + if !ok { + return + } + id, ok := parseUUIDParam(c, "exportId") + if !ok { + return + } + job, xerr := h.OrgTransferService.GetExport(c.Request.Context(), orgID, id) + if xerr != nil { + errx.JSON(c, xerr) + return + } + c.JSON(http.StatusOK, job) +} + +// AdminDownloadOrgExport streams a finished archive. +// +// GET /admin/organizations/:id/exports/:exportId/download +func (h *Handler) AdminDownloadOrgExport(c *gin.Context) { + orgID, ok := h.adminTransferOrg(c) + if !ok { + return + } + id, ok := parseUUIDParam(c, "exportId") + if !ok { + return + } + + body, job, xerr := h.OrgTransferService.OpenExport(c.Request.Context(), orgID, id) + if xerr != nil { + errx.JSON(c, xerr) + return + } + defer body.Close() + + name := "workspace-" + job.ID.String()[:8] + ".warmbly.zip" + if org, err := h.OrgRepo.GetByID(c.Request.Context(), orgID); err == nil && org != nil { + name = orgtransfer.ArchiveFilename(org.Name, job.ID) + } + + h.audit(c, models.AuditActionExport, models.AuditEntityOrgArchive, &job.ID, map[string]string{ + "organization_id": orgID.String(), + "downloaded": "true", + }) + + c.Header("Content-Type", "application/zip") + c.Header("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, name)) + c.Header("X-Content-Type-Options", "nosniff") + if job.ArchiveBytes != nil { + c.Header("Content-Length", fmt.Sprintf("%d", *job.ArchiveBytes)) + } + if job.ArchiveSHA256 != nil { + c.Header("X-Archive-SHA256", *job.ArchiveSHA256) + } + + if _, err := io.Copy(c.Writer, body); err != nil { + // The client hung up mid-download; the status line is already sent. + return + } +} + +// AdminDeleteOrgExport removes an archive and its stored object. +// +// DELETE /admin/organizations/:id/exports/:exportId +func (h *Handler) AdminDeleteOrgExport(c *gin.Context) { + orgID, ok := h.adminTransferOrg(c) + if !ok { + return + } + id, ok := parseUUIDParam(c, "exportId") + if !ok { + return + } + if xerr := h.OrgTransferService.DeleteExport(c.Request.Context(), orgID, id); xerr != nil { + errx.JSON(c, xerr) + return + } + h.audit(c, models.AuditActionDelete, models.AuditEntityOrgArchive, &id, map[string]string{ + "organization_id": orgID.String(), + }) + c.Status(http.StatusNoContent) +} + +// ---------- import ---------- + +// AdminListOrgImports returns a workspace's recent imports. +// +// GET /admin/organizations/:id/imports +func (h *Handler) AdminListOrgImports(c *gin.Context) { + orgID, ok := h.adminTransferOrg(c) + if !ok { + return + } + jobs, xerr := h.OrgTransferService.ListImports(c.Request.Context(), orgID) + if xerr != nil { + errx.JSON(c, xerr) + return + } + c.JSON(http.StatusOK, gin.H{"data": jobs}) +} + +// AdminPreflightOrgImport reports what an uploaded archive would do to the +// workspace, writing nothing. +// +// POST /admin/organizations/:id/imports/preflight +func (h *Handler) AdminPreflightOrgImport(c *gin.Context) { + orgID, ok := h.adminTransferOrg(c) + if !ok { + return + } + spooled, xerr := spoolArchiveUpload(c) + if xerr != nil { + errx.JSON(c, xerr) + return + } + defer spooled.Close() + + report, xerr := h.OrgTransferService.Preflight( + c.Request.Context(), orgID, spooled, c.Request.FormValue("passphrase")) + if xerr != nil { + errx.JSON(c, xerr) + return + } + c.JSON(http.StatusOK, report) +} + +// AdminCreateOrgImport applies an uploaded archive to the workspace. +// +// POST /admin/organizations/:id/imports +func (h *Handler) AdminCreateOrgImport(c *gin.Context) { + orgID, ok := h.adminTransferOrg(c) + if !ok { + return + } + + // Ownership of the spooled file passes to the service on success; every + // failure path before that closes it here. + spooled, xerr := spoolArchiveUpload(c) + if xerr != nil { + errx.JSON(c, xerr) + return + } + + var req models.CreateOrgImportRequest + if opts := c.Request.FormValue("options"); opts != "" { + if err := jsonUnmarshalString(opts, &req); err != nil { + _ = spooled.Close() + errx.JSON(c, errx.New(errx.BadRequest, "invalid 'options' JSON: "+err.Error())) + return + } + } + req.Passphrase = c.Request.FormValue("passphrase") + + job, xerr := h.OrgTransferService.RequestImport(c.Request.Context(), orgID, middleware.GetAdminUserID(c), spooled, &req) + if xerr != nil { + _ = spooled.Close() + errx.JSON(c, xerr) + return + } + + h.audit(c, models.AuditActionImport, models.AuditEntityOrgArchive, &job.ID, map[string]string{ + "organization_id": orgID.String(), + "conflict_strategy": string(job.ConflictStrategy), + }) + c.JSON(http.StatusAccepted, job) +} diff --git a/internal/api/handler/handler.go b/internal/api/handler/handler.go index 39c14277..0fab70f7 100644 --- a/internal/api/handler/handler.go +++ b/internal/api/handler/handler.go @@ -45,7 +45,6 @@ import ( "github.com/warmbly/warmbly/internal/app/poollink" "github.com/warmbly/warmbly/internal/app/ratelimit" "github.com/warmbly/warmbly/internal/app/referral" - "github.com/warmbly/warmbly/internal/app/releases" "github.com/warmbly/warmbly/internal/app/research" "github.com/warmbly/warmbly/internal/app/segment" "github.com/warmbly/warmbly/internal/app/sequence" @@ -163,7 +162,6 @@ type Handler struct { WorkerOrchestrator *worker_orchestrator.Orchestrator WorkerRepo repository.WorkerRepository CredentialsRepo repository.CredentialsRepository - ReleasesService *releases.Service // UpdatesService backs the admin panel's update indicator and button. UpdatesService *updates.Service @@ -320,6 +318,18 @@ type Handler struct { // Wired in cmd/backend/main.go where the concrete clients live. SystemChecker *sysstatus.Checker + // Admin operations pages: cross-workspace reads of mailbox sync, the send + // outcome loop, fleet placement and abuse signals, plus the scheduled job + // registry every background loop records to. Nil-safe: the endpoints + // answer 501 when the repository is not wired. + AdminSyncRepo repository.AdminSyncRepository + AdminSendsRepo repository.AdminSendsRepository + AdminFleetRepo repository.AdminFleetRepository + AdminInsightRepo repository.AdminInsightRepository + JobRuns repository.JobRunRepository + // WebhookRepo backs the operator's "reclaim stuck deliveries" action. + WebhookRepo repository.WebhookRepository + // Operator visibility (admin panel, Instance section). // // InstanceRuntime carries the facts only boot knows (the resolved CORS diff --git a/internal/api/middleware/admin.go b/internal/api/middleware/admin.go index e040f1b8..41390bf0 100644 --- a/internal/api/middleware/admin.go +++ b/internal/api/middleware/admin.go @@ -112,9 +112,9 @@ func IsAdmin(c *gin.Context) bool { return GetAdminPermissions(c) > 0 } -// IsSuperAdmin returns true if the current user has all admin permissions +// IsSuperAdmin returns true if the current user holds every live admin bit. func IsSuperAdmin(c *gin.Context) bool { - return GetAdminPermissions(c) == models.AllAdminPermissions + return GetAdminPermissions(c).IsSuperAdmin() } // getAdminPermissions fetches admin permissions for a user diff --git a/internal/api/routes.go b/internal/api/routes.go index d4aa56ec..8fdbb8de 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -1509,11 +1509,56 @@ func Run( adminRoutes.GET("/analytics/emails/hourly", middleware.RequireAdminPermission(models.AdminPermViewAnalytics), h.AdminGetHourlyEmailStats) adminRoutes.GET("/analytics/users/growth", middleware.RequireAdminPermission(models.AdminPermViewAnalytics), h.AdminGetUserGrowthStats) - // Removed for self-host: worker load + email-distribution analytics - // (premised on multi-worker IP spread, moot when the mail provider owns the - // egress IP), and the SaaS commercial surfaces — plans, discount/promo - // codes, and the enterprise-sales inquiry queue — which have no role in a - // single-org, billing-disabled deployment. + // Signups by channel and trial conversion, from organization_acquisition. + adminRoutes.GET("/analytics/acquisition", middleware.RequireAdminPermission(models.AdminPermViewAnalytics), h.AdminGetAcquisition) + + // Mailbox sync governor: the platform copy of every mailbox's sync + // state. Actions re-ship the mailbox to its worker so the worker's live + // copy follows. + adminRoutes.GET("/sync", middleware.RequireAdminPermission(models.AdminPermViewUsers), h.AdminSearchSync) + adminRoutes.POST("/sync/:id/clear-throttle", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminSyncClearThrottle) + adminRoutes.POST("/sync/:id/restart-backfill", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminSyncRestartBackfill) + + // Send outcome loop and task queues. + adminRoutes.GET("/sends/in-flight", middleware.RequireAdminPermission(models.AdminPermViewCampaigns), h.AdminInFlightSends) + adminRoutes.GET("/tasks/dead-letters", middleware.RequireAdminPermission(models.AdminPermViewCampaigns), h.AdminListDeadLetters) + adminRoutes.POST("/tasks/dead-letters/:id/replay", middleware.RequireAdminPermission(models.AdminPermStopCampaigns), h.AdminReplayDeadLetter) + adminRoutes.GET("/tasks/failures", middleware.RequireAdminPermission(models.AdminPermViewCampaigns), h.AdminRecentTaskFailures) + adminRoutes.GET("/webhooks/health", middleware.RequireAdminPermission(models.AdminPermViewOrganizations), h.AdminWebhookHealth) + adminRoutes.POST("/webhooks/reclaim", middleware.RequireAdminPermission(models.AdminPermManageOrganizations), h.AdminWebhookReclaim) + + // Scheduled jobs: every background loop on the instance, with "run now". + adminRoutes.GET("/jobs", middleware.RequireAdminPermission(models.AdminPermViewAnalytics), h.AdminListJobs) + adminRoutes.POST("/jobs/:name/run", middleware.RequireAdminPermission(models.AdminPermManageSettings), h.AdminRunJob) + + // Fleet placement: capacity per worker, the control loops' decision + // log, and dedicated worker bindings. + adminRoutes.GET("/fleet/capacity", middleware.RequireAdminPermission(models.AdminPermViewWorkers), h.AdminFleetCapacity) + adminRoutes.GET("/fleet/decisions", middleware.RequireAdminPermission(models.AdminPermViewWorkers), h.AdminFleetDecisions) + adminRoutes.GET("/fleet/dedicated", middleware.RequireAdminPermission(models.AdminPermViewWorkers), h.AdminFleetDedicated) + adminRoutes.POST("/fleet/dedicated/:orgId/release", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminFleetReleaseDedicated) + adminRoutes.POST("/workers/:id/convert-dedicated", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminConvertWorkerToDedicated) + + // Workspace transfers: the same archive service the owner uses from + // Settings > Data, driven by the operator for any workspace. + adminRoutes.GET("/transfers", middleware.RequireAdminPermission(models.AdminPermViewOrganizations), h.AdminListTransfers) + adminRoutes.GET("/organizations/:id/exports", middleware.RequireAdminPermission(models.AdminPermViewOrganizations), h.AdminListOrgExports) + adminRoutes.POST("/organizations/:id/exports", middleware.RequireAdminPermission(models.AdminPermManageOrganizations), h.AdminCreateOrgExport) + adminRoutes.GET("/organizations/:id/exports/:exportId", middleware.RequireAdminPermission(models.AdminPermViewOrganizations), h.AdminGetOrgExport) + adminRoutes.GET("/organizations/:id/exports/:exportId/download", middleware.RequireAdminPermission(models.AdminPermManageOrganizations), h.AdminDownloadOrgExport) + adminRoutes.DELETE("/organizations/:id/exports/:exportId", middleware.RequireAdminPermission(models.AdminPermManageOrganizations), h.AdminDeleteOrgExport) + adminRoutes.GET("/organizations/:id/imports", middleware.RequireAdminPermission(models.AdminPermViewOrganizations), h.AdminListOrgImports) + adminRoutes.POST("/organizations/:id/imports/preflight", middleware.RequireAdminPermission(models.AdminPermManageOrganizations), h.AdminPreflightOrgImport) + adminRoutes.POST("/organizations/:id/imports", middleware.RequireAdminPermission(models.AdminPermManageOrganizations), h.AdminCreateOrgImport) + + // Per-workspace developer surface: keys and webhook endpoints. + adminRoutes.GET("/organizations/:id/api-keys", middleware.RequireAdminPermission(models.AdminPermViewOrganizations), h.AdminListOrgAPIKeys) + adminRoutes.DELETE("/organizations/:id/api-keys/:keyId", middleware.RequireAdminPermission(models.AdminPermManageOrganizations), h.AdminRevokeOrgAPIKey) + adminRoutes.GET("/organizations/:id/webhooks", middleware.RequireAdminPermission(models.AdminPermViewOrganizations), h.AdminListOrgWebhooks) + + // Warmup abuse signals and the block/unblock history. + adminRoutes.GET("/warmup/abuse", middleware.RequireAdminPermission(models.AdminPermViewWarmupPool), h.AdminWarmupAbuse) + adminRoutes.GET("/warmup/actions", middleware.RequireAdminPermission(models.AdminPermViewWarmupPool), h.AdminWarmupActions) // Admin Management adminRoutes.GET("/admins", middleware.RequireAdminPermission(models.AdminPermGrantAdminAccess), h.AdminListAdmins) diff --git a/internal/app/admin/service.go b/internal/app/admin/service.go index f4a98ea6..4871717c 100644 --- a/internal/app/admin/service.go +++ b/internal/app/admin/service.go @@ -58,23 +58,8 @@ type AdminService interface { GetAnalyticsTrends(ctx context.Context) (*models.AnalyticsTrends, *errx.Error) GetDailyEmailStats(ctx context.Context, startDate, endDate time.Time) ([]models.DailyEmailStats, *errx.Error) GetHourlyEmailStats(ctx context.Context, date time.Time) ([]models.HourlyEmailStats, *errx.Error) - GetWorkerLoadStats(ctx context.Context) ([]models.WorkerLoadStats, *errx.Error) - GetEmailDistribution(ctx context.Context) ([]models.EmailDistribution, *errx.Error) GetUserGrowthStats(ctx context.Context, startDate, endDate time.Time) ([]models.UserGrowthStats, *errx.Error) - // Plans - ListPlans(ctx context.Context, includePrivate bool) ([]models.Plan, *errx.Error) - SearchPlansForAdmin(ctx context.Context, search *models.AdminPlanSearch) (*models.AdminPlansResult, *errx.Error) - CreatePlan(ctx context.Context, adminID uuid.UUID, req *models.CreatePlanRequest, ipAddress, userAgent string) (*models.Plan, *errx.Error) - GetPlan(ctx context.Context, planID uuid.UUID) (*models.Plan, *errx.Error) - UpdatePlan(ctx context.Context, adminID, planID uuid.UUID, req *models.UpdatePlanRequest, ipAddress, userAgent string) (*models.Plan, *errx.Error) - DeletePlan(ctx context.Context, adminID, planID uuid.UUID, ipAddress, userAgent string) *errx.Error - - // Enterprise Inquiries - ListEnterpriseInquiries(ctx context.Context, search *models.AdminEnterpriseInquirySearch) (*models.AdminEnterpriseInquiriesResult, *errx.Error) - GetEnterpriseInquiry(ctx context.Context, id uuid.UUID) (*models.AdminEnterpriseInquiry, *errx.Error) - UpdateEnterpriseInquiry(ctx context.Context, adminID, inquiryID uuid.UUID, update *models.UpdateEnterpriseInquiryRequest, ipAddress, userAgent string) *errx.Error - // Admin Management ListAdmins(ctx context.Context, cursor *uuid.UUID, limit int) (*models.AdminsResult, *errx.Error) GrantAdminPermissions(ctx context.Context, adminID, targetUserID uuid.UUID, permissions models.AdminPermission, ipAddress, userAgent string) *errx.Error @@ -559,24 +544,6 @@ func (s *adminService) GetHourlyEmailStats(ctx context.Context, date time.Time) return stats, nil } -func (s *adminService) GetWorkerLoadStats(ctx context.Context) ([]models.WorkerLoadStats, *errx.Error) { - stats, err := s.repo.GetWorkerLoadStats(ctx) - if err != nil { - errs.CaptureException(err) - return nil, errx.New(errx.Internal, "failed to get worker load stats") - } - return stats, nil -} - -func (s *adminService) GetEmailDistribution(ctx context.Context) ([]models.EmailDistribution, *errx.Error) { - dist, err := s.repo.GetEmailDistribution(ctx) - if err != nil { - errs.CaptureException(err) - return nil, errx.New(errx.Internal, "failed to get email distribution") - } - return dist, nil -} - func (s *adminService) GetUserGrowthStats(ctx context.Context, startDate, endDate time.Time) ([]models.UserGrowthStats, *errx.Error) { stats, err := s.repo.GetUserGrowthStats(ctx, startDate, endDate) if err != nil { @@ -586,213 +553,6 @@ func (s *adminService) GetUserGrowthStats(ctx context.Context, startDate, endDat return stats, nil } -// Plans - -func (s *adminService) ListPlans(ctx context.Context, includePrivate bool) ([]models.Plan, *errx.Error) { - plans, err := s.repo.ListPlans(ctx, includePrivate) - if err != nil { - errs.CaptureException(err) - return nil, errx.New(errx.Internal, "failed to list plans") - } - return plans, nil -} - -func (s *adminService) SearchPlansForAdmin(ctx context.Context, search *models.AdminPlanSearch) (*models.AdminPlansResult, *errx.Error) { - result, err := s.repo.SearchPlansForAdmin(ctx, search) - if err != nil { - errs.CaptureException(err) - return nil, errx.New(errx.Internal, "failed to search plans") - } - return result, nil -} - -// resolveDuration maps the API's billing period onto a durations row. An -// unknown period is the caller's mistake, not an internal error. -func (s *adminService) resolveDuration(ctx context.Context, d models.Duration) (*uuid.UUID, *errx.Error) { - if d == "" { - return nil, errx.New(errx.BadRequest, "duration is required") - } - id, err := s.repo.DurationIDByTitle(ctx, string(d)) - if err != nil { - errs.CaptureException(err) - return nil, errx.New(errx.Internal, "failed to resolve plan duration") - } - if id == nil { - return nil, errx.New(errx.BadRequest, "unknown plan duration") - } - return id, nil -} - -func (s *adminService) CreatePlan(ctx context.Context, adminID uuid.UUID, req *models.CreatePlanRequest, ipAddress, userAgent string) (*models.Plan, *errx.Error) { - plan := &models.Plan{ - ID: uuid.New(), - Name: &req.Name, - MaxContacts: req.MaxContacts, - DailyEmails: req.DailyEmails, - AIGeneration: req.AIGeneration, - AccountLimit: req.AccountLimit, - Price: req.Price, - DiscountedPrice: req.DiscountedPrice, - Duration: req.Duration, - Public: req.Public, - DedicatedWorkers: req.DedicatedWorkers, - DailyCampaignLimit: req.DailyCampaignLimit, - MaxCampaigns: req.MaxCampaigns, - MaxActiveCampaigns: req.MaxActiveCampaigns, - MaxTeamMembers: req.MaxTeamMembers, - MaxEmailAccounts: req.MaxEmailAccounts, - } - - durationID, xerr := s.resolveDuration(ctx, plan.Duration) - if xerr != nil { - return nil, xerr - } - - if err := s.repo.CreatePlan(ctx, plan, *durationID); err != nil { - errs.CaptureException(err) - return nil, errx.New(errx.Internal, "failed to create plan") - } - - s.logAction(ctx, adminID, "create_plan", "plan", plan.ID, map[string]any{"name": req.Name}, ipAddress, userAgent) - return plan, nil -} - -func (s *adminService) GetPlan(ctx context.Context, planID uuid.UUID) (*models.Plan, *errx.Error) { - plan, err := s.repo.GetPlan(ctx, planID) - if err != nil { - errs.CaptureException(err) - return nil, errx.New(errx.Internal, "failed to get plan") - } - if plan == nil { - return nil, errx.ErrNotFound - } - return plan, nil -} - -func (s *adminService) UpdatePlan(ctx context.Context, adminID, planID uuid.UUID, req *models.UpdatePlanRequest, ipAddress, userAgent string) (*models.Plan, *errx.Error) { - plan, err := s.repo.GetPlan(ctx, planID) - if err != nil { - errs.CaptureException(err) - return nil, errx.New(errx.Internal, "failed to get plan") - } - if plan == nil { - return nil, errx.ErrNotFound - } - - // Apply updates - if req.Name != nil { - plan.Name = req.Name - } - if req.MaxContacts != nil { - plan.MaxContacts = *req.MaxContacts - } - if req.DailyEmails != nil { - plan.DailyEmails = *req.DailyEmails - } - if req.AIGeneration != nil { - plan.AIGeneration = *req.AIGeneration - } - if req.AccountLimit != nil { - plan.AccountLimit = *req.AccountLimit - } - if req.Price != nil { - plan.Price = *req.Price - } - if req.DiscountedPrice != nil { - plan.DiscountedPrice = *req.DiscountedPrice - } - if req.Duration != nil { - plan.Duration = *req.Duration - } - if req.Public != nil { - plan.Public = *req.Public - } - if req.DedicatedWorkers != nil { - plan.DedicatedWorkers = *req.DedicatedWorkers - } - if req.DailyCampaignLimit != nil { - plan.DailyCampaignLimit = req.DailyCampaignLimit - } - if req.MaxCampaigns != nil { - plan.MaxCampaigns = req.MaxCampaigns - } - if req.MaxActiveCampaigns != nil { - plan.MaxActiveCampaigns = req.MaxActiveCampaigns - } - if req.MaxTeamMembers != nil { - plan.MaxTeamMembers = req.MaxTeamMembers - } - if req.MaxEmailAccounts != nil { - plan.MaxEmailAccounts = req.MaxEmailAccounts - } - - durationID, xerr := s.resolveDuration(ctx, plan.Duration) - if xerr != nil { - return nil, xerr - } - - if err := s.repo.UpdatePlan(ctx, plan, *durationID); err != nil { - errs.CaptureException(err) - return nil, errx.New(errx.Internal, "failed to update plan") - } - - s.logAction(ctx, adminID, "update_plan", "plan", planID, nil, ipAddress, userAgent) - return plan, nil -} - -func (s *adminService) DeletePlan(ctx context.Context, adminID, planID uuid.UUID, ipAddress, userAgent string) *errx.Error { - // Check if plan is in use - inUse, err := s.repo.IsPlanInUse(ctx, planID) - if err != nil { - errs.CaptureException(err) - return errx.New(errx.Internal, "failed to check plan usage") - } - if inUse { - return errx.New(errx.BadRequest, "cannot delete plan that is in use") - } - - if err := s.repo.DeletePlan(ctx, planID); err != nil { - errs.CaptureException(err) - return errx.New(errx.Internal, "failed to delete plan") - } - - s.logAction(ctx, adminID, "delete_plan", "plan", planID, nil, ipAddress, userAgent) - return nil -} - -// Enterprise Inquiries - -func (s *adminService) ListEnterpriseInquiries(ctx context.Context, search *models.AdminEnterpriseInquirySearch) (*models.AdminEnterpriseInquiriesResult, *errx.Error) { - result, err := s.repo.ListEnterpriseInquiries(ctx, search) - if err != nil { - errs.CaptureException(err) - return nil, errx.New(errx.Internal, "failed to list enterprise inquiries") - } - return result, nil -} - -func (s *adminService) GetEnterpriseInquiry(ctx context.Context, id uuid.UUID) (*models.AdminEnterpriseInquiry, *errx.Error) { - inquiry, err := s.repo.GetEnterpriseInquiry(ctx, id) - if err != nil { - errs.CaptureException(err) - return nil, errx.New(errx.Internal, "failed to get enterprise inquiry") - } - if inquiry == nil { - return nil, errx.ErrNotFound - } - return inquiry, nil -} - -func (s *adminService) UpdateEnterpriseInquiry(ctx context.Context, adminID, inquiryID uuid.UUID, update *models.UpdateEnterpriseInquiryRequest, ipAddress, userAgent string) *errx.Error { - if err := s.repo.UpdateEnterpriseInquiry(ctx, inquiryID, update); err != nil { - errs.CaptureException(err) - return errx.New(errx.Internal, "failed to update enterprise inquiry") - } - - s.logAction(ctx, adminID, "update_enterprise_inquiry", "enterprise_inquiry", inquiryID, map[string]any{"update": update}, ipAddress, userAgent) - return nil -} - // Admin Management func (s *adminService) ListAdmins(ctx context.Context, cursor *uuid.UUID, limit int) (*models.AdminsResult, *errx.Error) { diff --git a/internal/app/advisor/runner.go b/internal/app/advisor/runner.go index 6f060880..ef77cf61 100644 --- a/internal/app/advisor/runner.go +++ b/internal/app/advisor/runner.go @@ -8,6 +8,7 @@ import ( "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/repository" ) @@ -52,18 +53,10 @@ func (r *Runner) Run(ctx context.Context) { if r.BatchSize <= 0 { r.BatchSize = defaultBatchSize } - - tick := time.NewTicker(r.Interval) - defer tick.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-tick.C: - r.sweep(ctx) - } - } + jobrun.Loop(ctx, "advisor_sweep", r.Interval, false, func(ctx context.Context) error { + r.sweep(ctx) + return nil + }) } func (r *Runner) sweep(ctx context.Context) { diff --git a/internal/app/consumer/auth_check_sweep.go b/internal/app/consumer/auth_check_sweep.go index 53314856..fb466ce3 100644 --- a/internal/app/consumer/auth_check_sweep.go +++ b/internal/app/consumer/auth_check_sweep.go @@ -9,6 +9,7 @@ import ( "github.com/google/uuid" "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/pkg/dnsauth" ) @@ -26,20 +27,12 @@ func (s *JobsService) StartAuthCheckSweep(ctx context.Context, interval, staleAf if s.EmailRepository == nil { return } - - ticker := time.NewTicker(interval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - sweepCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) - s.runAuthCheckSweep(sweepCtx, staleAfter) - cancel() - } - } + jobrun.Loop(ctx, "auth_check_sweep", interval, false, func(ctx context.Context) error { + sweepCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + s.runAuthCheckSweep(sweepCtx, staleAfter) + return nil + }) } func (s *JobsService) runAuthCheckSweep(ctx context.Context, staleAfter time.Duration) { diff --git a/internal/app/consumer/dead_worker.go b/internal/app/consumer/dead_worker.go index 4af152c0..92fd870c 100644 --- a/internal/app/consumer/dead_worker.go +++ b/internal/app/consumer/dead_worker.go @@ -8,6 +8,7 @@ import ( "github.com/google/uuid" "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/models" ) @@ -91,20 +92,12 @@ func (s *JobsService) StartDeadWorkerDetection(ctx context.Context, interval tim if s.WorkerRepo == nil { return } - - ticker := time.NewTicker(interval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - detectCtx, cancel := context.WithTimeout(ctx, 30*time.Second) - s.detectDeadWorkers(detectCtx) - cancel() - } - } + jobrun.Loop(ctx, "dead_worker_detection", interval, false, func(ctx context.Context) error { + detectCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + s.detectDeadWorkers(detectCtx) + return nil + }) } func (s *JobsService) detectDeadWorkers(ctx context.Context) { diff --git a/internal/app/consumer/dlq_retry.go b/internal/app/consumer/dlq_retry.go index cb6d08be..de48ba13 100644 --- a/internal/app/consumer/dlq_retry.go +++ b/internal/app/consumer/dlq_retry.go @@ -5,6 +5,7 @@ import ( "time" "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/jobrun" ) // StartDLQRetryLoop polls for retryable dead-lettered tasks and replays them. @@ -13,25 +14,16 @@ func (s *JobsService) StartDLQRetryLoop(ctx context.Context, interval time.Durat if s.AdvancedService == nil { return } - - ticker := time.NewTicker(interval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - retryCtx, cancel := context.WithTimeout(ctx, 30*time.Second) - retried, xerr := s.AdvancedService.ProcessRetryableDeadLetters(retryCtx) - cancel() - if xerr != nil { - log.Warn().Str("error", xerr.Error()).Msg("DLQ retry processing failed") - continue - } - if retried > 0 { - log.Info().Int("retried", retried).Msg("DLQ auto-retry processed dead letters") - } + jobrun.Loop(ctx, "dlq_retry", interval, false, func(ctx context.Context) error { + retryCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + retried, xerr := s.AdvancedService.ProcessRetryableDeadLetters(retryCtx) + if xerr != nil { + return xerr } - } + if retried > 0 { + log.Info().Int("retried", retried).Msg("DLQ auto-retry processed dead letters") + } + return nil + }) } diff --git a/internal/app/consumer/lifecycle_rebalancer.go b/internal/app/consumer/lifecycle_rebalancer.go index 9298913a..dd91d08d 100644 --- a/internal/app/consumer/lifecycle_rebalancer.go +++ b/internal/app/consumer/lifecycle_rebalancer.go @@ -8,6 +8,7 @@ import ( "github.com/rs/zerolog/log" "github.com/warmbly/warmbly/internal/app/lifecycle" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/models" ) @@ -22,16 +23,10 @@ func (s *JobsService) StartLifecycleRebalancer(ctx context.Context, interval tim if s.LifecycleRepo == nil { return } - ticker := time.NewTicker(interval) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - s.rebalanceLifecycles(ctx) - } - } + jobrun.Loop(ctx, "lifecycle_rebalancer", interval, false, func(ctx context.Context) error { + s.rebalanceLifecycles(ctx) + return nil + }) } func (s *JobsService) rebalanceLifecycles(ctx context.Context) { diff --git a/internal/app/consumer/risk_rebalancer.go b/internal/app/consumer/risk_rebalancer.go index ced62272..6bba959d 100644 --- a/internal/app/consumer/risk_rebalancer.go +++ b/internal/app/consumer/risk_rebalancer.go @@ -6,6 +6,7 @@ import ( "github.com/google/uuid" "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/models" ) @@ -30,25 +31,19 @@ func (s *JobsService) StartRiskRebalancer(ctx context.Context, interval time.Dur return } - // Initial run right after boot so a fresh deploy converges quickly. - go func() { - boot, cancel := context.WithTimeout(ctx, 5*time.Minute) - defer cancel() - s.rebalanceRisk(boot) - }() - - ticker := time.NewTicker(interval) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - runCtx, cancel := context.WithTimeout(ctx, 10*time.Minute) - s.rebalanceRisk(runCtx) - cancel() + // The boot pass (so a fresh deploy converges quickly) keeps its shorter budget. + first := true + jobrun.Loop(ctx, "risk_rebalancer", interval, true, func(ctx context.Context) error { + timeout := 10 * time.Minute + if first { + first = false + timeout = 5 * time.Minute } - } + runCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + s.rebalanceRisk(runCtx) + return nil + }) } func (s *JobsService) rebalanceRisk(ctx context.Context) { diff --git a/internal/app/consumer/stuck_send_reclaimer.go b/internal/app/consumer/stuck_send_reclaimer.go index 26eb5105..d2c66cfb 100644 --- a/internal/app/consumer/stuck_send_reclaimer.go +++ b/internal/app/consumer/stuck_send_reclaimer.go @@ -7,6 +7,7 @@ import ( "github.com/rs/zerolog/log" "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/repository" ) @@ -31,18 +32,12 @@ func (s *JobsService) StartStuckSendReclaimer(ctx context.Context, interval time if s.TaskRepo == nil || s.CampaignProgressRepo == nil { return } - ticker := time.NewTicker(interval) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - sweepCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) - s.reclaimStuckSends(sweepCtx) - cancel() - } - } + jobrun.Loop(ctx, "stuck_send_reclaimer", interval, false, func(ctx context.Context) error { + sweepCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + s.reclaimStuckSends(sweepCtx) + return nil + }) } func (s *JobsService) reclaimStuckSends(ctx context.Context) { diff --git a/internal/app/consumer/warmup_engagement_poller.go b/internal/app/consumer/warmup_engagement_poller.go index 503e662e..6a062ad9 100644 --- a/internal/app/consumer/warmup_engagement_poller.go +++ b/internal/app/consumer/warmup_engagement_poller.go @@ -7,6 +7,7 @@ import ( "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/models" ) @@ -18,18 +19,10 @@ func (s *JobsService) StartWarmupEngagementPoller(ctx context.Context, interval if s.WarmupEngagementRepo == nil || s.Publisher == nil { return } - - ticker := time.NewTicker(interval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - s.drainDueEngagements(ctx) - } - } + jobrun.Loop(ctx, "warmup_engagement_poller", interval, false, func(ctx context.Context) error { + s.drainDueEngagements(ctx) + return nil + }) } func (s *JobsService) drainDueEngagements(ctx context.Context) { diff --git a/internal/app/consumer/warmup_health_sweep.go b/internal/app/consumer/warmup_health_sweep.go index 90999823..7eea6fae 100644 --- a/internal/app/consumer/warmup_health_sweep.go +++ b/internal/app/consumer/warmup_health_sweep.go @@ -5,6 +5,7 @@ import ( "time" "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/jobrun" ) // StartWarmupHealthSweep runs a periodic health evaluation across all warmup pool participants. @@ -13,29 +14,19 @@ func (s *JobsService) StartWarmupHealthSweep(ctx context.Context, interval time. if s.WarmupService == nil { return } - - ticker := time.NewTicker(interval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - sweepCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) - evaluated, changes, xerr := s.WarmupService.EvaluateAllParticipants(sweepCtx) - if xerr != nil { - cancel() - log.Warn().Str("error", xerr.Error()).Msg("warmup health sweep failed") - continue - } - if evaluated > 0 { - log.Info().Int("evaluated", evaluated).Int("state_changes", changes).Msg("warmup health sweep completed") - } - if changes > 0 { - s.rebalanceRisk(sweepCtx) - } - cancel() + jobrun.Loop(ctx, "warmup_health_sweep", interval, false, func(ctx context.Context) error { + sweepCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + evaluated, changes, xerr := s.WarmupService.EvaluateAllParticipants(sweepCtx) + if xerr != nil { + return xerr } - } + if evaluated > 0 { + log.Info().Int("evaluated", evaluated).Int("state_changes", changes).Msg("warmup health sweep completed") + } + if changes > 0 { + s.rebalanceRisk(sweepCtx) + } + return nil + }) } diff --git a/internal/app/consumer/worker_heartbeat.go b/internal/app/consumer/worker_heartbeat.go index 196311d6..12b41ff3 100644 --- a/internal/app/consumer/worker_heartbeat.go +++ b/internal/app/consumer/worker_heartbeat.go @@ -7,6 +7,7 @@ import ( "github.com/redis/go-redis/v9" "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/jobrun" ) // StartWorkerHeartbeatSync mirrors workers' Redis heartbeat timestamps into @@ -19,20 +20,12 @@ func (s *JobsService) StartWorkerHeartbeatSync(ctx context.Context, interval tim if s.WorkerRepo == nil || s.Cache == nil { return } - - ticker := time.NewTicker(interval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - runCtx, cancel := context.WithTimeout(ctx, 15*time.Second) - s.syncHeartbeats(runCtx) - cancel() - } - } + jobrun.Loop(ctx, "worker_heartbeat_sync", interval, false, func(ctx context.Context) error { + runCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + s.syncHeartbeats(runCtx) + return nil + }) } func (s *JobsService) syncHeartbeats(ctx context.Context) { diff --git a/internal/app/correlate/correlate.go b/internal/app/correlate/correlate.go index e8046391..e5b43967 100644 --- a/internal/app/correlate/correlate.go +++ b/internal/app/correlate/correlate.go @@ -13,6 +13,7 @@ import ( "github.com/rs/zerolog/log" "github.com/warmbly/warmbly/internal/app/orgrisk" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/repository" ) @@ -74,16 +75,10 @@ func (s *Service) Start(ctx context.Context, interval time.Duration) { if s == nil || s.repo == nil || s.orgRisk == nil { return } - ticker := time.NewTicker(interval) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - s.Run(ctx) - } - } + jobrun.Loop(ctx, "correlation_sweep", interval, false, func(ctx context.Context) error { + s.Run(ctx) + return nil + }) } // finding is one detector's verdict on one organization. diff --git a/internal/app/fleet/quarantine.go b/internal/app/fleet/quarantine.go index 221ce2d4..9a2562a9 100644 --- a/internal/app/fleet/quarantine.go +++ b/internal/app/fleet/quarantine.go @@ -6,6 +6,7 @@ import ( "time" "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/repository" ) @@ -43,18 +44,7 @@ func (q *QuarantineEvaluator) defaults() { func (q *QuarantineEvaluator) Run(ctx context.Context) { q.defaults() - tick := time.NewTicker(q.Interval) - defer tick.Stop() - for { - select { - case <-ctx.Done(): - return - case <-tick.C: - if err := q.tick(ctx); err != nil { - log.Warn().Err(err).Msg("quarantine tick failed") - } - } - } + jobrun.Loop(ctx, "fleet_quarantine", q.Interval, false, q.tick) } func (q *QuarantineEvaluator) tick(ctx context.Context) error { diff --git a/internal/app/fleet/rebalance.go b/internal/app/fleet/rebalance.go index 10b63bb4..678101ce 100644 --- a/internal/app/fleet/rebalance.go +++ b/internal/app/fleet/rebalance.go @@ -15,6 +15,7 @@ import ( "github.com/google/uuid" "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/repository" ) @@ -60,18 +61,7 @@ func (r *Rebalancer) defaults() { // Run blocks until ctx is cancelled, ticking every Interval. func (r *Rebalancer) Run(ctx context.Context) { r.defaults() - tick := time.NewTicker(r.Interval) - defer tick.Stop() - for { - select { - case <-ctx.Done(): - return - case <-tick.C: - if err := r.tick(ctx); err != nil { - log.Warn().Err(err).Msg("fleet rebalance tick failed") - } - } - } + jobrun.Loop(ctx, "fleet_rebalance", r.Interval, false, r.tick) } func (r *Rebalancer) tick(ctx context.Context) error { diff --git a/internal/app/fleet/scale.go b/internal/app/fleet/scale.go index 865099d1..50a087ba 100644 --- a/internal/app/fleet/scale.go +++ b/internal/app/fleet/scale.go @@ -7,6 +7,7 @@ import ( "time" "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/repository" ) @@ -46,21 +47,8 @@ func (s *Scaler) defaults() { func (s *Scaler) Run(ctx context.Context) { s.defaults() - tick := time.NewTicker(s.Interval) - defer tick.Stop() - // Run once immediately on boot so an admin doesn't wait an hour for the - // first signal. - _ = s.tick(ctx) - for { - select { - case <-ctx.Done(): - return - case <-tick.C: - if err := s.tick(ctx); err != nil { - log.Warn().Err(err).Msg("fleet scale tick failed") - } - } - } + // Runs once on boot so an admin doesn't wait an hour for the first signal. + jobrun.Loop(ctx, "fleet_scale", s.Interval, true, s.tick) } func (s *Scaler) tick(ctx context.Context) error { diff --git a/internal/app/orgrisk/service.go b/internal/app/orgrisk/service.go index b8d04020..f4d50225 100644 --- a/internal/app/orgrisk/service.go +++ b/internal/app/orgrisk/service.go @@ -14,6 +14,7 @@ import ( "github.com/rs/zerolog/log" "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/repository" ) @@ -341,16 +342,10 @@ func StartExpirySweep(ctx context.Context, svc Service, interval time.Duration) if svc == nil || interval <= 0 { return } - ticker := time.NewTicker(interval) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - svc.SweepExpired(ctx) - } - } + jobrun.Loop(ctx, "org_risk_expiry_sweep", interval, false, func(ctx context.Context) error { + svc.SweepExpired(ctx) + return nil + }) } // apply mutates the signal set and re-derives score, band and reason from it, diff --git a/internal/infrastructure/db/migrations/000135_scheduled_job_runs.down.sql b/internal/infrastructure/db/migrations/000135_scheduled_job_runs.down.sql new file mode 100644 index 00000000..d868a5ec --- /dev/null +++ b/internal/infrastructure/db/migrations/000135_scheduled_job_runs.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS scheduled_job_runs; diff --git a/internal/infrastructure/db/migrations/000135_scheduled_job_runs.up.sql b/internal/infrastructure/db/migrations/000135_scheduled_job_runs.up.sql new file mode 100644 index 00000000..1c62e405 --- /dev/null +++ b/internal/infrastructure/db/migrations/000135_scheduled_job_runs.up.sql @@ -0,0 +1,25 @@ +-- scheduled_job_runs: one row per background loop on the instance, written by +-- the process that runs it (backend or consumer) and read by the admin panel's +-- Jobs page. Instance-level, not workspace data: it never travels in an +-- organization archive. +CREATE TABLE IF NOT EXISTS scheduled_job_runs ( + name text PRIMARY KEY, + service text NOT NULL DEFAULT '', + interval_seconds integer NOT NULL DEFAULT 0, + last_started_at timestamptz, + last_finished_at timestamptz, + last_duration_ms bigint NOT NULL DEFAULT 0, + last_status text NOT NULL DEFAULT 'idle' + CHECK (last_status IN ('idle', 'running', 'ok', 'error')), + last_error text NOT NULL DEFAULT '', + run_count bigint NOT NULL DEFAULT 0, + error_count bigint NOT NULL DEFAULT 0, + -- Set by the panel's "run now"; cleared by the owning loop when it picks + -- the request up on its next poll. + run_requested_at timestamptz, + next_run_at timestamptz, + updated_at timestamptz NOT NULL DEFAULT now() +); + +COMMENT ON TABLE scheduled_job_runs IS + 'Last run, next run and last error of every background loop, for the admin Jobs page. Instance-level.'; diff --git a/internal/jobrun/jobrun.go b/internal/jobrun/jobrun.go new file mode 100644 index 00000000..279b169d --- /dev/null +++ b/internal/jobrun/jobrun.go @@ -0,0 +1,152 @@ +// Package jobrun runs a background loop and records what it did, so the admin +// panel can list every scheduled job on the instance with its last run, its +// next run and its last error, and ask one to run now. +// +// Every service that hosts loops (backend, consumer) calls Configure once at +// boot with the store and its own name, then wraps each loop in Loop. Without +// a store the loops still run; nothing is recorded and "run now" has nowhere +// to land. +package jobrun + +import ( + "context" + "sync" + "time" + + "github.com/rs/zerolog/log" + + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/observability/errs" +) + +// Store persists job runs. Implemented by repository.JobRunRepository. +type Store interface { + // Register creates or refreshes the job's row at boot. + Register(ctx context.Context, name, service string, interval time.Duration, nextRunAt time.Time) error + MarkStarted(ctx context.Context, name string, at time.Time) error + MarkFinished(ctx context.Context, name string, startedAt, finishedAt time.Time, runErr error, nextRunAt time.Time) error + // RequestRun asks the owning loop to run at its next poll. False when no + // job by that name has ever registered. + RequestRun(ctx context.Context, name string) (bool, error) + // TakeRunRequest clears a pending request and reports whether there was one. + TakeRunRequest(ctx context.Context, name string) (bool, error) + List(ctx context.Context) ([]models.ScheduledJobRun, error) +} + +// requestPoll is how often a loop checks for a "run now" request. +const requestPoll = 15 * time.Second + +var ( + mu sync.RWMutex + store Store + service string +) + +// Configure sets the store every loop in this process records to and the +// name of this service. Safe to call once; later calls replace both. +func Configure(s Store, serviceName string) { + mu.Lock() + defer mu.Unlock() + store = s + service = serviceName +} + +func current() (Store, string) { + mu.RLock() + defer mu.RUnlock() + return store, service +} + +// Loop runs fn every interval until ctx ends, records each run, and also runs +// fn when the panel requests it. runOnBoot runs fn once before the first tick, +// which is what most retention and reconcile loops want. +func Loop(ctx context.Context, name string, interval time.Duration, runOnBoot bool, fn func(ctx context.Context) error) { + if interval <= 0 { + interval = time.Minute + } + st, svc := current() + if st != nil { + next := time.Now().Add(interval) + if runOnBoot { + next = time.Now() + } + if err := st.Register(ctx, name, svc, interval, next); err != nil { + log.Warn().Err(err).Str("job", name).Msg("jobrun: register failed") + } + } + + run := func() { + Run(ctx, name, interval, fn) + } + + if runOnBoot { + run() + } + + ticker := time.NewTicker(interval) + defer ticker.Stop() + poll := time.NewTicker(requestPoll) + defer poll.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + run() + case <-poll.C: + if st == nil { + continue + } + requested, err := st.TakeRunRequest(ctx, name) + if err != nil { + log.Warn().Err(err).Str("job", name).Msg("jobrun: poll failed") + continue + } + if requested { + run() + } + } + } +} + +// Run executes fn once and records it. For loops that keep their own ticker. +func Run(ctx context.Context, name string, interval time.Duration, fn func(ctx context.Context) error) { + st, _ := current() + started := time.Now() + if st != nil { + if err := st.MarkStarted(ctx, name, started); err != nil { + log.Warn().Err(err).Str("job", name).Msg("jobrun: mark started failed") + } + } + runErr := safeRun(ctx, name, fn) + if runErr != nil { + errs.CaptureException(runErr) + log.Warn().Err(runErr).Str("job", name).Msg("scheduled job failed") + } + if st != nil { + finished := time.Now() + if err := st.MarkFinished(ctx, name, started, finished, runErr, finished.Add(interval)); err != nil { + log.Warn().Err(err).Str("job", name).Msg("jobrun: mark finished failed") + } + } +} + +// safeRun turns a panic in a job into an error so one bad pass cannot take +// the service down. +func safeRun(ctx context.Context, name string, fn func(ctx context.Context) error) (err error) { + defer func() { + if r := recover(); r != nil { + err = &panicError{job: name, value: r} + } + }() + return fn(ctx) +} + +type panicError struct { + job string + value any +} + +func (p *panicError) Error() string { + return "job " + p.job + " panicked" +} diff --git a/internal/jobs/audit_retention.go b/internal/jobs/audit_retention.go index 068818e8..bf78762f 100644 --- a/internal/jobs/audit_retention.go +++ b/internal/jobs/audit_retention.go @@ -5,6 +5,7 @@ import ( "time" "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/observability/errs" "github.com/warmbly/warmbly/internal/repository" ) @@ -66,25 +67,9 @@ func NewAuditRetentionScheduler(job *AuditRetentionJob, interval time.Duration) // Start begins scheduled execution, running once immediately on boot. func (s *AuditRetentionScheduler) Start(ctx context.Context) { - ticker := time.NewTicker(s.interval) - defer ticker.Stop() - - if err := s.job.Run(ctx); err != nil { - errs.CaptureException(err) - } - - for { - select { - case <-ticker.C: - if err := s.job.Run(ctx); err != nil { - errs.CaptureException(err) - } - case <-s.stopCh: - return - case <-ctx.Done(): - return - } - } + ctx, cancel := stopContext(ctx, s.stopCh) + defer cancel() + jobrun.Loop(ctx, "audit_retention", s.interval, true, s.job.Run) } // Stop halts the scheduled execution. diff --git a/internal/jobs/danger_zone.go b/internal/jobs/danger_zone.go index 5bdd3b7f..e856a4c6 100644 --- a/internal/jobs/danger_zone.go +++ b/internal/jobs/danger_zone.go @@ -7,6 +7,7 @@ import ( "github.com/warmbly/warmbly/internal/observability/errs" "github.com/warmbly/warmbly/internal/app/dangerzone" + "github.com/warmbly/warmbly/internal/jobrun" ) // DangerZoneJob ticks the dangerzone subsystem: it executes any pending @@ -53,21 +54,12 @@ func NewDangerZoneScheduler(job *DangerZoneJob, interval time.Duration) *DangerZ // Start runs Run() on every tick until ctx is cancelled or Stop() is called. func (s *DangerZoneScheduler) Start(ctx context.Context) { - ticker := time.NewTicker(s.interval) - defer ticker.Stop() - - s.job.Run(ctx) - - for { - select { - case <-ticker.C: - s.job.Run(ctx) - case <-s.stopCh: - return - case <-ctx.Done(): - return - } - } + ctx, cancel := stopContext(ctx, s.stopCh) + defer cancel() + jobrun.Loop(ctx, "danger_zone", s.interval, true, func(ctx context.Context) error { + s.job.Run(ctx) + return nil + }) } // Stop halts the scheduler. diff --git a/internal/jobs/email_verification.go b/internal/jobs/email_verification.go index 792e6d06..2e1bd164 100644 --- a/internal/jobs/email_verification.go +++ b/internal/jobs/email_verification.go @@ -2,11 +2,13 @@ package jobs import ( "context" + "sync" "time" "github.com/warmbly/warmbly/internal/observability/errs" emailverifyapp "github.com/warmbly/warmbly/internal/app/emailverify" + "github.com/warmbly/warmbly/internal/jobrun" ) // EmailVerificationJob verifies a batch of contacts due for a check each run, @@ -54,6 +56,7 @@ type EmailVerificationScheduler struct { job *EmailVerificationJob interval time.Duration stopCh chan struct{} + mu sync.Mutex } // NewEmailVerificationScheduler creates the scheduler. @@ -67,26 +70,29 @@ func NewEmailVerificationScheduler(job *EmailVerificationJob, interval time.Dura // Start begins scheduled execution. func (s *EmailVerificationScheduler) Start(ctx context.Context) { - ticker := time.NewTicker(s.interval) - defer ticker.Stop() - - var wake <-chan struct{} + ctx, cancel := stopContext(ctx, s.stopCh) + defer cancel() + // A wake (import, re-verify) runs the same pass outside the tick; the + // mutex keeps it from overlapping a tick or a "run now". + run := func(ctx context.Context) error { + s.mu.Lock() + defer s.mu.Unlock() + return s.job.Run(ctx) + } if s.job != nil && s.job.svc != nil { - wake = s.job.svc.Wake() - } - for { - select { - case <-ticker.C: - case <-wake: - case <-s.stopCh: - return - case <-ctx.Done(): - return - } - if err := s.job.Run(ctx); err != nil { - errs.CaptureException(err) - } + wake := s.job.svc.Wake() + go func() { + for { + select { + case <-ctx.Done(): + return + case <-wake: + jobrun.Run(ctx, "email_verification", s.interval, run) + } + } + }() } + jobrun.Loop(ctx, "email_verification", s.interval, false, run) } // Stop halts the scheduled execution. diff --git a/internal/jobs/form_events_retention.go b/internal/jobs/form_events_retention.go index 9d75f525..9a654d83 100644 --- a/internal/jobs/form_events_retention.go +++ b/internal/jobs/form_events_retention.go @@ -5,6 +5,7 @@ import ( "time" "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/repository" ) @@ -44,15 +45,5 @@ func (j *FormEventsRetentionJob) Run(ctx context.Context) error { // Start runs the job once on boot and then on the interval until ctx ends. func (j *FormEventsRetentionJob) Start(ctx context.Context, interval time.Duration) { - ticker := time.NewTicker(interval) - defer ticker.Stop() - _ = j.Run(ctx) - for { - select { - case <-ticker.C: - _ = j.Run(ctx) - case <-ctx.Done(): - return - } - } + jobrun.Loop(ctx, "form_events_retention", interval, true, j.Run) } diff --git a/internal/jobs/forms_domain_sweep.go b/internal/jobs/forms_domain_sweep.go index 46865760..00bc75d8 100644 --- a/internal/jobs/forms_domain_sweep.go +++ b/internal/jobs/forms_domain_sweep.go @@ -7,6 +7,7 @@ import ( "github.com/rs/zerolog/log" "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/pkg/trackdns" "github.com/warmbly/warmbly/internal/repository" ) @@ -71,15 +72,5 @@ func (j *FormsDomainSweep) Run(ctx context.Context) error { // Start runs the sweep once on boot and then on the interval until ctx ends. func (j *FormsDomainSweep) Start(ctx context.Context, interval time.Duration) { - ticker := time.NewTicker(interval) - defer ticker.Stop() - _ = j.Run(ctx) - for { - select { - case <-ticker.C: - _ = j.Run(ctx) - case <-ctx.Done(): - return - } - } + jobrun.Loop(ctx, "forms_domain_sweep", interval, true, j.Run) } diff --git a/internal/jobs/guardrail.go b/internal/jobs/guardrail.go index f71b0dd3..ff468871 100644 --- a/internal/jobs/guardrail.go +++ b/internal/jobs/guardrail.go @@ -8,6 +8,7 @@ import ( "github.com/warmbly/warmbly/internal/observability/errs" "github.com/warmbly/warmbly/internal/app/guardrail" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/repository" ) @@ -69,21 +70,12 @@ func NewGuardrailScheduler(job *GuardrailJob, interval time.Duration) *Guardrail // Start runs Run() on every tick until ctx is cancelled or Stop() is called. func (s *GuardrailScheduler) Start(ctx context.Context) { - ticker := time.NewTicker(s.interval) - defer ticker.Stop() - - s.job.Run(ctx) - - for { - select { - case <-ticker.C: - s.job.Run(ctx) - case <-s.stopCh: - return - case <-ctx.Done(): - return - } - } + ctx, cancel := stopContext(ctx, s.stopCh) + defer cancel() + jobrun.Loop(ctx, "guardrail_sweep", s.interval, true, func(ctx context.Context) error { + s.job.Run(ctx) + return nil + }) } // Stop halts the scheduler. diff --git a/internal/jobs/org_transfer.go b/internal/jobs/org_transfer.go index 08ba2bcd..6780de8d 100644 --- a/internal/jobs/org_transfer.go +++ b/internal/jobs/org_transfer.go @@ -7,6 +7,7 @@ import ( "github.com/warmbly/warmbly/internal/observability/errs" "github.com/warmbly/warmbly/internal/app/orgtransfer" + "github.com/warmbly/warmbly/internal/jobrun" ) // OrgTransferJob keeps the workspace-archive tables honest. It deletes archives @@ -57,21 +58,12 @@ func NewOrgTransferScheduler(job *OrgTransferJob, interval time.Duration) *OrgTr // Start runs Run() on every tick until ctx is cancelled or Stop() is called. func (s *OrgTransferScheduler) Start(ctx context.Context) { - ticker := time.NewTicker(s.interval) - defer ticker.Stop() - - s.job.Run(ctx) - - for { - select { - case <-ticker.C: - s.job.Run(ctx) - case <-s.stopCh: - return - case <-ctx.Done(): - return - } - } + ctx, cancel := stopContext(ctx, s.stopCh) + defer cancel() + jobrun.Loop(ctx, "org_transfer_housekeeping", s.interval, true, func(ctx context.Context) error { + s.job.Run(ctx) + return nil + }) } // Stop halts the scheduler. diff --git a/internal/jobs/placement_poller.go b/internal/jobs/placement_poller.go index 3f558100..6db47fa5 100644 --- a/internal/jobs/placement_poller.go +++ b/internal/jobs/placement_poller.go @@ -7,6 +7,7 @@ import ( "github.com/warmbly/warmbly/internal/observability/errs" "github.com/warmbly/warmbly/internal/app/placement" + "github.com/warmbly/warmbly/internal/jobrun" ) // PlacementPoller reconciles pending seed inbox-placement results: each tick it @@ -49,21 +50,9 @@ func (p *PlacementPoller) Run(ctx context.Context) error { // Start begins scheduled execution on the configured interval. func (p *PlacementPoller) Start(ctx context.Context) { - ticker := time.NewTicker(p.interval) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - if err := p.Run(ctx); err != nil { - errs.CaptureException(err) - } - case <-p.stopCh: - return - case <-ctx.Done(): - return - } - } + ctx, cancel := stopContext(ctx, p.stopCh) + defer cancel() + jobrun.Loop(ctx, "placement_poller", p.interval, false, p.Run) } // Stop halts scheduled execution. diff --git a/internal/jobs/schedule.go b/internal/jobs/schedule.go new file mode 100644 index 00000000..f24f7689 --- /dev/null +++ b/internal/jobs/schedule.go @@ -0,0 +1,17 @@ +package jobs + +import "context" + +// stopContext derives a context that ends when the scheduler's Stop() closes +// stopCh, so a jobrun loop keeps the old Stop() semantics. +func stopContext(ctx context.Context, stopCh <-chan struct{}) (context.Context, context.CancelFunc) { + ctx, cancel := context.WithCancel(ctx) + go func() { + select { + case <-stopCh: + cancel() + case <-ctx.Done(): + } + }() + return ctx, cancel +} diff --git a/internal/jobs/trial_expiration.go b/internal/jobs/trial_expiration.go index 56731f39..43433a75 100644 --- a/internal/jobs/trial_expiration.go +++ b/internal/jobs/trial_expiration.go @@ -7,6 +7,7 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5/pgxpool" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/notify" "github.com/warmbly/warmbly/internal/notify/templates" @@ -154,26 +155,9 @@ func NewTrialExpirationScheduler(job *TrialExpirationJob, interval time.Duration // Start begins the scheduled execution func (s *TrialExpirationScheduler) Start(ctx context.Context) { - ticker := time.NewTicker(s.interval) - defer ticker.Stop() - - // Run immediately on start - if err := s.job.Run(ctx); err != nil { - errs.CaptureException(err) - } - - for { - select { - case <-ticker.C: - if err := s.job.Run(ctx); err != nil { - errs.CaptureException(err) - } - case <-s.stopCh: - return - case <-ctx.Done(): - return - } - } + ctx, cancel := stopContext(ctx, s.stopCh) + defer cancel() + jobrun.Loop(ctx, "trial_expiration", s.interval, true, s.job.Run) } // Stop halts the scheduled execution diff --git a/internal/jobs/verification_evidence.go b/internal/jobs/verification_evidence.go index 630d4da5..e5fd1144 100644 --- a/internal/jobs/verification_evidence.go +++ b/internal/jobs/verification_evidence.go @@ -4,9 +4,8 @@ import ( "context" "time" - "github.com/warmbly/warmbly/internal/observability/errs" - emailverifyapp "github.com/warmbly/warmbly/internal/app/emailverify" + "github.com/warmbly/warmbly/internal/jobrun" ) // DeliveryEvidenceJob turns campaign sends that never bounced into @@ -28,23 +27,15 @@ func NewDeliveryEvidenceJob(evidence *emailverifyapp.Evidence, interval time.Dur // Start runs the job on its interval until ctx ends. A full batch repeats // at once so a backlog drains. func (j *DeliveryEvidenceJob) Start(ctx context.Context) { - ticker := time.NewTicker(j.interval) - defer ticker.Stop() - for { - select { - case <-ticker.C: - case <-ctx.Done(): - return - } + jobrun.Loop(ctx, "delivery_evidence", j.interval, false, func(ctx context.Context) error { for { n, err := j.evidence.CreditCleanDeliveries(ctx, j.batch) if err != nil { - errs.CaptureException(err) - break + return err } if n < j.batch || ctx.Err() != nil { - break + return nil } } - } + }) } diff --git a/internal/jobs/warmup_batch_poller.go b/internal/jobs/warmup_batch_poller.go index f02248d4..4e0834bc 100644 --- a/internal/jobs/warmup_batch_poller.go +++ b/internal/jobs/warmup_batch_poller.go @@ -7,6 +7,7 @@ import ( "github.com/warmbly/warmbly/internal/observability/errs" "github.com/warmbly/warmbly/internal/app/warmupcontent" + "github.com/warmbly/warmbly/internal/jobrun" ) // WarmupBatchPoller reconciles in-flight OpenAI Batch API warmup-generation jobs: @@ -47,21 +48,9 @@ func (p *WarmupBatchPoller) Run(ctx context.Context) error { // Start begins scheduled execution on the configured interval. func (p *WarmupBatchPoller) Start(ctx context.Context) { - ticker := time.NewTicker(p.interval) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - if err := p.Run(ctx); err != nil { - errs.CaptureException(err) - } - case <-p.stopCh: - return - case <-ctx.Done(): - return - } - } + ctx, cancel := stopContext(ctx, p.stopCh) + defer cancel() + jobrun.Loop(ctx, "warmup_batch_poller", p.interval, false, p.Run) } // Stop halts scheduled execution. diff --git a/internal/jobs/warmup_generation.go b/internal/jobs/warmup_generation.go index b0776d85..f5102201 100644 --- a/internal/jobs/warmup_generation.go +++ b/internal/jobs/warmup_generation.go @@ -8,6 +8,7 @@ import ( "github.com/warmbly/warmbly/internal/observability/errs" "github.com/warmbly/warmbly/internal/app/warmupcontent" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/repository" ) @@ -79,21 +80,9 @@ func NewWarmupGenerationScheduler(job *WarmupGenerationJob, interval time.Durati // Start begins scheduled execution. func (s *WarmupGenerationScheduler) Start(ctx context.Context) { - ticker := time.NewTicker(s.interval) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - if err := s.job.Run(ctx); err != nil { - errs.CaptureException(err) - } - case <-s.stopCh: - return - case <-ctx.Done(): - return - } - } + ctx, cancel := stopContext(ctx, s.stopCh) + defer cancel() + jobrun.Loop(ctx, "warmup_generation", s.interval, false, s.job.Run) } // Stop halts the scheduled execution. diff --git a/internal/jobs/website_tracking_retention.go b/internal/jobs/website_tracking_retention.go index c705d31d..224d5bc2 100644 --- a/internal/jobs/website_tracking_retention.go +++ b/internal/jobs/website_tracking_retention.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/observability/errs" "github.com/warmbly/warmbly/internal/repository" ) @@ -39,15 +40,5 @@ func (j *WebsiteTrackingRetentionJob) Run(ctx context.Context) error { // Start runs the job once on boot and then on the interval until ctx ends. func (j *WebsiteTrackingRetentionJob) Start(ctx context.Context, interval time.Duration) { - ticker := time.NewTicker(interval) - defer ticker.Stop() - _ = j.Run(ctx) - for { - select { - case <-ticker.C: - _ = j.Run(ctx) - case <-ctx.Done(): - return - } - } + jobrun.Loop(ctx, "website_tracking_retention", interval, true, j.Run) } diff --git a/internal/models/admin.go b/internal/models/admin.go index fbf921a7..74b621ca 100644 --- a/internal/models/admin.go +++ b/internal/models/admin.go @@ -494,17 +494,6 @@ type HourlyEmailStats struct { TotalSent int64 `json:"total_sent"` } -// WorkerLoadStats represents worker load statistics -type WorkerLoadStats struct { - WorkerID uuid.UUID `json:"worker_id"` - WorkerName string `json:"worker_name"` - EmailsSentToday int64 `json:"emails_sent_today"` - QueuedEmails int64 `json:"queued_emails"` - ConnectedEmails int64 `json:"connected_emails"` - CPUUsage float64 `json:"cpu_usage,omitempty"` - MemoryUsage float64 `json:"memory_usage,omitempty"` -} - // UserGrowthStats represents user growth statistics type UserGrowthStats struct { Date time.Time `json:"date"` @@ -520,111 +509,6 @@ type AnalyticsTrends struct { RevenueGrowthPercent float64 `json:"revenue_growth_percent"` } -// CreatePlanRequest represents the request to create a custom plan -type CreatePlanRequest struct { - Name string `json:"name" binding:"required"` - MaxContacts uint `json:"max_contacts"` - DailyEmails uint `json:"daily_emails"` - AIGeneration bool `json:"ai_generation"` - AccountLimit uint `json:"account_limit"` - Price float32 `json:"price"` - DiscountedPrice float32 `json:"discounted_price"` - Duration Duration `json:"duration"` // month/year - DedicatedWorkers int `json:"dedicated_workers"` - DailyCampaignLimit *int `json:"daily_campaign_limit,omitempty"` - MaxCampaigns *int `json:"max_campaigns,omitempty"` - MaxActiveCampaigns *int `json:"max_active_campaigns,omitempty"` - MaxTeamMembers *int `json:"max_team_members,omitempty"` - MaxEmailAccounts *int `json:"max_email_accounts,omitempty"` - Public bool `json:"public"` // false for enterprise-only -} - -// UpdatePlanRequest represents the request to update a plan -type UpdatePlanRequest struct { - Name *string `json:"name,omitempty"` - MaxContacts *uint `json:"max_contacts,omitempty"` - DailyEmails *uint `json:"daily_emails,omitempty"` - AIGeneration *bool `json:"ai_generation,omitempty"` - AccountLimit *uint `json:"account_limit,omitempty"` - Price *float32 `json:"price,omitempty"` - DiscountedPrice *float32 `json:"discounted_price,omitempty"` - Duration *Duration `json:"duration,omitempty"` - DedicatedWorkers *int `json:"dedicated_workers,omitempty"` - DailyCampaignLimit *int `json:"daily_campaign_limit,omitempty"` - MaxCampaigns *int `json:"max_campaigns,omitempty"` - MaxActiveCampaigns *int `json:"max_active_campaigns,omitempty"` - MaxTeamMembers *int `json:"max_team_members,omitempty"` - MaxEmailAccounts *int `json:"max_email_accounts,omitempty"` - Public *bool `json:"public,omitempty"` -} - -// AdminEnterpriseInquiry represents an enterprise inquiry with admin details -type AdminEnterpriseInquiry struct { - ID uuid.UUID `json:"id"` - UserID *uuid.UUID `json:"user_id,omitempty"` - CompanyName string `json:"company_name"` - ContactName string `json:"contact_name"` - ContactEmail string `json:"contact_email"` - Phone *string `json:"phone,omitempty"` - TeamSize *string `json:"team_size,omitempty"` - EstimatedVolume *int `json:"estimated_volume,omitempty"` - MonthlyEmailVolume *string `json:"monthly_email_volume,omitempty"` - Message *string `json:"message,omitempty"` - Status string `json:"status"` - AssignedTo *uuid.UUID `json:"assigned_to,omitempty"` - Notes *string `json:"notes,omitempty"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - - // Joined data - User *AdminUserSummary `json:"user,omitempty"` - AssignedAdmin *AdminUserSummary `json:"assigned_admin,omitempty"` -} - -// AdminEnterpriseInquiriesResult represents paginated inquiries -type AdminEnterpriseInquiriesResult struct { - Data []AdminEnterpriseInquiry `json:"data"` - Pagination Pagination `json:"pagination"` -} - -// AdminEnterpriseInquirySearch are the query params for the enterprise inquiry -// queue. Mirrors AdminOrgSearch tag conventions. -type AdminEnterpriseInquirySearch struct { - Query string `form:"q"` - Status string `form:"status"` // pending, contacted, converted, declined, all - Assignment string `form:"assignment"` // assigned, unassigned - Linkage string `form:"linkage"` // linked, anonymous - - HasNotes bool `form:"has_notes"` - HasPhone bool `form:"has_phone"` - Processed bool `form:"processed"` - - // Count ranges - TeamSizeMin *int `form:"team_size_min"` - TeamSizeMax *int `form:"team_size_max"` - EstimatedVolumeMin *int `form:"estimated_volume_min"` - EstimatedVolumeMax *int `form:"estimated_volume_max"` - - // Date ranges (YYYY-MM-DD, UTC) - CreatedWithin int `form:"created_within"` - CreatedAfter *time.Time `form:"created_after" time_format:"2006-01-02" time_utc:"true"` - CreatedBefore *time.Time `form:"created_before" time_format:"2006-01-02" time_utc:"true"` - UpdatedAfter *time.Time `form:"updated_after" time_format:"2006-01-02" time_utc:"true"` - UpdatedBefore *time.Time `form:"updated_before" time_format:"2006-01-02" time_utc:"true"` - - Cursor *uuid.UUID `form:"cursor"` - Limit int `form:"limit"` - SortBy string `form:"sort_by"` // created_at, updated_at, company_name, contact_email, status, team_size, estimated_volume - SortDesc bool `form:"sort_desc"` -} - -// UpdateEnterpriseInquiryRequest represents the request to update an inquiry -type UpdateEnterpriseInquiryRequest struct { - Status *string `json:"status,omitempty"` - AssignedTo *uuid.UUID `json:"assigned_to,omitempty"` - Notes *string `json:"notes,omitempty"` -} - // AdminInfo represents an admin user for listing type AdminInfo struct { ID uuid.UUID `json:"id"` @@ -766,14 +650,6 @@ type WorkerStats struct { QueueDepth int64 `json:"queue_depth"` } -// EmailDistribution represents email distribution across workers -type EmailDistribution struct { - WorkerID uuid.UUID `json:"worker_id"` - WorkerName string `json:"worker_name"` - EmailCount int64 `json:"email_count"` - Percentage float64 `json:"percentage"` -} - // AdminOrgSearch are the query params for the admin organization listing. type AdminOrgSearch struct { Query string `form:"q"` @@ -923,43 +799,6 @@ type AdminLimitRequestsResult struct { Pagination Pagination `json:"pagination"` } -// AdminPlanSearch are the query params for the admin plan catalog listing. -// Mirrors AdminOrgSearch tag conventions. price_* bind to *int (whole-dollar -// filtering against the numeric price column). -type AdminPlanSearch struct { - Query string `form:"q"` - Visibility string `form:"visibility"` // public, private, "" = any - Duration string `form:"duration"` // month, year, "" = any (durations.title) - - AIGeneration bool `form:"ai_generation"` - HasStripe bool `form:"has_stripe"` - HasSubscribers bool `form:"has_subscribers"` - - // Numeric ranges - PriceMin *int `form:"price_min"` - PriceMax *int `form:"price_max"` - DailyEmailsMin *int `form:"daily_emails_min"` - DailyEmailsMax *int `form:"daily_emails_max"` - AccountLimitMin *int `form:"account_limit_min"` - AccountLimitMax *int `form:"account_limit_max"` - - // Date range - CreatedWithin int `form:"created_within"` // days; 0 = any - CreatedAfter *time.Time `form:"created_after" time_format:"2006-01-02" time_utc:"true"` - CreatedBefore *time.Time `form:"created_before" time_format:"2006-01-02" time_utc:"true"` - - Cursor *uuid.UUID `form:"cursor"` - Limit int `form:"limit"` - SortBy string `form:"sort_by"` // price, name, daily_emails, account_limit, created_at - SortDesc bool `form:"sort_desc"` -} - -// AdminPlansResult is the paginated response for the admin plan listing. -type AdminPlansResult struct { - Data []Plan `json:"data"` - Pagination Pagination `json:"pagination"` -} - // AdminOrgDetail is the full payload for the org detail page. Carries // three limit blocks side-by-side so the UI can explain *why* each // effective number is what it is: diff --git a/internal/models/admin_ops.go b/internal/models/admin_ops.go new file mode 100644 index 00000000..ed33c4bd --- /dev/null +++ b/internal/models/admin_ops.go @@ -0,0 +1,357 @@ +package models + +import ( + "encoding/json" + "time" + + "github.com/google/uuid" +) + +// Operator-facing read models for the admin panel's operations pages: mailbox +// sync, in-flight sends, dead letters, scheduled jobs, fleet placement, +// workspace transfers and abuse signals. Every row here is instance-wide and +// carries the organization it belongs to, because the operator reads across +// workspaces. + +// ---- mailbox sync ---- + +// AdminSyncSearch filters the sync governor page. +type AdminSyncSearch struct { + // State: all | throttled | backfilling | stalled | pending | complete. + State string `form:"state"` + Q string `form:"q"` + Cursor string `form:"cursor"` + Limit int `form:"limit"` +} + +// AdminSyncSummary counts every mailbox that has reported sync state once. +type AdminSyncSummary struct { + Total int `json:"total"` + Throttled int `json:"throttled"` + Backfilling int `json:"backfilling"` + Stalled int `json:"stalled"` + Pending int `json:"pending"` + Complete int `json:"complete"` + Deferred int `json:"deferred"` +} + +// AdminSyncRow is one mailbox's sync state with its owner attached. +type AdminSyncRow struct { + EmailID uuid.UUID `json:"email_id"` + UserID uuid.UUID `json:"user_id"` + Email string `json:"email"` + Provider string `json:"provider"` + AccountStatus string `json:"account_status"` + OrganizationID *uuid.UUID `json:"organization_id,omitempty"` + OrganizationName string `json:"organization_name"` + WorkerID *uuid.UUID `json:"worker_id,omitempty"` + + BackfillStatus string `json:"backfill_status"` + BackfillSynced int `json:"backfill_synced"` + BackfillSince *time.Time `json:"backfill_since,omitempty"` + BackfillStartedAt *time.Time `json:"backfill_started_at,omitempty"` + BackfillCompletedAt *time.Time `json:"backfill_completed_at,omitempty"` + + ThrottledUntil *time.Time `json:"throttled_until,omitempty"` + ThrottleReason string `json:"throttle_reason"` + Deferred int `json:"deferred"` + // Stalled is a running backfill whose state has not moved for an hour. + Stalled bool `json:"stalled"` + LastSyncedAt *time.Time `json:"last_synced_at,omitempty"` + UpdatedAt time.Time `json:"updated_at"` +} + +type AdminSyncResult struct { + Data []AdminSyncRow `json:"data"` + Pagination *Pagination `json:"pagination"` + Summary AdminSyncSummary `json:"summary"` +} + +// ---- in-flight sends ---- + +// AdminInFlightSend is a reserved campaign send no worker result has resolved. +type AdminInFlightSend struct { + CampaignID uuid.UUID `json:"campaign_id"` + CampaignName string `json:"campaign_name"` + OrganizationID *uuid.UUID `json:"organization_id,omitempty"` + OrganizationName string `json:"organization_name"` + ContactID uuid.UUID `json:"contact_id"` + ContactEmail string `json:"contact_email"` + SequenceID uuid.UUID `json:"sequence_id"` + TaskID *uuid.UUID `json:"task_id,omitempty"` + TaskStatus string `json:"task_status"` + // HasMessageID means the worker did put the mail on the wire and only the + // stamp was lost; the reclaimer will stamp it rather than retry. + HasMessageID bool `json:"has_message_id"` + EmailAccountID *uuid.UUID `json:"email_account_id,omitempty"` + MailboxEmail string `json:"mailbox_email"` + WorkerID *uuid.UUID `json:"worker_id,omitempty"` + DispatchedAt time.Time `json:"dispatched_at"` + AgeSeconds int64 `json:"age_seconds"` +} + +type AdminInFlightSummary struct { + Total int `json:"total"` + // Age buckets, in minutes since dispatch. + Under5m int `json:"under_5m"` + Under30m int `json:"under_30m"` + PastReclaimWindow int `json:"past_reclaim_window"` + OldestDispatched *time.Time `json:"oldest_dispatched_at,omitempty"` + // ReclaimAfterMinutes is config.CampaignSendReclaimAfterMinutes, so the + // page can say when the sweep will pick a row up. + ReclaimAfterMinutes int `json:"reclaim_after_minutes"` +} + +type AdminInFlightResult struct { + Summary AdminInFlightSummary `json:"summary"` + Data []AdminInFlightSend `json:"data"` +} + +// AdminDeadLetterRow is a task dead letter with the workspace it belongs to. +type AdminDeadLetterRow struct { + TaskDeadLetter + OrganizationID *uuid.UUID `json:"organization_id,omitempty"` + OrganizationName string `json:"organization_name"` +} + +type AdminDeadLettersResult struct { + Data []AdminDeadLetterRow `json:"data"` + Pagination *Pagination `json:"pagination"` + // Counts by status across the instance, independent of the filter. + Pending int `json:"pending"` + Replayed int `json:"replayed"` + Failed int `json:"failed"` +} + +// AdminTaskFailureRow is one row of task_failures with the task it names. +type AdminTaskFailureRow struct { + TaskID uuid.UUID `json:"task_id"` + TaskType string `json:"task_type"` + TaskStatus string `json:"task_status"` + Title string `json:"title"` + Message string `json:"message"` + EmailAccountID uuid.UUID `json:"email_account_id"` + MailboxEmail string `json:"mailbox_email"` + OrganizationID *uuid.UUID `json:"organization_id,omitempty"` + OrganizationName string `json:"organization_name"` + OccurredAt time.Time `json:"occurred_at"` +} + +// AdminWebhookEndpointRow is a customer webhook endpoint as the operator sees it. +type AdminWebhookEndpointRow struct { + ID uuid.UUID `json:"id"` + OrganizationID uuid.UUID `json:"organization_id"` + OrganizationName string `json:"organization_name"` + URL string `json:"url"` + Description string `json:"description"` + Enabled bool `json:"enabled"` + EventTypes []string `json:"event_types"` + ConsecutiveFailures int `json:"consecutive_failures"` + LastSuccessAt *time.Time `json:"last_success_at,omitempty"` + LastFailureAt *time.Time `json:"last_failure_at,omitempty"` + LastFailureReason string `json:"last_failure_reason"` + DeliveriesLast7d int64 `json:"deliveries_last_7d"` + FailedLast7d int64 `json:"failed_last_7d"` + DropsLast7d int64 `json:"drops_last_7d"` +} + +// AdminWebhookHealth is the instance-wide delivery picture. +type AdminWebhookHealth struct { + // InFlightStale counts deliveries claimed longer ago than the lease. + InFlightStale int64 `json:"in_flight_stale"` + PendingDue int64 `json:"pending_due"` + DeliveredLast24h int64 `json:"delivered_last_24h"` + FailedLast24h int64 `json:"failed_last_24h"` + AbandonedLast24h int64 `json:"abandoned_last_24h"` + DropsLast7d int64 `json:"drops_last_7d"` + LeaseMinutes int `json:"lease_minutes"` + // FailingEndpoints lists endpoints with consecutive failures, worst first. + FailingEndpoints []AdminWebhookEndpointRow `json:"failing_endpoints"` +} + +// ---- scheduled jobs ---- + +// ScheduledJobRun is the persisted record of one background loop: what it is, +// when it last ran and how it went. One row per job name, shared by every +// service that runs loops (backend, consumer). +type ScheduledJobRun struct { + Name string `json:"name"` + Service string `json:"service"` + IntervalSeconds int `json:"interval_seconds"` + LastStartedAt *time.Time `json:"last_started_at,omitempty"` + LastFinishedAt *time.Time `json:"last_finished_at,omitempty"` + LastDurationMs int64 `json:"last_duration_ms"` + // LastStatus: idle | running | ok | error. + LastStatus string `json:"last_status"` + LastError string `json:"last_error"` + RunCount int64 `json:"run_count"` + ErrorCount int64 `json:"error_count"` + // RunRequestedAt is set by the panel's "run now"; the owning loop clears it + // when it picks the request up. + RunRequestedAt *time.Time `json:"run_requested_at,omitempty"` + NextRunAt *time.Time `json:"next_run_at,omitempty"` + UpdatedAt time.Time `json:"updated_at"` +} + +// ---- fleet ---- + +// AdminFleetWorkerRow is a worker with its capacity-view row attached. +type AdminFleetWorkerRow struct { + WorkerID uuid.UUID `json:"worker_id"` + Name string `json:"name"` + IPAddr string `json:"ip_addr"` + Active bool `json:"active"` + FreeTier bool `json:"free_tier"` + WorkerType WorkerType `json:"worker_type"` + RiskPool WorkerRiskPool `json:"risk_pool"` + EgressKind WorkerEgressKind `json:"egress_kind"` + HealthState WorkerHealthState `json:"health_state"` + InstallState string `json:"install_state"` + LastSeenAt *time.Time `json:"last_seen_at,omitempty"` + Live bool `json:"live"` + AccountCount int `json:"account_count"` + Tags []string `json:"tags"` + + LoadScore float64 `json:"load_score"` + BaseCapacity float64 `json:"base_capacity"` + HealthMultiplier float64 `json:"health_multiplier"` + AgeMultiplier float64 `json:"age_multiplier"` + EffectiveCapacity float64 `json:"effective_capacity"` + // Utilization is load over effective capacity; the rebalancer calls a + // worker hot above 0.8 and cold below 0.5. + Utilization float64 `json:"utilization"` + SendsAttempted1h int64 `json:"sends_attempted_1h"` + SendsSucceeded1h int64 `json:"sends_succeeded_1h"` + BouncesHard1h int64 `json:"bounces_hard_1h"` + BouncesSoft1h int64 `json:"bounces_soft_1h"` + Complaints1h int64 `json:"complaints_1h"` + AuthErrors1h int64 `json:"auth_errors_1h"` +} + +// AdminFleetDecision is one decision_log row: what a control loop did and why. +type AdminFleetDecision struct { + ID int64 `json:"id"` + Kind string `json:"kind"` + WorkerID *uuid.UUID `json:"worker_id,omitempty"` + WorkerName string `json:"worker_name"` + MailboxID *uuid.UUID `json:"mailbox_id,omitempty"` + Before json.RawMessage `json:"before,omitempty"` + After json.RawMessage `json:"after,omitempty"` + Reason string `json:"reason"` + TriggeredBy string `json:"triggered_by"` + CreatedAt time.Time `json:"created_at"` +} + +// AdminDedicatedAssignment is an active worker-to-organization binding. +type AdminDedicatedAssignment struct { + ID uuid.UUID `json:"id"` + WorkerID uuid.UUID `json:"worker_id"` + WorkerName string `json:"worker_name"` + WorkerLive bool `json:"worker_live"` + OrganizationID uuid.UUID `json:"organization_id"` + OrganizationName string `json:"organization_name"` + SubscriptionID uuid.UUID `json:"subscription_id"` + AssignedAt time.Time `json:"assigned_at"` + ReleasedAt *time.Time `json:"released_at,omitempty"` + AccountCount int `json:"account_count"` +} + +// AdminConvertDedicatedRequest is the body of POST /admin/workers/:id/convert-dedicated. +type AdminConvertDedicatedRequest struct { + OrganizationID string `json:"organization_id"` + SubscriptionID string `json:"subscription_id"` + DrainToWorkerID *string `json:"drain_to_worker_id"` +} + +// ---- workspace transfers ---- + +// AdminTransferJob is an export or import job with its workspace attached. +type AdminTransferJob struct { + // Kind: export | import. + Kind string `json:"kind"` + ID uuid.UUID `json:"id"` + OrganizationID uuid.UUID `json:"organization_id"` + OrganizationName string `json:"organization_name"` + RequestedBy *uuid.UUID `json:"requested_by,omitempty"` + RequestedByEmail string `json:"requested_by_email"` + Status OrgTransferStatus `json:"status"` + Groups []OrgDataGroup `json:"groups"` + IncludeSecrets bool `json:"include_secrets"` + ProgressPercent int `json:"progress_percent"` + ProgressStage string `json:"progress_stage"` + ArchiveBytes *int64 `json:"archive_bytes,omitempty"` + ErrorMessage *string `json:"error_message,omitempty"` + StartedAt *time.Time `json:"started_at,omitempty"` + CompletedAt *time.Time `json:"completed_at,omitempty"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` + CreatedAt time.Time `json:"created_at"` +} + +// ---- abuse and insight ---- + +// AdminWarmupAbuseRow ranks a mailbox by invalid warmup-token attempts. +type AdminWarmupAbuseRow struct { + EmailAccountID uuid.UUID `json:"email_account_id"` + Email string `json:"email"` + OrganizationID *uuid.UUID `json:"organization_id,omitempty"` + OrganizationName string `json:"organization_name"` + Attempts int `json:"attempts"` + LastAttemptAt time.Time `json:"last_attempt_at"` + Blocked bool `json:"blocked"` + SpamScore int `json:"spam_score"` + HealthState string `json:"health_state"` +} + +// AdminWarmupAction is one warmup_admin_actions row with both parties named. +type AdminWarmupAction struct { + ID uuid.UUID `json:"id"` + AdminUserID uuid.UUID `json:"admin_user_id"` + AdminEmail string `json:"admin_email"` + EmailAccountID uuid.UUID `json:"email_account_id"` + Email string `json:"email"` + Action string `json:"action"` + Reason *string `json:"reason,omitempty"` + CreatedAt time.Time `json:"created_at"` +} + +// AdminOrgAPIKey is an API key as the operator sees it: never the secret. +type AdminOrgAPIKey struct { + ID uuid.UUID `json:"id"` + Name string `json:"name"` + KeyPrefix string `json:"key_prefix"` + KeySuffix string `json:"key_suffix"` + Status string `json:"status"` + Permissions int64 `json:"permissions"` + UserID uuid.UUID `json:"user_id"` + UserEmail string `json:"user_email"` + LastUsedAt *time.Time `json:"last_used_at,omitempty"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` + RevokedAt *time.Time `json:"revoked_at,omitempty"` + CreatedAt time.Time `json:"created_at"` + RequestsLast7d int64 `json:"requests_last_7d"` +} + +// AdminAcquisitionChannel is signups grouped by UTM source and medium. +type AdminAcquisitionChannel struct { + Source string `json:"source"` + Medium string `json:"medium"` + Signups int `json:"signups"` + Converted int `json:"converted"` +} + +type AdminAcquisitionReferrer struct { + Host string `json:"host"` + Signups int `json:"signups"` +} + +// AdminAcquisition is signups by channel over a window, with how many of them +// went on to a paid subscription and how many trials are about to end. +type AdminAcquisition struct { + Days int `json:"days"` + Signups int `json:"signups"` + WithChannel int `json:"with_channel"` + Converted int `json:"converted"` + TrialsExpiring7d int `json:"trials_expiring_7d"` + Channels []AdminAcquisitionChannel `json:"channels"` + Referrers []AdminAcquisitionReferrer `json:"referrers"` +} diff --git a/internal/models/admin_permission.go b/internal/models/admin_permission.go index 4cb3c157..6474bf48 100644 --- a/internal/models/admin_permission.go +++ b/internal/models/admin_permission.go @@ -3,12 +3,14 @@ package models // AdminPermission represents platform-level admin permissions as a bitmask type AdminPermission uint32 +// Bit positions are stored in users.admin_permissions, so a retired bit keeps +// its placeholder here and is never reused. const ( // User Management (bits 0-3) - AdminPermViewUsers AdminPermission = 1 << iota // View user profiles - AdminPermBanUsers // Ban/unban users - AdminPermEditUsers // Edit user details - AdminPermImpersonateUsers // Login as user (support) + AdminPermViewUsers AdminPermission = 1 << iota // View user profiles + AdminPermBanUsers // Ban/unban users + adminPermReserved2 // Retired (edit users), never reuse + adminPermReserved3 // Retired (impersonate users), never reuse // Worker Management (bits 4-5) AdminPermViewWorkers // View worker list @@ -31,11 +33,11 @@ const ( AdminPermManageRateLimits // User rate limits AdminPermManageSettings // Platform settings - // Enterprise (bits 15-18) - AdminPermViewEnterpriseInquiries // View inquiries - AdminPermManageEnterpriseInquiries // Process inquiries - AdminPermManagePlans // Create/edit custom plans - AdminPermManageBilling // Refunds, adjust billing + // Retired enterprise bits (15-18), never reuse + adminPermReserved15 // Retired (view enterprise inquiries) + adminPermReserved16 // Retired (manage enterprise inquiries) + adminPermReserved17 // Retired (manage plans) + adminPermReserved18 // Retired (manage billing) // Super Admin (bit 19) AdminPermGrantAdminAccess // Grant/revoke admin permissions @@ -45,10 +47,28 @@ const ( AdminPermManageOrganizations // Set per-org limit overrides, ban scope, etc. ) -// AllAdminPermissions contains all admin permissions. Bump the shift -// whenever a new bit is added above. +// AllAdminPermissions is the full mask, retired bits included: it is what +// existing super admins hold in the database. Bump the shift whenever a new +// bit is added above. const AllAdminPermissions AdminPermission = (1 << 22) - 1 +// LiveAdminPermissions ORs every bit that still gates a route. +const LiveAdminPermissions AdminPermission = AdminPermViewUsers | AdminPermBanUsers | + AdminPermViewWorkers | AdminPermManageWorkers | + AdminPermViewWarmupPool | AdminPermManageWarmupBans | AdminPermReviewAppeals | + AdminPermViewCampaigns | AdminPermStopCampaigns | + AdminPermViewAnalytics | AdminPermViewAuditLogs | + AdminPermManageRateLimits | AdminPermManageSettings | + AdminPermGrantAdminAccess | + AdminPermViewOrganizations | AdminPermManageOrganizations + +// retiredAdminPermissions are the bits that no longer gate anything. +const retiredAdminPermissions = adminPermReserved2 | adminPermReserved3 | + adminPermReserved15 | adminPermReserved16 | adminPermReserved17 | adminPermReserved18 + +// Compile-time check: every bit in AllAdminPermissions is live or retired. +var _ [0]struct{} = [AllAdminPermissions ^ (LiveAdminPermissions | retiredAdminPermissions)]struct{}{} + // HasPermission checks if the permission bitmask contains the specified permission func (p AdminPermission) HasPermission(perm AdminPermission) bool { return p&perm == perm @@ -69,9 +89,9 @@ func (p AdminPermission) IsAdmin() bool { return p > 0 } -// IsSuperAdmin returns true if the user has all admin permissions +// IsSuperAdmin returns true if the user holds every live admin permission func (p AdminPermission) IsSuperAdmin() bool { - return p == AllAdminPermissions + return p&LiveAdminPermissions == LiveAdminPermissions } // AdminRoleName represents predefined admin role names @@ -89,7 +109,7 @@ var AdminRolePermissions = map[AdminRoleName]AdminPermission{ AdminRoleSuper: AllAdminPermissions, AdminRoleSupport: AdminPermViewUsers | AdminPermViewCampaigns | AdminPermViewWarmupPool | AdminPermManageWarmupBans | AdminPermReviewAppeals | AdminPermViewAuditLogs | - AdminPermViewEnterpriseInquiries | AdminPermViewOrganizations, + AdminPermViewOrganizations, AdminRoleOps: AdminPermViewWorkers | AdminPermManageWorkers | AdminPermViewAnalytics | AdminPermViewAuditLogs | AdminPermManageRateLimits | AdminPermViewOrganizations, AdminRoleAnalyst: AdminPermViewUsers | AdminPermViewCampaigns | AdminPermViewAnalytics | @@ -112,14 +132,12 @@ type PermissionInfo struct { Category string `json:"category"` } -// GetAllPermissionInfos returns information about all admin permissions +// GetAllPermissionInfos returns information about all live admin permissions func GetAllPermissionInfos() []PermissionInfo { return []PermissionInfo{ // User Management {Name: "view_users", Permission: AdminPermViewUsers, Description: "View user profiles and details", Category: "User Management"}, {Name: "ban_users", Permission: AdminPermBanUsers, Description: "Ban and unban users", Category: "User Management"}, - {Name: "edit_users", Permission: AdminPermEditUsers, Description: "Edit user details", Category: "User Management"}, - {Name: "impersonate_users", Permission: AdminPermImpersonateUsers, Description: "Login as user for support", Category: "User Management"}, // Worker Management {Name: "view_workers", Permission: AdminPermViewWorkers, Description: "View worker list and status", Category: "Worker Management"}, @@ -142,12 +160,6 @@ func GetAllPermissionInfos() []PermissionInfo { {Name: "manage_rate_limits", Permission: AdminPermManageRateLimits, Description: "Manage user rate limits", Category: "Settings"}, {Name: "manage_settings", Permission: AdminPermManageSettings, Description: "Manage platform settings", Category: "Settings"}, - // Enterprise - {Name: "view_enterprise_inquiries", Permission: AdminPermViewEnterpriseInquiries, Description: "View enterprise inquiries", Category: "Enterprise"}, - {Name: "manage_enterprise_inquiries", Permission: AdminPermManageEnterpriseInquiries, Description: "Process enterprise inquiries", Category: "Enterprise"}, - {Name: "manage_plans", Permission: AdminPermManagePlans, Description: "Create and edit custom plans", Category: "Enterprise"}, - {Name: "manage_billing", Permission: AdminPermManageBilling, Description: "Manage refunds and billing", Category: "Enterprise"}, - // Super Admin {Name: "grant_admin_access", Permission: AdminPermGrantAdminAccess, Description: "Grant or revoke admin permissions", Category: "Super Admin"}, diff --git a/internal/repository/admin_schema_live_test.go b/internal/repository/admin_schema_live_test.go index e7f2d2a9..d38c45f8 100644 --- a/internal/repository/admin_schema_live_test.go +++ b/internal/repository/admin_schema_live_test.go @@ -85,28 +85,6 @@ func newAdminFixture(t *testing.T, pool *pgxpool.Pool) *adminFixture { return f } -// ensureDuration returns the id of the durations row with this title, creating -// it for the duration of the test if the instance does not have one. -func ensureDuration(t *testing.T, pool *pgxpool.Pool, title string) uuid.UUID { - t.Helper() - ctx := context.Background() - var id uuid.UUID - err := pool.QueryRow(ctx, `SELECT id FROM durations WHERE title = $1`, title).Scan(&id) - if err == nil { - return id - } - id = uuid.New() - if _, err := pool.Exec(ctx, `INSERT INTO durations (id, title) VALUES ($1, $2)`, id, title); err != nil { - t.Fatalf("create durations row %q: %v", title, err) - } - t.Cleanup(func() { - if _, err := pool.Exec(context.Background(), `DELETE FROM durations WHERE id = $1`, id); err != nil { - t.Errorf("cleanup durations row %q: %v", title, err) - } - }) - return id -} - // The email-account section of the user preview compared a uuid column against // a text parameter, so the query errored and the caller swallowed it: every // user looked like they had no mailboxes. @@ -220,82 +198,6 @@ func TestLiveAdminForceStopLeavesTerminalCampaignsAlone(t *testing.T) { } } -// plans stores the billing period as duration_id (FK to durations); the admin -// writes named a `duration` column that does not exist. -func TestLiveAdminPlanRoundTripsDuration(t *testing.T) { - _, pool := liveContactDB(t) - repo := NewAdminRepository(pool) - ctx := context.Background() - - // A bare install ships only the monthly duration (migration 000080); the - // yearly one arrives with the seed. Add whatever is missing and take it - // back out again, so this runs on either. - monthID := ensureDuration(t, pool, "month") - yearID := ensureDuration(t, pool, "year") - - resolved, err := repo.DurationIDByTitle(ctx, "month") - if err != nil { - t.Fatalf("DurationIDByTitle(month): %v", err) - } - if resolved == nil || *resolved != monthID { - t.Fatalf("DurationIDByTitle(month) = %v, want %v", resolved, monthID) - } - if unknown, err := repo.DurationIDByTitle(ctx, "fortnight"); err != nil || unknown != nil { - t.Fatalf("DurationIDByTitle(fortnight) = %v, %v; want nil, nil so the API can answer 400", unknown, err) - } - - name := "Issue 209 plan" - plan := &models.Plan{ - ID: uuid.New(), - Name: &name, - MaxContacts: 1000, - DailyEmails: 50, - AccountLimit: 3, - Price: 49, - Duration: models.DurationMonth, - MonthlyCredits: 25, - } - t.Cleanup(func() { - if _, err := pool.Exec(context.Background(), `DELETE FROM plans WHERE id = $1`, plan.ID); err != nil { - t.Errorf("cleanup plan: %v", err) - } - }) - - if err := repo.CreatePlan(ctx, plan, monthID); err != nil { - t.Fatalf("CreatePlan: %v", err) - } - - got, err := repo.GetPlan(ctx, plan.ID) - if err != nil { - t.Fatalf("GetPlan: %v", err) - } - if got == nil { - t.Fatal("GetPlan found nothing for a plan that was just created") - } - if got.Duration != models.DurationMonth { - t.Fatalf("plan reads back duration %q, want %q", got.Duration, models.DurationMonth) - } - if got.MonthlyCredits != 25 { - t.Fatalf("plan reads back %d monthly credits, want 25", got.MonthlyCredits) - } - - got.Duration = models.DurationYear - got.Price = 490 - if err := repo.UpdatePlan(ctx, got, yearID); err != nil { - t.Fatalf("UpdatePlan: %v", err) - } - after, err := repo.GetPlan(ctx, plan.ID) - if err != nil { - t.Fatalf("GetPlan after update: %v", err) - } - if after.Duration != models.DurationYear { - t.Fatalf("plan reads back duration %q after switching to yearly, want %q", after.Duration, models.DurationYear) - } - if after.Price != 490 { - t.Fatalf("plan reads back price %v, want 490", after.Price) - } -} - // user_rate_limits has limit_api_calls_daily / limit_bulk_ops_daily and no // daily_email_limit, so both the read and the write failed every time. func TestLiveAdminUserRateLimitsRoundTrip(t *testing.T) { diff --git a/internal/repository/pg_admin.go b/internal/repository/pg_admin.go index 3eb25d09..704ca377 100644 --- a/internal/repository/pg_admin.go +++ b/internal/repository/pg_admin.go @@ -66,25 +66,8 @@ type AdminRepository interface { GetPlatformOverview(ctx context.Context) (*models.PlatformOverview, error) GetDailyEmailStats(ctx context.Context, startDate, endDate time.Time) ([]models.DailyEmailStats, error) GetHourlyEmailStats(ctx context.Context, date time.Time) ([]models.HourlyEmailStats, error) - GetWorkerLoadStats(ctx context.Context) ([]models.WorkerLoadStats, error) GetUserGrowthStats(ctx context.Context, startDate, endDate time.Time) ([]models.UserGrowthStats, error) GetAnalyticsTrends(ctx context.Context) (*models.AnalyticsTrends, error) - GetEmailDistribution(ctx context.Context) ([]models.EmailDistribution, error) - - // Plans - ListPlans(ctx context.Context, includePrivate bool) ([]models.Plan, error) - SearchPlansForAdmin(ctx context.Context, search *models.AdminPlanSearch) (*models.AdminPlansResult, error) - DurationIDByTitle(ctx context.Context, title string) (*uuid.UUID, error) - CreatePlan(ctx context.Context, plan *models.Plan, durationID uuid.UUID) error - GetPlan(ctx context.Context, planID uuid.UUID) (*models.Plan, error) - UpdatePlan(ctx context.Context, plan *models.Plan, durationID uuid.UUID) error - DeletePlan(ctx context.Context, planID uuid.UUID) error - IsPlanInUse(ctx context.Context, planID uuid.UUID) (bool, error) - - // Enterprise Inquiries - ListEnterpriseInquiries(ctx context.Context, search *models.AdminEnterpriseInquirySearch) (*models.AdminEnterpriseInquiriesResult, error) - GetEnterpriseInquiry(ctx context.Context, id uuid.UUID) (*models.AdminEnterpriseInquiry, error) - UpdateEnterpriseInquiry(ctx context.Context, id uuid.UUID, update *models.UpdateEnterpriseInquiryRequest) error // User Rate Limits GetUserRateLimits(ctx context.Context, userID uuid.UUID) (*models.AdminUserRateLimits, error) @@ -1793,34 +1776,6 @@ func (r *adminRepository) GetHourlyEmailStats(ctx context.Context, date time.Tim return []models.HourlyEmailStats{}, nil } -// GetWorkerLoadStats gets worker load statistics -func (r *adminRepository) GetWorkerLoadStats(ctx context.Context) ([]models.WorkerLoadStats, error) { - query := ` - SELECT w.id, w.name, - (SELECT COUNT(*) FROM email_accounts ea WHERE ea.worker_id = w.id) as connected_emails - FROM workers w - WHERE w.active = true - ` - - rows, err := r.db.Query(ctx, query) - if err != nil { - return nil, err - } - defer rows.Close() - - var stats []models.WorkerLoadStats - for rows.Next() { - var s models.WorkerLoadStats - err := rows.Scan(&s.WorkerID, &s.WorkerName, &s.ConnectedEmails) - if err != nil { - return nil, err - } - stats = append(stats, s) - } - - return stats, nil -} - // GetUserGrowthStats gets user growth statistics func (r *adminRepository) GetUserGrowthStats(ctx context.Context, startDate, endDate time.Time) ([]models.UserGrowthStats, error) { query := ` @@ -1866,606 +1821,6 @@ func (r *adminRepository) GetAnalyticsTrends(ctx context.Context) (*models.Analy return trends, nil } -// GetEmailDistribution gets email distribution across workers -func (r *adminRepository) GetEmailDistribution(ctx context.Context) ([]models.EmailDistribution, error) { - query := ` - SELECT w.id, w.name, COUNT(ea.id) as email_count - FROM workers w - LEFT JOIN email_accounts ea ON ea.worker_id = w.id - GROUP BY w.id, w.name - ORDER BY email_count DESC - ` - - rows, err := r.db.Query(ctx, query) - if err != nil { - return nil, err - } - defer rows.Close() - - var total int64 - var distributions []models.EmailDistribution - for rows.Next() { - var d models.EmailDistribution - err := rows.Scan(&d.WorkerID, &d.WorkerName, &d.EmailCount) - if err != nil { - return nil, err - } - total += d.EmailCount - distributions = append(distributions, d) - } - - // Calculate percentages - for i := range distributions { - if total > 0 { - distributions[i].Percentage = float64(distributions[i].EmailCount) / float64(total) * 100 - } - } - - return distributions, nil -} - -// ListPlans lists all plans -func (r *adminRepository) ListPlans(ctx context.Context, includePrivate bool) ([]models.Plan, error) { - whereClause := "" - if !includePrivate { - whereClause = "WHERE p.public = true" - } - - query := ` - SELECT p.id, p.name, p.max_contacts, p.daily_emails, p.ai_generation, p.account_limit, - p.price, p.discounted_price, d.title, p.savings, p.public, - p.stripe_price_id, p.stripe_product_id, p.dedicated_workers, p.daily_campaign_limit, - p.max_campaigns, p.max_active_campaigns, p.max_team_members, p.max_email_accounts, - p.updated_at, p.created_at - FROM plans p - LEFT JOIN durations d ON d.id = p.duration_id - ` + whereClause + ` - ORDER BY p.price ASC - ` - - rows, err := r.db.Query(ctx, query) - if err != nil { - return nil, err - } - defer rows.Close() - - var plans []models.Plan - for rows.Next() { - var p models.Plan - var duration *string - err := rows.Scan( - &p.ID, &p.Name, &p.MaxContacts, &p.DailyEmails, &p.AIGeneration, &p.AccountLimit, - &p.Price, &p.DiscountedPrice, &duration, &p.Savings, &p.Public, - &p.StripePriceID, &p.StripeProductID, &p.DedicatedWorkers, &p.DailyCampaignLimit, - &p.MaxCampaigns, &p.MaxActiveCampaigns, &p.MaxTeamMembers, &p.MaxEmailAccounts, - &p.UpdatedAt, &p.CreatedAt, - ) - if err != nil { - return nil, err - } - if duration != nil { - p.Duration = models.Duration(*duration) - } - plans = append(plans, p) - } - - return plans, nil -} - -// SearchPlansForAdmin is the faceted + cursor-paginated plan catalog query. -// Mirrors SearchOrganizationsForAdmin. Plans is a small table so the pager is -// usually inert, but the {data,pagination} envelope keeps the Explorer stack -// uniform. -func (r *adminRepository) SearchPlansForAdmin(ctx context.Context, search *models.AdminPlanSearch) (*models.AdminPlansResult, error) { - limit := search.Limit - if limit <= 0 || limit > 100 { - limit = 50 - } - - args := []interface{}{} - argNum := 1 - where := "WHERE 1=1" - - if search.Query != "" { - where += ` AND (p.name ILIKE $` + itoa(argNum) + ` OR p.stripe_price_id ILIKE $` + itoa(argNum) + ` OR p.stripe_product_id ILIKE $` + itoa(argNum) + `)` - args = append(args, "%"+search.Query+"%") - argNum++ - } - switch search.Visibility { - case "public": - where += ` AND p.public = TRUE` - case "private": - where += ` AND p.public = FALSE` - } - if search.Duration != "" { - where += ` AND d.title = $` + itoa(argNum) - args = append(args, search.Duration) - argNum++ - } - if search.AIGeneration { - where += ` AND p.ai_generation = TRUE` - } - if search.HasStripe { - where += ` AND p.stripe_price_id IS NOT NULL AND p.stripe_price_id <> ''` - } - if search.HasSubscribers { - where += ` AND EXISTS (SELECT 1 FROM subscriptions s WHERE s.plan_id = p.id)` - } - - addInt := func(frag string, v *int) { - if v != nil { - where += " AND " + fmt.Sprintf(frag, argNum) - args = append(args, *v) - argNum++ - } - } - addAfter := func(col string, v *time.Time) { - if v != nil { - where += " AND " + col + " >= $" + itoa(argNum) - args = append(args, *v) - argNum++ - } - } - addBefore := func(col string, v *time.Time) { - if v != nil { - where += " AND " + col + " < ($" + itoa(argNum) + " + INTERVAL '1 day')" - args = append(args, *v) - argNum++ - } - } - - addInt(`p.price >= $%d`, search.PriceMin) - addInt(`p.price <= $%d`, search.PriceMax) - addInt(`p.daily_emails >= $%d`, search.DailyEmailsMin) - addInt(`p.daily_emails <= $%d`, search.DailyEmailsMax) - addInt(`p.account_limit >= $%d`, search.AccountLimitMin) - addInt(`p.account_limit <= $%d`, search.AccountLimitMax) - - if search.CreatedWithin > 0 { - where += ` AND p.created_at >= NOW() - ($` + itoa(argNum) + `::int * INTERVAL '1 day')` - args = append(args, search.CreatedWithin) - argNum++ - } - addAfter("p.created_at", search.CreatedAfter) - addBefore("p.created_at", search.CreatedBefore) - - if search.Cursor != nil { - where += ` AND p.id < $` + itoa(argNum) - args = append(args, *search.Cursor) - argNum++ - } - - orderCol := "p.price" - switch search.SortBy { - case "name": - orderCol = "p.name" - case "daily_emails": - orderCol = "p.daily_emails" - case "account_limit": - orderCol = "p.account_limit" - case "created_at": - orderCol = "p.created_at" - } - // Catalog defaults to cheapest-first (ASC); explicit sorts honor sort_desc. - orderDir := "ASC" - if search.SortBy != "" && search.SortDesc { - orderDir = "DESC" - } - orderBy := "ORDER BY " + orderCol + " " + orderDir + ", p.id DESC" - - args = append(args, limit+1) - - query := ` - SELECT p.id, p.name, p.max_contacts, p.daily_emails, p.ai_generation, p.account_limit, - p.price, p.discounted_price, d.title, p.savings, p.public, - p.stripe_price_id, p.stripe_product_id, p.dedicated_workers, p.daily_campaign_limit, - p.max_campaigns, p.max_active_campaigns, p.max_team_members, p.max_email_accounts, - p.updated_at, p.created_at - FROM plans p - LEFT JOIN durations d ON d.id = p.duration_id - ` + where + ` - ` + orderBy + ` - LIMIT $` + itoa(argNum) - - rows, err := r.db.Query(ctx, query, args...) - if err != nil { - return nil, err - } - defer rows.Close() - - plans := []models.Plan{} - for rows.Next() { - var p models.Plan - var duration *string - if err := rows.Scan( - &p.ID, &p.Name, &p.MaxContacts, &p.DailyEmails, &p.AIGeneration, &p.AccountLimit, - &p.Price, &p.DiscountedPrice, &duration, &p.Savings, &p.Public, - &p.StripePriceID, &p.StripeProductID, &p.DedicatedWorkers, &p.DailyCampaignLimit, - &p.MaxCampaigns, &p.MaxActiveCampaigns, &p.MaxTeamMembers, &p.MaxEmailAccounts, - &p.UpdatedAt, &p.CreatedAt, - ); err != nil { - return nil, err - } - if duration != nil { - p.Duration = models.Duration(*duration) - } - plans = append(plans, p) - } - - result := &models.AdminPlansResult{ - Data: plans, - Pagination: models.Pagination{HasMore: len(plans) > limit}, - } - if len(plans) > limit { - result.Data = plans[:limit] - last := plans[limit-1].ID - result.Pagination.NextCursor = paging.UUIDString(last) - } - - countQuery := `SELECT COUNT(*) FROM plans p LEFT JOIN durations d ON d.id = p.duration_id ` + where - var total int64 - if err := r.db.QueryRow(ctx, countQuery, args[:len(args)-1]...).Scan(&total); err == nil { - result.Pagination.Total = &total - } - - return result, nil -} - -// DurationIDByTitle resolves a durations.title to its id, or (nil, nil) when -// there is no such period, so the caller can answer 400 not a NOT NULL violation. -func (r *adminRepository) DurationIDByTitle(ctx context.Context, title string) (*uuid.UUID, error) { - var id uuid.UUID - err := r.db.QueryRow(ctx, `SELECT id FROM durations WHERE title = $1`, title).Scan(&id) - if errors.Is(err, pgx.ErrNoRows) { - return nil, nil - } - if err != nil { - return nil, err - } - return &id, nil -} - -// CreatePlan creates a new plan. The billing period is duration_id, an FK. -func (r *adminRepository) CreatePlan(ctx context.Context, plan *models.Plan, durationID uuid.UUID) error { - query := ` - INSERT INTO plans (id, name, max_contacts, daily_emails, ai_generation, account_limit, - price, discounted_price, duration_id, savings, public, dedicated_workers, daily_campaign_limit, - max_campaigns, max_active_campaigns, max_team_members, max_email_accounts, - monthly_credits, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20) - ` - now := time.Now() - _, err := r.db.Exec(ctx, query, - plan.ID, plan.Name, plan.MaxContacts, plan.DailyEmails, plan.AIGeneration, plan.AccountLimit, - plan.Price, plan.DiscountedPrice, durationID, plan.Savings, plan.Public, - plan.DedicatedWorkers, plan.DailyCampaignLimit, - plan.MaxCampaigns, plan.MaxActiveCampaigns, plan.MaxTeamMembers, plan.MaxEmailAccounts, - plan.MonthlyCredits, now, now, - ) - return err -} - -// GetPlan gets a plan by ID -func (r *adminRepository) GetPlan(ctx context.Context, planID uuid.UUID) (*models.Plan, error) { - query := ` - SELECT p.id, p.name, p.max_contacts, p.daily_emails, p.ai_generation, p.account_limit, - p.price, p.discounted_price, d.title, p.savings, p.public, - p.stripe_price_id, p.stripe_price_id_yearly, p.stripe_product_id, - p.dedicated_workers, p.daily_campaign_limit, - p.max_campaigns, p.max_active_campaigns, p.max_team_members, p.max_email_accounts, - p.monthly_credits, p.referral_reward_percent, p.updated_at, p.created_at - FROM plans p - LEFT JOIN durations d ON d.id = p.duration_id - WHERE p.id = $1 - ` - - var p models.Plan - var duration *string - err := r.db.QueryRow(ctx, query, planID).Scan( - &p.ID, &p.Name, &p.MaxContacts, &p.DailyEmails, &p.AIGeneration, &p.AccountLimit, - &p.Price, &p.DiscountedPrice, &duration, &p.Savings, &p.Public, - &p.StripePriceID, &p.StripePriceIDYearly, &p.StripeProductID, - &p.DedicatedWorkers, &p.DailyCampaignLimit, - &p.MaxCampaigns, &p.MaxActiveCampaigns, &p.MaxTeamMembers, &p.MaxEmailAccounts, - &p.MonthlyCredits, &p.ReferralRewardPercent, &p.UpdatedAt, &p.CreatedAt, - ) - if errors.Is(err, pgx.ErrNoRows) { - return nil, nil - } - if err != nil { - return nil, err - } - if duration != nil { - p.Duration = models.Duration(*duration) - } - return &p, nil -} - -// UpdatePlan updates a plan. -func (r *adminRepository) UpdatePlan(ctx context.Context, plan *models.Plan, durationID uuid.UUID) error { - query := ` - UPDATE plans SET name = $2, max_contacts = $3, daily_emails = $4, ai_generation = $5, - account_limit = $6, price = $7, discounted_price = $8, duration_id = $9, public = $10, - dedicated_workers = $11, daily_campaign_limit = $12, max_campaigns = $13, - max_active_campaigns = $14, max_team_members = $15, max_email_accounts = $16, - updated_at = $17 - WHERE id = $1 - ` - _, err := r.db.Exec(ctx, query, - plan.ID, plan.Name, plan.MaxContacts, plan.DailyEmails, plan.AIGeneration, - plan.AccountLimit, plan.Price, plan.DiscountedPrice, durationID, plan.Public, - plan.DedicatedWorkers, plan.DailyCampaignLimit, plan.MaxCampaigns, - plan.MaxActiveCampaigns, plan.MaxTeamMembers, plan.MaxEmailAccounts, - time.Now(), - ) - return err -} - -// DeletePlan deletes a plan -func (r *adminRepository) DeletePlan(ctx context.Context, planID uuid.UUID) error { - _, err := r.db.Exec(ctx, `DELETE FROM plans WHERE id = $1`, planID) - return err -} - -// IsPlanInUse checks if a plan is being used by any subscription -func (r *adminRepository) IsPlanInUse(ctx context.Context, planID uuid.UUID) (bool, error) { - var count int - err := r.db.QueryRow(ctx, `SELECT COUNT(*) FROM subscriptions WHERE plan_id = $1`, planID).Scan(&count) - return count > 0, err -} - -// ListEnterpriseInquiries lists enterprise inquiries with the shared faceted -// search params; mirrors SearchOrganizationsForAdmin (incremental WHERE builder, -// id keyset, LIMIT+1 has_more, separate COUNT). -func (r *adminRepository) ListEnterpriseInquiries(ctx context.Context, search *models.AdminEnterpriseInquirySearch) (*models.AdminEnterpriseInquiriesResult, error) { - limit := search.Limit - if limit <= 0 || limit > 100 { - limit = 50 - } - - args := []interface{}{} - argNum := 1 - whereClause := "WHERE 1=1" - - if search.Query != "" { - whereClause += " AND (ei.company_name ILIKE $" + itoa(argNum) + " OR ei.contact_name ILIKE $" + itoa(argNum) + " OR ei.contact_email ILIKE $" + itoa(argNum) + " OR COALESCE(u.email,'') ILIKE $" + itoa(argNum) + ")" - args = append(args, "%"+search.Query+"%") - argNum++ - } - if search.Status != "" && search.Status != "all" { - whereClause += " AND ei.status = $" + itoa(argNum) - args = append(args, search.Status) - argNum++ - } - switch search.Assignment { - case "assigned": - whereClause += " AND ei.assigned_to IS NOT NULL" - case "unassigned": - whereClause += " AND ei.assigned_to IS NULL" - } - switch search.Linkage { - case "linked": - whereClause += " AND ei.user_id IS NOT NULL" - case "anonymous": - whereClause += " AND ei.user_id IS NULL" - } - if search.HasNotes { - whereClause += " AND ei.notes IS NOT NULL AND ei.notes <> ''" - } - if search.HasPhone { - whereClause += " AND ei.phone IS NOT NULL AND ei.phone <> ''" - } - if search.Processed { - whereClause += " AND ei.processed_at IS NOT NULL" - } - - addInt := func(frag string, v *int) { - if v != nil { - whereClause += " AND " + fmt.Sprintf(frag, argNum) - args = append(args, *v) - argNum++ - } - } - addAfter := func(col string, v *time.Time) { - if v != nil { - whereClause += " AND " + col + " >= $" + itoa(argNum) - args = append(args, *v) - argNum++ - } - } - addBefore := func(col string, v *time.Time) { - if v != nil { - whereClause += " AND " + col + " < ($" + itoa(argNum) + " + INTERVAL '1 day')" - args = append(args, *v) - argNum++ - } - } - - addInt(`ei.team_size >= $%d`, search.TeamSizeMin) - addInt(`ei.team_size <= $%d`, search.TeamSizeMax) - addInt(`ei.estimated_volume >= $%d`, search.EstimatedVolumeMin) - addInt(`ei.estimated_volume <= $%d`, search.EstimatedVolumeMax) - - if search.CreatedWithin > 0 { - whereClause += " AND ei.created_at >= NOW() - ($" + itoa(argNum) + "::int * INTERVAL '1 day')" - args = append(args, search.CreatedWithin) - argNum++ - } - addAfter("ei.created_at", search.CreatedAfter) - addBefore("ei.created_at", search.CreatedBefore) - addAfter("COALESCE(ei.updated_at, ei.created_at)", search.UpdatedAfter) - addBefore("COALESCE(ei.updated_at, ei.created_at)", search.UpdatedBefore) - - if search.Cursor != nil { - whereClause += " AND ei.id < $" + itoa(argNum) - args = append(args, *search.Cursor) - argNum++ - } - - orderCol := "ei.created_at" - switch search.SortBy { - case "updated_at": - orderCol = "COALESCE(ei.updated_at, ei.created_at)" - case "company_name": - orderCol = "ei.company_name" - case "contact_email": - orderCol = "ei.contact_email" - case "status": - orderCol = "ei.status" - case "team_size": - orderCol = "ei.team_size" - case "estimated_volume": - orderCol = "ei.estimated_volume" - } - orderDir := "DESC" - if search.SortBy != "" && !search.SortDesc { - orderDir = "ASC" - } - orderBy := "ORDER BY " + orderCol + " " + orderDir + ", ei.id DESC" - - args = append(args, limit+1) - - query := ` - SELECT ei.id, ei.user_id, ei.company_name, ei.contact_name, ei.contact_email, - ei.phone, ei.team_size, ei.estimated_volume, ei.monthly_email_volume, ei.message, - ei.notes, ei.status, ei.assigned_to, - ei.created_at, COALESCE(ei.updated_at, ei.created_at) as updated_at, - u.id, u.first_name, u.last_name, u.email, - au.id, au.first_name, au.last_name, au.email - FROM enterprise_inquiries ei - LEFT JOIN users u ON u.id = ei.user_id - LEFT JOIN users au ON au.id = ei.assigned_to - ` + whereClause + ` - ` + orderBy + ` - LIMIT $` + itoa(argNum) - - rows, err := r.db.Query(ctx, query, args...) - if err != nil { - return nil, err - } - defer rows.Close() - - var inquiries []models.AdminEnterpriseInquiry - for rows.Next() { - var inq models.AdminEnterpriseInquiry - var teamSize *int // team_size is an integer column; the DTO exposes it as a string - var userID, assignedUserID *uuid.UUID - var userFirstName, userLastName, userEmail *string - var assignedFirstName, assignedLastName, assignedEmail *string - - err := rows.Scan( - &inq.ID, &inq.UserID, &inq.CompanyName, &inq.ContactName, &inq.ContactEmail, - &inq.Phone, &teamSize, &inq.EstimatedVolume, &inq.MonthlyEmailVolume, &inq.Message, - &inq.Notes, &inq.Status, &inq.AssignedTo, - &inq.CreatedAt, &inq.UpdatedAt, - &userID, &userFirstName, &userLastName, &userEmail, - &assignedUserID, &assignedFirstName, &assignedLastName, &assignedEmail, - ) - if err != nil { - return nil, err - } - - if teamSize != nil { - s := fmt.Sprintf("%d", *teamSize) - inq.TeamSize = &s - } - - if userID != nil { - inq.User = &models.AdminUserSummary{ - ID: *userID, - FirstName: *userFirstName, - LastName: *userLastName, - Email: *userEmail, - } - } - - if assignedUserID != nil { - inq.AssignedAdmin = &models.AdminUserSummary{ - ID: *assignedUserID, - FirstName: *assignedFirstName, - LastName: *assignedLastName, - Email: *assignedEmail, - } - } - - inquiries = append(inquiries, inq) - } - - result := &models.AdminEnterpriseInquiriesResult{ - Data: inquiries, - Pagination: models.Pagination{ - HasMore: len(inquiries) > limit, - }, - } - - if len(inquiries) > limit { - result.Data = inquiries[:limit] - lastID := inquiries[limit-1].ID - result.Pagination.NextCursor = paging.UUIDString(lastID) - } - - // Total count for the same filter — drop the trailing LIMIT arg. - countQuery := `SELECT COUNT(*) FROM enterprise_inquiries ei LEFT JOIN users u ON u.id = ei.user_id LEFT JOIN users au ON au.id = ei.assigned_to ` + whereClause - var total int64 - if err := r.db.QueryRow(ctx, countQuery, args[:len(args)-1]...).Scan(&total); err == nil { - result.Pagination.Total = &total - } - - return result, nil -} - -// GetEnterpriseInquiry gets a specific enterprise inquiry -func (r *adminRepository) GetEnterpriseInquiry(ctx context.Context, id uuid.UUID) (*models.AdminEnterpriseInquiry, error) { - query := ` - SELECT ei.id, ei.user_id, ei.company_name, ei.contact_name, ei.contact_email, - ei.phone, ei.team_size, ei.notes, ei.status, ei.assigned_to, - ei.created_at, COALESCE(ei.updated_at, ei.created_at) as updated_at - FROM enterprise_inquiries ei - WHERE ei.id = $1 - ` - - var inq models.AdminEnterpriseInquiry - err := r.db.QueryRow(ctx, query, id).Scan( - &inq.ID, &inq.UserID, &inq.CompanyName, &inq.ContactName, &inq.ContactEmail, - &inq.Phone, &inq.TeamSize, &inq.Notes, &inq.Status, &inq.AssignedTo, - &inq.CreatedAt, &inq.UpdatedAt, - ) - if err == pgx.ErrNoRows { - return nil, nil - } - if err != nil { - return nil, err - } - return &inq, nil -} - -// UpdateEnterpriseInquiry updates an enterprise inquiry -func (r *adminRepository) UpdateEnterpriseInquiry(ctx context.Context, id uuid.UUID, update *models.UpdateEnterpriseInquiryRequest) error { - setClauses := []string{"updated_at = NOW()"} - args := []interface{}{id} - argNum := 2 - - if update.Status != nil { - setClauses = append(setClauses, "status = $"+itoa(argNum)) - args = append(args, *update.Status) - argNum++ - } - if update.AssignedTo != nil { - setClauses = append(setClauses, "assigned_to = $"+itoa(argNum)) - args = append(args, *update.AssignedTo) - argNum++ - } - if update.Notes != nil { - setClauses = append(setClauses, "notes = $"+itoa(argNum)) - args = append(args, *update.Notes) - argNum++ - } - - query := "UPDATE enterprise_inquiries SET " + joinStrings(setClauses, ", ") + " WHERE id = $1" - _, err := r.db.Exec(ctx, query, args...) - return err -} - // GetUserRateLimits gets a user's API and realtime throughput limits. Mail // volume is not one of them: that is a per-mailbox budget and a plan entitlement. func (r *adminRepository) GetUserRateLimits(ctx context.Context, userID uuid.UUID) (*models.AdminUserRateLimits, error) { diff --git a/internal/repository/pg_admin_fleet.go b/internal/repository/pg_admin_fleet.go new file mode 100644 index 00000000..fce26cf4 --- /dev/null +++ b/internal/repository/pg_admin_fleet.go @@ -0,0 +1,171 @@ +package repository + +import ( + "context" + "sort" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/infrastructure/db" + "github.com/warmbly/warmbly/internal/models" +) + +// AdminFleetRepository is the operator's view of automatic placement: every +// worker with its capacity-view row, the decision log the control loops write, +// and the dedicated worker bindings. +type AdminFleetRepository interface { + // Capacity lists every worker (active or not) left-joined to + // worker_capacity_view, hottest first. + Capacity(ctx context.Context) ([]models.AdminFleetWorkerRow, error) + // Decisions lists decision_log newest first. kind "" means every kind; + // workerID nil means every worker. + Decisions(ctx context.Context, kind string, workerID *uuid.UUID, limit int) ([]models.AdminFleetDecision, error) + // DedicatedAssignments lists active bindings (released_at IS NULL) with the + // worker and workspace named and the worker's current mailbox count. + DedicatedAssignments(ctx context.Context) ([]models.AdminDedicatedAssignment, error) +} + +type adminFleetRepository struct { + db *db.DB +} + +func NewAdminFleetRepository(d *db.DB) AdminFleetRepository { + return &adminFleetRepository{db: d} +} + +// Capacity joins every worker row to the materialized capacity view; the view +// only carries active workers, so inactive ones come back with zeroed metrics. +func (r *adminFleetRepository) Capacity(ctx context.Context) ([]models.AdminFleetWorkerRow, error) { + rows, err := r.db.Query(ctx, ` + SELECT w.id, w.name, w.ip_addr, COALESCE(w.active, false), w.free_tier, w.worker_type, + w.risk_pool, w.egress_kind, w.health_state, w.install_state, + w.last_seen_at, + (COALESCE(w.active, false) AND w.last_seen_at > now() - $1::interval) AS live, + w.account_count, + COALESCE(t.tags, '{}'::text[]) AS tags, + COALESCE(v.load_score, w.load_score, 0)::float8, + COALESCE(v.base_capacity, 0)::float8, + COALESCE(v.health_multiplier, 1)::float8, + COALESCE(v.age_multiplier, 1)::float8, + COALESCE(v.sends_attempted_1h, 0), COALESCE(v.sends_succeeded_1h, 0), + COALESCE(v.bounces_hard_1h, 0), COALESCE(v.bounces_soft_1h, 0), + COALESCE(v.complaints_1h, 0), COALESCE(v.auth_errors_1h, 0) + FROM workers w + LEFT JOIN worker_capacity_view v ON v.worker_id = w.id + LEFT JOIN ( + SELECT worker_id, array_agg(tag::text ORDER BY tag) AS tags + FROM worker_tags GROUP BY worker_id + ) t ON t.worker_id = w.id + `, WorkerLivenessWindow) + if err != nil { + return nil, err + } + defer rows.Close() + + out := make([]models.AdminFleetWorkerRow, 0) + for rows.Next() { + var row models.AdminFleetWorkerRow + var live *bool + if err := rows.Scan( + &row.WorkerID, &row.Name, &row.IPAddr, &row.Active, &row.FreeTier, &row.WorkerType, + &row.RiskPool, &row.EgressKind, &row.HealthState, &row.InstallState, + &row.LastSeenAt, &live, &row.AccountCount, &row.Tags, + &row.LoadScore, &row.BaseCapacity, &row.HealthMultiplier, &row.AgeMultiplier, + &row.SendsAttempted1h, &row.SendsSucceeded1h, + &row.BouncesHard1h, &row.BouncesSoft1h, &row.Complaints1h, &row.AuthErrors1h, + ); err != nil { + return nil, err + } + // A NULL last_seen_at makes the AND NULL; that worker has never heartbeated. + row.Live = live != nil && *live + if row.Tags == nil { + row.Tags = []string{} + } + row.EffectiveCapacity = row.BaseCapacity * row.HealthMultiplier * row.AgeMultiplier + if row.EffectiveCapacity <= 0 { + row.EffectiveCapacity = 1 + } + row.Utilization = row.LoadScore / row.EffectiveCapacity + out = append(out, row) + } + if err := rows.Err(); err != nil { + return nil, err + } + sort.SliceStable(out, func(i, j int) bool { return out[i].Utilization > out[j].Utilization }) + return out, nil +} + +// Decisions lists decision_log newest first; kind and workerID are optional +// filters and limit is clamped to 1..500 with a default of 100. +func (r *adminFleetRepository) Decisions(ctx context.Context, kind string, workerID *uuid.UUID, limit int) ([]models.AdminFleetDecision, error) { + if limit <= 0 { + limit = 100 + } + if limit > 500 { + limit = 500 + } + rows, err := r.db.Query(ctx, ` + SELECT d.id, d.kind, d.worker_id, COALESCE(w.name, ''), d.mailbox_id, + d.before, d.after, COALESCE(d.reason, ''), COALESCE(d.triggered_by, ''), d.created_at + FROM decision_log d + LEFT JOIN workers w ON w.id = d.worker_id + WHERE ($1 = '' OR d.kind = $1) + AND ($2::uuid IS NULL OR d.worker_id = $2) + ORDER BY d.created_at DESC, d.id DESC + LIMIT $3 + `, kind, workerID, limit) + if err != nil { + return nil, err + } + defer rows.Close() + + out := make([]models.AdminFleetDecision, 0) + for rows.Next() { + var d models.AdminFleetDecision + var before, after []byte + if err := rows.Scan( + &d.ID, &d.Kind, &d.WorkerID, &d.WorkerName, &d.MailboxID, + &before, &after, &d.Reason, &d.TriggeredBy, &d.CreatedAt, + ); err != nil { + return nil, err + } + d.Before = before + d.After = after + out = append(out, d) + } + return out, rows.Err() +} + +// DedicatedAssignments lists active bindings with the worker's live mailbox +// count taken from email_accounts rather than the cached account_count. +func (r *adminFleetRepository) DedicatedAssignments(ctx context.Context) ([]models.AdminDedicatedAssignment, error) { + rows, err := r.db.Query(ctx, ` + SELECT a.id, a.worker_id, COALESCE(w.name, ''), + COALESCE(w.active AND w.last_seen_at > now() - $1::interval, false), + a.organization_id, COALESCE(o.name, ''), + a.subscription_id, a.assigned_at, a.released_at, + (SELECT count(*) FROM email_accounts e WHERE e.worker_id = a.worker_id) + FROM dedicated_worker_assignments a + LEFT JOIN workers w ON w.id = a.worker_id + LEFT JOIN organizations o ON o.id = a.organization_id + WHERE a.released_at IS NULL + ORDER BY a.assigned_at DESC + `, WorkerLivenessWindow) + if err != nil { + return nil, err + } + defer rows.Close() + + out := make([]models.AdminDedicatedAssignment, 0) + for rows.Next() { + var a models.AdminDedicatedAssignment + if err := rows.Scan( + &a.ID, &a.WorkerID, &a.WorkerName, &a.WorkerLive, + &a.OrganizationID, &a.OrganizationName, + &a.SubscriptionID, &a.AssignedAt, &a.ReleasedAt, &a.AccountCount, + ); err != nil { + return nil, err + } + out = append(out, a) + } + return out, rows.Err() +} diff --git a/internal/repository/pg_admin_insight.go b/internal/repository/pg_admin_insight.go new file mode 100644 index 00000000..657bd25c --- /dev/null +++ b/internal/repository/pg_admin_insight.go @@ -0,0 +1,293 @@ +package repository + +import ( + "context" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/infrastructure/db" + "github.com/warmbly/warmbly/internal/models" +) + +// AdminInsightRepository holds the smaller cross-workspace reads the panel +// shows on existing pages: warmup abuse signals, warmup admin actions, a +// workspace's API keys, workspace transfers and signup acquisition. +type AdminInsightRepository interface { + // WarmupAbuse ranks mailboxes by invalid warmup-token attempts since the + // given time, most attempts first. + WarmupAbuse(ctx context.Context, since time.Time, limit int) ([]models.AdminWarmupAbuseRow, error) + // WarmupActions lists warmup_admin_actions newest first. + WarmupActions(ctx context.Context, limit int) ([]models.AdminWarmupAction, error) + // OrgAPIKeys lists a workspace's API keys, active first, never the hash. + OrgAPIKeys(ctx context.Context, orgID uuid.UUID) ([]models.AdminOrgAPIKey, error) + // ListTransfers lists export and import jobs across every workspace, + // newest first, capped at limit. + ListTransfers(ctx context.Context, limit int) ([]models.AdminTransferJob, error) + // Acquisition groups signups from the last days by channel and reports + // how many converted to a paid subscription. + Acquisition(ctx context.Context, days int) (*models.AdminAcquisition, error) +} + +type adminInsightRepository struct { + db *db.DB +} + +func NewAdminInsightRepository(d *db.DB) AdminInsightRepository { + return &adminInsightRepository{db: d} +} + +func (r *adminInsightRepository) WarmupAbuse(ctx context.Context, since time.Time, limit int) ([]models.AdminWarmupAbuseRow, error) { + if limit <= 0 { + limit = 50 + } + // warmup_pool_participants is unique per mailbox (000097), so the left + // join adds at most one row. + const q = ` + SELECT a.email_account_id, ea.email, ea.organization_id, COALESCE(o.name, ''), + COUNT(*)::int, MAX(a.created_at), + COALESCE(p.blocked_at IS NOT NULL AND (p.blocked_until IS NULL OR p.blocked_until > now()), false), + COALESCE(p.spam_score, 0), COALESCE(p.health_state::text, '') + FROM warmup_invalid_token_attempts a + JOIN email_accounts ea ON ea.id = a.email_account_id + LEFT JOIN organizations o ON o.id = ea.organization_id + LEFT JOIN warmup_pool_participants p ON p.email_account_id = a.email_account_id + WHERE a.created_at >= $1 + GROUP BY a.email_account_id, ea.email, ea.organization_id, o.name, + p.blocked_at, p.blocked_until, p.spam_score, p.health_state + ORDER BY COUNT(*) DESC, MAX(a.created_at) DESC + LIMIT $2 + ` + rows, err := r.db.Query(ctx, q, since, limit) + if err != nil { + return nil, fmt.Errorf("admin insight: warmup abuse: %w", err) + } + defer rows.Close() + out := []models.AdminWarmupAbuseRow{} + for rows.Next() { + var row models.AdminWarmupAbuseRow + if err := rows.Scan( + &row.EmailAccountID, &row.Email, &row.OrganizationID, &row.OrganizationName, + &row.Attempts, &row.LastAttemptAt, &row.Blocked, &row.SpamScore, &row.HealthState, + ); err != nil { + return nil, fmt.Errorf("admin insight: warmup abuse scan: %w", err) + } + out = append(out, row) + } + return out, rows.Err() +} + +func (r *adminInsightRepository) WarmupActions(ctx context.Context, limit int) ([]models.AdminWarmupAction, error) { + if limit <= 0 { + limit = 100 + } + const q = ` + SELECT a.id, a.admin_user_id, COALESCE(u.email, ''), a.email_account_id, COALESCE(ea.email, ''), + a.action, a.reason, a.created_at + FROM warmup_admin_actions a + LEFT JOIN users u ON u.id = a.admin_user_id + LEFT JOIN email_accounts ea ON ea.id = a.email_account_id + ORDER BY a.created_at DESC, a.id DESC + LIMIT $1 + ` + rows, err := r.db.Query(ctx, q, limit) + if err != nil { + return nil, fmt.Errorf("admin insight: warmup actions: %w", err) + } + defer rows.Close() + out := []models.AdminWarmupAction{} + for rows.Next() { + var row models.AdminWarmupAction + if err := rows.Scan( + &row.ID, &row.AdminUserID, &row.AdminEmail, &row.EmailAccountID, &row.Email, + &row.Action, &row.Reason, &row.CreatedAt, + ); err != nil { + return nil, fmt.Errorf("admin insight: warmup actions scan: %w", err) + } + out = append(out, row) + } + return out, rows.Err() +} + +func (r *adminInsightRepository) OrgAPIKeys(ctx context.Context, orgID uuid.UUID) ([]models.AdminOrgAPIKey, error) { + const q = ` + SELECT k.id, k.name, k.key_prefix, k.key_suffix, k.status, k.permissions, + k.user_id, COALESCE(u.email, ''), k.last_used_at, k.expires_at, k.revoked_at, k.created_at, + (SELECT COUNT(*) FROM api_key_usage_logs l + WHERE l.api_key_id = k.id AND l.created_at >= now() - interval '7 days') + FROM api_keys k + LEFT JOIN users u ON u.id = k.user_id + WHERE k.organization_id = $1 + ORDER BY (k.status = 'active') DESC, k.created_at DESC + ` + rows, err := r.db.Query(ctx, q, orgID) + if err != nil { + return nil, fmt.Errorf("admin insight: org api keys: %w", err) + } + defer rows.Close() + out := []models.AdminOrgAPIKey{} + for rows.Next() { + var row models.AdminOrgAPIKey + if err := rows.Scan( + &row.ID, &row.Name, &row.KeyPrefix, &row.KeySuffix, &row.Status, &row.Permissions, + &row.UserID, &row.UserEmail, &row.LastUsedAt, &row.ExpiresAt, &row.RevokedAt, &row.CreatedAt, + &row.RequestsLast7d, + ); err != nil { + return nil, fmt.Errorf("admin insight: org api keys scan: %w", err) + } + out = append(out, row) + } + return out, rows.Err() +} + +func (r *adminInsightRepository) ListTransfers(ctx context.Context, limit int) ([]models.AdminTransferJob, error) { + if limit <= 0 { + limit = 100 + } + const q = ` + SELECT j.kind, j.id, j.organization_id, COALESCE(o.name, ''), j.requested_by, COALESCE(u.email, ''), + j.status, j.groups, j.include_secrets, j.progress_percent, j.progress_stage, + j.archive_bytes, j.error_message, j.started_at, j.completed_at, j.expires_at, j.created_at + FROM ( + SELECT 'export' AS kind, e.id, e.organization_id, e.requested_by, e.status, e.groups, + e.include_secrets, e.progress_percent::int AS progress_percent, e.progress_stage, + e.archive_bytes, e.error_message, e.started_at, e.completed_at, e.expires_at, e.created_at + FROM org_export_jobs e + UNION ALL + SELECT 'import', i.id, i.organization_id, i.requested_by, i.status, i.groups, + false, i.progress_percent::int, i.progress_stage, + i.archive_bytes, i.error_message, i.started_at, i.completed_at, NULL::timestamptz, i.created_at + FROM org_import_jobs i + ) j + LEFT JOIN organizations o ON o.id = j.organization_id + LEFT JOIN users u ON u.id = j.requested_by + ORDER BY j.created_at DESC, j.id DESC + LIMIT $1 + ` + rows, err := r.db.Query(ctx, q, limit) + if err != nil { + return nil, fmt.Errorf("admin insight: transfers: %w", err) + } + defer rows.Close() + out := []models.AdminTransferJob{} + for rows.Next() { + var ( + row models.AdminTransferJob + status string + groups []string + ) + if err := rows.Scan( + &row.Kind, &row.ID, &row.OrganizationID, &row.OrganizationName, &row.RequestedBy, &row.RequestedByEmail, + &status, &groups, &row.IncludeSecrets, &row.ProgressPercent, &row.ProgressStage, + &row.ArchiveBytes, &row.ErrorMessage, &row.StartedAt, &row.CompletedAt, &row.ExpiresAt, &row.CreatedAt, + ); err != nil { + return nil, fmt.Errorf("admin insight: transfers scan: %w", err) + } + row.Status = models.OrgTransferStatus(status) + row.Groups = make([]models.OrgDataGroup, 0, len(groups)) + for _, g := range groups { + row.Groups = append(row.Groups, models.OrgDataGroup(g)) + } + out = append(out, row) + } + return out, rows.Err() +} + +func (r *adminInsightRepository) Acquisition(ctx context.Context, days int) (*models.AdminAcquisition, error) { + if days <= 0 { + days = 30 + } + // Paid means a live Stripe subscription; trialing rows carry no Stripe id. + const cohort = ` + WITH cohort AS ( + SELECT o.id, + COALESCE(a.utm_source, '') AS source, + COALESCE(a.utm_medium, '') AS medium, + COALESCE(a.referrer_host, '') AS referrer, + EXISTS ( + SELECT 1 FROM subscriptions s + WHERE s.organization_id = o.id + AND s.status IN ('active', 'past_due') + AND s.stripe_subscription_id IS NOT NULL + ) AS converted + FROM organizations o + LEFT JOIN organization_acquisition a ON a.organization_id = o.id + WHERE o.created_at >= now() - ($1::int * interval '1 day') + ) + ` + out := &models.AdminAcquisition{ + Days: days, + Channels: []models.AdminAcquisitionChannel{}, + Referrers: []models.AdminAcquisitionReferrer{}, + } + + const totals = cohort + ` + SELECT COUNT(*)::int, COUNT(*) FILTER (WHERE source <> '')::int, COUNT(*) FILTER (WHERE converted)::int + FROM cohort + ` + if err := r.db.QueryRow(ctx, totals, days).Scan(&out.Signups, &out.WithChannel, &out.Converted); err != nil { + return nil, fmt.Errorf("admin insight: acquisition totals: %w", err) + } + + // Same predicate the trial expiration job uses, looking forward instead. + const trials = ` + SELECT COUNT(*)::int FROM subscriptions + WHERE free_trial_ends_at IS NOT NULL + AND free_trial_ends_at BETWEEN now() AND now() + interval '7 days' + AND stripe_subscription_id IS NULL + AND status NOT IN ('canceled', 'incomplete_expired') + ` + if err := r.db.QueryRow(ctx, trials).Scan(&out.TrialsExpiring7d); err != nil { + return nil, fmt.Errorf("admin insight: acquisition trials: %w", err) + } + + const channels = cohort + ` + SELECT source, medium, COUNT(*)::int, COUNT(*) FILTER (WHERE converted)::int + FROM cohort + WHERE NOT (source = '' AND medium = '') + GROUP BY source, medium + ORDER BY COUNT(*) DESC, source, medium + ` + rows, err := r.db.Query(ctx, channels, days) + if err != nil { + return nil, fmt.Errorf("admin insight: acquisition channels: %w", err) + } + for rows.Next() { + var ch models.AdminAcquisitionChannel + if err := rows.Scan(&ch.Source, &ch.Medium, &ch.Signups, &ch.Converted); err != nil { + rows.Close() + return nil, fmt.Errorf("admin insight: acquisition channels scan: %w", err) + } + out.Channels = append(out.Channels, ch) + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("admin insight: acquisition channels: %w", err) + } + + const referrers = cohort + ` + SELECT referrer, COUNT(*)::int + FROM cohort + WHERE referrer <> '' + GROUP BY referrer + ORDER BY COUNT(*) DESC, referrer + LIMIT 10 + ` + rows, err = r.db.Query(ctx, referrers, days) + if err != nil { + return nil, fmt.Errorf("admin insight: acquisition referrers: %w", err) + } + for rows.Next() { + var ref models.AdminAcquisitionReferrer + if err := rows.Scan(&ref.Host, &ref.Signups); err != nil { + rows.Close() + return nil, fmt.Errorf("admin insight: acquisition referrers scan: %w", err) + } + out.Referrers = append(out.Referrers, ref) + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("admin insight: acquisition referrers: %w", err) + } + return out, nil +} diff --git a/internal/repository/pg_admin_sends.go b/internal/repository/pg_admin_sends.go new file mode 100644 index 00000000..616cc7d5 --- /dev/null +++ b/internal/repository/pg_admin_sends.go @@ -0,0 +1,326 @@ +package repository + +import ( + "context" + "encoding/json" + "errors" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/warmbly/warmbly/internal/infrastructure/db" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/utils/paging" +) + +// AdminSendsRepository reads the send outcome loop and the task queues across +// every workspace: reservations no worker answered, dead letters, task +// failures and customer webhook delivery health. +type AdminSendsRepository interface { + // InFlight lists reserved sends without a result, oldest first, capped at + // limit, with the age buckets computed against reclaimAfter. + InFlight(ctx context.Context, reclaimAfter time.Duration, limit int) (*models.AdminInFlightResult, error) + // ListDeadLetters pages task_dead_letters newest first. status "" means + // every status; cursor is the id of the last row of the previous page. + ListDeadLetters(ctx context.Context, status string, cursor *uuid.UUID, limit int) (*models.AdminDeadLettersResult, error) + GetDeadLetter(ctx context.Context, id uuid.UUID) (*models.AdminDeadLetterRow, error) + // RecentTaskFailures lists the newest task_failures rows with their task + // and mailbox, capped at limit. + RecentTaskFailures(ctx context.Context, limit int) ([]models.AdminTaskFailureRow, error) + // WebhookHealth is the instance-wide delivery picture; a delivery is stale + // when it has been in_flight longer than lease. + WebhookHealth(ctx context.Context, lease time.Duration) (*models.AdminWebhookHealth, error) + // OrgWebhooks lists one workspace's endpoints with their recent delivery + // and drop counts. + OrgWebhooks(ctx context.Context, orgID uuid.UUID) ([]models.AdminWebhookEndpointRow, error) +} + +type adminSendsRepository struct { + db *db.DB +} + +func NewAdminSendsRepository(d *db.DB) AdminSendsRepository { + return &adminSendsRepository{db: d} +} + +func (r *adminSendsRepository) InFlight(ctx context.Context, reclaimAfter time.Duration, limit int) (*models.AdminInFlightResult, error) { + if limit <= 0 { + limit = 100 + } + if limit > 500 { + limit = 500 + } + reclaimSecs := reclaimAfter.Seconds() + out := &models.AdminInFlightResult{ + Summary: models.AdminInFlightSummary{ReclaimAfterMinutes: int(reclaimAfter / time.Minute)}, + Data: []models.AdminInFlightSend{}, + } + + // The summary counts every reservation, not just the page. + err := r.db.QueryRow(ctx, ` + SELECT COUNT(*), + COUNT(*) FILTER (WHERE dispatched_at > NOW() - INTERVAL '5 minutes'), + COUNT(*) FILTER (WHERE dispatched_at <= NOW() - INTERVAL '5 minutes' + AND dispatched_at > NOW() - INTERVAL '30 minutes'), + COUNT(*) FILTER (WHERE dispatched_at <= NOW() - make_interval(secs => $1)), + MIN(dispatched_at) + FROM campaign_contact_progress + WHERE sent_at IS NULL AND dispatched_at IS NOT NULL + `, reclaimSecs).Scan( + &out.Summary.Total, &out.Summary.Under5m, &out.Summary.Under30m, + &out.Summary.PastReclaimWindow, &out.Summary.OldestDispatched, + ) + if err != nil { + return nil, err + } + + rows, err := r.db.Query(ctx, ` + SELECT p.campaign_id, COALESCE(c.name, ''), c.organization_id, COALESCE(o.name, ''), + p.contact_id, COALESCE(ct.email, ''), p.sequence_id, + p.dispatch_task_id, COALESCE(t.status::text, ''), COALESCE(t.message_id <> '', false), + t.email_account_id, COALESCE(ea.email, ''), ea.worker_id, + p.dispatched_at + FROM campaign_contact_progress p + JOIN campaigns c ON c.id = p.campaign_id + LEFT JOIN organizations o ON o.id = c.organization_id + LEFT JOIN contacts ct ON ct.id = p.contact_id + LEFT JOIN tasks t ON t.id = p.dispatch_task_id + LEFT JOIN email_accounts ea ON ea.id = t.email_account_id + WHERE p.sent_at IS NULL AND p.dispatched_at IS NOT NULL + ORDER BY p.dispatched_at ASC + LIMIT $1 + `, limit) + if err != nil { + return nil, err + } + defer rows.Close() + now := time.Now() + for rows.Next() { + var s models.AdminInFlightSend + if err := rows.Scan( + &s.CampaignID, &s.CampaignName, &s.OrganizationID, &s.OrganizationName, + &s.ContactID, &s.ContactEmail, &s.SequenceID, + &s.TaskID, &s.TaskStatus, &s.HasMessageID, + &s.EmailAccountID, &s.MailboxEmail, &s.WorkerID, + &s.DispatchedAt, + ); err != nil { + return nil, err + } + s.AgeSeconds = int64(now.Sub(s.DispatchedAt).Seconds()) + out.Data = append(out.Data, s) + } + return out, rows.Err() +} + +// deadLetterColumns is the select list shared by the list and get queries; a +// dead letter reaches its workspace through the task's mailbox. +const deadLetterColumns = ` + SELECT d.id, d.task_id, d.task_type, d.payload, d.last_error, d.attempts, d.max_attempts, + d.status, d.next_retry_at, d.replayed_at, d.created_at, d.updated_at, + ea.organization_id, COALESCE(o.name, '') + FROM task_dead_letters d + LEFT JOIN tasks t ON t.id = d.task_id + LEFT JOIN email_accounts ea ON ea.id = t.email_account_id + LEFT JOIN organizations o ON o.id = ea.organization_id` + +func scanDeadLetter(row pgx.Row) (*models.AdminDeadLetterRow, error) { + var d models.AdminDeadLetterRow + var payload []byte + if err := row.Scan( + &d.ID, &d.TaskID, &d.TaskType, &payload, &d.LastError, &d.Attempts, &d.MaxAttempts, + &d.Status, &d.NextRetryAt, &d.ReplayedAt, &d.CreatedAt, &d.UpdatedAt, + &d.OrganizationID, &d.OrganizationName, + ); err != nil { + return nil, err + } + if len(payload) > 0 { + _ = json.Unmarshal(payload, &d.Payload) + } + if d.Payload == nil { + d.Payload = map[string]interface{}{} + } + return &d, nil +} + +func (r *adminSendsRepository) ListDeadLetters(ctx context.Context, status string, cursor *uuid.UUID, limit int) (*models.AdminDeadLettersResult, error) { + if limit <= 0 || limit > 200 { + limit = 50 + } + out := &models.AdminDeadLettersResult{ + Data: []models.AdminDeadLetterRow{}, + Pagination: &models.Pagination{}, + } + if err := r.db.QueryRow(ctx, ` + SELECT COUNT(*) FILTER (WHERE status = 'pending'), + COUNT(*) FILTER (WHERE status = 'replayed'), + COUNT(*) FILTER (WHERE status = 'failed') + FROM task_dead_letters + `).Scan(&out.Pending, &out.Replayed, &out.Failed); err != nil { + return nil, err + } + + rows, err := r.db.Query(ctx, deadLetterColumns+` + WHERE ($1 = '' OR d.status = $1) + AND ($2::uuid IS NULL OR (d.created_at, d.id) < ( + SELECT created_at, id FROM task_dead_letters WHERE id = $2)) + ORDER BY d.created_at DESC, d.id DESC + LIMIT $3 + `, status, cursor, limit+1) + if err != nil { + return nil, err + } + defer rows.Close() + for rows.Next() { + d, err := scanDeadLetter(rows) + if err != nil { + return nil, err + } + out.Data = append(out.Data, *d) + } + if err := rows.Err(); err != nil { + return nil, err + } + if len(out.Data) > limit { + out.Data = out.Data[:limit] + out.Pagination.HasMore = true + out.Pagination.NextCursor = paging.UUIDString(out.Data[limit-1].ID) + } + return out, nil +} + +func (r *adminSendsRepository) GetDeadLetter(ctx context.Context, id uuid.UUID) (*models.AdminDeadLetterRow, error) { + d, err := scanDeadLetter(r.db.QueryRow(ctx, deadLetterColumns+` WHERE d.id = $1`, id)) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + return nil, err + } + return d, nil +} + +// RecentTaskFailures orders by the task's updated_at: task_failures carries no +// timestamp of its own and the failure is the last write to the task. +func (r *adminSendsRepository) RecentTaskFailures(ctx context.Context, limit int) ([]models.AdminTaskFailureRow, error) { + if limit <= 0 || limit > 500 { + limit = 100 + } + rows, err := r.db.Query(ctx, ` + SELECT f.task_id, t.task_type::text, t.status::text, f.title, f.message, + t.email_account_id, COALESCE(ea.email, ''), ea.organization_id, COALESCE(o.name, ''), + t.updated_at + FROM task_failures f + JOIN tasks t ON t.id = f.task_id + LEFT JOIN email_accounts ea ON ea.id = t.email_account_id + LEFT JOIN organizations o ON o.id = ea.organization_id + ORDER BY t.updated_at DESC + LIMIT $1 + `, limit) + if err != nil { + return nil, err + } + defer rows.Close() + out := make([]models.AdminTaskFailureRow, 0, limit) + for rows.Next() { + var f models.AdminTaskFailureRow + if err := rows.Scan( + &f.TaskID, &f.TaskType, &f.TaskStatus, &f.Title, &f.Message, + &f.EmailAccountID, &f.MailboxEmail, &f.OrganizationID, &f.OrganizationName, + &f.OccurredAt, + ); err != nil { + return nil, err + } + out = append(out, f) + } + return out, rows.Err() +} + +// webhookEndpointColumns is the endpoint row with its 7-day counters. Drops +// are rolled up per (organization, event_type), so an endpoint's drops are +// the org's drops on the event types it subscribes to (all, when unfiltered). +const webhookEndpointColumns = ` + SELECT e.id, e.organization_id, COALESCE(o.name, ''), e.url, e.description, e.enabled, + e.event_types, e.consecutive_failures, e.last_success_at, e.last_failure_at, + COALESCE(e.last_failure_reason, ''), + (SELECT COUNT(*) FROM webhook_deliveries d + WHERE d.endpoint_id = e.id AND d.created_at >= NOW() - INTERVAL '7 days'), + (SELECT COUNT(*) FROM webhook_deliveries d + WHERE d.endpoint_id = e.id AND d.created_at >= NOW() - INTERVAL '7 days' + AND d.status IN ('failed', 'abandoned')), + (SELECT COALESCE(SUM(wd.dropped_windows), 0) FROM webhook_event_drops wd + WHERE wd.organization_id = e.organization_id AND wd.day >= CURRENT_DATE - 7 + AND (cardinality(e.event_types) = 0 OR wd.event_type = ANY(e.event_types))) + FROM webhook_endpoints e + LEFT JOIN organizations o ON o.id = e.organization_id` + +func scanWebhookEndpoints(rows pgx.Rows) ([]models.AdminWebhookEndpointRow, error) { + defer rows.Close() + out := []models.AdminWebhookEndpointRow{} + for rows.Next() { + var e models.AdminWebhookEndpointRow + if err := rows.Scan( + &e.ID, &e.OrganizationID, &e.OrganizationName, &e.URL, &e.Description, &e.Enabled, + &e.EventTypes, &e.ConsecutiveFailures, &e.LastSuccessAt, &e.LastFailureAt, + &e.LastFailureReason, &e.DeliveriesLast7d, &e.FailedLast7d, &e.DropsLast7d, + ); err != nil { + return nil, err + } + if e.EventTypes == nil { + e.EventTypes = []string{} + } + out = append(out, e) + } + return out, rows.Err() +} + +// WebhookHealth calls a delivery stale by updated_at, which is what both the +// claim and ReclaimStuckDeliveries use, so the count is what a reclaim sweeps. +func (r *adminSendsRepository) WebhookHealth(ctx context.Context, lease time.Duration) (*models.AdminWebhookHealth, error) { + out := &models.AdminWebhookHealth{ + LeaseMinutes: int(lease / time.Minute), + FailingEndpoints: []models.AdminWebhookEndpointRow{}, + } + if err := r.db.QueryRow(ctx, ` + SELECT COUNT(*) FILTER (WHERE status = 'in_flight' AND updated_at < NOW() - make_interval(secs => $1)), + COUNT(*) FILTER (WHERE status = 'pending' AND next_attempt_at <= NOW()), + COUNT(*) FILTER (WHERE status = 'delivered' AND updated_at >= NOW() - INTERVAL '24 hours'), + COUNT(*) FILTER (WHERE status = 'failed' AND updated_at >= NOW() - INTERVAL '24 hours'), + COUNT(*) FILTER (WHERE status = 'abandoned' AND updated_at >= NOW() - INTERVAL '24 hours') + FROM webhook_deliveries + `, lease.Seconds()).Scan( + &out.InFlightStale, &out.PendingDue, &out.DeliveredLast24h, &out.FailedLast24h, &out.AbandonedLast24h, + ); err != nil { + return nil, err + } + if err := r.db.QueryRow(ctx, ` + SELECT COALESCE(SUM(dropped_windows), 0) FROM webhook_event_drops WHERE day >= CURRENT_DATE - 7 + `).Scan(&out.DropsLast7d); err != nil { + return nil, err + } + + rows, err := r.db.Query(ctx, webhookEndpointColumns+` + WHERE e.consecutive_failures > 0 + ORDER BY e.consecutive_failures DESC, e.last_failure_at DESC NULLS LAST + LIMIT 50 + `) + if err != nil { + return nil, err + } + failing, err := scanWebhookEndpoints(rows) + if err != nil { + return nil, err + } + out.FailingEndpoints = failing + return out, nil +} + +func (r *adminSendsRepository) OrgWebhooks(ctx context.Context, orgID uuid.UUID) ([]models.AdminWebhookEndpointRow, error) { + rows, err := r.db.Query(ctx, webhookEndpointColumns+` + WHERE e.organization_id = $1 + ORDER BY e.created_at DESC + `, orgID) + if err != nil { + return nil, err + } + return scanWebhookEndpoints(rows) +} diff --git a/internal/repository/pg_admin_sync.go b/internal/repository/pg_admin_sync.go new file mode 100644 index 00000000..ab317d4a --- /dev/null +++ b/internal/repository/pg_admin_sync.go @@ -0,0 +1,211 @@ +package repository + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/infrastructure/db" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/utils/paging" +) + +// AdminSyncRepository is the operator's cross-workspace view of mailbox sync: +// every email_sync_state row joined to its mailbox, owner and workspace. +type AdminSyncRepository interface { + // Search lists sync rows newest-updated first, filtered by state and a + // free-text match on the mailbox address or workspace name, and returns + // the instance-wide summary alongside (the summary ignores the filter). + Search(ctx context.Context, search *models.AdminSyncSearch) (*models.AdminSyncResult, error) + // ClearThrottle lifts a fair-use throttle on the platform copy. False when + // the mailbox has no sync row or was not throttled. + ClearThrottle(ctx context.Context, emailID uuid.UUID) (bool, error) + // ResetBackfill puts the backfill back to pending with an empty cursor and + // zero progress, so the next load re-imports history from scratch. False + // when the mailbox has no sync row. + ResetBackfill(ctx context.Context, emailID uuid.UUID) (bool, error) +} + +type adminSyncRepository struct { + db *db.DB +} + +func NewAdminSyncRepository(d *db.DB) AdminSyncRepository { + return &adminSyncRepository{db: d} +} + +// Sentinel errors the handler maps to 400. +var ( + ErrAdminSyncBadCursor = errors.New("admin sync: invalid cursor") + ErrAdminSyncBadState = errors.New("admin sync: invalid state filter") +) + +const ( + adminSyncDefaultLimit = 50 + adminSyncMaxLimit = 200 +) + +// adminSyncStateWhere is the per-row predicate for each state filter. Expired +// throttles are not throttles, mirroring EmailSyncStateRepository.Get. +func adminSyncStateWhere(state string) (string, error) { + switch state { + case "", "all": + return "", nil + case "throttled": + return "s.throttled_until > now()", nil + case "backfilling": + return "s.backfill_status = 'running'", nil + case "stalled": + return "s.backfill_status = 'running' AND s.updated_at < now() - interval '1 hour'", nil + case "pending": + return "s.backfill_status = 'pending'", nil + case "complete": + return "s.backfill_status = 'complete'", nil + } + return "", ErrAdminSyncBadState +} + +func (r *adminSyncRepository) Search(ctx context.Context, search *models.AdminSyncSearch) (*models.AdminSyncResult, error) { + if search == nil { + search = &models.AdminSyncSearch{} + } + limit := search.Limit + if limit <= 0 { + limit = adminSyncDefaultLimit + } + if limit > adminSyncMaxLimit { + limit = adminSyncMaxLimit + } + + where := "WHERE TRUE" + args := []any{} + if cond, err := adminSyncStateWhere(search.State); err != nil { + return nil, err + } else if cond != "" { + where += " AND " + cond + } + if q := strings.TrimSpace(search.Q); q != "" { + args = append(args, "%"+q+"%") + n := itoa(len(args)) + where += " AND (ea.email ILIKE $" + n + " OR o.name ILIKE $" + n + ")" + } + if search.Cursor != "" { + cursor, err := uuid.Parse(search.Cursor) + if err != nil { + return nil, ErrAdminSyncBadCursor + } + args = append(args, cursor) + n := itoa(len(args)) + // Keyset on (updated_at, email_id); the cursor is the last row's id and + // its updated_at is looked up so the token stays a plain uuid. + where += " AND (s.updated_at, s.email_id) < ((SELECT c.updated_at FROM email_sync_state c WHERE c.email_id = $" + n + "), $" + n + "::uuid)" + } + args = append(args, limit+1) + + query := ` + SELECT s.email_id, s.user_id, ea.email, ea.provider::text, ea.status::text, + ea.organization_id, COALESCE(o.name, ''), ea.worker_id, + s.backfill_status, s.backfill_synced, s.backfill_since, + s.backfill_started_at, s.backfill_completed_at, + CASE WHEN s.throttled_until > now() THEN s.throttled_until END, + CASE WHEN s.throttled_until > now() THEN s.throttle_reason ELSE '' END, + s.deferred, + (s.backfill_status = 'running' AND s.updated_at < now() - interval '1 hour'), + s.last_synced_at, s.updated_at + FROM email_sync_state s + JOIN email_accounts ea ON ea.id = s.email_id + LEFT JOIN organizations o ON o.id = ea.organization_id + ` + where + ` + ORDER BY s.updated_at DESC, s.email_id DESC + LIMIT $` + itoa(len(args)) + + rows, err := r.db.Query(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("admin sync: search: %w", err) + } + defer rows.Close() + + items := []models.AdminSyncRow{} + for rows.Next() { + var row models.AdminSyncRow + if err := rows.Scan( + &row.EmailID, &row.UserID, &row.Email, &row.Provider, &row.AccountStatus, + &row.OrganizationID, &row.OrganizationName, &row.WorkerID, + &row.BackfillStatus, &row.BackfillSynced, &row.BackfillSince, + &row.BackfillStartedAt, &row.BackfillCompletedAt, + &row.ThrottledUntil, &row.ThrottleReason, + &row.Deferred, &row.Stalled, + &row.LastSyncedAt, &row.UpdatedAt, + ); err != nil { + return nil, fmt.Errorf("admin sync: scan: %w", err) + } + items = append(items, row) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("admin sync: rows: %w", err) + } + + result := &models.AdminSyncResult{ + Data: items, + Pagination: &models.Pagination{HasMore: len(items) > limit}, + } + if len(items) > limit { + result.Data = items[:limit] + result.Pagination.NextCursor = paging.UUIDString(items[limit-1].EmailID) + } + + const summary = ` + SELECT COUNT(*), + COUNT(*) FILTER (WHERE throttled_until > now()), + COUNT(*) FILTER (WHERE backfill_status = 'running'), + COUNT(*) FILTER (WHERE backfill_status = 'running' AND updated_at < now() - interval '1 hour'), + COUNT(*) FILTER (WHERE backfill_status = 'pending'), + COUNT(*) FILTER (WHERE backfill_status = 'complete'), + COALESCE(SUM(deferred), 0) + FROM email_sync_state + ` + var total, throttled, backfilling, stalled, pending, complete, deferred int64 + if err := r.db.QueryRow(ctx, summary).Scan(&total, &throttled, &backfilling, &stalled, &pending, &complete, &deferred); err != nil { + return nil, fmt.Errorf("admin sync: summary: %w", err) + } + result.Summary = models.AdminSyncSummary{ + Total: int(total), + Throttled: int(throttled), + Backfilling: int(backfilling), + Stalled: int(stalled), + Pending: int(pending), + Complete: int(complete), + Deferred: int(deferred), + } + return result, nil +} + +func (r *adminSyncRepository) ClearThrottle(ctx context.Context, emailID uuid.UUID) (bool, error) { + const q = ` + UPDATE email_sync_state + SET throttled_until = NULL, throttle_reason = '', updated_at = now() + WHERE email_id = $1 AND throttled_until IS NOT NULL + ` + tag, err := r.db.Exec(ctx, q, emailID) + if err != nil { + return false, fmt.Errorf("admin sync: clear throttle: %w", err) + } + return tag.RowsAffected() > 0, nil +} + +func (r *adminSyncRepository) ResetBackfill(ctx context.Context, emailID uuid.UUID) (bool, error) { + const q = ` + UPDATE email_sync_state + SET backfill_status = 'pending', backfill_cursor = '{}'::jsonb, backfill_synced = 0, + backfill_since = NULL, backfill_started_at = NULL, backfill_completed_at = NULL, + updated_at = now() + WHERE email_id = $1 + ` + tag, err := r.db.Exec(ctx, q, emailID) + if err != nil { + return false, fmt.Errorf("admin sync: reset backfill: %w", err) + } + return tag.RowsAffected() > 0, nil +} diff --git a/internal/repository/pg_job_runs.go b/internal/repository/pg_job_runs.go new file mode 100644 index 00000000..e603bf59 --- /dev/null +++ b/internal/repository/pg_job_runs.go @@ -0,0 +1,120 @@ +package repository + +import ( + "context" + "time" + + "github.com/warmbly/warmbly/internal/infrastructure/db" + "github.com/warmbly/warmbly/internal/models" +) + +// JobRunRepository persists scheduled_job_runs, one row per background loop. +// It is the jobrun.Store the backend and consumer record to. +type JobRunRepository interface { + Register(ctx context.Context, name, service string, interval time.Duration, nextRunAt time.Time) error + MarkStarted(ctx context.Context, name string, at time.Time) error + MarkFinished(ctx context.Context, name string, startedAt, finishedAt time.Time, runErr error, nextRunAt time.Time) error + RequestRun(ctx context.Context, name string) (bool, error) + TakeRunRequest(ctx context.Context, name string) (bool, error) + List(ctx context.Context) ([]models.ScheduledJobRun, error) +} + +type jobRunRepository struct { + db *db.DB +} + +func NewJobRunRepository(d *db.DB) JobRunRepository { + return &jobRunRepository{db: d} +} + +func (r *jobRunRepository) Register(ctx context.Context, name, service string, interval time.Duration, nextRunAt time.Time) error { + _, err := r.db.Exec(ctx, ` + INSERT INTO scheduled_job_runs (name, service, interval_seconds, next_run_at, updated_at) + VALUES ($1, $2, $3, $4, now()) + ON CONFLICT (name) DO UPDATE SET + service = EXCLUDED.service, + interval_seconds = EXCLUDED.interval_seconds, + next_run_at = EXCLUDED.next_run_at, + last_status = CASE WHEN scheduled_job_runs.last_status = 'running' THEN 'idle' ELSE scheduled_job_runs.last_status END, + updated_at = now() + `, name, service, int(interval.Seconds()), nextRunAt) + return err +} + +func (r *jobRunRepository) MarkStarted(ctx context.Context, name string, at time.Time) error { + _, err := r.db.Exec(ctx, ` + UPDATE scheduled_job_runs + SET last_started_at = $2, last_status = 'running', updated_at = now() + WHERE name = $1 + `, name, at) + return err +} + +func (r *jobRunRepository) MarkFinished(ctx context.Context, name string, startedAt, finishedAt time.Time, runErr error, nextRunAt time.Time) error { + status, msg := "ok", "" + if runErr != nil { + status, msg = "error", runErr.Error() + if len(msg) > 2000 { + msg = msg[:2000] + } + } + _, err := r.db.Exec(ctx, ` + UPDATE scheduled_job_runs + SET last_finished_at = $2, + last_duration_ms = $3, + last_status = $4, + last_error = $5, + run_count = run_count + 1, + error_count = error_count + CASE WHEN $4 = 'error' THEN 1 ELSE 0 END, + next_run_at = $6, + updated_at = now() + WHERE name = $1 + `, name, finishedAt, finishedAt.Sub(startedAt).Milliseconds(), status, msg, nextRunAt) + return err +} + +func (r *jobRunRepository) RequestRun(ctx context.Context, name string) (bool, error) { + tag, err := r.db.Exec(ctx, ` + UPDATE scheduled_job_runs SET run_requested_at = now(), updated_at = now() WHERE name = $1 + `, name) + if err != nil { + return false, err + } + return tag.RowsAffected() > 0, nil +} + +func (r *jobRunRepository) TakeRunRequest(ctx context.Context, name string) (bool, error) { + tag, err := r.db.Exec(ctx, ` + UPDATE scheduled_job_runs SET run_requested_at = NULL, updated_at = now() + WHERE name = $1 AND run_requested_at IS NOT NULL + `, name) + if err != nil { + return false, err + } + return tag.RowsAffected() > 0, nil +} + +func (r *jobRunRepository) List(ctx context.Context) ([]models.ScheduledJobRun, error) { + rows, err := r.db.Query(ctx, ` + SELECT name, service, interval_seconds, last_started_at, last_finished_at, last_duration_ms, + last_status, last_error, run_count, error_count, run_requested_at, next_run_at, updated_at + FROM scheduled_job_runs + ORDER BY service, name + `) + if err != nil { + return nil, err + } + defer rows.Close() + out := []models.ScheduledJobRun{} + for rows.Next() { + var j models.ScheduledJobRun + if err := rows.Scan( + &j.Name, &j.Service, &j.IntervalSeconds, &j.LastStartedAt, &j.LastFinishedAt, &j.LastDurationMs, + &j.LastStatus, &j.LastError, &j.RunCount, &j.ErrorCount, &j.RunRequestedAt, &j.NextRunAt, &j.UpdatedAt, + ); err != nil { + return nil, err + } + out = append(out, j) + } + return out, rows.Err() +} From 9729f608c01c76d40b69d8604dc5b3dc3b34c728 Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Mon, 7 Sep 2026 21:56:23 -0700 Subject: [PATCH 2/2] feat: address the CodeRabbit review on the admin backend PR by carrying the keyset boundary inside the sync cursor token instead of re-reading the cursor row's mutable updated_at, clearing only a throttle that is still active so an expired one is not reported as cleared, counting a medium-only UTM attribution in the acquisition with_channel total the way the channel list already does, releasing the exact dedicated assignment the handler read by id so a binding created meanwhile is never released, and returning the underlying failure from the warmup engagement poller, heartbeat sync, danger zone and org transfer housekeeping loops so scheduled_job_runs records an error instead of ok --- internal/api/handler/admin_fleet.go | 6 +++--- .../app/consumer/warmup_engagement_poller.go | 12 +++++------- internal/app/consumer/worker_heartbeat.go | 10 +++++----- internal/jobs/danger_zone.go | 17 ++++++++--------- internal/jobs/org_transfer.go | 15 ++++++--------- internal/repository/pg_admin_insight.go | 2 +- internal/repository/pg_admin_sync.go | 18 +++++++++--------- internal/repository/pg_worker.go | 15 +++++++++++++++ 8 files changed, 52 insertions(+), 43 deletions(-) diff --git a/internal/api/handler/admin_fleet.go b/internal/api/handler/admin_fleet.go index 06ab38db..a289e680 100644 --- a/internal/api/handler/admin_fleet.go +++ b/internal/api/handler/admin_fleet.go @@ -119,9 +119,9 @@ func (h *Handler) AdminFleetReleaseDedicated(c *gin.Context) { errx.JSON(c, errx.New(errx.Internal, "migrate to shared: "+err.Error())) return } - // MigrateOrgToShared swallows its own release error, so release again; the - // UPDATE is a no-op when the row is already released. - if err := h.WorkerAssignmentService.ReleaseDedicatedWorker(ctx, orgID); err != nil { + // MigrateOrgToShared swallows its own release error, so release the exact + // row read above by id: a binding created meanwhile is never touched. + if _, err := h.WorkerRepo.ReleaseDedicatedAssignmentByID(ctx, assignment.ID); err != nil { errx.JSON(c, errx.New(errx.Internal, "release assignment: "+err.Error())) return } diff --git a/internal/app/consumer/warmup_engagement_poller.go b/internal/app/consumer/warmup_engagement_poller.go index 6a062ad9..d787ffab 100644 --- a/internal/app/consumer/warmup_engagement_poller.go +++ b/internal/app/consumer/warmup_engagement_poller.go @@ -3,6 +3,7 @@ package jobs import ( "context" "encoding/json" + "fmt" "time" "github.com/rs/zerolog/log" @@ -19,20 +20,16 @@ func (s *JobsService) StartWarmupEngagementPoller(ctx context.Context, interval if s.WarmupEngagementRepo == nil || s.Publisher == nil { return } - jobrun.Loop(ctx, "warmup_engagement_poller", interval, false, func(ctx context.Context) error { - s.drainDueEngagements(ctx) - return nil - }) + jobrun.Loop(ctx, "warmup_engagement_poller", interval, false, s.drainDueEngagements) } -func (s *JobsService) drainDueEngagements(ctx context.Context) { +func (s *JobsService) drainDueEngagements(ctx context.Context) error { cctx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() due, err := s.WarmupEngagementRepo.ClaimDuePendingEngagements(cctx, 200) if err != nil { - log.Warn().Err(err).Msg("warmup engagement poller: claim failed") - return + return fmt.Errorf("warmup engagement poller: claim: %w", err) } for _, p := range due { @@ -56,4 +53,5 @@ func (s *JobsService) drainDueEngagements(ctx context.Context) { action.DelaySeconds = 0 // dwell already elapsed; run immediately s.Publisher.PublishWarmupAction(cctx, *account.WorkerID, &action) } + return nil } diff --git a/internal/app/consumer/worker_heartbeat.go b/internal/app/consumer/worker_heartbeat.go index 12b41ff3..d6a72e3d 100644 --- a/internal/app/consumer/worker_heartbeat.go +++ b/internal/app/consumer/worker_heartbeat.go @@ -3,6 +3,7 @@ package jobs import ( "context" "errors" + "fmt" "time" "github.com/redis/go-redis/v9" @@ -23,16 +24,14 @@ func (s *JobsService) StartWorkerHeartbeatSync(ctx context.Context, interval tim jobrun.Loop(ctx, "worker_heartbeat_sync", interval, false, func(ctx context.Context) error { runCtx, cancel := context.WithTimeout(ctx, 15*time.Second) defer cancel() - s.syncHeartbeats(runCtx) - return nil + return s.syncHeartbeats(runCtx) }) } -func (s *JobsService) syncHeartbeats(ctx context.Context) { +func (s *JobsService) syncHeartbeats(ctx context.Context) error { workers, err := s.WorkerRepo.GetAllActiveWorkers(ctx) if err != nil { - log.Warn().Err(err).Msg("heartbeat sync: list workers failed") - return + return fmt.Errorf("heartbeat sync: list workers: %w", err) } for _, w := range workers { @@ -52,4 +51,5 @@ func (s *JobsService) syncHeartbeats(ctx context.Context) { log.Warn().Err(err).Str("worker_id", w.ID.String()).Msg("heartbeat sync: update failed") } } + return nil } diff --git a/internal/jobs/danger_zone.go b/internal/jobs/danger_zone.go index e856a4c6..ad7be5f5 100644 --- a/internal/jobs/danger_zone.go +++ b/internal/jobs/danger_zone.go @@ -2,10 +2,10 @@ package jobs import ( "context" + "errors" + "fmt" "time" - "github.com/warmbly/warmbly/internal/observability/errs" - "github.com/warmbly/warmbly/internal/app/dangerzone" "github.com/warmbly/warmbly/internal/jobrun" ) @@ -25,13 +25,15 @@ func NewDangerZoneJob(svc dangerzone.Service) *DangerZoneJob { // Run performs one tick. Errors are logged to Sentry and swallowed so // the scheduler keeps ticking; the next tick will retry anything that // got marked failed. -func (j *DangerZoneJob) Run(ctx context.Context) { +func (j *DangerZoneJob) Run(ctx context.Context) error { + var failures []error if _, _, err := j.svc.ExecuteDuePendingDeletions(ctx); err != nil { - errs.CaptureException(err) + failures = append(failures, fmt.Errorf("execute deletions: %w", err)) } if err := j.svc.DispatchReminders(ctx); err != nil { - errs.CaptureException(err) + failures = append(failures, fmt.Errorf("dispatch reminders: %w", err)) } + return errors.Join(failures...) } // DangerZoneScheduler runs the job on a fixed interval. @@ -56,10 +58,7 @@ func NewDangerZoneScheduler(job *DangerZoneJob, interval time.Duration) *DangerZ func (s *DangerZoneScheduler) Start(ctx context.Context) { ctx, cancel := stopContext(ctx, s.stopCh) defer cancel() - jobrun.Loop(ctx, "danger_zone", s.interval, true, func(ctx context.Context) error { - s.job.Run(ctx) - return nil - }) + jobrun.Loop(ctx, "danger_zone", s.interval, true, s.job.Run) } // Stop halts the scheduler. diff --git a/internal/jobs/org_transfer.go b/internal/jobs/org_transfer.go index 6780de8d..fb6da979 100644 --- a/internal/jobs/org_transfer.go +++ b/internal/jobs/org_transfer.go @@ -2,10 +2,9 @@ package jobs import ( "context" + "fmt" "time" - "github.com/warmbly/warmbly/internal/observability/errs" - "github.com/warmbly/warmbly/internal/app/orgtransfer" "github.com/warmbly/warmbly/internal/jobrun" ) @@ -30,13 +29,14 @@ func NewOrgTransferJob(svc orgtransfer.Service) *OrgTransferJob { // Run performs one tick. Errors are reported and swallowed so the scheduler // keeps ticking; the next tick retries whatever did not land. -func (j *OrgTransferJob) Run(ctx context.Context) { +func (j *OrgTransferJob) Run(ctx context.Context) error { if j.svc == nil { - return + return nil } if _, err := j.svc.PurgeExpiredExports(ctx); err != nil { - errs.CaptureException(err) + return fmt.Errorf("purge expired exports: %w", err) } + return nil } // OrgTransferScheduler runs the job on a fixed interval. @@ -60,10 +60,7 @@ func NewOrgTransferScheduler(job *OrgTransferJob, interval time.Duration) *OrgTr func (s *OrgTransferScheduler) Start(ctx context.Context) { ctx, cancel := stopContext(ctx, s.stopCh) defer cancel() - jobrun.Loop(ctx, "org_transfer_housekeeping", s.interval, true, func(ctx context.Context) error { - s.job.Run(ctx) - return nil - }) + jobrun.Loop(ctx, "org_transfer_housekeeping", s.interval, true, s.job.Run) } // Stop halts the scheduler. diff --git a/internal/repository/pg_admin_insight.go b/internal/repository/pg_admin_insight.go index 657bd25c..986e8162 100644 --- a/internal/repository/pg_admin_insight.go +++ b/internal/repository/pg_admin_insight.go @@ -222,7 +222,7 @@ func (r *adminInsightRepository) Acquisition(ctx context.Context, days int) (*mo } const totals = cohort + ` - SELECT COUNT(*)::int, COUNT(*) FILTER (WHERE source <> '')::int, COUNT(*) FILTER (WHERE converted)::int + SELECT COUNT(*)::int, COUNT(*) FILTER (WHERE source <> '' OR medium <> '')::int, COUNT(*) FILTER (WHERE converted)::int FROM cohort ` if err := r.db.QueryRow(ctx, totals, days).Scan(&out.Signups, &out.WithChannel, &out.Converted); err != nil { diff --git a/internal/repository/pg_admin_sync.go b/internal/repository/pg_admin_sync.go index ab317d4a..07a21910 100644 --- a/internal/repository/pg_admin_sync.go +++ b/internal/repository/pg_admin_sync.go @@ -92,15 +92,15 @@ func (r *adminSyncRepository) Search(ctx context.Context, search *models.AdminSy where += " AND (ea.email ILIKE $" + n + " OR o.name ILIKE $" + n + ")" } if search.Cursor != "" { - cursor, err := uuid.Parse(search.Cursor) - if err != nil { + // The token carries the boundary itself, so a row that moves or is + // deleted between pages cannot shift or empty the next page. + at, id, xerr := paging.DecodeTimeCursor(search.Cursor) + if xerr != nil { return nil, ErrAdminSyncBadCursor } - args = append(args, cursor) - n := itoa(len(args)) - // Keyset on (updated_at, email_id); the cursor is the last row's id and - // its updated_at is looked up so the token stays a plain uuid. - where += " AND (s.updated_at, s.email_id) < ((SELECT c.updated_at FROM email_sync_state c WHERE c.email_id = $" + n + "), $" + n + "::uuid)" + args = append(args, at, id) + n := len(args) + where += " AND (s.updated_at, s.email_id) < ($" + itoa(n-1) + "::timestamptz, $" + itoa(n) + "::uuid)" } args = append(args, limit+1) @@ -153,7 +153,7 @@ func (r *adminSyncRepository) Search(ctx context.Context, search *models.AdminSy } if len(items) > limit { result.Data = items[:limit] - result.Pagination.NextCursor = paging.UUIDString(items[limit-1].EmailID) + result.Pagination.NextCursor = paging.EncodeTime(items[limit-1].UpdatedAt, items[limit-1].EmailID) } const summary = ` @@ -186,7 +186,7 @@ func (r *adminSyncRepository) ClearThrottle(ctx context.Context, emailID uuid.UU const q = ` UPDATE email_sync_state SET throttled_until = NULL, throttle_reason = '', updated_at = now() - WHERE email_id = $1 AND throttled_until IS NOT NULL + WHERE email_id = $1 AND throttled_until > now() ` tag, err := r.db.Exec(ctx, q, emailID) if err != nil { diff --git a/internal/repository/pg_worker.go b/internal/repository/pg_worker.go index 4cadd298..4c3d5f24 100644 --- a/internal/repository/pg_worker.go +++ b/internal/repository/pg_worker.go @@ -59,6 +59,9 @@ type WorkerRepository interface { GetActiveDedicatedAssignment(ctx context.Context, userID uuid.UUID) (*models.DedicatedWorkerAssignment, error) GetDedicatedWorkerByOrgID(ctx context.Context, orgID uuid.UUID) (*models.Worker, error) ReleaseDedicatedAssignment(ctx context.Context, userID uuid.UUID) error + // ReleaseDedicatedAssignmentByID releases one specific binding; false when + // it was already released, so a caller never releases a newer one by accident. + ReleaseDedicatedAssignmentByID(ctx context.Context, id uuid.UUID) (bool, error) // Email account worker queries GetEmailAccountsByWorkerID(ctx context.Context, workerID uuid.UUID) ([]uuid.UUID, error) @@ -413,6 +416,18 @@ func (r *workerRepository) ReleaseDedicatedAssignment(ctx context.Context, userI return err } +func (r *workerRepository) ReleaseDedicatedAssignmentByID(ctx context.Context, id uuid.UUID) (bool, error) { + tag, err := r.db.Exec(ctx, ` + UPDATE dedicated_worker_assignments + SET released_at = now() + WHERE id = $1 AND released_at IS NULL + `, id) + if err != nil { + return false, err + } + return tag.RowsAffected() > 0, nil +} + // GetEmailAccountsByWorkerID retrieves all email account IDs assigned to a worker func (r *workerRepository) GetEmailAccountsByWorkerID(ctx context.Context, workerID uuid.UUID) ([]uuid.UUID, error) { query := `SELECT id FROM email_accounts WHERE worker_id = $1`