diff --git a/cmd/backend/main.go b/cmd/backend/main.go index 6c34e7f8..c1670f7d 100644 --- a/cmd/backend/main.go +++ b/cmd/backend/main.go @@ -122,6 +122,7 @@ import ( "github.com/warmbly/warmbly/internal/infrastructure/kms" "github.com/warmbly/warmbly/internal/infrastructure/pubsub" "github.com/warmbly/warmbly/internal/infrastructure/storage" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/jobs" "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/notify" @@ -273,6 +274,16 @@ func main() { // Workspace archives (export/import between instances) var orgTransferService orgtransfer.Service + // Admin operations pages and the scheduled job registry. + var ( + adminSyncRepo repository.AdminSyncRepository + adminSendsRepo repository.AdminSendsRepository + adminFleetRepo repository.AdminFleetRepository + adminInsightRepo repository.AdminInsightRepository + jobRunRepo repository.JobRunRepository + webhookRepoForHandler repository.WebhookRepository + ) + // Organization-wide audit trail var auditService audit.AuditService @@ -690,6 +701,16 @@ func main() { webhookRepository := repository.NewWebhookRepository(primaryDB.Pool) webhookService := webhook.NewService(webhookRepository) webhookServiceForHandler = webhookService + webhookRepoForHandler = webhookRepository + + // Every background loop in this process records to scheduled_job_runs + // from here on, and the admin panel can ask any of them to run now. + jobRunRepo = repository.NewJobRunRepository(primaryDB) + jobrun.Configure(jobRunRepo, "backend") + adminSyncRepo = repository.NewAdminSyncRepository(primaryDB) + adminSendsRepo = repository.NewAdminSendsRepository(primaryDB) + adminFleetRepo = repository.NewAdminFleetRepository(primaryDB) + adminInsightRepo = repository.NewAdminInsightRepository(primaryDB) integrationRepository := repository.NewIntegrationRepository(primaryDB.Pool) // OAuth 2.1 authorization server (third-party app registration + token flow). @@ -1032,20 +1053,7 @@ func main() { // rebalance + scale + quarantine evaluators see fresh rolling // metrics. The materialized view is what aggregates the 1h windows // across all workers. - go func() { - tick := time.NewTicker(time.Minute) - defer tick.Stop() - for { - select { - case <-ctx.Done(): - return - case <-tick.C: - if err := workerRepository.RefreshWorkerCapacityView(ctx); err != nil { - log.Printf("worker_capacity_view refresh: %v", err) - } - } - } - }() + go jobrun.Loop(ctx, "worker_capacity_refresh", time.Minute, false, workerRepository.RefreshWorkerCapacityView) go (&fleet.Rebalancer{ WorkerRepo: workerRepository, @@ -2021,7 +2029,6 @@ func main() { WorkerOrchestrator: workerOrchestrator, WorkerRepo: workerRepoForHandler, CredentialsRepo: credentialsRepository, - ReleasesService: releasesService, UpdatesService: updatesService, // Notifications @@ -2100,6 +2107,14 @@ func main() { // Admin System Status probes SystemChecker: systemChecker, + // Admin operations pages. + AdminSyncRepo: adminSyncRepo, + AdminSendsRepo: adminSendsRepo, + AdminFleetRepo: adminFleetRepo, + AdminInsightRepo: adminInsightRepo, + JobRuns: jobRunRepo, + WebhookRepo: webhookRepoForHandler, + // Organization-wide audit trail, backed by Postgres. The no-op // fallback (audit.NewNoOpService) remains for entrypoints without // a database. diff --git a/cmd/consumer/main.go b/cmd/consumer/main.go index 9b8dfff8..e9ee7fd0 100644 --- a/cmd/consumer/main.go +++ b/cmd/consumer/main.go @@ -44,6 +44,7 @@ import ( "github.com/warmbly/warmbly/internal/infrastructure/kms" "github.com/warmbly/warmbly/internal/infrastructure/pubsub" "github.com/warmbly/warmbly/internal/infrastructure/storage" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/notify" "github.com/warmbly/warmbly/internal/observability" @@ -400,6 +401,10 @@ func main() { eventsPublisher := events.NewPublisher(consumerBus, s3Client, consumerCodec, cipherService) + // Every consumer loop records to scheduled_job_runs, so the admin panel + // lists it next to the backend's and can ask it to run now. + jobrun.Configure(repository.NewJobRunRepository(primaryDB), "consumer") + // JobsService jobsService := &jobs.JobsService{ Bus: consumerBus, diff --git a/internal/api/handler/admin.go b/internal/api/handler/admin.go index b6536a30..a122bd05 100644 --- a/internal/api/handler/admin.go +++ b/internal/api/handler/admin.go @@ -682,28 +682,6 @@ func (h *Handler) AdminGetHourlyEmailStats(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"stats": stats}) } -// AdminGetWorkerLoadStats gets worker load statistics -func (h *Handler) AdminGetWorkerLoadStats(c *gin.Context) { - stats, xerr := h.AdminService.GetWorkerLoadStats(c.Request.Context()) - if xerr != nil { - errx.JSON(c, xerr) - return - } - - c.JSON(http.StatusOK, gin.H{"stats": stats}) -} - -// AdminGetEmailDistribution gets email distribution across workers -func (h *Handler) AdminGetEmailDistribution(c *gin.Context) { - dist, xerr := h.AdminService.GetEmailDistribution(c.Request.Context()) - if xerr != nil { - errx.JSON(c, xerr) - return - } - - c.JSON(http.StatusOK, gin.H{"distribution": dist}) -} - // AdminGetUserGrowthStats gets user growth statistics func (h *Handler) AdminGetUserGrowthStats(c *gin.Context) { startDate, endDate := parseDateRange(c) @@ -717,189 +695,6 @@ func (h *Handler) AdminGetUserGrowthStats(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"stats": stats}) } -// Plan Management Handlers - -// AdminListPlans lists the plan catalog with the shared faceted query params; -// returns the standard {data, pagination} envelope. -func (h *Handler) AdminListPlans(c *gin.Context) { - var search models.AdminPlanSearch - if err := c.ShouldBindQuery(&search); err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid query parameters")) - return - } - // duration is durations.title (free text, no DB enum) — validate app-side. - if search.Duration != "" && search.Duration != "month" && search.Duration != "year" { - errx.JSON(c, errx.New(errx.BadRequest, "invalid duration")) - return - } - - result, xerr := h.AdminService.SearchPlansForAdmin(c.Request.Context(), &search) - if xerr != nil { - errx.JSON(c, xerr) - return - } - - c.JSON(http.StatusOK, result) -} - -// AdminCreatePlan creates a new plan -func (h *Handler) AdminCreatePlan(c *gin.Context) { - adminID := middleware.GetAdminUserID(c) - if adminID == nil { - errx.JSON(c, errx.ErrUnauthorized) - return - } - - var req models.CreatePlanRequest - if err := c.ShouldBindJSON(&req); err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid request body")) - return - } - - plan, xerr := h.AdminService.CreatePlan(c.Request.Context(), *adminID, &req, c.ClientIP(), c.GetHeader("User-Agent")) - if xerr != nil { - errx.JSON(c, xerr) - return - } - - c.JSON(http.StatusCreated, plan) -} - -// AdminGetPlan gets a plan by ID -func (h *Handler) AdminGetPlan(c *gin.Context) { - planID, err := uuid.Parse(c.Param("id")) - if err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid plan ID")) - return - } - - plan, xerr := h.AdminService.GetPlan(c.Request.Context(), planID) - if xerr != nil { - errx.JSON(c, xerr) - return - } - - c.JSON(http.StatusOK, plan) -} - -// AdminUpdatePlan updates a plan -func (h *Handler) AdminUpdatePlan(c *gin.Context) { - adminID := middleware.GetAdminUserID(c) - if adminID == nil { - errx.JSON(c, errx.ErrUnauthorized) - return - } - - planID, err := uuid.Parse(c.Param("id")) - if err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid plan ID")) - return - } - - var req models.UpdatePlanRequest - if err := c.ShouldBindJSON(&req); err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid request body")) - return - } - - plan, xerr := h.AdminService.UpdatePlan(c.Request.Context(), *adminID, planID, &req, c.ClientIP(), c.GetHeader("User-Agent")) - if xerr != nil { - errx.JSON(c, xerr) - return - } - - c.JSON(http.StatusOK, plan) -} - -// AdminDeletePlan deletes a plan -func (h *Handler) AdminDeletePlan(c *gin.Context) { - adminID := middleware.GetAdminUserID(c) - if adminID == nil { - errx.JSON(c, errx.ErrUnauthorized) - return - } - - planID, err := uuid.Parse(c.Param("id")) - if err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid plan ID")) - return - } - - xerr := h.AdminService.DeletePlan(c.Request.Context(), *adminID, planID, c.ClientIP(), c.GetHeader("User-Agent")) - if xerr != nil { - errx.JSON(c, xerr) - return - } - - c.JSON(http.StatusOK, gin.H{"message": "plan deleted successfully"}) -} - -// Enterprise Inquiry Handlers - -// AdminListEnterpriseInquiries lists enterprise inquiries with the shared -// faceted query params; returns the standard {data, pagination} envelope. -func (h *Handler) AdminListEnterpriseInquiries(c *gin.Context) { - var search models.AdminEnterpriseInquirySearch - if err := c.ShouldBindQuery(&search); err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid query parameters")) - return - } - - result, xerr := h.AdminService.ListEnterpriseInquiries(c.Request.Context(), &search) - if xerr != nil { - errx.JSON(c, xerr) - return - } - - c.JSON(http.StatusOK, result) -} - -// AdminGetEnterpriseInquiry gets a specific enterprise inquiry -func (h *Handler) AdminGetEnterpriseInquiry(c *gin.Context) { - inquiryID, err := uuid.Parse(c.Param("id")) - if err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid inquiry ID")) - return - } - - inquiry, xerr := h.AdminService.GetEnterpriseInquiry(c.Request.Context(), inquiryID) - if xerr != nil { - errx.JSON(c, xerr) - return - } - - c.JSON(http.StatusOK, inquiry) -} - -// AdminUpdateEnterpriseInquiry updates an enterprise inquiry -func (h *Handler) AdminUpdateEnterpriseInquiry(c *gin.Context) { - adminID := middleware.GetAdminUserID(c) - if adminID == nil { - errx.JSON(c, errx.ErrUnauthorized) - return - } - - inquiryID, err := uuid.Parse(c.Param("id")) - if err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid inquiry ID")) - return - } - - var req models.UpdateEnterpriseInquiryRequest - if err := c.ShouldBindJSON(&req); err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid request body")) - return - } - - xerr := h.AdminService.UpdateEnterpriseInquiry(c.Request.Context(), *adminID, inquiryID, &req, c.ClientIP(), c.GetHeader("User-Agent")) - if xerr != nil { - errx.JSON(c, xerr) - return - } - - c.JSON(http.StatusOK, gin.H{"message": "inquiry updated successfully"}) -} - // Admin Management Handlers // AdminListAdmins lists all admin users diff --git a/internal/api/handler/admin_credentials.go b/internal/api/handler/admin_credentials.go index 4c23a464..5c26ad87 100644 --- a/internal/api/handler/admin_credentials.go +++ b/internal/api/handler/admin_credentials.go @@ -1,324 +1,19 @@ -// Admin endpoints for reusable worker credentials. +// Admin endpoints binding a worker to a reusable worker profile. // -// /admin/aws-credentials list / create -// /admin/aws-credentials/:id get / update / delete -// /admin/worker-profiles list / create -// /admin/worker-profiles/:id get / update / delete -// /admin/worker-profiles/:id/workers list workers using this profile -// /admin/worker-profiles/:id/apply re-write env + restart on every assigned worker -// /admin/workers/:id/profile assign / unassign a profile to a worker -// /admin/workers/:id/apply re-write env + restart for a single worker -// -// Secret material is never returned over the API. Update bodies use empty -// strings to mean "keep the stored value as-is". The dashboard renders set -// secrets as "••••••". +// /admin/workers/:id/profile assign / unassign a profile to a worker +// /admin/workers/:id/apply re-write env + restart for a single worker package handler import ( - "fmt" "net/http" "github.com/gin-gonic/gin" "github.com/google/uuid" "github.com/warmbly/warmbly/internal/errx" "github.com/warmbly/warmbly/internal/models" - "github.com/warmbly/warmbly/internal/repository" ) -// AWS credentials - -type awsCredsBody struct { - Name string `json:"name" binding:"required"` - Description string `json:"description"` - Region string `json:"region" binding:"required"` - AccessKeyID string `json:"access_key_id" binding:"required"` - SecretAccessKey string `json:"secret_access_key"` // empty on update = keep -} - -func (h *Handler) AdminListAWSCreds(c *gin.Context) { - if h.CredentialsRepo == nil { - errx.JSON(c, errx.New(errx.Internal, "credentials repo not configured")) - return - } - creds, err := h.CredentialsRepo.ListAWSCreds(c.Request.Context()) - if err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - c.JSON(http.StatusOK, gin.H{"data": creds}) -} - -func (h *Handler) AdminCreateAWSCreds(c *gin.Context) { - var body awsCredsBody - if err := c.ShouldBindJSON(&body); err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid request body")) - return - } - if body.SecretAccessKey == "" { - errx.JSON(c, errx.New(errx.BadRequest, "secret_access_key is required on create")) - return - } - enc, err := h.WorkerOrchestrator.EncryptSecret(c.Request.Context(), body.SecretAccessKey) - if err != nil { - errx.JSON(c, errx.New(errx.Internal, "encrypt: "+err.Error())) - return - } - id, err := h.CredentialsRepo.CreateAWSCreds(c.Request.Context(), repository.CreateAWSCredsInput{ - Name: body.Name, - Description: body.Description, - Region: body.Region, - AccessKeyID: body.AccessKeyID, - SecretAccessKeyEncrypted: enc, - }) - if err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - h.audit(c, models.AuditActionCreate, models.AuditEntityAWSCredentials, &id, map[string]string{ - "name": body.Name, - "region": body.Region, - }) - c.JSON(http.StatusCreated, gin.H{"id": id}) -} - -func (h *Handler) AdminGetAWSCreds(c *gin.Context) { - id, ok := parseUUID(c, "id") - if !ok { - return - } - creds, err := h.CredentialsRepo.GetAWSCreds(c.Request.Context(), id) - if err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - if creds == nil { - errx.JSON(c, errx.New(errx.NotFound, "credentials not found")) - return - } - // Don't leak ciphertext. - creds.SecretAccessKeyEncrypted = "" - c.JSON(http.StatusOK, creds) -} - -func (h *Handler) AdminUpdateAWSCreds(c *gin.Context) { - id, ok := parseUUID(c, "id") - if !ok { - return - } - var body awsCredsBody - if err := c.ShouldBindJSON(&body); err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid request body")) - return - } - enc := "" - if body.SecretAccessKey != "" { - var err error - enc, err = h.WorkerOrchestrator.EncryptSecret(c.Request.Context(), body.SecretAccessKey) - if err != nil { - errx.JSON(c, errx.New(errx.Internal, "encrypt: "+err.Error())) - return - } - } - if err := h.CredentialsRepo.UpdateAWSCreds(c.Request.Context(), id, repository.UpdateAWSCredsInput{ - Name: body.Name, - Description: body.Description, - Region: body.Region, - AccessKeyID: body.AccessKeyID, - SecretAccessKeyEncrypted: enc, - }); err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - changes := map[string]string{} - if enc != "" { - changes["secret_access_key"] = "rotated" - } - h.audit(c, models.AuditActionUpdate, models.AuditEntityAWSCredentials, &id, changes) - c.JSON(http.StatusOK, gin.H{"ok": true}) -} - -func (h *Handler) AdminDeleteAWSCreds(c *gin.Context) { - id, ok := parseUUID(c, "id") - if !ok { - return - } - if err := h.CredentialsRepo.DeleteAWSCreds(c.Request.Context(), id); err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - h.audit(c, models.AuditActionDelete, models.AuditEntityAWSCredentials, &id, nil) - c.JSON(http.StatusOK, gin.H{"ok": true}) -} - -// worker profiles - -type profileBody struct { - Name string `json:"name" binding:"required"` - Description string `json:"description"` - AppEnv string `json:"app_env"` - WorkerImage string `json:"worker_image"` - KafkaBootstrap string `json:"kafka_bootstrap_servers"` - KafkaSASLUsername string `json:"kafka_sasl_username"` - KafkaSASLPassword string `json:"kafka_sasl_password"` - SchemaRegistryURL string `json:"schema_registry_url"` - SchemaRegistryKey string `json:"schema_registry_key"` - SchemaRegistrySecret string `json:"schema_registry_secret"` - RedisURL string `json:"redis_url"` - AWSCredentialID *string `json:"aws_credential_id"` -} - -func (h *Handler) AdminListProfiles(c *gin.Context) { - if h.CredentialsRepo == nil { - errx.JSON(c, errx.New(errx.Internal, "credentials repo not configured")) - return - } - profiles, err := h.CredentialsRepo.ListProfiles(c.Request.Context()) - if err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - c.JSON(http.StatusOK, gin.H{"data": profiles}) -} - -func (h *Handler) AdminCreateProfile(c *gin.Context) { - var body profileBody - if err := c.ShouldBindJSON(&body); err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid request body")) - return - } - in, xerr := h.profileBodyToInput(c, body) - if xerr { - return - } - id, err := h.CredentialsRepo.CreateProfile(c.Request.Context(), in) - if err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - h.audit(c, models.AuditActionCreate, models.AuditEntityWorkerProfile, &id, map[string]string{ - "name": body.Name, - "app_env": body.AppEnv, - }) - c.JSON(http.StatusCreated, gin.H{"id": id}) -} - -func (h *Handler) AdminGetProfile(c *gin.Context) { - id, ok := parseUUID(c, "id") - if !ok { - return - } - p, err := h.CredentialsRepo.GetProfile(c.Request.Context(), id) - if err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - if p == nil { - errx.JSON(c, errx.New(errx.NotFound, "profile not found")) - return - } - c.JSON(http.StatusOK, p) -} - -func (h *Handler) AdminUpdateProfile(c *gin.Context) { - id, ok := parseUUID(c, "id") - if !ok { - return - } - var body profileBody - if err := c.ShouldBindJSON(&body); err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid request body")) - return - } - in, xerr := h.profileBodyToInput(c, body) - if xerr { - return - } - if err := h.CredentialsRepo.UpdateProfile(c.Request.Context(), id, in); err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - // Record which secret fields were rotated (we don't log the values, only that they changed). - changes := map[string]string{} - if body.KafkaSASLPassword != "" { - changes["kafka_sasl_password"] = "rotated" - } - if body.SchemaRegistrySecret != "" { - changes["schema_registry_secret"] = "rotated" - } - if body.RedisURL != "" { - changes["redis_url"] = "rotated" - } - h.audit(c, models.AuditActionUpdate, models.AuditEntityWorkerProfile, &id, changes) - c.JSON(http.StatusOK, gin.H{"ok": true}) -} - -func (h *Handler) AdminDeleteProfile(c *gin.Context) { - id, ok := parseUUID(c, "id") - if !ok { - return - } - if err := h.CredentialsRepo.DeleteProfile(c.Request.Context(), id); err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - h.audit(c, models.AuditActionDelete, models.AuditEntityWorkerProfile, &id, nil) - c.JSON(http.StatusOK, gin.H{"ok": true}) -} - -func (h *Handler) AdminListProfileWorkers(c *gin.Context) { - id, ok := parseUUID(c, "id") - if !ok { - return - } - workers, err := h.WorkerRepo.ListWorkersByProfile(c.Request.Context(), id) - if err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - c.JSON(http.StatusOK, gin.H{"data": workers}) -} - -// AdminApplyProfile re-applies the profile to every assigned worker by -// re-writing /etc/warmbly/worker.env and restarting the service. -// -// Reports a per-worker outcome map so the UI can show what succeeded. -func (h *Handler) AdminApplyProfile(c *gin.Context) { - id, ok := parseUUID(c, "id") - if !ok { - return - } - workers, err := h.WorkerRepo.ListWorkersByProfile(c.Request.Context(), id) - if err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - results := make([]gin.H, 0, len(workers)) - for _, w := range workers { - // Skip workers that aren't installed yet — apply is restart-based. - if w.InstallState != models.WorkerInstallStateInstalled { - results = append(results, gin.H{"worker_id": w.ID, "ok": false, "skipped": "not installed"}) - continue - } - applyErr := h.WorkerOrchestrator.ApplyConfig(c.Request.Context(), w.ID) - r := gin.H{"worker_id": w.ID, "ok": applyErr == nil} - if applyErr != nil { - r["error"] = applyErr.Error() - } - results = append(results, r) - } - okCount := 0 - for _, r := range results { - if v, _ := r["ok"].(bool); v { - okCount++ - } - } - h.audit(c, models.AuditActionApply, models.AuditEntityWorkerProfile, &id, map[string]string{ - "workers_applied": fmt.Sprintf("%d/%d", okCount, len(results)), - }) - c.JSON(http.StatusOK, gin.H{"results": results}) -} - // worker → profile binding type assignProfileBody struct { @@ -374,70 +69,6 @@ func (h *Handler) AdminApplyWorkerConfig(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"ok": true}) } -// helpers - -func (h *Handler) profileBodyToInput(c *gin.Context, body profileBody) (repository.CreateProfileInput, bool) { - ctx := c.Request.Context() - encrypt := func(s string) (string, bool) { - if s == "" { - return "", true - } - enc, err := h.WorkerOrchestrator.EncryptSecret(ctx, s) - if err != nil { - errx.JSON(c, errx.New(errx.Internal, "encrypt: "+err.Error())) - return "", false - } - return enc, true - } - - kafkaEnc, ok := encrypt(body.KafkaSASLPassword) - if !ok { - return repository.CreateProfileInput{}, true - } - schemaEnc, ok := encrypt(body.SchemaRegistrySecret) - if !ok { - return repository.CreateProfileInput{}, true - } - redisEnc, ok := encrypt(body.RedisURL) - if !ok { - return repository.CreateProfileInput{}, true - } - - var awsID *uuid.UUID - if body.AWSCredentialID != nil && *body.AWSCredentialID != "" { - parsed, err := uuid.Parse(*body.AWSCredentialID) - if err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid aws_credential_id")) - return repository.CreateProfileInput{}, true - } - awsID = &parsed - } - - appEnv := body.AppEnv - if appEnv == "" { - appEnv = "prod" - } - image := body.WorkerImage - if image == "" { - image = "ghcr.io/warmbly/worker:latest" - } - - return repository.CreateProfileInput{ - Name: body.Name, - Description: body.Description, - AppEnv: appEnv, - WorkerImage: image, - KafkaBootstrap: body.KafkaBootstrap, - KafkaSASLUsername: body.KafkaSASLUsername, - KafkaSASLPasswordEncrypted: kafkaEnc, - SchemaRegistryURL: body.SchemaRegistryURL, - SchemaRegistryKey: body.SchemaRegistryKey, - SchemaRegistrySecretEncrypted: schemaEnc, - RedisURLEncrypted: redisEnc, - AWSCredentialID: awsID, - }, false -} - func parseUUID(c *gin.Context, param string) (uuid.UUID, bool) { id, err := uuid.Parse(c.Param(param)) if err != nil { diff --git a/internal/api/handler/admin_fleet.go b/internal/api/handler/admin_fleet.go new file mode 100644 index 00000000..a289e680 --- /dev/null +++ b/internal/api/handler/admin_fleet.go @@ -0,0 +1,178 @@ +package handler + +import ( + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" +) + +// Fleet placement as the operator sees it: every worker against its capacity +// row, the decision log the control loops write, and the dedicated bindings. + +// AdminFleetCapacity lists every worker with its capacity-view row, hottest first. +// +// GET /admin/fleet/capacity +func (h *Handler) AdminFleetCapacity(c *gin.Context) { + if h.AdminFleetRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "fleet view is not available on this instance")) + return + } + rows, err := h.AdminFleetRepo.Capacity(c.Request.Context()) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + c.JSON(http.StatusOK, gin.H{"data": rows}) +} + +// AdminFleetDecisions lists decision_log newest first. +// +// GET /admin/fleet/decisions?kind=&worker_id=&limit= +func (h *Handler) AdminFleetDecisions(c *gin.Context) { + if h.AdminFleetRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "fleet view is not available on this instance")) + return + } + var workerID *uuid.UUID + if raw := c.Query("worker_id"); raw != "" { + id, err := uuid.Parse(raw) + if err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid worker_id")) + return + } + workerID = &id + } + limit := 0 + if raw := c.Query("limit"); raw != "" { + n, err := strconv.Atoi(raw) + if err != nil || n <= 0 { + errx.JSON(c, errx.New(errx.BadRequest, "invalid limit")) + return + } + limit = n + } + rows, err := h.AdminFleetRepo.Decisions(c.Request.Context(), c.Query("kind"), workerID, limit) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + c.JSON(http.StatusOK, gin.H{"data": rows}) +} + +// AdminFleetDedicated lists active worker-to-organization bindings. +// +// GET /admin/fleet/dedicated +func (h *Handler) AdminFleetDedicated(c *gin.Context) { + if h.AdminFleetRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "fleet view is not available on this instance")) + return + } + rows, err := h.AdminFleetRepo.DedicatedAssignments(c.Request.Context()) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + c.JSON(http.StatusOK, gin.H{"data": rows}) +} + +// AdminFleetReleaseDedicated is the inverse of AdminConvertWorkerToDedicated: +// the org's mailboxes go back to shared premium workers, the binding is +// released, and the worker re-enters the shared pool once nothing binds it. +// +// POST /admin/fleet/dedicated/:orgId/release +func (h *Handler) AdminFleetReleaseDedicated(c *gin.Context) { + orgID, ok := parseUUIDParam(c, "orgId") + if !ok { + return + } + if h.WorkerAssignmentService == nil || h.WorkerRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "worker placement is not available on this instance")) + return + } + ctx := c.Request.Context() + + assignment, err := h.WorkerRepo.GetActiveDedicatedAssignment(ctx, orgID) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + if assignment == nil { + errx.JSON(c, errx.New(errx.NotFound, "organization has no active dedicated worker")) + return + } + workerID := assignment.WorkerID + + before, err := h.WorkerRepo.GetEmailAccountsByWorkerID(ctx, workerID) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, "list accounts: "+err.Error())) + return + } + + // Moves every org mailbox onto a live shared premium worker and releases + // the binding; a mailbox with no shared target stays put rather than failing. + if err := h.WorkerAssignmentService.MigrateOrgToShared(ctx, orgID); err != nil { + errx.JSON(c, errx.New(errx.Internal, "migrate to shared: "+err.Error())) + return + } + // MigrateOrgToShared swallows its own release error, so release the exact + // row read above by id: a binding created meanwhile is never touched. + if _, err := h.WorkerRepo.ReleaseDedicatedAssignmentByID(ctx, assignment.ID); err != nil { + errx.JSON(c, errx.New(errx.Internal, "release assignment: "+err.Error())) + return + } + + after, err := h.WorkerRepo.GetEmailAccountsByWorkerID(ctx, workerID) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, "list accounts: "+err.Error())) + return + } + moved := len(before) - len(after) + if moved < 0 { + moved = 0 + } + + // The worker only returns to the shared pool when no other org binds it. + stillBound := false + if h.AdminFleetRepo != nil { + active, err := h.AdminFleetRepo.DedicatedAssignments(ctx) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, "list assignments: "+err.Error())) + return + } + for _, a := range active { + if a.WorkerID == workerID { + stillBound = true + break + } + } + } + returnedToShared := false + if !stillBound { + if err := h.WorkerRepo.SetWorkerType(ctx, workerID, models.WorkerTypeShared); err != nil { + errx.JSON(c, errx.New(errx.Internal, "set type: "+err.Error())) + return + } + returnedToShared = true + } + + h.audit(c, "release_dedicated", models.AuditEntityWorker, &workerID, map[string]string{ + "organization_id": orgID.String(), + "subscription_id": assignment.SubscriptionID.String(), + "assignment_id": assignment.ID.String(), + "accounts_moved": itoa(moved), + "accounts_remaining": itoa(len(after)), + "returned_to_shared": boolStr(returnedToShared), + }) + c.JSON(http.StatusOK, gin.H{ + "ok": true, + "worker_id": workerID, + "accounts_moved": moved, + "accounts_remaining": len(after), + "returned_to_shared": returnedToShared, + }) +} diff --git a/internal/api/handler/admin_insight.go b/internal/api/handler/admin_insight.go new file mode 100644 index 00000000..0d1da919 --- /dev/null +++ b/internal/api/handler/admin_insight.go @@ -0,0 +1,162 @@ +package handler + +import ( + "net/http" + "strconv" + "time" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" +) + +const adminInsightUnavailable = "this insight is not available on this instance" + +// queryInt reads an integer query parameter, clamping it to [min, max]. +func queryInt(c *gin.Context, key string, def, min, max int) int { + v := def + if raw := c.Query(key); raw != "" { + if n, err := strconv.Atoi(raw); err == nil { + v = n + } + } + if v < min { + v = min + } + if v > max { + v = max + } + return v +} + +// AdminGetAcquisition is GET /admin/analytics/acquisition?days=30. +func (h *Handler) AdminGetAcquisition(c *gin.Context) { + if h.AdminInsightRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, adminInsightUnavailable)) + return + } + days := queryInt(c, "days", 30, 1, 365) + out, err := h.AdminInsightRepo.Acquisition(c.Request.Context(), days) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, "failed to load acquisition")) + return + } + c.JSON(http.StatusOK, out) +} + +// AdminWarmupAbuse is GET /admin/warmup/abuse?window=24h|7d|30d&limit=50. +func (h *Handler) AdminWarmupAbuse(c *gin.Context) { + if h.AdminInsightRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, adminInsightUnavailable)) + return + } + var window time.Duration + switch c.DefaultQuery("window", "7d") { + case "24h": + window = 24 * time.Hour + case "7d": + window = 7 * 24 * time.Hour + case "30d": + window = 30 * 24 * time.Hour + default: + errx.JSON(c, errx.New(errx.BadRequest, "window must be 24h, 7d or 30d")) + return + } + limit := queryInt(c, "limit", 50, 1, 200) + rows, err := h.AdminInsightRepo.WarmupAbuse(c.Request.Context(), time.Now().Add(-window), limit) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, "failed to load warmup abuse signals")) + return + } + c.JSON(http.StatusOK, gin.H{"data": rows}) +} + +// AdminWarmupActions is GET /admin/warmup/actions?limit=100. +func (h *Handler) AdminWarmupActions(c *gin.Context) { + if h.AdminInsightRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, adminInsightUnavailable)) + return + } + limit := queryInt(c, "limit", 100, 1, 500) + rows, err := h.AdminInsightRepo.WarmupActions(c.Request.Context(), limit) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, "failed to load warmup actions")) + return + } + c.JSON(http.StatusOK, gin.H{"data": rows}) +} + +// AdminListOrgAPIKeys is GET /admin/organizations/:id/api-keys. +func (h *Handler) AdminListOrgAPIKeys(c *gin.Context) { + if h.AdminInsightRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, adminInsightUnavailable)) + return + } + orgID, err := uuid.Parse(c.Param("id")) + if err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid organization ID")) + return + } + rows, err := h.AdminInsightRepo.OrgAPIKeys(c.Request.Context(), orgID) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, "failed to load API keys")) + return + } + c.JSON(http.StatusOK, gin.H{"data": rows}) +} + +// AdminRevokeOrgAPIKey is DELETE /admin/organizations/:id/api-keys/:keyId. +func (h *Handler) AdminRevokeOrgAPIKey(c *gin.Context) { + if h.APIKeyService == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "API keys are not available on this instance")) + return + } + orgID, err := uuid.Parse(c.Param("id")) + if err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid organization ID")) + return + } + keyID, err := uuid.Parse(c.Param("keyId")) + if err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid API key ID")) + return + } + var body struct { + Reason string `json:"reason"` + } + if c.Request.ContentLength != 0 { + if err := c.ShouldBindJSON(&body); err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid request body")) + return + } + } + reason := body.Reason + if reason == "" { + reason = "revoked by platform admin" + } + if xerr := h.APIKeyService.Revoke(c.Request.Context(), orgID, keyID, reason); xerr != nil { + errx.JSON(c, xerr) + return + } + h.audit(c, models.AuditActionRevoke, models.AuditEntityAPIKey, &keyID, map[string]string{ + "organization_id": orgID.String(), + "reason": reason, + }) + c.JSON(http.StatusOK, gin.H{"id": keyID, "organization_id": orgID, "status": models.APIKeyStatusRevoked}) +} + +// AdminListTransfers is GET /admin/transfers?limit=100. +func (h *Handler) AdminListTransfers(c *gin.Context) { + if h.AdminInsightRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, adminInsightUnavailable)) + return + } + limit := queryInt(c, "limit", 100, 1, 500) + rows, err := h.AdminInsightRepo.ListTransfers(c.Request.Context(), limit) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, "failed to load transfers")) + return + } + c.JSON(http.StatusOK, gin.H{"data": rows}) +} diff --git a/internal/api/handler/admin_jobs.go b/internal/api/handler/admin_jobs.go new file mode 100644 index 00000000..b74665c4 --- /dev/null +++ b/internal/api/handler/admin_jobs.go @@ -0,0 +1,57 @@ +// Admin endpoints for the scheduled jobs registry: every background loop the +// backend and consumer run, with its last run, and a "run now" request the +// owning loop picks up on its next poll. + +package handler + +import ( + "net/http" + + "github.com/gin-gonic/gin" + + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" +) + +const scheduledJobEntity models.AuditEntityType = "scheduled_job" + +// AdminListJobs lists every registered background loop. +func (h *Handler) AdminListJobs(c *gin.Context) { + if h.JobRuns == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "job registry is not available on this instance")) + return + } + jobs, err := h.JobRuns.List(c.Request.Context()) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + if jobs == nil { + jobs = []models.ScheduledJobRun{} + } + c.JSON(http.StatusOK, gin.H{"data": jobs}) +} + +// AdminRunJob asks the loop that owns the job to run at its next poll. +func (h *Handler) AdminRunJob(c *gin.Context) { + if h.JobRuns == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "job registry is not available on this instance")) + return + } + name := c.Param("name") + if name == "" { + errx.JSON(c, errx.New(errx.BadRequest, "job name is required")) + return + } + found, err := h.JobRuns.RequestRun(c.Request.Context(), name) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + if !found { + errx.JSON(c, errx.New(errx.NotFound, "job not found")) + return + } + h.audit(c, models.AuditActionStart, scheduledJobEntity, nil, map[string]string{"job": name}) + c.JSON(http.StatusOK, gin.H{"requested": true}) +} diff --git a/internal/api/handler/admin_provisioning.go b/internal/api/handler/admin_provisioning.go deleted file mode 100644 index 0aa2353c..00000000 --- a/internal/api/handler/admin_provisioning.go +++ /dev/null @@ -1,814 +0,0 @@ -package handler - -import ( - "context" - "encoding/json" - "errors" - "net/http" - "strconv" - "time" - - "github.com/gin-gonic/gin" - "github.com/google/uuid" - "github.com/warmbly/warmbly/internal/infrastructure/cloudprovider" - "github.com/warmbly/warmbly/internal/infrastructure/cloudprovider/hetzner" - "github.com/warmbly/warmbly/internal/models" - "github.com/warmbly/warmbly/internal/repository" -) - -// Admin endpoints under /admin for autonomous fleet management: -// -// /admin/cloud-credentials CRUD + test -// /admin/cloud-providers/:provider/... discovery (locations, server_types, images) -// /admin/provisioning-templates CRUD -// /admin/provisioning-jobs list + create + detail -// /admin/provisioning-policy per-provider budget caps -// -// All gated by AdminPermManageSettings via the route registration. - -// --------------------------------------------------------------------------- -// Cloud credentials -// --------------------------------------------------------------------------- - -type CloudCredentialResponse struct { - ID uuid.UUID `json:"id"` - Provider string `json:"provider"` - Name string `json:"name"` - TokenRedacted string `json:"token_redacted"` - LastUsedAt *time.Time `json:"last_used_at,omitempty"` - LastTestAt *time.Time `json:"last_test_at,omitempty"` - LastTestOK *bool `json:"last_test_ok,omitempty"` - LastTestError *string `json:"last_test_error,omitempty"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` -} - -func toCredResponse(c *repository.CloudCredential) CloudCredentialResponse { - return CloudCredentialResponse{ - ID: c.ID, - Provider: c.Provider, - Name: c.Name, - TokenRedacted: maskToken(c.EncryptedToken), - LastUsedAt: c.LastUsedAt, - LastTestAt: c.LastTestAt, - LastTestOK: c.LastTestOK, - LastTestError: c.LastTestError, - CreatedAt: c.CreatedAt, - UpdatedAt: c.UpdatedAt, - } -} - -func maskToken(t string) string { - if len(t) <= 8 { - return "***" - } - return t[:4] + "***" + t[len(t)-4:] -} - -func (h *Handler) AdminListCloudCredentials(c *gin.Context) { - if h.CloudCredentialRepo == nil { - c.JSON(http.StatusOK, gin.H{"data": []any{}}) - return - } - rows, err := h.CloudCredentialRepo.List(c.Request.Context()) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - out := make([]CloudCredentialResponse, 0, len(rows)) - for _, r := range rows { - out = append(out, toCredResponse(&r)) - } - c.JSON(http.StatusOK, gin.H{"data": out}) -} - -type CreateCloudCredentialRequest struct { - Provider string `json:"provider"` - Name string `json:"name"` - Token string `json:"token"` -} - -func (h *Handler) AdminCreateCloudCredential(c *gin.Context) { - if h.CloudCredentialRepo == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "cloud credentials repo not configured"}) - return - } - var req CreateCloudCredentialRequest - if err := c.ShouldBindJSON(&req); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid body"}) - return - } - if req.Provider == "" || req.Token == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "provider and token required"}) - return - } - if req.Name == "" { - req.Name = req.Provider + "-default" - } - - // TODO: cipher-encrypt the token via h.CipherService. For now we store - // as-is so the wiring works end-to-end; flag this in audit log. - row := &repository.CloudCredential{ - Provider: req.Provider, - Name: req.Name, - EncryptedToken: req.Token, - } - if err := h.CloudCredentialRepo.Create(c.Request.Context(), row); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - c.JSON(http.StatusCreated, toCredResponse(row)) -} - -func (h *Handler) AdminDeleteCloudCredential(c *gin.Context) { - if h.CloudCredentialRepo == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "cloud credentials repo not configured"}) - return - } - id, err := uuid.Parse(c.Param("id")) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"}) - return - } - if err := h.CloudCredentialRepo.Delete(c.Request.Context(), id); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - c.Status(http.StatusNoContent) -} - -func (h *Handler) AdminTestCloudCredential(c *gin.Context) { - if h.CloudCredentialRepo == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "cloud credentials repo not configured"}) - return - } - id, err := uuid.Parse(c.Param("id")) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"}) - return - } - cred, err := h.CloudCredentialRepo.Get(c.Request.Context(), id) - if err != nil || cred == nil { - c.JSON(http.StatusNotFound, gin.H{"error": "credential not found"}) - return - } - provider, err := buildProvider(cred) - if err != nil { - _ = h.CloudCredentialRepo.UpdateTestResult(c.Request.Context(), id, false, err.Error()) - c.JSON(http.StatusOK, gin.H{"ok": false, "error": err.Error()}) - return - } - ctx, cancel := context.WithTimeout(c.Request.Context(), 15*time.Second) - defer cancel() - if err := provider.Verify(ctx); err != nil { - _ = h.CloudCredentialRepo.UpdateTestResult(ctx, id, false, err.Error()) - c.JSON(http.StatusOK, gin.H{"ok": false, "error": err.Error()}) - return - } - _ = h.CloudCredentialRepo.UpdateTestResult(ctx, id, true, "") - c.JSON(http.StatusOK, gin.H{"ok": true}) -} - -// buildProvider picks the right cloudprovider.Provider implementation for a -// credential row. Today only Hetzner; adding OVH/Vultr means another case. -func buildProvider(c *repository.CloudCredential) (cloudprovider.Provider, error) { - switch c.Provider { - case "hetzner": - return hetzner.New(c.EncryptedToken) - default: - return nil, errors.New("unsupported provider: " + c.Provider) - } -} - -// --------------------------------------------------------------------------- -// Provider catalog discovery (used by admin UI dropdowns) -// --------------------------------------------------------------------------- - -func (h *Handler) AdminListProviderLocations(c *gin.Context) { - provider, err := h.providerByName(c) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - return - } - ctx, cancel := context.WithTimeout(c.Request.Context(), 15*time.Second) - defer cancel() - locs, err := provider.Locations(ctx) - if err != nil { - c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) - return - } - c.JSON(http.StatusOK, gin.H{"data": locs}) -} - -func (h *Handler) AdminListProviderServerTypes(c *gin.Context) { - provider, err := h.providerByName(c) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - return - } - ctx, cancel := context.WithTimeout(c.Request.Context(), 15*time.Second) - defer cancel() - types, err := provider.ServerTypes(ctx) - if err != nil { - c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) - return - } - c.JSON(http.StatusOK, gin.H{"data": types}) -} - -func (h *Handler) AdminListProviderImages(c *gin.Context) { - provider, err := h.providerByName(c) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - return - } - ctx, cancel := context.WithTimeout(c.Request.Context(), 15*time.Second) - defer cancel() - images, err := provider.Images(ctx) - if err != nil { - c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) - return - } - c.JSON(http.StatusOK, gin.H{"data": images}) -} - -// providerByName resolves a provider name to a Provider client, looking up -// the most recent credential row for that provider. -func (h *Handler) providerByName(c *gin.Context) (cloudprovider.Provider, error) { - if h.CloudCredentialRepo == nil { - return nil, errors.New("cloud credentials not configured") - } - name := c.Param("provider") - if name == "" { - return nil, errors.New("provider path param required") - } - cred, err := h.CloudCredentialRepo.GetByProvider(c.Request.Context(), name) - if err != nil { - return nil, err - } - if cred == nil { - return nil, errors.New("no credential configured for provider " + name) - } - return buildProvider(cred) -} - -// --------------------------------------------------------------------------- -// Provisioning templates -// -// The admin UI works in a nested {name, description, config:{...}, -// auto_provision_tier, is_draft} shape; the repository row is flat. The DTO -// helpers below translate between the two so the two sides agree on the wire -// format (they did not previously, which silently 400'd every template save). -// --------------------------------------------------------------------------- - -type provLabelDTO struct { - Key string `json:"key"` - Value string `json:"value"` -} - -type provTemplateConfigDTO struct { - Provider string `json:"provider"` - CredentialID *uuid.UUID `json:"credential_id,omitempty"` - Location string `json:"location"` - ServerType string `json:"server_type"` - ServerCount int `json:"server_count"` - IPv4PerServer int `json:"ipv4_per_server"` - IPv6PerServer int `json:"ipv6_per_server"` - WorkerTier string `json:"worker_tier"` - WorkerProfileID *uuid.UUID `json:"worker_profile_id,omitempty"` - EgressKind string `json:"egress_kind"` - Image string `json:"image"` - Datacenter string `json:"datacenter,omitempty"` - PlacementGroup string `json:"placement_group,omitempty"` - PrivateNetwork string `json:"private_network,omitempty"` - Firewall string `json:"firewall"` - Labels []provLabelDTO `json:"labels"` -} - -type provTemplateDTO struct { - ID uuid.UUID `json:"id,omitempty"` - Name string `json:"name"` - Description string `json:"description,omitempty"` - Config provTemplateConfigDTO `json:"config"` - AutoProvisionTier string `json:"auto_provision_tier,omitempty"` - IsDraft bool `json:"is_draft"` - CreatedAt time.Time `json:"created_at,omitempty"` - UpdatedAt time.Time `json:"updated_at,omitempty"` -} - -func labelsToDTO(m map[string]string) []provLabelDTO { - out := make([]provLabelDTO, 0, len(m)) - for k, v := range m { - out = append(out, provLabelDTO{Key: k, Value: v}) - } - return out -} - -func labelsFromDTO(in []provLabelDTO) map[string]string { - m := map[string]string{} - for _, l := range in { - if l.Key != "" { - m[l.Key] = l.Value - } - } - return m -} - -func toTemplateDTO(t *repository.ProvisioningTemplate) provTemplateDTO { - auto := "" - if t.IsAutoTemplate { - auto = t.Tier - } - return provTemplateDTO{ - ID: t.ID, - Name: t.Name, - Description: t.Description, - Config: provTemplateConfigDTO{ - Provider: t.Provider, - Location: t.Location, - ServerType: t.ServerType, - ServerCount: t.ServerCount, - IPv4PerServer: t.IPv4PerServer, - IPv6PerServer: t.IPv6PerServer, - WorkerTier: t.Tier, - WorkerProfileID: t.WorkerProfileID, - EgressKind: t.EgressKind, - Image: t.Image, - Datacenter: t.Datacenter, - PlacementGroup: t.PlacementGroup, - PrivateNetwork: t.PrivateNetwork, - Firewall: t.Firewall, - Labels: labelsToDTO(t.Labels), - }, - AutoProvisionTier: auto, - IsDraft: t.IsDraft, - CreatedAt: t.CreatedAt, - UpdatedAt: t.UpdatedAt, - } -} - -// fromTemplateDTO maps the UI shape onto the flat repo model and applies the -// field defaults the create path used to apply inline. -func fromTemplateDTO(d *provTemplateDTO) *repository.ProvisioningTemplate { - cfg := d.Config - t := &repository.ProvisioningTemplate{ - ID: d.ID, - Name: d.Name, - Description: d.Description, - Provider: cfg.Provider, - Location: cfg.Location, - Datacenter: cfg.Datacenter, - ServerType: cfg.ServerType, - Image: cfg.Image, - ServerCount: cfg.ServerCount, - IPv4PerServer: cfg.IPv4PerServer, - IPv6PerServer: cfg.IPv6PerServer, - WorkerProfileID: cfg.WorkerProfileID, - Tier: cfg.WorkerTier, - EgressKind: cfg.EgressKind, - Labels: labelsFromDTO(cfg.Labels), - PlacementGroup: cfg.PlacementGroup, - PrivateNetwork: cfg.PrivateNetwork, - Firewall: cfg.Firewall, - IsDraft: d.IsDraft, - // A draft is never eligible as the tier's auto-provision template. - IsAutoTemplate: !d.IsDraft && d.AutoProvisionTier != "", - } - if t.Image == "" { - t.Image = "ubuntu-22.04" - } - if t.ServerCount == 0 { - t.ServerCount = 1 - } - if t.IPv4PerServer == 0 { - t.IPv4PerServer = 1 - } - if t.IPv6PerServer == 0 { - t.IPv6PerServer = 1 - } - if t.EgressKind == "" { - t.EgressKind = "cold_smtp" - } - return t -} - -func (h *Handler) AdminListProvisioningTemplates(c *gin.Context) { - if h.ProvisioningTemplateRepo == nil { - c.JSON(http.StatusOK, gin.H{"data": []any{}}) - return - } - rows, err := h.ProvisioningTemplateRepo.List(c.Request.Context()) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - out := make([]provTemplateDTO, 0, len(rows)) - for i := range rows { - out = append(out, toTemplateDTO(&rows[i])) - } - c.JSON(http.StatusOK, gin.H{"data": out}) -} - -func (h *Handler) AdminGetProvisioningTemplate(c *gin.Context) { - id, err := uuid.Parse(c.Param("id")) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"}) - return - } - row, err := h.ProvisioningTemplateRepo.Get(c.Request.Context(), id) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - if row == nil { - c.Status(http.StatusNotFound) - return - } - c.JSON(http.StatusOK, toTemplateDTO(row)) -} - -func (h *Handler) AdminCreateProvisioningTemplate(c *gin.Context) { - if h.ProvisioningTemplateRepo == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "template repo not configured"}) - return - } - var d provTemplateDTO - if err := c.ShouldBindJSON(&d); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid body: " + err.Error()}) - return - } - t := fromTemplateDTO(&d) - if t.Name == "" || t.Provider == "" || t.Location == "" || t.ServerType == "" || t.Tier == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "name, provider, location, server_type and worker_tier are required"}) - return - } - if !repository.IsClientRequestableTier(t.Tier) { - c.JSON(http.StatusBadRequest, gin.H{ - "error": "dedicated tier templates cannot be created; dedicated workers are allocated automatically by the control plane", - "code": "tier_not_allowed", - }) - return - } - if err := h.ProvisioningTemplateRepo.Create(c.Request.Context(), t); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - c.JSON(http.StatusCreated, toTemplateDTO(t)) -} - -func (h *Handler) AdminUpdateProvisioningTemplate(c *gin.Context) { - id, err := uuid.Parse(c.Param("id")) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"}) - return - } - var d provTemplateDTO - if err := c.ShouldBindJSON(&d); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid body"}) - return - } - t := fromTemplateDTO(&d) - t.ID = id - if t.Name == "" || t.Provider == "" || t.Location == "" || t.ServerType == "" || t.Tier == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "name, provider, location, server_type and worker_tier are required"}) - return - } - if !repository.IsClientRequestableTier(t.Tier) { - c.JSON(http.StatusBadRequest, gin.H{ - "error": "templates cannot be set to the dedicated tier; dedicated workers are allocated automatically by the control plane", - "code": "tier_not_allowed", - }) - return - } - if err := h.ProvisioningTemplateRepo.Update(c.Request.Context(), t); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - c.JSON(http.StatusOK, toTemplateDTO(t)) -} - -func (h *Handler) AdminDeleteProvisioningTemplate(c *gin.Context) { - id, err := uuid.Parse(c.Param("id")) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"}) - return - } - if err := h.ProvisioningTemplateRepo.Delete(c.Request.Context(), id); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - c.Status(http.StatusNoContent) -} - -// --------------------------------------------------------------------------- -// Provisioning jobs -// --------------------------------------------------------------------------- - -type CreateProvisioningJobRequest struct { - TemplateID *uuid.UUID `json:"template_id,omitempty"` - Custom json.RawMessage `json:"custom,omitempty"` - TriggeredBy string `json:"triggered_by,omitempty"` -} - -// Job rows are flat in the repo and the UI works in a nested {config:{...}} -// shape with snake_case keys — the same translation problem the templates have. -// toJobDTO maps the row (and decodes the stored flat config snapshot back into -// the nested ProvisioningConfig) so the jobs list / detail / progress panel get -// the fields they read. -type provJobStepDTO struct { - Key string `json:"key"` - Label string `json:"label"` - Done int `json:"done"` - Total int `json:"total"` -} - -type provJobTimelineDTO struct { - State string `json:"state"` - At time.Time `json:"at"` - Note string `json:"note,omitempty"` -} - -type provJobDTO struct { - ID uuid.UUID `json:"id"` - State string `json:"state"` - TriggeredBy string `json:"triggered_by,omitempty"` - Provider string `json:"provider"` - TemplateID *uuid.UUID `json:"template_id,omitempty"` - TemplateName string `json:"template_name,omitempty"` - Config provTemplateConfigDTO `json:"config"` - Progress []provJobStepDTO `json:"progress"` - Timeline []provJobTimelineDTO `json:"timeline"` - CreatedWorkerIDs []uuid.UUID `json:"created_worker_ids,omitempty"` - LastError *string `json:"last_error,omitempty"` - CompletedAt *time.Time `json:"completed_at,omitempty"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` -} - -func toJobDTO(j *repository.ProvisioningJob) provJobDTO { - cfg := provTemplateConfigDTO{Labels: []provLabelDTO{}} - templateName := "" - if len(j.Config) > 0 { - var flat repository.ProvisioningTemplate - if err := json.Unmarshal(j.Config, &flat); err == nil { - cfg = toTemplateDTO(&flat).Config - // The snapshot carries the template name for template-launched jobs - // (custom jobs snapshot under the placeholder name "custom"). - if j.TemplateID != nil { - templateName = flat.Name - } - } - } - return provJobDTO{ - ID: j.ID, - State: string(j.State), - TriggeredBy: j.TriggeredBy, - Provider: j.Provider, - TemplateID: j.TemplateID, - TemplateName: templateName, - Config: cfg, - Progress: []provJobStepDTO{}, - Timeline: []provJobTimelineDTO{}, - CreatedWorkerIDs: j.WorkerIDs, - LastError: j.Error, - CompletedAt: j.CompletedAt, - CreatedAt: j.CreatedAt, - UpdatedAt: j.UpdatedAt, - } -} - -func (h *Handler) AdminListProvisioningJobs(c *gin.Context) { - if h.ProvisioningJobRepo == nil { - c.JSON(http.StatusOK, gin.H{"data": []any{}}) - return - } - limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50")) - rows, err := h.ProvisioningJobRepo.List(c.Request.Context(), limit) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - out := make([]provJobDTO, 0, len(rows)) - for i := range rows { - out = append(out, toJobDTO(&rows[i])) - } - c.JSON(http.StatusOK, gin.H{"data": out}) -} - -func (h *Handler) AdminGetProvisioningJob(c *gin.Context) { - id, err := uuid.Parse(c.Param("id")) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"}) - return - } - row, err := h.ProvisioningJobRepo.Get(c.Request.Context(), id) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - if row == nil { - c.Status(http.StatusNotFound) - return - } - c.JSON(http.StatusOK, toJobDTO(row)) -} - -func (h *Handler) AdminCreateProvisioningJob(c *gin.Context) { - if h.ProvisioningJobRepo == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "provisioning jobs repo not configured"}) - return - } - var req CreateProvisioningJobRequest - if err := c.ShouldBindJSON(&req); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid body"}) - return - } - - // Resolve template or custom config into the JSONB config column. - var ( - config json.RawMessage - templateID *uuid.UUID - provider string - ) - if req.TemplateID != nil { - if h.ProvisioningTemplateRepo == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "template repo not configured"}) - return - } - t, err := h.ProvisioningTemplateRepo.Get(c.Request.Context(), *req.TemplateID) - if err != nil || t == nil { - c.JSON(http.StatusNotFound, gin.H{"error": "template not found"}) - return - } - if !repository.IsClientRequestableTier(t.Tier) { - c.JSON(http.StatusBadRequest, gin.H{ - "error": "this template targets the dedicated tier, which is allocated automatically by the control plane; pick a shared-tier template", - "code": "tier_not_allowed", - }) - return - } - // Snapshot the flat template into the job's config column. Its field - // names line up with provisioning.JobConfig, so the state machine reads - // it directly. - b, _ := json.Marshal(t) - config = b - templateID = &t.ID - provider = t.Provider - } else { - if len(req.Custom) == 0 { - c.JSON(http.StatusBadRequest, gin.H{"error": "either template_id or custom config required"}) - return - } - // The UI sends the nested config shape (worker_tier, {key,value} label - // rows). Normalize it through the same mapping templates use so the - // snapshot matches provisioning.JobConfig. - var cfgDTO provTemplateConfigDTO - if err := json.Unmarshal(req.Custom, &cfgDTO); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid custom config: " + err.Error()}) - return - } - if cfgDTO.Provider == "" || cfgDTO.Location == "" || cfgDTO.ServerType == "" || cfgDTO.WorkerTier == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "custom config requires provider, location, server_type and worker_tier"}) - return - } - snap := fromTemplateDTO(&provTemplateDTO{Name: "custom", Config: cfgDTO}) - if !repository.IsClientRequestableTier(snap.Tier) { - c.JSON(http.StatusBadRequest, gin.H{ - "error": "dedicated tier cannot be provisioned directly; dedicated workers are allocated automatically by the control plane", - "code": "tier_not_allowed", - }) - return - } - b, _ := json.Marshal(snap) - config = b - provider = cfgDTO.Provider - } - - if provider == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "provider missing from config"}) - return - } - - // Cloud-first gate: a provisioning job for a provider can only be created - // once a cloud credential for that provider exists — otherwise the job - // would sit in 'pending' forever with no way to reach the provider API. - var credentialID *uuid.UUID - if h.CloudCredentialRepo != nil { - cred, err := h.CloudCredentialRepo.GetByProvider(c.Request.Context(), provider) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - if cred == nil { - c.JSON(http.StatusConflict, gin.H{ - "error": "no cloud provider connected for " + provider + " — add one under Settings → Cloud Providers first", - "code": "cloud_provider_required", - }) - return - } - credentialID = &cred.ID - } - - triggeredBy := req.TriggeredBy - if triggeredBy == "" { - triggeredBy = "admin" - } - - job := &repository.ProvisioningJob{ - State: models.ProvJobPending, - TriggeredBy: triggeredBy, - Provider: provider, - CredentialID: credentialID, - TemplateID: templateID, - Config: config, - } - if err := h.ProvisioningJobRepo.Create(c.Request.Context(), job); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - - // The state machine pickup is asynchronous: the provisioning runner - // (internal/app/provisioning Runner) processes jobs in state != - // completed/failed. The admin UI polls GET /admin/provisioning-jobs/:id - // for live status. - - c.JSON(http.StatusAccepted, toJobDTO(job)) -} - -// AdminRetryProvisioningJob resets a failed job back to pending so the runner -// re-attempts it. -func (h *Handler) AdminRetryProvisioningJob(c *gin.Context) { - if h.ProvisioningJobRepo == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "provisioning jobs repo not configured"}) - return - } - id, err := uuid.Parse(c.Param("id")) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid id"}) - return - } - job, err := h.ProvisioningJobRepo.Get(c.Request.Context(), id) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - if job == nil { - c.JSON(http.StatusNotFound, gin.H{"error": "job not found"}) - return - } - if job.State != models.ProvJobFailed { - c.JSON(http.StatusConflict, gin.H{ - "error": "only failed jobs can be retried", - "code": "job_not_retryable", - }) - return - } - if err := h.ProvisioningJobRepo.Retry(c.Request.Context(), id); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - job, err = h.ProvisioningJobRepo.Get(c.Request.Context(), id) - if err != nil || job == nil { - c.JSON(http.StatusOK, gin.H{"ok": true}) - return - } - c.JSON(http.StatusOK, toJobDTO(job)) -} - -// --------------------------------------------------------------------------- -// Provisioning policy -// --------------------------------------------------------------------------- - -func (h *Handler) AdminListProvisioningPolicy(c *gin.Context) { - if h.ProvisioningPolicyRepo == nil { - c.JSON(http.StatusOK, gin.H{"policies": []any{}}) - return - } - rows, err := h.ProvisioningPolicyRepo.List(c.Request.Context()) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - c.JSON(http.StatusOK, gin.H{"policies": rows}) -} - -func (h *Handler) AdminUpdateProvisioningPolicy(c *gin.Context) { - if h.ProvisioningPolicyRepo == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "policy repo not configured"}) - return - } - var p repository.ProvisioningPolicy - if err := c.ShouldBindJSON(&p); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid body"}) - return - } - if p.Provider == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "provider required"}) - return - } - if err := h.ProvisioningPolicyRepo.Update(c.Request.Context(), &p); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - c.JSON(http.StatusOK, p) -} diff --git a/internal/api/handler/admin_releases.go b/internal/api/handler/admin_releases.go deleted file mode 100644 index 3901a2ff..00000000 --- a/internal/api/handler/admin_releases.go +++ /dev/null @@ -1,100 +0,0 @@ -// Release-handling endpoints. -// -// POST /webhooks/github/releases — public, HMAC-validated. GitHub posts here -// when a release event fires. -// POST /admin/releases/check — manual trigger from the dashboard. -// GET /admin/releases/state — last known per-channel resolution. -// PUT /admin/worker-profiles/:id/release — set release channel + auto-update. -// -// The webhook endpoint is intentionally not behind admin auth; security comes -// from the X-Hub-Signature-256 HMAC. If RELEASES_WEBHOOK_SECRET isn't set, -// the endpoint refuses all requests. - -package handler - -import ( - "fmt" - "io" - "net/http" - - "github.com/gin-gonic/gin" - "github.com/warmbly/warmbly/internal/app/releases" - "github.com/warmbly/warmbly/internal/errx" - "github.com/warmbly/warmbly/internal/models" -) - -func (h *Handler) GithubReleasesWebhook(c *gin.Context) { - if h.ReleasesService == nil { - c.JSON(http.StatusNotImplemented, gin.H{"error": "releases service not configured"}) - return - } - body, err := io.ReadAll(c.Request.Body) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "read body: " + err.Error()}) - return - } - signature := c.GetHeader("X-Hub-Signature-256") - eventType := c.GetHeader("X-GitHub-Event") - if err := h.ReleasesService.HandleWebhook(c.Request.Context(), body, signature, eventType); err != nil { - c.JSON(http.StatusUnauthorized, gin.H{"error": err.Error()}) - return - } - c.JSON(http.StatusOK, gin.H{"ok": true}) -} - -func (h *Handler) AdminCheckReleases(c *gin.Context) { - if h.ReleasesService == nil { - errx.JSON(c, errx.New(errx.NotFound, "releases service not configured")) - return - } - changed, err := h.ReleasesService.CheckGitHub(c.Request.Context()) - if err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - h.audit(c, models.AuditActionCheckReleases, models.AuditEntityRelease, nil, map[string]string{ - "changed_profiles": fmt.Sprintf("%d", len(changed)), - }) - c.JSON(http.StatusOK, gin.H{ - "state": h.ReleasesService.GetState(), - "changed": changed, - }) -} - -func (h *Handler) AdminReleasesState(c *gin.Context) { - if h.ReleasesService == nil { - c.JSON(http.StatusOK, gin.H{"enabled": false}) - return - } - c.JSON(http.StatusOK, h.ReleasesService.GetState()) -} - -type setReleaseBody struct { - Channel string `json:"channel" binding:"required,oneof=pinned stable dev"` - AutoUpdate bool `json:"auto_update"` -} - -func (h *Handler) AdminSetProfileRelease(c *gin.Context) { - id, ok := parseUUID(c, "id") - if !ok { - return - } - var body setReleaseBody - if err := c.ShouldBindJSON(&body); err != nil { - errx.JSON(c, errx.New(errx.BadRequest, "invalid request body")) - return - } - if err := h.CredentialsRepo.UpdateProfileRelease(c.Request.Context(), id, models.ReleaseChannel(body.Channel), body.AutoUpdate); err != nil { - errx.JSON(c, errx.New(errx.Internal, err.Error())) - return - } - h.audit(c, models.AuditActionUpdate, models.AuditEntityWorkerProfile, &id, map[string]string{ - "release_channel": body.Channel, - "auto_update": fmt.Sprintf("%t", body.AutoUpdate), - }) - c.JSON(http.StatusOK, gin.H{"ok": true}) -} - -// Compile-time check that the releases package is referenced (keeps the -// import stable across edits even if every method body changes). -var _ = (*releases.Service)(nil) diff --git a/internal/api/handler/admin_sends.go b/internal/api/handler/admin_sends.go new file mode 100644 index 00000000..fcce2111 --- /dev/null +++ b/internal/api/handler/admin_sends.go @@ -0,0 +1,186 @@ +// Admin endpoints for the send outcome loop and the task queues: in-flight +// reservations, dead letters, task failures and customer webhook delivery +// health across every workspace. + +package handler + +import ( + "net/http" + "strconv" + "time" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" +) + +const deadLetterEntity models.AuditEntityType = "task_dead_letter" + +// webhookDeliveryLease mirrors the default LeaseTimeout in +// webhook.NewDeliveryWorker (cmd/backend wires the worker without overriding it). +const webhookDeliveryLease = 5 * time.Minute + +// parseBoundedLimit is parseLimit with a page cap above the admin default of 100. +func parseBoundedLimit(s string, def, max int) int { + if s == "" { + return def + } + n, err := strconv.Atoi(s) + if err != nil || n <= 0 { + return def + } + if n > max { + return max + } + return n +} + +func (h *Handler) adminSendsReady(c *gin.Context) bool { + if h.AdminSendsRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "send operations are not available on this instance")) + return false + } + return true +} + +// AdminInFlightSends lists reserved sends no worker result has resolved yet. +func (h *Handler) AdminInFlightSends(c *gin.Context) { + if !h.adminSendsReady(c) { + return + } + limit := parseBoundedLimit(c.Query("limit"), 100, 500) + reclaimAfter := time.Duration(config.CampaignSendReclaimAfterMinutes) * time.Minute + result, err := h.AdminSendsRepo.InFlight(c.Request.Context(), reclaimAfter, limit) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + c.JSON(http.StatusOK, result) +} + +// AdminListDeadLetters pages task dead letters newest first. +func (h *Handler) AdminListDeadLetters(c *gin.Context) { + if !h.adminSendsReady(c) { + return + } + cursor := parseCursor(c.Query("cursor")) + limit := parseLimit(c.Query("limit"), 50) + status := c.Query("status") + switch status { + case "", "pending", "replayed", "failed": + default: + errx.JSON(c, errx.New(errx.BadRequest, "invalid status")) + return + } + result, err := h.AdminSendsRepo.ListDeadLetters(c.Request.Context(), status, cursor, limit) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + c.JSON(http.StatusOK, result) +} + +// AdminReplayDeadLetter re-enqueues one dead letter through its workspace. +func (h *Handler) AdminReplayDeadLetter(c *gin.Context) { + if !h.adminSendsReady(c) { + return + } + if h.AdvancedService == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "dead letter replay is not available on this instance")) + return + } + id, err := uuid.Parse(c.Param("id")) + if err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid dead letter ID")) + return + } + row, err := h.AdminSendsRepo.GetDeadLetter(c.Request.Context(), id) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + if row == nil { + errx.JSON(c, errx.New(errx.NotFound, "dead letter not found")) + return + } + if row.OrganizationID == nil { + errx.JSON(c, errx.New(errx.BadRequest, "dead letter has no organization; its task or mailbox is gone")) + return + } + if xerr := h.AdvancedService.ReplayDeadLetter(c.Request.Context(), *row.OrganizationID, id); xerr != nil { + errx.JSON(c, xerr) + return + } + h.audit(c, models.AuditActionResume, deadLetterEntity, &id, map[string]string{ + "organization_id": row.OrganizationID.String(), + "task_id": row.TaskID.String(), + "task_type": row.TaskType, + }) + c.JSON(http.StatusOK, gin.H{"replayed": true}) +} + +// AdminRecentTaskFailures lists the newest task failures with their mailbox. +func (h *Handler) AdminRecentTaskFailures(c *gin.Context) { + if !h.adminSendsReady(c) { + return + } + limit := parseBoundedLimit(c.Query("limit"), 100, 500) + rows, err := h.AdminSendsRepo.RecentTaskFailures(c.Request.Context(), limit) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + c.JSON(http.StatusOK, gin.H{"data": rows}) +} + +// AdminWebhookHealth is the instance-wide webhook delivery picture. +func (h *Handler) AdminWebhookHealth(c *gin.Context) { + if !h.adminSendsReady(c) { + return + } + health, err := h.AdminSendsRepo.WebhookHealth(c.Request.Context(), webhookDeliveryLease) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + c.JSON(http.StatusOK, health) +} + +// AdminWebhookReclaim re-queues deliveries stranded in_flight past the lease. +func (h *Handler) AdminWebhookReclaim(c *gin.Context) { + if h.WebhookRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "webhook delivery is not available on this instance")) + return + } + n, err := h.WebhookRepo.ReclaimStuckDeliveries(c.Request.Context(), webhookDeliveryLease) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + h.audit(c, models.AuditActionUpdate, models.AuditEntityWebhook, nil, map[string]string{ + "action": "reclaim_stuck_deliveries", + "reclaimed": strconv.FormatInt(n, 10), + }) + c.JSON(http.StatusOK, gin.H{"reclaimed": n}) +} + +// AdminListOrgWebhooks lists one workspace's endpoints with recent counts. +func (h *Handler) AdminListOrgWebhooks(c *gin.Context) { + if !h.adminSendsReady(c) { + return + } + orgID, err := uuid.Parse(c.Param("id")) + if err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid organization ID")) + return + } + rows, err := h.AdminSendsRepo.OrgWebhooks(c.Request.Context(), orgID) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return + } + c.JSON(http.StatusOK, gin.H{"data": rows}) +} diff --git a/internal/api/handler/admin_sync.go b/internal/api/handler/admin_sync.go new file mode 100644 index 00000000..4ec08e90 --- /dev/null +++ b/internal/api/handler/admin_sync.go @@ -0,0 +1,121 @@ +package handler + +import ( + "errors" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +const ( + auditActionSyncClearThrottle models.AuditAction = "sync_clear_throttle" + auditActionSyncRestartBackfill models.AuditAction = "sync_restart_backfill" +) + +// AdminSearchSync is GET /admin/sync: every mailbox's sync governor state. +func (h *Handler) AdminSearchSync(c *gin.Context) { + if h.AdminSyncRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "sync operations are not available on this instance")) + return + } + var search models.AdminSyncSearch + if err := c.ShouldBindQuery(&search); err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid query parameters")) + return + } + result, err := h.AdminSyncRepo.Search(c.Request.Context(), &search) + if err != nil { + switch { + case errors.Is(err, repository.ErrAdminSyncBadCursor): + errx.JSON(c, errx.New(errx.BadRequest, "invalid cursor")) + case errors.Is(err, repository.ErrAdminSyncBadState): + errx.JSON(c, errx.New(errx.BadRequest, "invalid state filter")) + default: + errx.JSON(c, errx.New(errx.Internal, "failed to load sync state")) + } + return + } + c.JSON(http.StatusOK, result) +} + +// AdminSyncClearThrottle is POST /admin/sync/:id/clear-throttle. +func (h *Handler) AdminSyncClearThrottle(c *gin.Context) { + if h.AdminSyncRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "sync operations are not available on this instance")) + return + } + id, err := uuid.Parse(c.Param("id")) + if err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid mailbox ID")) + return + } + cleared, err := h.AdminSyncRepo.ClearThrottle(c.Request.Context(), id) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, "failed to clear throttle")) + return + } + if !cleared { + errx.JSON(c, errx.New(errx.NotFound, "mailbox has no sync state or is not throttled")) + return + } + resp := gin.H{"email_id": id, "cleared": true} + h.reloadSyncedMailbox(c, id, resp) + h.audit(c, auditActionSyncClearThrottle, models.AuditEntityEmailAccount, &id, map[string]string{ + "reloaded": boolString(resp["reloaded"].(bool)), + }) + c.JSON(http.StatusOK, resp) +} + +// AdminSyncRestartBackfill is POST /admin/sync/:id/restart-backfill. +func (h *Handler) AdminSyncRestartBackfill(c *gin.Context) { + if h.AdminSyncRepo == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "sync operations are not available on this instance")) + return + } + id, err := uuid.Parse(c.Param("id")) + if err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid mailbox ID")) + return + } + reset, err := h.AdminSyncRepo.ResetBackfill(c.Request.Context(), id) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, "failed to reset backfill")) + return + } + if !reset { + errx.JSON(c, errx.New(errx.NotFound, "mailbox has no sync state")) + return + } + resp := gin.H{"email_id": id, "reset": true} + h.reloadSyncedMailbox(c, id, resp) + h.audit(c, auditActionSyncRestartBackfill, models.AuditEntityEmailAccount, &id, map[string]string{ + "reloaded": boolString(resp["reloaded"].(bool)), + }) + c.JSON(http.StatusOK, resp) +} + +// reloadSyncedMailbox re-ships the mailbox so the worker's live copy follows +// the platform copy; a failure is reported in the response, not as an error. +func (h *Handler) reloadSyncedMailbox(c *gin.Context, id uuid.UUID, resp gin.H) { + resp["reloaded"] = false + if h.EmailService == nil { + resp["reload_error"] = "email service not configured" + return + } + if err := h.EmailService.LoadAccountOntoWorker(c.Request.Context(), id); err != nil { + resp["reload_error"] = err.Error() + return + } + resp["reloaded"] = true +} + +func boolString(b bool) string { + if b { + return "true" + } + return "false" +} diff --git a/internal/api/handler/admin_transfer.go b/internal/api/handler/admin_transfer.go new file mode 100644 index 00000000..fe6ca182 --- /dev/null +++ b/internal/api/handler/admin_transfer.go @@ -0,0 +1,260 @@ +package handler + +import ( + "fmt" + "io" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/api/middleware" + "github.com/warmbly/warmbly/internal/app/orgtransfer" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" +) + +// Operator-side workspace archives: the same export and import flow as +// org_transfer.go, keyed by the :id path parameter instead of the caller's own +// organization, and audited to the admin log rather than the org's. + +// adminTransferOrg resolves :id to an existing organization and confirms the +// transfer service is wired, answering the request itself otherwise. +func (h *Handler) adminTransferOrg(c *gin.Context) (uuid.UUID, bool) { + orgID, ok := parseUUIDParam(c, "id") + if !ok { + return uuid.Nil, false + } + org, err := h.OrgRepo.GetByID(c.Request.Context(), orgID) + if err != nil { + errx.JSON(c, errx.New(errx.Internal, err.Error())) + return uuid.Nil, false + } + if org == nil { + errx.JSON(c, errx.New(errx.NotFound, "organization not found")) + return uuid.Nil, false + } + if h.OrgTransferService == nil { + errx.JSON(c, errx.New(errx.NotImplemented, "Workspace transfer is not available on this instance.")) + return uuid.Nil, false + } + return orgID, true +} + +// ---------- export ---------- + +// AdminListOrgExports returns a workspace's recent archive builds. +// +// GET /admin/organizations/:id/exports +func (h *Handler) AdminListOrgExports(c *gin.Context) { + orgID, ok := h.adminTransferOrg(c) + if !ok { + return + } + jobs, xerr := h.OrgTransferService.ListExports(c.Request.Context(), orgID) + if xerr != nil { + errx.JSON(c, xerr) + return + } + c.JSON(http.StatusOK, gin.H{"data": jobs}) +} + +// AdminCreateOrgExport starts an archive build on the operator's behalf. +// +// POST /admin/organizations/:id/exports +func (h *Handler) AdminCreateOrgExport(c *gin.Context) { + orgID, ok := h.adminTransferOrg(c) + if !ok { + return + } + var req models.CreateOrgExportRequest + if err := c.ShouldBindJSON(&req); err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid request body")) + return + } + + job, xerr := h.OrgTransferService.RequestExport(c.Request.Context(), orgID, middleware.GetAdminUserID(c), &req) + if xerr != nil { + errx.JSON(c, xerr) + return + } + + h.audit(c, models.AuditActionExport, models.AuditEntityOrgArchive, &job.ID, map[string]string{ + "organization_id": orgID.String(), + "include_secrets": fmt.Sprintf("%t", job.IncludeSecrets), + "groups": fmt.Sprintf("%d", len(job.Groups)), + }) + c.JSON(http.StatusAccepted, job) +} + +// AdminGetOrgExport returns one archive build, for progress polling. +// +// GET /admin/organizations/:id/exports/:exportId +func (h *Handler) AdminGetOrgExport(c *gin.Context) { + orgID, ok := h.adminTransferOrg(c) + if !ok { + return + } + id, ok := parseUUIDParam(c, "exportId") + if !ok { + return + } + job, xerr := h.OrgTransferService.GetExport(c.Request.Context(), orgID, id) + if xerr != nil { + errx.JSON(c, xerr) + return + } + c.JSON(http.StatusOK, job) +} + +// AdminDownloadOrgExport streams a finished archive. +// +// GET /admin/organizations/:id/exports/:exportId/download +func (h *Handler) AdminDownloadOrgExport(c *gin.Context) { + orgID, ok := h.adminTransferOrg(c) + if !ok { + return + } + id, ok := parseUUIDParam(c, "exportId") + if !ok { + return + } + + body, job, xerr := h.OrgTransferService.OpenExport(c.Request.Context(), orgID, id) + if xerr != nil { + errx.JSON(c, xerr) + return + } + defer body.Close() + + name := "workspace-" + job.ID.String()[:8] + ".warmbly.zip" + if org, err := h.OrgRepo.GetByID(c.Request.Context(), orgID); err == nil && org != nil { + name = orgtransfer.ArchiveFilename(org.Name, job.ID) + } + + h.audit(c, models.AuditActionExport, models.AuditEntityOrgArchive, &job.ID, map[string]string{ + "organization_id": orgID.String(), + "downloaded": "true", + }) + + c.Header("Content-Type", "application/zip") + c.Header("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, name)) + c.Header("X-Content-Type-Options", "nosniff") + if job.ArchiveBytes != nil { + c.Header("Content-Length", fmt.Sprintf("%d", *job.ArchiveBytes)) + } + if job.ArchiveSHA256 != nil { + c.Header("X-Archive-SHA256", *job.ArchiveSHA256) + } + + if _, err := io.Copy(c.Writer, body); err != nil { + // The client hung up mid-download; the status line is already sent. + return + } +} + +// AdminDeleteOrgExport removes an archive and its stored object. +// +// DELETE /admin/organizations/:id/exports/:exportId +func (h *Handler) AdminDeleteOrgExport(c *gin.Context) { + orgID, ok := h.adminTransferOrg(c) + if !ok { + return + } + id, ok := parseUUIDParam(c, "exportId") + if !ok { + return + } + if xerr := h.OrgTransferService.DeleteExport(c.Request.Context(), orgID, id); xerr != nil { + errx.JSON(c, xerr) + return + } + h.audit(c, models.AuditActionDelete, models.AuditEntityOrgArchive, &id, map[string]string{ + "organization_id": orgID.String(), + }) + c.Status(http.StatusNoContent) +} + +// ---------- import ---------- + +// AdminListOrgImports returns a workspace's recent imports. +// +// GET /admin/organizations/:id/imports +func (h *Handler) AdminListOrgImports(c *gin.Context) { + orgID, ok := h.adminTransferOrg(c) + if !ok { + return + } + jobs, xerr := h.OrgTransferService.ListImports(c.Request.Context(), orgID) + if xerr != nil { + errx.JSON(c, xerr) + return + } + c.JSON(http.StatusOK, gin.H{"data": jobs}) +} + +// AdminPreflightOrgImport reports what an uploaded archive would do to the +// workspace, writing nothing. +// +// POST /admin/organizations/:id/imports/preflight +func (h *Handler) AdminPreflightOrgImport(c *gin.Context) { + orgID, ok := h.adminTransferOrg(c) + if !ok { + return + } + spooled, xerr := spoolArchiveUpload(c) + if xerr != nil { + errx.JSON(c, xerr) + return + } + defer spooled.Close() + + report, xerr := h.OrgTransferService.Preflight( + c.Request.Context(), orgID, spooled, c.Request.FormValue("passphrase")) + if xerr != nil { + errx.JSON(c, xerr) + return + } + c.JSON(http.StatusOK, report) +} + +// AdminCreateOrgImport applies an uploaded archive to the workspace. +// +// POST /admin/organizations/:id/imports +func (h *Handler) AdminCreateOrgImport(c *gin.Context) { + orgID, ok := h.adminTransferOrg(c) + if !ok { + return + } + + // Ownership of the spooled file passes to the service on success; every + // failure path before that closes it here. + spooled, xerr := spoolArchiveUpload(c) + if xerr != nil { + errx.JSON(c, xerr) + return + } + + var req models.CreateOrgImportRequest + if opts := c.Request.FormValue("options"); opts != "" { + if err := jsonUnmarshalString(opts, &req); err != nil { + _ = spooled.Close() + errx.JSON(c, errx.New(errx.BadRequest, "invalid 'options' JSON: "+err.Error())) + return + } + } + req.Passphrase = c.Request.FormValue("passphrase") + + job, xerr := h.OrgTransferService.RequestImport(c.Request.Context(), orgID, middleware.GetAdminUserID(c), spooled, &req) + if xerr != nil { + _ = spooled.Close() + errx.JSON(c, xerr) + return + } + + h.audit(c, models.AuditActionImport, models.AuditEntityOrgArchive, &job.ID, map[string]string{ + "organization_id": orgID.String(), + "conflict_strategy": string(job.ConflictStrategy), + }) + c.JSON(http.StatusAccepted, job) +} diff --git a/internal/api/handler/handler.go b/internal/api/handler/handler.go index 39c14277..0fab70f7 100644 --- a/internal/api/handler/handler.go +++ b/internal/api/handler/handler.go @@ -45,7 +45,6 @@ import ( "github.com/warmbly/warmbly/internal/app/poollink" "github.com/warmbly/warmbly/internal/app/ratelimit" "github.com/warmbly/warmbly/internal/app/referral" - "github.com/warmbly/warmbly/internal/app/releases" "github.com/warmbly/warmbly/internal/app/research" "github.com/warmbly/warmbly/internal/app/segment" "github.com/warmbly/warmbly/internal/app/sequence" @@ -163,7 +162,6 @@ type Handler struct { WorkerOrchestrator *worker_orchestrator.Orchestrator WorkerRepo repository.WorkerRepository CredentialsRepo repository.CredentialsRepository - ReleasesService *releases.Service // UpdatesService backs the admin panel's update indicator and button. UpdatesService *updates.Service @@ -320,6 +318,18 @@ type Handler struct { // Wired in cmd/backend/main.go where the concrete clients live. SystemChecker *sysstatus.Checker + // Admin operations pages: cross-workspace reads of mailbox sync, the send + // outcome loop, fleet placement and abuse signals, plus the scheduled job + // registry every background loop records to. Nil-safe: the endpoints + // answer 501 when the repository is not wired. + AdminSyncRepo repository.AdminSyncRepository + AdminSendsRepo repository.AdminSendsRepository + AdminFleetRepo repository.AdminFleetRepository + AdminInsightRepo repository.AdminInsightRepository + JobRuns repository.JobRunRepository + // WebhookRepo backs the operator's "reclaim stuck deliveries" action. + WebhookRepo repository.WebhookRepository + // Operator visibility (admin panel, Instance section). // // InstanceRuntime carries the facts only boot knows (the resolved CORS diff --git a/internal/api/middleware/admin.go b/internal/api/middleware/admin.go index e040f1b8..41390bf0 100644 --- a/internal/api/middleware/admin.go +++ b/internal/api/middleware/admin.go @@ -112,9 +112,9 @@ func IsAdmin(c *gin.Context) bool { return GetAdminPermissions(c) > 0 } -// IsSuperAdmin returns true if the current user has all admin permissions +// IsSuperAdmin returns true if the current user holds every live admin bit. func IsSuperAdmin(c *gin.Context) bool { - return GetAdminPermissions(c) == models.AllAdminPermissions + return GetAdminPermissions(c).IsSuperAdmin() } // getAdminPermissions fetches admin permissions for a user diff --git a/internal/api/routes.go b/internal/api/routes.go index d4aa56ec..8fdbb8de 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -1509,11 +1509,56 @@ func Run( adminRoutes.GET("/analytics/emails/hourly", middleware.RequireAdminPermission(models.AdminPermViewAnalytics), h.AdminGetHourlyEmailStats) adminRoutes.GET("/analytics/users/growth", middleware.RequireAdminPermission(models.AdminPermViewAnalytics), h.AdminGetUserGrowthStats) - // Removed for self-host: worker load + email-distribution analytics - // (premised on multi-worker IP spread, moot when the mail provider owns the - // egress IP), and the SaaS commercial surfaces — plans, discount/promo - // codes, and the enterprise-sales inquiry queue — which have no role in a - // single-org, billing-disabled deployment. + // Signups by channel and trial conversion, from organization_acquisition. + adminRoutes.GET("/analytics/acquisition", middleware.RequireAdminPermission(models.AdminPermViewAnalytics), h.AdminGetAcquisition) + + // Mailbox sync governor: the platform copy of every mailbox's sync + // state. Actions re-ship the mailbox to its worker so the worker's live + // copy follows. + adminRoutes.GET("/sync", middleware.RequireAdminPermission(models.AdminPermViewUsers), h.AdminSearchSync) + adminRoutes.POST("/sync/:id/clear-throttle", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminSyncClearThrottle) + adminRoutes.POST("/sync/:id/restart-backfill", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminSyncRestartBackfill) + + // Send outcome loop and task queues. + adminRoutes.GET("/sends/in-flight", middleware.RequireAdminPermission(models.AdminPermViewCampaigns), h.AdminInFlightSends) + adminRoutes.GET("/tasks/dead-letters", middleware.RequireAdminPermission(models.AdminPermViewCampaigns), h.AdminListDeadLetters) + adminRoutes.POST("/tasks/dead-letters/:id/replay", middleware.RequireAdminPermission(models.AdminPermStopCampaigns), h.AdminReplayDeadLetter) + adminRoutes.GET("/tasks/failures", middleware.RequireAdminPermission(models.AdminPermViewCampaigns), h.AdminRecentTaskFailures) + adminRoutes.GET("/webhooks/health", middleware.RequireAdminPermission(models.AdminPermViewOrganizations), h.AdminWebhookHealth) + adminRoutes.POST("/webhooks/reclaim", middleware.RequireAdminPermission(models.AdminPermManageOrganizations), h.AdminWebhookReclaim) + + // Scheduled jobs: every background loop on the instance, with "run now". + adminRoutes.GET("/jobs", middleware.RequireAdminPermission(models.AdminPermViewAnalytics), h.AdminListJobs) + adminRoutes.POST("/jobs/:name/run", middleware.RequireAdminPermission(models.AdminPermManageSettings), h.AdminRunJob) + + // Fleet placement: capacity per worker, the control loops' decision + // log, and dedicated worker bindings. + adminRoutes.GET("/fleet/capacity", middleware.RequireAdminPermission(models.AdminPermViewWorkers), h.AdminFleetCapacity) + adminRoutes.GET("/fleet/decisions", middleware.RequireAdminPermission(models.AdminPermViewWorkers), h.AdminFleetDecisions) + adminRoutes.GET("/fleet/dedicated", middleware.RequireAdminPermission(models.AdminPermViewWorkers), h.AdminFleetDedicated) + adminRoutes.POST("/fleet/dedicated/:orgId/release", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminFleetReleaseDedicated) + adminRoutes.POST("/workers/:id/convert-dedicated", middleware.RequireAdminPermission(models.AdminPermManageWorkers), h.AdminConvertWorkerToDedicated) + + // Workspace transfers: the same archive service the owner uses from + // Settings > Data, driven by the operator for any workspace. + adminRoutes.GET("/transfers", middleware.RequireAdminPermission(models.AdminPermViewOrganizations), h.AdminListTransfers) + adminRoutes.GET("/organizations/:id/exports", middleware.RequireAdminPermission(models.AdminPermViewOrganizations), h.AdminListOrgExports) + adminRoutes.POST("/organizations/:id/exports", middleware.RequireAdminPermission(models.AdminPermManageOrganizations), h.AdminCreateOrgExport) + adminRoutes.GET("/organizations/:id/exports/:exportId", middleware.RequireAdminPermission(models.AdminPermViewOrganizations), h.AdminGetOrgExport) + adminRoutes.GET("/organizations/:id/exports/:exportId/download", middleware.RequireAdminPermission(models.AdminPermManageOrganizations), h.AdminDownloadOrgExport) + adminRoutes.DELETE("/organizations/:id/exports/:exportId", middleware.RequireAdminPermission(models.AdminPermManageOrganizations), h.AdminDeleteOrgExport) + adminRoutes.GET("/organizations/:id/imports", middleware.RequireAdminPermission(models.AdminPermViewOrganizations), h.AdminListOrgImports) + adminRoutes.POST("/organizations/:id/imports/preflight", middleware.RequireAdminPermission(models.AdminPermManageOrganizations), h.AdminPreflightOrgImport) + adminRoutes.POST("/organizations/:id/imports", middleware.RequireAdminPermission(models.AdminPermManageOrganizations), h.AdminCreateOrgImport) + + // Per-workspace developer surface: keys and webhook endpoints. + adminRoutes.GET("/organizations/:id/api-keys", middleware.RequireAdminPermission(models.AdminPermViewOrganizations), h.AdminListOrgAPIKeys) + adminRoutes.DELETE("/organizations/:id/api-keys/:keyId", middleware.RequireAdminPermission(models.AdminPermManageOrganizations), h.AdminRevokeOrgAPIKey) + adminRoutes.GET("/organizations/:id/webhooks", middleware.RequireAdminPermission(models.AdminPermViewOrganizations), h.AdminListOrgWebhooks) + + // Warmup abuse signals and the block/unblock history. + adminRoutes.GET("/warmup/abuse", middleware.RequireAdminPermission(models.AdminPermViewWarmupPool), h.AdminWarmupAbuse) + adminRoutes.GET("/warmup/actions", middleware.RequireAdminPermission(models.AdminPermViewWarmupPool), h.AdminWarmupActions) // Admin Management adminRoutes.GET("/admins", middleware.RequireAdminPermission(models.AdminPermGrantAdminAccess), h.AdminListAdmins) diff --git a/internal/app/admin/service.go b/internal/app/admin/service.go index f4a98ea6..4871717c 100644 --- a/internal/app/admin/service.go +++ b/internal/app/admin/service.go @@ -58,23 +58,8 @@ type AdminService interface { GetAnalyticsTrends(ctx context.Context) (*models.AnalyticsTrends, *errx.Error) GetDailyEmailStats(ctx context.Context, startDate, endDate time.Time) ([]models.DailyEmailStats, *errx.Error) GetHourlyEmailStats(ctx context.Context, date time.Time) ([]models.HourlyEmailStats, *errx.Error) - GetWorkerLoadStats(ctx context.Context) ([]models.WorkerLoadStats, *errx.Error) - GetEmailDistribution(ctx context.Context) ([]models.EmailDistribution, *errx.Error) GetUserGrowthStats(ctx context.Context, startDate, endDate time.Time) ([]models.UserGrowthStats, *errx.Error) - // Plans - ListPlans(ctx context.Context, includePrivate bool) ([]models.Plan, *errx.Error) - SearchPlansForAdmin(ctx context.Context, search *models.AdminPlanSearch) (*models.AdminPlansResult, *errx.Error) - CreatePlan(ctx context.Context, adminID uuid.UUID, req *models.CreatePlanRequest, ipAddress, userAgent string) (*models.Plan, *errx.Error) - GetPlan(ctx context.Context, planID uuid.UUID) (*models.Plan, *errx.Error) - UpdatePlan(ctx context.Context, adminID, planID uuid.UUID, req *models.UpdatePlanRequest, ipAddress, userAgent string) (*models.Plan, *errx.Error) - DeletePlan(ctx context.Context, adminID, planID uuid.UUID, ipAddress, userAgent string) *errx.Error - - // Enterprise Inquiries - ListEnterpriseInquiries(ctx context.Context, search *models.AdminEnterpriseInquirySearch) (*models.AdminEnterpriseInquiriesResult, *errx.Error) - GetEnterpriseInquiry(ctx context.Context, id uuid.UUID) (*models.AdminEnterpriseInquiry, *errx.Error) - UpdateEnterpriseInquiry(ctx context.Context, adminID, inquiryID uuid.UUID, update *models.UpdateEnterpriseInquiryRequest, ipAddress, userAgent string) *errx.Error - // Admin Management ListAdmins(ctx context.Context, cursor *uuid.UUID, limit int) (*models.AdminsResult, *errx.Error) GrantAdminPermissions(ctx context.Context, adminID, targetUserID uuid.UUID, permissions models.AdminPermission, ipAddress, userAgent string) *errx.Error @@ -559,24 +544,6 @@ func (s *adminService) GetHourlyEmailStats(ctx context.Context, date time.Time) return stats, nil } -func (s *adminService) GetWorkerLoadStats(ctx context.Context) ([]models.WorkerLoadStats, *errx.Error) { - stats, err := s.repo.GetWorkerLoadStats(ctx) - if err != nil { - errs.CaptureException(err) - return nil, errx.New(errx.Internal, "failed to get worker load stats") - } - return stats, nil -} - -func (s *adminService) GetEmailDistribution(ctx context.Context) ([]models.EmailDistribution, *errx.Error) { - dist, err := s.repo.GetEmailDistribution(ctx) - if err != nil { - errs.CaptureException(err) - return nil, errx.New(errx.Internal, "failed to get email distribution") - } - return dist, nil -} - func (s *adminService) GetUserGrowthStats(ctx context.Context, startDate, endDate time.Time) ([]models.UserGrowthStats, *errx.Error) { stats, err := s.repo.GetUserGrowthStats(ctx, startDate, endDate) if err != nil { @@ -586,213 +553,6 @@ func (s *adminService) GetUserGrowthStats(ctx context.Context, startDate, endDat return stats, nil } -// Plans - -func (s *adminService) ListPlans(ctx context.Context, includePrivate bool) ([]models.Plan, *errx.Error) { - plans, err := s.repo.ListPlans(ctx, includePrivate) - if err != nil { - errs.CaptureException(err) - return nil, errx.New(errx.Internal, "failed to list plans") - } - return plans, nil -} - -func (s *adminService) SearchPlansForAdmin(ctx context.Context, search *models.AdminPlanSearch) (*models.AdminPlansResult, *errx.Error) { - result, err := s.repo.SearchPlansForAdmin(ctx, search) - if err != nil { - errs.CaptureException(err) - return nil, errx.New(errx.Internal, "failed to search plans") - } - return result, nil -} - -// resolveDuration maps the API's billing period onto a durations row. An -// unknown period is the caller's mistake, not an internal error. -func (s *adminService) resolveDuration(ctx context.Context, d models.Duration) (*uuid.UUID, *errx.Error) { - if d == "" { - return nil, errx.New(errx.BadRequest, "duration is required") - } - id, err := s.repo.DurationIDByTitle(ctx, string(d)) - if err != nil { - errs.CaptureException(err) - return nil, errx.New(errx.Internal, "failed to resolve plan duration") - } - if id == nil { - return nil, errx.New(errx.BadRequest, "unknown plan duration") - } - return id, nil -} - -func (s *adminService) CreatePlan(ctx context.Context, adminID uuid.UUID, req *models.CreatePlanRequest, ipAddress, userAgent string) (*models.Plan, *errx.Error) { - plan := &models.Plan{ - ID: uuid.New(), - Name: &req.Name, - MaxContacts: req.MaxContacts, - DailyEmails: req.DailyEmails, - AIGeneration: req.AIGeneration, - AccountLimit: req.AccountLimit, - Price: req.Price, - DiscountedPrice: req.DiscountedPrice, - Duration: req.Duration, - Public: req.Public, - DedicatedWorkers: req.DedicatedWorkers, - DailyCampaignLimit: req.DailyCampaignLimit, - MaxCampaigns: req.MaxCampaigns, - MaxActiveCampaigns: req.MaxActiveCampaigns, - MaxTeamMembers: req.MaxTeamMembers, - MaxEmailAccounts: req.MaxEmailAccounts, - } - - durationID, xerr := s.resolveDuration(ctx, plan.Duration) - if xerr != nil { - return nil, xerr - } - - if err := s.repo.CreatePlan(ctx, plan, *durationID); err != nil { - errs.CaptureException(err) - return nil, errx.New(errx.Internal, "failed to create plan") - } - - s.logAction(ctx, adminID, "create_plan", "plan", plan.ID, map[string]any{"name": req.Name}, ipAddress, userAgent) - return plan, nil -} - -func (s *adminService) GetPlan(ctx context.Context, planID uuid.UUID) (*models.Plan, *errx.Error) { - plan, err := s.repo.GetPlan(ctx, planID) - if err != nil { - errs.CaptureException(err) - return nil, errx.New(errx.Internal, "failed to get plan") - } - if plan == nil { - return nil, errx.ErrNotFound - } - return plan, nil -} - -func (s *adminService) UpdatePlan(ctx context.Context, adminID, planID uuid.UUID, req *models.UpdatePlanRequest, ipAddress, userAgent string) (*models.Plan, *errx.Error) { - plan, err := s.repo.GetPlan(ctx, planID) - if err != nil { - errs.CaptureException(err) - return nil, errx.New(errx.Internal, "failed to get plan") - } - if plan == nil { - return nil, errx.ErrNotFound - } - - // Apply updates - if req.Name != nil { - plan.Name = req.Name - } - if req.MaxContacts != nil { - plan.MaxContacts = *req.MaxContacts - } - if req.DailyEmails != nil { - plan.DailyEmails = *req.DailyEmails - } - if req.AIGeneration != nil { - plan.AIGeneration = *req.AIGeneration - } - if req.AccountLimit != nil { - plan.AccountLimit = *req.AccountLimit - } - if req.Price != nil { - plan.Price = *req.Price - } - if req.DiscountedPrice != nil { - plan.DiscountedPrice = *req.DiscountedPrice - } - if req.Duration != nil { - plan.Duration = *req.Duration - } - if req.Public != nil { - plan.Public = *req.Public - } - if req.DedicatedWorkers != nil { - plan.DedicatedWorkers = *req.DedicatedWorkers - } - if req.DailyCampaignLimit != nil { - plan.DailyCampaignLimit = req.DailyCampaignLimit - } - if req.MaxCampaigns != nil { - plan.MaxCampaigns = req.MaxCampaigns - } - if req.MaxActiveCampaigns != nil { - plan.MaxActiveCampaigns = req.MaxActiveCampaigns - } - if req.MaxTeamMembers != nil { - plan.MaxTeamMembers = req.MaxTeamMembers - } - if req.MaxEmailAccounts != nil { - plan.MaxEmailAccounts = req.MaxEmailAccounts - } - - durationID, xerr := s.resolveDuration(ctx, plan.Duration) - if xerr != nil { - return nil, xerr - } - - if err := s.repo.UpdatePlan(ctx, plan, *durationID); err != nil { - errs.CaptureException(err) - return nil, errx.New(errx.Internal, "failed to update plan") - } - - s.logAction(ctx, adminID, "update_plan", "plan", planID, nil, ipAddress, userAgent) - return plan, nil -} - -func (s *adminService) DeletePlan(ctx context.Context, adminID, planID uuid.UUID, ipAddress, userAgent string) *errx.Error { - // Check if plan is in use - inUse, err := s.repo.IsPlanInUse(ctx, planID) - if err != nil { - errs.CaptureException(err) - return errx.New(errx.Internal, "failed to check plan usage") - } - if inUse { - return errx.New(errx.BadRequest, "cannot delete plan that is in use") - } - - if err := s.repo.DeletePlan(ctx, planID); err != nil { - errs.CaptureException(err) - return errx.New(errx.Internal, "failed to delete plan") - } - - s.logAction(ctx, adminID, "delete_plan", "plan", planID, nil, ipAddress, userAgent) - return nil -} - -// Enterprise Inquiries - -func (s *adminService) ListEnterpriseInquiries(ctx context.Context, search *models.AdminEnterpriseInquirySearch) (*models.AdminEnterpriseInquiriesResult, *errx.Error) { - result, err := s.repo.ListEnterpriseInquiries(ctx, search) - if err != nil { - errs.CaptureException(err) - return nil, errx.New(errx.Internal, "failed to list enterprise inquiries") - } - return result, nil -} - -func (s *adminService) GetEnterpriseInquiry(ctx context.Context, id uuid.UUID) (*models.AdminEnterpriseInquiry, *errx.Error) { - inquiry, err := s.repo.GetEnterpriseInquiry(ctx, id) - if err != nil { - errs.CaptureException(err) - return nil, errx.New(errx.Internal, "failed to get enterprise inquiry") - } - if inquiry == nil { - return nil, errx.ErrNotFound - } - return inquiry, nil -} - -func (s *adminService) UpdateEnterpriseInquiry(ctx context.Context, adminID, inquiryID uuid.UUID, update *models.UpdateEnterpriseInquiryRequest, ipAddress, userAgent string) *errx.Error { - if err := s.repo.UpdateEnterpriseInquiry(ctx, inquiryID, update); err != nil { - errs.CaptureException(err) - return errx.New(errx.Internal, "failed to update enterprise inquiry") - } - - s.logAction(ctx, adminID, "update_enterprise_inquiry", "enterprise_inquiry", inquiryID, map[string]any{"update": update}, ipAddress, userAgent) - return nil -} - // Admin Management func (s *adminService) ListAdmins(ctx context.Context, cursor *uuid.UUID, limit int) (*models.AdminsResult, *errx.Error) { diff --git a/internal/app/advisor/runner.go b/internal/app/advisor/runner.go index 6f060880..ef77cf61 100644 --- a/internal/app/advisor/runner.go +++ b/internal/app/advisor/runner.go @@ -8,6 +8,7 @@ import ( "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/repository" ) @@ -52,18 +53,10 @@ func (r *Runner) Run(ctx context.Context) { if r.BatchSize <= 0 { r.BatchSize = defaultBatchSize } - - tick := time.NewTicker(r.Interval) - defer tick.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-tick.C: - r.sweep(ctx) - } - } + jobrun.Loop(ctx, "advisor_sweep", r.Interval, false, func(ctx context.Context) error { + r.sweep(ctx) + return nil + }) } func (r *Runner) sweep(ctx context.Context) { diff --git a/internal/app/consumer/auth_check_sweep.go b/internal/app/consumer/auth_check_sweep.go index 53314856..fb466ce3 100644 --- a/internal/app/consumer/auth_check_sweep.go +++ b/internal/app/consumer/auth_check_sweep.go @@ -9,6 +9,7 @@ import ( "github.com/google/uuid" "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/pkg/dnsauth" ) @@ -26,20 +27,12 @@ func (s *JobsService) StartAuthCheckSweep(ctx context.Context, interval, staleAf if s.EmailRepository == nil { return } - - ticker := time.NewTicker(interval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - sweepCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) - s.runAuthCheckSweep(sweepCtx, staleAfter) - cancel() - } - } + jobrun.Loop(ctx, "auth_check_sweep", interval, false, func(ctx context.Context) error { + sweepCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + s.runAuthCheckSweep(sweepCtx, staleAfter) + return nil + }) } func (s *JobsService) runAuthCheckSweep(ctx context.Context, staleAfter time.Duration) { diff --git a/internal/app/consumer/dead_worker.go b/internal/app/consumer/dead_worker.go index 4af152c0..92fd870c 100644 --- a/internal/app/consumer/dead_worker.go +++ b/internal/app/consumer/dead_worker.go @@ -8,6 +8,7 @@ import ( "github.com/google/uuid" "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/models" ) @@ -91,20 +92,12 @@ func (s *JobsService) StartDeadWorkerDetection(ctx context.Context, interval tim if s.WorkerRepo == nil { return } - - ticker := time.NewTicker(interval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - detectCtx, cancel := context.WithTimeout(ctx, 30*time.Second) - s.detectDeadWorkers(detectCtx) - cancel() - } - } + jobrun.Loop(ctx, "dead_worker_detection", interval, false, func(ctx context.Context) error { + detectCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + s.detectDeadWorkers(detectCtx) + return nil + }) } func (s *JobsService) detectDeadWorkers(ctx context.Context) { diff --git a/internal/app/consumer/dlq_retry.go b/internal/app/consumer/dlq_retry.go index cb6d08be..de48ba13 100644 --- a/internal/app/consumer/dlq_retry.go +++ b/internal/app/consumer/dlq_retry.go @@ -5,6 +5,7 @@ import ( "time" "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/jobrun" ) // StartDLQRetryLoop polls for retryable dead-lettered tasks and replays them. @@ -13,25 +14,16 @@ func (s *JobsService) StartDLQRetryLoop(ctx context.Context, interval time.Durat if s.AdvancedService == nil { return } - - ticker := time.NewTicker(interval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - retryCtx, cancel := context.WithTimeout(ctx, 30*time.Second) - retried, xerr := s.AdvancedService.ProcessRetryableDeadLetters(retryCtx) - cancel() - if xerr != nil { - log.Warn().Str("error", xerr.Error()).Msg("DLQ retry processing failed") - continue - } - if retried > 0 { - log.Info().Int("retried", retried).Msg("DLQ auto-retry processed dead letters") - } + jobrun.Loop(ctx, "dlq_retry", interval, false, func(ctx context.Context) error { + retryCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + retried, xerr := s.AdvancedService.ProcessRetryableDeadLetters(retryCtx) + if xerr != nil { + return xerr } - } + if retried > 0 { + log.Info().Int("retried", retried).Msg("DLQ auto-retry processed dead letters") + } + return nil + }) } diff --git a/internal/app/consumer/lifecycle_rebalancer.go b/internal/app/consumer/lifecycle_rebalancer.go index 9298913a..dd91d08d 100644 --- a/internal/app/consumer/lifecycle_rebalancer.go +++ b/internal/app/consumer/lifecycle_rebalancer.go @@ -8,6 +8,7 @@ import ( "github.com/rs/zerolog/log" "github.com/warmbly/warmbly/internal/app/lifecycle" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/models" ) @@ -22,16 +23,10 @@ func (s *JobsService) StartLifecycleRebalancer(ctx context.Context, interval tim if s.LifecycleRepo == nil { return } - ticker := time.NewTicker(interval) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - s.rebalanceLifecycles(ctx) - } - } + jobrun.Loop(ctx, "lifecycle_rebalancer", interval, false, func(ctx context.Context) error { + s.rebalanceLifecycles(ctx) + return nil + }) } func (s *JobsService) rebalanceLifecycles(ctx context.Context) { diff --git a/internal/app/consumer/risk_rebalancer.go b/internal/app/consumer/risk_rebalancer.go index ced62272..6bba959d 100644 --- a/internal/app/consumer/risk_rebalancer.go +++ b/internal/app/consumer/risk_rebalancer.go @@ -6,6 +6,7 @@ import ( "github.com/google/uuid" "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/models" ) @@ -30,25 +31,19 @@ func (s *JobsService) StartRiskRebalancer(ctx context.Context, interval time.Dur return } - // Initial run right after boot so a fresh deploy converges quickly. - go func() { - boot, cancel := context.WithTimeout(ctx, 5*time.Minute) - defer cancel() - s.rebalanceRisk(boot) - }() - - ticker := time.NewTicker(interval) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - runCtx, cancel := context.WithTimeout(ctx, 10*time.Minute) - s.rebalanceRisk(runCtx) - cancel() + // The boot pass (so a fresh deploy converges quickly) keeps its shorter budget. + first := true + jobrun.Loop(ctx, "risk_rebalancer", interval, true, func(ctx context.Context) error { + timeout := 10 * time.Minute + if first { + first = false + timeout = 5 * time.Minute } - } + runCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + s.rebalanceRisk(runCtx) + return nil + }) } func (s *JobsService) rebalanceRisk(ctx context.Context) { diff --git a/internal/app/consumer/stuck_send_reclaimer.go b/internal/app/consumer/stuck_send_reclaimer.go index 26eb5105..d2c66cfb 100644 --- a/internal/app/consumer/stuck_send_reclaimer.go +++ b/internal/app/consumer/stuck_send_reclaimer.go @@ -7,6 +7,7 @@ import ( "github.com/rs/zerolog/log" "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/repository" ) @@ -31,18 +32,12 @@ func (s *JobsService) StartStuckSendReclaimer(ctx context.Context, interval time if s.TaskRepo == nil || s.CampaignProgressRepo == nil { return } - ticker := time.NewTicker(interval) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - sweepCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) - s.reclaimStuckSends(sweepCtx) - cancel() - } - } + jobrun.Loop(ctx, "stuck_send_reclaimer", interval, false, func(ctx context.Context) error { + sweepCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + s.reclaimStuckSends(sweepCtx) + return nil + }) } func (s *JobsService) reclaimStuckSends(ctx context.Context) { diff --git a/internal/app/consumer/warmup_engagement_poller.go b/internal/app/consumer/warmup_engagement_poller.go index 503e662e..d787ffab 100644 --- a/internal/app/consumer/warmup_engagement_poller.go +++ b/internal/app/consumer/warmup_engagement_poller.go @@ -3,10 +3,12 @@ package jobs import ( "context" "encoding/json" + "fmt" "time" "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/models" ) @@ -18,28 +20,16 @@ func (s *JobsService) StartWarmupEngagementPoller(ctx context.Context, interval if s.WarmupEngagementRepo == nil || s.Publisher == nil { return } - - ticker := time.NewTicker(interval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - s.drainDueEngagements(ctx) - } - } + jobrun.Loop(ctx, "warmup_engagement_poller", interval, false, s.drainDueEngagements) } -func (s *JobsService) drainDueEngagements(ctx context.Context) { +func (s *JobsService) drainDueEngagements(ctx context.Context) error { cctx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() due, err := s.WarmupEngagementRepo.ClaimDuePendingEngagements(cctx, 200) if err != nil { - log.Warn().Err(err).Msg("warmup engagement poller: claim failed") - return + return fmt.Errorf("warmup engagement poller: claim: %w", err) } for _, p := range due { @@ -63,4 +53,5 @@ func (s *JobsService) drainDueEngagements(ctx context.Context) { action.DelaySeconds = 0 // dwell already elapsed; run immediately s.Publisher.PublishWarmupAction(cctx, *account.WorkerID, &action) } + return nil } diff --git a/internal/app/consumer/warmup_health_sweep.go b/internal/app/consumer/warmup_health_sweep.go index 90999823..7eea6fae 100644 --- a/internal/app/consumer/warmup_health_sweep.go +++ b/internal/app/consumer/warmup_health_sweep.go @@ -5,6 +5,7 @@ import ( "time" "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/jobrun" ) // StartWarmupHealthSweep runs a periodic health evaluation across all warmup pool participants. @@ -13,29 +14,19 @@ func (s *JobsService) StartWarmupHealthSweep(ctx context.Context, interval time. if s.WarmupService == nil { return } - - ticker := time.NewTicker(interval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - sweepCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) - evaluated, changes, xerr := s.WarmupService.EvaluateAllParticipants(sweepCtx) - if xerr != nil { - cancel() - log.Warn().Str("error", xerr.Error()).Msg("warmup health sweep failed") - continue - } - if evaluated > 0 { - log.Info().Int("evaluated", evaluated).Int("state_changes", changes).Msg("warmup health sweep completed") - } - if changes > 0 { - s.rebalanceRisk(sweepCtx) - } - cancel() + jobrun.Loop(ctx, "warmup_health_sweep", interval, false, func(ctx context.Context) error { + sweepCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + evaluated, changes, xerr := s.WarmupService.EvaluateAllParticipants(sweepCtx) + if xerr != nil { + return xerr } - } + if evaluated > 0 { + log.Info().Int("evaluated", evaluated).Int("state_changes", changes).Msg("warmup health sweep completed") + } + if changes > 0 { + s.rebalanceRisk(sweepCtx) + } + return nil + }) } diff --git a/internal/app/consumer/worker_heartbeat.go b/internal/app/consumer/worker_heartbeat.go index 196311d6..d6a72e3d 100644 --- a/internal/app/consumer/worker_heartbeat.go +++ b/internal/app/consumer/worker_heartbeat.go @@ -3,10 +3,12 @@ package jobs import ( "context" "errors" + "fmt" "time" "github.com/redis/go-redis/v9" "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/jobrun" ) // StartWorkerHeartbeatSync mirrors workers' Redis heartbeat timestamps into @@ -19,27 +21,17 @@ func (s *JobsService) StartWorkerHeartbeatSync(ctx context.Context, interval tim if s.WorkerRepo == nil || s.Cache == nil { return } - - ticker := time.NewTicker(interval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - runCtx, cancel := context.WithTimeout(ctx, 15*time.Second) - s.syncHeartbeats(runCtx) - cancel() - } - } + jobrun.Loop(ctx, "worker_heartbeat_sync", interval, false, func(ctx context.Context) error { + runCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + return s.syncHeartbeats(runCtx) + }) } -func (s *JobsService) syncHeartbeats(ctx context.Context) { +func (s *JobsService) syncHeartbeats(ctx context.Context) error { workers, err := s.WorkerRepo.GetAllActiveWorkers(ctx) if err != nil { - log.Warn().Err(err).Msg("heartbeat sync: list workers failed") - return + return fmt.Errorf("heartbeat sync: list workers: %w", err) } for _, w := range workers { @@ -59,4 +51,5 @@ func (s *JobsService) syncHeartbeats(ctx context.Context) { log.Warn().Err(err).Str("worker_id", w.ID.String()).Msg("heartbeat sync: update failed") } } + return nil } diff --git a/internal/app/correlate/correlate.go b/internal/app/correlate/correlate.go index e8046391..e5b43967 100644 --- a/internal/app/correlate/correlate.go +++ b/internal/app/correlate/correlate.go @@ -13,6 +13,7 @@ import ( "github.com/rs/zerolog/log" "github.com/warmbly/warmbly/internal/app/orgrisk" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/repository" ) @@ -74,16 +75,10 @@ func (s *Service) Start(ctx context.Context, interval time.Duration) { if s == nil || s.repo == nil || s.orgRisk == nil { return } - ticker := time.NewTicker(interval) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - s.Run(ctx) - } - } + jobrun.Loop(ctx, "correlation_sweep", interval, false, func(ctx context.Context) error { + s.Run(ctx) + return nil + }) } // finding is one detector's verdict on one organization. diff --git a/internal/app/fleet/quarantine.go b/internal/app/fleet/quarantine.go index 221ce2d4..9a2562a9 100644 --- a/internal/app/fleet/quarantine.go +++ b/internal/app/fleet/quarantine.go @@ -6,6 +6,7 @@ import ( "time" "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/repository" ) @@ -43,18 +44,7 @@ func (q *QuarantineEvaluator) defaults() { func (q *QuarantineEvaluator) Run(ctx context.Context) { q.defaults() - tick := time.NewTicker(q.Interval) - defer tick.Stop() - for { - select { - case <-ctx.Done(): - return - case <-tick.C: - if err := q.tick(ctx); err != nil { - log.Warn().Err(err).Msg("quarantine tick failed") - } - } - } + jobrun.Loop(ctx, "fleet_quarantine", q.Interval, false, q.tick) } func (q *QuarantineEvaluator) tick(ctx context.Context) error { diff --git a/internal/app/fleet/rebalance.go b/internal/app/fleet/rebalance.go index 10b63bb4..678101ce 100644 --- a/internal/app/fleet/rebalance.go +++ b/internal/app/fleet/rebalance.go @@ -15,6 +15,7 @@ import ( "github.com/google/uuid" "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/repository" ) @@ -60,18 +61,7 @@ func (r *Rebalancer) defaults() { // Run blocks until ctx is cancelled, ticking every Interval. func (r *Rebalancer) Run(ctx context.Context) { r.defaults() - tick := time.NewTicker(r.Interval) - defer tick.Stop() - for { - select { - case <-ctx.Done(): - return - case <-tick.C: - if err := r.tick(ctx); err != nil { - log.Warn().Err(err).Msg("fleet rebalance tick failed") - } - } - } + jobrun.Loop(ctx, "fleet_rebalance", r.Interval, false, r.tick) } func (r *Rebalancer) tick(ctx context.Context) error { diff --git a/internal/app/fleet/scale.go b/internal/app/fleet/scale.go index 865099d1..50a087ba 100644 --- a/internal/app/fleet/scale.go +++ b/internal/app/fleet/scale.go @@ -7,6 +7,7 @@ import ( "time" "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/repository" ) @@ -46,21 +47,8 @@ func (s *Scaler) defaults() { func (s *Scaler) Run(ctx context.Context) { s.defaults() - tick := time.NewTicker(s.Interval) - defer tick.Stop() - // Run once immediately on boot so an admin doesn't wait an hour for the - // first signal. - _ = s.tick(ctx) - for { - select { - case <-ctx.Done(): - return - case <-tick.C: - if err := s.tick(ctx); err != nil { - log.Warn().Err(err).Msg("fleet scale tick failed") - } - } - } + // Runs once on boot so an admin doesn't wait an hour for the first signal. + jobrun.Loop(ctx, "fleet_scale", s.Interval, true, s.tick) } func (s *Scaler) tick(ctx context.Context) error { diff --git a/internal/app/orgrisk/service.go b/internal/app/orgrisk/service.go index b8d04020..f4d50225 100644 --- a/internal/app/orgrisk/service.go +++ b/internal/app/orgrisk/service.go @@ -14,6 +14,7 @@ import ( "github.com/rs/zerolog/log" "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/repository" ) @@ -341,16 +342,10 @@ func StartExpirySweep(ctx context.Context, svc Service, interval time.Duration) if svc == nil || interval <= 0 { return } - ticker := time.NewTicker(interval) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - svc.SweepExpired(ctx) - } - } + jobrun.Loop(ctx, "org_risk_expiry_sweep", interval, false, func(ctx context.Context) error { + svc.SweepExpired(ctx) + return nil + }) } // apply mutates the signal set and re-derives score, band and reason from it, diff --git a/internal/infrastructure/db/migrations/000135_scheduled_job_runs.down.sql b/internal/infrastructure/db/migrations/000135_scheduled_job_runs.down.sql new file mode 100644 index 00000000..d868a5ec --- /dev/null +++ b/internal/infrastructure/db/migrations/000135_scheduled_job_runs.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS scheduled_job_runs; diff --git a/internal/infrastructure/db/migrations/000135_scheduled_job_runs.up.sql b/internal/infrastructure/db/migrations/000135_scheduled_job_runs.up.sql new file mode 100644 index 00000000..1c62e405 --- /dev/null +++ b/internal/infrastructure/db/migrations/000135_scheduled_job_runs.up.sql @@ -0,0 +1,25 @@ +-- scheduled_job_runs: one row per background loop on the instance, written by +-- the process that runs it (backend or consumer) and read by the admin panel's +-- Jobs page. Instance-level, not workspace data: it never travels in an +-- organization archive. +CREATE TABLE IF NOT EXISTS scheduled_job_runs ( + name text PRIMARY KEY, + service text NOT NULL DEFAULT '', + interval_seconds integer NOT NULL DEFAULT 0, + last_started_at timestamptz, + last_finished_at timestamptz, + last_duration_ms bigint NOT NULL DEFAULT 0, + last_status text NOT NULL DEFAULT 'idle' + CHECK (last_status IN ('idle', 'running', 'ok', 'error')), + last_error text NOT NULL DEFAULT '', + run_count bigint NOT NULL DEFAULT 0, + error_count bigint NOT NULL DEFAULT 0, + -- Set by the panel's "run now"; cleared by the owning loop when it picks + -- the request up on its next poll. + run_requested_at timestamptz, + next_run_at timestamptz, + updated_at timestamptz NOT NULL DEFAULT now() +); + +COMMENT ON TABLE scheduled_job_runs IS + 'Last run, next run and last error of every background loop, for the admin Jobs page. Instance-level.'; diff --git a/internal/jobrun/jobrun.go b/internal/jobrun/jobrun.go new file mode 100644 index 00000000..279b169d --- /dev/null +++ b/internal/jobrun/jobrun.go @@ -0,0 +1,152 @@ +// Package jobrun runs a background loop and records what it did, so the admin +// panel can list every scheduled job on the instance with its last run, its +// next run and its last error, and ask one to run now. +// +// Every service that hosts loops (backend, consumer) calls Configure once at +// boot with the store and its own name, then wraps each loop in Loop. Without +// a store the loops still run; nothing is recorded and "run now" has nowhere +// to land. +package jobrun + +import ( + "context" + "sync" + "time" + + "github.com/rs/zerolog/log" + + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/observability/errs" +) + +// Store persists job runs. Implemented by repository.JobRunRepository. +type Store interface { + // Register creates or refreshes the job's row at boot. + Register(ctx context.Context, name, service string, interval time.Duration, nextRunAt time.Time) error + MarkStarted(ctx context.Context, name string, at time.Time) error + MarkFinished(ctx context.Context, name string, startedAt, finishedAt time.Time, runErr error, nextRunAt time.Time) error + // RequestRun asks the owning loop to run at its next poll. False when no + // job by that name has ever registered. + RequestRun(ctx context.Context, name string) (bool, error) + // TakeRunRequest clears a pending request and reports whether there was one. + TakeRunRequest(ctx context.Context, name string) (bool, error) + List(ctx context.Context) ([]models.ScheduledJobRun, error) +} + +// requestPoll is how often a loop checks for a "run now" request. +const requestPoll = 15 * time.Second + +var ( + mu sync.RWMutex + store Store + service string +) + +// Configure sets the store every loop in this process records to and the +// name of this service. Safe to call once; later calls replace both. +func Configure(s Store, serviceName string) { + mu.Lock() + defer mu.Unlock() + store = s + service = serviceName +} + +func current() (Store, string) { + mu.RLock() + defer mu.RUnlock() + return store, service +} + +// Loop runs fn every interval until ctx ends, records each run, and also runs +// fn when the panel requests it. runOnBoot runs fn once before the first tick, +// which is what most retention and reconcile loops want. +func Loop(ctx context.Context, name string, interval time.Duration, runOnBoot bool, fn func(ctx context.Context) error) { + if interval <= 0 { + interval = time.Minute + } + st, svc := current() + if st != nil { + next := time.Now().Add(interval) + if runOnBoot { + next = time.Now() + } + if err := st.Register(ctx, name, svc, interval, next); err != nil { + log.Warn().Err(err).Str("job", name).Msg("jobrun: register failed") + } + } + + run := func() { + Run(ctx, name, interval, fn) + } + + if runOnBoot { + run() + } + + ticker := time.NewTicker(interval) + defer ticker.Stop() + poll := time.NewTicker(requestPoll) + defer poll.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + run() + case <-poll.C: + if st == nil { + continue + } + requested, err := st.TakeRunRequest(ctx, name) + if err != nil { + log.Warn().Err(err).Str("job", name).Msg("jobrun: poll failed") + continue + } + if requested { + run() + } + } + } +} + +// Run executes fn once and records it. For loops that keep their own ticker. +func Run(ctx context.Context, name string, interval time.Duration, fn func(ctx context.Context) error) { + st, _ := current() + started := time.Now() + if st != nil { + if err := st.MarkStarted(ctx, name, started); err != nil { + log.Warn().Err(err).Str("job", name).Msg("jobrun: mark started failed") + } + } + runErr := safeRun(ctx, name, fn) + if runErr != nil { + errs.CaptureException(runErr) + log.Warn().Err(runErr).Str("job", name).Msg("scheduled job failed") + } + if st != nil { + finished := time.Now() + if err := st.MarkFinished(ctx, name, started, finished, runErr, finished.Add(interval)); err != nil { + log.Warn().Err(err).Str("job", name).Msg("jobrun: mark finished failed") + } + } +} + +// safeRun turns a panic in a job into an error so one bad pass cannot take +// the service down. +func safeRun(ctx context.Context, name string, fn func(ctx context.Context) error) (err error) { + defer func() { + if r := recover(); r != nil { + err = &panicError{job: name, value: r} + } + }() + return fn(ctx) +} + +type panicError struct { + job string + value any +} + +func (p *panicError) Error() string { + return "job " + p.job + " panicked" +} diff --git a/internal/jobs/audit_retention.go b/internal/jobs/audit_retention.go index 068818e8..bf78762f 100644 --- a/internal/jobs/audit_retention.go +++ b/internal/jobs/audit_retention.go @@ -5,6 +5,7 @@ import ( "time" "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/observability/errs" "github.com/warmbly/warmbly/internal/repository" ) @@ -66,25 +67,9 @@ func NewAuditRetentionScheduler(job *AuditRetentionJob, interval time.Duration) // Start begins scheduled execution, running once immediately on boot. func (s *AuditRetentionScheduler) Start(ctx context.Context) { - ticker := time.NewTicker(s.interval) - defer ticker.Stop() - - if err := s.job.Run(ctx); err != nil { - errs.CaptureException(err) - } - - for { - select { - case <-ticker.C: - if err := s.job.Run(ctx); err != nil { - errs.CaptureException(err) - } - case <-s.stopCh: - return - case <-ctx.Done(): - return - } - } + ctx, cancel := stopContext(ctx, s.stopCh) + defer cancel() + jobrun.Loop(ctx, "audit_retention", s.interval, true, s.job.Run) } // Stop halts the scheduled execution. diff --git a/internal/jobs/danger_zone.go b/internal/jobs/danger_zone.go index 5bdd3b7f..ad7be5f5 100644 --- a/internal/jobs/danger_zone.go +++ b/internal/jobs/danger_zone.go @@ -2,11 +2,12 @@ package jobs import ( "context" + "errors" + "fmt" "time" - "github.com/warmbly/warmbly/internal/observability/errs" - "github.com/warmbly/warmbly/internal/app/dangerzone" + "github.com/warmbly/warmbly/internal/jobrun" ) // DangerZoneJob ticks the dangerzone subsystem: it executes any pending @@ -24,13 +25,15 @@ func NewDangerZoneJob(svc dangerzone.Service) *DangerZoneJob { // Run performs one tick. Errors are logged to Sentry and swallowed so // the scheduler keeps ticking; the next tick will retry anything that // got marked failed. -func (j *DangerZoneJob) Run(ctx context.Context) { +func (j *DangerZoneJob) Run(ctx context.Context) error { + var failures []error if _, _, err := j.svc.ExecuteDuePendingDeletions(ctx); err != nil { - errs.CaptureException(err) + failures = append(failures, fmt.Errorf("execute deletions: %w", err)) } if err := j.svc.DispatchReminders(ctx); err != nil { - errs.CaptureException(err) + failures = append(failures, fmt.Errorf("dispatch reminders: %w", err)) } + return errors.Join(failures...) } // DangerZoneScheduler runs the job on a fixed interval. @@ -53,21 +56,9 @@ func NewDangerZoneScheduler(job *DangerZoneJob, interval time.Duration) *DangerZ // Start runs Run() on every tick until ctx is cancelled or Stop() is called. func (s *DangerZoneScheduler) Start(ctx context.Context) { - ticker := time.NewTicker(s.interval) - defer ticker.Stop() - - s.job.Run(ctx) - - for { - select { - case <-ticker.C: - s.job.Run(ctx) - case <-s.stopCh: - return - case <-ctx.Done(): - return - } - } + ctx, cancel := stopContext(ctx, s.stopCh) + defer cancel() + jobrun.Loop(ctx, "danger_zone", s.interval, true, s.job.Run) } // Stop halts the scheduler. diff --git a/internal/jobs/email_verification.go b/internal/jobs/email_verification.go index 792e6d06..2e1bd164 100644 --- a/internal/jobs/email_verification.go +++ b/internal/jobs/email_verification.go @@ -2,11 +2,13 @@ package jobs import ( "context" + "sync" "time" "github.com/warmbly/warmbly/internal/observability/errs" emailverifyapp "github.com/warmbly/warmbly/internal/app/emailverify" + "github.com/warmbly/warmbly/internal/jobrun" ) // EmailVerificationJob verifies a batch of contacts due for a check each run, @@ -54,6 +56,7 @@ type EmailVerificationScheduler struct { job *EmailVerificationJob interval time.Duration stopCh chan struct{} + mu sync.Mutex } // NewEmailVerificationScheduler creates the scheduler. @@ -67,26 +70,29 @@ func NewEmailVerificationScheduler(job *EmailVerificationJob, interval time.Dura // Start begins scheduled execution. func (s *EmailVerificationScheduler) Start(ctx context.Context) { - ticker := time.NewTicker(s.interval) - defer ticker.Stop() - - var wake <-chan struct{} + ctx, cancel := stopContext(ctx, s.stopCh) + defer cancel() + // A wake (import, re-verify) runs the same pass outside the tick; the + // mutex keeps it from overlapping a tick or a "run now". + run := func(ctx context.Context) error { + s.mu.Lock() + defer s.mu.Unlock() + return s.job.Run(ctx) + } if s.job != nil && s.job.svc != nil { - wake = s.job.svc.Wake() - } - for { - select { - case <-ticker.C: - case <-wake: - case <-s.stopCh: - return - case <-ctx.Done(): - return - } - if err := s.job.Run(ctx); err != nil { - errs.CaptureException(err) - } + wake := s.job.svc.Wake() + go func() { + for { + select { + case <-ctx.Done(): + return + case <-wake: + jobrun.Run(ctx, "email_verification", s.interval, run) + } + } + }() } + jobrun.Loop(ctx, "email_verification", s.interval, false, run) } // Stop halts the scheduled execution. diff --git a/internal/jobs/form_events_retention.go b/internal/jobs/form_events_retention.go index 9d75f525..9a654d83 100644 --- a/internal/jobs/form_events_retention.go +++ b/internal/jobs/form_events_retention.go @@ -5,6 +5,7 @@ import ( "time" "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/repository" ) @@ -44,15 +45,5 @@ func (j *FormEventsRetentionJob) Run(ctx context.Context) error { // Start runs the job once on boot and then on the interval until ctx ends. func (j *FormEventsRetentionJob) Start(ctx context.Context, interval time.Duration) { - ticker := time.NewTicker(interval) - defer ticker.Stop() - _ = j.Run(ctx) - for { - select { - case <-ticker.C: - _ = j.Run(ctx) - case <-ctx.Done(): - return - } - } + jobrun.Loop(ctx, "form_events_retention", interval, true, j.Run) } diff --git a/internal/jobs/forms_domain_sweep.go b/internal/jobs/forms_domain_sweep.go index 46865760..00bc75d8 100644 --- a/internal/jobs/forms_domain_sweep.go +++ b/internal/jobs/forms_domain_sweep.go @@ -7,6 +7,7 @@ import ( "github.com/rs/zerolog/log" "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/pkg/trackdns" "github.com/warmbly/warmbly/internal/repository" ) @@ -71,15 +72,5 @@ func (j *FormsDomainSweep) Run(ctx context.Context) error { // Start runs the sweep once on boot and then on the interval until ctx ends. func (j *FormsDomainSweep) Start(ctx context.Context, interval time.Duration) { - ticker := time.NewTicker(interval) - defer ticker.Stop() - _ = j.Run(ctx) - for { - select { - case <-ticker.C: - _ = j.Run(ctx) - case <-ctx.Done(): - return - } - } + jobrun.Loop(ctx, "forms_domain_sweep", interval, true, j.Run) } diff --git a/internal/jobs/guardrail.go b/internal/jobs/guardrail.go index f71b0dd3..ff468871 100644 --- a/internal/jobs/guardrail.go +++ b/internal/jobs/guardrail.go @@ -8,6 +8,7 @@ import ( "github.com/warmbly/warmbly/internal/observability/errs" "github.com/warmbly/warmbly/internal/app/guardrail" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/repository" ) @@ -69,21 +70,12 @@ func NewGuardrailScheduler(job *GuardrailJob, interval time.Duration) *Guardrail // Start runs Run() on every tick until ctx is cancelled or Stop() is called. func (s *GuardrailScheduler) Start(ctx context.Context) { - ticker := time.NewTicker(s.interval) - defer ticker.Stop() - - s.job.Run(ctx) - - for { - select { - case <-ticker.C: - s.job.Run(ctx) - case <-s.stopCh: - return - case <-ctx.Done(): - return - } - } + ctx, cancel := stopContext(ctx, s.stopCh) + defer cancel() + jobrun.Loop(ctx, "guardrail_sweep", s.interval, true, func(ctx context.Context) error { + s.job.Run(ctx) + return nil + }) } // Stop halts the scheduler. diff --git a/internal/jobs/org_transfer.go b/internal/jobs/org_transfer.go index 08ba2bcd..fb6da979 100644 --- a/internal/jobs/org_transfer.go +++ b/internal/jobs/org_transfer.go @@ -2,11 +2,11 @@ package jobs import ( "context" + "fmt" "time" - "github.com/warmbly/warmbly/internal/observability/errs" - "github.com/warmbly/warmbly/internal/app/orgtransfer" + "github.com/warmbly/warmbly/internal/jobrun" ) // OrgTransferJob keeps the workspace-archive tables honest. It deletes archives @@ -29,13 +29,14 @@ func NewOrgTransferJob(svc orgtransfer.Service) *OrgTransferJob { // Run performs one tick. Errors are reported and swallowed so the scheduler // keeps ticking; the next tick retries whatever did not land. -func (j *OrgTransferJob) Run(ctx context.Context) { +func (j *OrgTransferJob) Run(ctx context.Context) error { if j.svc == nil { - return + return nil } if _, err := j.svc.PurgeExpiredExports(ctx); err != nil { - errs.CaptureException(err) + return fmt.Errorf("purge expired exports: %w", err) } + return nil } // OrgTransferScheduler runs the job on a fixed interval. @@ -57,21 +58,9 @@ func NewOrgTransferScheduler(job *OrgTransferJob, interval time.Duration) *OrgTr // Start runs Run() on every tick until ctx is cancelled or Stop() is called. func (s *OrgTransferScheduler) Start(ctx context.Context) { - ticker := time.NewTicker(s.interval) - defer ticker.Stop() - - s.job.Run(ctx) - - for { - select { - case <-ticker.C: - s.job.Run(ctx) - case <-s.stopCh: - return - case <-ctx.Done(): - return - } - } + ctx, cancel := stopContext(ctx, s.stopCh) + defer cancel() + jobrun.Loop(ctx, "org_transfer_housekeeping", s.interval, true, s.job.Run) } // Stop halts the scheduler. diff --git a/internal/jobs/placement_poller.go b/internal/jobs/placement_poller.go index 3f558100..6db47fa5 100644 --- a/internal/jobs/placement_poller.go +++ b/internal/jobs/placement_poller.go @@ -7,6 +7,7 @@ import ( "github.com/warmbly/warmbly/internal/observability/errs" "github.com/warmbly/warmbly/internal/app/placement" + "github.com/warmbly/warmbly/internal/jobrun" ) // PlacementPoller reconciles pending seed inbox-placement results: each tick it @@ -49,21 +50,9 @@ func (p *PlacementPoller) Run(ctx context.Context) error { // Start begins scheduled execution on the configured interval. func (p *PlacementPoller) Start(ctx context.Context) { - ticker := time.NewTicker(p.interval) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - if err := p.Run(ctx); err != nil { - errs.CaptureException(err) - } - case <-p.stopCh: - return - case <-ctx.Done(): - return - } - } + ctx, cancel := stopContext(ctx, p.stopCh) + defer cancel() + jobrun.Loop(ctx, "placement_poller", p.interval, false, p.Run) } // Stop halts scheduled execution. diff --git a/internal/jobs/schedule.go b/internal/jobs/schedule.go new file mode 100644 index 00000000..f24f7689 --- /dev/null +++ b/internal/jobs/schedule.go @@ -0,0 +1,17 @@ +package jobs + +import "context" + +// stopContext derives a context that ends when the scheduler's Stop() closes +// stopCh, so a jobrun loop keeps the old Stop() semantics. +func stopContext(ctx context.Context, stopCh <-chan struct{}) (context.Context, context.CancelFunc) { + ctx, cancel := context.WithCancel(ctx) + go func() { + select { + case <-stopCh: + cancel() + case <-ctx.Done(): + } + }() + return ctx, cancel +} diff --git a/internal/jobs/trial_expiration.go b/internal/jobs/trial_expiration.go index 56731f39..43433a75 100644 --- a/internal/jobs/trial_expiration.go +++ b/internal/jobs/trial_expiration.go @@ -7,6 +7,7 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5/pgxpool" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/notify" "github.com/warmbly/warmbly/internal/notify/templates" @@ -154,26 +155,9 @@ func NewTrialExpirationScheduler(job *TrialExpirationJob, interval time.Duration // Start begins the scheduled execution func (s *TrialExpirationScheduler) Start(ctx context.Context) { - ticker := time.NewTicker(s.interval) - defer ticker.Stop() - - // Run immediately on start - if err := s.job.Run(ctx); err != nil { - errs.CaptureException(err) - } - - for { - select { - case <-ticker.C: - if err := s.job.Run(ctx); err != nil { - errs.CaptureException(err) - } - case <-s.stopCh: - return - case <-ctx.Done(): - return - } - } + ctx, cancel := stopContext(ctx, s.stopCh) + defer cancel() + jobrun.Loop(ctx, "trial_expiration", s.interval, true, s.job.Run) } // Stop halts the scheduled execution diff --git a/internal/jobs/verification_evidence.go b/internal/jobs/verification_evidence.go index 630d4da5..e5fd1144 100644 --- a/internal/jobs/verification_evidence.go +++ b/internal/jobs/verification_evidence.go @@ -4,9 +4,8 @@ import ( "context" "time" - "github.com/warmbly/warmbly/internal/observability/errs" - emailverifyapp "github.com/warmbly/warmbly/internal/app/emailverify" + "github.com/warmbly/warmbly/internal/jobrun" ) // DeliveryEvidenceJob turns campaign sends that never bounced into @@ -28,23 +27,15 @@ func NewDeliveryEvidenceJob(evidence *emailverifyapp.Evidence, interval time.Dur // Start runs the job on its interval until ctx ends. A full batch repeats // at once so a backlog drains. func (j *DeliveryEvidenceJob) Start(ctx context.Context) { - ticker := time.NewTicker(j.interval) - defer ticker.Stop() - for { - select { - case <-ticker.C: - case <-ctx.Done(): - return - } + jobrun.Loop(ctx, "delivery_evidence", j.interval, false, func(ctx context.Context) error { for { n, err := j.evidence.CreditCleanDeliveries(ctx, j.batch) if err != nil { - errs.CaptureException(err) - break + return err } if n < j.batch || ctx.Err() != nil { - break + return nil } } - } + }) } diff --git a/internal/jobs/warmup_batch_poller.go b/internal/jobs/warmup_batch_poller.go index f02248d4..4e0834bc 100644 --- a/internal/jobs/warmup_batch_poller.go +++ b/internal/jobs/warmup_batch_poller.go @@ -7,6 +7,7 @@ import ( "github.com/warmbly/warmbly/internal/observability/errs" "github.com/warmbly/warmbly/internal/app/warmupcontent" + "github.com/warmbly/warmbly/internal/jobrun" ) // WarmupBatchPoller reconciles in-flight OpenAI Batch API warmup-generation jobs: @@ -47,21 +48,9 @@ func (p *WarmupBatchPoller) Run(ctx context.Context) error { // Start begins scheduled execution on the configured interval. func (p *WarmupBatchPoller) Start(ctx context.Context) { - ticker := time.NewTicker(p.interval) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - if err := p.Run(ctx); err != nil { - errs.CaptureException(err) - } - case <-p.stopCh: - return - case <-ctx.Done(): - return - } - } + ctx, cancel := stopContext(ctx, p.stopCh) + defer cancel() + jobrun.Loop(ctx, "warmup_batch_poller", p.interval, false, p.Run) } // Stop halts scheduled execution. diff --git a/internal/jobs/warmup_generation.go b/internal/jobs/warmup_generation.go index b0776d85..f5102201 100644 --- a/internal/jobs/warmup_generation.go +++ b/internal/jobs/warmup_generation.go @@ -8,6 +8,7 @@ import ( "github.com/warmbly/warmbly/internal/observability/errs" "github.com/warmbly/warmbly/internal/app/warmupcontent" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/repository" ) @@ -79,21 +80,9 @@ func NewWarmupGenerationScheduler(job *WarmupGenerationJob, interval time.Durati // Start begins scheduled execution. func (s *WarmupGenerationScheduler) Start(ctx context.Context) { - ticker := time.NewTicker(s.interval) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - if err := s.job.Run(ctx); err != nil { - errs.CaptureException(err) - } - case <-s.stopCh: - return - case <-ctx.Done(): - return - } - } + ctx, cancel := stopContext(ctx, s.stopCh) + defer cancel() + jobrun.Loop(ctx, "warmup_generation", s.interval, false, s.job.Run) } // Stop halts the scheduled execution. diff --git a/internal/jobs/website_tracking_retention.go b/internal/jobs/website_tracking_retention.go index c705d31d..224d5bc2 100644 --- a/internal/jobs/website_tracking_retention.go +++ b/internal/jobs/website_tracking_retention.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/observability/errs" "github.com/warmbly/warmbly/internal/repository" ) @@ -39,15 +40,5 @@ func (j *WebsiteTrackingRetentionJob) Run(ctx context.Context) error { // Start runs the job once on boot and then on the interval until ctx ends. func (j *WebsiteTrackingRetentionJob) Start(ctx context.Context, interval time.Duration) { - ticker := time.NewTicker(interval) - defer ticker.Stop() - _ = j.Run(ctx) - for { - select { - case <-ticker.C: - _ = j.Run(ctx) - case <-ctx.Done(): - return - } - } + jobrun.Loop(ctx, "website_tracking_retention", interval, true, j.Run) } diff --git a/internal/models/admin.go b/internal/models/admin.go index fbf921a7..74b621ca 100644 --- a/internal/models/admin.go +++ b/internal/models/admin.go @@ -494,17 +494,6 @@ type HourlyEmailStats struct { TotalSent int64 `json:"total_sent"` } -// WorkerLoadStats represents worker load statistics -type WorkerLoadStats struct { - WorkerID uuid.UUID `json:"worker_id"` - WorkerName string `json:"worker_name"` - EmailsSentToday int64 `json:"emails_sent_today"` - QueuedEmails int64 `json:"queued_emails"` - ConnectedEmails int64 `json:"connected_emails"` - CPUUsage float64 `json:"cpu_usage,omitempty"` - MemoryUsage float64 `json:"memory_usage,omitempty"` -} - // UserGrowthStats represents user growth statistics type UserGrowthStats struct { Date time.Time `json:"date"` @@ -520,111 +509,6 @@ type AnalyticsTrends struct { RevenueGrowthPercent float64 `json:"revenue_growth_percent"` } -// CreatePlanRequest represents the request to create a custom plan -type CreatePlanRequest struct { - Name string `json:"name" binding:"required"` - MaxContacts uint `json:"max_contacts"` - DailyEmails uint `json:"daily_emails"` - AIGeneration bool `json:"ai_generation"` - AccountLimit uint `json:"account_limit"` - Price float32 `json:"price"` - DiscountedPrice float32 `json:"discounted_price"` - Duration Duration `json:"duration"` // month/year - DedicatedWorkers int `json:"dedicated_workers"` - DailyCampaignLimit *int `json:"daily_campaign_limit,omitempty"` - MaxCampaigns *int `json:"max_campaigns,omitempty"` - MaxActiveCampaigns *int `json:"max_active_campaigns,omitempty"` - MaxTeamMembers *int `json:"max_team_members,omitempty"` - MaxEmailAccounts *int `json:"max_email_accounts,omitempty"` - Public bool `json:"public"` // false for enterprise-only -} - -// UpdatePlanRequest represents the request to update a plan -type UpdatePlanRequest struct { - Name *string `json:"name,omitempty"` - MaxContacts *uint `json:"max_contacts,omitempty"` - DailyEmails *uint `json:"daily_emails,omitempty"` - AIGeneration *bool `json:"ai_generation,omitempty"` - AccountLimit *uint `json:"account_limit,omitempty"` - Price *float32 `json:"price,omitempty"` - DiscountedPrice *float32 `json:"discounted_price,omitempty"` - Duration *Duration `json:"duration,omitempty"` - DedicatedWorkers *int `json:"dedicated_workers,omitempty"` - DailyCampaignLimit *int `json:"daily_campaign_limit,omitempty"` - MaxCampaigns *int `json:"max_campaigns,omitempty"` - MaxActiveCampaigns *int `json:"max_active_campaigns,omitempty"` - MaxTeamMembers *int `json:"max_team_members,omitempty"` - MaxEmailAccounts *int `json:"max_email_accounts,omitempty"` - Public *bool `json:"public,omitempty"` -} - -// AdminEnterpriseInquiry represents an enterprise inquiry with admin details -type AdminEnterpriseInquiry struct { - ID uuid.UUID `json:"id"` - UserID *uuid.UUID `json:"user_id,omitempty"` - CompanyName string `json:"company_name"` - ContactName string `json:"contact_name"` - ContactEmail string `json:"contact_email"` - Phone *string `json:"phone,omitempty"` - TeamSize *string `json:"team_size,omitempty"` - EstimatedVolume *int `json:"estimated_volume,omitempty"` - MonthlyEmailVolume *string `json:"monthly_email_volume,omitempty"` - Message *string `json:"message,omitempty"` - Status string `json:"status"` - AssignedTo *uuid.UUID `json:"assigned_to,omitempty"` - Notes *string `json:"notes,omitempty"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - - // Joined data - User *AdminUserSummary `json:"user,omitempty"` - AssignedAdmin *AdminUserSummary `json:"assigned_admin,omitempty"` -} - -// AdminEnterpriseInquiriesResult represents paginated inquiries -type AdminEnterpriseInquiriesResult struct { - Data []AdminEnterpriseInquiry `json:"data"` - Pagination Pagination `json:"pagination"` -} - -// AdminEnterpriseInquirySearch are the query params for the enterprise inquiry -// queue. Mirrors AdminOrgSearch tag conventions. -type AdminEnterpriseInquirySearch struct { - Query string `form:"q"` - Status string `form:"status"` // pending, contacted, converted, declined, all - Assignment string `form:"assignment"` // assigned, unassigned - Linkage string `form:"linkage"` // linked, anonymous - - HasNotes bool `form:"has_notes"` - HasPhone bool `form:"has_phone"` - Processed bool `form:"processed"` - - // Count ranges - TeamSizeMin *int `form:"team_size_min"` - TeamSizeMax *int `form:"team_size_max"` - EstimatedVolumeMin *int `form:"estimated_volume_min"` - EstimatedVolumeMax *int `form:"estimated_volume_max"` - - // Date ranges (YYYY-MM-DD, UTC) - CreatedWithin int `form:"created_within"` - CreatedAfter *time.Time `form:"created_after" time_format:"2006-01-02" time_utc:"true"` - CreatedBefore *time.Time `form:"created_before" time_format:"2006-01-02" time_utc:"true"` - UpdatedAfter *time.Time `form:"updated_after" time_format:"2006-01-02" time_utc:"true"` - UpdatedBefore *time.Time `form:"updated_before" time_format:"2006-01-02" time_utc:"true"` - - Cursor *uuid.UUID `form:"cursor"` - Limit int `form:"limit"` - SortBy string `form:"sort_by"` // created_at, updated_at, company_name, contact_email, status, team_size, estimated_volume - SortDesc bool `form:"sort_desc"` -} - -// UpdateEnterpriseInquiryRequest represents the request to update an inquiry -type UpdateEnterpriseInquiryRequest struct { - Status *string `json:"status,omitempty"` - AssignedTo *uuid.UUID `json:"assigned_to,omitempty"` - Notes *string `json:"notes,omitempty"` -} - // AdminInfo represents an admin user for listing type AdminInfo struct { ID uuid.UUID `json:"id"` @@ -766,14 +650,6 @@ type WorkerStats struct { QueueDepth int64 `json:"queue_depth"` } -// EmailDistribution represents email distribution across workers -type EmailDistribution struct { - WorkerID uuid.UUID `json:"worker_id"` - WorkerName string `json:"worker_name"` - EmailCount int64 `json:"email_count"` - Percentage float64 `json:"percentage"` -} - // AdminOrgSearch are the query params for the admin organization listing. type AdminOrgSearch struct { Query string `form:"q"` @@ -923,43 +799,6 @@ type AdminLimitRequestsResult struct { Pagination Pagination `json:"pagination"` } -// AdminPlanSearch are the query params for the admin plan catalog listing. -// Mirrors AdminOrgSearch tag conventions. price_* bind to *int (whole-dollar -// filtering against the numeric price column). -type AdminPlanSearch struct { - Query string `form:"q"` - Visibility string `form:"visibility"` // public, private, "" = any - Duration string `form:"duration"` // month, year, "" = any (durations.title) - - AIGeneration bool `form:"ai_generation"` - HasStripe bool `form:"has_stripe"` - HasSubscribers bool `form:"has_subscribers"` - - // Numeric ranges - PriceMin *int `form:"price_min"` - PriceMax *int `form:"price_max"` - DailyEmailsMin *int `form:"daily_emails_min"` - DailyEmailsMax *int `form:"daily_emails_max"` - AccountLimitMin *int `form:"account_limit_min"` - AccountLimitMax *int `form:"account_limit_max"` - - // Date range - CreatedWithin int `form:"created_within"` // days; 0 = any - CreatedAfter *time.Time `form:"created_after" time_format:"2006-01-02" time_utc:"true"` - CreatedBefore *time.Time `form:"created_before" time_format:"2006-01-02" time_utc:"true"` - - Cursor *uuid.UUID `form:"cursor"` - Limit int `form:"limit"` - SortBy string `form:"sort_by"` // price, name, daily_emails, account_limit, created_at - SortDesc bool `form:"sort_desc"` -} - -// AdminPlansResult is the paginated response for the admin plan listing. -type AdminPlansResult struct { - Data []Plan `json:"data"` - Pagination Pagination `json:"pagination"` -} - // AdminOrgDetail is the full payload for the org detail page. Carries // three limit blocks side-by-side so the UI can explain *why* each // effective number is what it is: diff --git a/internal/models/admin_ops.go b/internal/models/admin_ops.go new file mode 100644 index 00000000..ed33c4bd --- /dev/null +++ b/internal/models/admin_ops.go @@ -0,0 +1,357 @@ +package models + +import ( + "encoding/json" + "time" + + "github.com/google/uuid" +) + +// Operator-facing read models for the admin panel's operations pages: mailbox +// sync, in-flight sends, dead letters, scheduled jobs, fleet placement, +// workspace transfers and abuse signals. Every row here is instance-wide and +// carries the organization it belongs to, because the operator reads across +// workspaces. + +// ---- mailbox sync ---- + +// AdminSyncSearch filters the sync governor page. +type AdminSyncSearch struct { + // State: all | throttled | backfilling | stalled | pending | complete. + State string `form:"state"` + Q string `form:"q"` + Cursor string `form:"cursor"` + Limit int `form:"limit"` +} + +// AdminSyncSummary counts every mailbox that has reported sync state once. +type AdminSyncSummary struct { + Total int `json:"total"` + Throttled int `json:"throttled"` + Backfilling int `json:"backfilling"` + Stalled int `json:"stalled"` + Pending int `json:"pending"` + Complete int `json:"complete"` + Deferred int `json:"deferred"` +} + +// AdminSyncRow is one mailbox's sync state with its owner attached. +type AdminSyncRow struct { + EmailID uuid.UUID `json:"email_id"` + UserID uuid.UUID `json:"user_id"` + Email string `json:"email"` + Provider string `json:"provider"` + AccountStatus string `json:"account_status"` + OrganizationID *uuid.UUID `json:"organization_id,omitempty"` + OrganizationName string `json:"organization_name"` + WorkerID *uuid.UUID `json:"worker_id,omitempty"` + + BackfillStatus string `json:"backfill_status"` + BackfillSynced int `json:"backfill_synced"` + BackfillSince *time.Time `json:"backfill_since,omitempty"` + BackfillStartedAt *time.Time `json:"backfill_started_at,omitempty"` + BackfillCompletedAt *time.Time `json:"backfill_completed_at,omitempty"` + + ThrottledUntil *time.Time `json:"throttled_until,omitempty"` + ThrottleReason string `json:"throttle_reason"` + Deferred int `json:"deferred"` + // Stalled is a running backfill whose state has not moved for an hour. + Stalled bool `json:"stalled"` + LastSyncedAt *time.Time `json:"last_synced_at,omitempty"` + UpdatedAt time.Time `json:"updated_at"` +} + +type AdminSyncResult struct { + Data []AdminSyncRow `json:"data"` + Pagination *Pagination `json:"pagination"` + Summary AdminSyncSummary `json:"summary"` +} + +// ---- in-flight sends ---- + +// AdminInFlightSend is a reserved campaign send no worker result has resolved. +type AdminInFlightSend struct { + CampaignID uuid.UUID `json:"campaign_id"` + CampaignName string `json:"campaign_name"` + OrganizationID *uuid.UUID `json:"organization_id,omitempty"` + OrganizationName string `json:"organization_name"` + ContactID uuid.UUID `json:"contact_id"` + ContactEmail string `json:"contact_email"` + SequenceID uuid.UUID `json:"sequence_id"` + TaskID *uuid.UUID `json:"task_id,omitempty"` + TaskStatus string `json:"task_status"` + // HasMessageID means the worker did put the mail on the wire and only the + // stamp was lost; the reclaimer will stamp it rather than retry. + HasMessageID bool `json:"has_message_id"` + EmailAccountID *uuid.UUID `json:"email_account_id,omitempty"` + MailboxEmail string `json:"mailbox_email"` + WorkerID *uuid.UUID `json:"worker_id,omitempty"` + DispatchedAt time.Time `json:"dispatched_at"` + AgeSeconds int64 `json:"age_seconds"` +} + +type AdminInFlightSummary struct { + Total int `json:"total"` + // Age buckets, in minutes since dispatch. + Under5m int `json:"under_5m"` + Under30m int `json:"under_30m"` + PastReclaimWindow int `json:"past_reclaim_window"` + OldestDispatched *time.Time `json:"oldest_dispatched_at,omitempty"` + // ReclaimAfterMinutes is config.CampaignSendReclaimAfterMinutes, so the + // page can say when the sweep will pick a row up. + ReclaimAfterMinutes int `json:"reclaim_after_minutes"` +} + +type AdminInFlightResult struct { + Summary AdminInFlightSummary `json:"summary"` + Data []AdminInFlightSend `json:"data"` +} + +// AdminDeadLetterRow is a task dead letter with the workspace it belongs to. +type AdminDeadLetterRow struct { + TaskDeadLetter + OrganizationID *uuid.UUID `json:"organization_id,omitempty"` + OrganizationName string `json:"organization_name"` +} + +type AdminDeadLettersResult struct { + Data []AdminDeadLetterRow `json:"data"` + Pagination *Pagination `json:"pagination"` + // Counts by status across the instance, independent of the filter. + Pending int `json:"pending"` + Replayed int `json:"replayed"` + Failed int `json:"failed"` +} + +// AdminTaskFailureRow is one row of task_failures with the task it names. +type AdminTaskFailureRow struct { + TaskID uuid.UUID `json:"task_id"` + TaskType string `json:"task_type"` + TaskStatus string `json:"task_status"` + Title string `json:"title"` + Message string `json:"message"` + EmailAccountID uuid.UUID `json:"email_account_id"` + MailboxEmail string `json:"mailbox_email"` + OrganizationID *uuid.UUID `json:"organization_id,omitempty"` + OrganizationName string `json:"organization_name"` + OccurredAt time.Time `json:"occurred_at"` +} + +// AdminWebhookEndpointRow is a customer webhook endpoint as the operator sees it. +type AdminWebhookEndpointRow struct { + ID uuid.UUID `json:"id"` + OrganizationID uuid.UUID `json:"organization_id"` + OrganizationName string `json:"organization_name"` + URL string `json:"url"` + Description string `json:"description"` + Enabled bool `json:"enabled"` + EventTypes []string `json:"event_types"` + ConsecutiveFailures int `json:"consecutive_failures"` + LastSuccessAt *time.Time `json:"last_success_at,omitempty"` + LastFailureAt *time.Time `json:"last_failure_at,omitempty"` + LastFailureReason string `json:"last_failure_reason"` + DeliveriesLast7d int64 `json:"deliveries_last_7d"` + FailedLast7d int64 `json:"failed_last_7d"` + DropsLast7d int64 `json:"drops_last_7d"` +} + +// AdminWebhookHealth is the instance-wide delivery picture. +type AdminWebhookHealth struct { + // InFlightStale counts deliveries claimed longer ago than the lease. + InFlightStale int64 `json:"in_flight_stale"` + PendingDue int64 `json:"pending_due"` + DeliveredLast24h int64 `json:"delivered_last_24h"` + FailedLast24h int64 `json:"failed_last_24h"` + AbandonedLast24h int64 `json:"abandoned_last_24h"` + DropsLast7d int64 `json:"drops_last_7d"` + LeaseMinutes int `json:"lease_minutes"` + // FailingEndpoints lists endpoints with consecutive failures, worst first. + FailingEndpoints []AdminWebhookEndpointRow `json:"failing_endpoints"` +} + +// ---- scheduled jobs ---- + +// ScheduledJobRun is the persisted record of one background loop: what it is, +// when it last ran and how it went. One row per job name, shared by every +// service that runs loops (backend, consumer). +type ScheduledJobRun struct { + Name string `json:"name"` + Service string `json:"service"` + IntervalSeconds int `json:"interval_seconds"` + LastStartedAt *time.Time `json:"last_started_at,omitempty"` + LastFinishedAt *time.Time `json:"last_finished_at,omitempty"` + LastDurationMs int64 `json:"last_duration_ms"` + // LastStatus: idle | running | ok | error. + LastStatus string `json:"last_status"` + LastError string `json:"last_error"` + RunCount int64 `json:"run_count"` + ErrorCount int64 `json:"error_count"` + // RunRequestedAt is set by the panel's "run now"; the owning loop clears it + // when it picks the request up. + RunRequestedAt *time.Time `json:"run_requested_at,omitempty"` + NextRunAt *time.Time `json:"next_run_at,omitempty"` + UpdatedAt time.Time `json:"updated_at"` +} + +// ---- fleet ---- + +// AdminFleetWorkerRow is a worker with its capacity-view row attached. +type AdminFleetWorkerRow struct { + WorkerID uuid.UUID `json:"worker_id"` + Name string `json:"name"` + IPAddr string `json:"ip_addr"` + Active bool `json:"active"` + FreeTier bool `json:"free_tier"` + WorkerType WorkerType `json:"worker_type"` + RiskPool WorkerRiskPool `json:"risk_pool"` + EgressKind WorkerEgressKind `json:"egress_kind"` + HealthState WorkerHealthState `json:"health_state"` + InstallState string `json:"install_state"` + LastSeenAt *time.Time `json:"last_seen_at,omitempty"` + Live bool `json:"live"` + AccountCount int `json:"account_count"` + Tags []string `json:"tags"` + + LoadScore float64 `json:"load_score"` + BaseCapacity float64 `json:"base_capacity"` + HealthMultiplier float64 `json:"health_multiplier"` + AgeMultiplier float64 `json:"age_multiplier"` + EffectiveCapacity float64 `json:"effective_capacity"` + // Utilization is load over effective capacity; the rebalancer calls a + // worker hot above 0.8 and cold below 0.5. + Utilization float64 `json:"utilization"` + SendsAttempted1h int64 `json:"sends_attempted_1h"` + SendsSucceeded1h int64 `json:"sends_succeeded_1h"` + BouncesHard1h int64 `json:"bounces_hard_1h"` + BouncesSoft1h int64 `json:"bounces_soft_1h"` + Complaints1h int64 `json:"complaints_1h"` + AuthErrors1h int64 `json:"auth_errors_1h"` +} + +// AdminFleetDecision is one decision_log row: what a control loop did and why. +type AdminFleetDecision struct { + ID int64 `json:"id"` + Kind string `json:"kind"` + WorkerID *uuid.UUID `json:"worker_id,omitempty"` + WorkerName string `json:"worker_name"` + MailboxID *uuid.UUID `json:"mailbox_id,omitempty"` + Before json.RawMessage `json:"before,omitempty"` + After json.RawMessage `json:"after,omitempty"` + Reason string `json:"reason"` + TriggeredBy string `json:"triggered_by"` + CreatedAt time.Time `json:"created_at"` +} + +// AdminDedicatedAssignment is an active worker-to-organization binding. +type AdminDedicatedAssignment struct { + ID uuid.UUID `json:"id"` + WorkerID uuid.UUID `json:"worker_id"` + WorkerName string `json:"worker_name"` + WorkerLive bool `json:"worker_live"` + OrganizationID uuid.UUID `json:"organization_id"` + OrganizationName string `json:"organization_name"` + SubscriptionID uuid.UUID `json:"subscription_id"` + AssignedAt time.Time `json:"assigned_at"` + ReleasedAt *time.Time `json:"released_at,omitempty"` + AccountCount int `json:"account_count"` +} + +// AdminConvertDedicatedRequest is the body of POST /admin/workers/:id/convert-dedicated. +type AdminConvertDedicatedRequest struct { + OrganizationID string `json:"organization_id"` + SubscriptionID string `json:"subscription_id"` + DrainToWorkerID *string `json:"drain_to_worker_id"` +} + +// ---- workspace transfers ---- + +// AdminTransferJob is an export or import job with its workspace attached. +type AdminTransferJob struct { + // Kind: export | import. + Kind string `json:"kind"` + ID uuid.UUID `json:"id"` + OrganizationID uuid.UUID `json:"organization_id"` + OrganizationName string `json:"organization_name"` + RequestedBy *uuid.UUID `json:"requested_by,omitempty"` + RequestedByEmail string `json:"requested_by_email"` + Status OrgTransferStatus `json:"status"` + Groups []OrgDataGroup `json:"groups"` + IncludeSecrets bool `json:"include_secrets"` + ProgressPercent int `json:"progress_percent"` + ProgressStage string `json:"progress_stage"` + ArchiveBytes *int64 `json:"archive_bytes,omitempty"` + ErrorMessage *string `json:"error_message,omitempty"` + StartedAt *time.Time `json:"started_at,omitempty"` + CompletedAt *time.Time `json:"completed_at,omitempty"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` + CreatedAt time.Time `json:"created_at"` +} + +// ---- abuse and insight ---- + +// AdminWarmupAbuseRow ranks a mailbox by invalid warmup-token attempts. +type AdminWarmupAbuseRow struct { + EmailAccountID uuid.UUID `json:"email_account_id"` + Email string `json:"email"` + OrganizationID *uuid.UUID `json:"organization_id,omitempty"` + OrganizationName string `json:"organization_name"` + Attempts int `json:"attempts"` + LastAttemptAt time.Time `json:"last_attempt_at"` + Blocked bool `json:"blocked"` + SpamScore int `json:"spam_score"` + HealthState string `json:"health_state"` +} + +// AdminWarmupAction is one warmup_admin_actions row with both parties named. +type AdminWarmupAction struct { + ID uuid.UUID `json:"id"` + AdminUserID uuid.UUID `json:"admin_user_id"` + AdminEmail string `json:"admin_email"` + EmailAccountID uuid.UUID `json:"email_account_id"` + Email string `json:"email"` + Action string `json:"action"` + Reason *string `json:"reason,omitempty"` + CreatedAt time.Time `json:"created_at"` +} + +// AdminOrgAPIKey is an API key as the operator sees it: never the secret. +type AdminOrgAPIKey struct { + ID uuid.UUID `json:"id"` + Name string `json:"name"` + KeyPrefix string `json:"key_prefix"` + KeySuffix string `json:"key_suffix"` + Status string `json:"status"` + Permissions int64 `json:"permissions"` + UserID uuid.UUID `json:"user_id"` + UserEmail string `json:"user_email"` + LastUsedAt *time.Time `json:"last_used_at,omitempty"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` + RevokedAt *time.Time `json:"revoked_at,omitempty"` + CreatedAt time.Time `json:"created_at"` + RequestsLast7d int64 `json:"requests_last_7d"` +} + +// AdminAcquisitionChannel is signups grouped by UTM source and medium. +type AdminAcquisitionChannel struct { + Source string `json:"source"` + Medium string `json:"medium"` + Signups int `json:"signups"` + Converted int `json:"converted"` +} + +type AdminAcquisitionReferrer struct { + Host string `json:"host"` + Signups int `json:"signups"` +} + +// AdminAcquisition is signups by channel over a window, with how many of them +// went on to a paid subscription and how many trials are about to end. +type AdminAcquisition struct { + Days int `json:"days"` + Signups int `json:"signups"` + WithChannel int `json:"with_channel"` + Converted int `json:"converted"` + TrialsExpiring7d int `json:"trials_expiring_7d"` + Channels []AdminAcquisitionChannel `json:"channels"` + Referrers []AdminAcquisitionReferrer `json:"referrers"` +} diff --git a/internal/models/admin_permission.go b/internal/models/admin_permission.go index 4cb3c157..6474bf48 100644 --- a/internal/models/admin_permission.go +++ b/internal/models/admin_permission.go @@ -3,12 +3,14 @@ package models // AdminPermission represents platform-level admin permissions as a bitmask type AdminPermission uint32 +// Bit positions are stored in users.admin_permissions, so a retired bit keeps +// its placeholder here and is never reused. const ( // User Management (bits 0-3) - AdminPermViewUsers AdminPermission = 1 << iota // View user profiles - AdminPermBanUsers // Ban/unban users - AdminPermEditUsers // Edit user details - AdminPermImpersonateUsers // Login as user (support) + AdminPermViewUsers AdminPermission = 1 << iota // View user profiles + AdminPermBanUsers // Ban/unban users + adminPermReserved2 // Retired (edit users), never reuse + adminPermReserved3 // Retired (impersonate users), never reuse // Worker Management (bits 4-5) AdminPermViewWorkers // View worker list @@ -31,11 +33,11 @@ const ( AdminPermManageRateLimits // User rate limits AdminPermManageSettings // Platform settings - // Enterprise (bits 15-18) - AdminPermViewEnterpriseInquiries // View inquiries - AdminPermManageEnterpriseInquiries // Process inquiries - AdminPermManagePlans // Create/edit custom plans - AdminPermManageBilling // Refunds, adjust billing + // Retired enterprise bits (15-18), never reuse + adminPermReserved15 // Retired (view enterprise inquiries) + adminPermReserved16 // Retired (manage enterprise inquiries) + adminPermReserved17 // Retired (manage plans) + adminPermReserved18 // Retired (manage billing) // Super Admin (bit 19) AdminPermGrantAdminAccess // Grant/revoke admin permissions @@ -45,10 +47,28 @@ const ( AdminPermManageOrganizations // Set per-org limit overrides, ban scope, etc. ) -// AllAdminPermissions contains all admin permissions. Bump the shift -// whenever a new bit is added above. +// AllAdminPermissions is the full mask, retired bits included: it is what +// existing super admins hold in the database. Bump the shift whenever a new +// bit is added above. const AllAdminPermissions AdminPermission = (1 << 22) - 1 +// LiveAdminPermissions ORs every bit that still gates a route. +const LiveAdminPermissions AdminPermission = AdminPermViewUsers | AdminPermBanUsers | + AdminPermViewWorkers | AdminPermManageWorkers | + AdminPermViewWarmupPool | AdminPermManageWarmupBans | AdminPermReviewAppeals | + AdminPermViewCampaigns | AdminPermStopCampaigns | + AdminPermViewAnalytics | AdminPermViewAuditLogs | + AdminPermManageRateLimits | AdminPermManageSettings | + AdminPermGrantAdminAccess | + AdminPermViewOrganizations | AdminPermManageOrganizations + +// retiredAdminPermissions are the bits that no longer gate anything. +const retiredAdminPermissions = adminPermReserved2 | adminPermReserved3 | + adminPermReserved15 | adminPermReserved16 | adminPermReserved17 | adminPermReserved18 + +// Compile-time check: every bit in AllAdminPermissions is live or retired. +var _ [0]struct{} = [AllAdminPermissions ^ (LiveAdminPermissions | retiredAdminPermissions)]struct{}{} + // HasPermission checks if the permission bitmask contains the specified permission func (p AdminPermission) HasPermission(perm AdminPermission) bool { return p&perm == perm @@ -69,9 +89,9 @@ func (p AdminPermission) IsAdmin() bool { return p > 0 } -// IsSuperAdmin returns true if the user has all admin permissions +// IsSuperAdmin returns true if the user holds every live admin permission func (p AdminPermission) IsSuperAdmin() bool { - return p == AllAdminPermissions + return p&LiveAdminPermissions == LiveAdminPermissions } // AdminRoleName represents predefined admin role names @@ -89,7 +109,7 @@ var AdminRolePermissions = map[AdminRoleName]AdminPermission{ AdminRoleSuper: AllAdminPermissions, AdminRoleSupport: AdminPermViewUsers | AdminPermViewCampaigns | AdminPermViewWarmupPool | AdminPermManageWarmupBans | AdminPermReviewAppeals | AdminPermViewAuditLogs | - AdminPermViewEnterpriseInquiries | AdminPermViewOrganizations, + AdminPermViewOrganizations, AdminRoleOps: AdminPermViewWorkers | AdminPermManageWorkers | AdminPermViewAnalytics | AdminPermViewAuditLogs | AdminPermManageRateLimits | AdminPermViewOrganizations, AdminRoleAnalyst: AdminPermViewUsers | AdminPermViewCampaigns | AdminPermViewAnalytics | @@ -112,14 +132,12 @@ type PermissionInfo struct { Category string `json:"category"` } -// GetAllPermissionInfos returns information about all admin permissions +// GetAllPermissionInfos returns information about all live admin permissions func GetAllPermissionInfos() []PermissionInfo { return []PermissionInfo{ // User Management {Name: "view_users", Permission: AdminPermViewUsers, Description: "View user profiles and details", Category: "User Management"}, {Name: "ban_users", Permission: AdminPermBanUsers, Description: "Ban and unban users", Category: "User Management"}, - {Name: "edit_users", Permission: AdminPermEditUsers, Description: "Edit user details", Category: "User Management"}, - {Name: "impersonate_users", Permission: AdminPermImpersonateUsers, Description: "Login as user for support", Category: "User Management"}, // Worker Management {Name: "view_workers", Permission: AdminPermViewWorkers, Description: "View worker list and status", Category: "Worker Management"}, @@ -142,12 +160,6 @@ func GetAllPermissionInfos() []PermissionInfo { {Name: "manage_rate_limits", Permission: AdminPermManageRateLimits, Description: "Manage user rate limits", Category: "Settings"}, {Name: "manage_settings", Permission: AdminPermManageSettings, Description: "Manage platform settings", Category: "Settings"}, - // Enterprise - {Name: "view_enterprise_inquiries", Permission: AdminPermViewEnterpriseInquiries, Description: "View enterprise inquiries", Category: "Enterprise"}, - {Name: "manage_enterprise_inquiries", Permission: AdminPermManageEnterpriseInquiries, Description: "Process enterprise inquiries", Category: "Enterprise"}, - {Name: "manage_plans", Permission: AdminPermManagePlans, Description: "Create and edit custom plans", Category: "Enterprise"}, - {Name: "manage_billing", Permission: AdminPermManageBilling, Description: "Manage refunds and billing", Category: "Enterprise"}, - // Super Admin {Name: "grant_admin_access", Permission: AdminPermGrantAdminAccess, Description: "Grant or revoke admin permissions", Category: "Super Admin"}, diff --git a/internal/repository/admin_schema_live_test.go b/internal/repository/admin_schema_live_test.go index e7f2d2a9..d38c45f8 100644 --- a/internal/repository/admin_schema_live_test.go +++ b/internal/repository/admin_schema_live_test.go @@ -85,28 +85,6 @@ func newAdminFixture(t *testing.T, pool *pgxpool.Pool) *adminFixture { return f } -// ensureDuration returns the id of the durations row with this title, creating -// it for the duration of the test if the instance does not have one. -func ensureDuration(t *testing.T, pool *pgxpool.Pool, title string) uuid.UUID { - t.Helper() - ctx := context.Background() - var id uuid.UUID - err := pool.QueryRow(ctx, `SELECT id FROM durations WHERE title = $1`, title).Scan(&id) - if err == nil { - return id - } - id = uuid.New() - if _, err := pool.Exec(ctx, `INSERT INTO durations (id, title) VALUES ($1, $2)`, id, title); err != nil { - t.Fatalf("create durations row %q: %v", title, err) - } - t.Cleanup(func() { - if _, err := pool.Exec(context.Background(), `DELETE FROM durations WHERE id = $1`, id); err != nil { - t.Errorf("cleanup durations row %q: %v", title, err) - } - }) - return id -} - // The email-account section of the user preview compared a uuid column against // a text parameter, so the query errored and the caller swallowed it: every // user looked like they had no mailboxes. @@ -220,82 +198,6 @@ func TestLiveAdminForceStopLeavesTerminalCampaignsAlone(t *testing.T) { } } -// plans stores the billing period as duration_id (FK to durations); the admin -// writes named a `duration` column that does not exist. -func TestLiveAdminPlanRoundTripsDuration(t *testing.T) { - _, pool := liveContactDB(t) - repo := NewAdminRepository(pool) - ctx := context.Background() - - // A bare install ships only the monthly duration (migration 000080); the - // yearly one arrives with the seed. Add whatever is missing and take it - // back out again, so this runs on either. - monthID := ensureDuration(t, pool, "month") - yearID := ensureDuration(t, pool, "year") - - resolved, err := repo.DurationIDByTitle(ctx, "month") - if err != nil { - t.Fatalf("DurationIDByTitle(month): %v", err) - } - if resolved == nil || *resolved != monthID { - t.Fatalf("DurationIDByTitle(month) = %v, want %v", resolved, monthID) - } - if unknown, err := repo.DurationIDByTitle(ctx, "fortnight"); err != nil || unknown != nil { - t.Fatalf("DurationIDByTitle(fortnight) = %v, %v; want nil, nil so the API can answer 400", unknown, err) - } - - name := "Issue 209 plan" - plan := &models.Plan{ - ID: uuid.New(), - Name: &name, - MaxContacts: 1000, - DailyEmails: 50, - AccountLimit: 3, - Price: 49, - Duration: models.DurationMonth, - MonthlyCredits: 25, - } - t.Cleanup(func() { - if _, err := pool.Exec(context.Background(), `DELETE FROM plans WHERE id = $1`, plan.ID); err != nil { - t.Errorf("cleanup plan: %v", err) - } - }) - - if err := repo.CreatePlan(ctx, plan, monthID); err != nil { - t.Fatalf("CreatePlan: %v", err) - } - - got, err := repo.GetPlan(ctx, plan.ID) - if err != nil { - t.Fatalf("GetPlan: %v", err) - } - if got == nil { - t.Fatal("GetPlan found nothing for a plan that was just created") - } - if got.Duration != models.DurationMonth { - t.Fatalf("plan reads back duration %q, want %q", got.Duration, models.DurationMonth) - } - if got.MonthlyCredits != 25 { - t.Fatalf("plan reads back %d monthly credits, want 25", got.MonthlyCredits) - } - - got.Duration = models.DurationYear - got.Price = 490 - if err := repo.UpdatePlan(ctx, got, yearID); err != nil { - t.Fatalf("UpdatePlan: %v", err) - } - after, err := repo.GetPlan(ctx, plan.ID) - if err != nil { - t.Fatalf("GetPlan after update: %v", err) - } - if after.Duration != models.DurationYear { - t.Fatalf("plan reads back duration %q after switching to yearly, want %q", after.Duration, models.DurationYear) - } - if after.Price != 490 { - t.Fatalf("plan reads back price %v, want 490", after.Price) - } -} - // user_rate_limits has limit_api_calls_daily / limit_bulk_ops_daily and no // daily_email_limit, so both the read and the write failed every time. func TestLiveAdminUserRateLimitsRoundTrip(t *testing.T) { diff --git a/internal/repository/pg_admin.go b/internal/repository/pg_admin.go index 3eb25d09..704ca377 100644 --- a/internal/repository/pg_admin.go +++ b/internal/repository/pg_admin.go @@ -66,25 +66,8 @@ type AdminRepository interface { GetPlatformOverview(ctx context.Context) (*models.PlatformOverview, error) GetDailyEmailStats(ctx context.Context, startDate, endDate time.Time) ([]models.DailyEmailStats, error) GetHourlyEmailStats(ctx context.Context, date time.Time) ([]models.HourlyEmailStats, error) - GetWorkerLoadStats(ctx context.Context) ([]models.WorkerLoadStats, error) GetUserGrowthStats(ctx context.Context, startDate, endDate time.Time) ([]models.UserGrowthStats, error) GetAnalyticsTrends(ctx context.Context) (*models.AnalyticsTrends, error) - GetEmailDistribution(ctx context.Context) ([]models.EmailDistribution, error) - - // Plans - ListPlans(ctx context.Context, includePrivate bool) ([]models.Plan, error) - SearchPlansForAdmin(ctx context.Context, search *models.AdminPlanSearch) (*models.AdminPlansResult, error) - DurationIDByTitle(ctx context.Context, title string) (*uuid.UUID, error) - CreatePlan(ctx context.Context, plan *models.Plan, durationID uuid.UUID) error - GetPlan(ctx context.Context, planID uuid.UUID) (*models.Plan, error) - UpdatePlan(ctx context.Context, plan *models.Plan, durationID uuid.UUID) error - DeletePlan(ctx context.Context, planID uuid.UUID) error - IsPlanInUse(ctx context.Context, planID uuid.UUID) (bool, error) - - // Enterprise Inquiries - ListEnterpriseInquiries(ctx context.Context, search *models.AdminEnterpriseInquirySearch) (*models.AdminEnterpriseInquiriesResult, error) - GetEnterpriseInquiry(ctx context.Context, id uuid.UUID) (*models.AdminEnterpriseInquiry, error) - UpdateEnterpriseInquiry(ctx context.Context, id uuid.UUID, update *models.UpdateEnterpriseInquiryRequest) error // User Rate Limits GetUserRateLimits(ctx context.Context, userID uuid.UUID) (*models.AdminUserRateLimits, error) @@ -1793,34 +1776,6 @@ func (r *adminRepository) GetHourlyEmailStats(ctx context.Context, date time.Tim return []models.HourlyEmailStats{}, nil } -// GetWorkerLoadStats gets worker load statistics -func (r *adminRepository) GetWorkerLoadStats(ctx context.Context) ([]models.WorkerLoadStats, error) { - query := ` - SELECT w.id, w.name, - (SELECT COUNT(*) FROM email_accounts ea WHERE ea.worker_id = w.id) as connected_emails - FROM workers w - WHERE w.active = true - ` - - rows, err := r.db.Query(ctx, query) - if err != nil { - return nil, err - } - defer rows.Close() - - var stats []models.WorkerLoadStats - for rows.Next() { - var s models.WorkerLoadStats - err := rows.Scan(&s.WorkerID, &s.WorkerName, &s.ConnectedEmails) - if err != nil { - return nil, err - } - stats = append(stats, s) - } - - return stats, nil -} - // GetUserGrowthStats gets user growth statistics func (r *adminRepository) GetUserGrowthStats(ctx context.Context, startDate, endDate time.Time) ([]models.UserGrowthStats, error) { query := ` @@ -1866,606 +1821,6 @@ func (r *adminRepository) GetAnalyticsTrends(ctx context.Context) (*models.Analy return trends, nil } -// GetEmailDistribution gets email distribution across workers -func (r *adminRepository) GetEmailDistribution(ctx context.Context) ([]models.EmailDistribution, error) { - query := ` - SELECT w.id, w.name, COUNT(ea.id) as email_count - FROM workers w - LEFT JOIN email_accounts ea ON ea.worker_id = w.id - GROUP BY w.id, w.name - ORDER BY email_count DESC - ` - - rows, err := r.db.Query(ctx, query) - if err != nil { - return nil, err - } - defer rows.Close() - - var total int64 - var distributions []models.EmailDistribution - for rows.Next() { - var d models.EmailDistribution - err := rows.Scan(&d.WorkerID, &d.WorkerName, &d.EmailCount) - if err != nil { - return nil, err - } - total += d.EmailCount - distributions = append(distributions, d) - } - - // Calculate percentages - for i := range distributions { - if total > 0 { - distributions[i].Percentage = float64(distributions[i].EmailCount) / float64(total) * 100 - } - } - - return distributions, nil -} - -// ListPlans lists all plans -func (r *adminRepository) ListPlans(ctx context.Context, includePrivate bool) ([]models.Plan, error) { - whereClause := "" - if !includePrivate { - whereClause = "WHERE p.public = true" - } - - query := ` - SELECT p.id, p.name, p.max_contacts, p.daily_emails, p.ai_generation, p.account_limit, - p.price, p.discounted_price, d.title, p.savings, p.public, - p.stripe_price_id, p.stripe_product_id, p.dedicated_workers, p.daily_campaign_limit, - p.max_campaigns, p.max_active_campaigns, p.max_team_members, p.max_email_accounts, - p.updated_at, p.created_at - FROM plans p - LEFT JOIN durations d ON d.id = p.duration_id - ` + whereClause + ` - ORDER BY p.price ASC - ` - - rows, err := r.db.Query(ctx, query) - if err != nil { - return nil, err - } - defer rows.Close() - - var plans []models.Plan - for rows.Next() { - var p models.Plan - var duration *string - err := rows.Scan( - &p.ID, &p.Name, &p.MaxContacts, &p.DailyEmails, &p.AIGeneration, &p.AccountLimit, - &p.Price, &p.DiscountedPrice, &duration, &p.Savings, &p.Public, - &p.StripePriceID, &p.StripeProductID, &p.DedicatedWorkers, &p.DailyCampaignLimit, - &p.MaxCampaigns, &p.MaxActiveCampaigns, &p.MaxTeamMembers, &p.MaxEmailAccounts, - &p.UpdatedAt, &p.CreatedAt, - ) - if err != nil { - return nil, err - } - if duration != nil { - p.Duration = models.Duration(*duration) - } - plans = append(plans, p) - } - - return plans, nil -} - -// SearchPlansForAdmin is the faceted + cursor-paginated plan catalog query. -// Mirrors SearchOrganizationsForAdmin. Plans is a small table so the pager is -// usually inert, but the {data,pagination} envelope keeps the Explorer stack -// uniform. -func (r *adminRepository) SearchPlansForAdmin(ctx context.Context, search *models.AdminPlanSearch) (*models.AdminPlansResult, error) { - limit := search.Limit - if limit <= 0 || limit > 100 { - limit = 50 - } - - args := []interface{}{} - argNum := 1 - where := "WHERE 1=1" - - if search.Query != "" { - where += ` AND (p.name ILIKE $` + itoa(argNum) + ` OR p.stripe_price_id ILIKE $` + itoa(argNum) + ` OR p.stripe_product_id ILIKE $` + itoa(argNum) + `)` - args = append(args, "%"+search.Query+"%") - argNum++ - } - switch search.Visibility { - case "public": - where += ` AND p.public = TRUE` - case "private": - where += ` AND p.public = FALSE` - } - if search.Duration != "" { - where += ` AND d.title = $` + itoa(argNum) - args = append(args, search.Duration) - argNum++ - } - if search.AIGeneration { - where += ` AND p.ai_generation = TRUE` - } - if search.HasStripe { - where += ` AND p.stripe_price_id IS NOT NULL AND p.stripe_price_id <> ''` - } - if search.HasSubscribers { - where += ` AND EXISTS (SELECT 1 FROM subscriptions s WHERE s.plan_id = p.id)` - } - - addInt := func(frag string, v *int) { - if v != nil { - where += " AND " + fmt.Sprintf(frag, argNum) - args = append(args, *v) - argNum++ - } - } - addAfter := func(col string, v *time.Time) { - if v != nil { - where += " AND " + col + " >= $" + itoa(argNum) - args = append(args, *v) - argNum++ - } - } - addBefore := func(col string, v *time.Time) { - if v != nil { - where += " AND " + col + " < ($" + itoa(argNum) + " + INTERVAL '1 day')" - args = append(args, *v) - argNum++ - } - } - - addInt(`p.price >= $%d`, search.PriceMin) - addInt(`p.price <= $%d`, search.PriceMax) - addInt(`p.daily_emails >= $%d`, search.DailyEmailsMin) - addInt(`p.daily_emails <= $%d`, search.DailyEmailsMax) - addInt(`p.account_limit >= $%d`, search.AccountLimitMin) - addInt(`p.account_limit <= $%d`, search.AccountLimitMax) - - if search.CreatedWithin > 0 { - where += ` AND p.created_at >= NOW() - ($` + itoa(argNum) + `::int * INTERVAL '1 day')` - args = append(args, search.CreatedWithin) - argNum++ - } - addAfter("p.created_at", search.CreatedAfter) - addBefore("p.created_at", search.CreatedBefore) - - if search.Cursor != nil { - where += ` AND p.id < $` + itoa(argNum) - args = append(args, *search.Cursor) - argNum++ - } - - orderCol := "p.price" - switch search.SortBy { - case "name": - orderCol = "p.name" - case "daily_emails": - orderCol = "p.daily_emails" - case "account_limit": - orderCol = "p.account_limit" - case "created_at": - orderCol = "p.created_at" - } - // Catalog defaults to cheapest-first (ASC); explicit sorts honor sort_desc. - orderDir := "ASC" - if search.SortBy != "" && search.SortDesc { - orderDir = "DESC" - } - orderBy := "ORDER BY " + orderCol + " " + orderDir + ", p.id DESC" - - args = append(args, limit+1) - - query := ` - SELECT p.id, p.name, p.max_contacts, p.daily_emails, p.ai_generation, p.account_limit, - p.price, p.discounted_price, d.title, p.savings, p.public, - p.stripe_price_id, p.stripe_product_id, p.dedicated_workers, p.daily_campaign_limit, - p.max_campaigns, p.max_active_campaigns, p.max_team_members, p.max_email_accounts, - p.updated_at, p.created_at - FROM plans p - LEFT JOIN durations d ON d.id = p.duration_id - ` + where + ` - ` + orderBy + ` - LIMIT $` + itoa(argNum) - - rows, err := r.db.Query(ctx, query, args...) - if err != nil { - return nil, err - } - defer rows.Close() - - plans := []models.Plan{} - for rows.Next() { - var p models.Plan - var duration *string - if err := rows.Scan( - &p.ID, &p.Name, &p.MaxContacts, &p.DailyEmails, &p.AIGeneration, &p.AccountLimit, - &p.Price, &p.DiscountedPrice, &duration, &p.Savings, &p.Public, - &p.StripePriceID, &p.StripeProductID, &p.DedicatedWorkers, &p.DailyCampaignLimit, - &p.MaxCampaigns, &p.MaxActiveCampaigns, &p.MaxTeamMembers, &p.MaxEmailAccounts, - &p.UpdatedAt, &p.CreatedAt, - ); err != nil { - return nil, err - } - if duration != nil { - p.Duration = models.Duration(*duration) - } - plans = append(plans, p) - } - - result := &models.AdminPlansResult{ - Data: plans, - Pagination: models.Pagination{HasMore: len(plans) > limit}, - } - if len(plans) > limit { - result.Data = plans[:limit] - last := plans[limit-1].ID - result.Pagination.NextCursor = paging.UUIDString(last) - } - - countQuery := `SELECT COUNT(*) FROM plans p LEFT JOIN durations d ON d.id = p.duration_id ` + where - var total int64 - if err := r.db.QueryRow(ctx, countQuery, args[:len(args)-1]...).Scan(&total); err == nil { - result.Pagination.Total = &total - } - - return result, nil -} - -// DurationIDByTitle resolves a durations.title to its id, or (nil, nil) when -// there is no such period, so the caller can answer 400 not a NOT NULL violation. -func (r *adminRepository) DurationIDByTitle(ctx context.Context, title string) (*uuid.UUID, error) { - var id uuid.UUID - err := r.db.QueryRow(ctx, `SELECT id FROM durations WHERE title = $1`, title).Scan(&id) - if errors.Is(err, pgx.ErrNoRows) { - return nil, nil - } - if err != nil { - return nil, err - } - return &id, nil -} - -// CreatePlan creates a new plan. The billing period is duration_id, an FK. -func (r *adminRepository) CreatePlan(ctx context.Context, plan *models.Plan, durationID uuid.UUID) error { - query := ` - INSERT INTO plans (id, name, max_contacts, daily_emails, ai_generation, account_limit, - price, discounted_price, duration_id, savings, public, dedicated_workers, daily_campaign_limit, - max_campaigns, max_active_campaigns, max_team_members, max_email_accounts, - monthly_credits, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20) - ` - now := time.Now() - _, err := r.db.Exec(ctx, query, - plan.ID, plan.Name, plan.MaxContacts, plan.DailyEmails, plan.AIGeneration, plan.AccountLimit, - plan.Price, plan.DiscountedPrice, durationID, plan.Savings, plan.Public, - plan.DedicatedWorkers, plan.DailyCampaignLimit, - plan.MaxCampaigns, plan.MaxActiveCampaigns, plan.MaxTeamMembers, plan.MaxEmailAccounts, - plan.MonthlyCredits, now, now, - ) - return err -} - -// GetPlan gets a plan by ID -func (r *adminRepository) GetPlan(ctx context.Context, planID uuid.UUID) (*models.Plan, error) { - query := ` - SELECT p.id, p.name, p.max_contacts, p.daily_emails, p.ai_generation, p.account_limit, - p.price, p.discounted_price, d.title, p.savings, p.public, - p.stripe_price_id, p.stripe_price_id_yearly, p.stripe_product_id, - p.dedicated_workers, p.daily_campaign_limit, - p.max_campaigns, p.max_active_campaigns, p.max_team_members, p.max_email_accounts, - p.monthly_credits, p.referral_reward_percent, p.updated_at, p.created_at - FROM plans p - LEFT JOIN durations d ON d.id = p.duration_id - WHERE p.id = $1 - ` - - var p models.Plan - var duration *string - err := r.db.QueryRow(ctx, query, planID).Scan( - &p.ID, &p.Name, &p.MaxContacts, &p.DailyEmails, &p.AIGeneration, &p.AccountLimit, - &p.Price, &p.DiscountedPrice, &duration, &p.Savings, &p.Public, - &p.StripePriceID, &p.StripePriceIDYearly, &p.StripeProductID, - &p.DedicatedWorkers, &p.DailyCampaignLimit, - &p.MaxCampaigns, &p.MaxActiveCampaigns, &p.MaxTeamMembers, &p.MaxEmailAccounts, - &p.MonthlyCredits, &p.ReferralRewardPercent, &p.UpdatedAt, &p.CreatedAt, - ) - if errors.Is(err, pgx.ErrNoRows) { - return nil, nil - } - if err != nil { - return nil, err - } - if duration != nil { - p.Duration = models.Duration(*duration) - } - return &p, nil -} - -// UpdatePlan updates a plan. -func (r *adminRepository) UpdatePlan(ctx context.Context, plan *models.Plan, durationID uuid.UUID) error { - query := ` - UPDATE plans SET name = $2, max_contacts = $3, daily_emails = $4, ai_generation = $5, - account_limit = $6, price = $7, discounted_price = $8, duration_id = $9, public = $10, - dedicated_workers = $11, daily_campaign_limit = $12, max_campaigns = $13, - max_active_campaigns = $14, max_team_members = $15, max_email_accounts = $16, - updated_at = $17 - WHERE id = $1 - ` - _, err := r.db.Exec(ctx, query, - plan.ID, plan.Name, plan.MaxContacts, plan.DailyEmails, plan.AIGeneration, - plan.AccountLimit, plan.Price, plan.DiscountedPrice, durationID, plan.Public, - plan.DedicatedWorkers, plan.DailyCampaignLimit, plan.MaxCampaigns, - plan.MaxActiveCampaigns, plan.MaxTeamMembers, plan.MaxEmailAccounts, - time.Now(), - ) - return err -} - -// DeletePlan deletes a plan -func (r *adminRepository) DeletePlan(ctx context.Context, planID uuid.UUID) error { - _, err := r.db.Exec(ctx, `DELETE FROM plans WHERE id = $1`, planID) - return err -} - -// IsPlanInUse checks if a plan is being used by any subscription -func (r *adminRepository) IsPlanInUse(ctx context.Context, planID uuid.UUID) (bool, error) { - var count int - err := r.db.QueryRow(ctx, `SELECT COUNT(*) FROM subscriptions WHERE plan_id = $1`, planID).Scan(&count) - return count > 0, err -} - -// ListEnterpriseInquiries lists enterprise inquiries with the shared faceted -// search params; mirrors SearchOrganizationsForAdmin (incremental WHERE builder, -// id keyset, LIMIT+1 has_more, separate COUNT). -func (r *adminRepository) ListEnterpriseInquiries(ctx context.Context, search *models.AdminEnterpriseInquirySearch) (*models.AdminEnterpriseInquiriesResult, error) { - limit := search.Limit - if limit <= 0 || limit > 100 { - limit = 50 - } - - args := []interface{}{} - argNum := 1 - whereClause := "WHERE 1=1" - - if search.Query != "" { - whereClause += " AND (ei.company_name ILIKE $" + itoa(argNum) + " OR ei.contact_name ILIKE $" + itoa(argNum) + " OR ei.contact_email ILIKE $" + itoa(argNum) + " OR COALESCE(u.email,'') ILIKE $" + itoa(argNum) + ")" - args = append(args, "%"+search.Query+"%") - argNum++ - } - if search.Status != "" && search.Status != "all" { - whereClause += " AND ei.status = $" + itoa(argNum) - args = append(args, search.Status) - argNum++ - } - switch search.Assignment { - case "assigned": - whereClause += " AND ei.assigned_to IS NOT NULL" - case "unassigned": - whereClause += " AND ei.assigned_to IS NULL" - } - switch search.Linkage { - case "linked": - whereClause += " AND ei.user_id IS NOT NULL" - case "anonymous": - whereClause += " AND ei.user_id IS NULL" - } - if search.HasNotes { - whereClause += " AND ei.notes IS NOT NULL AND ei.notes <> ''" - } - if search.HasPhone { - whereClause += " AND ei.phone IS NOT NULL AND ei.phone <> ''" - } - if search.Processed { - whereClause += " AND ei.processed_at IS NOT NULL" - } - - addInt := func(frag string, v *int) { - if v != nil { - whereClause += " AND " + fmt.Sprintf(frag, argNum) - args = append(args, *v) - argNum++ - } - } - addAfter := func(col string, v *time.Time) { - if v != nil { - whereClause += " AND " + col + " >= $" + itoa(argNum) - args = append(args, *v) - argNum++ - } - } - addBefore := func(col string, v *time.Time) { - if v != nil { - whereClause += " AND " + col + " < ($" + itoa(argNum) + " + INTERVAL '1 day')" - args = append(args, *v) - argNum++ - } - } - - addInt(`ei.team_size >= $%d`, search.TeamSizeMin) - addInt(`ei.team_size <= $%d`, search.TeamSizeMax) - addInt(`ei.estimated_volume >= $%d`, search.EstimatedVolumeMin) - addInt(`ei.estimated_volume <= $%d`, search.EstimatedVolumeMax) - - if search.CreatedWithin > 0 { - whereClause += " AND ei.created_at >= NOW() - ($" + itoa(argNum) + "::int * INTERVAL '1 day')" - args = append(args, search.CreatedWithin) - argNum++ - } - addAfter("ei.created_at", search.CreatedAfter) - addBefore("ei.created_at", search.CreatedBefore) - addAfter("COALESCE(ei.updated_at, ei.created_at)", search.UpdatedAfter) - addBefore("COALESCE(ei.updated_at, ei.created_at)", search.UpdatedBefore) - - if search.Cursor != nil { - whereClause += " AND ei.id < $" + itoa(argNum) - args = append(args, *search.Cursor) - argNum++ - } - - orderCol := "ei.created_at" - switch search.SortBy { - case "updated_at": - orderCol = "COALESCE(ei.updated_at, ei.created_at)" - case "company_name": - orderCol = "ei.company_name" - case "contact_email": - orderCol = "ei.contact_email" - case "status": - orderCol = "ei.status" - case "team_size": - orderCol = "ei.team_size" - case "estimated_volume": - orderCol = "ei.estimated_volume" - } - orderDir := "DESC" - if search.SortBy != "" && !search.SortDesc { - orderDir = "ASC" - } - orderBy := "ORDER BY " + orderCol + " " + orderDir + ", ei.id DESC" - - args = append(args, limit+1) - - query := ` - SELECT ei.id, ei.user_id, ei.company_name, ei.contact_name, ei.contact_email, - ei.phone, ei.team_size, ei.estimated_volume, ei.monthly_email_volume, ei.message, - ei.notes, ei.status, ei.assigned_to, - ei.created_at, COALESCE(ei.updated_at, ei.created_at) as updated_at, - u.id, u.first_name, u.last_name, u.email, - au.id, au.first_name, au.last_name, au.email - FROM enterprise_inquiries ei - LEFT JOIN users u ON u.id = ei.user_id - LEFT JOIN users au ON au.id = ei.assigned_to - ` + whereClause + ` - ` + orderBy + ` - LIMIT $` + itoa(argNum) - - rows, err := r.db.Query(ctx, query, args...) - if err != nil { - return nil, err - } - defer rows.Close() - - var inquiries []models.AdminEnterpriseInquiry - for rows.Next() { - var inq models.AdminEnterpriseInquiry - var teamSize *int // team_size is an integer column; the DTO exposes it as a string - var userID, assignedUserID *uuid.UUID - var userFirstName, userLastName, userEmail *string - var assignedFirstName, assignedLastName, assignedEmail *string - - err := rows.Scan( - &inq.ID, &inq.UserID, &inq.CompanyName, &inq.ContactName, &inq.ContactEmail, - &inq.Phone, &teamSize, &inq.EstimatedVolume, &inq.MonthlyEmailVolume, &inq.Message, - &inq.Notes, &inq.Status, &inq.AssignedTo, - &inq.CreatedAt, &inq.UpdatedAt, - &userID, &userFirstName, &userLastName, &userEmail, - &assignedUserID, &assignedFirstName, &assignedLastName, &assignedEmail, - ) - if err != nil { - return nil, err - } - - if teamSize != nil { - s := fmt.Sprintf("%d", *teamSize) - inq.TeamSize = &s - } - - if userID != nil { - inq.User = &models.AdminUserSummary{ - ID: *userID, - FirstName: *userFirstName, - LastName: *userLastName, - Email: *userEmail, - } - } - - if assignedUserID != nil { - inq.AssignedAdmin = &models.AdminUserSummary{ - ID: *assignedUserID, - FirstName: *assignedFirstName, - LastName: *assignedLastName, - Email: *assignedEmail, - } - } - - inquiries = append(inquiries, inq) - } - - result := &models.AdminEnterpriseInquiriesResult{ - Data: inquiries, - Pagination: models.Pagination{ - HasMore: len(inquiries) > limit, - }, - } - - if len(inquiries) > limit { - result.Data = inquiries[:limit] - lastID := inquiries[limit-1].ID - result.Pagination.NextCursor = paging.UUIDString(lastID) - } - - // Total count for the same filter — drop the trailing LIMIT arg. - countQuery := `SELECT COUNT(*) FROM enterprise_inquiries ei LEFT JOIN users u ON u.id = ei.user_id LEFT JOIN users au ON au.id = ei.assigned_to ` + whereClause - var total int64 - if err := r.db.QueryRow(ctx, countQuery, args[:len(args)-1]...).Scan(&total); err == nil { - result.Pagination.Total = &total - } - - return result, nil -} - -// GetEnterpriseInquiry gets a specific enterprise inquiry -func (r *adminRepository) GetEnterpriseInquiry(ctx context.Context, id uuid.UUID) (*models.AdminEnterpriseInquiry, error) { - query := ` - SELECT ei.id, ei.user_id, ei.company_name, ei.contact_name, ei.contact_email, - ei.phone, ei.team_size, ei.notes, ei.status, ei.assigned_to, - ei.created_at, COALESCE(ei.updated_at, ei.created_at) as updated_at - FROM enterprise_inquiries ei - WHERE ei.id = $1 - ` - - var inq models.AdminEnterpriseInquiry - err := r.db.QueryRow(ctx, query, id).Scan( - &inq.ID, &inq.UserID, &inq.CompanyName, &inq.ContactName, &inq.ContactEmail, - &inq.Phone, &inq.TeamSize, &inq.Notes, &inq.Status, &inq.AssignedTo, - &inq.CreatedAt, &inq.UpdatedAt, - ) - if err == pgx.ErrNoRows { - return nil, nil - } - if err != nil { - return nil, err - } - return &inq, nil -} - -// UpdateEnterpriseInquiry updates an enterprise inquiry -func (r *adminRepository) UpdateEnterpriseInquiry(ctx context.Context, id uuid.UUID, update *models.UpdateEnterpriseInquiryRequest) error { - setClauses := []string{"updated_at = NOW()"} - args := []interface{}{id} - argNum := 2 - - if update.Status != nil { - setClauses = append(setClauses, "status = $"+itoa(argNum)) - args = append(args, *update.Status) - argNum++ - } - if update.AssignedTo != nil { - setClauses = append(setClauses, "assigned_to = $"+itoa(argNum)) - args = append(args, *update.AssignedTo) - argNum++ - } - if update.Notes != nil { - setClauses = append(setClauses, "notes = $"+itoa(argNum)) - args = append(args, *update.Notes) - argNum++ - } - - query := "UPDATE enterprise_inquiries SET " + joinStrings(setClauses, ", ") + " WHERE id = $1" - _, err := r.db.Exec(ctx, query, args...) - return err -} - // GetUserRateLimits gets a user's API and realtime throughput limits. Mail // volume is not one of them: that is a per-mailbox budget and a plan entitlement. func (r *adminRepository) GetUserRateLimits(ctx context.Context, userID uuid.UUID) (*models.AdminUserRateLimits, error) { diff --git a/internal/repository/pg_admin_fleet.go b/internal/repository/pg_admin_fleet.go new file mode 100644 index 00000000..fce26cf4 --- /dev/null +++ b/internal/repository/pg_admin_fleet.go @@ -0,0 +1,171 @@ +package repository + +import ( + "context" + "sort" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/infrastructure/db" + "github.com/warmbly/warmbly/internal/models" +) + +// AdminFleetRepository is the operator's view of automatic placement: every +// worker with its capacity-view row, the decision log the control loops write, +// and the dedicated worker bindings. +type AdminFleetRepository interface { + // Capacity lists every worker (active or not) left-joined to + // worker_capacity_view, hottest first. + Capacity(ctx context.Context) ([]models.AdminFleetWorkerRow, error) + // Decisions lists decision_log newest first. kind "" means every kind; + // workerID nil means every worker. + Decisions(ctx context.Context, kind string, workerID *uuid.UUID, limit int) ([]models.AdminFleetDecision, error) + // DedicatedAssignments lists active bindings (released_at IS NULL) with the + // worker and workspace named and the worker's current mailbox count. + DedicatedAssignments(ctx context.Context) ([]models.AdminDedicatedAssignment, error) +} + +type adminFleetRepository struct { + db *db.DB +} + +func NewAdminFleetRepository(d *db.DB) AdminFleetRepository { + return &adminFleetRepository{db: d} +} + +// Capacity joins every worker row to the materialized capacity view; the view +// only carries active workers, so inactive ones come back with zeroed metrics. +func (r *adminFleetRepository) Capacity(ctx context.Context) ([]models.AdminFleetWorkerRow, error) { + rows, err := r.db.Query(ctx, ` + SELECT w.id, w.name, w.ip_addr, COALESCE(w.active, false), w.free_tier, w.worker_type, + w.risk_pool, w.egress_kind, w.health_state, w.install_state, + w.last_seen_at, + (COALESCE(w.active, false) AND w.last_seen_at > now() - $1::interval) AS live, + w.account_count, + COALESCE(t.tags, '{}'::text[]) AS tags, + COALESCE(v.load_score, w.load_score, 0)::float8, + COALESCE(v.base_capacity, 0)::float8, + COALESCE(v.health_multiplier, 1)::float8, + COALESCE(v.age_multiplier, 1)::float8, + COALESCE(v.sends_attempted_1h, 0), COALESCE(v.sends_succeeded_1h, 0), + COALESCE(v.bounces_hard_1h, 0), COALESCE(v.bounces_soft_1h, 0), + COALESCE(v.complaints_1h, 0), COALESCE(v.auth_errors_1h, 0) + FROM workers w + LEFT JOIN worker_capacity_view v ON v.worker_id = w.id + LEFT JOIN ( + SELECT worker_id, array_agg(tag::text ORDER BY tag) AS tags + FROM worker_tags GROUP BY worker_id + ) t ON t.worker_id = w.id + `, WorkerLivenessWindow) + if err != nil { + return nil, err + } + defer rows.Close() + + out := make([]models.AdminFleetWorkerRow, 0) + for rows.Next() { + var row models.AdminFleetWorkerRow + var live *bool + if err := rows.Scan( + &row.WorkerID, &row.Name, &row.IPAddr, &row.Active, &row.FreeTier, &row.WorkerType, + &row.RiskPool, &row.EgressKind, &row.HealthState, &row.InstallState, + &row.LastSeenAt, &live, &row.AccountCount, &row.Tags, + &row.LoadScore, &row.BaseCapacity, &row.HealthMultiplier, &row.AgeMultiplier, + &row.SendsAttempted1h, &row.SendsSucceeded1h, + &row.BouncesHard1h, &row.BouncesSoft1h, &row.Complaints1h, &row.AuthErrors1h, + ); err != nil { + return nil, err + } + // A NULL last_seen_at makes the AND NULL; that worker has never heartbeated. + row.Live = live != nil && *live + if row.Tags == nil { + row.Tags = []string{} + } + row.EffectiveCapacity = row.BaseCapacity * row.HealthMultiplier * row.AgeMultiplier + if row.EffectiveCapacity <= 0 { + row.EffectiveCapacity = 1 + } + row.Utilization = row.LoadScore / row.EffectiveCapacity + out = append(out, row) + } + if err := rows.Err(); err != nil { + return nil, err + } + sort.SliceStable(out, func(i, j int) bool { return out[i].Utilization > out[j].Utilization }) + return out, nil +} + +// Decisions lists decision_log newest first; kind and workerID are optional +// filters and limit is clamped to 1..500 with a default of 100. +func (r *adminFleetRepository) Decisions(ctx context.Context, kind string, workerID *uuid.UUID, limit int) ([]models.AdminFleetDecision, error) { + if limit <= 0 { + limit = 100 + } + if limit > 500 { + limit = 500 + } + rows, err := r.db.Query(ctx, ` + SELECT d.id, d.kind, d.worker_id, COALESCE(w.name, ''), d.mailbox_id, + d.before, d.after, COALESCE(d.reason, ''), COALESCE(d.triggered_by, ''), d.created_at + FROM decision_log d + LEFT JOIN workers w ON w.id = d.worker_id + WHERE ($1 = '' OR d.kind = $1) + AND ($2::uuid IS NULL OR d.worker_id = $2) + ORDER BY d.created_at DESC, d.id DESC + LIMIT $3 + `, kind, workerID, limit) + if err != nil { + return nil, err + } + defer rows.Close() + + out := make([]models.AdminFleetDecision, 0) + for rows.Next() { + var d models.AdminFleetDecision + var before, after []byte + if err := rows.Scan( + &d.ID, &d.Kind, &d.WorkerID, &d.WorkerName, &d.MailboxID, + &before, &after, &d.Reason, &d.TriggeredBy, &d.CreatedAt, + ); err != nil { + return nil, err + } + d.Before = before + d.After = after + out = append(out, d) + } + return out, rows.Err() +} + +// DedicatedAssignments lists active bindings with the worker's live mailbox +// count taken from email_accounts rather than the cached account_count. +func (r *adminFleetRepository) DedicatedAssignments(ctx context.Context) ([]models.AdminDedicatedAssignment, error) { + rows, err := r.db.Query(ctx, ` + SELECT a.id, a.worker_id, COALESCE(w.name, ''), + COALESCE(w.active AND w.last_seen_at > now() - $1::interval, false), + a.organization_id, COALESCE(o.name, ''), + a.subscription_id, a.assigned_at, a.released_at, + (SELECT count(*) FROM email_accounts e WHERE e.worker_id = a.worker_id) + FROM dedicated_worker_assignments a + LEFT JOIN workers w ON w.id = a.worker_id + LEFT JOIN organizations o ON o.id = a.organization_id + WHERE a.released_at IS NULL + ORDER BY a.assigned_at DESC + `, WorkerLivenessWindow) + if err != nil { + return nil, err + } + defer rows.Close() + + out := make([]models.AdminDedicatedAssignment, 0) + for rows.Next() { + var a models.AdminDedicatedAssignment + if err := rows.Scan( + &a.ID, &a.WorkerID, &a.WorkerName, &a.WorkerLive, + &a.OrganizationID, &a.OrganizationName, + &a.SubscriptionID, &a.AssignedAt, &a.ReleasedAt, &a.AccountCount, + ); err != nil { + return nil, err + } + out = append(out, a) + } + return out, rows.Err() +} diff --git a/internal/repository/pg_admin_insight.go b/internal/repository/pg_admin_insight.go new file mode 100644 index 00000000..986e8162 --- /dev/null +++ b/internal/repository/pg_admin_insight.go @@ -0,0 +1,293 @@ +package repository + +import ( + "context" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/infrastructure/db" + "github.com/warmbly/warmbly/internal/models" +) + +// AdminInsightRepository holds the smaller cross-workspace reads the panel +// shows on existing pages: warmup abuse signals, warmup admin actions, a +// workspace's API keys, workspace transfers and signup acquisition. +type AdminInsightRepository interface { + // WarmupAbuse ranks mailboxes by invalid warmup-token attempts since the + // given time, most attempts first. + WarmupAbuse(ctx context.Context, since time.Time, limit int) ([]models.AdminWarmupAbuseRow, error) + // WarmupActions lists warmup_admin_actions newest first. + WarmupActions(ctx context.Context, limit int) ([]models.AdminWarmupAction, error) + // OrgAPIKeys lists a workspace's API keys, active first, never the hash. + OrgAPIKeys(ctx context.Context, orgID uuid.UUID) ([]models.AdminOrgAPIKey, error) + // ListTransfers lists export and import jobs across every workspace, + // newest first, capped at limit. + ListTransfers(ctx context.Context, limit int) ([]models.AdminTransferJob, error) + // Acquisition groups signups from the last days by channel and reports + // how many converted to a paid subscription. + Acquisition(ctx context.Context, days int) (*models.AdminAcquisition, error) +} + +type adminInsightRepository struct { + db *db.DB +} + +func NewAdminInsightRepository(d *db.DB) AdminInsightRepository { + return &adminInsightRepository{db: d} +} + +func (r *adminInsightRepository) WarmupAbuse(ctx context.Context, since time.Time, limit int) ([]models.AdminWarmupAbuseRow, error) { + if limit <= 0 { + limit = 50 + } + // warmup_pool_participants is unique per mailbox (000097), so the left + // join adds at most one row. + const q = ` + SELECT a.email_account_id, ea.email, ea.organization_id, COALESCE(o.name, ''), + COUNT(*)::int, MAX(a.created_at), + COALESCE(p.blocked_at IS NOT NULL AND (p.blocked_until IS NULL OR p.blocked_until > now()), false), + COALESCE(p.spam_score, 0), COALESCE(p.health_state::text, '') + FROM warmup_invalid_token_attempts a + JOIN email_accounts ea ON ea.id = a.email_account_id + LEFT JOIN organizations o ON o.id = ea.organization_id + LEFT JOIN warmup_pool_participants p ON p.email_account_id = a.email_account_id + WHERE a.created_at >= $1 + GROUP BY a.email_account_id, ea.email, ea.organization_id, o.name, + p.blocked_at, p.blocked_until, p.spam_score, p.health_state + ORDER BY COUNT(*) DESC, MAX(a.created_at) DESC + LIMIT $2 + ` + rows, err := r.db.Query(ctx, q, since, limit) + if err != nil { + return nil, fmt.Errorf("admin insight: warmup abuse: %w", err) + } + defer rows.Close() + out := []models.AdminWarmupAbuseRow{} + for rows.Next() { + var row models.AdminWarmupAbuseRow + if err := rows.Scan( + &row.EmailAccountID, &row.Email, &row.OrganizationID, &row.OrganizationName, + &row.Attempts, &row.LastAttemptAt, &row.Blocked, &row.SpamScore, &row.HealthState, + ); err != nil { + return nil, fmt.Errorf("admin insight: warmup abuse scan: %w", err) + } + out = append(out, row) + } + return out, rows.Err() +} + +func (r *adminInsightRepository) WarmupActions(ctx context.Context, limit int) ([]models.AdminWarmupAction, error) { + if limit <= 0 { + limit = 100 + } + const q = ` + SELECT a.id, a.admin_user_id, COALESCE(u.email, ''), a.email_account_id, COALESCE(ea.email, ''), + a.action, a.reason, a.created_at + FROM warmup_admin_actions a + LEFT JOIN users u ON u.id = a.admin_user_id + LEFT JOIN email_accounts ea ON ea.id = a.email_account_id + ORDER BY a.created_at DESC, a.id DESC + LIMIT $1 + ` + rows, err := r.db.Query(ctx, q, limit) + if err != nil { + return nil, fmt.Errorf("admin insight: warmup actions: %w", err) + } + defer rows.Close() + out := []models.AdminWarmupAction{} + for rows.Next() { + var row models.AdminWarmupAction + if err := rows.Scan( + &row.ID, &row.AdminUserID, &row.AdminEmail, &row.EmailAccountID, &row.Email, + &row.Action, &row.Reason, &row.CreatedAt, + ); err != nil { + return nil, fmt.Errorf("admin insight: warmup actions scan: %w", err) + } + out = append(out, row) + } + return out, rows.Err() +} + +func (r *adminInsightRepository) OrgAPIKeys(ctx context.Context, orgID uuid.UUID) ([]models.AdminOrgAPIKey, error) { + const q = ` + SELECT k.id, k.name, k.key_prefix, k.key_suffix, k.status, k.permissions, + k.user_id, COALESCE(u.email, ''), k.last_used_at, k.expires_at, k.revoked_at, k.created_at, + (SELECT COUNT(*) FROM api_key_usage_logs l + WHERE l.api_key_id = k.id AND l.created_at >= now() - interval '7 days') + FROM api_keys k + LEFT JOIN users u ON u.id = k.user_id + WHERE k.organization_id = $1 + ORDER BY (k.status = 'active') DESC, k.created_at DESC + ` + rows, err := r.db.Query(ctx, q, orgID) + if err != nil { + return nil, fmt.Errorf("admin insight: org api keys: %w", err) + } + defer rows.Close() + out := []models.AdminOrgAPIKey{} + for rows.Next() { + var row models.AdminOrgAPIKey + if err := rows.Scan( + &row.ID, &row.Name, &row.KeyPrefix, &row.KeySuffix, &row.Status, &row.Permissions, + &row.UserID, &row.UserEmail, &row.LastUsedAt, &row.ExpiresAt, &row.RevokedAt, &row.CreatedAt, + &row.RequestsLast7d, + ); err != nil { + return nil, fmt.Errorf("admin insight: org api keys scan: %w", err) + } + out = append(out, row) + } + return out, rows.Err() +} + +func (r *adminInsightRepository) ListTransfers(ctx context.Context, limit int) ([]models.AdminTransferJob, error) { + if limit <= 0 { + limit = 100 + } + const q = ` + SELECT j.kind, j.id, j.organization_id, COALESCE(o.name, ''), j.requested_by, COALESCE(u.email, ''), + j.status, j.groups, j.include_secrets, j.progress_percent, j.progress_stage, + j.archive_bytes, j.error_message, j.started_at, j.completed_at, j.expires_at, j.created_at + FROM ( + SELECT 'export' AS kind, e.id, e.organization_id, e.requested_by, e.status, e.groups, + e.include_secrets, e.progress_percent::int AS progress_percent, e.progress_stage, + e.archive_bytes, e.error_message, e.started_at, e.completed_at, e.expires_at, e.created_at + FROM org_export_jobs e + UNION ALL + SELECT 'import', i.id, i.organization_id, i.requested_by, i.status, i.groups, + false, i.progress_percent::int, i.progress_stage, + i.archive_bytes, i.error_message, i.started_at, i.completed_at, NULL::timestamptz, i.created_at + FROM org_import_jobs i + ) j + LEFT JOIN organizations o ON o.id = j.organization_id + LEFT JOIN users u ON u.id = j.requested_by + ORDER BY j.created_at DESC, j.id DESC + LIMIT $1 + ` + rows, err := r.db.Query(ctx, q, limit) + if err != nil { + return nil, fmt.Errorf("admin insight: transfers: %w", err) + } + defer rows.Close() + out := []models.AdminTransferJob{} + for rows.Next() { + var ( + row models.AdminTransferJob + status string + groups []string + ) + if err := rows.Scan( + &row.Kind, &row.ID, &row.OrganizationID, &row.OrganizationName, &row.RequestedBy, &row.RequestedByEmail, + &status, &groups, &row.IncludeSecrets, &row.ProgressPercent, &row.ProgressStage, + &row.ArchiveBytes, &row.ErrorMessage, &row.StartedAt, &row.CompletedAt, &row.ExpiresAt, &row.CreatedAt, + ); err != nil { + return nil, fmt.Errorf("admin insight: transfers scan: %w", err) + } + row.Status = models.OrgTransferStatus(status) + row.Groups = make([]models.OrgDataGroup, 0, len(groups)) + for _, g := range groups { + row.Groups = append(row.Groups, models.OrgDataGroup(g)) + } + out = append(out, row) + } + return out, rows.Err() +} + +func (r *adminInsightRepository) Acquisition(ctx context.Context, days int) (*models.AdminAcquisition, error) { + if days <= 0 { + days = 30 + } + // Paid means a live Stripe subscription; trialing rows carry no Stripe id. + const cohort = ` + WITH cohort AS ( + SELECT o.id, + COALESCE(a.utm_source, '') AS source, + COALESCE(a.utm_medium, '') AS medium, + COALESCE(a.referrer_host, '') AS referrer, + EXISTS ( + SELECT 1 FROM subscriptions s + WHERE s.organization_id = o.id + AND s.status IN ('active', 'past_due') + AND s.stripe_subscription_id IS NOT NULL + ) AS converted + FROM organizations o + LEFT JOIN organization_acquisition a ON a.organization_id = o.id + WHERE o.created_at >= now() - ($1::int * interval '1 day') + ) + ` + out := &models.AdminAcquisition{ + Days: days, + Channels: []models.AdminAcquisitionChannel{}, + Referrers: []models.AdminAcquisitionReferrer{}, + } + + const totals = cohort + ` + SELECT COUNT(*)::int, COUNT(*) FILTER (WHERE source <> '' OR medium <> '')::int, COUNT(*) FILTER (WHERE converted)::int + FROM cohort + ` + if err := r.db.QueryRow(ctx, totals, days).Scan(&out.Signups, &out.WithChannel, &out.Converted); err != nil { + return nil, fmt.Errorf("admin insight: acquisition totals: %w", err) + } + + // Same predicate the trial expiration job uses, looking forward instead. + const trials = ` + SELECT COUNT(*)::int FROM subscriptions + WHERE free_trial_ends_at IS NOT NULL + AND free_trial_ends_at BETWEEN now() AND now() + interval '7 days' + AND stripe_subscription_id IS NULL + AND status NOT IN ('canceled', 'incomplete_expired') + ` + if err := r.db.QueryRow(ctx, trials).Scan(&out.TrialsExpiring7d); err != nil { + return nil, fmt.Errorf("admin insight: acquisition trials: %w", err) + } + + const channels = cohort + ` + SELECT source, medium, COUNT(*)::int, COUNT(*) FILTER (WHERE converted)::int + FROM cohort + WHERE NOT (source = '' AND medium = '') + GROUP BY source, medium + ORDER BY COUNT(*) DESC, source, medium + ` + rows, err := r.db.Query(ctx, channels, days) + if err != nil { + return nil, fmt.Errorf("admin insight: acquisition channels: %w", err) + } + for rows.Next() { + var ch models.AdminAcquisitionChannel + if err := rows.Scan(&ch.Source, &ch.Medium, &ch.Signups, &ch.Converted); err != nil { + rows.Close() + return nil, fmt.Errorf("admin insight: acquisition channels scan: %w", err) + } + out.Channels = append(out.Channels, ch) + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("admin insight: acquisition channels: %w", err) + } + + const referrers = cohort + ` + SELECT referrer, COUNT(*)::int + FROM cohort + WHERE referrer <> '' + GROUP BY referrer + ORDER BY COUNT(*) DESC, referrer + LIMIT 10 + ` + rows, err = r.db.Query(ctx, referrers, days) + if err != nil { + return nil, fmt.Errorf("admin insight: acquisition referrers: %w", err) + } + for rows.Next() { + var ref models.AdminAcquisitionReferrer + if err := rows.Scan(&ref.Host, &ref.Signups); err != nil { + rows.Close() + return nil, fmt.Errorf("admin insight: acquisition referrers scan: %w", err) + } + out.Referrers = append(out.Referrers, ref) + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("admin insight: acquisition referrers: %w", err) + } + return out, nil +} diff --git a/internal/repository/pg_admin_sends.go b/internal/repository/pg_admin_sends.go new file mode 100644 index 00000000..616cc7d5 --- /dev/null +++ b/internal/repository/pg_admin_sends.go @@ -0,0 +1,326 @@ +package repository + +import ( + "context" + "encoding/json" + "errors" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/warmbly/warmbly/internal/infrastructure/db" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/utils/paging" +) + +// AdminSendsRepository reads the send outcome loop and the task queues across +// every workspace: reservations no worker answered, dead letters, task +// failures and customer webhook delivery health. +type AdminSendsRepository interface { + // InFlight lists reserved sends without a result, oldest first, capped at + // limit, with the age buckets computed against reclaimAfter. + InFlight(ctx context.Context, reclaimAfter time.Duration, limit int) (*models.AdminInFlightResult, error) + // ListDeadLetters pages task_dead_letters newest first. status "" means + // every status; cursor is the id of the last row of the previous page. + ListDeadLetters(ctx context.Context, status string, cursor *uuid.UUID, limit int) (*models.AdminDeadLettersResult, error) + GetDeadLetter(ctx context.Context, id uuid.UUID) (*models.AdminDeadLetterRow, error) + // RecentTaskFailures lists the newest task_failures rows with their task + // and mailbox, capped at limit. + RecentTaskFailures(ctx context.Context, limit int) ([]models.AdminTaskFailureRow, error) + // WebhookHealth is the instance-wide delivery picture; a delivery is stale + // when it has been in_flight longer than lease. + WebhookHealth(ctx context.Context, lease time.Duration) (*models.AdminWebhookHealth, error) + // OrgWebhooks lists one workspace's endpoints with their recent delivery + // and drop counts. + OrgWebhooks(ctx context.Context, orgID uuid.UUID) ([]models.AdminWebhookEndpointRow, error) +} + +type adminSendsRepository struct { + db *db.DB +} + +func NewAdminSendsRepository(d *db.DB) AdminSendsRepository { + return &adminSendsRepository{db: d} +} + +func (r *adminSendsRepository) InFlight(ctx context.Context, reclaimAfter time.Duration, limit int) (*models.AdminInFlightResult, error) { + if limit <= 0 { + limit = 100 + } + if limit > 500 { + limit = 500 + } + reclaimSecs := reclaimAfter.Seconds() + out := &models.AdminInFlightResult{ + Summary: models.AdminInFlightSummary{ReclaimAfterMinutes: int(reclaimAfter / time.Minute)}, + Data: []models.AdminInFlightSend{}, + } + + // The summary counts every reservation, not just the page. + err := r.db.QueryRow(ctx, ` + SELECT COUNT(*), + COUNT(*) FILTER (WHERE dispatched_at > NOW() - INTERVAL '5 minutes'), + COUNT(*) FILTER (WHERE dispatched_at <= NOW() - INTERVAL '5 minutes' + AND dispatched_at > NOW() - INTERVAL '30 minutes'), + COUNT(*) FILTER (WHERE dispatched_at <= NOW() - make_interval(secs => $1)), + MIN(dispatched_at) + FROM campaign_contact_progress + WHERE sent_at IS NULL AND dispatched_at IS NOT NULL + `, reclaimSecs).Scan( + &out.Summary.Total, &out.Summary.Under5m, &out.Summary.Under30m, + &out.Summary.PastReclaimWindow, &out.Summary.OldestDispatched, + ) + if err != nil { + return nil, err + } + + rows, err := r.db.Query(ctx, ` + SELECT p.campaign_id, COALESCE(c.name, ''), c.organization_id, COALESCE(o.name, ''), + p.contact_id, COALESCE(ct.email, ''), p.sequence_id, + p.dispatch_task_id, COALESCE(t.status::text, ''), COALESCE(t.message_id <> '', false), + t.email_account_id, COALESCE(ea.email, ''), ea.worker_id, + p.dispatched_at + FROM campaign_contact_progress p + JOIN campaigns c ON c.id = p.campaign_id + LEFT JOIN organizations o ON o.id = c.organization_id + LEFT JOIN contacts ct ON ct.id = p.contact_id + LEFT JOIN tasks t ON t.id = p.dispatch_task_id + LEFT JOIN email_accounts ea ON ea.id = t.email_account_id + WHERE p.sent_at IS NULL AND p.dispatched_at IS NOT NULL + ORDER BY p.dispatched_at ASC + LIMIT $1 + `, limit) + if err != nil { + return nil, err + } + defer rows.Close() + now := time.Now() + for rows.Next() { + var s models.AdminInFlightSend + if err := rows.Scan( + &s.CampaignID, &s.CampaignName, &s.OrganizationID, &s.OrganizationName, + &s.ContactID, &s.ContactEmail, &s.SequenceID, + &s.TaskID, &s.TaskStatus, &s.HasMessageID, + &s.EmailAccountID, &s.MailboxEmail, &s.WorkerID, + &s.DispatchedAt, + ); err != nil { + return nil, err + } + s.AgeSeconds = int64(now.Sub(s.DispatchedAt).Seconds()) + out.Data = append(out.Data, s) + } + return out, rows.Err() +} + +// deadLetterColumns is the select list shared by the list and get queries; a +// dead letter reaches its workspace through the task's mailbox. +const deadLetterColumns = ` + SELECT d.id, d.task_id, d.task_type, d.payload, d.last_error, d.attempts, d.max_attempts, + d.status, d.next_retry_at, d.replayed_at, d.created_at, d.updated_at, + ea.organization_id, COALESCE(o.name, '') + FROM task_dead_letters d + LEFT JOIN tasks t ON t.id = d.task_id + LEFT JOIN email_accounts ea ON ea.id = t.email_account_id + LEFT JOIN organizations o ON o.id = ea.organization_id` + +func scanDeadLetter(row pgx.Row) (*models.AdminDeadLetterRow, error) { + var d models.AdminDeadLetterRow + var payload []byte + if err := row.Scan( + &d.ID, &d.TaskID, &d.TaskType, &payload, &d.LastError, &d.Attempts, &d.MaxAttempts, + &d.Status, &d.NextRetryAt, &d.ReplayedAt, &d.CreatedAt, &d.UpdatedAt, + &d.OrganizationID, &d.OrganizationName, + ); err != nil { + return nil, err + } + if len(payload) > 0 { + _ = json.Unmarshal(payload, &d.Payload) + } + if d.Payload == nil { + d.Payload = map[string]interface{}{} + } + return &d, nil +} + +func (r *adminSendsRepository) ListDeadLetters(ctx context.Context, status string, cursor *uuid.UUID, limit int) (*models.AdminDeadLettersResult, error) { + if limit <= 0 || limit > 200 { + limit = 50 + } + out := &models.AdminDeadLettersResult{ + Data: []models.AdminDeadLetterRow{}, + Pagination: &models.Pagination{}, + } + if err := r.db.QueryRow(ctx, ` + SELECT COUNT(*) FILTER (WHERE status = 'pending'), + COUNT(*) FILTER (WHERE status = 'replayed'), + COUNT(*) FILTER (WHERE status = 'failed') + FROM task_dead_letters + `).Scan(&out.Pending, &out.Replayed, &out.Failed); err != nil { + return nil, err + } + + rows, err := r.db.Query(ctx, deadLetterColumns+` + WHERE ($1 = '' OR d.status = $1) + AND ($2::uuid IS NULL OR (d.created_at, d.id) < ( + SELECT created_at, id FROM task_dead_letters WHERE id = $2)) + ORDER BY d.created_at DESC, d.id DESC + LIMIT $3 + `, status, cursor, limit+1) + if err != nil { + return nil, err + } + defer rows.Close() + for rows.Next() { + d, err := scanDeadLetter(rows) + if err != nil { + return nil, err + } + out.Data = append(out.Data, *d) + } + if err := rows.Err(); err != nil { + return nil, err + } + if len(out.Data) > limit { + out.Data = out.Data[:limit] + out.Pagination.HasMore = true + out.Pagination.NextCursor = paging.UUIDString(out.Data[limit-1].ID) + } + return out, nil +} + +func (r *adminSendsRepository) GetDeadLetter(ctx context.Context, id uuid.UUID) (*models.AdminDeadLetterRow, error) { + d, err := scanDeadLetter(r.db.QueryRow(ctx, deadLetterColumns+` WHERE d.id = $1`, id)) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + return nil, err + } + return d, nil +} + +// RecentTaskFailures orders by the task's updated_at: task_failures carries no +// timestamp of its own and the failure is the last write to the task. +func (r *adminSendsRepository) RecentTaskFailures(ctx context.Context, limit int) ([]models.AdminTaskFailureRow, error) { + if limit <= 0 || limit > 500 { + limit = 100 + } + rows, err := r.db.Query(ctx, ` + SELECT f.task_id, t.task_type::text, t.status::text, f.title, f.message, + t.email_account_id, COALESCE(ea.email, ''), ea.organization_id, COALESCE(o.name, ''), + t.updated_at + FROM task_failures f + JOIN tasks t ON t.id = f.task_id + LEFT JOIN email_accounts ea ON ea.id = t.email_account_id + LEFT JOIN organizations o ON o.id = ea.organization_id + ORDER BY t.updated_at DESC + LIMIT $1 + `, limit) + if err != nil { + return nil, err + } + defer rows.Close() + out := make([]models.AdminTaskFailureRow, 0, limit) + for rows.Next() { + var f models.AdminTaskFailureRow + if err := rows.Scan( + &f.TaskID, &f.TaskType, &f.TaskStatus, &f.Title, &f.Message, + &f.EmailAccountID, &f.MailboxEmail, &f.OrganizationID, &f.OrganizationName, + &f.OccurredAt, + ); err != nil { + return nil, err + } + out = append(out, f) + } + return out, rows.Err() +} + +// webhookEndpointColumns is the endpoint row with its 7-day counters. Drops +// are rolled up per (organization, event_type), so an endpoint's drops are +// the org's drops on the event types it subscribes to (all, when unfiltered). +const webhookEndpointColumns = ` + SELECT e.id, e.organization_id, COALESCE(o.name, ''), e.url, e.description, e.enabled, + e.event_types, e.consecutive_failures, e.last_success_at, e.last_failure_at, + COALESCE(e.last_failure_reason, ''), + (SELECT COUNT(*) FROM webhook_deliveries d + WHERE d.endpoint_id = e.id AND d.created_at >= NOW() - INTERVAL '7 days'), + (SELECT COUNT(*) FROM webhook_deliveries d + WHERE d.endpoint_id = e.id AND d.created_at >= NOW() - INTERVAL '7 days' + AND d.status IN ('failed', 'abandoned')), + (SELECT COALESCE(SUM(wd.dropped_windows), 0) FROM webhook_event_drops wd + WHERE wd.organization_id = e.organization_id AND wd.day >= CURRENT_DATE - 7 + AND (cardinality(e.event_types) = 0 OR wd.event_type = ANY(e.event_types))) + FROM webhook_endpoints e + LEFT JOIN organizations o ON o.id = e.organization_id` + +func scanWebhookEndpoints(rows pgx.Rows) ([]models.AdminWebhookEndpointRow, error) { + defer rows.Close() + out := []models.AdminWebhookEndpointRow{} + for rows.Next() { + var e models.AdminWebhookEndpointRow + if err := rows.Scan( + &e.ID, &e.OrganizationID, &e.OrganizationName, &e.URL, &e.Description, &e.Enabled, + &e.EventTypes, &e.ConsecutiveFailures, &e.LastSuccessAt, &e.LastFailureAt, + &e.LastFailureReason, &e.DeliveriesLast7d, &e.FailedLast7d, &e.DropsLast7d, + ); err != nil { + return nil, err + } + if e.EventTypes == nil { + e.EventTypes = []string{} + } + out = append(out, e) + } + return out, rows.Err() +} + +// WebhookHealth calls a delivery stale by updated_at, which is what both the +// claim and ReclaimStuckDeliveries use, so the count is what a reclaim sweeps. +func (r *adminSendsRepository) WebhookHealth(ctx context.Context, lease time.Duration) (*models.AdminWebhookHealth, error) { + out := &models.AdminWebhookHealth{ + LeaseMinutes: int(lease / time.Minute), + FailingEndpoints: []models.AdminWebhookEndpointRow{}, + } + if err := r.db.QueryRow(ctx, ` + SELECT COUNT(*) FILTER (WHERE status = 'in_flight' AND updated_at < NOW() - make_interval(secs => $1)), + COUNT(*) FILTER (WHERE status = 'pending' AND next_attempt_at <= NOW()), + COUNT(*) FILTER (WHERE status = 'delivered' AND updated_at >= NOW() - INTERVAL '24 hours'), + COUNT(*) FILTER (WHERE status = 'failed' AND updated_at >= NOW() - INTERVAL '24 hours'), + COUNT(*) FILTER (WHERE status = 'abandoned' AND updated_at >= NOW() - INTERVAL '24 hours') + FROM webhook_deliveries + `, lease.Seconds()).Scan( + &out.InFlightStale, &out.PendingDue, &out.DeliveredLast24h, &out.FailedLast24h, &out.AbandonedLast24h, + ); err != nil { + return nil, err + } + if err := r.db.QueryRow(ctx, ` + SELECT COALESCE(SUM(dropped_windows), 0) FROM webhook_event_drops WHERE day >= CURRENT_DATE - 7 + `).Scan(&out.DropsLast7d); err != nil { + return nil, err + } + + rows, err := r.db.Query(ctx, webhookEndpointColumns+` + WHERE e.consecutive_failures > 0 + ORDER BY e.consecutive_failures DESC, e.last_failure_at DESC NULLS LAST + LIMIT 50 + `) + if err != nil { + return nil, err + } + failing, err := scanWebhookEndpoints(rows) + if err != nil { + return nil, err + } + out.FailingEndpoints = failing + return out, nil +} + +func (r *adminSendsRepository) OrgWebhooks(ctx context.Context, orgID uuid.UUID) ([]models.AdminWebhookEndpointRow, error) { + rows, err := r.db.Query(ctx, webhookEndpointColumns+` + WHERE e.organization_id = $1 + ORDER BY e.created_at DESC + `, orgID) + if err != nil { + return nil, err + } + return scanWebhookEndpoints(rows) +} diff --git a/internal/repository/pg_admin_sync.go b/internal/repository/pg_admin_sync.go new file mode 100644 index 00000000..07a21910 --- /dev/null +++ b/internal/repository/pg_admin_sync.go @@ -0,0 +1,211 @@ +package repository + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/infrastructure/db" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/utils/paging" +) + +// AdminSyncRepository is the operator's cross-workspace view of mailbox sync: +// every email_sync_state row joined to its mailbox, owner and workspace. +type AdminSyncRepository interface { + // Search lists sync rows newest-updated first, filtered by state and a + // free-text match on the mailbox address or workspace name, and returns + // the instance-wide summary alongside (the summary ignores the filter). + Search(ctx context.Context, search *models.AdminSyncSearch) (*models.AdminSyncResult, error) + // ClearThrottle lifts a fair-use throttle on the platform copy. False when + // the mailbox has no sync row or was not throttled. + ClearThrottle(ctx context.Context, emailID uuid.UUID) (bool, error) + // ResetBackfill puts the backfill back to pending with an empty cursor and + // zero progress, so the next load re-imports history from scratch. False + // when the mailbox has no sync row. + ResetBackfill(ctx context.Context, emailID uuid.UUID) (bool, error) +} + +type adminSyncRepository struct { + db *db.DB +} + +func NewAdminSyncRepository(d *db.DB) AdminSyncRepository { + return &adminSyncRepository{db: d} +} + +// Sentinel errors the handler maps to 400. +var ( + ErrAdminSyncBadCursor = errors.New("admin sync: invalid cursor") + ErrAdminSyncBadState = errors.New("admin sync: invalid state filter") +) + +const ( + adminSyncDefaultLimit = 50 + adminSyncMaxLimit = 200 +) + +// adminSyncStateWhere is the per-row predicate for each state filter. Expired +// throttles are not throttles, mirroring EmailSyncStateRepository.Get. +func adminSyncStateWhere(state string) (string, error) { + switch state { + case "", "all": + return "", nil + case "throttled": + return "s.throttled_until > now()", nil + case "backfilling": + return "s.backfill_status = 'running'", nil + case "stalled": + return "s.backfill_status = 'running' AND s.updated_at < now() - interval '1 hour'", nil + case "pending": + return "s.backfill_status = 'pending'", nil + case "complete": + return "s.backfill_status = 'complete'", nil + } + return "", ErrAdminSyncBadState +} + +func (r *adminSyncRepository) Search(ctx context.Context, search *models.AdminSyncSearch) (*models.AdminSyncResult, error) { + if search == nil { + search = &models.AdminSyncSearch{} + } + limit := search.Limit + if limit <= 0 { + limit = adminSyncDefaultLimit + } + if limit > adminSyncMaxLimit { + limit = adminSyncMaxLimit + } + + where := "WHERE TRUE" + args := []any{} + if cond, err := adminSyncStateWhere(search.State); err != nil { + return nil, err + } else if cond != "" { + where += " AND " + cond + } + if q := strings.TrimSpace(search.Q); q != "" { + args = append(args, "%"+q+"%") + n := itoa(len(args)) + where += " AND (ea.email ILIKE $" + n + " OR o.name ILIKE $" + n + ")" + } + if search.Cursor != "" { + // The token carries the boundary itself, so a row that moves or is + // deleted between pages cannot shift or empty the next page. + at, id, xerr := paging.DecodeTimeCursor(search.Cursor) + if xerr != nil { + return nil, ErrAdminSyncBadCursor + } + args = append(args, at, id) + n := len(args) + where += " AND (s.updated_at, s.email_id) < ($" + itoa(n-1) + "::timestamptz, $" + itoa(n) + "::uuid)" + } + args = append(args, limit+1) + + query := ` + SELECT s.email_id, s.user_id, ea.email, ea.provider::text, ea.status::text, + ea.organization_id, COALESCE(o.name, ''), ea.worker_id, + s.backfill_status, s.backfill_synced, s.backfill_since, + s.backfill_started_at, s.backfill_completed_at, + CASE WHEN s.throttled_until > now() THEN s.throttled_until END, + CASE WHEN s.throttled_until > now() THEN s.throttle_reason ELSE '' END, + s.deferred, + (s.backfill_status = 'running' AND s.updated_at < now() - interval '1 hour'), + s.last_synced_at, s.updated_at + FROM email_sync_state s + JOIN email_accounts ea ON ea.id = s.email_id + LEFT JOIN organizations o ON o.id = ea.organization_id + ` + where + ` + ORDER BY s.updated_at DESC, s.email_id DESC + LIMIT $` + itoa(len(args)) + + rows, err := r.db.Query(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("admin sync: search: %w", err) + } + defer rows.Close() + + items := []models.AdminSyncRow{} + for rows.Next() { + var row models.AdminSyncRow + if err := rows.Scan( + &row.EmailID, &row.UserID, &row.Email, &row.Provider, &row.AccountStatus, + &row.OrganizationID, &row.OrganizationName, &row.WorkerID, + &row.BackfillStatus, &row.BackfillSynced, &row.BackfillSince, + &row.BackfillStartedAt, &row.BackfillCompletedAt, + &row.ThrottledUntil, &row.ThrottleReason, + &row.Deferred, &row.Stalled, + &row.LastSyncedAt, &row.UpdatedAt, + ); err != nil { + return nil, fmt.Errorf("admin sync: scan: %w", err) + } + items = append(items, row) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("admin sync: rows: %w", err) + } + + result := &models.AdminSyncResult{ + Data: items, + Pagination: &models.Pagination{HasMore: len(items) > limit}, + } + if len(items) > limit { + result.Data = items[:limit] + result.Pagination.NextCursor = paging.EncodeTime(items[limit-1].UpdatedAt, items[limit-1].EmailID) + } + + const summary = ` + SELECT COUNT(*), + COUNT(*) FILTER (WHERE throttled_until > now()), + COUNT(*) FILTER (WHERE backfill_status = 'running'), + COUNT(*) FILTER (WHERE backfill_status = 'running' AND updated_at < now() - interval '1 hour'), + COUNT(*) FILTER (WHERE backfill_status = 'pending'), + COUNT(*) FILTER (WHERE backfill_status = 'complete'), + COALESCE(SUM(deferred), 0) + FROM email_sync_state + ` + var total, throttled, backfilling, stalled, pending, complete, deferred int64 + if err := r.db.QueryRow(ctx, summary).Scan(&total, &throttled, &backfilling, &stalled, &pending, &complete, &deferred); err != nil { + return nil, fmt.Errorf("admin sync: summary: %w", err) + } + result.Summary = models.AdminSyncSummary{ + Total: int(total), + Throttled: int(throttled), + Backfilling: int(backfilling), + Stalled: int(stalled), + Pending: int(pending), + Complete: int(complete), + Deferred: int(deferred), + } + return result, nil +} + +func (r *adminSyncRepository) ClearThrottle(ctx context.Context, emailID uuid.UUID) (bool, error) { + const q = ` + UPDATE email_sync_state + SET throttled_until = NULL, throttle_reason = '', updated_at = now() + WHERE email_id = $1 AND throttled_until > now() + ` + tag, err := r.db.Exec(ctx, q, emailID) + if err != nil { + return false, fmt.Errorf("admin sync: clear throttle: %w", err) + } + return tag.RowsAffected() > 0, nil +} + +func (r *adminSyncRepository) ResetBackfill(ctx context.Context, emailID uuid.UUID) (bool, error) { + const q = ` + UPDATE email_sync_state + SET backfill_status = 'pending', backfill_cursor = '{}'::jsonb, backfill_synced = 0, + backfill_since = NULL, backfill_started_at = NULL, backfill_completed_at = NULL, + updated_at = now() + WHERE email_id = $1 + ` + tag, err := r.db.Exec(ctx, q, emailID) + if err != nil { + return false, fmt.Errorf("admin sync: reset backfill: %w", err) + } + return tag.RowsAffected() > 0, nil +} diff --git a/internal/repository/pg_job_runs.go b/internal/repository/pg_job_runs.go new file mode 100644 index 00000000..e603bf59 --- /dev/null +++ b/internal/repository/pg_job_runs.go @@ -0,0 +1,120 @@ +package repository + +import ( + "context" + "time" + + "github.com/warmbly/warmbly/internal/infrastructure/db" + "github.com/warmbly/warmbly/internal/models" +) + +// JobRunRepository persists scheduled_job_runs, one row per background loop. +// It is the jobrun.Store the backend and consumer record to. +type JobRunRepository interface { + Register(ctx context.Context, name, service string, interval time.Duration, nextRunAt time.Time) error + MarkStarted(ctx context.Context, name string, at time.Time) error + MarkFinished(ctx context.Context, name string, startedAt, finishedAt time.Time, runErr error, nextRunAt time.Time) error + RequestRun(ctx context.Context, name string) (bool, error) + TakeRunRequest(ctx context.Context, name string) (bool, error) + List(ctx context.Context) ([]models.ScheduledJobRun, error) +} + +type jobRunRepository struct { + db *db.DB +} + +func NewJobRunRepository(d *db.DB) JobRunRepository { + return &jobRunRepository{db: d} +} + +func (r *jobRunRepository) Register(ctx context.Context, name, service string, interval time.Duration, nextRunAt time.Time) error { + _, err := r.db.Exec(ctx, ` + INSERT INTO scheduled_job_runs (name, service, interval_seconds, next_run_at, updated_at) + VALUES ($1, $2, $3, $4, now()) + ON CONFLICT (name) DO UPDATE SET + service = EXCLUDED.service, + interval_seconds = EXCLUDED.interval_seconds, + next_run_at = EXCLUDED.next_run_at, + last_status = CASE WHEN scheduled_job_runs.last_status = 'running' THEN 'idle' ELSE scheduled_job_runs.last_status END, + updated_at = now() + `, name, service, int(interval.Seconds()), nextRunAt) + return err +} + +func (r *jobRunRepository) MarkStarted(ctx context.Context, name string, at time.Time) error { + _, err := r.db.Exec(ctx, ` + UPDATE scheduled_job_runs + SET last_started_at = $2, last_status = 'running', updated_at = now() + WHERE name = $1 + `, name, at) + return err +} + +func (r *jobRunRepository) MarkFinished(ctx context.Context, name string, startedAt, finishedAt time.Time, runErr error, nextRunAt time.Time) error { + status, msg := "ok", "" + if runErr != nil { + status, msg = "error", runErr.Error() + if len(msg) > 2000 { + msg = msg[:2000] + } + } + _, err := r.db.Exec(ctx, ` + UPDATE scheduled_job_runs + SET last_finished_at = $2, + last_duration_ms = $3, + last_status = $4, + last_error = $5, + run_count = run_count + 1, + error_count = error_count + CASE WHEN $4 = 'error' THEN 1 ELSE 0 END, + next_run_at = $6, + updated_at = now() + WHERE name = $1 + `, name, finishedAt, finishedAt.Sub(startedAt).Milliseconds(), status, msg, nextRunAt) + return err +} + +func (r *jobRunRepository) RequestRun(ctx context.Context, name string) (bool, error) { + tag, err := r.db.Exec(ctx, ` + UPDATE scheduled_job_runs SET run_requested_at = now(), updated_at = now() WHERE name = $1 + `, name) + if err != nil { + return false, err + } + return tag.RowsAffected() > 0, nil +} + +func (r *jobRunRepository) TakeRunRequest(ctx context.Context, name string) (bool, error) { + tag, err := r.db.Exec(ctx, ` + UPDATE scheduled_job_runs SET run_requested_at = NULL, updated_at = now() + WHERE name = $1 AND run_requested_at IS NOT NULL + `, name) + if err != nil { + return false, err + } + return tag.RowsAffected() > 0, nil +} + +func (r *jobRunRepository) List(ctx context.Context) ([]models.ScheduledJobRun, error) { + rows, err := r.db.Query(ctx, ` + SELECT name, service, interval_seconds, last_started_at, last_finished_at, last_duration_ms, + last_status, last_error, run_count, error_count, run_requested_at, next_run_at, updated_at + FROM scheduled_job_runs + ORDER BY service, name + `) + if err != nil { + return nil, err + } + defer rows.Close() + out := []models.ScheduledJobRun{} + for rows.Next() { + var j models.ScheduledJobRun + if err := rows.Scan( + &j.Name, &j.Service, &j.IntervalSeconds, &j.LastStartedAt, &j.LastFinishedAt, &j.LastDurationMs, + &j.LastStatus, &j.LastError, &j.RunCount, &j.ErrorCount, &j.RunRequestedAt, &j.NextRunAt, &j.UpdatedAt, + ); err != nil { + return nil, err + } + out = append(out, j) + } + return out, rows.Err() +} diff --git a/internal/repository/pg_worker.go b/internal/repository/pg_worker.go index 4cadd298..4c3d5f24 100644 --- a/internal/repository/pg_worker.go +++ b/internal/repository/pg_worker.go @@ -59,6 +59,9 @@ type WorkerRepository interface { GetActiveDedicatedAssignment(ctx context.Context, userID uuid.UUID) (*models.DedicatedWorkerAssignment, error) GetDedicatedWorkerByOrgID(ctx context.Context, orgID uuid.UUID) (*models.Worker, error) ReleaseDedicatedAssignment(ctx context.Context, userID uuid.UUID) error + // ReleaseDedicatedAssignmentByID releases one specific binding; false when + // it was already released, so a caller never releases a newer one by accident. + ReleaseDedicatedAssignmentByID(ctx context.Context, id uuid.UUID) (bool, error) // Email account worker queries GetEmailAccountsByWorkerID(ctx context.Context, workerID uuid.UUID) ([]uuid.UUID, error) @@ -413,6 +416,18 @@ func (r *workerRepository) ReleaseDedicatedAssignment(ctx context.Context, userI return err } +func (r *workerRepository) ReleaseDedicatedAssignmentByID(ctx context.Context, id uuid.UUID) (bool, error) { + tag, err := r.db.Exec(ctx, ` + UPDATE dedicated_worker_assignments + SET released_at = now() + WHERE id = $1 AND released_at IS NULL + `, id) + if err != nil { + return false, err + } + return tag.RowsAffected() > 0, nil +} + // GetEmailAccountsByWorkerID retrieves all email account IDs assigned to a worker func (r *workerRepository) GetEmailAccountsByWorkerID(ctx context.Context, workerID uuid.UUID) ([]uuid.UUID, error) { query := `SELECT id FROM email_accounts WHERE worker_id = $1`