mirror of
https://github.com/warmbly/warmbly.git
synced 2026-11-17 16:19:13 +00:00
274 lines
8.9 KiB
Go
274 lines
8.9 KiB
Go
// Admin endpoints for the send outcome loop and the task queues: in-flight
|
|
// reservations, dead letters, task failures and customer webhook delivery
|
|
// health across every workspace.
|
|
|
|
package handler
|
|
|
|
import (
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/gin-gonic/gin"
|
|
"github.com/google/uuid"
|
|
|
|
"github.com/warmbly/warmbly/internal/config"
|
|
"github.com/warmbly/warmbly/internal/errx"
|
|
"github.com/warmbly/warmbly/internal/models"
|
|
)
|
|
|
|
const (
|
|
deadLetterEntity models.AuditEntityType = "task_dead_letter"
|
|
sendHoldEntity models.AuditEntityType = "send_recovery_hold"
|
|
)
|
|
|
|
// webhookDeliveryLease mirrors the default LeaseTimeout in
|
|
// webhook.NewDeliveryWorker (cmd/backend wires the worker without overriding it).
|
|
const webhookDeliveryLease = 5 * time.Minute
|
|
|
|
// parseBoundedLimit is parseLimit with a page cap above the admin default of 100.
|
|
func parseBoundedLimit(s string, def, max int) int {
|
|
if s == "" {
|
|
return def
|
|
}
|
|
n, err := strconv.Atoi(s)
|
|
if err != nil || n <= 0 {
|
|
return def
|
|
}
|
|
if n > max {
|
|
return max
|
|
}
|
|
return n
|
|
}
|
|
|
|
func (h *Handler) adminSendsReady(c *gin.Context) bool {
|
|
if h.AdminSendsRepo == nil {
|
|
errx.JSON(c, errx.New(errx.NotImplemented, "send operations are not available on this instance"))
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
// AdminInFlightSends lists reserved sends no worker result has resolved yet.
|
|
func (h *Handler) AdminInFlightSends(c *gin.Context) {
|
|
if !h.adminSendsReady(c) {
|
|
return
|
|
}
|
|
limit := parseBoundedLimit(c.Query("limit"), 100, 500)
|
|
reclaimAfter := time.Duration(config.CampaignSendReclaimAfterMinutes) * time.Minute
|
|
result, err := h.AdminSendsRepo.InFlight(c.Request.Context(), reclaimAfter, limit)
|
|
if err != nil {
|
|
errx.JSON(c, errx.New(errx.Internal, err.Error()))
|
|
return
|
|
}
|
|
c.JSON(http.StatusOK, result)
|
|
}
|
|
|
|
// AdminListDeadLetters pages task dead letters newest first.
|
|
func (h *Handler) AdminListDeadLetters(c *gin.Context) {
|
|
if !h.adminSendsReady(c) {
|
|
return
|
|
}
|
|
cursor := parseCursor(c.Query("cursor"))
|
|
limit := parseLimit(c.Query("limit"), 50)
|
|
status := c.Query("status")
|
|
switch status {
|
|
case "", "pending", "replayed", "failed":
|
|
default:
|
|
errx.JSON(c, errx.New(errx.BadRequest, "invalid status"))
|
|
return
|
|
}
|
|
result, err := h.AdminSendsRepo.ListDeadLetters(c.Request.Context(), status, cursor, limit)
|
|
if err != nil {
|
|
errx.JSON(c, errx.New(errx.Internal, err.Error()))
|
|
return
|
|
}
|
|
c.JSON(http.StatusOK, result)
|
|
}
|
|
|
|
// AdminReplayDeadLetter re-enqueues one dead letter through its workspace.
|
|
func (h *Handler) AdminReplayDeadLetter(c *gin.Context) {
|
|
if !h.adminSendsReady(c) {
|
|
return
|
|
}
|
|
if h.AdvancedService == nil {
|
|
errx.JSON(c, errx.New(errx.NotImplemented, "dead letter replay is not available on this instance"))
|
|
return
|
|
}
|
|
id, err := uuid.Parse(c.Param("id"))
|
|
if err != nil {
|
|
errx.JSON(c, errx.New(errx.BadRequest, "invalid dead letter ID"))
|
|
return
|
|
}
|
|
row, err := h.AdminSendsRepo.GetDeadLetter(c.Request.Context(), id)
|
|
if err != nil {
|
|
errx.JSON(c, errx.New(errx.Internal, err.Error()))
|
|
return
|
|
}
|
|
if row == nil {
|
|
errx.JSON(c, errx.New(errx.NotFound, "dead letter not found"))
|
|
return
|
|
}
|
|
if row.OrganizationID == nil {
|
|
errx.JSON(c, errx.New(errx.BadRequest, "dead letter has no organization; its task or mailbox is gone"))
|
|
return
|
|
}
|
|
if xerr := h.AdvancedService.ReplayDeadLetter(c.Request.Context(), *row.OrganizationID, id); xerr != nil {
|
|
errx.JSON(c, xerr)
|
|
return
|
|
}
|
|
h.audit(c, models.AuditActionResume, deadLetterEntity, &id, map[string]string{
|
|
"organization_id": row.OrganizationID.String(),
|
|
"task_id": row.TaskID.String(),
|
|
"task_type": row.TaskType,
|
|
})
|
|
c.JSON(http.StatusOK, gin.H{"replayed": true})
|
|
}
|
|
|
|
// AdminSendHolds lists mailboxes held from sending and what holds each one.
|
|
func (h *Handler) AdminSendHolds(c *gin.Context) {
|
|
if !h.adminSendsReady(c) {
|
|
return
|
|
}
|
|
result, err := h.AdminSendsRepo.SendHolds(c.Request.Context(), parseBoundedLimit(c.Query("limit"), 100, 500))
|
|
if err != nil {
|
|
errx.JSON(c, errx.InternalError())
|
|
return
|
|
}
|
|
c.JSON(http.StatusOK, result)
|
|
}
|
|
|
|
// AdminResolveSendHold records an operator's provider-checked verdict on a held send through the mailbox's own resolution path.
|
|
func (h *Handler) AdminResolveSendHold(c *gin.Context) {
|
|
if !h.adminSendsReady(c) {
|
|
return
|
|
}
|
|
if h.EmailService == nil {
|
|
errx.JSON(c, errx.New(errx.NotImplemented, "send hold resolution is not available on this instance"))
|
|
return
|
|
}
|
|
mailboxID, err := uuid.Parse(c.Param("id"))
|
|
if err != nil {
|
|
errx.JSON(c, errx.New(errx.BadRequest, "invalid mailbox ID"))
|
|
return
|
|
}
|
|
var body models.AdminResolveSendHold
|
|
if err := c.ShouldBindJSON(&body); err != nil {
|
|
errx.JSON(c, errx.New(errx.BadRequest, "invalid request body"))
|
|
return
|
|
}
|
|
body.ConfirmationReference = strings.TrimSpace(body.ConfirmationReference)
|
|
body.MessageID = strings.TrimSpace(body.MessageID)
|
|
if body.ConfirmationReference == "" || strings.ContainsAny(body.ConfirmationReference, "\r\n") {
|
|
errx.JSON(c, errx.New(errx.BadRequest, "confirmation_reference must be one non-empty line"))
|
|
return
|
|
}
|
|
if body.Outcome == "sent" && body.MessageID == "" {
|
|
errx.JSON(c, errx.New(errx.BadRequest, "message_id is required when confirming a send went out"))
|
|
return
|
|
}
|
|
resolution, ok := body.Resolution()
|
|
if !ok {
|
|
errx.JSON(c, errx.New(errx.BadRequest, "outcome does not apply to this hold reason"))
|
|
return
|
|
}
|
|
held, err := h.AdminSendsRepo.GetSendHold(c.Request.Context(), mailboxID)
|
|
if err != nil {
|
|
errx.JSON(c, errx.InternalError())
|
|
return
|
|
}
|
|
if held == nil {
|
|
errx.JSON(c, errx.NewWithIdentifier(errx.NotFound, "send_hold_not_found", "This mailbox is not held from sending."))
|
|
return
|
|
}
|
|
if held.OrganizationID == nil || held.HeldTaskID == nil || *held.HeldTaskID != body.HeldTaskID || held.Reason != body.HeldReason {
|
|
errx.JSON(c, errx.NewWithIdentifier(errx.Conflict, "send_hold_changed", "This hold has changed since it was loaded. Refresh and check it again."))
|
|
return
|
|
}
|
|
if !held.Resolvable {
|
|
errx.JSON(c, errx.NewWithIdentifier(errx.Conflict, "send_hold_not_resolvable", "This hold cannot be resolved right now (blocker: "+held.Blocker+"). The Holds list says what it is waiting for."))
|
|
return
|
|
}
|
|
update := &models.UpdateEmail{SendRecoveryResolution: resolution}
|
|
if _, xerr := h.EmailService.Update(c.Request.Context(), held.OrganizationID.String(), "", mailboxID.String(), update); xerr != nil {
|
|
if xerr == errx.ErrInvalid {
|
|
errx.JSON(c, errx.NewWithIdentifier(errx.Conflict, "send_hold_not_resolvable", "The hold or its send changed and the evidence no longer applies. Refresh and check it again."))
|
|
return
|
|
}
|
|
errx.JSON(c, xerr)
|
|
return
|
|
}
|
|
h.audit(c, models.AuditActionUpdate, sendHoldEntity, &mailboxID, map[string]string{
|
|
"organization_id": held.OrganizationID.String(),
|
|
"held_task_id": body.HeldTaskID.String(),
|
|
"held_reason": body.HeldReason,
|
|
"outcome": body.Outcome,
|
|
})
|
|
// An unknown verdict is applied by the result consumer, so the hold lifts shortly after, not now.
|
|
c.JSON(http.StatusOK, gin.H{"accepted": true, "applied_by_consumer": body.HeldReason == "unknown"})
|
|
}
|
|
|
|
// AdminRecentTaskFailures lists the newest task failures with their mailbox.
|
|
func (h *Handler) AdminRecentTaskFailures(c *gin.Context) {
|
|
if !h.adminSendsReady(c) {
|
|
return
|
|
}
|
|
limit := parseBoundedLimit(c.Query("limit"), 100, 500)
|
|
rows, err := h.AdminSendsRepo.RecentTaskFailures(c.Request.Context(), limit)
|
|
if err != nil {
|
|
errx.JSON(c, errx.New(errx.Internal, err.Error()))
|
|
return
|
|
}
|
|
c.JSON(http.StatusOK, gin.H{"data": rows})
|
|
}
|
|
|
|
// AdminWebhookHealth is the instance-wide webhook delivery picture.
|
|
func (h *Handler) AdminWebhookHealth(c *gin.Context) {
|
|
if !h.adminSendsReady(c) {
|
|
return
|
|
}
|
|
health, err := h.AdminSendsRepo.WebhookHealth(c.Request.Context(), webhookDeliveryLease)
|
|
if err != nil {
|
|
errx.JSON(c, errx.New(errx.Internal, err.Error()))
|
|
return
|
|
}
|
|
c.JSON(http.StatusOK, health)
|
|
}
|
|
|
|
// AdminWebhookReclaim re-queues deliveries stranded in_flight past the lease.
|
|
func (h *Handler) AdminWebhookReclaim(c *gin.Context) {
|
|
if h.WebhookRepo == nil {
|
|
errx.JSON(c, errx.New(errx.NotImplemented, "webhook delivery is not available on this instance"))
|
|
return
|
|
}
|
|
n, err := h.WebhookRepo.ReclaimStuckDeliveries(c.Request.Context(), webhookDeliveryLease)
|
|
if err != nil {
|
|
errx.JSON(c, errx.New(errx.Internal, err.Error()))
|
|
return
|
|
}
|
|
h.audit(c, models.AuditActionUpdate, models.AuditEntityWebhook, nil, map[string]string{
|
|
"action": "reclaim_stuck_deliveries",
|
|
"reclaimed": strconv.FormatInt(n, 10),
|
|
})
|
|
c.JSON(http.StatusOK, gin.H{"reclaimed": n})
|
|
}
|
|
|
|
// AdminListOrgWebhooks lists one workspace's endpoints with recent counts.
|
|
func (h *Handler) AdminListOrgWebhooks(c *gin.Context) {
|
|
if !h.adminSendsReady(c) {
|
|
return
|
|
}
|
|
orgID, err := uuid.Parse(c.Param("id"))
|
|
if err != nil {
|
|
errx.JSON(c, errx.New(errx.BadRequest, "invalid organization ID"))
|
|
return
|
|
}
|
|
rows, err := h.AdminSendsRepo.OrgWebhooks(c.Request.Context(), orgID)
|
|
if err != nil {
|
|
errx.JSON(c, errx.New(errx.Internal, err.Error()))
|
|
return
|
|
}
|
|
c.JSON(http.StatusOK, gin.H{"data": rows})
|
|
}
|