mirror of
https://github.com/warmbly/warmbly.git
synced 2026-10-22 08:03:26 +00:00
288 lines
65 KiB
Go
288 lines
65 KiB
Go
package repository
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
"github.com/warmbly/warmbly/internal/app/monitoring"
|
|
"github.com/warmbly/warmbly/internal/models"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
type MonitoringRepository struct{ pool *pgxpool.Pool }
|
|
|
|
func NewMonitoringRepository(pool *pgxpool.Pool) *MonitoringRepository {
|
|
return &MonitoringRepository{pool: pool}
|
|
}
|
|
|
|
type monitoringQuery struct {
|
|
id, title, unit string
|
|
condition models.MonitoringCondition
|
|
severity, note, query string
|
|
window time.Duration
|
|
}
|
|
|
|
type monitoringSection struct {
|
|
permission models.AdminPermission
|
|
investigate string
|
|
queries []monitoringQuery
|
|
}
|
|
|
|
func (r *MonitoringRepository) Sources() []monitoring.Source {
|
|
out := make([]monitoring.Source, 0, len(monitoringSections)+3)
|
|
for _, id := range monitoringSectionOrder {
|
|
section := monitoringSections[id]
|
|
out = append(out, monitoring.Source{ID: id, Permission: section.permission, Collect: func(ctx context.Context, at time.Time) (models.MonitoringSource, error) {
|
|
return r.collect(ctx, at, id, section)
|
|
}})
|
|
}
|
|
out = append(out, r.workerSource(), r.domainSource(), r.loadingSource())
|
|
return out
|
|
}
|
|
|
|
func (r *MonitoringRepository) collect(ctx context.Context, at time.Time, id string, section monitoringSection) (models.MonitoringSource, error) {
|
|
out := models.MonitoringSource{Metrics: []models.MonitoringMetric{}}
|
|
if r == nil || r.pool == nil {
|
|
out.Availability = models.MonitoringUnavailable
|
|
out.Coverage = "unavailable"
|
|
out.Reason = "dependency_missing"
|
|
return out, nil
|
|
}
|
|
if id == "arrivals" {
|
|
var exists bool
|
|
if err := r.pool.QueryRow(ctx, "SELECT to_regclass('public.sync_arrival_outbox') IS NOT NULL").Scan(&exists); err != nil {
|
|
return out, err
|
|
}
|
|
if !exists {
|
|
out.Availability = models.MonitoringUnavailable
|
|
out.Coverage = "unavailable"
|
|
out.Reason = "schema_absent"
|
|
for _, q := range section.queries {
|
|
m := monitoringMetric(q, at, section.investigate)
|
|
m.Availability = models.MonitoringUnavailable
|
|
m.Condition = models.MonitoringUnknown
|
|
m.Reason = "schema_absent"
|
|
m.ObservedAt = nil
|
|
out.Metrics = append(out.Metrics, m)
|
|
}
|
|
return out, nil
|
|
}
|
|
}
|
|
queries := ""
|
|
for _, q := range section.queries {
|
|
if queries != "" {
|
|
queries += " UNION ALL "
|
|
}
|
|
queries += strings.Replace(strings.ReplaceAll(q.query, "$1", "$1::timestamptz"), "SELECT ", "SELECT '"+q.id+"', ", 1) + " AND $1::timestamptz IS NOT NULL"
|
|
}
|
|
rows, err := r.pool.Query(ctx, queries, at)
|
|
if err != nil {
|
|
return out, err
|
|
}
|
|
defer rows.Close()
|
|
index := 0
|
|
for rows.Next() {
|
|
if index >= len(section.queries) {
|
|
return out, errors.New("unexpected monitoring rows")
|
|
}
|
|
var id string
|
|
m := models.MonitoringMetric{}
|
|
if err := rows.Scan(&id, &m.Count, &m.AffectedMailboxes, &m.AffectedOrganizations, &m.UnknownOrganizationRows, &m.EvidenceAt, &m.LatestEvidenceAt, &m.NextEligibleAt); err != nil {
|
|
return out, err
|
|
}
|
|
found := false
|
|
for _, q := range section.queries {
|
|
if q.id == id {
|
|
v := monitoringMetric(q, at, section.investigate)
|
|
v.Count = m.Count
|
|
v.AffectedMailboxes = m.AffectedMailboxes
|
|
v.AffectedOrganizations = m.AffectedOrganizations
|
|
v.UnknownOrganizationRows = m.UnknownOrganizationRows
|
|
v.EvidenceAt = m.EvidenceAt
|
|
v.LatestEvidenceAt = m.LatestEvidenceAt
|
|
v.NextEligibleAt = m.NextEligibleAt
|
|
m = v
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
return out, errors.New("unexpected monitoring metric")
|
|
}
|
|
classifyMonitoringMetric(&m)
|
|
out.Metrics = append(out.Metrics, m)
|
|
index++
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return out, err
|
|
}
|
|
if index != len(section.queries) {
|
|
return out, errors.New("incomplete monitoring rows")
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func monitoringMetric(q monitoringQuery, at time.Time, path string) models.MonitoringMetric {
|
|
m := models.MonitoringMetric{ID: q.id, Title: q.title, Unit: q.unit, Condition: q.condition, Severity: q.severity, Note: q.note, Investigate: path, ObservedAt: &at, Availability: models.MonitoringFresh}
|
|
if seconds, ok := monitoringThresholds[q.id]; ok {
|
|
m.ThresholdSeconds = &seconds
|
|
}
|
|
if q.window > 0 {
|
|
start := at.Add(-q.window)
|
|
m.WindowStart = &start
|
|
m.WindowEnd = &at
|
|
}
|
|
return m
|
|
}
|
|
|
|
var monitoringThresholds = map[string]int64{
|
|
"reservations_under_5m": 300, "reservations_5_to_30m": 1800, "reservations_over_30m": 1800,
|
|
"campaign_reserved_aging": 1800, "dispatch_persistent": 600, "dispatch_old_failure": 900,
|
|
"original_slot_overdue_1h": 3600, "sync_stale": 1200, "sync_held_stale": 1200,
|
|
"backfill_stale_running": 3600, "worker_samples_missing": 1200,
|
|
"webhook_stale_claim": 300, "notification_aging": 120, "notification_stale": 600,
|
|
"arrival_aging": 300,
|
|
}
|
|
|
|
func classifyMonitoringMetric(m *models.MonitoringMetric) {
|
|
if m.Count == nil {
|
|
m.Availability = models.MonitoringUnavailable
|
|
m.Condition = models.MonitoringUnknown
|
|
m.Reason = "no_recent_evidence"
|
|
return
|
|
}
|
|
if *m.Count == 0 {
|
|
m.Condition = models.MonitoringNoEvidence
|
|
m.Severity = "info"
|
|
if m.ID == "confirmed_warmup_1h" || m.ID == "confirmed_other_1h" || m.ID == "confirmed_warmup_proxy_1h" {
|
|
m.Note += " Zero recent confirmations establishes neither recovery nor an outage."
|
|
}
|
|
}
|
|
}
|
|
|
|
var monitoringSectionOrder = []string{"send_safety", "sends", "campaign_reservations", "dispatch", "sync", "mailbox_errors", "worker_samples", "jobs", "dead_letters", "webhooks", "notifications", "result_effects", "arrivals"}
|
|
var monitoringSections = map[string]monitoringSection{
|
|
"campaign_reservations": {permission: models.AdminPermViewCampaigns, investigate: "/sends?tab=in-flight", queries: []monitoringQuery{
|
|
{id: "campaign_reserved", title: "Active campaign progress reservations", unit: "reservations", condition: models.MonitoringInFlight, severity: "info", note: "Native dispatched_at without sent_at progress state, distinct from unknown provider results.", query: `SELECT COUNT(*),COUNT(DISTINCT t.email_account_id),COUNT(DISTINCT c.organization_id),COUNT(*) FILTER(WHERE c.organization_id IS NULL),MIN(p.dispatched_at),MAX(p.dispatched_at),NULL::timestamptz FROM campaign_contact_progress p LEFT JOIN tasks t ON t.id=p.dispatch_task_id JOIN campaigns c ON c.id=p.campaign_id WHERE p.sent_at IS NULL AND p.dispatched_at IS NOT NULL`},
|
|
{id: "campaign_reserved_aging", title: "Campaign progress reservations older than 30m", unit: "reservations", condition: models.MonitoringRecoveryUnverified, severity: "warning", note: "30m investigation policy, not a native timeout or permission to replay.", query: `SELECT COUNT(*),COUNT(DISTINCT t.email_account_id),COUNT(DISTINCT c.organization_id),COUNT(*) FILTER(WHERE c.organization_id IS NULL),MIN(p.dispatched_at),MAX(p.dispatched_at),NULL::timestamptz FROM campaign_contact_progress p LEFT JOIN tasks t ON t.id=p.dispatch_task_id JOIN campaigns c ON c.id=p.campaign_id WHERE p.sent_at IS NULL AND p.dispatched_at<=$1-INTERVAL '30 minutes'`},
|
|
}},
|
|
"worker_samples": {permission: models.AdminPermViewWorkers, investigate: "/fleet", queries: []monitoringQuery{
|
|
{id: "worker_samples_missing", title: "Active workers without fresh health samples", unit: "workers", condition: models.MonitoringTelemetryStale, severity: "warning", note: "Monitoring 20m receipt-freshness policy; samples are stored with database receipt time, not worker clock. Zero counters in a fresh sample may be uninstrumented.", query: `SELECT COUNT(*),NULL::bigint,NULL::bigint,NULL::bigint,MIN(s.at),MAX(s.at),NULL::timestamptz FROM fleet_nodes n LEFT JOIN LATERAL(SELECT MAX(observed_at) AS at FROM worker_health_samples WHERE worker_id=n.id) s ON true WHERE n.active AND n.role='worker' AND (s.at IS NULL OR s.at<$1-INTERVAL '20 minutes' OR s.at>$1)`},
|
|
{id: "worker_sample_auth_errors_1h", title: "Reported worker credential errors (1h)", unit: "reports", condition: models.MonitoringRecentFailure, severity: "warning", note: "Worker-reported window counters, independent of mailbox reports. Zero counters do not establish instrumented healthy credentials.", window: time.Hour, query: `SELECT SUM(h.auth_errors)::bigint,NULL::bigint,NULL::bigint,NULL::bigint,MIN(h.observed_at),MAX(h.observed_at),NULL::timestamptz FROM worker_health_samples h WHERE h.observed_at BETWEEN $1-INTERVAL '1 hour' AND $1`},
|
|
{id: "worker_sample_rate_errors_1h", title: "Reported worker rate-limit errors (1h)", unit: "reports", condition: models.MonitoringRecentFailure, severity: "warning", note: "Reported window counters may be uninstrumented; not cumulative retry attempts.", window: time.Hour, query: `SELECT SUM(h.rate_limit_errors)::bigint,NULL::bigint,NULL::bigint,NULL::bigint,MIN(h.observed_at),MAX(h.observed_at),NULL::timestamptz FROM worker_health_samples h WHERE h.observed_at BETWEEN $1-INTERVAL '1 hour' AND $1`},
|
|
{id: "worker_sample_sends_1h", title: "Reported worker successful sends (1h)", unit: "reports", condition: models.MonitoringUnknown, severity: "info", note: "Worker-reported counter, not provider-confirmed warmup delivery; token-confirmed tasks are measured separately.", window: time.Hour, query: `SELECT SUM(h.sends_succeeded)::bigint,NULL::bigint,NULL::bigint,NULL::bigint,MIN(h.observed_at),MAX(h.observed_at),NULL::timestamptz FROM worker_health_samples h WHERE h.observed_at BETWEEN $1-INTERVAL '1 hour' AND $1`},
|
|
}},
|
|
"send_safety": {permission: models.AdminPermViewUsers, investigate: "/mailboxes", queries: []monitoringQuery{
|
|
{id: "holds_unknown", title: "Unknown send outcome protection", unit: "mailboxes", condition: "safety_hold", severity: "warning", note: "Protection can be set before execution; not a provider failure or permission to resend.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT e.id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(NULL::timestamptz), MAX(NULL::timestamptz), MIN(NULL::timestamptz) FROM email_accounts e WHERE e.send_recovery_hold AND COALESCE(e.send_recovery_reason,'unknown')='unknown'`},
|
|
{id: "holds_conflict", title: "conflict send protection", unit: "mailboxes", condition: "safety_hold", severity: "warning", note: "Current persisted safety restriction, not proof of a new rejection.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT e.id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(NULL::timestamptz), MAX(NULL::timestamptz), MIN(NULL::timestamptz) FROM email_accounts e WHERE e.send_recovery_hold AND e.send_recovery_reason='conflict'`},
|
|
{id: "holds_permanent", title: "permanent send protection", unit: "mailboxes", condition: "safety_hold", severity: "warning", note: "Current persisted safety restriction, not proof of a new rejection.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT e.id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(NULL::timestamptz), MAX(NULL::timestamptz), MIN(NULL::timestamptz) FROM email_accounts e WHERE e.send_recovery_hold AND e.send_recovery_reason='permanent'`},
|
|
{id: "holds_authentication", title: "authentication send protection", unit: "mailboxes", condition: "safety_hold", severity: "warning", note: "Current persisted safety restriction, not proof of a new rejection.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT e.id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(NULL::timestamptz), MAX(NULL::timestamptz), MIN(NULL::timestamptz) FROM email_accounts e WHERE e.send_recovery_hold AND e.send_recovery_reason='authentication'`},
|
|
{id: "send_cooldowns", title: "Active provider cooldowns", unit: "mailboxes", condition: "waiting_retry", severity: "info", note: "Expiry is next eligibility, not recovery confirmation.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT e.id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(NULL::timestamptz), MAX(NULL::timestamptz), MIN(e.send_cooldown_until::timestamptz) FROM email_accounts e WHERE e.send_cooldown_until>$1`},
|
|
{id: "send_paused", title: "Paused warmup", unit: "mailboxes", condition: "waiting_schedule", severity: "info", note: "Explicit operator pause, not a failure.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT e.id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(e.warmup_paused_at::timestamptz), MAX(e.warmup_paused_at::timestamptz), MIN(NULL::timestamptz) FROM email_accounts e WHERE e.status='active' AND e.warmup IS NOT NULL AND e.warmup_paused_at IS NOT NULL`},
|
|
}},
|
|
"sends": {permission: models.AdminPermViewCampaigns, queries: []monitoringQuery{
|
|
{id: "confirmed_warmup_1h", title: "Provider-confirmed warmup tasks (1h)", unit: "tasks", condition: "healthy", severity: "info", note: "Provider acceptance only; legacy completed_at is a time proxy after independent token confirmation.", window: 3600 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(COALESCE(t.send_result_applied_at,t.completed_at)::timestamptz), MAX(COALESCE(t.send_result_applied_at,t.completed_at)::timestamptz), MIN(NULL::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.task_type='warmup' AND EXISTS(SELECT 1 FROM warmup_tokens wt WHERE wt.task_id=t.id AND wt.sent_message_id<>'') AND COALESCE(t.send_result_applied_at,t.completed_at) BETWEEN $1-INTERVAL '1 hour' AND $1`},
|
|
{id: "confirmed_other_1h", title: "Reconciled non-warmup sent tasks (1h)", unit: "tasks", condition: "healthy", severity: "info", note: "Native reconciliation evidence, not receiver delivery or inbox placement.", window: 3600 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.send_result_applied_at::timestamptz), MAX(t.send_result_applied_at::timestamptz), MIN(NULL::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.task_type<>'warmup' AND t.send_result_state='sent' AND t.send_result_applied_at BETWEEN $1-INTERVAL '1 hour' AND $1`},
|
|
{id: "confirmed_warmup_proxy_1h", title: "Legacy-clock confirmed warmup tasks (1h)", unit: "tasks", condition: "healthy", severity: "info", note: "Subset of confirmed warmup using completed_at proxy.", window: 3600 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.completed_at::timestamptz), MAX(t.completed_at::timestamptz), MIN(NULL::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.task_type='warmup' AND t.send_result_applied_at IS NULL AND EXISTS(SELECT 1 FROM warmup_tokens wt WHERE wt.task_id=t.id AND wt.sent_message_id<>'') AND t.completed_at BETWEEN $1-INTERVAL '1 hour' AND $1`},
|
|
{id: "unknown_results", title: "Unverified send results", unit: "tasks", condition: "recovery_unverified", severity: "warning", note: "Includes legacy uncertainty; unknown is not a failed send.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.send_reserved_at::timestamptz), MAX(t.send_reserved_at::timestamptz), MIN(NULL::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.send_result_state='unknown'`},
|
|
{id: "reservations_under_5m", title: "Outstanding reservation: under_5m", unit: "tasks", condition: "in_flight", severity: "info", note: "Reservation precedes provider execution; age is not a native timeout guarantee.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.send_reserved_at::timestamptz), MAX(t.send_executor_started_at::timestamptz), MIN(NULL::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.send_reserved_at IS NOT NULL AND t.send_released_at IS NULL AND t.send_result_state='unknown' AND t.send_result_applied_at IS NULL AND t.send_reserved_at>$1-INTERVAL '5 minutes' AND t.send_reserved_at<=$1`},
|
|
{id: "reservations_5_to_30m", title: "Outstanding reservation: 5_to_30m", unit: "tasks", condition: "in_flight", severity: "info", note: "Reservation precedes provider execution; age is not a native timeout guarantee.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.send_reserved_at::timestamptz), MAX(t.send_executor_started_at::timestamptz), MIN(NULL::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.send_reserved_at IS NOT NULL AND t.send_released_at IS NULL AND t.send_result_state='unknown' AND t.send_result_applied_at IS NULL AND t.send_reserved_at<=$1-INTERVAL '5 minutes' AND t.send_reserved_at>$1-INTERVAL '30 minutes'`},
|
|
{id: "reservations_over_30m", title: "Outstanding reservation: over_30m", unit: "tasks", condition: "recovery_unverified", severity: "warning", note: "Reservation precedes provider execution; age is not a native timeout guarantee.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.send_reserved_at::timestamptz), MAX(t.send_executor_started_at::timestamptz), MIN(NULL::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.send_reserved_at IS NOT NULL AND t.send_released_at IS NULL AND t.send_result_state='unknown' AND t.send_result_applied_at IS NULL AND t.send_reserved_at<=$1-INTERVAL '30 minutes'`},
|
|
{id: "failed_results_1h", title: "Reconciled failed send results (1h)", unit: "tasks", condition: "recent_failure", severity: "warning", note: "Failure of these tasks, not a continuing mailbox outage.", window: 3600 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.send_result_applied_at::timestamptz), MAX(t.send_result_applied_at::timestamptz), MIN(NULL::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.send_result_state='failed' AND t.send_result_applied_at BETWEEN $1-INTERVAL '1 hour' AND $1`},
|
|
{id: "failed_results_24h", title: "Reconciled failed send results (24h)", unit: "tasks", condition: "recovery_unverified", severity: "warning", note: "Historical task outcomes; mailbox recovery requires scoped confirmation.", window: 86400 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.send_result_applied_at::timestamptz), MAX(t.send_result_applied_at::timestamptz), MIN(NULL::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.send_result_state='failed' AND t.send_result_applied_at BETWEEN $1-INTERVAL '24 hours' AND $1`},
|
|
}},
|
|
"dispatch": {permission: models.AdminPermViewCampaigns, queries: []monitoringQuery{
|
|
{id: "dispatch_due", title: "Effectively due pending tasks", unit: "tasks", condition: "backlog", severity: "info", note: "Database dispatch backlog, not cloud provider queue depth or outage.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(GREATEST(t.scheduled_at,COALESCE(t.dispatch_retry_at,t.scheduled_at))::timestamptz), MAX(t.dispatch_failure_at::timestamptz), MIN(NULL::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.status='pending' AND GREATEST(t.scheduled_at,COALESCE(t.dispatch_retry_at,t.scheduled_at))<=$1`},
|
|
{id: "future_schedule", title: "Future scheduled tasks", unit: "tasks", condition: "waiting_schedule", severity: "info", note: "A future slot does not establish whether pacing, daily cap or business hours chose it.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(NULL::timestamptz), MAX(NULL::timestamptz), MIN(GREATEST(t.scheduled_at,COALESCE(t.dispatch_retry_at,t.scheduled_at))::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.status='pending' AND t.scheduled_at>$1`},
|
|
{id: "retry_wait", title: "Dispatch retry waits", unit: "tasks", condition: "waiting_retry", severity: "info", note: "Local scheduler retry evidence; count is tasks, not attempts.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.dispatch_failure_since::timestamptz), MAX(t.dispatch_failure_at::timestamptz), MIN(t.dispatch_retry_at::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.status='pending' AND t.scheduled_at<=$1 AND t.dispatch_retry_at>$1`},
|
|
{id: "dispatch_persistent", title: "Repeated recent dispatch failure evidence", unit: "tasks", condition: "persistent_failure", severity: "warning", note: "Monitoring policy: repeated stamped dispatch failure over 10m with recurrence within 15m; not provider failure.", window: 900 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.dispatch_failure_since::timestamptz), MAX(t.dispatch_failure_at::timestamptz), MIN(t.dispatch_retry_at::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.status='pending' AND t.dispatch_failure_since<=$1-INTERVAL '10 minutes' AND t.dispatch_failure_at>$1-INTERVAL '15 minutes' AND t.dispatch_failure_at<=$1 AND t.dispatch_failure_at>t.dispatch_failure_since`},
|
|
{id: "dispatch_old_failure", title: "Old dispatch failure, recovery unverified", unit: "tasks", condition: "recovery_unverified", severity: "warning", note: "Old failure does not prove continuing failure.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.dispatch_failure_since::timestamptz), MAX(t.dispatch_failure_at::timestamptz), MIN(t.dispatch_retry_at::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.status='pending' AND t.dispatch_failure_at<=$1-INTERVAL '15 minutes'`},
|
|
{id: "original_slot_overdue_1h", title: "Original warmup slot overdue (1h)", unit: "tasks", condition: "backlog", severity: "info", note: "Original slot delay includes retry waits and persisted safety/cooldown gates; not unexplained dispatcher failure.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.scheduled_at::timestamptz), MAX(NULL::timestamptz), MIN(t.dispatch_retry_at::timestamptz) FROM tasks t JOIN email_accounts e ON e.id=t.email_account_id WHERE t.status='pending' AND t.task_type='warmup' AND t.scheduled_at<=$1-INTERVAL '1 hour' AND e.status='active' AND e.warmup IS NOT NULL AND e.warmup_paused_at IS NULL AND (e.test_mode IS NULL OR e.test_mode='legacy' OR e.test_mode='diagnostic' AND e.test_send_enabled)`},
|
|
{id: "unscheduled_pending", title: "Pending tasks without a schedule", unit: "tasks", condition: "unknown", severity: "info", note: "No due time means overdue age cannot be inferred.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.created_at::timestamptz), MAX(NULL::timestamptz), MIN(NULL::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.status='pending' AND t.scheduled_at IS NULL AND t.dispatch_retry_at IS NULL`},
|
|
}},
|
|
"sync": {permission: models.AdminPermViewUsers, investigate: "/sync", queries: []monitoringQuery{
|
|
{id: "sync_eligible", title: "Active assigned supported mailboxes", unit: "mailboxes", condition: "unknown", severity: "info", note: "Population for sync state coverage, not a throughput assertion.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT e.id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(NULL::timestamptz), MAX(NULL::timestamptz), MIN(NULL::timestamptz) FROM email_accounts e WHERE e.status='active' AND e.worker_id IS NOT NULL AND e.provider::text IN ('gmail','outlook','smtp_imap')`},
|
|
{id: "sync_missing", title: "Eligible mailboxes without sync state", unit: "mailboxes", condition: "unknown", severity: "warning", note: "Absent worker state is unobserved, not a sync outage.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT e.id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(NULL::timestamptz), MAX(NULL::timestamptz), MIN(NULL::timestamptz) FROM email_accounts e LEFT JOIN email_sync_state s ON s.email_id=e.id WHERE e.status='active' AND e.worker_id IS NOT NULL AND e.provider::text IN ('gmail','outlook','smtp_imap') AND s.email_id IS NULL`},
|
|
{id: "sync_unassigned", title: "Active supported mailboxes unassigned", unit: "mailboxes", condition: "unknown", severity: "warning", note: "Assignment issue; no provider sync inference.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT e.id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(NULL::timestamptz), MAX(NULL::timestamptz), MIN(NULL::timestamptz) FROM email_accounts e WHERE e.status='active' AND e.worker_id IS NULL AND e.provider::text IN ('gmail','outlook','smtp_imap')`},
|
|
{id: "sync_state_rows", title: "Persisted sync state rows", unit: "mailboxes", condition: "unknown", severity: "info", note: "Fleet-wide persisted population; zero rows is not healthy sync.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.last_synced_at::timestamptz), MAX(s.last_synced_at::timestamptz), MIN(NULL::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE TRUE`},
|
|
{id: "sync_hold_burst", title: "Active sync burst hold", unit: "mailboxes", condition: "budget", severity: "info", note: "Persisted governor hold, not transport failure; expiry is eligibility only.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.last_synced_at::timestamptz), MAX(s.last_synced_at::timestamptz), MIN(s.throttled_until::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE s.throttled_until>$1 AND s.throttle_reason='burst'`},
|
|
{id: "sync_hold_hourly", title: "Active sync hourly hold", unit: "mailboxes", condition: "budget", severity: "info", note: "Persisted governor hold, not transport failure; expiry is eligibility only.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.last_synced_at::timestamptz), MAX(s.last_synced_at::timestamptz), MIN(s.throttled_until::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE s.throttled_until>$1 AND s.throttle_reason='hourly'`},
|
|
{id: "sync_hold_daily", title: "Active sync daily hold", unit: "mailboxes", condition: "daily_limit", severity: "info", note: "Persisted governor hold, not transport failure; expiry is eligibility only.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.last_synced_at::timestamptz), MAX(s.last_synced_at::timestamptz), MIN(s.throttled_until::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE s.throttled_until>$1 AND s.throttle_reason='daily'`},
|
|
{id: "sync_hold_org_daily", title: "Active sync org_daily hold", unit: "mailboxes", condition: "daily_limit", severity: "info", note: "Persisted governor hold, not transport failure; expiry is eligibility only.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.last_synced_at::timestamptz), MAX(s.last_synced_at::timestamptz), MIN(s.throttled_until::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE s.throttled_until>$1 AND s.throttle_reason='org_daily'`},
|
|
{id: "sync_hold_priority_daily", title: "Active sync priority_daily hold", unit: "mailboxes", condition: "daily_limit", severity: "info", note: "Persisted governor hold, not transport failure; expiry is eligibility only.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.last_synced_at::timestamptz), MAX(s.last_synced_at::timestamptz), MIN(s.throttled_until::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE s.throttled_until>$1 AND s.throttle_reason='priority_daily'`},
|
|
{id: "sync_hold_other", title: "Other active sync holds", unit: "mailboxes", condition: "budget", severity: "info", note: "Unknown reason is not promoted to a provider error.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.last_synced_at::timestamptz), MAX(s.last_synced_at::timestamptz), MIN(s.throttled_until::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE s.throttled_until>$1 AND s.throttle_reason NOT IN ('burst','hourly','daily','org_daily','priority_daily')`},
|
|
{id: "sync_stale", title: "Eligible sync state older than 20m", unit: "mailboxes", condition: "stale_telemetry", severity: "warning", note: "20m monitoring freshness policy, not provider SLA; base tick timestamps do not prove successful sync.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.last_synced_at::timestamptz), MAX(s.last_synced_at::timestamptz), MIN(NULL::timestamptz) FROM email_sync_state s JOIN email_accounts e ON e.id=s.email_id WHERE e.status='active' AND e.worker_id IS NOT NULL AND e.provider::text IN ('gmail','outlook','smtp_imap') AND (s.last_synced_at IS NULL OR s.last_synced_at<$1-INTERVAL '20 minutes') AND (s.throttled_until IS NULL OR s.throttled_until<=$1)`},
|
|
{id: "sync_held_stale", title: "Stale sync observations under active holds", unit: "mailboxes", condition: "budget", severity: "info", note: "Hold explains waiting but does not refresh old activity evidence.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.last_synced_at::timestamptz), MAX(s.last_synced_at::timestamptz), MIN(s.throttled_until::timestamptz) FROM email_sync_state s JOIN email_accounts e ON e.id=s.email_id WHERE e.status='active' AND e.worker_id IS NOT NULL AND (s.last_synced_at IS NULL OR s.last_synced_at<$1-INTERVAL '20 minutes') AND s.throttled_until>$1`},
|
|
{id: "sync_future_clock", title: "Future-dated sync activity", unit: "mailboxes", condition: "unknown", severity: "warning", note: "Clock anomaly; never a negative-age healthy reading.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.last_synced_at::timestamptz), MAX(s.last_synced_at::timestamptz), MIN(NULL::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE s.last_synced_at>$1`},
|
|
{id: "backfill_running", title: "Backfill running states", unit: "mailboxes", condition: "waiting_schedule", severity: "info", note: "Persisted import state; complete can mean configured cap reached, not current sync health.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.backfill_started_at::timestamptz), MAX(s.updated_at::timestamptz), MIN(NULL::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE s.backfill_status='running'`},
|
|
{id: "backfill_pending", title: "Backfill pending states", unit: "mailboxes", condition: "waiting_schedule", severity: "info", note: "Persisted import state; complete can mean configured cap reached, not current sync health.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.backfill_started_at::timestamptz), MAX(s.updated_at::timestamptz), MIN(NULL::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE s.backfill_status='pending'`},
|
|
{id: "backfill_complete", title: "Backfill complete states", unit: "mailboxes", condition: "unknown", severity: "info", note: "Persisted import state; complete can mean configured cap reached, not current sync health.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.backfill_started_at::timestamptz), MAX(s.updated_at::timestamptz), MIN(NULL::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE s.backfill_status='complete'`},
|
|
{id: "backfill_stale_running", title: "Stale running backfill without active hold", unit: "mailboxes", condition: "stale_telemetry", severity: "warning", note: "Native 1h stale running-state rule, potential stall rather than proved outage.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.updated_at::timestamptz), MAX(s.updated_at::timestamptz), MIN(NULL::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE s.backfill_status='running' AND s.updated_at<$1-INTERVAL '1 hour' AND (s.throttled_until IS NULL OR s.throttled_until<=$1)`},
|
|
{id: "sync_deferred", title: "Last reported budget-deferred live messages", unit: "messages", condition: "budget", severity: "info", note: "Latest pass count, not remaining provider backlog; incomplete ticks can overwrite it.", window: 0 * time.Second, query: `SELECT COALESCE(SUM(s.deferred),0)::bigint, COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.last_synced_at::timestamptz), MAX(s.last_synced_at::timestamptz), MIN(s.throttled_until::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE s.deferred>0`},
|
|
}},
|
|
"mailbox_errors": {permission: models.AdminPermViewUsers, investigate: "/mailboxes", queries: []monitoringQuery{
|
|
{id: "credential_errors", title: "Unresolved mailbox credential reports", unit: "reports", condition: "recovery_unverified", severity: "warning", note: "Deduplicated first stored report, not latest failing attempt or recurrence. Includes inactive accounts.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT r.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(r.created_at::timestamptz), MAX(r.created_at::timestamptz), MIN(NULL::timestamptz) FROM email_account_errors r LEFT JOIN email_accounts e ON e.id=r.email_account_id WHERE r.resolved_at IS NULL AND r.error_code IN ('GOOGLE_AUTHENTICATION_FAILED','AUTHENTICATION_FAILED','AUTHORIZATION_FAILED','INVALID_CREDENTIALS')`},
|
|
{id: "sync_errors", title: "Unresolved sync transport reports", unit: "reports", condition: "recovery_unverified", severity: "warning", note: "Deduplicated first stored report, not latest failing attempt or recurrence. Includes inactive accounts.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT r.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(r.created_at::timestamptz), MAX(r.created_at::timestamptz), MIN(NULL::timestamptz) FROM email_account_errors r LEFT JOIN email_accounts e ON e.id=r.email_account_id WHERE r.resolved_at IS NULL AND r.error_code IN ('SYNC_ERROR','SYNC_AUTH_ERROR','SYNC_CONNECTION_ERROR')`},
|
|
{id: "sync_safety_reports", title: "Unresolved sync flood/fair-use reports", unit: "reports", condition: "safety_hold", severity: "warning", note: "Deduplicated first stored report, not latest failing attempt or recurrence. Includes inactive accounts.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT r.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(r.created_at::timestamptz), MAX(r.created_at::timestamptz), MIN(NULL::timestamptz) FROM email_account_errors r LEFT JOIN email_accounts e ON e.id=r.email_account_id WHERE r.resolved_at IS NULL AND r.error_code IN ('SYNC_FLOOD','SYNC_FAIR_USE')`},
|
|
{id: "other_mailbox_errors", title: "Other unresolved mailbox reports", unit: "reports", condition: "recovery_unverified", severity: "warning", note: "Deduplicated first stored report, not latest failing attempt or recurrence. Includes inactive accounts.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT r.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(r.created_at::timestamptz), MAX(r.created_at::timestamptz), MIN(NULL::timestamptz) FROM email_account_errors r LEFT JOIN email_accounts e ON e.id=r.email_account_id WHERE r.resolved_at IS NULL AND r.error_code NOT IN ('GOOGLE_AUTHENTICATION_FAILED','AUTHENTICATION_FAILED','AUTHORIZATION_FAILED','INVALID_CREDENTIALS','SYNC_ERROR','SYNC_AUTH_ERROR','SYNC_CONNECTION_ERROR','SYNC_FLOOD','SYNC_FAIR_USE')`},
|
|
{id: "inactive_credential_errors", title: "Inactive mailboxes with unresolved credential reports", unit: "reports", condition: "safety_hold", severity: "warning", note: "Inactive without matching error is not an authentication failure.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT r.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(r.created_at::timestamptz), MAX(r.created_at::timestamptz), MIN(NULL::timestamptz) FROM email_account_errors r JOIN email_accounts e ON e.id=r.email_account_id WHERE r.resolved_at IS NULL AND e.status<>'active' AND r.error_code IN ('GOOGLE_AUTHENTICATION_FAILED','AUTHENTICATION_FAILED','AUTHORIZATION_FAILED','INVALID_CREDENTIALS')`},
|
|
}},
|
|
"jobs": {permission: models.AdminPermViewAnalytics, investigate: "/jobs", queries: []monitoringQuery{
|
|
{id: "job_error_recent", title: "Jobs whose latest completion errored within 1h", unit: "jobs", condition: "recent_failure", severity: "warning", note: "Name-keyed instance rows, not replica heartbeats; wrapper return does not prove subsystem throughput.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, NULL::bigint, NULL::bigint, MIN(j.last_finished_at::timestamptz), MAX(j.last_finished_at::timestamptz), MIN(j.next_run_at::timestamptz) FROM scheduled_job_runs j WHERE j.last_status='error' AND j.last_finished_at BETWEEN $1-INTERVAL '1 hour' AND $1`},
|
|
{id: "job_error_old", title: "Jobs with older recorded error", unit: "jobs", condition: "recovery_unverified", severity: "warning", note: "Name-keyed instance rows, not replica heartbeats; wrapper return does not prove subsystem throughput.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, NULL::bigint, NULL::bigint, MIN(j.last_finished_at::timestamptz), MAX(j.last_finished_at::timestamptz), MIN(j.next_run_at::timestamptz) FROM scheduled_job_runs j WHERE j.last_status='error' AND (j.last_finished_at IS NULL OR j.last_finished_at<$1-INTERVAL '1 hour')`},
|
|
{id: "job_running", title: "Recorded running jobs", unit: "jobs", condition: "in_flight", severity: "info", note: "Name-keyed instance rows, not replica heartbeats; wrapper return does not prove subsystem throughput.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, NULL::bigint, NULL::bigint, MIN(j.last_started_at::timestamptz), MAX(j.last_finished_at::timestamptz), MIN(j.next_run_at::timestamptz) FROM scheduled_job_runs j WHERE j.last_status='running'`},
|
|
{id: "job_future", title: "Recorded future job schedules", unit: "jobs", condition: "waiting_schedule", severity: "info", note: "Name-keyed instance rows, not replica heartbeats; wrapper return does not prove subsystem throughput.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, NULL::bigint, NULL::bigint, MIN(j.last_finished_at::timestamptz), MAX(j.last_finished_at::timestamptz), MIN(j.next_run_at::timestamptz) FROM scheduled_job_runs j WHERE j.last_status<>'running' AND j.next_run_at>$1`},
|
|
{id: "job_overdue", title: "Recorded job schedule past grace", unit: "jobs", condition: "backlog", severity: "warning", note: "Name-keyed instance rows, not replica heartbeats; wrapper return does not prove subsystem throughput.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, NULL::bigint, NULL::bigint, MIN(j.last_finished_at::timestamptz), MAX(j.last_finished_at::timestamptz), MIN(j.next_run_at::timestamptz) FROM scheduled_job_runs j WHERE j.last_status<>'running' AND j.next_run_at IS NOT NULL AND j.next_run_at+make_interval(secs=>GREATEST(60,LEAST(j.interval_seconds,300)))<$1`},
|
|
{id: "job_ok", title: "Jobs with latest successful wrapper completion", unit: "jobs", condition: "healthy", severity: "info", note: "Name-keyed instance rows, not replica heartbeats; wrapper return does not prove subsystem throughput.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, NULL::bigint, NULL::bigint, MIN(j.last_finished_at::timestamptz), MAX(j.last_finished_at::timestamptz), MIN(j.next_run_at::timestamptz) FROM scheduled_job_runs j WHERE j.last_status='ok'`},
|
|
}},
|
|
"dead_letters": {permission: models.AdminPermViewCampaigns, investigate: "/sends?tab=dead-letters", queries: []monitoringQuery{
|
|
{id: "dlq_due", title: "Dead letters due for retry", unit: "records", condition: "backlog", severity: "warning", note: "Durable queue state; attempts are delivery/replay attempts, replay is not provider confirmation.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(d.next_retry_at::timestamptz), MAX(d.updated_at::timestamptz), MIN(d.next_retry_at::timestamptz) FROM task_dead_letters d LEFT JOIN tasks t ON t.id=d.task_id LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE d.status='pending' AND d.next_retry_at<=$1 AND d.attempts<d.max_attempts`},
|
|
{id: "dlq_wait", title: "Dead letters awaiting retry slot", unit: "records", condition: "waiting_retry", severity: "info", note: "Durable queue state; attempts are delivery/replay attempts, replay is not provider confirmation.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(d.created_at::timestamptz), MAX(d.updated_at::timestamptz), MIN(d.next_retry_at::timestamptz) FROM task_dead_letters d LEFT JOIN tasks t ON t.id=d.task_id LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE d.status='pending' AND d.next_retry_at>$1 AND d.attempts<d.max_attempts`},
|
|
{id: "dlq_exhausted", title: "Dead letters with exhausted attempts", unit: "records", condition: "recovery_unverified", severity: "warning", note: "Durable queue state; attempts are delivery/replay attempts, replay is not provider confirmation.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(d.created_at::timestamptz), MAX(d.updated_at::timestamptz), MIN(d.next_retry_at::timestamptz) FROM task_dead_letters d LEFT JOIN tasks t ON t.id=d.task_id LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE d.status IN ('pending','failed') AND d.attempts>=d.max_attempts`},
|
|
{id: "dlq_unscheduled", title: "Pending dead letters without retry slot", unit: "records", condition: "unknown", severity: "warning", note: "Durable queue state; attempts are delivery/replay attempts, replay is not provider confirmation.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(d.created_at::timestamptz), MAX(d.updated_at::timestamptz), MIN(d.next_retry_at::timestamptz) FROM task_dead_letters d LEFT JOIN tasks t ON t.id=d.task_id LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE d.status='pending' AND d.next_retry_at IS NULL AND d.attempts<d.max_attempts`},
|
|
{id: "dlq_replayed", title: "Replayed dead letters", unit: "records", condition: "unknown", severity: "info", note: "Durable queue state; attempts are delivery/replay attempts, replay is not provider confirmation.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(d.created_at::timestamptz), MAX(d.updated_at::timestamptz), MIN(d.next_retry_at::timestamptz) FROM task_dead_letters d LEFT JOIN tasks t ON t.id=d.task_id LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE d.status='replayed'`},
|
|
{id: "task_failures_recent", title: "Currently failed tasks updated within 1h", unit: "tasks", condition: "recent_failure", severity: "warning", note: "Mutable task update proxy; not immutable failure-event time.", window: 3600 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.updated_at::timestamptz), MAX(t.updated_at::timestamptz), MIN(NULL::timestamptz) FROM task_failures f JOIN tasks t ON t.id=f.task_id LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.status='failed' AND t.updated_at BETWEEN $1-INTERVAL '1 hour' AND $1`},
|
|
}},
|
|
"webhooks": {permission: models.AdminPermViewOrganizations, investigate: "", queries: []monitoringQuery{
|
|
{id: "webhook_due", title: "Due webhook deliveries", unit: "deliveries", condition: "backlog", severity: "info", note: "Current row state, not HTTP attempt history; native lease default 5m.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT d.organization_id), COUNT(*) FILTER(WHERE d.organization_id IS NULL), MIN(d.next_attempt_at::timestamptz), MAX(d.updated_at::timestamptz), MIN(d.next_attempt_at::timestamptz) FROM webhook_deliveries d WHERE d.status='pending' AND d.next_attempt_at<=$1`},
|
|
{id: "webhook_wait", title: "Future webhook retry slots", unit: "deliveries", condition: "waiting_retry", severity: "info", note: "Current row state, not HTTP attempt history; native lease default 5m.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT d.organization_id), COUNT(*) FILTER(WHERE d.organization_id IS NULL), MIN(d.updated_at::timestamptz), MAX(d.updated_at::timestamptz), MIN(d.next_attempt_at::timestamptz) FROM webhook_deliveries d WHERE d.status='pending' AND d.next_attempt_at>$1`},
|
|
{id: "webhook_retry", title: "Pending webhooks with prior attempts", unit: "deliveries", condition: "waiting_retry", severity: "info", note: "Current row state, not HTTP attempt history; native lease default 5m.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT d.organization_id), COUNT(*) FILTER(WHERE d.organization_id IS NULL), MIN(d.updated_at::timestamptz), MAX(d.updated_at::timestamptz), MIN(d.next_attempt_at::timestamptz) FROM webhook_deliveries d WHERE d.status='pending' AND d.attempt_count>0`},
|
|
{id: "webhook_stale_claim", title: "Webhook claims older than 5m", unit: "deliveries", condition: "stale_telemetry", severity: "warning", note: "Current row state, not HTTP attempt history; native lease default 5m.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT d.organization_id), COUNT(*) FILTER(WHERE d.organization_id IS NULL), MIN(d.updated_at::timestamptz), MAX(d.updated_at::timestamptz), MIN(d.next_attempt_at::timestamptz) FROM webhook_deliveries d WHERE d.status='in_flight' AND d.updated_at<$1-INTERVAL '5 minutes'`},
|
|
{id: "webhook_abandoned_24h", title: "Abandoned webhook rows settled within 24h", unit: "deliveries", condition: "recent_failure", severity: "warning", note: "Current row state, not HTTP attempt history; native lease default 5m.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT d.organization_id), COUNT(*) FILTER(WHERE d.organization_id IS NULL), MIN(d.updated_at::timestamptz), MAX(d.updated_at::timestamptz), MIN(d.next_attempt_at::timestamptz) FROM webhook_deliveries d WHERE d.status='abandoned' AND d.updated_at BETWEEN $1-INTERVAL '24 hours' AND $1`},
|
|
{id: "webhook_delivered_24h", title: "Delivered webhook rows settled within 24h", unit: "deliveries", condition: "healthy", severity: "info", note: "Current row state, not HTTP attempt history; native lease default 5m.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT d.organization_id), COUNT(*) FILTER(WHERE d.organization_id IS NULL), MIN(d.updated_at::timestamptz), MAX(d.updated_at::timestamptz), MIN(d.next_attempt_at::timestamptz) FROM webhook_deliveries d WHERE d.status='delivered' AND d.updated_at BETWEEN $1-INTERVAL '24 hours' AND $1`},
|
|
{id: "webhook_endpoint_recent", title: "Endpoints with recent failure evidence", unit: "endpoints", condition: "recent_failure", severity: "warning", note: "Endpoint-level evidence is independent of individual delivery outcomes; exact aggregate, not capped examples.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(e.first_failure_at::timestamptz), MAX(e.last_failure_at::timestamptz), MIN(NULL::timestamptz) FROM webhook_endpoints e WHERE e.consecutive_failures>0 AND e.last_failure_at BETWEEN $1-INTERVAL '1 hour' AND $1`},
|
|
{id: "webhook_endpoint_persistent", title: "Endpoints with repeated sustained recent failures", unit: "endpoints", condition: "persistent_failure", severity: "warning", note: "Endpoint-level evidence is independent of individual delivery outcomes; exact aggregate, not capped examples.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(e.first_failure_at::timestamptz), MAX(e.last_failure_at::timestamptz), MIN(NULL::timestamptz) FROM webhook_endpoints e WHERE e.consecutive_failures>=5 AND e.first_failure_at<=$1-INTERVAL '72 hours' AND e.last_failure_at BETWEEN $1-INTERVAL '1 hour' AND $1`},
|
|
{id: "webhook_endpoint_old", title: "Endpoints with old unrecovered failure evidence", unit: "endpoints", condition: "recovery_unverified", severity: "warning", note: "Endpoint-level evidence is independent of individual delivery outcomes; exact aggregate, not capped examples.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(e.first_failure_at::timestamptz), MAX(e.last_failure_at::timestamptz), MIN(NULL::timestamptz) FROM webhook_endpoints e WHERE e.consecutive_failures>0 AND (e.last_failure_at IS NULL OR e.last_failure_at<$1-INTERVAL '1 hour')`},
|
|
{id: "webhook_auto_disabled", title: "Auto-disabled webhook endpoints", unit: "endpoints", condition: "safety_hold", severity: "warning", note: "Endpoint-level evidence is independent of individual delivery outcomes; exact aggregate, not capped examples.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(e.first_failure_at::timestamptz), MAX(e.last_failure_at::timestamptz), MIN(NULL::timestamptz) FROM webhook_endpoints e WHERE e.auto_disabled_at IS NOT NULL AND NOT e.enabled`},
|
|
{id: "webhook_drop_windows", title: "Rate-limited webhook windows (8 UTC dates)", unit: "windows", condition: "budget", severity: "info", note: "Today plus seven preceding UTC dates; throttled windows, not individual dropped events.", window: 0 * time.Second, query: `SELECT COALESCE(SUM(d.dropped_windows),0)::bigint, NULL::bigint, COUNT(DISTINCT d.organization_id), COUNT(*) FILTER(WHERE d.organization_id IS NULL), MIN(d.last_dropped_at::timestamptz), MAX(d.last_dropped_at::timestamptz), MIN(NULL::timestamptz) FROM webhook_event_drops d WHERE d.day>=($1 AT TIME ZONE 'UTC')::date-7 AND d.day<=($1 AT TIME ZONE 'UTC')::date`},
|
|
}},
|
|
"notifications": {permission: models.AdminPermManageSettings, investigate: "/health", queries: []monitoringQuery{
|
|
{id: "notification_due", title: "Due notification email rows", unit: "rows", condition: "backlog", severity: "info", note: "Digest rows are not physical emails; sent is settlement, skipped reasons/exhaustion time are unavailable.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT n.organization_id), COUNT(*) FILTER(WHERE n.organization_id IS NULL), MIN(n.email_due_at::timestamptz), MAX(n.email_due_at::timestamptz), MIN(CASE WHEN n.email_state='pending' THEN n.email_due_at END::timestamptz) FROM notifications n WHERE n.email_state='pending' AND n.email_due_at<=$1`},
|
|
{id: "notification_aging", title: "Notification due rows older than 2m", unit: "rows", condition: "backlog", severity: "warning", note: "Digest rows are not physical emails; sent is settlement, skipped reasons/exhaustion time are unavailable.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT n.organization_id), COUNT(*) FILTER(WHERE n.organization_id IS NULL), MIN(n.email_due_at::timestamptz), MAX(n.email_due_at::timestamptz), MIN(CASE WHEN n.email_state='pending' THEN n.email_due_at END::timestamptz) FROM notifications n WHERE n.email_state='pending' AND n.email_due_at<$1-INTERVAL '2 minutes'`},
|
|
{id: "notification_wait", title: "Future notification email rows", unit: "rows", condition: "waiting_schedule", severity: "info", note: "Digest rows are not physical emails; sent is settlement, skipped reasons/exhaustion time are unavailable.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT n.organization_id), COUNT(*) FILTER(WHERE n.organization_id IS NULL), MIN(n.email_due_at::timestamptz), MAX(n.email_due_at::timestamptz), MIN(CASE WHEN n.email_state='pending' THEN n.email_due_at END::timestamptz) FROM notifications n WHERE n.email_state='pending' AND n.email_due_at>$1`},
|
|
{id: "notification_retry", title: "Requeued notification email rows", unit: "rows", condition: "waiting_retry", severity: "info", note: "Digest rows are not physical emails; sent is settlement, skipped reasons/exhaustion time are unavailable.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT n.organization_id), COUNT(*) FILTER(WHERE n.organization_id IS NULL), MIN(n.email_due_at::timestamptz), MAX(n.email_due_at::timestamptz), MIN(CASE WHEN n.email_state='pending' THEN n.email_due_at END::timestamptz) FROM notifications n WHERE n.email_state='pending' AND n.email_attempts>0`},
|
|
{id: "notification_sending", title: "Claimed notification email rows", unit: "rows", condition: "in_flight", severity: "info", note: "Digest rows are not physical emails; sent is settlement, skipped reasons/exhaustion time are unavailable.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT n.organization_id), COUNT(*) FILTER(WHERE n.organization_id IS NULL), MIN(n.email_due_at::timestamptz), MAX(n.email_due_at::timestamptz), MIN(CASE WHEN n.email_state='pending' THEN n.email_due_at END::timestamptz) FROM notifications n WHERE n.email_state='sending'`},
|
|
{id: "notification_stale", title: "Notification claims older than 10m", unit: "rows", condition: "stale_telemetry", severity: "warning", note: "Digest rows are not physical emails; sent is settlement, skipped reasons/exhaustion time are unavailable.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT n.organization_id), COUNT(*) FILTER(WHERE n.organization_id IS NULL), MIN(n.email_due_at::timestamptz), MAX(n.email_due_at::timestamptz), MIN(CASE WHEN n.email_state='pending' THEN n.email_due_at END::timestamptz) FROM notifications n WHERE n.email_state='sending' AND n.email_due_at<$1-INTERVAL '10 minutes'`},
|
|
{id: "notification_exhausted", title: "Skipped notification rows with exhausted requeues", unit: "rows", condition: "recovery_unverified", severity: "warning", note: "Digest rows are not physical emails; sent is settlement, skipped reasons/exhaustion time are unavailable.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT n.organization_id), COUNT(*) FILTER(WHERE n.organization_id IS NULL), MIN(NULL::timestamptz), MAX(NULL::timestamptz), MIN(CASE WHEN n.email_state='pending' THEN n.email_due_at END::timestamptz) FROM notifications n WHERE n.email_state='skipped' AND n.email_attempts>=3`},
|
|
{id: "notification_settled_24h", title: "Notification rows settled within 24h", unit: "rows", condition: "unknown", severity: "info", note: "Digest rows are not physical emails; sent is settlement, skipped reasons/exhaustion time are unavailable.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT n.organization_id), COUNT(*) FILTER(WHERE n.organization_id IS NULL), MIN(n.email_due_at::timestamptz), MAX(n.email_due_at::timestamptz), MIN(CASE WHEN n.email_state='pending' THEN n.email_due_at END::timestamptz) FROM notifications n WHERE n.email_state='sent' AND n.email_due_at BETWEEN $1-INTERVAL '24 hours' AND $1`},
|
|
}},
|
|
"result_effects": {permission: models.AdminPermViewCampaigns, investigate: "/sends?tab=in-flight", queries: []monitoringQuery{
|
|
{id: "effects_pending", title: "Pending send result effects", unit: "effects", condition: "backlog", severity: "info", note: "Outbox effect delivery, not provider execution or sent email count.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(e.created_at::timestamptz), MAX(NULL::timestamptz), MIN(e.locked_until::timestamptz) FROM send_result_effects e LEFT JOIN tasks t ON t.id=e.task_id WHERE e.delivered_at IS NULL`},
|
|
{id: "effects_unleased", title: "Pending unleased result effects", unit: "effects", condition: "backlog", severity: "info", note: "Outbox effect delivery, not provider execution or sent email count.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(e.created_at::timestamptz), MAX(NULL::timestamptz), MIN(e.locked_until::timestamptz) FROM send_result_effects e LEFT JOIN tasks t ON t.id=e.task_id WHERE e.delivered_at IS NULL AND (e.locked_until IS NULL OR e.locked_until<=$1)`},
|
|
{id: "effects_leased", title: "Currently leased result effects", unit: "effects", condition: "in_flight", severity: "info", note: "Outbox effect delivery, not provider execution or sent email count.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(e.created_at::timestamptz), MAX(NULL::timestamptz), MIN(e.locked_until::timestamptz) FROM send_result_effects e LEFT JOIN tasks t ON t.id=e.task_id WHERE e.delivered_at IS NULL AND e.locked_until>$1`},
|
|
{id: "effects_repeated", title: "Pending effects with repeated attempts", unit: "effects", condition: "waiting_retry", severity: "warning", note: "Outbox effect delivery, not provider execution or sent email count.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(e.created_at::timestamptz), MAX(NULL::timestamptz), MIN(e.locked_until::timestamptz) FROM send_result_effects e LEFT JOIN tasks t ON t.id=e.task_id WHERE e.delivered_at IS NULL AND e.attempts>1`},
|
|
}},
|
|
"arrivals": {permission: models.AdminPermViewUsers, investigate: "/sync", queries: []monitoringQuery{
|
|
{id: "arrival_pending", title: "Pending durable arrivals", unit: "arrivals", condition: "backlog", severity: "info", note: "Optional outbox coverage excludes legacy published arrivals; stages/leases/slots are not historical retries.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT a.email_id), COUNT(DISTINCT a.organization_id), COUNT(*) FILTER(WHERE a.organization_id IS NULL), MIN(a.created_at::timestamptz), MAX(NULL::timestamptz), MIN(a.retry_at::timestamptz) FROM sync_arrival_outbox a WHERE TRUE`},
|
|
{id: "arrival_due", title: "Due unleased durable arrivals", unit: "arrivals", condition: "backlog", severity: "info", note: "Optional outbox coverage excludes legacy published arrivals; stages/leases/slots are not historical retries.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT a.email_id), COUNT(DISTINCT a.organization_id), COUNT(*) FILTER(WHERE a.organization_id IS NULL), MIN(a.created_at::timestamptz), MAX(NULL::timestamptz), MIN(a.retry_at::timestamptz) FROM sync_arrival_outbox a WHERE a.retry_at<=$1 AND (a.locked_until IS NULL OR a.locked_until<$1)`},
|
|
{id: "arrival_leased", title: "Leased durable arrivals", unit: "arrivals", condition: "in_flight", severity: "info", note: "Optional outbox coverage excludes legacy published arrivals; stages/leases/slots are not historical retries.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT a.email_id), COUNT(DISTINCT a.organization_id), COUNT(*) FILTER(WHERE a.organization_id IS NULL), MIN(a.created_at::timestamptz), MAX(NULL::timestamptz), MIN(a.retry_at::timestamptz) FROM sync_arrival_outbox a WHERE a.locked_until>$1`},
|
|
{id: "arrival_wait", title: "Future arrival retry slots", unit: "arrivals", condition: "waiting_retry", severity: "info", note: "Optional outbox coverage excludes legacy published arrivals; stages/leases/slots are not historical retries.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT a.email_id), COUNT(DISTINCT a.organization_id), COUNT(*) FILTER(WHERE a.organization_id IS NULL), MIN(a.created_at::timestamptz), MAX(NULL::timestamptz), MIN(a.retry_at::timestamptz) FROM sync_arrival_outbox a WHERE a.retry_at>$1`},
|
|
{id: "arrival_aging", title: "Durable arrivals older than 5m", unit: "arrivals", condition: "backlog", severity: "warning", note: "Optional outbox coverage excludes legacy published arrivals; stages/leases/slots are not historical retries.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT a.email_id), COUNT(DISTINCT a.organization_id), COUNT(*) FILTER(WHERE a.organization_id IS NULL), MIN(a.created_at::timestamptz), MAX(NULL::timestamptz), MIN(a.retry_at::timestamptz) FROM sync_arrival_outbox a WHERE a.created_at<=$1-INTERVAL '5 minutes'`},
|
|
{id: "arrival_stage_0", title: "Durable arrivals at stage 0", unit: "arrivals", condition: "backlog", severity: "info", note: "Optional outbox coverage excludes legacy published arrivals; stages/leases/slots are not historical retries.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT a.email_id), COUNT(DISTINCT a.organization_id), COUNT(*) FILTER(WHERE a.organization_id IS NULL), MIN(a.created_at::timestamptz), MAX(NULL::timestamptz), MIN(a.retry_at::timestamptz) FROM sync_arrival_outbox a WHERE a.stage=0`},
|
|
{id: "arrival_stage_1", title: "Durable arrivals at stage 1", unit: "arrivals", condition: "backlog", severity: "info", note: "Optional outbox coverage excludes legacy published arrivals; stages/leases/slots are not historical retries.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT a.email_id), COUNT(DISTINCT a.organization_id), COUNT(*) FILTER(WHERE a.organization_id IS NULL), MIN(a.created_at::timestamptz), MAX(NULL::timestamptz), MIN(a.retry_at::timestamptz) FROM sync_arrival_outbox a WHERE a.stage=1`},
|
|
{id: "arrival_stage_2", title: "Durable arrivals at stage 2", unit: "arrivals", condition: "backlog", severity: "info", note: "Optional outbox coverage excludes legacy published arrivals; stages/leases/slots are not historical retries.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT a.email_id), COUNT(DISTINCT a.organization_id), COUNT(*) FILTER(WHERE a.organization_id IS NULL), MIN(a.created_at::timestamptz), MAX(NULL::timestamptz), MIN(a.retry_at::timestamptz) FROM sync_arrival_outbox a WHERE a.stage=2`},
|
|
{id: "arrival_stage_3", title: "Durable arrivals at stage 3", unit: "arrivals", condition: "backlog", severity: "info", note: "Optional outbox coverage excludes legacy published arrivals; stages/leases/slots are not historical retries.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT a.email_id), COUNT(DISTINCT a.organization_id), COUNT(*) FILTER(WHERE a.organization_id IS NULL), MIN(a.created_at::timestamptz), MAX(NULL::timestamptz), MIN(a.retry_at::timestamptz) FROM sync_arrival_outbox a WHERE a.stage=3`},
|
|
}},
|
|
}
|