diff --git a/admin/src/app/dashboard/configuration/SettingsTab.tsx b/admin/src/app/dashboard/configuration/SettingsTab.tsx
index 8912691d5..2291c948d 100644
--- a/admin/src/app/dashboard/configuration/SettingsTab.tsx
+++ b/admin/src/app/dashboard/configuration/SettingsTab.tsx
@@ -109,7 +109,28 @@ const RETENTION_FIELDS = [
setting: "warmup_event_days",
label: "Warmup records (days)",
min: 30,
- help: "Per-message warmup records: tokens, receipts, tampering events and spam reports. The pool health bands read the last 30 days, which is the floor. The daily sent and received counts behind the analytics are separate and never pruned.",
+ help: "Per-message warmup records: tokens, receipts, tampering events, spam reports and the record of which messages belong to a warmup thread. The pool health bands read the last 30 days, which is the floor. The daily sent and received counts behind the analytics are separate and never pruned.",
+ },
+ {
+ key: "campaignLogDays",
+ setting: "campaign_log_days",
+ label: "Campaign activity feed (days)",
+ min: RETENTION_MIN_DAYS,
+ help: "Entries in each campaign's activity feed: sends that failed or were deferred, pauses and the reasons for them. Campaign stats, progress and sent mail are separate and unaffected.",
+ },
+ {
+ key: "notificationDays",
+ setting: "notification_days",
+ label: "Notifications (days)",
+ min: RETENTION_MIN_DAYS,
+ help: "In-app notifications, read or unread. One still waiting to be emailed is kept until it is sent or skipped.",
+ },
+ {
+ key: "webhookDeliveryDays",
+ setting: "webhook_delivery_days",
+ label: "Webhook delivery log (days)",
+ min: RETENTION_MIN_DAYS,
+ help: "Delivered and abandoned webhook deliveries, with the payload and the endpoint's response excerpt. A delivery still being retried is kept until it settles. A delivery past this window can no longer be redelivered from the log.",
},
] as const;
@@ -120,25 +141,31 @@ const RETENTION_PRESETS = [
{
id: "default",
label: "Defaults",
- description: "365 / 180 / 90 days, warmup 30 / 365",
+ description: "365 / 180 / 90 days, warmup 30 / 365, feeds 365 / 180 / 90",
values: {
engagementDays: "365",
formDays: "180",
auditDays: "90",
warmupMailDays: "30",
warmupEventDays: "365",
+ campaignLogDays: "365",
+ notificationDays: "180",
+ webhookDeliveryDays: "90",
},
},
{
id: "minimal",
label: "Minimal retention",
- description: "30 / 30 / 30 days, warmup 7 / 30",
+ description: "30 / 30 / 30 days, warmup 7 / 30, feeds 30 / 30 / 30",
values: {
engagementDays: "30",
formDays: "30",
auditDays: "30",
warmupMailDays: "7",
warmupEventDays: "30",
+ campaignLogDays: "30",
+ notificationDays: "30",
+ webhookDeliveryDays: "30",
},
},
] as const;
@@ -304,6 +331,9 @@ function toForm(s: InstanceSettings): FormState {
auditDays: String(s.retention.audit_log_days),
warmupMailDays: String(s.retention.warmup_mail_days),
warmupEventDays: String(s.retention.warmup_event_days),
+ campaignLogDays: String(s.retention.campaign_log_days),
+ notificationDays: String(s.retention.notification_days),
+ webhookDeliveryDays: String(s.retention.webhook_delivery_days),
},
tracking: {
machineWindowOpen: String(s.tracking.machine_window_open_seconds),
@@ -482,6 +512,9 @@ export function SettingsTab({ onDirtyChange, onSwitchTab }: SettingsTabProps) {
audit_log_days: Number(form.retention.auditDays),
warmup_mail_days: Number(form.retention.warmupMailDays),
warmup_event_days: Number(form.retention.warmupEventDays),
+ campaign_log_days: Number(form.retention.campaignLogDays),
+ notification_days: Number(form.retention.notificationDays),
+ webhook_delivery_days: Number(form.retention.webhookDeliveryDays),
},
tracking: {
machine_window_open_seconds: Number(form.tracking.machineWindowOpen),
diff --git a/admin/src/lib/api/client/admin/instance.ts b/admin/src/lib/api/client/admin/instance.ts
index b5e9ee90f..9c1830497 100644
--- a/admin/src/lib/api/client/admin/instance.ts
+++ b/admin/src/lib/api/client/admin/instance.ts
@@ -135,6 +135,11 @@ export interface InstanceSettings {
// second. The daily warmup statistics are never pruned.
warmup_mail_days: number;
warmup_event_days: number;
+ // History shown in the product: a campaign's activity feed, in-app
+ // notifications, and each webhook endpoint's delivery log.
+ campaign_log_days: number;
+ notification_days: number;
+ webhook_delivery_days: number;
};
// How soon after a send an open or click is recorded as automated. The
// clock starts at dispatch to the worker, so the window also covers the
diff --git a/cmd/backend/main.go b/cmd/backend/main.go
index f4ca9e736..ab2693b39 100644
--- a/cmd/backend/main.go
+++ b/cmd/backend/main.go
@@ -2218,6 +2218,9 @@ func main() {
auditRetentionScheduler := jobs.NewAuditRetentionScheduler(auditRetentionJob, 6*time.Hour)
go auditRetentionScheduler.Start(ctx)
+ // Feeds, logs and finished bookkeeping rows past their windows, in batches.
+ go jobs.NewTableRetentionJob(repository.NewTableRetentionRepository(primaryDB.Pool), instanceSettings).Start(ctx, 6*time.Hour)
+
// Website tracking: the snippet's settings, the ingest path the tracking
// service forwards page views to, and the per-workspace retention sweep
// that keeps the window promised on the settings page.
diff --git a/docs/content/docs/development/configuration.mdx b/docs/content/docs/development/configuration.mdx
index 0cecd9a87..3027e64d0 100644
--- a/docs/content/docs/development/configuration.mdx
+++ b/docs/content/docs/development/configuration.mdx
@@ -1120,7 +1120,10 @@ These are the only settings a browser can change, and no environment variable ow
| `retention.form_event_days` | integer, 1 to 3650 | `180` | How long form funnel events (views, starts, field-level drop-off) are kept. Funnel reports range up to 90 days, so anything shorter shortens the report too |
| `retention.audit_log_days` | integer, 1 to 3650 | `90` | How long the audit trail is kept. It carries IP addresses, user agents and change payloads, so this is also how long that data is held |
| `retention.warmup_mail_days` | integer, 3 to 3650 | `30` | How long warmup mail stays in each mailbox before Warmbly deletes it, wherever the mailbox's filing setting keeps it (Trash on Gmail), with the stored copy of its body. A mailbox can set its own window on the Warmup tab of its drawer or through `warmup_retention_days` on the API; this is what every other mailbox follows. The floor leaves room for the engagement and for a reply in the thread to finish |
-| `retention.warmup_event_days` | integer, 30 to 3650 | `365` | How long the per-message warmup records (tokens, receipts, tampering events, spam reports, and a linked instance's closed commands and reported arrivals) are kept. The pool health bands read the last 30 days, which is the floor. The daily sent and received counts behind the warmup analytics are separate and never pruned |
+| `retention.warmup_event_days` | integer, 30 to 3650 | `365` | How long the per-message warmup records (tokens, receipts, tampering events, spam reports, warmup thread turns, and a linked instance's closed commands and reported arrivals) are kept. The pool health bands read the last 30 days, which is the floor. The daily sent and received counts behind the warmup analytics are separate and never pruned |
+| `retention.campaign_log_days` | integer, 1 to 3650 | `365` | How long each campaign's activity feed keeps an entry. Campaign stats, progress and sent mail are separate and unaffected |
+| `retention.notification_days` | integer, 1 to 3650 | `180` | How long in-app notifications are kept, read or unread. One still waiting to be emailed is kept until it is sent or skipped |
+| `retention.webhook_delivery_days` | integer, 1 to 3650 | `90` | How long a settled webhook delivery (delivered or abandoned) stays in its endpoint's delivery log. A delivery still being retried is kept until it settles, and one past this window can no longer be redelivered |
| `tracking.machine_window_open_seconds` | integer, 1 to 900 | `60` | How soon after a send was dispatched an open is recorded as automated rather than a person's. The clock starts when the send is handed to a worker, so this window also covers the sending provider's queue and the transit to the recipient, not just reading time. Raise it when delivery-time scanners are being counted as opens; lower it when recipients who read immediately are being missed. A change applies within a minute and only to events recorded after it |
| `tracking.machine_window_click_seconds` | integer, 1 to 900 | `30` | The same window for click tickets. Kept separate because the two mistakes cost different things: a misjudged open loses a metric, a misjudged click loses the automation behind an interested lead |
| `tracking.machine_window_probable_seconds` | integer, 1 to 86400 | `600` | The window used instead of the two above when the tracking service recognised the source as a mail-security network that also renders clicked pages for people, which Proofpoint, Mimecast and Cisco do through browser isolation. Such a match cannot settle whether a person is behind the request, so it widens the window rather than deciding: inside it the event is the delivery-time scan, past it the recipient who got to the mail later. Never applied shorter than the window for the kind of event in hand, so naming a network can only ever catch more scans. Its ceiling reaches a day because how long a vendor takes to detonate a link is the vendor's property, not the instance's |
@@ -1139,7 +1142,7 @@ These are the only settings a browser can change, and no environment variable ow
The four `sync.*` values are read by the backend when a mailbox is loaded onto a worker (on connect, on reassignment, and by the reconciler's periodic republish), so a change reaches every mailbox within 30 minutes without a restart (the republish gives each mailbox one turn per 30 minutes, spread out so no worker gets them all at once). The fixed pacing numbers around them (burst per five minutes, hourly, backfill pace, the flood threshold and the chronic-overage rule) are compiled constants listed under **Instance > Configuration > Effective limits**; see [Mailboxes](/guides/mailboxes/#what-gets-synced) for how the budgets behave. The four `sync.*` values also have a companion read view on **Operations > Sync**, which shows each mailbox's backfill progress and fair-use throttle against them and can clear a throttle or restart a backfill.
-The five `retention.*` values are read by the pruning sweeps on every pass, so shortening one takes effect on the next sweep rather than at the next restart. Deletion is permanent and there is no grace period: what already sits outside a shortened window goes on that sweep. The warmup mail window is the one that reaches into customers' mailboxes: the consumer retires each message and the worker holding the mailbox deletes it there, so shortening it empties the warmup folders of the whole instance down to the new window within a few hours. See [data control](/development/data-control/#what-is-kept-and-for-how-long) for what each log holds and what a shorter window costs.
+The eight `retention.*` values are read by the pruning sweeps on every pass, so shortening one takes effect on the next sweep rather than at the next restart. Deletion is permanent and there is no grace period: what already sits outside a shortened window goes on that sweep. The warmup mail window is the one that reaches into customers' mailboxes: the consumer retires each message and the worker holding the mailbox deletes it there, so shortening it empties the warmup folders of the whole instance down to the new window within a few hours. See [data control](/development/data-control/#what-is-kept-and-for-how-long) for what each log holds and what a shorter window costs.
An unattended install can seed the whole document before anyone signs in, with `WARMBLY_SETTINGS_BOOTSTRAP` holding the same partial JSON the admin API takes:
diff --git a/docs/content/docs/development/data-control.mdx b/docs/content/docs/development/data-control.mdx
index d94d0e860..31f6c9b36 100644
--- a/docs/content/docs/development/data-control.mdx
+++ b/docs/content/docs/development/data-control.mdx
@@ -100,17 +100,24 @@ See [Automatic inbox tagging](/guides/inbox-tagging/), [Advisor](/guides/advisor
| `retention.form_event_days` | 180 | Form funnel events: views, starts, field-level drop-off |
| `retention.audit_log_days` | 90 | The audit trail: actor, IP address, user agent, change payload |
| `retention.warmup_mail_days` | 30 | Warmup mail in the mailboxes themselves, and the stored copy of each message's body. Deleted by the platform once older than this, wherever the mailbox files it; a mailbox can set its own window |
-| `retention.warmup_event_days` | 365 | Per-message warmup records: tokens, receipts, tampering events, spam moves and their attribution, spam reports, and for a linked instance that runs its own mailboxes, the commands Cloud queued for it once they are closed and the warmup mail it reported once retired. The daily sent and received counts behind the analytics are kept. When each mailbox's owner last acted in it (five-minute marks, no message content) is kept 30 days whatever this is set to |
+| `retention.warmup_event_days` | 365 | Per-message warmup records: tokens, receipts, tampering events, spam moves and their attribution, spam reports, and which messages belong to a warmup thread, and for a linked instance that runs its own mailboxes, the commands Cloud queued for it once they are closed and the warmup mail it reported once retired. The daily sent and received counts behind the analytics are kept. When each mailbox's owner last acted in it (five-minute marks, no message content) is kept 30 days whatever this is set to |
+| `retention.campaign_log_days` | 365 | Each campaign's activity feed: failed and deferred sends, pauses and their reasons |
+| `retention.notification_days` | 180 | In-app notifications, read or unread. One still waiting to be emailed is kept until it is sent or skipped |
+| `retention.webhook_delivery_days` | 90 | Each webhook endpoint's delivery log of settled deliveries, with the payload and the response excerpt. A delivery still being retried is kept until it settles |
-The first three are between 1 and 3,650 days, and they are the settings a retention or privacy policy applies to, because each window is also how long the personal data in that log is held. The warmup mail window starts at 3 days and the warmup records window at 30, the least the engagement legs and the pool health bands need.
+The event logs and the three history feeds are between 1 and 3,650 days, and they are the settings a retention or privacy policy applies to, because each window is also how long the personal data in that log is held. The warmup mail window starts at 3 days and the warmup records window at 30, the least the engagement legs and the pool health bands need.
None of them change a number anyone reads. Campaign progress keeps its own summary of opens and clicks that outlives the per-event log, so counts, filters and branching are unaffected by shortening any of these. What gets shorter is what a contact's timeline can show, how far a funnel report reaches, and how far back an admin can audit.
+The history feeds are the same: a campaign's stats and sent mail are not in its activity feed, and an old notification that was never read stops counting toward the unread badge when it goes. A webhook delivery past its window can no longer be redelivered from the log.
+
+A few bookkeeping tables are trimmed on fixed windows with no setting: the marks the consumer uses to count each open or click once follow `retention.engagement_event_days`, worker health samples are kept 7 days, and a send's queued follow-up effects 7 days after they ran. The send record itself (each task and its message, which tracking, threading and campaign stats read), the message map and synced mail have no window and stay until their mailbox, campaign or workspace is deleted.
+
There is no grace period and no copy. Take a backup first if you are not sure.
-The **minimal retention** preset in the admin panel and in the installer sets the three event logs to 30 days, warmup mail to 7 and warmup records to 30.
+The **minimal retention** preset in the admin panel and in the installer sets the three event logs to 30 days, warmup mail to 7 and warmup records to 30. The admin panel's preset also sets the three history feeds to 30; an install leaves them at their defaults.
### Warmup mail in mailboxes
diff --git a/docs/content/docs/guides/campaigns.mdx b/docs/content/docs/guides/campaigns.mdx
index 46bc5b3e4..99df9fc24 100644
--- a/docs/content/docs/guides/campaigns.mdx
+++ b/docs/content/docs/guides/campaigns.mdx
@@ -71,7 +71,7 @@ You need at least one active mailbox and a contact list. Warm new mailboxes befo
## Shared workspace access
-Campaigns belong to the workspace. Members with **View campaigns** can read campaign details, steps and activity logs regardless of who created them. **Manage campaigns** allows members to edit settings, create or edit steps and delete steps. Campaign overview metrics require **View analytics**. See [Team and roles](/guides/team-roles/) for permission details.
+Campaigns belong to the workspace. Members with **View campaigns** can read campaign details, steps and activity logs regardless of who created them. **Manage campaigns** allows members to edit settings, create or edit steps and delete steps. Campaign overview metrics require **View analytics**. See [Team and roles](/guides/team-roles/) for permission details. A campaign's activity log keeps its entries for 365 days by default (`retention.campaign_log_days` on a self-hosted instance); its stats and sent mail are kept separately and are unaffected.
Select the workspace that owns the campaign before opening it. Creating a campaign does not grant access to it from a different workspace.
diff --git a/docs/content/docs/guides/notifications.mdx b/docs/content/docs/guides/notifications.mdx
index 4558e7cb1..bab0b35bb 100644
--- a/docs/content/docs/guides/notifications.mdx
+++ b/docs/content/docs/guides/notifications.mdx
@@ -45,6 +45,8 @@ The bell in the dashboard chrome shows your unread count. Every time you open it
**Feeds are per user**, not shared across the workspace, so two teammates see different things depending on what they own and what they enabled.
+Notifications are kept for 180 days by default, read or not, so one older than that leaves the feed and the bell count. One still waiting to be emailed is kept until it is sent. A self-hosted instance sets this window with `retention.notification_days` (see [data control](/development/data-control/#event-history)).
+
The feed floats above the dashboard instead of being clipped by the header or page. It stays within the screen on narrow displays, with the list scrolling while its filters and settings link remain accessible. Press outside it, press Escape, or open another dashboard menu to close it.
diff --git a/docs/content/docs/guides/webhooks.mdx b/docs/content/docs/guides/webhooks.mdx
index a94099458..4777eadeb 100644
--- a/docs/content/docs/guides/webhooks.mdx
+++ b/docs/content/docs/guides/webhooks.mdx
@@ -130,6 +130,8 @@ Every delivery attempt is logged. Open an endpoint in Settings -> Webhooks and g
You can also query deliveries across all endpoints with `GET /webhooks/deliveries`, or scope it to one endpoint with `GET /webhooks/{id}/deliveries`. Both support filtering by status and event type, and return opaque cursor pagination.
+A settled delivery stays in the log for 90 days by default after its last attempt, after which it can no longer be redelivered. A delivery still being retried stays until it settles. A self-hosted instance sets this window with `retention.webhook_delivery_days` (see [data control](/development/data-control/#event-history)).
+
If an event was throttled rather than delivered, it appears in `GET /webhooks/throttle-drops` instead of the delivery log.
diff --git a/internal/api/handler/admin_instance.go b/internal/api/handler/admin_instance.go
index 36d16662e..9630677c8 100644
--- a/internal/api/handler/admin_instance.go
+++ b/internal/api/handler/admin_instance.go
@@ -161,6 +161,9 @@ func instanceSettingsAuditDetails(doc instancesettings.Document) map[string]any
"retention_engagement_event_days": doc.Retention.EngagementEventDays,
"retention_form_event_days": doc.Retention.FormEventDays,
"retention_audit_log_days": doc.Retention.AuditLogDays,
+ "retention_campaign_log_days": doc.Retention.CampaignLogDays,
+ "retention_notification_days": doc.Retention.NotificationDays,
+ "retention_webhook_delivery_days": doc.Retention.WebhookDeliveryDays,
"tracking_machine_window_open": doc.Tracking.MachineWindowOpenSeconds,
"tracking_machine_window_click": doc.Tracking.MachineWindowClickSeconds,
"tracking_machine_window_probable": doc.Tracking.MachineWindowProbableSeconds,
diff --git a/internal/app/instancesettings/document.go b/internal/app/instancesettings/document.go
index a104b4356..2b2112b30 100644
--- a/internal/app/instancesettings/document.go
+++ b/internal/app/instancesettings/document.go
@@ -75,6 +75,12 @@ type Retention struct {
// receipts, tampering and spam reports) are kept. The daily warmup
// statistics behind the analytics are separate and never pruned.
WarmupEventDays int `json:"warmup_event_days"`
+ // CampaignLogDays is how long a campaign's activity feed keeps an entry.
+ CampaignLogDays int `json:"campaign_log_days"`
+ // NotificationDays is how long in-app notifications are kept, read or not.
+ NotificationDays int `json:"notification_days"`
+ // WebhookDeliveryDays is how long a settled webhook delivery stays in its endpoint's log.
+ WebhookDeliveryDays int `json:"webhook_delivery_days"`
}
// Tracking holds the engagement-classification windows. Zero means "compiled
@@ -346,6 +352,9 @@ func DefaultRetention() Retention {
AuditLogDays: config.AuditLogRetentionDaysDefault,
WarmupMailDays: config.WarmupMailRetentionDaysDefault,
WarmupEventDays: config.WarmupEventRetentionDaysDefault,
+ CampaignLogDays: config.CampaignLogRetentionDaysDefault,
+ NotificationDays: config.NotificationRetentionDaysDefault,
+ WebhookDeliveryDays: config.WebhookDeliveryRetentionDaysDefault,
}
}
@@ -374,6 +383,9 @@ func (r *Retention) Normalize() {
// the thirty-day health bands that read them.
r.WarmupMailDays = clamp(r.WarmupMailDays, config.WarmupMailRetentionDaysDefault, config.WarmupMailRetentionDaysMin)
r.WarmupEventDays = clamp(r.WarmupEventDays, config.WarmupEventRetentionDaysDefault, config.WarmupEventRetentionDaysMin)
+ r.CampaignLogDays = clamp(r.CampaignLogDays, config.CampaignLogRetentionDaysDefault, config.RetentionDaysMin)
+ r.NotificationDays = clamp(r.NotificationDays, config.NotificationRetentionDaysDefault, config.RetentionDaysMin)
+ r.WebhookDeliveryDays = clamp(r.WebhookDeliveryDays, config.WebhookDeliveryRetentionDaysDefault, config.RetentionDaysMin)
}
// Normalize clamps a document into its accepted range. It is applied on read
@@ -463,6 +475,9 @@ type Patch struct {
AuditLogDays *int `json:"audit_log_days"`
WarmupMailDays *int `json:"warmup_mail_days"`
WarmupEventDays *int `json:"warmup_event_days"`
+ CampaignLogDays *int `json:"campaign_log_days"`
+ NotificationDays *int `json:"notification_days"`
+ WebhookDeliveryDays *int `json:"webhook_delivery_days"`
} `json:"retention"`
Tracking *struct {
MachineWindowOpenSeconds *int `json:"machine_window_open_seconds"`
@@ -541,6 +556,15 @@ func (p Patch) Apply(doc Document) Document {
if p.Retention.WarmupEventDays != nil {
doc.Retention.WarmupEventDays = *p.Retention.WarmupEventDays
}
+ if p.Retention.CampaignLogDays != nil {
+ doc.Retention.CampaignLogDays = *p.Retention.CampaignLogDays
+ }
+ if p.Retention.NotificationDays != nil {
+ doc.Retention.NotificationDays = *p.Retention.NotificationDays
+ }
+ if p.Retention.WebhookDeliveryDays != nil {
+ doc.Retention.WebhookDeliveryDays = *p.Retention.WebhookDeliveryDays
+ }
}
if p.Tracking != nil {
if p.Tracking.MachineWindowOpenSeconds != nil {
diff --git a/internal/app/unibox/backfill_body_text.go b/internal/app/unibox/backfill_body_text.go
index 0ad67408f..829edb210 100644
--- a/internal/app/unibox/backfill_body_text.go
+++ b/internal/app/unibox/backfill_body_text.go
@@ -4,10 +4,10 @@ import (
"context"
"time"
- "github.com/google/uuid"
"github.com/rs/zerolog/log"
"github.com/warmbly/warmbly/internal/config"
"github.com/warmbly/warmbly/internal/pkg/mailhtml"
+ "github.com/warmbly/warmbly/internal/repository"
)
// Backfill pacing. A pass is one query plus one object-storage read per row, so
@@ -22,15 +22,23 @@ const (
// bodies were indexed, so unibox search covers the whole archive rather than
// only mail that arrives from now on.
//
-// The sweep walks the table once per process: it pages by id, and stops for
-// good when a pass turns up no rows left to visit. Rows whose stored body
-// really is empty stay at body_text = ” and are simply re-visited after the
-// next restart, which is cheaper than inventing a "tried and failed" marker.
+// The sweep walks the table once per instance: it pages by id, stores its
+// place after every page, and records itself done when a page comes back
+// empty, so a restart resumes rather than re-reading rows whose stored body
+// really is empty. New mail is written with its text, so finishing leaves
+// nothing behind.
func (s *uniboxService) StartBodyTextBackfill(ctx context.Context) {
+ progress, err := s.uniboxRepository.BodyBackfillProgress(ctx)
+ if err != nil {
+ log.Warn().Err(err).Msg("unibox body backfill: progress unreadable, starting over")
+ }
+ if progress.Done {
+ return
+ }
ticker := time.NewTicker(bodyBackfillInterval)
defer ticker.Stop()
- var cursor uuid.UUID
+ cursor := progress.After
var filled int
for {
select {
@@ -45,6 +53,9 @@ func (s *uniboxService) StartBodyTextBackfill(ctx context.Context) {
continue
}
if len(targets) == 0 {
+ if serr := s.uniboxRepository.SaveBodyBackfillProgress(ctx, repository.UniboxBodyBackfill{After: cursor, Done: true}); serr != nil {
+ log.Warn().Err(serr).Msg("unibox body backfill: could not record completion")
+ }
if filled > 0 {
log.Info().Int("messages", filled).Msg("unibox body backfill: complete")
}
@@ -69,5 +80,8 @@ func (s *uniboxService) StartBodyTextBackfill(ctx context.Context) {
}
filled++
}
+ if serr := s.uniboxRepository.SaveBodyBackfillProgress(ctx, repository.UniboxBodyBackfill{After: cursor}); serr != nil {
+ log.Warn().Err(serr).Msg("unibox body backfill: could not record progress")
+ }
}
}
diff --git a/internal/config/constants.go b/internal/config/constants.go
index 492200b06..980f4735b 100644
--- a/internal/config/constants.go
+++ b/internal/config/constants.go
@@ -339,6 +339,13 @@ const (
// shortens it under Instance settings.
AuditLogRetentionDaysDefault = 90
+ // CampaignLogRetentionDaysDefault is how long a campaign's activity feed keeps an entry.
+ CampaignLogRetentionDaysDefault = 365
+ // NotificationRetentionDaysDefault is how long in-app notifications are kept, read or not.
+ NotificationRetentionDaysDefault = 180
+ // WebhookDeliveryRetentionDaysDefault is how long a settled delivery stays in an endpoint's log.
+ WebhookDeliveryRetentionDaysDefault = 90
+
// Ten years is the ceiling every retention window shares. It is not a
// recommendation: it is the point past which "keep it" and "keep it
// forever" stop differing, and it bounds a typo. One day is the floor, so
diff --git a/internal/infrastructure/db/migrations/000341_email_opens_opened_at_index.down.sql b/internal/infrastructure/db/migrations/000341_email_opens_opened_at_index.down.sql
new file mode 100644
index 000000000..82504dbad
--- /dev/null
+++ b/internal/infrastructure/db/migrations/000341_email_opens_opened_at_index.down.sql
@@ -0,0 +1 @@
+DROP INDEX CONCURRENTLY IF EXISTS public.idx_email_opens_opened_at;
diff --git a/internal/infrastructure/db/migrations/000341_email_opens_opened_at_index.up.sql b/internal/infrastructure/db/migrations/000341_email_opens_opened_at_index.up.sql
new file mode 100644
index 000000000..d0534e14b
--- /dev/null
+++ b/internal/infrastructure/db/migrations/000341_email_opens_opened_at_index.up.sql
@@ -0,0 +1,3 @@
+-- Alone in its file for CONCURRENTLY. Serves the engagement-log prune, which deletes by age in batches.
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_email_opens_opened_at
+ ON public.email_opens (opened_at);
diff --git a/internal/infrastructure/db/migrations/000342_email_link_clicks_clicked_at_index.down.sql b/internal/infrastructure/db/migrations/000342_email_link_clicks_clicked_at_index.down.sql
new file mode 100644
index 000000000..132e096bf
--- /dev/null
+++ b/internal/infrastructure/db/migrations/000342_email_link_clicks_clicked_at_index.down.sql
@@ -0,0 +1 @@
+DROP INDEX CONCURRENTLY IF EXISTS public.idx_email_link_clicks_clicked_at;
diff --git a/internal/infrastructure/db/migrations/000342_email_link_clicks_clicked_at_index.up.sql b/internal/infrastructure/db/migrations/000342_email_link_clicks_clicked_at_index.up.sql
new file mode 100644
index 000000000..bda1e7dfb
--- /dev/null
+++ b/internal/infrastructure/db/migrations/000342_email_link_clicks_clicked_at_index.up.sql
@@ -0,0 +1,3 @@
+-- Alone in its file for CONCURRENTLY. Serves the engagement-log prune, which deletes by age in batches.
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_email_link_clicks_clicked_at
+ ON public.email_link_clicks (clicked_at);
diff --git a/internal/infrastructure/db/migrations/000343_worker_health_samples_observed_index.down.sql b/internal/infrastructure/db/migrations/000343_worker_health_samples_observed_index.down.sql
new file mode 100644
index 000000000..22830a32a
--- /dev/null
+++ b/internal/infrastructure/db/migrations/000343_worker_health_samples_observed_index.down.sql
@@ -0,0 +1 @@
+DROP INDEX CONCURRENTLY IF EXISTS public.idx_worker_health_samples_observed;
diff --git a/internal/infrastructure/db/migrations/000343_worker_health_samples_observed_index.up.sql b/internal/infrastructure/db/migrations/000343_worker_health_samples_observed_index.up.sql
new file mode 100644
index 000000000..7075089b8
--- /dev/null
+++ b/internal/infrastructure/db/migrations/000343_worker_health_samples_observed_index.up.sql
@@ -0,0 +1,3 @@
+-- Alone in its file for CONCURRENTLY. Serves the capacity view's last-hour window and the sample prune.
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_worker_health_samples_observed
+ ON public.worker_health_samples (observed_at);
diff --git a/internal/infrastructure/db/migrations/000344_campaign_logs_created_index.down.sql b/internal/infrastructure/db/migrations/000344_campaign_logs_created_index.down.sql
new file mode 100644
index 000000000..c82ef2231
--- /dev/null
+++ b/internal/infrastructure/db/migrations/000344_campaign_logs_created_index.down.sql
@@ -0,0 +1 @@
+DROP INDEX CONCURRENTLY IF EXISTS public.idx_campaign_logs_created;
diff --git a/internal/infrastructure/db/migrations/000344_campaign_logs_created_index.up.sql b/internal/infrastructure/db/migrations/000344_campaign_logs_created_index.up.sql
new file mode 100644
index 000000000..e5137f68b
--- /dev/null
+++ b/internal/infrastructure/db/migrations/000344_campaign_logs_created_index.up.sql
@@ -0,0 +1,3 @@
+-- Alone in its file for CONCURRENTLY. Serves the campaign activity-feed prune.
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_campaign_logs_created
+ ON public.campaign_logs (created_at);
diff --git a/internal/infrastructure/db/migrations/000345_notifications_created_index.down.sql b/internal/infrastructure/db/migrations/000345_notifications_created_index.down.sql
new file mode 100644
index 000000000..bb0525eaa
--- /dev/null
+++ b/internal/infrastructure/db/migrations/000345_notifications_created_index.down.sql
@@ -0,0 +1 @@
+DROP INDEX CONCURRENTLY IF EXISTS public.idx_notifications_created;
diff --git a/internal/infrastructure/db/migrations/000345_notifications_created_index.up.sql b/internal/infrastructure/db/migrations/000345_notifications_created_index.up.sql
new file mode 100644
index 000000000..dac121734
--- /dev/null
+++ b/internal/infrastructure/db/migrations/000345_notifications_created_index.up.sql
@@ -0,0 +1,3 @@
+-- Alone in its file for CONCURRENTLY. Serves the notification prune.
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_notifications_created
+ ON public.notifications (created_at);
diff --git a/internal/infrastructure/db/migrations/000346_webhook_deliveries_settled_index.down.sql b/internal/infrastructure/db/migrations/000346_webhook_deliveries_settled_index.down.sql
new file mode 100644
index 000000000..6977ce72c
--- /dev/null
+++ b/internal/infrastructure/db/migrations/000346_webhook_deliveries_settled_index.down.sql
@@ -0,0 +1 @@
+DROP INDEX CONCURRENTLY IF EXISTS public.idx_webhook_deliveries_settled;
diff --git a/internal/infrastructure/db/migrations/000346_webhook_deliveries_settled_index.up.sql b/internal/infrastructure/db/migrations/000346_webhook_deliveries_settled_index.up.sql
new file mode 100644
index 000000000..1c7947722
--- /dev/null
+++ b/internal/infrastructure/db/migrations/000346_webhook_deliveries_settled_index.up.sql
@@ -0,0 +1,4 @@
+-- Alone in its file for CONCURRENTLY. Serves the delivery-log prune, which only removes settled deliveries.
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_webhook_deliveries_settled
+ ON public.webhook_deliveries (updated_at)
+ WHERE status IN ('delivered', 'failed', 'abandoned');
diff --git a/internal/infrastructure/db/migrations/000347_warmup_received_account_live_index.down.sql b/internal/infrastructure/db/migrations/000347_warmup_received_account_live_index.down.sql
new file mode 100644
index 000000000..a14749d2b
--- /dev/null
+++ b/internal/infrastructure/db/migrations/000347_warmup_received_account_live_index.down.sql
@@ -0,0 +1 @@
+DROP INDEX CONCURRENTLY IF EXISTS public.idx_warmup_received_account_live;
diff --git a/internal/infrastructure/db/migrations/000347_warmup_received_account_live_index.up.sql b/internal/infrastructure/db/migrations/000347_warmup_received_account_live_index.up.sql
new file mode 100644
index 000000000..461c458b8
--- /dev/null
+++ b/internal/infrastructure/db/migrations/000347_warmup_received_account_live_index.up.sql
@@ -0,0 +1,4 @@
+-- Alone in its file for CONCURRENTLY. Serves the warmup mail retention listing, which walks each mailbox's unretired receipts.
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_warmup_received_account_live
+ ON public.warmup_received (email_account_id, created_at)
+ WHERE retired_at IS NULL;
diff --git a/internal/infrastructure/db/migrations/000348_warmup_tokens_sender_live_index.down.sql b/internal/infrastructure/db/migrations/000348_warmup_tokens_sender_live_index.down.sql
new file mode 100644
index 000000000..17f63f29e
--- /dev/null
+++ b/internal/infrastructure/db/migrations/000348_warmup_tokens_sender_live_index.down.sql
@@ -0,0 +1 @@
+DROP INDEX CONCURRENTLY IF EXISTS public.idx_warmup_tokens_sender_live;
diff --git a/internal/infrastructure/db/migrations/000348_warmup_tokens_sender_live_index.up.sql b/internal/infrastructure/db/migrations/000348_warmup_tokens_sender_live_index.up.sql
new file mode 100644
index 000000000..e57405b10
--- /dev/null
+++ b/internal/infrastructure/db/migrations/000348_warmup_tokens_sender_live_index.up.sql
@@ -0,0 +1,4 @@
+-- Alone in its file for CONCURRENTLY. Serves the warmup mail retention listing for each sender's own copies.
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_warmup_tokens_sender_live
+ ON public.warmup_tokens (sender_account_id, created_at)
+ WHERE sent_retired_at IS NULL AND sent_message_id <> '';
diff --git a/internal/jobs/table_retention.go b/internal/jobs/table_retention.go
new file mode 100644
index 000000000..69f257f43
--- /dev/null
+++ b/internal/jobs/table_retention.go
@@ -0,0 +1,90 @@
+package jobs
+
+import (
+ "context"
+ "errors"
+ "time"
+
+ "github.com/rs/zerolog/log"
+
+ "github.com/warmbly/warmbly/internal/app/instancesettings"
+ "github.com/warmbly/warmbly/internal/jobrun"
+)
+
+const (
+ // workerHealthSampleKeepDays covers every reader of the samples, the longest of which is one hour.
+ workerHealthSampleKeepDays = 7
+ // sendResultEffectKeepDays keeps a delivered effect long enough to explain a late duplicate.
+ sendResultEffectKeepDays = 7
+)
+
+// TableRetentionStore is the prune half of repository.TableRetentionRepository.
+type TableRetentionStore interface {
+ PruneCampaignLogs(ctx context.Context, before time.Time) (int64, error)
+ PruneNotifications(ctx context.Context, before time.Time) (int64, error)
+ PruneWebhookDeliveries(ctx context.Context, before time.Time) (int64, error)
+ PruneTrackingDedupe(ctx context.Context, before time.Time) (int64, error)
+ PruneWorkerHealthSamples(ctx context.Context, before time.Time) (int64, error)
+ PruneDeliveredSendResultEffects(ctx context.Context, before time.Time) (int64, error)
+}
+
+// TableRetentionJob prunes feeds, logs and finished bookkeeping past their
+// windows. The windows that bound product history are instance settings.
+type TableRetentionJob struct {
+ store TableRetentionStore
+ retention RetentionSource
+}
+
+// NewTableRetentionJob builds the job; without a settings source the compiled defaults apply.
+func NewTableRetentionJob(store TableRetentionStore, retention RetentionSource) *TableRetentionJob {
+ return &TableRetentionJob{store: store, retention: retention}
+}
+
+// Start runs the prune on every replica's loop; jobrun gives each slot to one of them.
+func (j *TableRetentionJob) Start(ctx context.Context, interval time.Duration) {
+ if j.store == nil {
+ return
+ }
+ jobrun.Loop(ctx, "table_retention", interval, true, j.Run)
+}
+
+// Run executes one pass and keeps going past a failing table.
+func (j *TableRetentionJob) Run(ctx context.Context) error {
+ windows := instancesettings.DefaultRetention()
+ if j.retention != nil {
+ windows = j.retention.RetentionWindows(ctx)
+ }
+ now := time.Now()
+ steps := []struct {
+ table string
+ prune func(context.Context, time.Time) (int64, error)
+ days int
+ }{
+ {"campaign_logs", j.store.PruneCampaignLogs, windows.CampaignLogDays},
+ {"notifications", j.store.PruneNotifications, windows.NotificationDays},
+ {"webhook_deliveries", j.store.PruneWebhookDeliveries, windows.WebhookDeliveryDays},
+ // A dedupe mark only matters while the open and click logs it guards are kept.
+ {"tracking_events_processed", j.store.PruneTrackingDedupe, windows.EngagementEventDays},
+ {"worker_health_samples", j.store.PruneWorkerHealthSamples, workerHealthSampleKeepDays},
+ {"send_result_effects", j.store.PruneDeliveredSendResultEffects, sendResultEffectKeepDays},
+ }
+ var failures []error
+ for _, s := range steps {
+ if s.days <= 0 {
+ continue
+ }
+ n, err := s.prune(ctx, now.AddDate(0, 0, -s.days))
+ if err != nil {
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+ failures = append(failures, err)
+ log.Warn().Err(err).Str("table", s.table).Msg("table retention: prune failed")
+ continue
+ }
+ if n > 0 {
+ log.Info().Int64("deleted", n).Str("table", s.table).Msg("table retention: pruned")
+ }
+ }
+ return errors.Join(failures...)
+}
diff --git a/internal/jobs/table_retention_test.go b/internal/jobs/table_retention_test.go
new file mode 100644
index 000000000..6a0ee5609
--- /dev/null
+++ b/internal/jobs/table_retention_test.go
@@ -0,0 +1,70 @@
+package jobs
+
+import (
+ "context"
+ "errors"
+ "testing"
+ "time"
+
+ "github.com/warmbly/warmbly/internal/app/instancesettings"
+)
+
+type fakeTableRetention struct {
+ before map[string]time.Time
+ fail string
+}
+
+func (f *fakeTableRetention) record(table string, before time.Time) (int64, error) {
+ f.before[table] = before
+ if table == f.fail {
+ return 0, errors.New("boom")
+ }
+ return 1, nil
+}
+
+func (f *fakeTableRetention) PruneCampaignLogs(_ context.Context, b time.Time) (int64, error) {
+ return f.record("campaign_logs", b)
+}
+func (f *fakeTableRetention) PruneNotifications(_ context.Context, b time.Time) (int64, error) {
+ return f.record("notifications", b)
+}
+func (f *fakeTableRetention) PruneWebhookDeliveries(_ context.Context, b time.Time) (int64, error) {
+ return f.record("webhook_deliveries", b)
+}
+func (f *fakeTableRetention) PruneTrackingDedupe(_ context.Context, b time.Time) (int64, error) {
+ return f.record("tracking_events_processed", b)
+}
+func (f *fakeTableRetention) PruneWorkerHealthSamples(_ context.Context, b time.Time) (int64, error) {
+ return f.record("worker_health_samples", b)
+}
+func (f *fakeTableRetention) PruneDeliveredSendResultEffects(_ context.Context, b time.Time) (int64, error) {
+ return f.record("send_result_effects", b)
+}
+
+type fixedRetention instancesettings.Retention
+
+func (r fixedRetention) RetentionWindows(context.Context) instancesettings.Retention {
+ return instancesettings.Retention(r)
+}
+
+func TestTableRetentionReadsWindowsAndSurvivesAFailingTable(t *testing.T) {
+ store := &fakeTableRetention{before: map[string]time.Time{}, fail: "notifications"}
+ windows := instancesettings.DefaultRetention()
+ windows.CampaignLogDays, windows.WebhookDeliveryDays, windows.EngagementEventDays = 10, 20, 30
+ err := NewTableRetentionJob(store, fixedRetention(windows)).Run(context.Background())
+ if err == nil {
+ t.Fatal("a failing table was not reported")
+ }
+ if len(store.before) != 6 {
+ t.Fatalf("pruned %d tables, want all six despite the failure: %v", len(store.before), store.before)
+ }
+ for table, days := range map[string]int{
+ "campaign_logs": 10, "webhook_deliveries": 20, "tracking_events_processed": 30,
+ "worker_health_samples": workerHealthSampleKeepDays, "send_result_effects": sendResultEffectKeepDays,
+ } {
+ age := time.Since(store.before[table])
+ if want := time.Duration(days) * 24 * time.Hour; age < want-2*time.Hour || age > want+2*time.Hour {
+ t.Errorf("%s pruned before %v ago, want %d days", table, age, days)
+ }
+ }
+}
diff --git a/internal/repository/pg_email_opens.go b/internal/repository/pg_email_opens.go
index 8491d7d1f..0abe39139 100644
--- a/internal/repository/pg_email_opens.go
+++ b/internal/repository/pg_email_opens.go
@@ -98,13 +98,8 @@ func (r *emailOpenRepository) HasHumanOpen(ctx context.Context, campaignID, cont
}
func (r *emailOpenRepository) Cleanup(ctx context.Context, olderThanDays int) (int64, error) {
- tag, err := r.db.Exec(ctx,
- `DELETE FROM email_opens WHERE opened_at < NOW() - $1 * INTERVAL '1 day'`,
- olderThanDays)
- if err != nil {
- return 0, err
- }
- return tag.RowsAffected(), nil
+ return DeleteInBatches(ctx, r.db, "email_opens",
+ `SELECT ctid FROM email_opens WHERE opened_at < NOW() - $2 * INTERVAL '1 day' LIMIT $1`, olderThanDays)
}
// OriginDeriver re-reads one logged event's origin from its stored user agent
diff --git a/internal/repository/pg_link_clicks.go b/internal/repository/pg_link_clicks.go
index 9205df22f..56a66aea3 100644
--- a/internal/repository/pg_link_clicks.go
+++ b/internal/repository/pg_link_clicks.go
@@ -179,13 +179,8 @@ func (r *linkClickRepository) ListPendingAnnouncements(ctx context.Context, befo
}
func (r *linkClickRepository) Cleanup(ctx context.Context, olderThanDays int) (int64, error) {
- tag, err := r.db.Exec(ctx,
- `DELETE FROM email_link_clicks WHERE clicked_at < NOW() - $1 * INTERVAL '1 day'`,
- olderThanDays)
- if err != nil {
- return 0, err
- }
- return tag.RowsAffected(), nil
+ return DeleteInBatches(ctx, r.db, "email_link_clicks",
+ `SELECT ctid FROM email_link_clicks WHERE clicked_at < NOW() - $2 * INTERVAL '1 day' LIMIT $1`, olderThanDays)
}
func (r *linkClickRepository) CountRecentOtherLinks(ctx context.Context, taskID uuid.UUID, ipHash string, linkID *uuid.UUID, destination string, since time.Time) (int, error) {
diff --git a/internal/repository/pg_table_retention.go b/internal/repository/pg_table_retention.go
new file mode 100644
index 000000000..8880a6ab7
--- /dev/null
+++ b/internal/repository/pg_table_retention.go
@@ -0,0 +1,87 @@
+package repository
+
+import (
+ "context"
+ "time"
+
+ "github.com/jackc/pgx/v5/pgxpool"
+)
+
+const (
+ // PruneBatch is how many rows one delete statement removes.
+ PruneBatch = 5000
+ // prunePause lets autovacuum, replication and live writes in between batches.
+ prunePause = 50 * time.Millisecond
+)
+
+// DeleteInBatches runs a ctid-selecting delete until it removes fewer than a
+// full batch. selectCtids is `SELECT ctid FROM t WHERE ... LIMIT $1` with the
+// batch size as $1 and its own arguments after it.
+func DeleteInBatches(ctx context.Context, db *pgxpool.Pool, table, selectCtids string, args ...any) (int64, error) {
+ query := `DELETE FROM ` + table + ` WHERE ctid = ANY(ARRAY(` + selectCtids + `))`
+ var total int64
+ for {
+ tag, err := db.Exec(ctx, query, append([]any{PruneBatch}, args...)...)
+ if err != nil {
+ return total, err
+ }
+ total += tag.RowsAffected()
+ if tag.RowsAffected() < PruneBatch {
+ return total, nil
+ }
+ select {
+ case <-ctx.Done():
+ return total, ctx.Err()
+ case <-time.After(prunePause):
+ }
+ }
+}
+
+// TableRetentionRepository prunes history that has a retention window and no
+// owner of its own: feeds, logs and finished bookkeeping rows.
+type TableRetentionRepository struct {
+ db *pgxpool.Pool
+}
+
+// NewTableRetentionRepository builds the repository.
+func NewTableRetentionRepository(db *pgxpool.Pool) *TableRetentionRepository {
+ return &TableRetentionRepository{db: db}
+}
+
+// PruneCampaignLogs drops campaign activity-feed entries older than before.
+func (r *TableRetentionRepository) PruneCampaignLogs(ctx context.Context, before time.Time) (int64, error) {
+ return DeleteInBatches(ctx, r.db, "campaign_logs",
+ `SELECT ctid FROM campaign_logs WHERE created_at < $2 LIMIT $1`, before)
+}
+
+// PruneNotifications drops notifications older than before, except one still owed an email.
+func (r *TableRetentionRepository) PruneNotifications(ctx context.Context, before time.Time) (int64, error) {
+ return DeleteInBatches(ctx, r.db, "notifications",
+ `SELECT ctid FROM notifications WHERE created_at < $2
+ AND email_state NOT IN ('pending', 'sending') LIMIT $1`, before)
+}
+
+// PruneWebhookDeliveries drops settled deliveries last touched before before; one still retrying stays.
+func (r *TableRetentionRepository) PruneWebhookDeliveries(ctx context.Context, before time.Time) (int64, error) {
+ return DeleteInBatches(ctx, r.db, "webhook_deliveries",
+ `SELECT ctid FROM webhook_deliveries WHERE status IN ('delivered', 'failed', 'abandoned')
+ AND updated_at < $2 LIMIT $1`, before)
+}
+
+// PruneTrackingDedupe drops consumer-side tracking dedupe marks older than before.
+func (r *TableRetentionRepository) PruneTrackingDedupe(ctx context.Context, before time.Time) (int64, error) {
+ return DeleteInBatches(ctx, r.db, "tracking_events_processed",
+ `SELECT ctid FROM tracking_events_processed WHERE processed_at < $2 LIMIT $1`, before)
+}
+
+// PruneWorkerHealthSamples drops worker health samples older than before.
+func (r *TableRetentionRepository) PruneWorkerHealthSamples(ctx context.Context, before time.Time) (int64, error) {
+ return DeleteInBatches(ctx, r.db, "worker_health_samples",
+ `SELECT ctid FROM worker_health_samples WHERE observed_at < $2 LIMIT $1`, before)
+}
+
+// PruneDeliveredSendResultEffects drops send-result effects delivered before before.
+func (r *TableRetentionRepository) PruneDeliveredSendResultEffects(ctx context.Context, before time.Time) (int64, error) {
+ return DeleteInBatches(ctx, r.db, "send_result_effects",
+ `SELECT ctid FROM send_result_effects WHERE delivered_at IS NOT NULL AND delivered_at < $2 LIMIT $1`, before)
+}
diff --git a/internal/repository/pg_table_retention_live_test.go b/internal/repository/pg_table_retention_live_test.go
new file mode 100644
index 000000000..4fa9b240e
--- /dev/null
+++ b/internal/repository/pg_table_retention_live_test.go
@@ -0,0 +1,151 @@
+package repository
+
+import (
+ "context"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/google/uuid"
+)
+
+// ancient is older than any fixture another live test writes, so a prune in
+// a shared test database only reaches this test's rows.
+const ancientDays = 4000
+
+func TestLiveTableRetentionPrunesOnlyPastWindow(t *testing.T) {
+ _, pool := liveContactDB(t)
+ ctx := context.Background()
+ user, org, campaign, worker, endpoint, account, task := uuid.New(), uuid.New(), uuid.New(), uuid.New(), uuid.New(), uuid.New(), uuid.New()
+ exec := func(query string, args ...any) {
+ t.Helper()
+ if _, err := pool.Exec(ctx, query, args...); err != nil {
+ t.Fatalf("fixture %q: %v", query[:min(60, len(query))], err)
+ }
+ }
+ exec(`INSERT INTO users (id, first_name, last_name, email) VALUES ($1, 'Table', 'Retention', $2)`, user, "table-retention-"+uuid.NewString()+"@example.test")
+ exec(`INSERT INTO organizations (id, name, owner_user_id) VALUES ($1, 'Table retention', $2)`, org, user)
+ exec(`INSERT INTO campaigns (id, user_id, organization_id, name, description, days, status, updated_at, created_at)
+ VALUES ($1, $2, $3, 'Retention', '', 62, 'active', NOW(), NOW())`, campaign, user, org)
+ exec(`INSERT INTO fleet_nodes (id, role, name) VALUES ($1, 'worker', 'table-retention')`, worker)
+ exec(`INSERT INTO workers (id) VALUES ($1)`, worker)
+ exec(`INSERT INTO webhook_endpoints (id, organization_id, url, secret) VALUES ($1, $2, 'https://example.test/hook', 's')`, endpoint, org)
+ exec(`INSERT INTO email_accounts (id, user_id, organization_id, email, name, signature_plain, signature_html, provider)
+ VALUES ($1, $2, $3, $4, 'Retention', '', '', 'smtp_imap')`, account, user, org, "table-retention-"+uuid.NewString()+"@example.test")
+ exec(`INSERT INTO tasks (id, task_type, email_account_id, status, message_id) VALUES ($1, 'campaign', $2, 'completed', '')`, task, account)
+ t.Cleanup(func() {
+ c := context.Background()
+ _, _ = pool.Exec(c, `DELETE FROM tracking_events_processed WHERE task_id = $1`, task)
+ _, _ = pool.Exec(c, `DELETE FROM tasks WHERE id = $1`, task)
+ _, _ = pool.Exec(c, `DELETE FROM email_accounts WHERE id = $1`, account)
+ _, _ = pool.Exec(c, `DELETE FROM workers WHERE id = $1`, worker)
+ _, _ = pool.Exec(c, `DELETE FROM fleet_nodes WHERE id = $1`, worker)
+ _, _ = pool.Exec(c, `DELETE FROM organizations WHERE id = $1`, org)
+ _, _ = pool.Exec(c, `DELETE FROM users WHERE id = $1`, user)
+ })
+
+ old := time.Now().AddDate(0, 0, -ancientDays-1)
+ fresh := time.Now()
+ exec(`INSERT INTO campaign_logs (campaign_id, event_type, created_at) VALUES ($1, 'old', $2), ($1, 'fresh', $3)`, campaign, old, fresh)
+ exec(`INSERT INTO notifications (user_id, organization_id, category, title, created_at, email_state)
+ VALUES ($1, $2, 'c', 'old', $3, 'none'), ($1, $2, 'c', 'owed', $3, 'pending'), ($1, $2, 'c', 'fresh', $4, 'none')`, user, org, old, fresh)
+ exec(`INSERT INTO webhook_deliveries (endpoint_id, organization_id, event_type, event_id, payload, status, updated_at)
+ VALUES ($1, $2, 'old', gen_random_uuid(), '{}', 'delivered', $3),
+ ($1, $2, 'retrying', gen_random_uuid(), '{}', 'pending', $3),
+ ($1, $2, 'fresh', gen_random_uuid(), '{}', 'abandoned', $4)`, endpoint, org, old, fresh)
+ exec(`INSERT INTO worker_health_samples (worker_id, observed_at) VALUES ($1, $2), ($1, $3)`, worker, old, fresh)
+ exec(`INSERT INTO send_result_effects (task_id, effect_key, organization_id, kind, payload, delivered_at)
+ VALUES ($1, 'old', $2, 'webhook', '{}', $3), ($1, 'undelivered', $2, 'webhook', '{}', NULL), ($1, 'fresh', $2, 'webhook', '{}', $4)`, task, org, old, fresh)
+ // More than one batch, so the loop has to come back for the rest.
+ exec(`INSERT INTO tracking_events_processed (task_id, event_type, url_hash, processed_at)
+ SELECT $1, 'click', g::text, $2 FROM generate_series(1, $3::int) g`, task, old, PruneBatch+7)
+ exec(`INSERT INTO tracking_events_processed (task_id, event_type, url_hash, processed_at) VALUES ($1, 'open', '', $2)`, task, fresh)
+
+ repo := NewTableRetentionRepository(pool)
+ before := time.Now().AddDate(0, 0, -ancientDays)
+ for name, prune := range map[string]func(context.Context, time.Time) (int64, error){
+ "campaign_logs": repo.PruneCampaignLogs, "notifications": repo.PruneNotifications, "webhook_deliveries": repo.PruneWebhookDeliveries,
+ "tracking_events_processed": repo.PruneTrackingDedupe, "worker_health_samples": repo.PruneWorkerHealthSamples,
+ "send_result_effects": repo.PruneDeliveredSendResultEffects,
+ } {
+ if _, err := prune(ctx, before); err != nil {
+ t.Fatalf("%s: %v", name, err)
+ }
+ }
+
+ count := func(query string, args ...any) int {
+ t.Helper()
+ var n int
+ if err := pool.QueryRow(ctx, query, args...).Scan(&n); err != nil {
+ t.Fatalf("count: %v", err)
+ }
+ return n
+ }
+ for _, c := range []struct {
+ what string
+ query string
+ arg uuid.UUID
+ want int
+ }{
+ {"campaign feed keeps only the fresh entry", `SELECT count(*) FROM campaign_logs WHERE campaign_id = $1 AND event_type = 'fresh'`, campaign, 1},
+ {"campaign feed drops the old entry", `SELECT count(*) FROM campaign_logs WHERE campaign_id = $1`, campaign, 1},
+ {"notifications keep the fresh one and the one owed an email", `SELECT count(*) FROM notifications WHERE user_id = $1 AND title IN ('fresh', 'owed')`, user, 2},
+ {"notifications drop the old one", `SELECT count(*) FROM notifications WHERE user_id = $1`, user, 2},
+ {"webhook log keeps the retrying and fresh deliveries", `SELECT count(*) FROM webhook_deliveries WHERE endpoint_id = $1 AND event_type IN ('retrying', 'fresh')`, endpoint, 2},
+ {"webhook log drops the old settled delivery", `SELECT count(*) FROM webhook_deliveries WHERE endpoint_id = $1`, endpoint, 2},
+ {"health samples keep the fresh one", `SELECT count(*) FROM worker_health_samples WHERE worker_id = $1`, worker, 1},
+ {"effects keep the undelivered and fresh ones", `SELECT count(*) FROM send_result_effects WHERE task_id = $1`, task, 2},
+ {"dedupe marks keep only the fresh one, across batches", `SELECT count(*) FROM tracking_events_processed WHERE task_id = $1`, task, 1},
+ } {
+ if got := count(c.query, c.arg); got != c.want {
+ t.Errorf("%s: got %d rows, want %d", c.what, got, c.want)
+ }
+ }
+}
+
+// Each prune's row selection is served by an index, so a prune never scans
+// the whole table it trims.
+func TestLiveRetentionPrunesUseTheirIndexes(t *testing.T) {
+ _, pool := liveContactDB(t)
+ ctx := context.Background()
+ conn, err := pool.Acquire(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer conn.Release()
+ if _, err := conn.Exec(ctx, `SET enable_seqscan = off`); err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _, _ = conn.Exec(context.Background(), `RESET enable_seqscan`) }()
+
+ for _, c := range []struct{ query, index string }{
+ {`SELECT ctid FROM email_opens WHERE opened_at < NOW() - 30 * INTERVAL '1 day' LIMIT 5000`, "idx_email_opens_opened_at"},
+ {`SELECT ctid FROM email_link_clicks WHERE clicked_at < NOW() - 30 * INTERVAL '1 day' LIMIT 5000`, "idx_email_link_clicks_clicked_at"},
+ {`SELECT ctid FROM campaign_logs WHERE created_at < NOW() LIMIT 5000`, "idx_campaign_logs_created"},
+ {`SELECT ctid FROM notifications WHERE created_at < NOW() AND email_state NOT IN ('pending', 'sending') LIMIT 5000`, "idx_notifications_created"},
+ {`SELECT ctid FROM webhook_deliveries WHERE status IN ('delivered', 'failed', 'abandoned') AND updated_at < NOW() LIMIT 5000`, "idx_webhook_deliveries_settled"},
+ {`SELECT ctid FROM tracking_events_processed WHERE processed_at < NOW() LIMIT 5000`, "idx_tracking_processed_cleanup"},
+ {`SELECT ctid FROM worker_health_samples WHERE observed_at < NOW() LIMIT 5000`, "idx_worker_health_samples_observed"},
+ {`SELECT worker_id, sum(sends_attempted) FROM worker_health_samples WHERE observed_at > now() - interval '1 hour' GROUP BY worker_id`, "idx_worker_health_samples_observed"},
+ {`SELECT ctid FROM send_result_effects WHERE delivered_at IS NOT NULL AND delivered_at < NOW() LIMIT 5000`, "idx_send_result_effects_delivered"},
+ {`SELECT internal_id FROM warmup_received WHERE email_account_id = '00000000-0000-0000-0000-000000000001' AND retired_at IS NULL AND created_at < NOW() ORDER BY created_at LIMIT 100`, "idx_warmup_received_account_live"},
+ {`SELECT token FROM warmup_tokens WHERE sender_account_id = '00000000-0000-0000-0000-000000000001' AND sent_message_id <> '' AND sent_retired_at IS NULL AND created_at < NOW() ORDER BY created_at LIMIT 100`, "idx_warmup_tokens_sender_live"},
+ } {
+ rows, err := conn.Query(ctx, `EXPLAIN `+c.query)
+ if err != nil {
+ t.Fatalf("explain %q: %v", c.query, err)
+ }
+ var plan strings.Builder
+ for rows.Next() {
+ var line string
+ if err := rows.Scan(&line); err != nil {
+ t.Fatal(err)
+ }
+ plan.WriteString(line + "\n")
+ }
+ rows.Close()
+ if !strings.Contains(plan.String(), c.index) {
+ t.Errorf("%s is not used by %q:\n%s", c.index, c.query, plan.String())
+ }
+ }
+}
diff --git a/internal/repository/pg_tracking_dedupe.go b/internal/repository/pg_tracking_dedupe.go
index b02317ca6..d086f5417 100644
--- a/internal/repository/pg_tracking_dedupe.go
+++ b/internal/repository/pg_tracking_dedupe.go
@@ -11,7 +11,6 @@ import (
type TrackingDedupeRepository interface {
IsProcessed(ctx context.Context, taskID uuid.UUID, eventType, urlHash string) (bool, error)
MarkProcessed(ctx context.Context, taskID uuid.UUID, eventType, urlHash string) error
- Cleanup(ctx context.Context, olderThanDays int) (int64, error)
}
type trackingDedupeRepository struct {
@@ -63,18 +62,3 @@ func (r *trackingDedupeRepository) MarkProcessed(ctx context.Context, taskID uui
_, err := r.db.Exec(ctx, query, taskID, eventType, urlHash)
return err
}
-
-// Cleanup removes old processed tracking events (for scheduled job)
-func (r *trackingDedupeRepository) Cleanup(ctx context.Context, olderThanDays int) (int64, error) {
- query := `
- DELETE FROM tracking_events_processed
- WHERE processed_at < NOW() - $1 * INTERVAL '1 day'
- `
-
- result, err := r.db.Exec(ctx, query, olderThanDays)
- if err != nil {
- return 0, err
- }
-
- return result.RowsAffected(), nil
-}
diff --git a/internal/repository/pg_unibox.go b/internal/repository/pg_unibox.go
index 26720aa60..10a45bedf 100644
--- a/internal/repository/pg_unibox.go
+++ b/internal/repository/pg_unibox.go
@@ -155,6 +155,9 @@ type UniboxRepository interface {
// over message text would only ever work for mail synced from now on.
ListMissingBodyText(ctx context.Context, afterID uuid.UUID, limit int) ([]UniboxBodyTarget, error)
SetBodyText(ctx context.Context, id uuid.UUID, bodyText string) error
+ // BodyBackfillProgress and SaveBodyBackfillProgress keep the walk's place across restarts.
+ BodyBackfillProgress(ctx context.Context) (UniboxBodyBackfill, error)
+ SaveBodyBackfillProgress(ctx context.Context, p UniboxBodyBackfill) error
// GroundingByThread and GroundingByAddress return message text for AI
// prompts. They are separate from the preview queries because they select
@@ -2153,6 +2156,41 @@ func (r *uniboxRepository) ListMissingBodyText(ctx context.Context, afterID uuid
return out, rows.Err()
}
+// uniboxBodyBackfillKey is the admin_settings row holding the backfill's place.
+const uniboxBodyBackfillKey = "unibox.body_text_backfill"
+
+// UniboxBodyBackfill is how far the body text backfill has walked, and whether it finished.
+type UniboxBodyBackfill struct {
+ After uuid.UUID `json:"after"`
+ Done bool `json:"done"`
+}
+
+// BodyBackfillProgress returns the stored place, or the start when none is stored.
+func (r *uniboxRepository) BodyBackfillProgress(ctx context.Context) (UniboxBodyBackfill, error) {
+ var p UniboxBodyBackfill
+ var raw []byte
+ err := r.db.QueryRow(ctx, `SELECT value FROM admin_settings WHERE key = $1`, uniboxBodyBackfillKey).Scan(&raw)
+ if errors.Is(err, pgx.ErrNoRows) {
+ return p, nil
+ }
+ if err != nil {
+ return p, err
+ }
+ return p, json.Unmarshal(raw, &p)
+}
+
+// SaveBodyBackfillProgress stores the place reached.
+func (r *uniboxRepository) SaveBodyBackfillProgress(ctx context.Context, p UniboxBodyBackfill) error {
+ raw, err := json.Marshal(p)
+ if err != nil {
+ return err
+ }
+ _, err = r.db.Exec(ctx, `
+ INSERT INTO admin_settings (key, value, updated_at) VALUES ($1, $2, now())
+ ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()`, uniboxBodyBackfillKey, raw)
+ return err
+}
+
// SetBodyText records the searchable rendering of a message body.
func (r *uniboxRepository) SetBodyText(ctx context.Context, id uuid.UUID, bodyText string) error {
_, err := r.db.Exec(ctx, `UPDATE unibox_emails SET body_text = $2 WHERE id = $1`, id, bodyText)
diff --git a/internal/repository/pg_warmup.go b/internal/repository/pg_warmup.go
index 9e08c6ed8..7b879b3bb 100644
--- a/internal/repository/pg_warmup.go
+++ b/internal/repository/pg_warmup.go
@@ -1990,13 +1990,20 @@ func (r *warmupRepository) GetWarmupReceived(ctx context.Context, accountID, int
// the mailbox's own when it set one, else the instance default handed in as
// $1, on active mailboxes that have a worker to act. The provider key comes
// from the message map by the internal id, which is what the worker's remove
-// and flag events are keyed on.
+// and flag events are keyed on. It starts from the mailboxes, so receipts
+// held by a mailbox that cannot act are never walked.
func (r *warmupRepository) ListWarmupMailToRetire(ctx context.Context, defaultDays, limit int) ([]WarmupMailToRetire, error) {
query := `
SELECT ea.user_id, ea.id, ` + actionWorkerSQL("ea") + `, wr.internal_id, wr.message_id,
COALESCE(m.message_id, ''), ea.warmup_placement, ea.warmup_folder
- FROM warmup_received wr
- JOIN email_accounts ea ON ea.id = wr.email_account_id
+ FROM email_accounts ea
+ CROSS JOIN LATERAL (
+ SELECT r.internal_id, r.message_id, r.created_at FROM warmup_received r
+ WHERE r.email_account_id = ea.id AND r.retired_at IS NULL
+ AND r.created_at < NOW() - make_interval(days => COALESCE(ea.warmup_retention_days, $1))
+ ORDER BY r.created_at
+ LIMIT $2
+ ) wr
LEFT JOIN LATERAL (
SELECT em.message_id FROM email_message_map em
WHERE em.email_id = ea.id AND em.id = wr.internal_id
@@ -2004,8 +2011,6 @@ func (r *warmupRepository) ListWarmupMailToRetire(ctx context.Context, defaultDa
) m ON true
WHERE ea.status = 'active'
AND ` + actionRoutableSQL("ea") + `
- AND wr.retired_at IS NULL
- AND wr.created_at < NOW() - make_interval(days => COALESCE(ea.warmup_retention_days, $1))
ORDER BY wr.created_at
LIMIT $2`
return r.scanMailToRetire(ctx, query, defaultDays, limit, false)
@@ -2017,12 +2022,19 @@ func (r *warmupRepository) ListWarmupMailToRetire(ctx context.Context, defaultDa
// senders are left out rather than searched for a message that does not
// exist. The map is keyed by the provider's id on Gmail and Graph, so the key
// is usually empty there and the worker searches by Message-ID instead.
+// Like the received side it starts from the mailboxes that can act.
func (r *warmupRepository) ListWarmupSentCopiesToRetire(ctx context.Context, defaultDays, limit int) ([]WarmupMailToRetire, error) {
query := `
SELECT ea.user_id, ea.id, ` + actionWorkerSQL("ea") + `, wt.token, wt.sent_message_id,
COALESCE(m.message_id, ''), ea.warmup_placement, ea.warmup_folder
- FROM warmup_tokens wt
- JOIN email_accounts ea ON ea.id = wt.sender_account_id
+ FROM email_accounts ea
+ CROSS JOIN LATERAL (
+ SELECT t.token, t.sent_message_id, t.created_at FROM warmup_tokens t
+ WHERE t.sender_account_id = ea.id AND t.sent_message_id <> '' AND t.sent_retired_at IS NULL
+ AND t.created_at < NOW() - make_interval(days => COALESCE(ea.warmup_retention_days, $1))
+ ORDER BY t.created_at
+ LIMIT $2
+ ) wt
LEFT JOIN LATERAL (
SELECT em.message_id FROM email_message_map em
WHERE em.email_id = ea.id AND em.message_id = wt.sent_message_id
@@ -2031,9 +2043,6 @@ func (r *warmupRepository) ListWarmupSentCopiesToRetire(ctx context.Context, def
WHERE ea.status = 'active'
AND ` + actionRoutableSQL("ea") + `
AND ea.provider <> 'smtp_imap'
- AND wt.sent_message_id <> ''
- AND wt.sent_retired_at IS NULL
- AND wt.created_at < NOW() - make_interval(days => COALESCE(ea.warmup_retention_days, $1))
ORDER BY wt.created_at
LIMIT $2`
return r.scanMailToRetire(ctx, query, defaultDays, limit, true)
@@ -2104,6 +2113,8 @@ func (r *warmupRepository) PruneWarmupEventsBefore(ctx context.Context, before t
AND (sent_message_id = '' OR sent_retired_at IS NOT NULL
OR sender_account_id IN (SELECT id FROM email_accounts WHERE provider = 'smtp_imap'))`, []any{before}},
{`DELETE FROM pool_link_commands WHERE status IN ('answered', 'expired', 'cancelled') AND created_at < $1`, []any{before}},
+ // Thread turns go at the age of the receipts and tokens they chain back to.
+ {`DELETE FROM warmup_thread_messages WHERE created_at < $1`, []any{before}},
// An arrival's handle is how a retiring delete reaches the instance, so it outlives the copy it names.
{`DELETE FROM pool_link_arrivals a WHERE a.reported_at < $1
AND NOT EXISTS (SELECT 1 FROM warmup_received wr WHERE wr.email_account_id = a.email_account_id
diff --git a/internal/repository/send_result_effects.go b/internal/repository/send_result_effects.go
index 979eebb99..377cf85ed 100644
--- a/internal/repository/send_result_effects.go
+++ b/internal/repository/send_result_effects.go
@@ -78,8 +78,7 @@ func (r *advancedOutreachRepository) DeliverSendResultEffects(ctx context.Contex
break
}
}
- _, err := r.db.Exec(ctx, `DELETE FROM send_result_effects WHERE delivered_at