Merge pull request #1084 from warmbly/feat/campaign-loading-deferrals

feat: keep campaign mailbox reconnect retries informational and preserve send attempts
This commit is contained in:
Matthew Meszaros
2026-10-28 14:51:01 +00:00
committed by GitHub
7 changed files with 287 additions and 6 deletions
+11
View File
@@ -247,6 +247,17 @@ jobs:
done
if grep -q -- '--- SKIP:' linked-mailbox-batch.log; then exit 1; fi
- name: Campaign mailbox reconnect retry regressions (PostgreSQL)
if: matrix.suite == 'live-1'
env:
WARMBLY_TEST_DB: postgres://warmbly:warmbly@127.0.0.1:5432/warmbly_test?sslmode=disable
run: |
set -o pipefail
go test -race -v -p 1 ./internal/app/consumer ./internal/repository -run '^TestLiveCampaign(MailboxLoadingDefersWithoutSpendingAttempts|LoadingLogsKeepHistoryWithoutCustomerAlerts)$' -count=1 | tee campaign-reconnect.log
grep -q '^--- PASS: TestLiveCampaignMailboxLoadingDefersWithoutSpendingAttempts ' campaign-reconnect.log
grep -q '^--- PASS: TestLiveCampaignLoadingLogsKeepHistoryWithoutCustomerAlerts ' campaign-reconnect.log
if grep -q -- '--- SKIP:' campaign-reconnect.log; then exit 1; fi
- name: Contact campaign empty-step regression (PostgreSQL)
if: matrix.suite == 'live-1'
env:
+2 -2
View File
@@ -347,7 +347,7 @@ When a step's branches are still undecided (`if they open within 3 days`), the p
One lead that cannot be sent right now never holds up the campaign. Each pass walks the queue in order and sends the first lead it can place, so a contact waiting for a same-provider mailbox, for the mailbox their conversation belongs to, or for their own working hours is skipped for that pass while the leads behind them go out. A pass tries up to 200 due leads this way. The campaign only waits when every lead it looked at is waiting, says so once a day in the activity log, and comes back and looks again within fifteen minutes.
A step counts as sent only once the sending worker has handed it to the mailbox provider. If the worker cannot send (the mailbox was still loading, the provider refused the message, a storage hiccup), the step goes back to the queue, the failure appears in **Needs attention** with the reason, and the next pass retries it. A mailbox that no running sending worker holds is passed over rather than picked, and a pass that fails for any reason is followed by another about a minute later, so a failure never leaves a campaign's leads queued until it is restarted. After five failed attempts the lead is marked **Failed** and dropped from the campaign, so a mailbox that can never send does not loop forever. A recipient the mail server refuses outright is not retried: it is recorded as a **Bounced** lead straight away and goes through the same bounce handling (suppression, guardrails, webhooks) as a bounce notice. A campaign that finished while a send was still in flight reopens to retry it.
A step counts as sent only once the sending worker has handed it to the mailbox provider. If a mailbox is still reconnecting after a worker restart or upgrade, the unsent step goes back to the queue automatically. This is an informational delay in the recent log, costs no failed-send attempt, and does not appear in **Needs attention**. Historical reconnect retries receive the same presentation only when stored task evidence confirms an internal preparation retry; unverified historical failures remain unchanged. Provider refusals and other send failures keep their reasons in **Needs attention**, and retryable failures return to the queue. A mailbox that no running sending worker holds is passed over rather than picked, and a pass that fails for any reason is followed by another about a minute later, so a failure never leaves a campaign's leads queued until it is restarted. After five counted send failures the lead is marked **Failed** and dropped from the campaign, so a mailbox that can never send does not loop forever. A recipient the mail server refuses outright is not retried: it is recorded as a **Bounced** lead straight away and goes through the same bounce handling (suppression, guardrails, webhooks) as a bounce notice. A campaign that finished while a send was still in flight reopens to retry it.
Every sent count counts emails only: the contact's **Sent** total, a lead's sent steps, the campaign's progress and the rates its guardrails judge. Wait, condition and action steps move a lead through the flow but send nothing, so a lead is **Done** once every email step has gone out, however many other steps sit between them.
@@ -465,7 +465,7 @@ A paused campaign shows the reason and the numbers behind it on the **Auto-pause
## Live activity
The detail view streams sends, opens, clicks, replies, bounces, and skips as they happen, for every teammate at once. The **Sending** card appears only while an email is actually being handed to a worker, and always names the contact; between sends the panel shows the campaign's progress and the log, not a send in flight. A **Needs attention** panel above it surfaces failures such as a step that could not be scheduled, a send the worker could not complete, or a mailbox whose sending worker is offline; these retry automatically and stay visible so nothing fails silently.
The detail view streams sends, opens, clicks, replies, bounces, and skips as they happen, for every teammate at once. The **Sending** card appears only while an email is actually being handed to a worker, and always names the contact; between sends the panel shows the campaign's progress and the log, not a send in flight. A **Needs attention** panel above it surfaces failures such as a step that could not be scheduled, a send the worker could not complete, or a mailbox whose sending worker is offline; retryable failures return to the queue automatically. Routine mailbox reconnect delays stay in the informational recent log while Warmbly retries, rather than asking the user to intervene.
If scheduling stalls on a transient infrastructure hiccup, Warmbly re-seeds within a few minutes. Pausing and resuming forces a fresh start.
@@ -0,0 +1,127 @@
package jobs
import (
"context"
"testing"
"time"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/repository"
)
func TestMailboxLoadingRequiresPreparationEvidence(t *testing.T) {
for _, tc := range []struct {
name string
code string
failure *errx.SendFailure
loading bool
}{
{name: "missing protocol evidence"},
{name: "preparation", failure: &errx.SendFailure{Protocol: "internal", Stage: "prepare", Disposition: errx.SendRetry}, loading: true},
{name: "provider retry", failure: &errx.SendFailure{Protocol: "smtp", Stage: "connect", Disposition: errx.SendRetry}},
{name: "ambiguous", failure: &errx.SendFailure{Protocol: "internal", Stage: "prepare", Disposition: errx.SendAmbiguous}},
{name: "authentication", code: string(errx.MailErrorCodeInvalidCredentials)},
} {
t.Run(tc.name, func(t *testing.T) {
_, code := sendFailureReason(models.SendEmailResult{Error: &models.EmailSendError{Code: tc.code, Message: models.MailboxNotLoadedPrefix, Failure: tc.failure}})
if (code == models.MailboxLoadingCode) != tc.loading {
t.Fatalf("code=%s, loading=%t", code, tc.loading)
}
})
}
}
func TestLiveCampaignMailboxLoadingDefersWithoutSpendingAttempts(t *testing.T) {
s, handle := liveWarmupService(t)
f := newSendResultFixture(t, handle)
s.TaskRepo = repository.NewTaskRepository(handle.Pool)
s.CampaignRepo = repository.NewCampaignRepostory(handle)
s.CampaignProgressRepo = repository.NewCampaignProgressRepository(handle.Pool)
s.CampaignLogRepo = repository.NewCampaignLogRepository(handle)
s.ContactRepo = repository.NewContactRepostory(handle)
ctx := context.Background()
var historicalTask string
for i := 0; i < 6; i++ {
task := f.stampSend(t, s)
result := models.SendEmailResult{TaskID: task, SentAt: time.Now(), Error: &models.EmailSendError{
Code: string(errx.MailErrorCodeServerUnreachable), Message: models.MailboxNotLoadedPrefix,
Failure: &errx.SendFailure{Protocol: "internal", Stage: "prepare", Disposition: errx.SendRetry, ObservedAt: time.Now()},
}}
for duplicate := 0; duplicate < 2; duplicate++ {
if err := s.HandleEmailFailed(ctx, result); err != nil {
t.Fatal(err)
}
}
if i == 0 {
historicalTask = task.String()
if _, err := handle.Exec(ctx, `UPDATE campaign_logs SET event_type='email_failed', metadata=(metadata-'mailbox_loading_retry') || '{"level":"error","code":"SERVER_UNREACHABLE"}'::jsonb WHERE campaign_id=$1 AND metadata->>'task_id'=$2`, f.campaign, historicalTask); err != nil {
t.Fatal(err)
}
}
var attempts, dailySends int
var sent, dispatched *time.Time
if err := handle.QueryRow(ctx, `SELECT send_attempts,sent_at,dispatched_at FROM campaign_contact_progress WHERE campaign_id=$1 AND contact_id=$2 AND sequence_id=$3`, f.campaign, f.contact, f.step).Scan(&attempts, &sent, &dispatched); err != nil {
t.Fatal(err)
}
if err := handle.QueryRow(ctx, `SELECT COALESCE(SUM(emails_sent),0) FROM campaign_daily_sends WHERE campaign_id=$1`, f.campaign).Scan(&dailySends); err != nil {
t.Fatal(err)
}
if attempts != 0 || sent != nil || dispatched != nil || dailySends != 0 {
t.Fatalf("unattempted send spent budget or stranded its reservation: attempts=%d daily=%d sent=%v dispatched=%v", attempts, dailySends, sent, dispatched)
}
}
logs, err := s.CampaignLogRepo.GetLogs(ctx, f.campaign, 100, nil)
if err != nil {
t.Fatal(err)
}
if len(logs.Data) != 6 {
t.Fatalf("duplicate results changed the log count: %d", len(logs.Data))
}
for _, entry := range logs.Data {
if entry.EventType != "email_deferred" || entry.Metadata["level"] != "info" || entry.Metadata["will_retry"] != true {
t.Fatalf("reconnect became a customer alert: %+v", entry)
}
if entry.Metadata["code"] == models.MailboxLoadingCode && entry.Metadata["mailbox_loading_retry"] != true {
t.Fatalf("confirmed retry marker missing: %+v", entry)
}
}
if _, err := handle.Exec(ctx, `UPDATE tasks SET send_result_evidence=jsonb_set(send_result_evidence,'{protocol}','"smtp"') WHERE id=$1::uuid`, historicalTask); err != nil {
t.Fatal(err)
}
logs, err = s.CampaignLogRepo.GetLogs(ctx, f.campaign, 100, nil)
if err != nil {
t.Fatal(err)
}
var providerRetained bool
for _, entry := range logs.Data {
if entry.Metadata["task_id"] == historicalTask {
providerRetained = entry.EventType == "email_failed" && entry.Metadata["level"] == "error"
}
}
if !providerRetained {
t.Fatal("provider evidence was reclassified from log text alone")
}
other := newSendResultFixture(t, handle)
if err := s.CampaignLogRepo.CreateLog(ctx, &repository.CampaignLogEntry{CampaignID: other.campaign, EventType: "email_failed", Message: "unconfirmed reconnect", Metadata: map[string]interface{}{
"level": "error", "code": string(errx.MailErrorCodeServerUnreachable), "error": models.MailboxNotLoadedPrefix, "will_retry": true, "task_id": historicalTask,
}}); err != nil {
t.Fatal(err)
}
otherLogs, err := s.CampaignLogRepo.GetLogs(ctx, other.campaign, 100, nil)
if err != nil || len(otherLogs.Data) != 1 || otherLogs.Data[0].Metadata["level"] != "error" {
t.Fatalf("another campaign's task must not authorize reclassification: %+v err=%v", otherLogs, err)
}
task := f.stampSend(t, s)
if err := s.HandleEmailFailed(ctx, models.SendEmailResult{TaskID: task, Error: &models.EmailSendError{Code: string(errx.MailErrorCodeServerUnreachable), Message: "provider connection failed", Failure: &errx.SendFailure{Protocol: "smtp", Stage: "connect", Disposition: errx.SendRetry, ObservedAt: time.Now()}}}); err != nil {
t.Fatal(err)
}
var attempts int
if err := handle.QueryRow(ctx, `SELECT send_attempts FROM campaign_contact_progress WHERE campaign_id=$1 AND contact_id=$2 AND sequence_id=$3`, f.campaign, f.contact, f.step).Scan(&attempts); err != nil || attempts != 1 {
t.Fatalf("provider failure must still spend an attempt: attempts=%d err=%v", attempts, err)
}
logs, err = s.CampaignLogRepo.GetLogs(ctx, f.campaign, 100, nil)
if err != nil || logs.Data[0].Metadata["level"] != "error" {
t.Fatalf("provider failure must stay actionable: %+v err=%v", logs, err)
}
}
+19 -4
View File
@@ -307,7 +307,7 @@ func (s *JobsService) failCampaignSend(ctx context.Context, task *repository.Tas
attempts, exhausted, rolledBack := 0, false, false
if ct.ContactID != nil && ct.SequenceID != nil && s.CampaignProgressRepo != nil {
if copyExcluded {
if copyExcluded || code == models.MailboxLoadingCode {
attempts, exhausted, rolledBack, err = s.CampaignProgressRepo.WalkBackSend(ctx, campaignID, *ct.ContactID, *ct.SequenceID, reason, false)
} else {
attempts, exhausted, rolledBack, err = s.CampaignProgressRepo.RecordSendFailure(ctx, campaignID, *ct.ContactID, *ct.SequenceID, reason)
@@ -473,8 +473,7 @@ func (s *JobsService) publishCampaignUpdated(ctx context.Context, campaign *mode
})
}
// logCampaignSendFailure writes the failure to the campaign activity log.
// metadata.level "error" tints it red in the dashboard's activity feed.
// logCampaignSendFailure records failures and automatic reconnect retries in campaign activity.
func (s *JobsService) logCampaignSendFailure(ctx context.Context, campaignID uuid.UUID, ct *repository.CampaignTask, recipient, reason, code string, attempts int, exhausted, rolledBack, reopened bool) {
if s.CampaignLogRepo == nil {
return
@@ -517,9 +516,16 @@ func (s *JobsService) logCampaignSendFailure(ctx context.Context, campaignID uui
if reopened {
meta["reopened"] = true
}
eventType := "email_failed"
if code == models.MailboxLoadingCode && rolledBack && !exhausted {
eventType = "email_deferred"
meta["level"] = "info"
meta["mailbox_loading_retry"] = true
msg = fmt.Sprintf("Sending to %s was delayed while the mailbox reconnected. An automatic retry was scheduled.", who)
}
_ = s.CampaignLogRepo.CreateLog(ctx, &repository.CampaignLogEntry{
CampaignID: campaignID,
EventType: "email_failed",
EventType: eventType,
Message: msg,
Metadata: meta,
})
@@ -558,6 +564,15 @@ func sendFailureReason(result models.SendEmailResult) (reason, code string) {
if reason == "" {
reason = "the sending worker reported an unknown error"
}
if strings.HasPrefix(reason, models.MailboxNotLoadedPrefix) && (code == "" || code == string(errx.MailErrorCodeServerUnreachable)) {
var failure *errx.SendFailure
if result.Error != nil {
failure = result.Error.Failure
}
if failure != nil && failure.Protocol == "internal" && failure.Stage == "prepare" && failure.Disposition == errx.SendRetry {
code = models.MailboxLoadingCode
}
}
if code == "" {
code = "SEND_FAILED"
}
+1
View File
@@ -4,6 +4,7 @@ import "time"
const (
MailboxNotLoadedPrefix = "The sending worker has not loaded this mailbox yet"
MailboxLoadingCode = "MAILBOX_LOADING"
WarmupFailureLoading = "mailbox_loading"
WarmupLoadingGracePeriod = time.Hour
)
@@ -0,0 +1,58 @@
package repository
import (
"context"
"testing"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/models"
)
func TestLiveCampaignLoadingLogsKeepHistoryWithoutCustomerAlerts(t *testing.T) {
handle, pool := liveContactDB(t)
f := newSharedOrgFixture(t, pool)
repo := NewCampaignLogRepository(handle)
ctx := context.Background()
t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM campaign_logs WHERE campaign_id=$1`, f.campaign) })
for _, tc := range []struct {
name string
code string
retry bool
text string
marker bool
}{
{name: "confirmed retry", code: models.MailboxLoadingCode, retry: true, text: models.MailboxNotLoadedPrefix, marker: true},
{name: "unconfirmed loading retry", code: string(errx.MailErrorCodeServerUnreachable), retry: true, text: models.MailboxNotLoadedPrefix},
{name: "exhausted", code: string(errx.MailErrorCodeServerUnreachable), text: models.MailboxNotLoadedPrefix},
{name: "provider failure", code: string(errx.MailErrorCodeServerUnreachable), retry: true, text: "provider connection failed"},
{name: "auth failure", code: string(errx.MailErrorCodeInvalidCredentials), retry: true, text: models.MailboxNotLoadedPrefix},
} {
if err := repo.CreateLog(ctx, &CampaignLogEntry{CampaignID: f.campaign, EventType: "email_failed", Message: tc.name, Metadata: map[string]interface{}{
"level": "error", "code": tc.code, "error": tc.text, "will_retry": tc.retry, "mailbox_loading_retry": tc.marker,
}}); err != nil {
t.Fatal(err)
}
}
logs, err := repo.GetLogs(ctx, f.campaign, 100, nil)
if err != nil || len(logs.Data) != 5 {
t.Fatalf("logs=%+v err=%v", logs, err)
}
normalized := 0
for _, entry := range logs.Data {
if entry.Message == "Sending was delayed while the mailbox reconnected. An automatic retry was scheduled." {
normalized++
if entry.EventType != "email_deferred" || entry.Metadata["level"] != "info" {
t.Fatalf("loading retry is still an alert: %+v", entry)
}
} else if entry.EventType != "email_failed" || entry.Metadata["level"] != "error" {
t.Fatalf("real or exhausted failure was hidden: %+v", entry)
}
}
if normalized != 1 {
t.Fatalf("normalized %d loading retries, want one", normalized)
}
var raw int
if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM campaign_logs WHERE campaign_id=$1 AND event_type='email_failed' AND metadata->>'level'='error'`, f.campaign).Scan(&raw); err != nil || raw != 5 {
t.Fatalf("historical diagnostics changed: raw=%d err=%v", raw, err)
}
}
+69
View File
@@ -2,9 +2,11 @@ package repository
import (
"context"
"strings"
"time"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/infrastructure/db"
"github.com/warmbly/warmbly/internal/models"
)
@@ -92,6 +94,13 @@ func (r *campaignLogRepository) GetLogs(ctx context.Context, campaignID uuid.UUI
}
logs = append(logs, l)
}
if err := rows.Err(); err != nil {
return nil, err
}
rows.Close()
if err := r.normalizeLoadingRetries(ctx, campaignID, logs); err != nil {
return nil, err
}
var nextCursor *string
hasMore := len(logs) > limit
@@ -110,6 +119,66 @@ func (r *campaignLogRepository) GetLogs(ctx context.Context, campaignID uuid.UUI
}, nil
}
// normalizeLoadingRetries requires a confirmed marker or same-campaign preparation evidence.
func (r *campaignLogRepository) normalizeLoadingRetries(ctx context.Context, campaignID uuid.UUID, logs []models.CampaignLog) error {
candidates := make(map[int]uuid.UUID)
confirmed := make(map[int]bool)
var taskIDs []uuid.UUID
for i := range logs {
l := &logs[i]
reason, _ := l.Metadata["error"].(string)
if l.EventType != "email_failed" || l.Metadata["will_retry"] != true || !strings.HasPrefix(reason, models.MailboxNotLoadedPrefix) {
continue
}
if l.Metadata["mailbox_loading_retry"] == true && l.Metadata["code"] == models.MailboxLoadingCode {
confirmed[i] = true
continue
}
if l.Metadata["code"] != string(errx.MailErrorCodeServerUnreachable) {
continue
}
rawID, _ := l.Metadata["task_id"].(string)
if id, err := uuid.Parse(rawID); err == nil && id != uuid.Nil {
candidates[i] = id
taskIDs = append(taskIDs, id)
}
}
proven := make(map[uuid.UUID]bool)
if len(taskIDs) > 0 {
rows, err := r.DB.Query(ctx, `SELECT ct.task_id FROM campaign_tasks ct JOIN tasks t ON t.id=ct.task_id
WHERE ct.campaign_id=$1 AND ct.task_id=ANY($2) AND t.task_type='campaign'
AND t.send_result_state='failed' AND t.send_result_applied_at IS NOT NULL
AND t.send_result_evidence->>'protocol'='internal'
AND t.send_result_evidence->>'stage'='prepare'
AND t.send_result_evidence->>'disposition'='retry'`, campaignID, taskIDs)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var id uuid.UUID
if err := rows.Scan(&id); err != nil {
return err
}
proven[id] = true
}
if err := rows.Err(); err != nil {
return err
}
}
for i, id := range candidates {
if proven[id] {
confirmed[i] = true
}
}
for i := range confirmed {
logs[i].EventType = "email_deferred"
logs[i].Metadata["level"] = "info"
logs[i].Message = "Sending was delayed while the mailbox reconnected. An automatic retry was scheduled."
}
return nil
}
func (r *campaignLogRepository) CreateLogOnce(ctx context.Context, entry *CampaignLogEntry, key, value string, since time.Time) (bool, error) {
metadata := entry.Metadata
if metadata == nil {