From b73d5b475ff1a1a4d66fd8d0fde9fac89e5ab685 Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Wed, 28 Oct 2026 15:22:37 +0100 Subject: [PATCH 1/2] feat: retry campaign mailbox reconnects without spending send attempts and present reconnect history as informational activity --- .github/workflows/ci.yml | 11 +++ docs/content/docs/guides/campaigns.mdx | 4 +- .../campaign_loading_deferral_test.go | 91 +++++++++++++++++++ internal/app/consumer/event_send_result.go | 22 ++++- internal/models/warmup_send_failure.go | 1 + .../campaign_loading_log_live_test.go | 56 ++++++++++++ internal/repository/pg_campaign_log.go | 9 ++ 7 files changed, 188 insertions(+), 6 deletions(-) create mode 100644 internal/app/consumer/campaign_loading_deferral_test.go create mode 100644 internal/repository/campaign_loading_log_live_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afe8d3704..334d88cd9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -247,6 +247,17 @@ jobs: done if grep -q -- '--- SKIP:' linked-mailbox-batch.log; then exit 1; fi + - name: Campaign mailbox reconnect retry regressions (PostgreSQL) + if: matrix.suite == 'live-1' + env: + WARMBLY_TEST_DB: postgres://warmbly:warmbly@127.0.0.1:5432/warmbly_test?sslmode=disable + run: | + set -o pipefail + go test -race -v -p 1 ./internal/app/consumer ./internal/repository -run '^TestLiveCampaign(MailboxLoadingDefersWithoutSpendingAttempts|LoadingLogsKeepHistoryWithoutCustomerAlerts)$' -count=1 | tee campaign-reconnect.log + grep -q '^--- PASS: TestLiveCampaignMailboxLoadingDefersWithoutSpendingAttempts ' campaign-reconnect.log + grep -q '^--- PASS: TestLiveCampaignLoadingLogsKeepHistoryWithoutCustomerAlerts ' campaign-reconnect.log + if grep -q -- '--- SKIP:' campaign-reconnect.log; then exit 1; fi + - name: Contact campaign empty-step regression (PostgreSQL) if: matrix.suite == 'live-1' env: diff --git a/docs/content/docs/guides/campaigns.mdx b/docs/content/docs/guides/campaigns.mdx index 992748653..5ea2d8d09 100644 --- a/docs/content/docs/guides/campaigns.mdx +++ b/docs/content/docs/guides/campaigns.mdx @@ -347,7 +347,7 @@ When a step's branches are still undecided (`if they open within 3 days`), the p One lead that cannot be sent right now never holds up the campaign. Each pass walks the queue in order and sends the first lead it can place, so a contact waiting for a same-provider mailbox, for the mailbox their conversation belongs to, or for their own working hours is skipped for that pass while the leads behind them go out. A pass tries up to 200 due leads this way. The campaign only waits when every lead it looked at is waiting, says so once a day in the activity log, and comes back and looks again within fifteen minutes. -A step counts as sent only once the sending worker has handed it to the mailbox provider. If the worker cannot send (the mailbox was still loading, the provider refused the message, a storage hiccup), the step goes back to the queue, the failure appears in **Needs attention** with the reason, and the next pass retries it. A mailbox that no running sending worker holds is passed over rather than picked, and a pass that fails for any reason is followed by another about a minute later, so a failure never leaves a campaign's leads queued until it is restarted. After five failed attempts the lead is marked **Failed** and dropped from the campaign, so a mailbox that can never send does not loop forever. A recipient the mail server refuses outright is not retried: it is recorded as a **Bounced** lead straight away and goes through the same bounce handling (suppression, guardrails, webhooks) as a bounce notice. A campaign that finished while a send was still in flight reopens to retry it. +A step counts as sent only once the sending worker has handed it to the mailbox provider. If a mailbox is still reconnecting after a worker restart or upgrade, the unsent step goes back to the queue automatically. This is an informational delay in the recent log, costs no failed-send attempt, and does not appear in **Needs attention**. The same presentation applies to historical reconnect retries. Provider refusals and other send failures keep their reasons in **Needs attention**, and retryable failures return to the queue. A mailbox that no running sending worker holds is passed over rather than picked, and a pass that fails for any reason is followed by another about a minute later, so a failure never leaves a campaign's leads queued until it is restarted. After five counted send failures the lead is marked **Failed** and dropped from the campaign, so a mailbox that can never send does not loop forever. A recipient the mail server refuses outright is not retried: it is recorded as a **Bounced** lead straight away and goes through the same bounce handling (suppression, guardrails, webhooks) as a bounce notice. A campaign that finished while a send was still in flight reopens to retry it. Every sent count counts emails only: the contact's **Sent** total, a lead's sent steps, the campaign's progress and the rates its guardrails judge. Wait, condition and action steps move a lead through the flow but send nothing, so a lead is **Done** once every email step has gone out, however many other steps sit between them. @@ -465,7 +465,7 @@ A paused campaign shows the reason and the numbers behind it on the **Auto-pause ## Live activity -The detail view streams sends, opens, clicks, replies, bounces, and skips as they happen, for every teammate at once. The **Sending** card appears only while an email is actually being handed to a worker, and always names the contact; between sends the panel shows the campaign's progress and the log, not a send in flight. A **Needs attention** panel above it surfaces failures such as a step that could not be scheduled, a send the worker could not complete, or a mailbox whose sending worker is offline; these retry automatically and stay visible so nothing fails silently. +The detail view streams sends, opens, clicks, replies, bounces, and skips as they happen, for every teammate at once. The **Sending** card appears only while an email is actually being handed to a worker, and always names the contact; between sends the panel shows the campaign's progress and the log, not a send in flight. A **Needs attention** panel above it surfaces failures such as a step that could not be scheduled, a send the worker could not complete, or a mailbox whose sending worker is offline; retryable failures return to the queue automatically. Routine mailbox reconnect delays stay in the informational recent log while Warmbly retries, rather than asking the user to intervene. If scheduling stalls on a transient infrastructure hiccup, Warmbly re-seeds within a few minutes. Pausing and resuming forces a fresh start. diff --git a/internal/app/consumer/campaign_loading_deferral_test.go b/internal/app/consumer/campaign_loading_deferral_test.go new file mode 100644 index 000000000..e7d04bf7f --- /dev/null +++ b/internal/app/consumer/campaign_loading_deferral_test.go @@ -0,0 +1,91 @@ +package jobs + +import ( + "context" + "testing" + "time" + + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +func TestMailboxLoadingRequiresPreparationEvidence(t *testing.T) { + for _, tc := range []struct { + name string + code string + failure *errx.SendFailure + loading bool + }{ + {name: "missing protocol evidence"}, + {name: "preparation", failure: &errx.SendFailure{Protocol: "internal", Stage: "prepare", Disposition: errx.SendRetry}, loading: true}, + {name: "provider retry", failure: &errx.SendFailure{Protocol: "smtp", Stage: "connect", Disposition: errx.SendRetry}}, + {name: "ambiguous", failure: &errx.SendFailure{Protocol: "internal", Stage: "prepare", Disposition: errx.SendAmbiguous}}, + {name: "authentication", code: string(errx.MailErrorCodeInvalidCredentials)}, + } { + t.Run(tc.name, func(t *testing.T) { + _, code := sendFailureReason(models.SendEmailResult{Error: &models.EmailSendError{Code: tc.code, Message: models.MailboxNotLoadedPrefix, Failure: tc.failure}}) + if (code == models.MailboxLoadingCode) != tc.loading { + t.Fatalf("code=%s, loading=%t", code, tc.loading) + } + }) + } +} + +func TestLiveCampaignMailboxLoadingDefersWithoutSpendingAttempts(t *testing.T) { + s, handle := liveWarmupService(t) + f := newSendResultFixture(t, handle) + s.TaskRepo = repository.NewTaskRepository(handle.Pool) + s.CampaignRepo = repository.NewCampaignRepostory(handle) + s.CampaignProgressRepo = repository.NewCampaignProgressRepository(handle.Pool) + s.CampaignLogRepo = repository.NewCampaignLogRepository(handle) + s.ContactRepo = repository.NewContactRepostory(handle) + ctx := context.Background() + for i := 0; i < 6; i++ { + task := f.stampSend(t, s) + result := models.SendEmailResult{TaskID: task, SentAt: time.Now(), Error: &models.EmailSendError{ + Code: string(errx.MailErrorCodeServerUnreachable), Message: models.MailboxNotLoadedPrefix, + Failure: &errx.SendFailure{Protocol: "internal", Stage: "prepare", Disposition: errx.SendRetry, ObservedAt: time.Now()}, + }} + for duplicate := 0; duplicate < 2; duplicate++ { + if err := s.HandleEmailFailed(ctx, result); err != nil { + t.Fatal(err) + } + } + var attempts, dailySends int + var sent, dispatched *time.Time + if err := handle.QueryRow(ctx, `SELECT send_attempts,sent_at,dispatched_at FROM campaign_contact_progress WHERE campaign_id=$1 AND contact_id=$2 AND sequence_id=$3`, f.campaign, f.contact, f.step).Scan(&attempts, &sent, &dispatched); err != nil { + t.Fatal(err) + } + if err := handle.QueryRow(ctx, `SELECT COALESCE(SUM(emails_sent),0) FROM campaign_daily_sends WHERE campaign_id=$1`, f.campaign).Scan(&dailySends); err != nil { + t.Fatal(err) + } + if attempts != 0 || sent != nil || dispatched != nil || dailySends != 0 { + t.Fatalf("unattempted send spent budget or stranded its reservation: attempts=%d daily=%d sent=%v dispatched=%v", attempts, dailySends, sent, dispatched) + } + } + logs, err := s.CampaignLogRepo.GetLogs(ctx, f.campaign, 100, nil) + if err != nil { + t.Fatal(err) + } + if len(logs.Data) != 6 { + t.Fatalf("duplicate results changed the log count: %d", len(logs.Data)) + } + for _, entry := range logs.Data { + if entry.EventType != "email_deferred" || entry.Metadata["level"] != "info" || entry.Metadata["will_retry"] != true { + t.Fatalf("reconnect became a customer alert: %+v", entry) + } + } + task := f.stampSend(t, s) + if err := s.HandleEmailFailed(ctx, models.SendEmailResult{TaskID: task, Error: &models.EmailSendError{Code: string(errx.MailErrorCodeServerUnreachable), Message: "provider connection failed", Failure: &errx.SendFailure{Protocol: "smtp", Stage: "connect", Disposition: errx.SendRetry, ObservedAt: time.Now()}}}); err != nil { + t.Fatal(err) + } + var attempts int + if err := handle.QueryRow(ctx, `SELECT send_attempts FROM campaign_contact_progress WHERE campaign_id=$1 AND contact_id=$2 AND sequence_id=$3`, f.campaign, f.contact, f.step).Scan(&attempts); err != nil || attempts != 1 { + t.Fatalf("provider failure must still spend an attempt: attempts=%d err=%v", attempts, err) + } + logs, err = s.CampaignLogRepo.GetLogs(ctx, f.campaign, 100, nil) + if err != nil || logs.Data[0].Metadata["level"] != "error" { + t.Fatalf("provider failure must stay actionable: %+v err=%v", logs, err) + } +} diff --git a/internal/app/consumer/event_send_result.go b/internal/app/consumer/event_send_result.go index 062286bcf..49291b6e7 100644 --- a/internal/app/consumer/event_send_result.go +++ b/internal/app/consumer/event_send_result.go @@ -307,7 +307,7 @@ func (s *JobsService) failCampaignSend(ctx context.Context, task *repository.Tas attempts, exhausted, rolledBack := 0, false, false if ct.ContactID != nil && ct.SequenceID != nil && s.CampaignProgressRepo != nil { - if copyExcluded { + if copyExcluded || code == models.MailboxLoadingCode { attempts, exhausted, rolledBack, err = s.CampaignProgressRepo.WalkBackSend(ctx, campaignID, *ct.ContactID, *ct.SequenceID, reason, false) } else { attempts, exhausted, rolledBack, err = s.CampaignProgressRepo.RecordSendFailure(ctx, campaignID, *ct.ContactID, *ct.SequenceID, reason) @@ -473,8 +473,7 @@ func (s *JobsService) publishCampaignUpdated(ctx context.Context, campaign *mode }) } -// logCampaignSendFailure writes the failure to the campaign activity log. -// metadata.level "error" tints it red in the dashboard's activity feed. +// logCampaignSendFailure records failures and automatic reconnect retries in campaign activity. func (s *JobsService) logCampaignSendFailure(ctx context.Context, campaignID uuid.UUID, ct *repository.CampaignTask, recipient, reason, code string, attempts int, exhausted, rolledBack, reopened bool) { if s.CampaignLogRepo == nil { return @@ -517,9 +516,15 @@ func (s *JobsService) logCampaignSendFailure(ctx context.Context, campaignID uui if reopened { meta["reopened"] = true } + eventType := "email_failed" + if code == models.MailboxLoadingCode && rolledBack && !exhausted { + eventType = "email_deferred" + meta["level"] = "info" + msg = fmt.Sprintf("Sending to %s was delayed while the mailbox reconnected. An automatic retry was scheduled.", who) + } _ = s.CampaignLogRepo.CreateLog(ctx, &repository.CampaignLogEntry{ CampaignID: campaignID, - EventType: "email_failed", + EventType: eventType, Message: msg, Metadata: meta, }) @@ -558,6 +563,15 @@ func sendFailureReason(result models.SendEmailResult) (reason, code string) { if reason == "" { reason = "the sending worker reported an unknown error" } + if strings.HasPrefix(reason, models.MailboxNotLoadedPrefix) && (code == "" || code == string(errx.MailErrorCodeServerUnreachable)) { + var failure *errx.SendFailure + if result.Error != nil { + failure = result.Error.Failure + } + if failure != nil && failure.Protocol == "internal" && failure.Stage == "prepare" && failure.Disposition == errx.SendRetry { + code = models.MailboxLoadingCode + } + } if code == "" { code = "SEND_FAILED" } diff --git a/internal/models/warmup_send_failure.go b/internal/models/warmup_send_failure.go index de3bea876..797085cb1 100644 --- a/internal/models/warmup_send_failure.go +++ b/internal/models/warmup_send_failure.go @@ -4,6 +4,7 @@ import "time" const ( MailboxNotLoadedPrefix = "The sending worker has not loaded this mailbox yet" + MailboxLoadingCode = "MAILBOX_LOADING" WarmupFailureLoading = "mailbox_loading" WarmupLoadingGracePeriod = time.Hour ) diff --git a/internal/repository/campaign_loading_log_live_test.go b/internal/repository/campaign_loading_log_live_test.go new file mode 100644 index 000000000..5f5bbe3f1 --- /dev/null +++ b/internal/repository/campaign_loading_log_live_test.go @@ -0,0 +1,56 @@ +package repository + +import ( + "context" + "testing" + + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" +) + +func TestLiveCampaignLoadingLogsKeepHistoryWithoutCustomerAlerts(t *testing.T) { + handle, pool := liveContactDB(t) + f := newSharedOrgFixture(t, pool) + repo := NewCampaignLogRepository(handle) + ctx := context.Background() + t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM campaign_logs WHERE campaign_id=$1`, f.campaign) }) + for _, tc := range []struct { + name string + code string + retry bool + text string + }{ + {name: "loading retry", code: string(errx.MailErrorCodeServerUnreachable), retry: true, text: models.MailboxNotLoadedPrefix}, + {name: "exhausted", code: string(errx.MailErrorCodeServerUnreachable), text: models.MailboxNotLoadedPrefix}, + {name: "provider failure", code: string(errx.MailErrorCodeServerUnreachable), retry: true, text: "provider connection failed"}, + {name: "auth failure", code: string(errx.MailErrorCodeInvalidCredentials), retry: true, text: models.MailboxNotLoadedPrefix}, + } { + if err := repo.CreateLog(ctx, &CampaignLogEntry{CampaignID: f.campaign, EventType: "email_failed", Message: tc.name, Metadata: map[string]interface{}{ + "level": "error", "code": tc.code, "error": tc.text, "will_retry": tc.retry, + }}); err != nil { + t.Fatal(err) + } + } + logs, err := repo.GetLogs(ctx, f.campaign, 100, nil) + if err != nil || len(logs.Data) != 4 { + t.Fatalf("logs=%+v err=%v", logs, err) + } + normalized := 0 + for _, entry := range logs.Data { + if entry.Message == "Sending was delayed while the mailbox reconnected. An automatic retry was scheduled." { + normalized++ + if entry.EventType != "email_deferred" || entry.Metadata["level"] != "info" { + t.Fatalf("loading retry is still an alert: %+v", entry) + } + } else if entry.EventType != "email_failed" || entry.Metadata["level"] != "error" { + t.Fatalf("real or exhausted failure was hidden: %+v", entry) + } + } + if normalized != 1 { + t.Fatalf("normalized %d loading retries, want one", normalized) + } + var raw int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM campaign_logs WHERE campaign_id=$1 AND event_type='email_failed' AND metadata->>'level'='error'`, f.campaign).Scan(&raw); err != nil || raw != 4 { + t.Fatalf("historical diagnostics changed: raw=%d err=%v", raw, err) + } +} diff --git a/internal/repository/pg_campaign_log.go b/internal/repository/pg_campaign_log.go index ff5a7564a..772996fd5 100644 --- a/internal/repository/pg_campaign_log.go +++ b/internal/repository/pg_campaign_log.go @@ -2,9 +2,11 @@ package repository import ( "context" + "strings" "time" "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/errx" "github.com/warmbly/warmbly/internal/infrastructure/db" "github.com/warmbly/warmbly/internal/models" ) @@ -90,6 +92,13 @@ func (r *campaignLogRepository) GetLogs(ctx context.Context, campaignID uuid.UUI db.CaptureError(err, "", nil, "scan") return nil, err } + // Historical reconnect retries stay informational without changing stored diagnostics. + if reason, ok := l.Metadata["error"].(string); ok && l.EventType == "email_failed" && + l.Metadata["will_retry"] == true && l.Metadata["code"] == string(errx.MailErrorCodeServerUnreachable) && strings.HasPrefix(reason, models.MailboxNotLoadedPrefix) { + l.EventType = "email_deferred" + l.Metadata["level"] = "info" + l.Message = "Sending was delayed while the mailbox reconnected. An automatic retry was scheduled." + } logs = append(logs, l) } From 6d0746850337bc738292e39866e90d9bf3edba75 Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Wed, 28 Oct 2026 15:38:52 +0100 Subject: [PATCH 2/2] feat: require confirmed retry markers or same-campaign task evidence for informational reconnect history --- docs/content/docs/guides/campaigns.mdx | 2 +- .../campaign_loading_deferral_test.go | 36 +++++++++ internal/app/consumer/event_send_result.go | 1 + .../campaign_loading_log_live_test.go | 18 +++-- internal/repository/pg_campaign_log.go | 74 +++++++++++++++++-- 5 files changed, 115 insertions(+), 16 deletions(-) diff --git a/docs/content/docs/guides/campaigns.mdx b/docs/content/docs/guides/campaigns.mdx index 5ea2d8d09..e81511ae5 100644 --- a/docs/content/docs/guides/campaigns.mdx +++ b/docs/content/docs/guides/campaigns.mdx @@ -347,7 +347,7 @@ When a step's branches are still undecided (`if they open within 3 days`), the p One lead that cannot be sent right now never holds up the campaign. Each pass walks the queue in order and sends the first lead it can place, so a contact waiting for a same-provider mailbox, for the mailbox their conversation belongs to, or for their own working hours is skipped for that pass while the leads behind them go out. A pass tries up to 200 due leads this way. The campaign only waits when every lead it looked at is waiting, says so once a day in the activity log, and comes back and looks again within fifteen minutes. -A step counts as sent only once the sending worker has handed it to the mailbox provider. If a mailbox is still reconnecting after a worker restart or upgrade, the unsent step goes back to the queue automatically. This is an informational delay in the recent log, costs no failed-send attempt, and does not appear in **Needs attention**. The same presentation applies to historical reconnect retries. Provider refusals and other send failures keep their reasons in **Needs attention**, and retryable failures return to the queue. A mailbox that no running sending worker holds is passed over rather than picked, and a pass that fails for any reason is followed by another about a minute later, so a failure never leaves a campaign's leads queued until it is restarted. After five counted send failures the lead is marked **Failed** and dropped from the campaign, so a mailbox that can never send does not loop forever. A recipient the mail server refuses outright is not retried: it is recorded as a **Bounced** lead straight away and goes through the same bounce handling (suppression, guardrails, webhooks) as a bounce notice. A campaign that finished while a send was still in flight reopens to retry it. +A step counts as sent only once the sending worker has handed it to the mailbox provider. If a mailbox is still reconnecting after a worker restart or upgrade, the unsent step goes back to the queue automatically. This is an informational delay in the recent log, costs no failed-send attempt, and does not appear in **Needs attention**. Historical reconnect retries receive the same presentation only when stored task evidence confirms an internal preparation retry; unverified historical failures remain unchanged. Provider refusals and other send failures keep their reasons in **Needs attention**, and retryable failures return to the queue. A mailbox that no running sending worker holds is passed over rather than picked, and a pass that fails for any reason is followed by another about a minute later, so a failure never leaves a campaign's leads queued until it is restarted. After five counted send failures the lead is marked **Failed** and dropped from the campaign, so a mailbox that can never send does not loop forever. A recipient the mail server refuses outright is not retried: it is recorded as a **Bounced** lead straight away and goes through the same bounce handling (suppression, guardrails, webhooks) as a bounce notice. A campaign that finished while a send was still in flight reopens to retry it. Every sent count counts emails only: the contact's **Sent** total, a lead's sent steps, the campaign's progress and the rates its guardrails judge. Wait, condition and action steps move a lead through the flow but send nothing, so a lead is **Done** once every email step has gone out, however many other steps sit between them. diff --git a/internal/app/consumer/campaign_loading_deferral_test.go b/internal/app/consumer/campaign_loading_deferral_test.go index e7d04bf7f..28d9717b1 100644 --- a/internal/app/consumer/campaign_loading_deferral_test.go +++ b/internal/app/consumer/campaign_loading_deferral_test.go @@ -41,6 +41,7 @@ func TestLiveCampaignMailboxLoadingDefersWithoutSpendingAttempts(t *testing.T) { s.CampaignLogRepo = repository.NewCampaignLogRepository(handle) s.ContactRepo = repository.NewContactRepostory(handle) ctx := context.Background() + var historicalTask string for i := 0; i < 6; i++ { task := f.stampSend(t, s) result := models.SendEmailResult{TaskID: task, SentAt: time.Now(), Error: &models.EmailSendError{ @@ -52,6 +53,12 @@ func TestLiveCampaignMailboxLoadingDefersWithoutSpendingAttempts(t *testing.T) { t.Fatal(err) } } + if i == 0 { + historicalTask = task.String() + if _, err := handle.Exec(ctx, `UPDATE campaign_logs SET event_type='email_failed', metadata=(metadata-'mailbox_loading_retry') || '{"level":"error","code":"SERVER_UNREACHABLE"}'::jsonb WHERE campaign_id=$1 AND metadata->>'task_id'=$2`, f.campaign, historicalTask); err != nil { + t.Fatal(err) + } + } var attempts, dailySends int var sent, dispatched *time.Time if err := handle.QueryRow(ctx, `SELECT send_attempts,sent_at,dispatched_at FROM campaign_contact_progress WHERE campaign_id=$1 AND contact_id=$2 AND sequence_id=$3`, f.campaign, f.contact, f.step).Scan(&attempts, &sent, &dispatched); err != nil { @@ -75,6 +82,35 @@ func TestLiveCampaignMailboxLoadingDefersWithoutSpendingAttempts(t *testing.T) { if entry.EventType != "email_deferred" || entry.Metadata["level"] != "info" || entry.Metadata["will_retry"] != true { t.Fatalf("reconnect became a customer alert: %+v", entry) } + if entry.Metadata["code"] == models.MailboxLoadingCode && entry.Metadata["mailbox_loading_retry"] != true { + t.Fatalf("confirmed retry marker missing: %+v", entry) + } + } + if _, err := handle.Exec(ctx, `UPDATE tasks SET send_result_evidence=jsonb_set(send_result_evidence,'{protocol}','"smtp"') WHERE id=$1::uuid`, historicalTask); err != nil { + t.Fatal(err) + } + logs, err = s.CampaignLogRepo.GetLogs(ctx, f.campaign, 100, nil) + if err != nil { + t.Fatal(err) + } + var providerRetained bool + for _, entry := range logs.Data { + if entry.Metadata["task_id"] == historicalTask { + providerRetained = entry.EventType == "email_failed" && entry.Metadata["level"] == "error" + } + } + if !providerRetained { + t.Fatal("provider evidence was reclassified from log text alone") + } + other := newSendResultFixture(t, handle) + if err := s.CampaignLogRepo.CreateLog(ctx, &repository.CampaignLogEntry{CampaignID: other.campaign, EventType: "email_failed", Message: "unconfirmed reconnect", Metadata: map[string]interface{}{ + "level": "error", "code": string(errx.MailErrorCodeServerUnreachable), "error": models.MailboxNotLoadedPrefix, "will_retry": true, "task_id": historicalTask, + }}); err != nil { + t.Fatal(err) + } + otherLogs, err := s.CampaignLogRepo.GetLogs(ctx, other.campaign, 100, nil) + if err != nil || len(otherLogs.Data) != 1 || otherLogs.Data[0].Metadata["level"] != "error" { + t.Fatalf("another campaign's task must not authorize reclassification: %+v err=%v", otherLogs, err) } task := f.stampSend(t, s) if err := s.HandleEmailFailed(ctx, models.SendEmailResult{TaskID: task, Error: &models.EmailSendError{Code: string(errx.MailErrorCodeServerUnreachable), Message: "provider connection failed", Failure: &errx.SendFailure{Protocol: "smtp", Stage: "connect", Disposition: errx.SendRetry, ObservedAt: time.Now()}}}); err != nil { diff --git a/internal/app/consumer/event_send_result.go b/internal/app/consumer/event_send_result.go index 49291b6e7..0e05b98d2 100644 --- a/internal/app/consumer/event_send_result.go +++ b/internal/app/consumer/event_send_result.go @@ -520,6 +520,7 @@ func (s *JobsService) logCampaignSendFailure(ctx context.Context, campaignID uui if code == models.MailboxLoadingCode && rolledBack && !exhausted { eventType = "email_deferred" meta["level"] = "info" + meta["mailbox_loading_retry"] = true msg = fmt.Sprintf("Sending to %s was delayed while the mailbox reconnected. An automatic retry was scheduled.", who) } _ = s.CampaignLogRepo.CreateLog(ctx, &repository.CampaignLogEntry{ diff --git a/internal/repository/campaign_loading_log_live_test.go b/internal/repository/campaign_loading_log_live_test.go index 5f5bbe3f1..f2e12a447 100644 --- a/internal/repository/campaign_loading_log_live_test.go +++ b/internal/repository/campaign_loading_log_live_test.go @@ -15,24 +15,26 @@ func TestLiveCampaignLoadingLogsKeepHistoryWithoutCustomerAlerts(t *testing.T) { ctx := context.Background() t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM campaign_logs WHERE campaign_id=$1`, f.campaign) }) for _, tc := range []struct { - name string - code string - retry bool - text string + name string + code string + retry bool + text string + marker bool }{ - {name: "loading retry", code: string(errx.MailErrorCodeServerUnreachable), retry: true, text: models.MailboxNotLoadedPrefix}, + {name: "confirmed retry", code: models.MailboxLoadingCode, retry: true, text: models.MailboxNotLoadedPrefix, marker: true}, + {name: "unconfirmed loading retry", code: string(errx.MailErrorCodeServerUnreachable), retry: true, text: models.MailboxNotLoadedPrefix}, {name: "exhausted", code: string(errx.MailErrorCodeServerUnreachable), text: models.MailboxNotLoadedPrefix}, {name: "provider failure", code: string(errx.MailErrorCodeServerUnreachable), retry: true, text: "provider connection failed"}, {name: "auth failure", code: string(errx.MailErrorCodeInvalidCredentials), retry: true, text: models.MailboxNotLoadedPrefix}, } { if err := repo.CreateLog(ctx, &CampaignLogEntry{CampaignID: f.campaign, EventType: "email_failed", Message: tc.name, Metadata: map[string]interface{}{ - "level": "error", "code": tc.code, "error": tc.text, "will_retry": tc.retry, + "level": "error", "code": tc.code, "error": tc.text, "will_retry": tc.retry, "mailbox_loading_retry": tc.marker, }}); err != nil { t.Fatal(err) } } logs, err := repo.GetLogs(ctx, f.campaign, 100, nil) - if err != nil || len(logs.Data) != 4 { + if err != nil || len(logs.Data) != 5 { t.Fatalf("logs=%+v err=%v", logs, err) } normalized := 0 @@ -50,7 +52,7 @@ func TestLiveCampaignLoadingLogsKeepHistoryWithoutCustomerAlerts(t *testing.T) { t.Fatalf("normalized %d loading retries, want one", normalized) } var raw int - if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM campaign_logs WHERE campaign_id=$1 AND event_type='email_failed' AND metadata->>'level'='error'`, f.campaign).Scan(&raw); err != nil || raw != 4 { + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM campaign_logs WHERE campaign_id=$1 AND event_type='email_failed' AND metadata->>'level'='error'`, f.campaign).Scan(&raw); err != nil || raw != 5 { t.Fatalf("historical diagnostics changed: raw=%d err=%v", raw, err) } } diff --git a/internal/repository/pg_campaign_log.go b/internal/repository/pg_campaign_log.go index 772996fd5..50e78ddfc 100644 --- a/internal/repository/pg_campaign_log.go +++ b/internal/repository/pg_campaign_log.go @@ -92,15 +92,15 @@ func (r *campaignLogRepository) GetLogs(ctx context.Context, campaignID uuid.UUI db.CaptureError(err, "", nil, "scan") return nil, err } - // Historical reconnect retries stay informational without changing stored diagnostics. - if reason, ok := l.Metadata["error"].(string); ok && l.EventType == "email_failed" && - l.Metadata["will_retry"] == true && l.Metadata["code"] == string(errx.MailErrorCodeServerUnreachable) && strings.HasPrefix(reason, models.MailboxNotLoadedPrefix) { - l.EventType = "email_deferred" - l.Metadata["level"] = "info" - l.Message = "Sending was delayed while the mailbox reconnected. An automatic retry was scheduled." - } logs = append(logs, l) } + if err := rows.Err(); err != nil { + return nil, err + } + rows.Close() + if err := r.normalizeLoadingRetries(ctx, campaignID, logs); err != nil { + return nil, err + } var nextCursor *string hasMore := len(logs) > limit @@ -119,6 +119,66 @@ func (r *campaignLogRepository) GetLogs(ctx context.Context, campaignID uuid.UUI }, nil } +// normalizeLoadingRetries requires a confirmed marker or same-campaign preparation evidence. +func (r *campaignLogRepository) normalizeLoadingRetries(ctx context.Context, campaignID uuid.UUID, logs []models.CampaignLog) error { + candidates := make(map[int]uuid.UUID) + confirmed := make(map[int]bool) + var taskIDs []uuid.UUID + for i := range logs { + l := &logs[i] + reason, _ := l.Metadata["error"].(string) + if l.EventType != "email_failed" || l.Metadata["will_retry"] != true || !strings.HasPrefix(reason, models.MailboxNotLoadedPrefix) { + continue + } + if l.Metadata["mailbox_loading_retry"] == true && l.Metadata["code"] == models.MailboxLoadingCode { + confirmed[i] = true + continue + } + if l.Metadata["code"] != string(errx.MailErrorCodeServerUnreachable) { + continue + } + rawID, _ := l.Metadata["task_id"].(string) + if id, err := uuid.Parse(rawID); err == nil && id != uuid.Nil { + candidates[i] = id + taskIDs = append(taskIDs, id) + } + } + proven := make(map[uuid.UUID]bool) + if len(taskIDs) > 0 { + rows, err := r.DB.Query(ctx, `SELECT ct.task_id FROM campaign_tasks ct JOIN tasks t ON t.id=ct.task_id + WHERE ct.campaign_id=$1 AND ct.task_id=ANY($2) AND t.task_type='campaign' + AND t.send_result_state='failed' AND t.send_result_applied_at IS NOT NULL + AND t.send_result_evidence->>'protocol'='internal' + AND t.send_result_evidence->>'stage'='prepare' + AND t.send_result_evidence->>'disposition'='retry'`, campaignID, taskIDs) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var id uuid.UUID + if err := rows.Scan(&id); err != nil { + return err + } + proven[id] = true + } + if err := rows.Err(); err != nil { + return err + } + } + for i, id := range candidates { + if proven[id] { + confirmed[i] = true + } + } + for i := range confirmed { + logs[i].EventType = "email_deferred" + logs[i].Metadata["level"] = "info" + logs[i].Message = "Sending was delayed while the mailbox reconnected. An automatic retry was scheduled." + } + return nil +} + func (r *campaignLogRepository) CreateLogOnce(ctx context.Context, entry *CampaignLogEntry, key, value string, since time.Time) (bool, error) { metadata := entry.Metadata if metadata == nil {