From f04d2a1badb03ad7a786015261fd542e511f02fb Mon Sep 17 00:00:00 2001 From: Jaseem Thayal Date: Thu, 5 Nov 2026 14:08:17 +0530 Subject: [PATCH] fix: log silent IMAP publish failures and dropped sync state reports (#1157) Two paths left last_synced_at stale with nothing in the logs, which is what shows as "No recent successful sync check": - The IMAP pass ended with nil when a MAILBOX_UPDATE, MAILBOX_DELETE or MAILBOX_RENAME publish failed. Those five sites now go through controlPlaneError, the helper the rest of the pass already uses for publish failures: it logs a warning with email_id and the error and emits ControlPlaneHeld evidence. It still returns nil and holds cursors. - HandleSyncState returned nil without a word when PutFromWorker rejected a report because the worker or transport generation no longer matched. It now logs a warning with email_id, worker_id, report_generation and transport_generation, and still returns nil. Logging only, no behaviour change. Co-Authored-By: Claude Opus 5.5 --- internal/app/consumer/event_sync_state.go | 9 +++++++ .../app/consumer/event_sync_state_test.go | 15 +++++++++++- internal/app/worker/wmail/evidence_test.go | 24 +++++++++++++++++++ internal/app/worker/wmail/sync_imap.go | 10 ++++---- 4 files changed, 52 insertions(+), 6 deletions(-) diff --git a/internal/app/consumer/event_sync_state.go b/internal/app/consumer/event_sync_state.go index 5c14bdcf1..d4e92e223 100644 --- a/internal/app/consumer/event_sync_state.go +++ b/internal/app/consumer/event_sync_state.go @@ -55,6 +55,15 @@ func (s *JobsService) HandleSyncState(ctx context.Context, e *models.JobEventSyn return err } if !accepted { + // The mailbox moved to another worker or transport generation. The + // report is dropped, so last_synced_at does not move; say so instead of + // leaving a stale "last synced" with nothing in the logs. + log.Warn(). + Str("email_id", e.EmailID.String()). + Str("worker_id", e.WorkerID.String()). + Str("report_generation", e.ReportGeneration.String()). + Str("transport_generation", e.State.TransportGeneration.String()). + Msg("sync state report dropped: reporting worker no longer owns the mailbox at this generation") return nil } diff --git a/internal/app/consumer/event_sync_state_test.go b/internal/app/consumer/event_sync_state_test.go index 40ff420cd..d398e75bb 100644 --- a/internal/app/consumer/event_sync_state_test.go +++ b/internal/app/consumer/event_sync_state_test.go @@ -1,13 +1,17 @@ package jobs import ( + "bytes" "context" "encoding/json" "errors" + "strings" "testing" "time" "github.com/google/uuid" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" "github.com/warmbly/warmbly/internal/errx" "github.com/warmbly/warmbly/internal/infrastructure/pubsub" "github.com/warmbly/warmbly/internal/models" @@ -96,9 +100,18 @@ func TestSyncStateChecksReportingWorkerBeforeSaving(t *testing.T) { at := time.Now().Add(-time.Minute) event := &models.JobEventSyncState{WorkerID: &workerID, ReportGeneration: syncTestGeneration, UserID: uuid.New(), EmailID: uuid.New(), State: models.SyncState{TransportGeneration: syncTestGeneration, LastSyncedAt: &at, BackfillStatus: models.SyncBackfillComplete, FoldersSkippedCap: 4}} - if err := s.HandleSyncState(t.Context(), event); err != nil { + var output bytes.Buffer + previous := log.Logger + log.Logger = zerolog.New(&output) + err := s.HandleSyncState(t.Context(), event) + log.Logger = previous + if err != nil { t.Fatal(err) } + // A dropped report leaves last_synced_at stale, so it must be logged (#1157). + if logged := strings.Contains(output.String(), "sync state report dropped") && strings.Contains(output.String(), workerID.String()); logged == accepted { + t.Fatalf("accepted=%t drop logged=%t: %s", accepted, logged, output.String()) + } if repo.reportWorker != workerID || repo.reportGeneration != syncTestGeneration || (repo.put != nil) != accepted { t.Fatalf("worker=%s accepted=%t saved=%+v", repo.reportWorker, accepted, repo.put) } diff --git a/internal/app/worker/wmail/evidence_test.go b/internal/app/worker/wmail/evidence_test.go index 400556c3e..e61351b4b 100644 --- a/internal/app/worker/wmail/evidence_test.go +++ b/internal/app/worker/wmail/evidence_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/pkg/nodeevidence" "github.com/warmbly/warmbly/internal/repository" ) @@ -22,3 +23,26 @@ func TestControlPlane503EvidenceHoldsCursorsAndIsNotProviderFailure(t *testing.T t.Fatal("missing cursor-held evidence", got) } } + +// A failed MAILBOX_DELETE publish used to end the IMAP pass with no log and no +// evidence, so the mailbox only showed up as "No recent successful sync check" +// (#1157). It still ends the pass and holds state, but now says why. +func TestImapSyncPublishFailureIsReported(t *testing.T) { + var got nodeevidence.Event + nodeevidence.Install(func(e nodeevidence.Event) { got = e }) + t.Cleanup(func() { nodeevidence.Install(nil) }) + conn := &fakeImapConn{folders: []models.Mailbox{{Name: "INBOX", UIDValidity: 7, HighestModSeq: 100}}} + w, _ := newIMAPTestMail(conn, &fixedBudget{allow: 10}, &models.Mailbox{Name: "INBOX", UIDValidity: 7, HighestModSeq: 100}) + w.SmtpImapData.Mailboxes = append(w.SmtpImapData.Mailboxes, &models.Mailbox{Name: "Old", UIDValidity: 5, HighestModSeq: 10}) + w.onEvent = func(models.JobEventType, any) error { return fmt.Errorf("bus down") } + + if err := w.Sync(t.Context()); err != nil { + t.Fatalf("Sync: %v", err) + } + if got.Name != nodeevidence.ControlPlaneHeld || got.MailboxID != w.ID { + t.Fatal("publish failure left no evidence", got) + } + if len(w.SmtpImapData.Mailboxes) != 2 { + t.Fatal("the folder was retired although its delete was never published") + } +} diff --git a/internal/app/worker/wmail/sync_imap.go b/internal/app/worker/wmail/sync_imap.go index fa427c2b8..7f4c74554 100644 --- a/internal/app/worker/wmail/sync_imap.go +++ b/internal/app/worker/wmail/sync_imap.go @@ -84,7 +84,7 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError { selectionComplete := client.FolderOverflow() == 0 && client.FolderConflicts() == 0 if selectionComplete { if err := w.imapFollowRenames(folders, listing.Present); err != nil { - return nil + return w.controlPlaneError(err, stats) } } @@ -112,7 +112,7 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError { return nil } if err := w.mboxEvent(&saved); err != nil { - return nil + return w.controlPlaneError(err, stats) } w.SmtpImapData.Mailboxes = append(w.SmtpImapData.Mailboxes, &saved) continue @@ -132,7 +132,7 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError { return nil } if err := w.mboxEvent(&saved); err != nil { - return nil + return w.controlPlaneError(err, stats) } *befBox = saved delete(w.flagScan, box.Name) @@ -188,7 +188,7 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError { advanceToView(&next, view) } if err := w.mboxEvent(&next); err != nil { - return nil + return w.controlPlaneError(err, stats) } // befBox is the stored copy itself, matched by name, so there is // nothing else in the list to keep in step with it. @@ -246,7 +246,7 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError { // the mail already stored from it along. Skipped: imap.SkipsFolder(*box, skip) || imapRetiredIntoSkipped(box, gone, skipped), }); err != nil { - return nil + return w.controlPlaneError(err, stats) } deleted = append(deleted, box.Name) }