fix: log silent IMAP publish failures and dropped sync state reports (#1157)

Two paths left last_synced_at stale with nothing in the logs, which is
what shows as "No recent successful sync check":

- The IMAP pass ended with nil when a MAILBOX_UPDATE, MAILBOX_DELETE or
  MAILBOX_RENAME publish failed. Those five sites now go through
  controlPlaneError, the helper the rest of the pass already uses for
  publish failures: it logs a warning with email_id and the error and
  emits ControlPlaneHeld evidence. It still returns nil and holds cursors.
- HandleSyncState returned nil without a word when PutFromWorker rejected
  a report because the worker or transport generation no longer matched.
  It now logs a warning with email_id, worker_id, report_generation and
  transport_generation, and still returns nil.

Logging only, no behaviour change.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Jaseem Thayal
2026-11-05 14:08:17 +05:30
co-authored by Claude Opus 5.5
parent 0df02b570c
commit f04d2a1bad
4 changed files with 52 additions and 6 deletions
@@ -55,6 +55,15 @@ func (s *JobsService) HandleSyncState(ctx context.Context, e *models.JobEventSyn
return err
}
if !accepted {
// The mailbox moved to another worker or transport generation. The
// report is dropped, so last_synced_at does not move; say so instead of
// leaving a stale "last synced" with nothing in the logs.
log.Warn().
Str("email_id", e.EmailID.String()).
Str("worker_id", e.WorkerID.String()).
Str("report_generation", e.ReportGeneration.String()).
Str("transport_generation", e.State.TransportGeneration.String()).
Msg("sync state report dropped: reporting worker no longer owns the mailbox at this generation")
return nil
}
+14 -1
View File
@@ -1,13 +1,17 @@
package jobs
import (
"bytes"
"context"
"encoding/json"
"errors"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/infrastructure/pubsub"
"github.com/warmbly/warmbly/internal/models"
@@ -96,9 +100,18 @@ func TestSyncStateChecksReportingWorkerBeforeSaving(t *testing.T) {
at := time.Now().Add(-time.Minute)
event := &models.JobEventSyncState{WorkerID: &workerID, ReportGeneration: syncTestGeneration, UserID: uuid.New(), EmailID: uuid.New(),
State: models.SyncState{TransportGeneration: syncTestGeneration, LastSyncedAt: &at, BackfillStatus: models.SyncBackfillComplete, FoldersSkippedCap: 4}}
if err := s.HandleSyncState(t.Context(), event); err != nil {
var output bytes.Buffer
previous := log.Logger
log.Logger = zerolog.New(&output)
err := s.HandleSyncState(t.Context(), event)
log.Logger = previous
if err != nil {
t.Fatal(err)
}
// A dropped report leaves last_synced_at stale, so it must be logged (#1157).
if logged := strings.Contains(output.String(), "sync state report dropped") && strings.Contains(output.String(), workerID.String()); logged == accepted {
t.Fatalf("accepted=%t drop logged=%t: %s", accepted, logged, output.String())
}
if repo.reportWorker != workerID || repo.reportGeneration != syncTestGeneration || (repo.put != nil) != accepted {
t.Fatalf("worker=%s accepted=%t saved=%+v", repo.reportWorker, accepted, repo.put)
}
@@ -5,6 +5,7 @@ import (
"testing"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/pkg/nodeevidence"
"github.com/warmbly/warmbly/internal/repository"
)
@@ -22,3 +23,26 @@ func TestControlPlane503EvidenceHoldsCursorsAndIsNotProviderFailure(t *testing.T
t.Fatal("missing cursor-held evidence", got)
}
}
// A failed MAILBOX_DELETE publish used to end the IMAP pass with no log and no
// evidence, so the mailbox only showed up as "No recent successful sync check"
// (#1157). It still ends the pass and holds state, but now says why.
func TestImapSyncPublishFailureIsReported(t *testing.T) {
var got nodeevidence.Event
nodeevidence.Install(func(e nodeevidence.Event) { got = e })
t.Cleanup(func() { nodeevidence.Install(nil) })
conn := &fakeImapConn{folders: []models.Mailbox{{Name: "INBOX", UIDValidity: 7, HighestModSeq: 100}}}
w, _ := newIMAPTestMail(conn, &fixedBudget{allow: 10}, &models.Mailbox{Name: "INBOX", UIDValidity: 7, HighestModSeq: 100})
w.SmtpImapData.Mailboxes = append(w.SmtpImapData.Mailboxes, &models.Mailbox{Name: "Old", UIDValidity: 5, HighestModSeq: 10})
w.onEvent = func(models.JobEventType, any) error { return fmt.Errorf("bus down") }
if err := w.Sync(t.Context()); err != nil {
t.Fatalf("Sync: %v", err)
}
if got.Name != nodeevidence.ControlPlaneHeld || got.MailboxID != w.ID {
t.Fatal("publish failure left no evidence", got)
}
if len(w.SmtpImapData.Mailboxes) != 2 {
t.Fatal("the folder was retired although its delete was never published")
}
}
+5 -5
View File
@@ -84,7 +84,7 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError {
selectionComplete := client.FolderOverflow() == 0 && client.FolderConflicts() == 0
if selectionComplete {
if err := w.imapFollowRenames(folders, listing.Present); err != nil {
return nil
return w.controlPlaneError(err, stats)
}
}
@@ -112,7 +112,7 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError {
return nil
}
if err := w.mboxEvent(&saved); err != nil {
return nil
return w.controlPlaneError(err, stats)
}
w.SmtpImapData.Mailboxes = append(w.SmtpImapData.Mailboxes, &saved)
continue
@@ -132,7 +132,7 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError {
return nil
}
if err := w.mboxEvent(&saved); err != nil {
return nil
return w.controlPlaneError(err, stats)
}
*befBox = saved
delete(w.flagScan, box.Name)
@@ -188,7 +188,7 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError {
advanceToView(&next, view)
}
if err := w.mboxEvent(&next); err != nil {
return nil
return w.controlPlaneError(err, stats)
}
// befBox is the stored copy itself, matched by name, so there is
// nothing else in the list to keep in step with it.
@@ -246,7 +246,7 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError {
// the mail already stored from it along.
Skipped: imap.SkipsFolder(*box, skip) || imapRetiredIntoSkipped(box, gone, skipped),
}); err != nil {
return nil
return w.controlPlaneError(err, stats)
}
deleted = append(deleted, box.Name)
}