mirror of
https://github.com/warmbly/warmbly.git
synced 2026-11-17 08:18:51 +00:00
fix: log silent IMAP publish failures and dropped sync state reports (#1157)
Two paths left last_synced_at stale with nothing in the logs, which is what shows as "No recent successful sync check": - The IMAP pass ended with nil when a MAILBOX_UPDATE, MAILBOX_DELETE or MAILBOX_RENAME publish failed. Those five sites now go through controlPlaneError, the helper the rest of the pass already uses for publish failures: it logs a warning with email_id and the error and emits ControlPlaneHeld evidence. It still returns nil and holds cursors. - HandleSyncState returned nil without a word when PutFromWorker rejected a report because the worker or transport generation no longer matched. It now logs a warning with email_id, worker_id, report_generation and transport_generation, and still returns nil. Logging only, no behaviour change. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
0df02b570c
commit
f04d2a1bad
@@ -55,6 +55,15 @@ func (s *JobsService) HandleSyncState(ctx context.Context, e *models.JobEventSyn
|
||||
return err
|
||||
}
|
||||
if !accepted {
|
||||
// The mailbox moved to another worker or transport generation. The
|
||||
// report is dropped, so last_synced_at does not move; say so instead of
|
||||
// leaving a stale "last synced" with nothing in the logs.
|
||||
log.Warn().
|
||||
Str("email_id", e.EmailID.String()).
|
||||
Str("worker_id", e.WorkerID.String()).
|
||||
Str("report_generation", e.ReportGeneration.String()).
|
||||
Str("transport_generation", e.State.TransportGeneration.String()).
|
||||
Msg("sync state report dropped: reporting worker no longer owns the mailbox at this generation")
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1,13 +1,17 @@
|
||||
package jobs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/warmbly/warmbly/internal/errx"
|
||||
"github.com/warmbly/warmbly/internal/infrastructure/pubsub"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
@@ -96,9 +100,18 @@ func TestSyncStateChecksReportingWorkerBeforeSaving(t *testing.T) {
|
||||
at := time.Now().Add(-time.Minute)
|
||||
event := &models.JobEventSyncState{WorkerID: &workerID, ReportGeneration: syncTestGeneration, UserID: uuid.New(), EmailID: uuid.New(),
|
||||
State: models.SyncState{TransportGeneration: syncTestGeneration, LastSyncedAt: &at, BackfillStatus: models.SyncBackfillComplete, FoldersSkippedCap: 4}}
|
||||
if err := s.HandleSyncState(t.Context(), event); err != nil {
|
||||
var output bytes.Buffer
|
||||
previous := log.Logger
|
||||
log.Logger = zerolog.New(&output)
|
||||
err := s.HandleSyncState(t.Context(), event)
|
||||
log.Logger = previous
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A dropped report leaves last_synced_at stale, so it must be logged (#1157).
|
||||
if logged := strings.Contains(output.String(), "sync state report dropped") && strings.Contains(output.String(), workerID.String()); logged == accepted {
|
||||
t.Fatalf("accepted=%t drop logged=%t: %s", accepted, logged, output.String())
|
||||
}
|
||||
if repo.reportWorker != workerID || repo.reportGeneration != syncTestGeneration || (repo.put != nil) != accepted {
|
||||
t.Fatalf("worker=%s accepted=%t saved=%+v", repo.reportWorker, accepted, repo.put)
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
"github.com/warmbly/warmbly/internal/pkg/nodeevidence"
|
||||
"github.com/warmbly/warmbly/internal/repository"
|
||||
)
|
||||
@@ -22,3 +23,26 @@ func TestControlPlane503EvidenceHoldsCursorsAndIsNotProviderFailure(t *testing.T
|
||||
t.Fatal("missing cursor-held evidence", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A failed MAILBOX_DELETE publish used to end the IMAP pass with no log and no
|
||||
// evidence, so the mailbox only showed up as "No recent successful sync check"
|
||||
// (#1157). It still ends the pass and holds state, but now says why.
|
||||
func TestImapSyncPublishFailureIsReported(t *testing.T) {
|
||||
var got nodeevidence.Event
|
||||
nodeevidence.Install(func(e nodeevidence.Event) { got = e })
|
||||
t.Cleanup(func() { nodeevidence.Install(nil) })
|
||||
conn := &fakeImapConn{folders: []models.Mailbox{{Name: "INBOX", UIDValidity: 7, HighestModSeq: 100}}}
|
||||
w, _ := newIMAPTestMail(conn, &fixedBudget{allow: 10}, &models.Mailbox{Name: "INBOX", UIDValidity: 7, HighestModSeq: 100})
|
||||
w.SmtpImapData.Mailboxes = append(w.SmtpImapData.Mailboxes, &models.Mailbox{Name: "Old", UIDValidity: 5, HighestModSeq: 10})
|
||||
w.onEvent = func(models.JobEventType, any) error { return fmt.Errorf("bus down") }
|
||||
|
||||
if err := w.Sync(t.Context()); err != nil {
|
||||
t.Fatalf("Sync: %v", err)
|
||||
}
|
||||
if got.Name != nodeevidence.ControlPlaneHeld || got.MailboxID != w.ID {
|
||||
t.Fatal("publish failure left no evidence", got)
|
||||
}
|
||||
if len(w.SmtpImapData.Mailboxes) != 2 {
|
||||
t.Fatal("the folder was retired although its delete was never published")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,7 +84,7 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError {
|
||||
selectionComplete := client.FolderOverflow() == 0 && client.FolderConflicts() == 0
|
||||
if selectionComplete {
|
||||
if err := w.imapFollowRenames(folders, listing.Present); err != nil {
|
||||
return nil
|
||||
return w.controlPlaneError(err, stats)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -112,7 +112,7 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError {
|
||||
return nil
|
||||
}
|
||||
if err := w.mboxEvent(&saved); err != nil {
|
||||
return nil
|
||||
return w.controlPlaneError(err, stats)
|
||||
}
|
||||
w.SmtpImapData.Mailboxes = append(w.SmtpImapData.Mailboxes, &saved)
|
||||
continue
|
||||
@@ -132,7 +132,7 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError {
|
||||
return nil
|
||||
}
|
||||
if err := w.mboxEvent(&saved); err != nil {
|
||||
return nil
|
||||
return w.controlPlaneError(err, stats)
|
||||
}
|
||||
*befBox = saved
|
||||
delete(w.flagScan, box.Name)
|
||||
@@ -188,7 +188,7 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError {
|
||||
advanceToView(&next, view)
|
||||
}
|
||||
if err := w.mboxEvent(&next); err != nil {
|
||||
return nil
|
||||
return w.controlPlaneError(err, stats)
|
||||
}
|
||||
// befBox is the stored copy itself, matched by name, so there is
|
||||
// nothing else in the list to keep in step with it.
|
||||
@@ -246,7 +246,7 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError {
|
||||
// the mail already stored from it along.
|
||||
Skipped: imap.SkipsFolder(*box, skip) || imapRetiredIntoSkipped(box, gone, skipped),
|
||||
}); err != nil {
|
||||
return nil
|
||||
return w.controlPlaneError(err, stats)
|
||||
}
|
||||
deleted = append(deleted, box.Name)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user