Merge pull request #984 from warmbly/fix/959-imap-modseq-bump-history

feat: cover the IMAP MODSEQ bump end to end with regression tests and document it in the mailboxes guide

docs/content/docs/guides/mailboxes.mdx

@@ -290,7 +290,7 @@ The IMAP folder cap limits which folders are opened, not which folders exist. Fo
 
 If Gmail rejects an ordinary import continuation token, the import restarts inside its original fixed date window. Already mapped mail is deduplicated, and the existing message cap and sync budgets still apply. Outlook recovers an expired delta checkpoint by replaying the configured recent-history window, reconciling known messages and following changes from a fresh checkpoint only after recovery completes. Older messages never imported are outside that recovery window. Authentication failures, throttles and transport errors are not treated as expired checkpoints.
 
-On IMAP, discovering a new folder or a changed UIDVALIDITY generation during or after the initial import schedules a bounded recent-history scan. Generation recovery reconciles known message handles and read state without restarting the global import count. In folders excluded from history imports, such as Junk and Trash, recovery updates only known messages and does not import unseen history. An unreadable or incomplete message body remains retryable rather than being stored as a successful empty message. A folder without usable modification sequences falls back to UIDNEXT checks and periodic flags even if the server advertises CONDSTORE. Changes are read with the standard CONDSTORE fetch. If a server answers that in a form Warmbly cannot read, the worker asks the other supported ways, and when none can be read it uses the same UIDNEXT checks. The mailbox keeps syncing either way. Failed flag publication keeps the previous baseline so it can retry. Background lookups do not mark unread messages as seen.
+On IMAP, discovering a new folder or a changed UIDVALIDITY generation during or after the initial import schedules a bounded recent-history scan. Generation recovery reconciles known message handles and read state without restarting the global import count. In folders excluded from history imports, such as Junk and Trash, recovery updates only known messages and does not import unseen history. An unreadable or incomplete message body remains retryable rather than being stored as a successful empty message. A folder without usable modification sequences falls back to UIDNEXT checks and periodic flags even if the server advertises CONDSTORE. Changes are read with the standard CONDSTORE fetch. If a server answers that in a form Warmbly cannot read, the worker asks the other supported ways, and when none can be read it uses the same UIDNEXT checks. The mailbox keeps syncing either way. When a server marks a whole folder as changed at once, only mail that arrived since the last pass is imported; older messages it reports again keep their read state in sync but are not imported, and they do not count toward the sync budget. Failed flag publication keeps the previous baseline so it can retry. Background lookups do not mark unread messages as seen.
 
 On upgraded instances, a newly admitted provider message is recorded together with an encrypted pending arrival before sync advances. The consumer replays that arrival under its original identity after a restart, including related bounce or complaint processing, and provider updates wait for an unresolved arrival. This is an at-least-once recovery path, not a promise that all historical mail can be reconstructed. Old map-only records have no replay payload, and mixed-version instances retain their legacy behavior. See the [self-host upgrade order and limits](/development/install/#mailbox-sync-upgrade-order).
 

internal/app/worker/wmail/sync_imap_test.go

@@ -1128,3 +1128,102 @@ func TestImapSyncLeavesHistoryBelowUIDNextOnAModSeqBump(t *testing.T) {
 		t.Error("mailbox was deactivated by re-reported history")
 	}
 }
+
+// throttleCountingBudget is a fixedBudget that counts the throttled days the
+// pass records, which is what deactivates a mailbox on the third.
+type throttleCountingBudget struct {
+	fixedBudget
+	throttledDays int
+}
+
+func (b *throttleCountingBudget) RecordThrottledDay(context.Context) bool {
+	b.throttledDays++
+	return false
+}
+
+// The #959 shape end to end: a bump re-reports thousands of old messages
+// against a nearly spent day. History must cost no budget, record no
+// throttled day and fetch no body, and the bump is walked once, not re-asked
+// every pass.
+func TestImapModSeqBumpOverHistoryRecordsNoThrottledDay(t *testing.T) {
+	history := uidRange(3*config.ImapFetchBatchSize + 50)
+	conn := &fakeImapConn{
+		folders: []models.Mailbox{{Name: "INBOX", UIDValidity: 7, UIDNext: 10_002, HighestModSeq: 90_000}},
+		changed: append(history, 10_000, 10_001),
+	}
+	budget := &throttleCountingBudget{fixedBudget: fixedBudget{allow: 2}}
+	w, events := newIMAPTestMail(conn, budget,
+		&models.Mailbox{Name: "INBOX", UIDValidity: 7, UIDNext: 10_000, HighestModSeq: 100})
+
+	for pass := range 2 {
+		if err := w.Sync(t.Context()); err != nil {
+			t.Fatalf("pass %d: Sync: %v", pass, err)
+		}
+	}
+
+	if got := len(newEmails(*events)); got != 2 {
+		t.Errorf("stored %d messages, want only the 2 arrivals", got)
+	}
+	if budget.throttledDays != 0 {
+		t.Errorf("recorded %d throttled days on re-reported history", budget.throttledDays)
+	}
+	if w.tracker.state.ThrottledUntil != nil {
+		t.Errorf("mailbox throttled (%s) by re-reported history", w.tracker.state.ThrottleReason)
+	}
+	if budget.observed != 2 {
+		t.Errorf("flood detection counted %d messages, want the 2 arrivals", budget.observed)
+	}
+	if conn.modseqSearches != 1 {
+		t.Errorf("searched the bump %d times over two passes, want once", conn.modseqSearches)
+	}
+	if got := w.SmtpImapData.Mailboxes[0].HighestModSeq; got != 90_000 {
+		t.Errorf("mod-sequence = %d, want 90000", got)
+	}
+}
+
+// An arrival the budget defers during a bump holds the cursor, UIDNEXT
+// included, so the retry still knows which UIDs are history and stores only
+// the arrival it owes.
+func TestImapModSeqBumpDeferredArrivalKeepsHistoryOut(t *testing.T) {
+	conn := &fakeImapConn{
+		folders: []models.Mailbox{{Name: "INBOX", UIDValidity: 7, UIDNext: 5003, HighestModSeq: 90_000}},
+		changed: append(uidRange(400), 5000, 5001, 5002),
+	}
+	budget := &fixedBudget{allow: 2}
+	w, events := newIMAPTestMail(conn, budget,
+		&models.Mailbox{Name: "INBOX", UIDValidity: 7, UIDNext: 5000, HighestModSeq: 100})
+
+	if err := w.Sync(t.Context()); err != nil {
+		t.Fatalf("first pass: %v", err)
+	}
+	held := w.SmtpImapData.Mailboxes[0]
+	if held.HighestModSeq != 100 || held.UIDNext != 5000 {
+		t.Fatalf("cursor = (%d, %d) with an arrival deferred, want it held at (100, 5000)", held.HighestModSeq, held.UIDNext)
+	}
+
+	// The consumer maps what the first pass stored before the retry looks.
+	known := map[string]string{}
+	for _, e := range newEmails(*events) {
+		known[e.Message.MessageID] = uuid.NewString()
+	}
+	w.EmailMessageMapRepository = someKnownMessageMap{known: known}
+	first := len(newEmails(*events))
+
+	budget.allow = 10
+	if err := w.Sync(t.Context()); err != nil {
+		t.Fatalf("second pass: %v", err)
+	}
+	var uids []uint32
+	for _, e := range newEmails(*events)[first:] {
+		if e.Message.UID < 5000 {
+			t.Errorf("retry admitted UID %d from below the held UIDNEXT", e.Message.UID)
+		}
+		uids = append(uids, e.Message.UID)
+	}
+	if first+len(uids) != 3 {
+		t.Errorf("stored %d then %v, want the 3 arrivals once each", first, uids)
+	}
+	if got := w.SmtpImapData.Mailboxes[0]; got.HighestModSeq != 90_000 || got.UIDNext != 5003 {
+		t.Errorf("cursor = (%d, %d) after the retry, want (90000, 5003)", got.HighestModSeq, got.UIDNext)
+	}
+}