Merge pull request #931 from warmbly/devin/1791640209-provider-sync-recovery

feat: preserve IMAP folders on incomplete status and honor Graph sync retry windows

docs/content/docs/guides/mailboxes.mdx

@@ -280,6 +280,10 @@ Connecting a mailbox does two things: it imports the mailbox's recent history, a
 
 The Sync card distinguishes a completed history import from a recent successful live check. An unfinished pass, provider failure, recovery scan or budget hold does not refresh the last successful check or erase the last observed deferred count. That count is an observation from sync, not a measurement of all mail still on the provider. A worker heartbeat alone is not evidence that a mailbox is current. Repeated connection failures refresh the warning's observation time without generating a new notification each pass; no later successful check means recovery is unverified.
 
+Sync respects an explicit provider retry window for temporary service failures as well as rate limits. On IMAP, an incomplete folder-status response or a temporary status failure keeps the existing folders and their sync positions for the next pass. Missing status metadata is requested separately, not treated as proof that a folder was deleted.
+
+The IMAP folder cap limits which folders are opened, not which folders exist. Folders outside the cap keep their saved sync positions. A capped pass cannot establish a rename or refresh the last successful live check. Folder retirement requires a complete name listing showing absence, an explicit provider not-found response, or your folder exclusion setting. On Outlook, an explicit retry window stops the remaining folder requests in that pass, even if an earlier folder failed without a retry window.
+
 If Gmail rejects an ordinary import continuation token, the import restarts inside its original fixed date window. Already mapped mail is deduplicated, and the existing message cap and sync budgets still apply. Outlook recovers an expired delta checkpoint by replaying the configured recent-history window, reconciling known messages and following changes from a fresh checkpoint only after recovery completes. Older messages never imported are outside that recovery window. Authentication failures, throttles and transport errors are not treated as expired checkpoints.
 
 On IMAP, discovering a new folder or a changed UIDVALIDITY generation during or after the initial import schedules a bounded recent-history scan. Generation recovery reconciles known message handles and read state without restarting the global import count. In folders excluded from history imports, such as Junk and Trash, recovery updates only known messages and does not import unseen history. An unreadable or incomplete message body remains retryable rather than being stored as a successful empty message. A folder without usable modification sequences falls back to UIDNEXT checks and periodic flags even if the server advertises CONDSTORE. Changes are read with the standard CONDSTORE fetch. If a server answers that in a form Warmbly cannot read, the worker asks the other supported ways, and when none can be read it uses the same UIDNEXT checks. The mailbox keeps syncing either way. Failed flag publication keeps the previous baseline so it can retry. Background lookups do not mark unread messages as seen.

internal/app/worker/wmail/governor_test.go

@@ -188,14 +188,17 @@ func TestNextSyncDelay(t *testing.T) {
 }
 
 func TestNextSyncDelayDoesNotUndercutProviderRetryAfter(t *testing.T) {
-	w := &WMail{}
-	retryAfter := 12 * time.Minute
-	mailErr := errx.MError(errx.MailErrorWarning, errx.MailErrorCodeSendingTooFast, "throttled", errx.MailErrorResolveMethodRetry)
-	mailErr.RetryAfter = retryAfter
-
-	for i := 0; i < 50; i++ {
-		if got := w.nextSyncDelay(time.Minute, mailErr); got < retryAfter {
-			t.Fatalf("delay %v retried before provider's %v window", got, retryAfter)
-		}
+	for _, code := range []errx.MailErrorCode{errx.MailErrorCodeSendingTooFast, errx.MailErrorCodeServerUnreachable, errx.MailErrorCodeConnectionLost} {
+		t.Run(string(code), func(t *testing.T) {
+			w := &WMail{transportFailures: 1}
+			retryAfter := 12 * time.Minute
+			mailErr := errx.MError(errx.MailErrorWarning, code, "retry later", errx.MailErrorResolveMethodRetry)
+			mailErr.RetryAfter = retryAfter
+			for i := 0; i < 50; i++ {
+				if got := w.nextSyncDelay(time.Minute, mailErr); got < retryAfter {
+					t.Fatalf("delay %v retried before provider's %v window", got, retryAfter)
+				}
+			}
+		})
 	}
 }

internal/app/worker/wmail/imap_conn.go

@@ -7,7 +7,6 @@ import (
 	goimap "github.com/emersion/go-imap/v2"
 	"github.com/warmbly/warmbly/internal/client/smtpimap/imap"
 	"github.com/warmbly/warmbly/internal/errx"
-	"github.com/warmbly/warmbly/internal/models"
 )
 
 // ImapConn is everything the worker drives on one IMAP connection.
@@ -16,7 +15,7 @@ import (
 // holds an *imap.Client.
 type ImapConn interface {
 	// Sync pass.
-	Folders() ([]models.Mailbox, *errx.MailError)
+	ListFolders() (imap.FolderListing, *errx.MailError)
 	// FolderOverflow is how many folders the last listing left out for the
 	// cap, FolderConflicts how many it left out for a duplicate UIDVALIDITY.
 	FolderOverflow() int

internal/app/worker/wmail/provider_presence_deadline_test.go

@@ -0,0 +1,170 @@
+package wmail
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/warmbly/warmbly/internal/errx"
+	"github.com/warmbly/warmbly/internal/models"
+)
+
+func TestSyncFolderCapDoesNotAuthorizeRename(t *testing.T) {
+	inbox := models.Mailbox{Name: "INBOX", UIDValidity: 7, UIDNext: 101, HighestModSeq: 100}
+	conn := &fakeImapConn{folders: []models.Mailbox{inbox, {Name: "NewlyKeptFolder", UIDValidity: 42, UIDNext: 101, HighestModSeq: 100}}, overflow: 1, present: []string{"StillOnServerButOutsideCap"}}
+	w, events := newIMAPTestMail(conn, &fixedBudget{allow: 10}, &inbox)
+	w.SmtpImapData.Mailboxes = append(w.SmtpImapData.Mailboxes, &models.Mailbox{Name: "StillOnServerButOutsideCap", UIDValidity: 42, UIDNext: 101, HighestModSeq: 100})
+	if err := w.Sync(t.Context()); err != nil {
+		t.Fatal(err)
+	}
+	for _, event := range *events {
+		if event.eventType == models.JobEventTypeMailboxRename {
+			t.Fatalf("bounded listing inferred false rename: %+v", event.body)
+		}
+	}
+}
+
+func TestSyncGraphRetryAfterStopsInflightRequests(t *testing.T) {
+	var requests atomic.Int32
+	srv := httptest.NewServer(http.HandlerFunc(func(out http.ResponseWriter, _ *http.Request) {
+		requests.Add(1)
+		out.Header().Set("Retry-After", "720")
+		out.Header().Set("Content-Type", "application/json")
+		out.WriteHeader(http.StatusServiceUnavailable)
+		_, _ = out.Write([]byte(`{"error":{"code":"ServiceUnavailable"}}`))
+	}))
+	defer srv.Close()
+	var events []captured
+	var relayed []models.SyncState
+	w := newGraphTestMail(t, srv, &events, &relayed)
+	previous := seedSyncTruth(w, &relayed)
+	err := w.syncOnce(t.Context())
+	if err == nil || err.RetryAfter != 12*time.Minute || requests.Load() != 1 || err.Failure == nil || err.Failure.RetryAt == nil {
+		t.Fatalf("early hidden retry or lost deadline: requests=%d err=%+v", requests.Load(), err)
+	}
+	if time.Now().Add(w.nextSyncDelay(time.Minute, err)).Before(*err.Failure.RetryAt) {
+		t.Fatal("next pass undercuts provider deadline")
+	}
+	assertNoSyncSuccess(t, w, previous)
+}
+
+func TestSyncGraphLaterFolderRetryAfterOverridesEarlierError(t *testing.T) {
+	var requests atomic.Int32
+	srv := httptest.NewServer(http.HandlerFunc(func(out http.ResponseWriter, _ *http.Request) {
+		if requests.Add(1) == 2 {
+			out.Header().Set("Retry-After", "720")
+		}
+		out.Header().Set("Content-Type", "application/json")
+		out.WriteHeader(http.StatusServiceUnavailable)
+		_, _ = out.Write([]byte(`{"error":{"code":"ServiceUnavailable"}}`))
+	}))
+	defer srv.Close()
+	var events []captured
+	var relayed []models.SyncState
+	w := newGraphTestMail(t, srv, &events, &relayed)
+	previous := seedSyncTruth(w, &relayed)
+	err := w.syncOnce(t.Context())
+	if err == nil || err.RetryAfter < 12*time.Minute || requests.Load() != 2 || err.Failure == nil || err.Failure.RetryAt == nil {
+		t.Fatalf("later provider deadline discarded: requests=%d err=%+v delay=%s", requests.Load(), err, w.nextSyncDelay(time.Minute, err))
+	}
+	if time.Now().Add(w.nextSyncDelay(time.Minute, err)).Before(*err.Failure.RetryAt) {
+		t.Fatal("later deadline was undercut by backoff")
+	}
+	assertNoSyncSuccess(t, w, previous)
+}
+
+func TestSyncFolderCapPreservesUnselectedStateWithoutClaimingRecovery(t *testing.T) {
+	inbox := models.Mailbox{Name: "INBOX", UIDValidity: 7, UIDNext: 101, HighestModSeq: 100}
+	conn := &fakeImapConn{folders: []models.Mailbox{inbox}, overflow: 1, present: []string{"StillOnServerButOutsideCap"}}
+	w, events := newIMAPTestMail(conn, &fixedBudget{allow: 10}, &inbox)
+	w.SmtpImapData.Mailboxes = append(w.SmtpImapData.Mailboxes, &models.Mailbox{Name: "StillOnServerButOutsideCap", UIDValidity: 42, UIDNext: 101, HighestModSeq: 100})
+	var relayed []models.SyncState
+	previous := seedSyncTruth(w, &relayed)
+	w.transportFailures = 2
+	w.EmailType = models.InboxProviderSMTPIMAP
+	if err := w.syncOnce(t.Context()); err != nil {
+		t.Fatal(err)
+	}
+	if gone := mailboxDeletes(*events); len(gone) != 0 {
+		t.Fatalf("bounded listing overflow=%d retired a folder without absence evidence: %v", conn.overflow, gone)
+	}
+	assertNoSyncSuccess(t, w, previous)
+	if w.transportFailures != 2 || len(w.SmtpImapData.Mailboxes) != 2 {
+		t.Fatal("capped pass erased saved folder or transport warning")
+	}
+	conn.overflow = 0
+	conn.folders = append(conn.folders, *w.SmtpImapData.Mailboxes[1])
+	if err := w.syncOnce(t.Context()); err != nil {
+		t.Fatal(err)
+	}
+	assertSyncCatchUp(t, w, previous)
+	if w.transportFailures != 0 {
+		t.Fatal("complete uncapped pass did not confirm recovery")
+	}
+}
+
+func TestSyncFolderCapRetiresOnlyCompleteAbsenceOrExplicitExclusion(t *testing.T) {
+	for _, excluded := range []bool{false, true} {
+		t.Run(map[bool]string{false: "absent", true: "excluded"}[excluded], func(t *testing.T) {
+			inbox := models.Mailbox{Name: "INBOX", UIDValidity: 7, UIDNext: 101, HighestModSeq: 100}
+			conn := &fakeImapConn{folders: []models.Mailbox{inbox}, overflow: 1, present: []string{"RetainedOutsideCap"}}
+			budget := &skipBudget{fixedBudget: &fixedBudget{allow: 10}}
+			if excluded {
+				budget.skip = []string{"Retired"}
+				conn.present = append(conn.present, "Retired")
+			}
+			w, events := newIMAPTestMail(conn, budget, &inbox)
+			w.SmtpImapData.Mailboxes = append(w.SmtpImapData.Mailboxes,
+				&models.Mailbox{Name: "Retired", UIDValidity: 42, UIDNext: 101, HighestModSeq: 100},
+				&models.Mailbox{Name: "RetainedOutsideCap", UIDValidity: 43, UIDNext: 101, HighestModSeq: 100})
+			if err := w.Sync(t.Context()); err != nil {
+				t.Fatal(err)
+			}
+			deletes := mailboxEvents(*events, models.JobEventTypeMailboxDelete)
+			if len(deletes) != 1 {
+				t.Fatalf("expected one positively authorized retirement: %+v", deletes)
+			}
+			retired := deletes[0].body.(*models.JobEventMailboxDelete)
+			if retired.Mailbox != "Retired" || retired.Skipped != excluded || len(w.SmtpImapData.Mailboxes) != 2 {
+				t.Fatalf("wrong cap retirement: %+v", retired)
+			}
+		})
+	}
+}
+
+func TestNextSyncDelayPreservesRetryAfterAcrossBackoff(t *testing.T) {
+	for _, code := range []errx.MailErrorCode{errx.MailErrorCodeSendingTooFast, errx.MailErrorCodeServerUnreachable, errx.MailErrorCodeConnectionLost} {
+		for _, deadline := range []time.Duration{30 * time.Second, 90 * time.Second, 12 * time.Minute, time.Hour} {
+			for _, failures := range []int{0, 1, 3, 9} {
+				w := &WMail{transportFailures: failures}
+				mailErr := errx.MError(errx.MailErrorWarning, code, "fixture", errx.MailErrorResolveMethodRetry)
+				mailErr.RetryAfter = deadline
+				for range 500 {
+					if got := w.nextSyncDelay(time.Minute, mailErr); got < deadline {
+						t.Fatalf("code=%s failures=%d deadline=%s delay=%s", code, failures, deadline, got)
+					}
+				}
+			}
+		}
+	}
+}
+
+func TestSyncPresentFolderWithoutStatusRetainsRecoveryEvidence(t *testing.T) {
+	inbox := models.Mailbox{Name: "INBOX", UIDValidity: 7, UIDNext: 101, HighestModSeq: 100}
+	conn := &fakeImapConn{folders: []models.Mailbox{inbox}, present: []string{"Unselectable"}}
+	w, events := newIMAPTestMail(conn, &fixedBudget{allow: 10}, &inbox)
+	w.SmtpImapData.Mailboxes = append(w.SmtpImapData.Mailboxes, &models.Mailbox{Name: "Unselectable", UIDValidity: 42, UIDNext: 101, HighestModSeq: 100})
+	var relayed []models.SyncState
+	previous := seedSyncTruth(w, &relayed)
+	w.EmailType = models.InboxProviderSMTPIMAP
+	w.transportFailures = 2
+	if err := w.syncOnce(t.Context()); err != nil {
+		t.Fatal(err)
+	}
+	assertNoSyncSuccess(t, w, previous)
+	if len(mailboxDeletes(*events)) != 0 || w.transportFailures != 2 {
+		t.Fatal("presence without usable status retired state or cleared warning")
+	}
+}

internal/app/worker/wmail/start_sync.go

@@ -46,13 +46,12 @@ func (w *WMail) nextSyncDelay(base time.Duration, last *errx.MailError) time.Dur
 	d := base
 	minimum := base / 2
 	switch {
+	case last != nil && last.RetryAfter > 0 && (isTransportError(last) || last.Code == errx.MailErrorCodeSendingTooFast):
+		// Service-unavailable responses can supply the same retry window as 429.
+		d = max(last.RetryAfter, base)
+		minimum = d
 	case last != nil && last.Code == errx.MailErrorCodeSendingTooFast:
-		// The provider returned 429. Respect its window when supplied.
 		d = syncBackoffMax
-		if last.RetryAfter > 0 {
-			d = max(last.RetryAfter, base)
-			minimum = d
-		}
 	case last != nil && isTransportError(last):
 		// The server is unreachable. Retry soon after the first failure (a
 		// dropped session reconnects on the next pass and costs one dial),

internal/app/worker/wmail/sync_imap.go

@@ -43,12 +43,16 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError {
 	// A mailbox left selected by the previous pass freezes LIST-STATUS on this
 	// connection, so release it before asking what changed.
 	client.ReleaseMailbox()
-	folders, err := client.Folders()
+	listing, err := client.ListFolders()
 	if err != nil {
 		return err
 	}
+	if listing.Present == nil {
+		return errx.ErrMailServerUnreachable
+	}
+	folders := listing.Folders
 	w.reportFolderOverflow()
-	// Folders() already drops Gmail's label views. Dropping them here too
+	// ListFolders already drops Gmail's label views. Dropping them here too
 	// costs nothing and keeps the pass correct against any listing: a view
 	// that reached it would re-file known mail as archive under a second UID.
 	folders = slices.DeleteFunc(folders, func(b models.Mailbox) bool { return imapVirtualFolder(&b) })
@@ -59,7 +63,8 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError {
 	// baselined. They are kept aside so mail that moves into one of them can
 	// be recognised as filed rather than lost.
 	var skipped []models.Mailbox
-	if skip := w.skipFolders(); len(skip) > 0 {
+	skip := w.skipFolders()
+	if len(skip) > 0 {
 		folders = slices.DeleteFunc(folders, func(b models.Mailbox) bool {
 			if !imap.SkipsFolder(b, skip) {
 				return false
@@ -76,12 +81,19 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError {
 	// changed. A rename read as a delete plus a first sighting would orphan
 	// every message filed under the old name and re-import the folder's
 	// history under the new one.
-	if err := w.imapFollowRenames(folders); err != nil {
-		return nil
+	selectionComplete := client.FolderOverflow() == 0 && client.FolderConflicts() == 0
+	if selectionComplete {
+		if err := w.imapFollowRenames(folders, listing.Present); err != nil {
+			return nil
+		}
 	}
 
 	// CONDSTORE is effective per folder; NOMODSEQ uses UIDNEXT and periodic flags.
-	caughtUp := true
+	caughtUp := selectionComplete
+	selectedNames := make(map[string]struct{}, len(folders))
+	for _, box := range folders {
+		selectedNames[box.Name] = struct{}{}
+	}
 
 	for i := range folders {
 		box := &folders[i]
@@ -212,17 +224,15 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError {
 		}
 	}
 
-	// Collect deletions first to avoid modifying the slice during iteration.
-	// Renames were already followed above, so a name missing from the listing
-	// at this point really is a folder that is gone.
+	// Only complete LIST presence or an explicit exclusion can retire state.
 	var deleted []string
 	var gone []*models.Mailbox
-outer:
 	for _, box := range w.SmtpImapData.Mailboxes {
-		for _, f := range folders {
-			if box.Name == f.Name {
-				continue outer
+		if _, present := listing.Present[box.Name]; present && !imapVirtualFolder(box) && !imap.SkipsFolder(*box, skip) {
+			if _, selected := selectedNames[box.Name]; !selected {
+				caughtUp = false
 			}
+			continue
 		}
 		gone = append(gone, box)
 	}
@@ -234,7 +244,7 @@ outer:
 			UIDValidity: box.UIDValidity,
 			// A folder that is still on the server but now excluded takes
 			// the mail already stored from it along.
-			Skipped: imapRetiredIntoSkipped(box, gone, skipped),
+			Skipped: imap.SkipsFolder(*box, skip) || imapRetiredIntoSkipped(box, gone, skipped),
 		}); err != nil {
 			return nil
 		}
@@ -1070,17 +1080,12 @@ func (w *WMail) setWalking(box *models.Mailbox) {
 // go missing while one arrives, and picking either would move the wrong
 // folder's mail into it. Those fall through to the ordinary delete and
 // first-sight paths, which lose nothing that was not already gone.
-func (w *WMail) imapFollowRenames(folders []models.Mailbox) error {
-	listed := make(map[string]struct{}, len(folders))
-	for i := range folders {
-		listed[folders[i].Name] = struct{}{}
-	}
-
+func (w *WMail) imapFollowRenames(folders []models.Mailbox, present map[string]struct{}) error {
 	// Group both sides by UIDVALIDITY: the stored folders that are no longer
 	// listed, and the listed folders that are not stored.
 	gone := map[uint32][]*models.Mailbox{}
 	for _, before := range w.SmtpImapData.Mailboxes {
-		if _, still := listed[before.Name]; still || before.UIDValidity == 0 {
+		if _, still := present[before.Name]; still || before.UIDValidity == 0 {
 			continue
 		}
 		gone[before.UIDValidity] = append(gone[before.UIDValidity], before)

internal/app/worker/wmail/sync_imap_test.go

@@ -21,6 +21,8 @@ import (
 type fakeImapConn struct {
 	ImapConn
 	folders   []models.Mailbox
+	present   []string
+	folderErr *errx.MailError
 	changed   []goimap.UID
 	fetches   int
 	released  int
@@ -54,9 +56,27 @@ type fakeImapConn struct {
 // in place, and a fake that shared its backing array would lose folders
 // between passes.
 func (c *fakeImapConn) Folders() ([]models.Mailbox, *errx.MailError) {
+	if c.folderErr != nil {
+		return nil, c.folderErr
+	}
 	return append([]models.Mailbox(nil), c.folders...), nil
 }
 
+func (c *fakeImapConn) ListFolders() (imap.FolderListing, *errx.MailError) {
+	folders, err := c.Folders()
+	if err != nil {
+		return imap.FolderListing{}, err
+	}
+	present := map[string]struct{}{}
+	for _, f := range folders {
+		present[f.Name] = struct{}{}
+	}
+	for _, name := range c.present {
+		present[name] = struct{}{}
+	}
+	return imap.FolderListing{Folders: folders, Present: present}, nil
+}
+
 func (c *fakeImapConn) FolderOverflow() int  { return c.overflow }
 func (c *fakeImapConn) FolderConflicts() int { return c.conflicts }
 
@@ -351,10 +371,17 @@ type backfillImapConn struct {
 }
 
 func (c *backfillImapConn) Folders() ([]models.Mailbox, *errx.MailError) { return c.folders, nil }
-func (c *backfillImapConn) FolderOverflow() int                          { return 0 }
-func (c *backfillImapConn) FolderConflicts() int                         { return 0 }
-func (c *backfillImapConn) HasCondStore() bool                           { return true }
-func (c *backfillImapConn) ReleaseMailbox()                              {}
+func (c *backfillImapConn) ListFolders() (imap.FolderListing, *errx.MailError) {
+	present := map[string]struct{}{}
+	for _, f := range c.folders {
+		present[f.Name] = struct{}{}
+	}
+	return imap.FolderListing{Folders: c.folders, Present: present}, nil
+}
+func (c *backfillImapConn) FolderOverflow() int  { return 0 }
+func (c *backfillImapConn) FolderConflicts() int { return 0 }
+func (c *backfillImapConn) HasCondStore() bool   { return true }
+func (c *backfillImapConn) ReleaseMailbox()      {}
 
 func (c *backfillImapConn) SelectForSyncState(folder string) (imap.Selected, *errx.MailError) {
 	count, err := c.SelectForSync(folder)

internal/app/worker/wmail/sync_truth_test.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -63,6 +64,81 @@ func assertSyncCatchUp(t *testing.T, w *WMail, previous time.Time) {
 	}
 }
 
+func TestSyncTruthGraphServiceUnavailableHonorsRetryWindowAndRecovers(t *testing.T) {
+	var recovered atomic.Bool
+	srv := httptest.NewServer(http.HandlerFunc(func(out http.ResponseWriter, req *http.Request) {
+		out.Header().Set("Content-Type", "application/json")
+		if !recovered.Load() {
+			out.Header().Set("Retry-After", "720")
+			out.WriteHeader(http.StatusServiceUnavailable)
+			_, _ = out.Write([]byte(`{"error":{"code":"ServiceUnavailable"}}`))
+			return
+		}
+		_, _ = fmt.Fprintf(out, `{"value":[],"@odata.deltaLink":"https://graph.microsoft.com%s?$deltatoken=recovered"}`, req.URL.Path)
+	}))
+	defer srv.Close()
+	var events []captured
+	var relayed []models.SyncState
+	w := newGraphTestMail(t, srv, &events, &relayed)
+	previous := seedSyncTruth(w, &relayed)
+	before := w.GraphData.Client.DeltaLinks[msgraph.FolderInbox]
+
+	for range 2 {
+		err := w.syncOnce(t.Context())
+		if err == nil || err.Code != errx.MailErrorCodeServerUnreachable || err.RetryAfter != 12*time.Minute {
+			t.Fatalf("missing retryable provider evidence: %v", err)
+		}
+		if delay := w.nextSyncDelay(time.Minute, err); delay < err.RetryAfter {
+			t.Fatalf("service retry window undercut: %v < %v", delay, err.RetryAfter)
+		}
+		assertNoSyncSuccess(t, w, previous)
+		if w.GraphData.Client.DeltaLinks[msgraph.FolderInbox] != before {
+			t.Fatal("failed service request advanced delta cursor")
+		}
+	}
+	if w.transportFailures != 2 {
+		t.Fatalf("failed passes reset transport evidence: %d", w.transportFailures)
+	}
+	recovered.Store(true)
+	if err := w.syncOnce(t.Context()); err != nil {
+		t.Fatal(err)
+	}
+	assertSyncCatchUp(t, w, previous)
+	if w.transportFailures != 0 || w.GraphData.Client.DeltaLinks[msgraph.FolderInbox] == before {
+		t.Fatal("provider-confirmed complete pass failed to update recovery state")
+	}
+}
+
+func TestSyncTruthImapFolderFailurePreservesSavedStateUntilRecovery(t *testing.T) {
+	box := &models.Mailbox{Name: "Sent", UIDValidity: 7, UIDNext: 10, HighestModSeq: 30}
+	conn := &fakeImapConn{folders: []models.Mailbox{*box}, folderErr: errx.ErrMailServerUnreachable}
+	w, events := newIMAPTestMail(conn, &fixedBudget{allow: 100}, box)
+	w.EmailType = models.InboxProviderSMTPIMAP
+	var relayed []models.SyncState
+	previous := seedSyncTruth(w, &relayed)
+	w.tracker.state.BackfillCursor.Folders = map[string]models.SyncFolderCursor{"Sent": {Done: true, UID: 7}}
+	for range 2 {
+		if err := w.syncOnce(t.Context()); err == nil || err.Code != errx.MailErrorCodeServerUnreachable {
+			t.Fatalf("folder failure hidden: %v", err)
+		}
+		assertNoSyncSuccess(t, w, previous)
+		if hasEvent(*events, models.JobEventTypeMailboxDelete) || len(w.SmtpImapData.Mailboxes) != 1 || w.SmtpImapData.Mailboxes[0] != box {
+			t.Fatal("failed folder listing retired saved mailbox state")
+		}
+		if cursor, ok := w.tracker.state.BackfillCursor.Folders["Sent"]; !ok || cursor.UID != 7 || !cursor.Done {
+			t.Fatal("failed folder listing lost the durable recovery cursor")
+		}
+	}
+	conn.folderErr = nil
+	if err := w.syncOnce(t.Context()); err != nil {
+		t.Fatal(err)
+	}
+	assertSyncCatchUp(t, w, previous)
+	if hasEvent(*events, models.JobEventTypeMailboxDelete) || w.transportFailures != 0 {
+		t.Fatal("complete provider recovery did not preserve folder state")
+	}
+}
+
 func TestSyncTruthGoogleFailureHeartbeatAndRecovery(t *testing.T) {
 	failing := true
 	srv := httptest.NewServer(http.HandlerFunc(func(out http.ResponseWriter, _ *http.Request) {

internal/client/msgraph/delta.go

@@ -95,7 +95,7 @@ func (c *Client) SyncPass(ctx context.Context) (bool, error) {
 			if folder == FolderArchive && errors.As(err, &mailErr) && mailErr.Code == errx.MailErrorCodeNotFound {
 				continue
 			}
-			if errors.As(err, &mailErr) && (mailErr.Type == errx.MailErrorCritical || mailErr.Code == errx.MailErrorCodeSendingTooFast) {
+			if errors.As(err, &mailErr) && (mailErr.Type == errx.MailErrorCritical || mailErr.Code == errx.MailErrorCodeSendingTooFast || mailErr.RetryAfter > 0) {
 				return false, err
 			}
 			complete = false

internal/client/smtpimap/imap/folders.go

@@ -1,8 +1,8 @@
 package imap
 
 import (
-	"errors"
 	"fmt"
+	"slices"
 	"sort"
 	"strings"
 	"unicode/utf8"
@@ -14,6 +14,16 @@ import (
 	"github.com/warmbly/warmbly/internal/models"
 )
 
+type FolderListing struct {
+	Folders []models.Mailbox
+	// Present includes names outside the sync cap, from the complete LIST.
+	Present map[string]struct{}
+}
+
+func (c *Client) ListFolders() (FolderListing, *errx.MailError) {
+	return c.listFoldersCapped(config.MaxEmailFolders)
+}
+
 // Folders lists the selectable folders of the account with the cursors the
 // sync loop keys on (UIDVALIDITY, UIDNEXT and, on a CONDSTORE server,
 // HIGHESTMODSEQ).
@@ -32,8 +42,13 @@ func (c *Client) Folders() ([]models.Mailbox, *errx.MailError) {
 // foldersCapped is Folders with the cap injectable, so a test can exercise
 // the overflow without standing up a hundred folders.
 func (c *Client) foldersCapped(limit int) ([]models.Mailbox, *errx.MailError) {
+	listing, err := c.listFoldersCapped(limit)
+	return listing.Folders, err
+}
+
+func (c *Client) listFoldersCapped(limit int) (FolderListing, *errx.MailError) {
 	if err := c.ensureConnected(); err != nil {
-		return nil, err
+		return FolderListing{}, err
 	}
 	c.lifecycle.RLock()
 	defer c.lifecycle.RUnlock()
@@ -47,12 +62,16 @@ func (c *Client) foldersCapped(limit int) ([]models.Mailbox, *errx.MailError) {
 		// Asking a server without CONDSTORE for HIGHESTMODSEQ is a BAD.
 		HighestModSeq: caps.Has(imap.CapCondStore),
 	}
-	opts, listStatus := listOptionsFor(caps, status)
+	opts, _ := listOptionsFor(caps, status)
 
 	var all []models.Mailbox
+	present := map[string]struct{}{}
 	statuses := map[string]*imap.StatusData{}
 	cmd := c.client.List("", "*", opts)
 	for f := cmd.Next(); f != nil; f = cmd.Next() {
+		if !slices.Contains(f.Attrs, imap.MailboxAttrNonExistent) {
+			present[f.Mailbox] = struct{}{}
+		}
 		attrs := make([]string, len(f.Attrs))
 		for i := range f.Attrs {
 			attrs[i] = string(f.Attrs[i])
@@ -67,7 +86,7 @@ func (c *Client) foldersCapped(limit int) ([]models.Mailbox, *errx.MailError) {
 		}
 	}
 	if err := cmd.Close(); err != nil {
-		return nil, c.handleError(err)
+		return FolderListing{}, c.handleError(err)
 	}
 
 	// Before the cap, not after: a name the server listed twice would
@@ -83,24 +102,21 @@ func (c *Client) foldersCapped(limit int) ([]models.Mailbox, *errx.MailError) {
 	for _, box := range kept {
 		st := statuses[box.Name]
 		if st == nil {
-			if listStatus {
-				// The server was asked and said nothing: the folder is not
-				// one it can open for us.
-				continue
-			}
 			data, err := c.client.Status(box.Name, status).Wait()
 			if err != nil {
-				var imapErr *imap.Error
-				if errors.As(err, &imapErr) {
-					// One folder the server will not report on must not
-					// take the rest of the account with it.
-					log.Warn().Err(err).Str("folder", box.Name).Msg("imap: STATUS refused; folder skipped")
+				merr := c.handleError(err)
+				if merr.Code == errx.ErrMailResourceNotFound.Code {
+					// Only explicit absence can retire a folder's stored state.
+					delete(present, box.Name)
 					continue
 				}
-				return nil, c.handleError(err)
+				return FolderListing{}, merr
 			}
 			st = data
 		}
+		if st == nil || st.UIDValidity == 0 || st.UIDNext == 0 {
+			return FolderListing{}, errx.ErrMailServerUnreachable
+		}
 		box.UIDValidity = st.UIDValidity
 		box.UIDNext = uint32(st.UIDNext)
 		box.HighestModSeq = st.HighestModSeq
@@ -110,7 +126,7 @@ func (c *Client) foldersCapped(limit int) ([]models.Mailbox, *errx.MailError) {
 		resp = append(resp, box)
 	}
 
-	return resp, nil
+	return FolderListing{Folders: resp, Present: present}, nil
 }
 
 // listOptionsFor is the LIST options a server's capabilities allow, and

internal/client/smtpimap/imap/server_test.go

@@ -3,6 +3,7 @@ package imap
 import (
 	"bufio"
 	"context"
+	"fmt"
 	"io"
 	"net"
 	"strings"
@@ -13,6 +14,7 @@ import (
 	"github.com/emersion/go-imap/v2"
 	"github.com/emersion/go-imap/v2/imapserver"
 	"github.com/emersion/go-imap/v2/imapserver/imapmemserver"
+	"github.com/warmbly/warmbly/internal/errx"
 	"github.com/warmbly/warmbly/internal/models"
 )
 
@@ -262,6 +264,87 @@ func TestFoldersWithoutListStatus(t *testing.T) {
 	}
 }
 
+func TestFoldersStatusFailureDoesNotBecomeFolderAbsence(t *testing.T) {
+	for _, tc := range []struct {
+		name, reply string
+		code        errx.MailErrorCode
+	}{
+		{"unavailable", "NO [UNAVAILABLE] Try later", errx.MailErrorCodeServerUnreachable},
+		{"server bug", "NO [SERVERBUG] Try later", errx.MailErrorCodeServerUnreachable},
+		{"connection limit", "NO [LIMIT] Try later", errx.MailErrorCodeSendingTooFast},
+		{"authentication", "NO [AUTHENTICATIONFAILED] Sign in again", errx.MailErrorCodeInvalidCredentials},
+		{"authorization", "NO [AUTHORIZATIONFAILED] Not permitted", errx.MailErrorCodeAuthorizationFailed},
+		{"unknown refusal", "NO Could not obtain status", errx.MailErrorCodeImapUnknown},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			c, _ := interceptingServer(t, nil, func(line string) string {
+				fields := strings.Fields(line)
+				if len(fields) >= 3 && strings.EqualFold(fields[1], "STATUS") && strings.Contains(line, "Sent") {
+					return fields[0] + " " + tc.reply
+				}
+				return ""
+			}, "Sent")
+			if err := c.Connect(); err != nil {
+				t.Fatal(err)
+			}
+			boxes, err := c.Folders()
+			if err == nil || err.Code != tc.code || len(boxes) != 0 {
+				t.Fatalf("partial listing could retire Sent: boxes=%v err=%v, want %s", boxes, err, tc.code)
+			}
+		})
+	}
+}
+
+func TestFoldersExplicitlyMissingStatusFolderCanBeSkipped(t *testing.T) {
+	c, _ := interceptingServer(t, nil, func(line string) string {
+		fields := strings.Fields(line)
+		if len(fields) >= 3 && strings.EqualFold(fields[1], "STATUS") && strings.Contains(line, "Sent") {
+			return fields[0] + " NO [NONEXISTENT] Folder removed"
+		}
+		return ""
+	}, "Sent")
+	boxes, err := c.Folders()
+	if err != nil || len(boxes) != 1 || boxes[0].Name != "INBOX" {
+		t.Fatalf("authoritative missing folder did not preserve remaining listing: %v %v", boxes, err)
+	}
+}
+
+func TestFoldersMissingListStatusFallsBackToStatus(t *testing.T) {
+	suppressed := 0
+	c, wire := rewritingServer(t, imap.CapSet{imap.CapIMAP4rev1: {}, imap.CapListExtended: {}, imap.CapListStatus: {}}, nil, func(line string) string {
+		if strings.HasPrefix(strings.ToUpper(line), "* STATUS ") && suppressed < 2 {
+			suppressed++
+			return ""
+		}
+		return line
+	}, "Sent")
+	boxes, err := c.Folders()
+	if err != nil || len(boxes) != 2 || len(wire.commands("STATUS")) != 2 {
+		t.Fatalf("missing LIST-STATUS metadata became folder absence: boxes=%v err=%v fallback=%v", boxes, err, wire.commands("STATUS"))
+	}
+}
+
+func TestFoldersIncompleteStatusRemainsRetryable(t *testing.T) {
+	for _, reply := range []string{"OK Status unavailable", "OK Status completed"} {
+		t.Run(reply, func(t *testing.T) {
+			c, _ := interceptingServer(t, nil, func(line string) string {
+				fields := strings.Fields(line)
+				if len(fields) >= 3 && strings.EqualFold(fields[1], "STATUS") && strings.Contains(line, "Sent") {
+					if reply == "OK Status completed" {
+						return "* STATUS Sent (MESSAGES 0)\r\n" + fields[0] + " " + reply
+					}
+					return fields[0] + " " + reply
+				}
+				return ""
+			}, "Sent")
+			boxes, err := c.Folders()
+			if err == nil || err.Code != errx.MailErrorCodeServerUnreachable || len(boxes) != 0 {
+				t.Fatalf("incomplete status was treated as authoritative: boxes=%v err=%v", boxes, err)
+			}
+		})
+	}
+}
+
 // A nested folder must be listed: "%" stopped at the top level, which is how
 // Gmail's [Gmail]/Sent Mail and Dovecot's INBOX.Sent went unsynced.
 func TestFoldersListsNested(t *testing.T) {
@@ -317,6 +400,72 @@ func TestFoldersCapKeepsInboxAndSpecialFolders(t *testing.T) {
 	}
 }
 
+func TestFolderListingPreservesCompleteNamesWithBoundedStatus(t *testing.T) {
+	for _, listStatus := range []bool{false, true} {
+		t.Run(fmt.Sprintf("LIST-STATUS=%t", listStatus), func(t *testing.T) {
+			var caps imap.CapSet
+			if listStatus {
+				caps = imap.CapSet{imap.CapIMAP4rev1: {}, imap.CapListExtended: {}, imap.CapListStatus: {}}
+			}
+			suppressed := 0
+			c, wire := rewritingServer(t, caps, nil, func(line string) string {
+				if listStatus && strings.HasPrefix(strings.ToUpper(line), "* STATUS ") && suppressed < 5 {
+					suppressed++
+					return ""
+				}
+				return line
+			}, "Sent", "Archive", "Outside", "Other")
+			listing, err := c.listFoldersCapped(2)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(listing.Folders) != 2 || c.FolderOverflow() != 3 || len(wire.commands("STATUS")) != 2 {
+				t.Fatalf("cap did not bound status work: listing=%+v commands=%v", listing, wire.commands("STATUS"))
+			}
+			for _, name := range []string{"INBOX", "Sent", "Archive", "Outside", "Other"} {
+				if _, present := listing.Present[name]; !present {
+					t.Fatalf("complete LIST presence lost %q outside cap", name)
+				}
+			}
+			for _, box := range listing.Folders {
+				if box.UIDValidity == 0 || box.UIDNext == 0 {
+					t.Fatalf("kept folder has incomplete status: %+v", box)
+				}
+			}
+		})
+	}
+}
+
+func TestFolderListingRequiresCompleteDiscoveryOrExplicitAbsence(t *testing.T) {
+	for _, reply := range []string{"NO [UNAVAILABLE] Try later", "NO [NONEXISTENT] Folder removed"} {
+		t.Run(reply, func(t *testing.T) {
+			c, _ := interceptingServer(t, nil, func(line string) string {
+				fields := strings.Fields(line)
+				if len(fields) >= 3 && strings.EqualFold(fields[1], "STATUS") && strings.Contains(line, "Sent") {
+					return fields[0] + " " + reply
+				}
+				return ""
+			}, "Sent")
+			listing, err := c.ListFolders()
+			if strings.Contains(reply, "UNAVAILABLE") {
+				if err == nil || listing.Present != nil || len(listing.Folders) != 0 {
+					t.Fatalf("failed pass leaked retirement authority: %+v err=%v", listing, err)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatal(err)
+			}
+			if _, present := listing.Present["Sent"]; present {
+				t.Fatal("explicit resource absence was not retained")
+			}
+			if _, present := listing.Present["INBOX"]; !present {
+				t.Fatal("remaining positive presence was lost")
+			}
+		})
+	}
+}
+
 // The UIDNEXT search is the incremental set without CONDSTORE: everything at
 // or above the cursor, and nothing below it.
 func TestSearchNewSince(t *testing.T) {