diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 087b1871f..06e66d94d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -205,7 +205,13 @@ jobs: go test -race ./internal/repository -run '^TestLive(LastWarmupSendFailure|PersistentMailboxLoadingFailure|DispatchRetry|LineageQueueRevisions|AdminPlatformOverview|WarmupRecoversOldPreparation|WarmupPreparationRecoveryRetains)' -count=1 go test -race ./internal/tasks -run '^Test(LiveWarmup(SafetyHoldReschedules|Recovery|TaskDoesNotApplyResults)|WarmupSenderDistinguishes)' -count=1 - - name: Test the Kafka codec and client lifecycle + - name: Mailbox sync admission, replay, truthful state and upgrade regressions (PostgreSQL) + env: + WARMBLY_TEST_DB: postgres://warmbly:warmbly@127.0.0.1:5432/warmbly_test?sslmode=disable + run: | + go test -race -p 1 ./internal/repository ./internal/app/consumer ./internal/app/orgtransfer ./internal/infrastructure/db -run '^TestLive(SyncArrival|SyncTruth|EmailErrorDedupe)' -count=1 + + - name: Test the Kafka codec, failed-handler replay and client lifecycle run: go test -race -tags kafka ./internal/infrastructure/codec/ ./internal/infrastructure/kafka/ ./internal/infrastructure/eventbus/ # fail_ci_if_error does not cover the uploader crashing before it runs diff --git a/cmd/backend/main.go b/cmd/backend/main.go index 1d2497af7..a1d9e5885 100644 --- a/cmd/backend/main.go +++ b/cmd/backend/main.go @@ -1137,6 +1137,7 @@ func main() { } cipherService = cipher.NewService(kms, cache, encryptedKeys) + emailMessageMapForHandler = repository.NewDurableEmailMessageMapRepository(primaryDB, cipherService) // Third-party integrations: OAuth connect flows + encrypted token // storage (sealed with the connecting user's DEK) + event-driven actions. diff --git a/cmd/consumer/main.go b/cmd/consumer/main.go index 2efa4372b..d00d402ca 100644 --- a/cmd/consumer/main.go +++ b/cmd/consumer/main.go @@ -526,6 +526,7 @@ func main() { EmailHistoryIDRepository: emailHistoryIDRepo, EmailGraphDeltaRepository: emailGraphDeltaRepo, EmailSyncStateRepository: repository.NewEmailSyncStateRepository(primaryDB), + ArrivalOutbox: repository.NewDurableEmailMessageMapRepository(primaryDB, cipherService), EmailAccountErrorRepository: emailAccountErrorRepo, WarmupRepo: warmupRepo, PoolLinkRepo: repository.NewPoolLinkRepository(primaryDB.Pool), diff --git a/docs/content/docs/development/data-control.mdx b/docs/content/docs/development/data-control.mdx index fe10a9538..f3412bd31 100644 --- a/docs/content/docs/development/data-control.mdx +++ b/docs/content/docs/development/data-control.mdx @@ -281,6 +281,16 @@ The `rsync` alternative works too and is sometimes simpler, as long as the stack [Workspace export and import](/guides/workspace-export-import/) moves a single organization's data between two running instances, re-sealing its secrets for the destination's keys. That is the per-customer tool; the bundle here is the instance-level one. They are not interchangeable: a bundle cannot be applied to one workspace, and a workspace archive cannot restore an instance. +### Mailbox sync recovery data + +New durable admissions retain an organization-encrypted arrival payload in `sync_arrival_outbox` alongside the provider map. The queue tracks delivery stages and retries consumer failures under the original message identity. A delivered arrival is removed; an unresolved arrival is retained for retry until it completes or its owning mailbox or organization is deleted. Deleting the mailbox intentionally removes that mailbox's pending work. This queue does not schedule new sends or authorize repeating a send. + +A full-instance backup must preserve the database, wrapped organization keys, the root key needed to unwrap them, and referenced message-body storage together. Restoring a queue without its keys or body objects cannot restore delivery. Use the normal consistent instance backup, not a separate copy of just the queue. A workspace archive excludes the queue and provider mappings whose arrivals are still pending. Drain pending work on the source before moving a workspace; importing an archive does not resume source-instance delivery. + +The consumer logs aggregate pending counts and oldest-arrival age. Arrivals older than five minutes, or delivery/inspection failures continuing for one minute, raise a deduplicated operator alert through the existing operator-notification integration when configured. Logs and notifications contain counts and age, not message bodies or addresses. Operator notifications repeat at most once per fifteen minutes; without an operator channel, monitor consumer logs for `sync arrival queue needs attention`. An empty queue is not proof of provider success or historical completeness. + +See [upgrade order and downgrade limits](/development/install/#mailbox-sync-upgrade-order). Legacy maps without stored arrival payloads cannot reconstruct historical lost mail. + ## Outbound calls ### Diagnostic and managed-consent portability diff --git a/docs/content/docs/development/install.mdx b/docs/content/docs/development/install.mdx index e6bf5d7cc..bb7173591 100644 --- a/docs/content/docs/development/install.mdx +++ b/docs/content/docs/development/install.mdx @@ -294,6 +294,16 @@ Put your own compose changes in a `docker-compose.override.yml`, which compose m Upgrade the backend first and confirm its database migrations complete before upgrading consumers. Consumers wait up to two minutes for the clean migration version required by their embedded code before starting database-dependent work. If the backend is still old or a migration failed, the consumer exits with an actionable schema-readiness error and can retry through the container restart policy. It does not run migrations or ignore missing tables. +### Mailbox sync upgrade order + +For durable mailbox arrivals, upgrade the backend, let its migrations complete, then upgrade and start the consumer before upgrading workers. The backend admits each new provider map and its organization-encrypted arrival payload in one database transaction; the consumer delivers pending stages after restart. Workers continue using internal HTTP endpoints and do not need database access. Preserve the instance's organization keys and body storage with the database, as described in [data control](/development/data-control/#mailbox-sync-recovery-data). + +An old worker with a new backend still writes legacy map-only records. A new worker with an old backend falls back only when the durable internal endpoint is absent (HTTP 404 or 405). Neither mixed-version path has the new durable-admission guarantee. A backend failure or an acknowledgement without the durable marker is not treated as success. When a conflicting map has no retained pending payload, the backend fails closed instead of claiming a durable admission; this includes a retry after the pending row has already been delivered and removed. Ordinary map lookup still deduplicates that message. + +Existing completed imports and maps are preserved. Pre-upgrade maps contain no replay payload, so an old lost arrival cannot be reconstructed from the map alone. Bounded provider recovery covers supported recent history, not an unlimited historical repair. [Mailbox sync](/guides/mailboxes/#what-gets-synced) explains the provider limits. + +Keep the upgraded consumer running until all pending arrivals finish before downgrading. The migration refuses to remove the arrival table while any pending row remains. Do not delete queued rows or force a migration version to bypass that refusal. An older consumer cannot drain this queue. Pending arrivals are instance-local and do not move in a workspace export. + Fleet auto-updates select releases no newer than the running backend. Existing fleet and node pins are checked again when answering heartbeats, so an incompatible stored target is not sent to nodes; new incompatible pins are rejected. Newly joined machines fall back to the backend's exact release, not the moving `prod` tag. Main builds use their published commit-SHA image tag. Unstamped custom builds must set `WARMBLY_VERSION` to their published image tag; unknown versions cannot be ordered safely. Upgrade the backend before moving the fleet to a newer release. This protection only takes effect after installing the compatibility release; it cannot change the behavior of an older backend already running. The dashboard discovers a separate realtime origin from the backend's existing public configuration at container startup or static build time. Bring the backend up first; if discovery warns that it is unavailable, restart the dashboard after backend readiness or rebuild the static deployment. See [configuration](/development/configuration/). diff --git a/docs/content/docs/guides/mailboxes.mdx b/docs/content/docs/guides/mailboxes.mdx index 80b562d78..21b0d5ced 100644 --- a/docs/content/docs/guides/mailboxes.mdx +++ b/docs/content/docs/guides/mailboxes.mdx @@ -207,7 +207,15 @@ Connecting a mailbox does two things: it imports the mailbox's recent history, a **The import** brings in the last `90` days, newest first, up to `5,000` messages per mailbox (the defaults; a self-hosted instance can change both under Instance settings). It reads the inbox, sent mail and archive (plus your own folders on Gmail and IMAP), and skips trash, drafts, spam and Gmail's "All Mail". You can watch it in the mailbox drawer's **Sync** card: a progress bar with a running count while it works, then "Up to date" with what was imported. Because it runs newest first, the mail you would look for is there within the first minute; a large mailbox finishes over the next while at a steady pace. A few IMAP servers (Seznam.cz, for one) refuse to search a folder by date; on those the import reads each message's arrival date instead, which makes the first pass over a large folder a little slower and changes nothing else. -**Live sync** then follows every folder the provider exposes, including junk (so placement problems are visible) and sent mail (so a conversation shows both sides). Read state, flags, deletions and moves are mirrored too. +**Live sync** follows the provider's supported folders, including junk (so placement problems are visible) and sent mail (so a conversation shows both sides). Outlook follows Inbox, Sent, Junk, Drafts and Archive where Archive is available; it does not follow every custom Graph folder. Read state, flags, deletions and moves are mirrored too. + +The Sync card distinguishes a completed history import from a recent successful live check. An unfinished pass, provider failure, recovery scan or budget hold does not refresh the last successful check or erase the last observed deferred count. That count is an observation from sync, not a measurement of all mail still on the provider. A worker heartbeat alone is not evidence that a mailbox is current. Repeated connection failures refresh the warning's observation time without generating a new notification each pass; no later successful check means recovery is unverified. + +If Gmail rejects an ordinary import continuation token, the import restarts inside its original fixed date window. Already mapped mail is deduplicated, and the existing message cap and sync budgets still apply. Outlook recovers an expired delta checkpoint by replaying the configured recent-history window, reconciling known messages and following changes from a fresh checkpoint only after recovery completes. Older messages never imported are outside that recovery window. Authentication failures, throttles and transport errors are not treated as expired checkpoints. + +On IMAP, discovering a new folder or a changed UIDVALIDITY generation during or after the initial import schedules a bounded recent-history scan. Generation recovery reconciles known message handles and read state without restarting the global import count. In folders excluded from history imports, such as Junk and Trash, recovery updates only known messages and does not import unseen history. An unreadable or incomplete message body remains retryable rather than being stored as a successful empty message. A folder without usable modification sequences falls back to UIDNEXT checks and periodic flags even if the server advertises CONDSTORE. Failed flag publication keeps the previous baseline so it can retry. Background lookups do not mark unread messages as seen. + +On upgraded instances, a newly admitted provider message is recorded together with an encrypted pending arrival before sync advances. The consumer replays that arrival under its original identity after a restart, including related bounce or complaint processing, and provider updates wait for an unresolved arrival. This is an at-least-once recovery path, not a promise that all historical mail can be reconstructed. Old map-only records have no replay payload, and mixed-version instances retain their legacy behavior. See the [self-host upgrade order and limits](/development/install/#mailbox-sync-upgrade-order). If Gmail expires its history checkpoint after a mailbox has been offline, Warmbly automatically scans the configured recent-history window for missed messages and checks every stored Gmail message for deletions, moves and read-state changes. Recovery resumes across worker restarts and respects the normal sync budget. Messages older than the configured import window that were never stored are not imported by recovery. Once the scan completes, changes made during recovery are followed from a fresh history checkpoint; no reconnection is needed. @@ -221,12 +229,12 @@ A folder the sync does not recognize files as inbox, so its mail shows up in the On an IMAP mailbox, the drawer's **Sync** card lists the folders the worker has seen under **Folders not synced**. Tick one and the sync stops opening it on its next pass, within a minute; mail already imported from it is removed from Warmbly, and mail that later moves into it from a synced folder is removed too. The mail itself is never touched at the provider. A folder that is not listed yet, because the mailbox has not synced or the folder was just created, can be typed in by name, exactly as your mail server lists it (`Warmer`, or `INBOX/Warmer` on a server that nests everything under the inbox). Case does not matter, and a skipped folder covers its subfolders. -The inbox, sent, drafts, spam, trash and archive folders cannot be skipped: a sync without them is a broken mailbox, not a quieter one. Un-ticking a folder brings it back the way a new folder arrives, syncing what lands in it from then on; the mail that accumulated while it was skipped is not imported. Up to `50` folders can be skipped per mailbox. Gmail and Outlook mailboxes have no such list. +The inbox, sent, drafts, spam, trash and archive folders cannot be skipped: a sync without them is a broken mailbox, not a quieter one. Un-ticking a folder brings it back with a bounded scan of the configured recent-history window; older mail outside that window is not imported. Up to `50` folders can be skipped per mailbox. Gmail and Outlook mailboxes have no such list. The same setting is `PUT /emails/:id/sync` in the [API](/api/endpoints/) and `warmbly mailbox skip-folders` in the CLI. - Some IMAP servers, including Outlook.com, Microsoft 365 over IMAP and Yahoo, cannot tell a client what changed since it last looked. New mail from those servers still arrives within a minute. Reading or flagging a message in another mail client shows up in Warmbly within about ten minutes rather than immediately. Nothing is lost either way, and Gmail, Outlook over OAuth, Fastmail and most self-hosted servers are immediate. + Some IMAP servers, including Outlook.com, Microsoft 365 over IMAP and Yahoo, cannot tell a client what changed since it last looked. Warmbly checks for new mail on each sync pass and scans read state and flags about every ten minutes. Those changes may take longer to appear than on a provider with usable modification sequences. Provider availability and sync budgets still apply. diff --git a/docs/content/docs/guides/warmup.mdx b/docs/content/docs/guides/warmup.mdx index 6e36c3e05..25fad0a52 100644 --- a/docs/content/docs/guides/warmup.mdx +++ b/docs/content/docs/guides/warmup.mdx @@ -208,6 +208,8 @@ At the volumes warmup runs, a few plaintext messages of a couple of kilobytes ea ### In the unibox +Mailbox-sync recovery replays stored arrivals, not sends. Recognized warmup remains hidden, and replay does not create another warmup send or count sync progress as proof that a provider sent mail. Receipt verification and duplicate-send protections remain separate from the mailbox's live-sync status. See [mailbox sync recovery and its limits](/guides/mailboxes/#what-gets-synced). + Warmup never appears in the unified inbox, in either direction. Warmbly recognizes it by its token, including for recipient-only mailboxes. A consumed or expired token still identifies warmup for filtering, but cannot trigger engagement again. When the header is missing, a recorded message identifier can identify older warmup during history sync; the sender-and-subject fallback stays limited to recent inbound sends so an ordinary conversation is not hidden just because it shares a subject. Existing verified warmup entries are removed from the unibox automatically in background batches, without deleting mail at the provider or changing warmup standing. Manual replies to verified diagnostic parents can stay hidden from the unibox and follow housekeeping. They do not authorize another automated turn or count as real campaign replies. diff --git a/docs/content/docs/guides/workspace-export-import.mdx b/docs/content/docs/guides/workspace-export-import.mdx index b1e9b73ec..ad39d26ea 100644 --- a/docs/content/docs/guides/workspace-export-import.mdx +++ b/docs/content/docs/guides/workspace-export-import.mdx @@ -150,6 +150,8 @@ If you have shell access to the source, `warmblyctl org export` writes the same ## What does not travel +Pending mailbox arrivals and their delivery queue are instance-local. Provider mappings whose arrivals are still pending are excluded too. Let the source consumer finish its pending arrivals before exporting; importing the archive does not replay that source queue. A [full-instance backup](/development/data-control/#mailbox-sync-recovery-data) instead preserves the queue together with its keys and message-body storage. + The Test workspace classification is instance-local. Importing an archive does not classify the destination as Test or grant a managed Test plan or testing credits. A few things belong to the instance rather than the workspace and are left out on purpose: the instance's link to Warmbly Cloud and which mailboxes it enrolled in the hosted pool ([Warmbly Cloud](/guides/warmbly-cloud/)), linked self-hosted instances on a cloud workspace, Google Workspace and Microsoft 365 admin grants, live sessions, in-flight OAuth handshakes and the organization's wrapped data key. Reconnect to Warmbly Cloud and make each admin grant again after the move. diff --git a/go.mod b/go.mod index 2b1aa2f15..551e34ae1 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssm v1.67.7 github.com/aws/smithy-go v1.28.1 github.com/confluentinc/confluent-kafka-go/v2 v2.12.0 - github.com/emersion/go-imap/v2 v2.0.0-beta.5 + github.com/emersion/go-imap/v2 v2.0.0-beta.7 github.com/emersion/go-message v0.18.1 github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 github.com/getsentry/sentry-go v0.36.2 diff --git a/go.sum b/go.sum index daee0c86c..f4a37ffcc 100644 --- a/go.sum +++ b/go.sum @@ -269,6 +269,8 @@ github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203 h1:XBBHcIb256gUJ github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203/go.mod h1:E1jcSv8FaEny+OP/5k9UxZVw9YFWGj7eI4KR/iOBqCg= github.com/emersion/go-imap/v2 v2.0.0-beta.5 h1:H3858DNmBuXyMK1++YrQIRdpKE1MwBc+ywBtg3n+0wA= github.com/emersion/go-imap/v2 v2.0.0-beta.5/go.mod h1:BZTFHsS1hmgBkFlHqbxGLXk2hnRqTItUgwjSSCsYNAk= +github.com/emersion/go-imap/v2 v2.0.0-beta.7 h1:lNznYWa5uhMrngnSYEklzCeye4DBq9TEJ+pr0K593+8= +github.com/emersion/go-imap/v2 v2.0.0-beta.7/go.mod h1:BZTFHsS1hmgBkFlHqbxGLXk2hnRqTItUgwjSSCsYNAk= github.com/emersion/go-message v0.18.1 h1:tfTxIoXFSFRwWaZsgnqS1DSZuGpYGzSmCZD8SK3QA2E= github.com/emersion/go-message v0.18.1/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA= github.com/emersion/go-msgauth v0.7.0 h1:vj2hMn6KhFtW41kshIBTXvp6KgYSqpA/ZN9Pv4g1INc= diff --git a/internal/api/handler/internal_arrival_test.go b/internal/api/handler/internal_arrival_test.go new file mode 100644 index 000000000..5a7c499a0 --- /dev/null +++ b/internal/api/handler/internal_arrival_test.go @@ -0,0 +1,70 @@ +package handler + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/repository" +) + +type internalArrivalMap struct { + calls, legacyCalls int + err error +} + +func (m *internalArrivalMap) Add(context.Context, repository.EmailMessageData) error { + m.legacyCalls++ + return nil +} +func (*internalArrivalMap) Get(context.Context, uuid.UUID, uuid.UUID, string) (*repository.EmailMessageData, error) { + return nil, nil +} +func (*internalArrivalMap) Del(context.Context, uuid.UUID, uuid.UUID, string, uuid.UUID) error { + return nil +} +func (m *internalArrivalMap) AdmitArrival(context.Context, repository.EmailMessageData, *repository.PendingArrival) error { + m.calls++ + return m.err +} + +func TestInternalArrivalAcknowledgesOnlyCompletedAdmission(t *testing.T) { + for _, tc := range []struct { + name string + err error + status int + ack string + }{ + {"success", nil, 204, "1"}, {"failure", errors.New("private payload or cipher failure"), 503, ""}, + {"unconfirmed collision", repository.ErrArrivalAdmissionUnconfirmed, 503, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + m := &internalArrivalMap{err: tc.err} + h := &Handler{EmailMessageMap: m} + r := gin.New() + r.POST("/arrival", h.InternalAdmitEmailArrival) + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/arrival", strings.NewReader(`{"map":{},"pending":{}}`))) + if m.calls != 1 || w.Code != tc.status || w.Header().Get("X-Warmbly-Arrival-Durable") != tc.ack || w.Body.Len() != 0 { + t.Fatalf("calls=%d status=%d body=%q", m.calls, w.Code, w.Body.String()) + } + }) + } +} + +func TestInternalArrivalPreservesLegacyWorkerProtocol(t *testing.T) { + m := &internalArrivalMap{} + h := &Handler{EmailMessageMap: m} + r := gin.New() + r.PUT("/map", h.InternalPutEmailMessageMap) + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodPut, "/map", strings.NewReader(`{"user_id":"fixture","email_id":"fixture","message_id":"fixture","id":"fixture","thread_id":"fixture"}`))) + if w.Code != 204 || m.legacyCalls != 1 || m.calls != 0 || w.Header().Get("X-Warmbly-Arrival-Durable") != "" { + t.Fatalf("legacy contract changed: %d", w.Code) + } +} diff --git a/internal/api/handler/internal_email_message_map.go b/internal/api/handler/internal_email_message_map.go index ca320d69f..b6fc83659 100644 --- a/internal/api/handler/internal_email_message_map.go +++ b/internal/api/handler/internal_email_message_map.go @@ -11,6 +11,29 @@ import ( "github.com/warmbly/warmbly/internal/repository" ) +func (h *Handler) InternalAdmitEmailArrival(c *gin.Context) { + var p struct { + Map emailMessageMapPayload `json:"map"` + Pending *repository.PendingArrival `json:"pending"` + } + if err := json.NewDecoder(http.MaxBytesReader(c.Writer, c.Request.Body, 4<<20)).Decode(&p); err != nil { + c.Status(http.StatusBadRequest) + return + } + repo, ok := h.EmailMessageMap.(repository.ArrivalAdmission) + if !ok { + c.Status(http.StatusServiceUnavailable) + return + } + if err := repo.AdmitArrival(c.Request.Context(), repository.EmailMessageData(p.Map), p.Pending); err != nil { + // Do not expose message content or org encryption failures over the protocol. + c.Status(http.StatusServiceUnavailable) + return + } + c.Header("X-Warmbly-Arrival-Durable", "1") + c.Status(http.StatusNoContent) +} + // Internal email-message-map endpoints. Workers call these instead of touching // Postgres directly (per CLAUDE.md), mirroring the DEK endpoints. Auth via // middleware.NodeBrokerAuthMiddleware (the node token). diff --git a/internal/api/routes.go b/internal/api/routes.go index d68d5feee..45fb66c7f 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -191,6 +191,7 @@ func Run( // Worker mailbox-sync messageId -> internal email map. node.GET("/email-message-map", h.InternalGetEmailMessageMap) node.PUT("/email-message-map", h.InternalPutEmailMessageMap) + node.POST("/email-message-map/arrival", h.InternalAdmitEmailArrival) node.DELETE("/email-message-map", h.InternalDeleteEmailMessageMap) // Sync governor priority lane: "is this new message a reply to diff --git a/internal/app/consumer/arrival_outbox.go b/internal/app/consumer/arrival_outbox.go new file mode 100644 index 000000000..1bb405685 --- /dev/null +++ b/internal/app/consumer/arrival_outbox.go @@ -0,0 +1,98 @@ +package jobs + +import ( + "context" + "fmt" + "strconv" + "time" + + "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +func (s *JobsService) runArrivalOutbox(ctx context.Context) { + if s.ArrivalOutbox == nil { + return + } + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + var failedSince, lastAlert time.Time + for { + pass, cancel := context.WithTimeout(ctx, 30*time.Second) + err := s.ArrivalOutbox.DeliverArrivals(pass, func(ctx context.Context, kind models.JobEventType, body any) error { + return s.deliverArrivalEvent(ctx, kind, body) + }) + if err != nil { + log.Warn().Msg("sync arrival delivery deferred") + } + s.observeArrivalBacklog(pass, err, time.Now(), &failedSince, &lastAlert) + cancel() + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +func (s *JobsService) observeArrivalBacklog(ctx context.Context, deliveryErr error, now time.Time, failedSince, lastAlert *time.Time) { + backlog := repository.ArrivalBacklog{} + known := false + if stats, ok := s.ArrivalOutbox.(repository.ArrivalBacklogRepository); ok { + var err error + backlog, err = stats.ArrivalBacklog(ctx) + if err != nil { + deliveryErr = err + } else { + known = true + } + } + if deliveryErr != nil { + if failedSince.IsZero() { + *failedSince = now + } + } else { + *failedSince = time.Time{} + } + age := time.Duration(0) + if backlog.Oldest != nil { + age = max(0, now.Sub(*backlog.Oldest)) + } + if age < 5*time.Minute && (failedSince.IsZero() || now.Sub(*failedSince) < time.Minute) { + return + } + if !lastAlert.IsZero() && now.Sub(*lastAlert) < 15*time.Minute { + return + } + *lastAlert = now + entry := log.Warn().Bool("backlog_known", known) + if known { + entry.Int("pending", backlog.Pending).Int64("oldest_age_seconds", int64(age.Seconds())) + } + entry.Msg("sync arrival queue needs attention") + if s.OpsNotifier == nil { + return + } + if s.Cache != nil { + ok, err := s.Cache.SetNX(ctx, "sync:arrival:opsnotify", "1", 15*time.Minute).Result() + if err == nil && !ok { + return + } + } + fields := map[string]string{"Pending": "unavailable", "Oldest age seconds": "unavailable"} + if known { + fields["Pending"] = strconv.Itoa(backlog.Pending) + fields["Oldest age seconds"] = strconv.FormatInt(int64(age.Seconds()), 10) + } + s.OpsNotifier.NotifyOperator("sync.arrival.backlog", "Mailbox arrival delivery delayed", + "Stored arrivals are waiting for delivery, or delivery checks are failing. Check the consumer and database before assuming sync is current.", + fields) +} + +func (s *JobsService) deliverArrivalEvent(ctx context.Context, kind models.JobEventType, body any) error { + if _, ok := s.eventHandlers[kind]; !ok { + return fmt.Errorf("durable arrival handler unavailable: %s", kind) + } + return s.HandleEvent(ctx, &models.JobEvent{Type: kind, Body: body}) +} diff --git a/internal/app/consumer/arrival_outbox_live_test.go b/internal/app/consumer/arrival_outbox_live_test.go new file mode 100644 index 000000000..ff4e98c29 --- /dev/null +++ b/internal/app/consumer/arrival_outbox_live_test.go @@ -0,0 +1,151 @@ +package jobs + +import ( + "context" + "errors" + "testing" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/app/advanced" + "github.com/warmbly/warmbly/internal/app/cipher" + "github.com/warmbly/warmbly/internal/app/warmup" + "github.com/warmbly/warmbly/internal/infrastructure/encryptedkeys" + "github.com/warmbly/warmbly/internal/infrastructure/kms" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +func TestLiveSyncArrivalReplayPreservesHiddenWarmupAndReceiptAuthority(t *testing.T) { + s, d := liveWarmupService(t) + f := newWarmupFixture(t, d) + ctx := t.Context() + f.mintToken(t, s.WarmupRepo) + k, err := kms.NewLocal(make([]byte, 32)) + if err != nil { + t.Fatal(err) + } + r := repository.NewDurableEmailMessageMapRepository(d, cipher.NewService(k, nil, encryptedkeys.NewPostgres(d))) + e := f.arrival(f.sentMsgID, nil) + e.UserID = f.senderUser + e.Message.EmailID = f.sender + e.Message.Folder = models.FolderSent + data := repository.EmailMessageData{UserID: e.UserID.String(), EmailID: e.Message.EmailID.String(), MessageID: e.Message.MessageID, ID: e.Message.ID.String(), ThreadID: e.Message.ThreadID} + if err := r.AdmitArrival(ctx, data, &repository.PendingArrival{Arrival: e}); err != nil { + t.Fatal(err) + } + s.InitEvents() + lost := errors.New("consumer terminated after handling") + if err := r.DeliverArrivals(ctx, func(ctx context.Context, kind models.JobEventType, body any) error { + if err := s.deliverArrivalEvent(ctx, kind, body); err != nil { + return err + } + return lost + }); !errors.Is(err, lost) { + t.Fatal(err) + } + if _, err := d.Exec(ctx, `UPDATE sync_arrival_outbox SET retry_at=now() WHERE email_id=$1`, f.sender); err != nil { + t.Fatal(err) + } + fresh, _ := liveWarmupService(t) + fresh.InitEvents() + if err := r.DeliverArrivals(ctx, fresh.deliverArrivalEvent); err != nil { + t.Fatal(err) + } + var visible, pending, receipts int + if err := d.QueryRow(ctx, `SELECT count(*) FROM unibox_emails WHERE id=$1`, e.Message.ID).Scan(&visible); err != nil || visible != 0 { + t.Fatalf("unconfirmed sent copy exposed: %d %v", visible, err) + } + if err := d.QueryRow(ctx, `SELECT count(*) FROM unibox_pending_emails WHERE id=$1`, e.Message.ID).Scan(&pending); err != nil || pending != 1 { + t.Fatalf("verification deferral lost original ID: %d %v", pending, err) + } + if err := d.QueryRow(ctx, `SELECT count(*) FROM warmup_received WHERE sender_account_id=$1`, f.sender).Scan(&receipts); err != nil || receipts != 0 { + t.Fatalf("sync invented receipt: %d %v", receipts, err) + } + if err := fresh.HandleEmailSent(ctx, models.SendEmailResult{TaskID: f.task, Success: true, MessageID: f.sentMsgID}); err != nil { + t.Fatal(err) + } + if _, err := d.Exec(ctx, `UPDATE unibox_pending_emails SET retry_at=now() WHERE email_account_id=$1`, f.sender); err != nil { + t.Fatal(err) + } + if err := fresh.retryPendingWarmupVerification(ctx); err != nil { + t.Fatal(err) + } + if err := d.QueryRow(ctx, `SELECT count(*) FROM unibox_emails WHERE id=$1`, e.Message.ID).Scan(&visible); err != nil || visible != 0 { + t.Fatalf("confirmed warmup exposed: %d %v", visible, err) + } + if err := d.QueryRow(ctx, `SELECT count(*) FROM warmup_received WHERE sender_account_id=$1`, f.sender).Scan(&receipts); err != nil || receipts != 0 { + t.Fatalf("sent copy invented recipient receipt: %d %v", receipts, err) + } + other := f.arrival("<"+uuid.NewString()+"@test.local>", nil) + other.Message.Subject = "ordinary customer question" + data = repository.EmailMessageData{UserID: other.UserID.String(), EmailID: other.Message.EmailID.String(), MessageID: other.Message.MessageID, ID: other.Message.ID.String()} + if err := r.AdmitArrival(ctx, data, &repository.PendingArrival{Arrival: other}); err != nil { + t.Fatal(err) + } + if err := r.DeliverArrivals(ctx, fresh.deliverArrivalEvent); err != nil { + t.Fatal(err) + } + if err := d.QueryRow(ctx, `SELECT count(*) FROM unibox_emails WHERE id=$1`, other.Message.ID).Scan(&visible); err != nil || visible != 1 { + t.Fatalf("ordinary arrival missing: %d %v", visible, err) + } +} + +func TestLiveSyncArrivalReportReplayIsIdempotent(t *testing.T) { + d := liveDB(t) + f := newSendResultFixture(t, d) + s := liveJobsService(d) + ctx := t.Context() + task := f.stampSend(t, s) + messageID := "<" + task.String() + "@test.local>" + if _, err := d.Exec(ctx, `UPDATE tasks SET message_id=$2 WHERE id=$1`, task, messageID); err != nil { + t.Fatal(err) + } + s.UniboxRepository = repository.NewUniboxRepository(d) + s.EmailRepository = repository.NewEmailRepostory(d, nil) + s.WarmupRepo = repository.NewWarmupRepository(d.Pool) + s.AdvancedService = advanced.NewService(repository.NewAdvancedOutreachRepository(d.Pool), s.CampaignRepo, s.EmailRepository, s.TaskRepo, repository.NewContactRepostory(d), s.CampaignProgressRepo, nil, nil, nil, nil, warmup.NewService(s.WarmupRepo)) + s.InitEvents() + k, err := kms.NewLocal(make([]byte, 32)) + if err != nil { + t.Fatal(err) + } + cipherService := cipher.NewService(k, nil, encryptedkeys.NewPostgres(d)) + r := repository.NewDurableEmailMessageMapRepository(d, cipherService) + id := uuid.New() + recipient := "lead-" + f.contact.String()[:8] + "@test.local" + p := &repository.PendingArrival{Arrival: &models.JobEventNewEmail{UserID: f.user, ReportOriginalMessageID: messageID, Message: &models.EmailMessageStoreData{ID: id, EmailID: f.mailbox, MessageID: "report", Folder: models.FolderInbox}}, Bounce: &models.JobEventInboundBounce{UserID: f.user, EmailID: f.mailbox, OriginalMessageID: messageID, FailedRecipient: recipient, Reason: "recipient refused"}, Complaint: &models.JobEventInboundComplaint{UserID: f.user, EmailID: f.mailbox, OriginalMessageID: messageID, ComplainedRecipient: recipient, Provider: "fixture"}} + data := repository.EmailMessageData{UserID: f.user.String(), EmailID: f.mailbox.String(), MessageID: "report", ID: id.String()} + if err := r.AdmitArrival(ctx, data, p); err != nil { + t.Fatal(err) + } + lost := errors.New("result lost after report was recorded") + for _, target := range []models.JobEventType{models.JobEventTypeInboundBounce, models.JobEventTypeInboundComplaint} { + if err := r.DeliverArrivals(ctx, func(ctx context.Context, kind models.JobEventType, body any) error { + if err := s.deliverArrivalEvent(ctx, kind, body); err != nil { + return err + } + if kind == target { + return lost + } + return nil + }); !errors.Is(err, lost) { + t.Fatal(err) + } + if _, err := d.Exec(ctx, `UPDATE sync_arrival_outbox SET retry_at=now() WHERE email_id=$1`, f.mailbox); err != nil { + t.Fatal(err) + } + } + if err := repository.NewDurableEmailMessageMapRepository(d, cipherService).DeliverArrivals(ctx, s.deliverArrivalEvent); err != nil { + t.Fatal(err) + } + var events, suppressed int + if err := d.QueryRow(ctx, `SELECT count(*) FROM deliverability_events WHERE organization_id=$1`, f.org).Scan(&events); err != nil { + t.Fatal(err) + } + if err := d.QueryRow(ctx, `SELECT count(*) FROM suppressed_recipients WHERE organization_id=$1`, f.org).Scan(&suppressed); err != nil { + t.Fatal(err) + } + if events != 2 || suppressed != 1 { + t.Fatalf("report replay duplicated effects: events=%d suppressed=%d", events, suppressed) + } +} diff --git a/internal/app/consumer/arrival_outbox_test.go b/internal/app/consumer/arrival_outbox_test.go new file mode 100644 index 000000000..857ffaa8b --- /dev/null +++ b/internal/app/consumer/arrival_outbox_test.go @@ -0,0 +1,81 @@ +package jobs + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +func TestDurableArrivalRequiresRegisteredHandler(t *testing.T) { + s := &JobsService{} + if err := s.deliverArrivalEvent(context.Background(), models.JobEventTypeNewEmail, &models.JobEventNewEmail{}); err == nil { + t.Fatal("unregistered durable event dropped") + } + called := false + s.InitEvents() + Register(s, models.JobEventTypeNewEmail, func(context.Context, *models.JobEventNewEmail) error { called = true; return nil }) + if err := s.deliverArrivalEvent(context.Background(), models.JobEventTypeNewEmail, &models.JobEventNewEmail{}); err != nil || !called { + t.Fatalf("called=%v err=%v", called, err) + } +} + +type arrivalBacklogStub struct { + repository.ArrivalOutbox + backlog repository.ArrivalBacklog + err error +} + +func (s *arrivalBacklogStub) ArrivalBacklog(context.Context) (repository.ArrivalBacklog, error) { + return s.backlog, s.err +} + +type arrivalOperatorStub struct { + calls int + content string + fields map[string]string +} + +func (s *arrivalOperatorStub) NotifyOperator(key, title, summary string, fields map[string]string) { + s.calls++ + s.content = key + title + summary + s.fields = fields + if len(fields) != 2 { + panic("unexpected private fields") + } +} + +func TestDurableArrivalBacklogAlertsAreAgedAndDeduplicated(t *testing.T) { + now := time.Now() + old := now.Add(-6 * time.Minute) + outbox := &arrivalBacklogStub{backlog: repository.ArrivalBacklog{Pending: 8, Oldest: &old}} + ops := &arrivalOperatorStub{} + s := &JobsService{ArrivalOutbox: outbox, OpsNotifier: ops} + var failed, alerted time.Time + s.observeArrivalBacklog(t.Context(), nil, now, &failed, &alerted) + s.observeArrivalBacklog(t.Context(), nil, now.Add(time.Minute), &failed, &alerted) + if ops.calls != 1 { + t.Fatalf("repeat alerts=%d", ops.calls) + } + outbox.backlog = repository.ArrivalBacklog{} + s.observeArrivalBacklog(t.Context(), nil, now.Add(16*time.Minute), &failed, &alerted) + if ops.calls != 1 { + t.Fatal("empty queue claimed ongoing backlog") + } + outbox.err = errors.New("private database error") + s.observeArrivalBacklog(t.Context(), nil, now.Add(17*time.Minute), &failed, &alerted) + if ops.calls != 1 { + t.Fatal("transient read failure alerted too soon") + } + s.observeArrivalBacklog(t.Context(), nil, now.Add(18*time.Minute), &failed, &alerted) + if ops.calls != 2 { + t.Fatal("persistent queue inspection failure was silent") + } + if strings.Contains(ops.content, "private database error") || ops.fields["Pending"] != "unavailable" || ops.fields["Oldest age seconds"] != "unavailable" { + t.Fatal("operator notice exposed an error or fabricated unavailable counts") + } +} diff --git a/internal/app/consumer/event_flags_update.go b/internal/app/consumer/event_flags_update.go index a2fe5f5a8..39cd3f73a 100644 --- a/internal/app/consumer/event_flags_update.go +++ b/internal/app/consumer/event_flags_update.go @@ -38,7 +38,7 @@ func (s *JobsService) HandleFlagsAdd(ctx context.Context, e *models.JobEventFlag } } - email, err := s.emailForSyncUpdate(ctx, e.UserID, e.ID, func(message *models.EmailMessageStoreData) { + email, err := s.emailForSyncUpdate(ctx, e.UserID, e.EmailID, e.ID, func(message *models.EmailMessageStoreData) { for _, flag := range e.Flags { if !slices.Contains(message.Flags, flag) { message.Flags = append(message.Flags, flag) @@ -155,7 +155,7 @@ func warmupTokenFromFlags(flags []string) string { } func (s *JobsService) HandleFlagsRemove(ctx context.Context, e *models.JobEventFlags) error { - email, err := s.emailForSyncUpdate(ctx, e.UserID, e.ID, func(message *models.EmailMessageStoreData) { + email, err := s.emailForSyncUpdate(ctx, e.UserID, e.EmailID, e.ID, func(message *models.EmailMessageStoreData) { message.Flags = slices.DeleteFunc(message.Flags, func(flag string) bool { return slices.Contains(e.Flags, flag) }) if models.SeenFromFlags(e.Flags) { message.Seen = false diff --git a/internal/app/consumer/event_remove_email.go b/internal/app/consumer/event_remove_email.go index 9ca4af0b0..0bf1a9509 100644 --- a/internal/app/consumer/event_remove_email.go +++ b/internal/app/consumer/event_remove_email.go @@ -2,6 +2,7 @@ package jobs import ( "context" + "errors" "time" "github.com/rs/zerolog/log" @@ -20,8 +21,17 @@ import ( // mailbox. A removal later than the window is housekeeping (see // warmupDeletionCounts) and is not held against anyone. // -// It also drops the local unibox entry for the removed message (best-effort). +// It drops the local unibox entry only after durable arrival replay is complete. func (s *JobsService) HandleRemoveEmail(ctx context.Context, e *models.JobEventRemoveEmail) error { + if s.ArrivalOutbox != nil { + pending, err := s.ArrivalOutbox.HasPendingArrival(ctx, e.UserID, e.EmailID, e.ID) + if err != nil { + return err + } + if pending { + return ErrSyncArrivalPending + } + } var checkErr error // A message the sync found in a folder the owner excluded is filed, not // deleted: it is still in the mailbox, so nothing is held against anyone. @@ -48,7 +58,9 @@ func (s *JobsService) HandleRemoveEmail(ctx context.Context, e *models.JobEventR } if s.UniboxRepository != nil { - _ = s.UniboxRepository.Delete(ctx, e.UserID, e.ID) + if err := s.UniboxRepository.Delete(ctx, e.UserID, e.ID); err != nil { + return errors.Join(checkErr, err) + } } s.publishInboxDeleted(ctx, e.UserID, e.EmailID, e.ID.String()) diff --git a/internal/app/consumer/event_sync_state.go b/internal/app/consumer/event_sync_state.go index dd6565523..17d381df4 100644 --- a/internal/app/consumer/event_sync_state.go +++ b/internal/app/consumer/event_sync_state.go @@ -34,12 +34,11 @@ func (s *JobsService) HandleSyncState(ctx context.Context, e *models.JobEventSyn return err } - // A relayed state means a pass completed: the worker reached the server, - // listed its folders and finished the tick. That is the only signal we - // get that an outage is over, and without it a five-minute blip left a - // red "needs attention" on the mailbox for good, because nothing but a - // credential reconnect ever resolved an error row. - s.resolveTransientMailErrors(ctx, e.EmailID, e.State.LastSyncedAt) + // Progress relays are not recovery evidence; require a newer successful check. + if e.State.LastSyncedAt != nil && e.State.Deferred == 0 && e.State.BackfillCursor.GoogleRecovery == nil && + (prev == nil || prev.LastSyncedAt == nil || e.State.LastSyncedAt.After(*prev.LastSyncedAt)) { + s.resolveTransientMailErrors(ctx, e.EmailID, e.State.LastSyncedAt) + } if s.StreamingPublisher == nil { return nil diff --git a/internal/app/consumer/event_sync_state_test.go b/internal/app/consumer/event_sync_state_test.go index 6218eacb4..9d077256d 100644 --- a/internal/app/consumer/event_sync_state_test.go +++ b/internal/app/consumer/event_sync_state_test.go @@ -14,11 +14,12 @@ import ( // stubSyncStateRepo accepts what the worker relays. type stubSyncStateRepo struct { repository.EmailSyncStateRepository - put *models.SyncState + put *models.SyncState + saved *models.SyncState } func (s *stubSyncStateRepo) Get(context.Context, uuid.UUID) (*models.SyncState, error) { - return nil, nil + return s.saved, nil } func (s *stubSyncStateRepo) Put(_ context.Context, _, _ uuid.UUID, st *models.SyncState) error { diff --git a/internal/app/consumer/event_update_email.go b/internal/app/consumer/event_update_email.go index 128545cf6..216995de3 100644 --- a/internal/app/consumer/event_update_email.go +++ b/internal/app/consumer/event_update_email.go @@ -13,7 +13,7 @@ import ( ) func (s *JobsService) HandleUpdateEmail(ctx context.Context, e *models.JobEventEmailUpdate) error { - email, err := s.emailForSyncUpdate(ctx, e.UserID, e.ID, func(message *models.EmailMessageStoreData) { + email, err := s.emailForSyncUpdate(ctx, e.UserID, e.EmailID, e.ID, func(message *models.EmailMessageStoreData) { token := warmupTokenFromMessage(message) message.Flags = append([]string{}, e.Flags...) if token != "" { @@ -98,7 +98,7 @@ func (s *JobsService) HandleFolderUpdate(ctx context.Context, e *models.JobEvent if e.Relayed { return s.applyRelayedFolder(ctx, e) } - email, err := s.emailForSyncUpdate(ctx, e.UserID, e.ID, func(message *models.EmailMessageStoreData) { + email, err := s.emailForSyncUpdate(ctx, e.UserID, e.EmailID, e.ID, func(message *models.EmailMessageStoreData) { // A pending row is not visible yet, so nothing has filed it locally. message.Folder = e.Folder }) @@ -144,16 +144,33 @@ func (s *JobsService) applyRelayedFolder(ctx context.Context, e *models.JobEvent } // emailForSyncUpdate rechecks visible mail if verification won the pending-row lock. -func (s *JobsService) emailForSyncUpdate(ctx context.Context, userID, id uuid.UUID, updatePending func(*models.EmailMessageStoreData)) (*models.EmailMessageStoreData, error) { - message, err := s.UniboxRepository.GetByID(ctx, userID, id) +var ErrSyncArrivalPending = errors.New("sync arrival pending delivery") + +func (s *JobsService) emailForSyncUpdate(ctx context.Context, userID, emailID, id uuid.UUID, updatePending func(*models.EmailMessageStoreData)) (*models.EmailMessageStoreData, error) { + message, err := s.UniboxRepository.GetForSync(ctx, userID, emailID, id) if !errors.Is(err, repository.ErrEmailNotFound) { return message, err } - updated, err := s.UniboxRepository.UpdatePendingEmail(ctx, userID, id, updatePending) + updated, err := s.UniboxRepository.UpdatePendingEmail(ctx, userID, id, func(message *models.EmailMessageStoreData) { + if message.EmailID == emailID { + updatePending(message) + } + }) if err != nil || updated { return nil, err } - message, err = s.UniboxRepository.GetByID(ctx, userID, id) + message, err = s.UniboxRepository.GetForSync(ctx, userID, emailID, id) + if errors.Is(err, repository.ErrEmailNotFound) && s.ArrivalOutbox != nil { + pending, lookupErr := s.ArrivalOutbox.HasPendingArrival(ctx, userID, emailID, id) + if lookupErr != nil { + return nil, lookupErr + } + if pending { + return nil, ErrSyncArrivalPending + } + // Delivery may have finished between the visible-row read and the marker read. + message, err = s.UniboxRepository.GetForSync(ctx, userID, emailID, id) + } if errors.Is(err, repository.ErrEmailNotFound) { return nil, nil } diff --git a/internal/app/consumer/service.go b/internal/app/consumer/service.go index c889872dc..69e7afd7b 100644 --- a/internal/app/consumer/service.go +++ b/internal/app/consumer/service.go @@ -41,6 +41,7 @@ type JobsService struct { EmailHistoryIDRepository repository.EmailHistoryIDRepository EmailGraphDeltaRepository repository.EmailGraphDeltaRepository EmailSyncStateRepository repository.EmailSyncStateRepository + ArrivalOutbox repository.ArrivalOutbox EmailAccountErrorRepository repository.EmailAccountErrorRepository WarmupRepo repository.WarmupRepository // PoolLinkRepo marks warmup-only mailboxes of linked instances; nil when unused. @@ -126,6 +127,7 @@ func (s *JobsService) Start(ctx context.Context) { // timestamps. go s.sweepFollowUps(ctx) go s.runSendResultEffects(ctx) + go s.runArrivalOutbox(ctx) if err := s.Bus.Subscribe(ctx, []string{kafka.TopicWorkerEvents}, "consumer-group", s.receive); err != nil { log.Error().Err(err).Msg("consumer: worker-events subscription ended") diff --git a/internal/app/consumer/sync_truth_live_test.go b/internal/app/consumer/sync_truth_live_test.go new file mode 100644 index 000000000..1105e090c --- /dev/null +++ b/internal/app/consumer/sync_truth_live_test.go @@ -0,0 +1,317 @@ +package jobs + +import ( + "context" + "errors" + "testing" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/app/cipher" + "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/infrastructure/encryptedkeys" + "github.com/warmbly/warmbly/internal/infrastructure/kms" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +func TestLiveSyncTruthRemovalWaitsForArrivalAndLostStageAcknowledgement(t *testing.T) { + for _, lostAck := range []bool{false, true} { + for _, kind := range []string{"ordinary", "confirmed warmup", "unconfirmed warmup"} { + name := kind + "/before ingestion" + if lostAck { + name = kind + "/lost stage acknowledgement" + } + t.Run(name, func(t *testing.T) { + s, d := liveWarmupService(t) + f := newWarmupFixture(t, d) + ctx := t.Context() + key, err := kms.NewLocal(make([]byte, 32)) + if err != nil { + t.Fatal(err) + } + r := repository.NewDurableEmailMessageMapRepository(d, cipher.NewService(key, nil, encryptedkeys.NewPostgres(d))) + s.ArrivalOutbox = r + s.InitEvents() + e := f.arrival("<"+uuid.NewString()+"@test.local>", nil) + e.Message.Subject = "ordinary customer question" + var token uuid.UUID + if kind != "ordinary" { + token = f.mintToken(t, s.WarmupRepo) + if _, err := d.Exec(ctx, `INSERT INTO warmup_tasks(task_id,lineage_version,subject,max_turns) VALUES($1,1,$2,3)`, f.task, f.subject); err != nil { + t.Fatal(err) + } + e = f.arrival(f.sentMsgID, []string{config.WarmupVerifyHeader + ":" + token.String()}) + if kind == "confirmed warmup" { + if err := s.HandleEmailSent(ctx, models.SendEmailResult{TaskID: f.task, Success: true, MessageID: f.sentMsgID}); err != nil { + t.Fatal(err) + } + } + } + mapping := repository.EmailMessageData{UserID: e.UserID.String(), EmailID: e.Message.EmailID.String(), ID: e.Message.ID.String(), MessageID: e.Message.MessageID} + if err := r.AdmitArrival(ctx, mapping, &repository.PendingArrival{Arrival: e}); err != nil { + t.Fatal(err) + } + if lostAck { + lost := errors.New("arrival handled but stage acknowledgement lost") + if err := r.DeliverArrivals(ctx, func(ctx context.Context, kind models.JobEventType, body any) error { + if err := s.deliverArrivalEvent(ctx, kind, body); err != nil { + return err + } + return lost + }); !errors.Is(err, lost) { + t.Fatalf("stage loss fixture: %v", err) + } + } + remove := &models.JobEventRemoveEmail{UserID: e.UserID, EmailID: e.Message.EmailID, ID: e.Message.ID} + if err := s.HandleRemoveEmail(ctx, remove); !errors.Is(err, ErrSyncArrivalPending) { + t.Fatalf("removal overtook pending original arrival: %v", err) + } + var visible int + if err := d.QueryRow(ctx, `SELECT count(*) FROM unibox_emails WHERE id=$1`, e.Message.ID).Scan(&visible); err != nil { + t.Fatal(err) + } + if lostAck && kind == "ordinary" && visible != 1 { + t.Fatal("blocked removal deleted already-ingested mail") + } + if _, err := d.Exec(ctx, `UPDATE sync_arrival_outbox SET retry_at=now() WHERE id=$1`, e.Message.ID); err != nil { + t.Fatal(err) + } + fresh, _ := liveWarmupService(t) + fresh.ArrivalOutbox = r + fresh.InitEvents() + if err := r.DeliverArrivals(ctx, fresh.deliverArrivalEvent); err != nil { + t.Fatal(err) + } + if pending, err := r.HasPendingArrival(ctx, e.UserID, e.Message.EmailID, e.Message.ID); err != nil || pending { + t.Fatalf("arrival did not finish: %v %v", pending, err) + } + unowned := *remove + unowned.UserID = f.senderUser + if err := fresh.HandleRemoveEmail(ctx, &unowned); err != nil { + t.Fatalf("unowned removal: %v", err) + } + if kind == "ordinary" { + if _, err := fresh.UniboxRepository.GetForSync(ctx, e.UserID, e.Message.EmailID, e.Message.ID); err != nil { + t.Fatalf("unowned removal deleted owned row: %v", err) + } + } + if err := fresh.HandleRemoveEmail(ctx, remove); err != nil { + t.Fatal(err) + } + if err := r.DeliverArrivals(ctx, fresh.deliverArrivalEvent); err != nil { + t.Fatal(err) + } + if err := fresh.HandleRemoveEmail(ctx, remove); err != nil { + t.Fatalf("removal retry: %v", err) + } + // Recovery updates to a retained map must not resurrect hidden or deleted mail. + if err := fresh.HandleUpdateEmail(ctx, &models.JobEventEmailUpdate{UserID: e.UserID, EmailID: e.Message.EmailID, ID: e.Message.ID, UID: 42, Mailbox: 9, ModSeq: 100, Flags: []string{models.FlagSeen}}); err != nil { + t.Fatal(err) + } + for _, table := range []string{"unibox_emails", "unibox_pending_emails", "sync_arrival_outbox"} { + var rows int + if err := d.QueryRow(ctx, "SELECT count(*) FROM "+table+" WHERE id=$1", e.Message.ID).Scan(&rows); err != nil || rows != 0 { + t.Fatalf("%s retained or recreated mail: %d %v", table, rows, err) + } + } + known, err := r.Get(ctx, e.UserID, e.Message.EmailID, e.Message.MessageID) + if err != nil || known == nil || known.ID != e.Message.ID.String() { + t.Fatalf("original mapping lost: %+v %v", known, err) + } + if kind != "ordinary" { + var receipts int + want := 0 + if kind == "confirmed warmup" { + want = 1 + } + if err := d.QueryRow(ctx, `SELECT count(*) FROM warmup_received WHERE sender_account_id=$1 AND email_account_id=$2`, f.sender, f.partner).Scan(&receipts); err != nil || receipts != want { + t.Fatalf("warmup authority changed: %d want=%d err=%v", receipts, want, err) + } + stored, err := fresh.WarmupRepo.FindWarmupToken(ctx, token) + if err != nil || stored == nil || (stored.ConsumedAt != nil) != (want == 1) { + t.Fatalf("warmup token authority changed: %+v %v", stored, err) + } + } + }) + } + } +} + +type failSyncArrivalCreate struct { + repository.UniboxRepository + fail bool +} + +func (r *failSyncArrivalCreate) CreateEntry(ctx context.Context, user uuid.UUID, message *models.EmailMessageStoreData) error { + if r.fail { + r.fail = false + return errors.New("transient arrival write failure") + } + return r.UniboxRepository.CreateEntry(ctx, user, message) +} + +func TestLiveSyncTruthPendingArrivalRetainsProviderUpdatesAcrossReload(t *testing.T) { + for _, kind := range []string{"seen", "folder", "full update"} { + t.Run(kind, func(t *testing.T) { + s, d := liveWarmupService(t) + f := newWarmupFixture(t, d) + ctx := t.Context() + key, err := kms.NewLocal(make([]byte, 32)) + if err != nil { + t.Fatal(err) + } + cipherService := cipher.NewService(key, nil, encryptedkeys.NewPostgres(d)) + r := repository.NewDurableEmailMessageMapRepository(d, cipherService) + s.ArrivalOutbox = r + s.UniboxRepository = &failSyncArrivalCreate{UniboxRepository: s.UniboxRepository, fail: true} + e := f.arrival("<"+uuid.NewString()+"@test.local>", nil) + e.Message.Subject = "ordinary customer question" + mapping := repository.EmailMessageData{UserID: e.UserID.String(), EmailID: e.Message.EmailID.String(), ID: e.Message.ID.String(), MessageID: e.Message.MessageID} + if err := r.AdmitArrival(ctx, mapping, &repository.PendingArrival{Arrival: e}); err != nil { + t.Fatal(err) + } + s.InitEvents() + if err := r.DeliverArrivals(ctx, s.deliverArrivalEvent); err == nil { + t.Fatal("expected transient arrival write failure") + } + apply := func(service *JobsService) error { + switch kind { + case "seen": + return service.HandleFlagsAdd(ctx, &models.JobEventFlags{UserID: e.UserID, EmailID: e.Message.EmailID, ID: e.Message.ID, Flags: []string{models.FlagSeen}}) + case "folder": + return service.HandleFolderUpdate(ctx, &models.JobEventFolderUpdate{UserID: e.UserID, EmailID: e.Message.EmailID, ID: e.Message.ID, Folder: models.FolderArchive}) + default: + return service.HandleUpdateEmail(ctx, &models.JobEventEmailUpdate{UserID: e.UserID, EmailID: e.Message.EmailID, ID: e.Message.ID, Flags: []string{models.FlagSeen}, Folder: models.FolderArchive, FolderPath: "Archive", UID: 42, Mailbox: 7}) + } + } + if err := apply(s); !errors.Is(err, ErrSyncArrivalPending) { + t.Fatalf("update was acknowledged ahead of failed arrival: %v", err) + } + if _, err := d.Exec(ctx, `UPDATE sync_arrival_outbox SET retry_at=now() WHERE id=$1`, e.Message.ID); err != nil { + t.Fatal(err) + } + fresh, _ := liveWarmupService(t) + fresh.ArrivalOutbox = repository.NewDurableEmailMessageMapRepository(d, cipherService) + fresh.InitEvents() + if err := fresh.ArrivalOutbox.DeliverArrivals(ctx, fresh.deliverArrivalEvent); err != nil { + t.Fatal(err) + } + if err := apply(fresh); err != nil { + t.Fatal(err) + } + message, err := fresh.UniboxRepository.GetForSync(ctx, e.UserID, e.Message.EmailID, e.Message.ID) + if err != nil { + t.Fatal(err) + } + if kind != "folder" && !message.Seen { + t.Fatal("provider Seen update lost after original-ID arrival replay") + } + if kind != "seen" && message.Folder != models.FolderArchive { + t.Fatalf("provider folder update lost: %q", message.Folder) + } + if kind == "full update" && (message.UID != 42 || message.Mailbox != 7 || message.FolderPath != "Archive") { + t.Fatalf("provider handles lost: %+v", message) + } + if _, err := d.Exec(ctx, `DELETE FROM email_accounts WHERE id=$1`, e.Message.EmailID); err != nil { + t.Fatal(err) + } + if err := apply(fresh); err != nil { + t.Fatalf("deleted mailbox update retried forever: %v", err) + } + }) + } +} + +func TestLiveSyncTruthBackgroundReadsKeepUnreadAndStayOwnerScoped(t *testing.T) { + s, d := liveWarmupService(t) + f := newWarmupFixture(t, d) + ctx := t.Context() + e := f.arrival("<"+uuid.NewString()+"@test.local>", nil) + if err := s.UniboxRepository.CreateEntry(ctx, e.UserID, e.Message); err != nil { + t.Fatal(err) + } + assertUnread := func() { + t.Helper() + var seen bool + if err := d.QueryRow(ctx, `SELECT seen FROM unibox_emails WHERE id=$1`, e.Message.ID).Scan(&seen); err != nil || seen { + t.Fatalf("background work marked unread mail seen: %v %v", seen, err) + } + } + for _, owner := range []uuid.UUID{f.senderUser, uuid.New()} { + if _, err := s.UniboxRepository.GetForSync(ctx, owner, e.Message.EmailID, e.Message.ID); !errors.Is(err, repository.ErrEmailNotFound) { + t.Fatalf("unowned sync lookup returned a row: %v", err) + } + } + for _, mailbox := range []uuid.UUID{f.sender, uuid.New(), uuid.Nil} { + if _, err := s.UniboxRepository.GetForSync(ctx, e.UserID, mailbox, e.Message.ID); !errors.Is(err, repository.ErrEmailNotFound) { + t.Fatalf("wrong-mailbox sync lookup returned a row: %v", err) + } + } + assertUnread() + if err := s.HandleFlagsAdd(ctx, &models.JobEventFlags{UserID: e.UserID, EmailID: e.Message.EmailID, ID: e.Message.ID, Flags: []string{models.FlagFlagged}}); err != nil { + t.Fatal(err) + } + assertUnread() + for _, folder := range []string{models.FolderInbox, models.FolderArchive, models.FolderArchive} { + if err := s.HandleFolderUpdate(ctx, &models.JobEventFolderUpdate{UserID: e.UserID, EmailID: e.Message.EmailID, ID: e.Message.ID, Folder: folder}); err != nil { + t.Fatal(err) + } + assertUnread() + } + if err := s.HandleFlagsAdd(ctx, &models.JobEventFlags{UserID: e.UserID, EmailID: e.Message.EmailID, ID: e.Message.ID, Flags: []string{models.FlagSeen}}); err != nil { + t.Fatal(err) + } + message, err := s.UniboxRepository.GetForSync(ctx, e.UserID, e.Message.EmailID, e.Message.ID) + if err != nil || message == nil || !message.Seen { + t.Fatalf("real provider Seen flag did not mark read: %v", err) + } + if err := s.UniboxRepository.MarkSeen(ctx, e.UserID, e.Message.ID, false); err != nil { + t.Fatal(err) + } + assertUnread() + message, err = s.UniboxRepository.GetByID(ctx, e.UserID, e.Message.ID) + if err != nil || message == nil || !message.Seen { + t.Fatalf("explicit user view no longer marks read: %v", err) + } +} + +func TestLiveSyncTruthLegacyArrivalHasNoPendingGuard(t *testing.T) { + s, d := liveWarmupService(t) + f := newWarmupFixture(t, d) + ctx := t.Context() + key, err := kms.NewLocal(make([]byte, 32)) + if err != nil { + t.Fatal(err) + } + r := repository.NewDurableEmailMessageMapRepository(d, cipher.NewService(key, nil, encryptedkeys.NewPostgres(d))) + s.ArrivalOutbox = r + s.UniboxRepository = &failSyncArrivalCreate{UniboxRepository: s.UniboxRepository, fail: true} + e := f.arrival("<"+uuid.NewString()+"@test.local>", nil) + e.Message.Subject = "ordinary legacy customer question" + // Legacy workers admit only the map, then publish NEW_EMAIL separately. + if err := r.Add(ctx, repository.EmailMessageData{UserID: e.UserID.String(), EmailID: e.Message.EmailID.String(), ID: e.Message.ID.String(), MessageID: e.Message.MessageID}); err != nil { + t.Fatal(err) + } + if err := s.HandleNewEmail(ctx, e); err == nil { + t.Fatal("expected legacy arrival creation failure") + } + if pending, err := r.HasPendingArrival(ctx, e.UserID, e.Message.EmailID, e.Message.ID); err != nil || pending { + t.Fatalf("legacy map unexpectedly acquired a durable arrival marker: %v %v", pending, err) + } + if err := s.HandleFlagsAdd(ctx, &models.JobEventFlags{UserID: e.UserID, EmailID: e.Message.EmailID, ID: e.Message.ID, Flags: []string{models.FlagSeen}}); err != nil { + t.Fatalf("legacy map alone inferred a pending arrival: %v", err) + } + if err := s.HandleFolderUpdate(ctx, &models.JobEventFolderUpdate{UserID: e.UserID, EmailID: e.Message.EmailID, ID: e.Message.ID, Folder: models.FolderArchive}); err != nil { + t.Fatalf("legacy map alone inferred a pending arrival: %v", err) + } + if err := s.HandleNewEmail(ctx, e); err != nil { + t.Fatal(err) + } + message, err := s.UniboxRepository.GetForSync(ctx, e.UserID, e.Message.EmailID, e.Message.ID) + if err != nil { + t.Fatal(err) + } + if message.Seen || message.Folder != models.FolderInbox { + t.Fatalf("legacy update ordering must not be reported as protected: %+v", message) + } +} diff --git a/internal/app/consumer/sync_truth_test.go b/internal/app/consumer/sync_truth_test.go new file mode 100644 index 000000000..148147e30 --- /dev/null +++ b/internal/app/consumer/sync_truth_test.go @@ -0,0 +1,253 @@ +package jobs + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/infrastructure/pubsub" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +func TestSyncTruthProgressRelaysDoNotResolveTransportWarnings(t *testing.T) { + previous := time.Now().Add(-time.Hour) + fresh := previous.Add(time.Minute) + for _, tc := range []struct { + name string + stamp *time.Time + deferred int + recovery *models.GoogleHistoryRecovery + resolved int + }{ + {"heartbeat", &previous, 0, nil, 0}, + {"older redelivery", nil, 0, nil, 0}, + {"deferred", &fresh, 5, nil, 0}, + {"recovering", &fresh, 0, &models.GoogleHistoryRecovery{HistoryID: 100}, 0}, + {"successful catch-up", &fresh, 0, nil, 1}, + } { + t.Run(tc.name, func(t *testing.T) { + errRepo := &stubErrorRepo{} + repo := &stubSyncStateRepo{saved: &models.SyncState{LastSyncedAt: &previous}} + s := &JobsService{EmailSyncStateRepository: repo, EmailAccountErrorRepository: errRepo} + if err := s.HandleSyncState(t.Context(), &models.JobEventSyncState{UserID: uuid.New(), EmailID: uuid.New(), State: models.SyncState{BackfillStatus: models.SyncBackfillComplete, LastSyncedAt: tc.stamp, Deferred: tc.deferred, BackfillCursor: models.SyncCursor{GoogleRecovery: tc.recovery}}}); err != nil { + t.Fatal(err) + } + if repo.put == nil || errRepo.calls != tc.resolved { + t.Fatalf("state publication/recovery outcome: saved=%v resolved=%d", repo.put != nil, errRepo.calls) + } + }) + } +} + +type removeSyncInbox struct { + repository.UniboxRepository + deletes int + err error +} + +func (r *removeSyncInbox) Delete(context.Context, uuid.UUID, uuid.UUID) error { + r.deletes++ + return r.err +} + +type removeSyncBus struct{ deleted int } + +type removeSyncWarmup struct { + repository.WarmupRepository + lookups int +} + +func (r *removeSyncWarmup) GetWarmupReceived(_ context.Context, mailbox, id uuid.UUID) (*repository.WarmupReceived, error) { + r.lookups++ + return &repository.WarmupReceived{EmailAccountID: mailbox, InternalID: id, MessageID: "", CreatedAt: time.Now()}, nil +} + +func (b *removeSyncBus) Publish(_ context.Context, _ string, data any, _ map[string]string) error { + if event, ok := data.(*pubsub.EmailInboxEvent); ok && event.EventType == pubsub.EventEmailDeleted { + b.deleted++ + } + return nil +} + +func TestSyncTruthRemovalWaitsForScopedPendingArrivals(t *testing.T) { + user, mailbox, id := uuid.New(), uuid.New(), uuid.New() + lookupErr := errors.New("pending marker unavailable") + for _, tc := range []struct { + name string + owner, account, message uuid.UUID + pending bool + err, want error + }{ + {"pending", user, mailbox, id, true, nil, ErrSyncArrivalPending}, + {"marker error", user, mailbox, id, false, lookupErr, lookupErr}, + {"finished or legacy", user, mailbox, id, false, nil, nil}, + {"other user", uuid.New(), mailbox, id, true, nil, nil}, + {"other mailbox", user, uuid.New(), id, true, nil, nil}, + {"other message", user, mailbox, uuid.New(), true, nil, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + inbox, bus := &removeSyncInbox{}, &removeSyncBus{} + s := &JobsService{UniboxRepository: inbox, ArrivalOutbox: &syncTruthOutbox{user: user, mailbox: mailbox, id: id, pending: tc.pending, err: tc.err}, EmailRepository: newEmailAccountRepo{}, StreamingPublisher: pubsub.NewStreamingPublisher(bus)} + err := s.HandleRemoveEmail(t.Context(), &models.JobEventRemoveEmail{UserID: tc.owner, EmailID: tc.account, ID: tc.message}) + if !errors.Is(err, tc.want) { + t.Fatalf("removal error=%v want=%v", err, tc.want) + } + want := 1 + if tc.want != nil { + want = 0 + } + if inbox.deletes != want || bus.deleted != want { + t.Fatalf("deletes=%d notifications=%d want=%d", inbox.deletes, bus.deleted, want) + } + }) + } +} + +func TestSyncTruthRemovalDeleteFailureRetriesBeforeNotification(t *testing.T) { + failure := errors.New("unibox delete unavailable") + inbox, bus := &removeSyncInbox{err: failure}, &removeSyncBus{} + warmup := &removeSyncWarmup{} + s := &JobsService{UniboxRepository: inbox, WarmupRepo: warmup, EmailRepository: newEmailAccountRepo{}, StreamingPublisher: pubsub.NewStreamingPublisher(bus)} + e := &models.JobEventRemoveEmail{UserID: uuid.New(), EmailID: uuid.New(), ID: uuid.New()} + if err := s.HandleRemoveEmail(t.Context(), e); !errors.Is(err, failure) { + t.Fatalf("delete failure acknowledged: %v", err) + } + if bus.deleted != 0 { + t.Fatal("failed deletion published success") + } + inbox.err = nil + if err := s.HandleRemoveEmail(t.Context(), e); err != nil { + t.Fatal(err) + } + if inbox.deletes != 2 || bus.deleted != 1 || warmup.lookups != 2 { + t.Fatalf("deletes=%d notifications=%d", inbox.deletes, bus.deleted) + } +} + +type syncTruthLookup struct { + repository.UniboxRepository + message *models.EmailMessageStoreData + reads int + lookups int + appearOn int +} + +func (r *syncTruthLookup) GetByID(context.Context, uuid.UUID, uuid.UUID) (*models.EmailMessageStoreData, error) { + r.reads++ + if r.message == nil { + return nil, repository.ErrEmailNotFound + } + r.message.Seen = true + return r.message, nil +} + +func (r *syncTruthLookup) GetForSync(context.Context, uuid.UUID, uuid.UUID, uuid.UUID) (*models.EmailMessageStoreData, error) { + r.lookups++ + if r.message == nil || (r.appearOn > 0 && r.lookups < r.appearOn) { + return nil, repository.ErrEmailNotFound + } + return r.message, nil +} + +func (*syncTruthLookup) UpdatePendingEmail(context.Context, uuid.UUID, uuid.UUID, func(*models.EmailMessageStoreData)) (bool, error) { + return false, nil +} + +func (r *syncTruthLookup) UpdateEntry(_ context.Context, _, _, _ uuid.UUID, update *repository.UpdateUniboxEntry) error { + if update.Seen != nil { + r.message.Seen = *update.Seen + } + return nil +} + +type syncTruthOutbox struct { + repository.ArrivalOutbox + user, mailbox, id uuid.UUID + pending bool + err error +} + +func (r *syncTruthOutbox) HasPendingArrival(_ context.Context, user, mailbox, id uuid.UUID) (bool, error) { + if user != r.user || mailbox != r.mailbox || id != r.id { + return false, nil + } + return r.pending, r.err +} + +func TestSyncTruthBackgroundUpdatesNeverViewUnreadMail(t *testing.T) { + for _, kind := range []string{"flagged", "folder", "same folder"} { + t.Run(kind, func(t *testing.T) { + r := &syncTruthLookup{message: &models.EmailMessageStoreData{Seen: false, Folder: models.FolderInbox, ProviderFolder: models.FolderInbox}} + s := &JobsService{UniboxRepository: r, EmailRepository: warmupInboxEmailRepo{}} + user, mailbox, id := uuid.New(), uuid.New(), uuid.New() + var err error + if kind == "flagged" { + err = s.HandleFlagsAdd(t.Context(), &models.JobEventFlags{UserID: user, EmailID: mailbox, ID: id, Flags: []string{models.FlagFlagged}}) + } else { + folder := models.FolderInbox + if kind == "folder" { + folder = models.FolderArchive + } + err = s.HandleFolderUpdate(t.Context(), &models.JobEventFolderUpdate{UserID: user, EmailID: mailbox, ID: id, Folder: folder}) + } + if err != nil || r.message.Seen || r.reads != 0 { + t.Fatalf("background update treated as view: seen=%v reads=%d err=%v", r.message.Seen, r.reads, err) + } + }) + } +} + +func TestSyncTruthPendingArrivalUpdatesRetryOnlyForOwnedUndeliveredMail(t *testing.T) { + user, mailbox, id := uuid.New(), uuid.New(), uuid.New() + for _, kind := range []string{"flags add", "flags remove", "folder", "full update"} { + t.Run(kind, func(t *testing.T) { + r := &syncTruthLookup{} + outbox := &syncTruthOutbox{user: user, mailbox: mailbox, id: id, pending: true} + s := &JobsService{UniboxRepository: r, ArrivalOutbox: outbox} + apply := func(owner, box uuid.UUID) error { + switch kind { + case "flags add": + return s.HandleFlagsAdd(t.Context(), &models.JobEventFlags{UserID: owner, EmailID: box, ID: id, Flags: []string{models.FlagSeen}}) + case "flags remove": + return s.HandleFlagsRemove(t.Context(), &models.JobEventFlags{UserID: owner, EmailID: box, ID: id, Flags: []string{models.FlagSeen}}) + case "folder": + return s.HandleFolderUpdate(t.Context(), &models.JobEventFolderUpdate{UserID: owner, EmailID: box, ID: id, Folder: models.FolderArchive}) + default: + return s.HandleUpdateEmail(t.Context(), &models.JobEventEmailUpdate{UserID: owner, EmailID: box, ID: id, Flags: []string{models.FlagSeen}}) + } + } + if err := apply(user, mailbox); !errors.Is(err, ErrSyncArrivalPending) { + t.Fatalf("pending arrival was acknowledged: %v", err) + } + if err := apply(uuid.New(), mailbox); err != nil { + t.Fatalf("unowned update retried: %v", err) + } + if err := apply(user, uuid.New()); err != nil { + t.Fatalf("other-mailbox update retried: %v", err) + } + outbox.pending = false + if err := apply(user, mailbox); err != nil { + t.Fatalf("deliberately hidden/deleted mail retried: %v", err) + } + outbox.err = errors.New("marker unavailable") + if err := apply(user, mailbox); !errors.Is(err, outbox.err) { + t.Fatalf("marker failure swallowed: %v", err) + } + }) + } +} + +func TestSyncTruthArrivalFinishingDuringLookupStillGetsUpdate(t *testing.T) { + user, mailbox, id := uuid.New(), uuid.New(), uuid.New() + r := &syncTruthLookup{message: &models.EmailMessageStoreData{EmailID: mailbox}, appearOn: 3} + s := &JobsService{UniboxRepository: r, EmailRepository: warmupInboxEmailRepo{}, ArrivalOutbox: &syncTruthOutbox{user: user, mailbox: mailbox, id: id}} + if err := s.HandleFlagsAdd(t.Context(), &models.JobEventFlags{UserID: user, EmailID: mailbox, ID: id, Flags: []string{models.FlagSeen}}); err != nil { + t.Fatal(err) + } + if !r.message.Seen { + t.Fatal("arrival committed between reads lost the later Seen update") + } +} diff --git a/internal/app/consumer/unibox_seen_sync_test.go b/internal/app/consumer/unibox_seen_sync_test.go index 070ffe858..a9ebc56cf 100644 --- a/internal/app/consumer/unibox_seen_sync_test.go +++ b/internal/app/consumer/unibox_seen_sync_test.go @@ -21,6 +21,10 @@ func (r *seenSyncRepo) GetByID(context.Context, uuid.UUID, uuid.UUID) (*models.E return r.stored, nil } +func (r *seenSyncRepo) GetForSync(context.Context, uuid.UUID, uuid.UUID, uuid.UUID) (*models.EmailMessageStoreData, error) { + return r.stored, nil +} + func (r *seenSyncRepo) UpdateEntry(_ context.Context, _, _, _ uuid.UUID, e *repository.UpdateUniboxEntry) error { r.updates = append(r.updates, *e) return nil diff --git a/internal/app/email/graph_checkpoint_test.go b/internal/app/email/graph_checkpoint_test.go new file mode 100644 index 000000000..c53cf882b --- /dev/null +++ b/internal/app/email/graph_checkpoint_test.go @@ -0,0 +1,66 @@ +package email + +import ( + "context" + "errors" + "testing" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +type graphCheckpointRepo struct { + links map[string]string + err error +} + +type graphLoaderState struct { + repository.EmailSyncStateRepository + state *models.SyncState + err error +} + +func (s *graphLoaderState) Get(context.Context, uuid.UUID) (*models.SyncState, error) { + return s.state, s.err +} + +func (r *graphCheckpointRepo) Get(context.Context, uuid.UUID, uuid.UUID) (map[string]string, error) { + return r.links, r.err +} +func (r *graphCheckpointRepo) Put(context.Context, uuid.UUID, uuid.UUID, string, string) error { + return nil +} + +func TestGraphCheckpointLoadFailureDoesNotPublishCursorlessWorker(t *testing.T) { + org, worker := uuid.New(), uuid.New() + acc := &models.Email{ID: uuid.New(), UserID: uuid.NewString(), Email: "box@outlook.test", Provider: string(models.InboxProviderOutlook), Status: "active", OrganizationID: &org, WorkerID: &worker} + repo := &stubReauthRepo{account: acc} + lookupErr := errors.New("checkpoint read failed") + checkpoints := &graphCheckpointRepo{links: map[string]string{"inbox": "saved-delta"}, err: lookupErr} + pub := &countingPublisher{} + s := &emailService{emailRepository: repo, workerAssignment: &countingAssignment{}, publisher: pub, graphDelta: checkpoints} + state := &graphLoaderState{state: &models.SyncState{BackfillStatus: models.SyncBackfillComplete}} + s.syncState = state + if err := s.LoadAccountOntoWorker(t.Context(), acc.ID); !errors.Is(err, lookupErr) || pub.added != 0 { + t.Fatalf("err=%v published=%d", err, pub.added) + } + checkpoints.err = nil + state.err = lookupErr + if err := s.LoadAccountOntoWorker(t.Context(), acc.ID); !errors.Is(err, lookupErr) || pub.added != 0 { + t.Fatalf("recovery state failure published unsafe worker: err=%v published=%d", err, pub.added) + } + state.err = nil + payload, err := s.buildAddWorkerEmail(t.Context(), acc) + if err != nil || payload.Graph.DeltaLinks["inbox"] != "saved-delta" || payload.Sync.State.BackfillStatus != models.SyncBackfillComplete { + t.Fatalf("reload did not recover saved checkpoint: %v %v", payload, err) + } + if err := s.LoadAccountOntoWorker(t.Context(), acc.ID); err != nil || pub.added != 1 { + t.Fatalf("retry err=%v published=%d", err, pub.added) + } + checkpoints.links = nil + payload, err = s.buildAddWorkerEmail(t.Context(), acc) + if err != nil || len(payload.Graph.DeltaLinks) != 0 { + t.Fatalf("genuinely missing checkpoint: %v %v", payload, err) + } +} diff --git a/internal/app/email/loader.go b/internal/app/email/loader.go index f58e795f7..b66abdb50 100644 --- a/internal/app/email/loader.go +++ b/internal/app/email/loader.go @@ -19,7 +19,7 @@ import ( // WireGraphDelta attaches the Graph delta-cursor repository so the reconciler can // seed a Graph mailbox's saved per-folder cursors when (re)loading it. Optional; -// when unset, Graph mailboxes prime from empty on load. +// when unset, Graph mailboxes recover from empty on load. func (s *emailService) WireGraphDelta(repo repository.EmailGraphDeltaRepository) { s.graphDelta = repo } @@ -352,6 +352,13 @@ func (s *emailService) buildAddWorkerEmail(ctx context.Context, acc *models.Emai // Only SMTP/IMAP acts on this; Gmail and Graph file their own copy. SaveToSent: &saveToSent, } + if provider == models.InboxProviderOutlook { + links, err := s.deltaLinksFor(ctx, userID, acc.ID) + if err != nil { + return nil, err + } + out.Graph = &models.AddWorkerEmailGraphData{DeltaLinks: links} + } // A mailbox under an administrator's grant has no stored credential // either; the worker draws tokens the control plane mints per use. @@ -368,7 +375,7 @@ func (s *emailService) buildAddWorkerEmail(ctx context.Context, acc *models.Emai case models.InboxProviderGoogle: out.Google = &models.AddWorkerEmailGoogleData{LastHistoryID: s.lastHistoryFor(ctx, userID, acc.ID, acc.LastID)} case models.InboxProviderOutlook: - out.Graph = &models.AddWorkerEmailGraphData{DeltaLinks: s.deltaLinksFor(ctx, userID, acc.ID), User: d.Subject} + out.Graph.User = d.Subject default: return nil, nil } @@ -383,7 +390,7 @@ func (s *emailService) buildAddWorkerEmail(ctx context.Context, acc *models.Emai case models.InboxProviderGoogle: out.Google = &models.AddWorkerEmailGoogleData{LastHistoryID: s.lastHistoryFor(ctx, userID, acc.ID, acc.LastID)} case models.InboxProviderOutlook: - out.Graph = &models.AddWorkerEmailGraphData{DeltaLinks: s.deltaLinksFor(ctx, userID, acc.ID)} + // The checkpoint was loaded before selecting the token owner. default: return nil, nil } @@ -406,10 +413,7 @@ func (s *emailService) buildAddWorkerEmail(ctx context.Context, acc *models.Emai if cerr != nil { return nil, cerr } - out.Graph = &models.AddWorkerEmailGraphData{ - Token: oauthToken(creds), - DeltaLinks: s.deltaLinksFor(ctx, userID, acc.ID), - } + out.Graph.Token = oauthToken(creds) case models.InboxProviderSMTPIMAP: creds, cerr := s.emailRepository.GetSMTPCredentials(ctx, acc.ID) if cerr != nil { @@ -498,11 +502,11 @@ func (s *emailService) syncDataFor(ctx context.Context, emailID uuid.UUID) (*mod } } if s.syncState != nil { - if saved, err := s.syncState.Get(ctx, emailID); err == nil { - data.State = saved - } else { - log.Warn().Err(err).Str("email_id", emailID.String()).Msg("sync state lookup failed; worker starts fresh") + saved, err := s.syncState.Get(ctx, emailID) + if err != nil { + return nil, fmt.Errorf("sync state lookup: %w", err) } + data.State = saved } return data, nil } @@ -522,15 +526,15 @@ func (s *emailService) mailboxesFor(ctx context.Context, userID, emailID uuid.UU return saved } -func (s *emailService) deltaLinksFor(ctx context.Context, userID, emailID uuid.UUID) map[string]string { +func (s *emailService) deltaLinksFor(ctx context.Context, userID, emailID uuid.UUID) (map[string]string, error) { if s.graphDelta == nil { - return nil + return nil, nil } links, err := s.graphDelta.Get(ctx, userID, emailID) if err != nil { - return nil + return nil, fmt.Errorf("graph checkpoint lookup: %w", err) } - return links + return links, nil } func oauthToken(c *repository.OAuthCredentials) *oauth2.Token { diff --git a/internal/app/orgtransfer/spec.go b/internal/app/orgtransfer/spec.go index 398f61fc2..7834f62e4 100644 --- a/internal/app/orgtransfer/spec.go +++ b/internal/app/orgtransfer/spec.go @@ -745,7 +745,8 @@ var Tables = []Table{ }, { Name: "email_message_map", Group: models.OrgDataGroupInbox, - Scope: `email_id IN ` + orgMailboxes, + Scope: `email_id IN ` + orgMailboxes + ` AND NOT EXISTS (SELECT 1 FROM sync_arrival_outbox o WHERE o.user_id=email_message_map.user_id AND o.email_id=email_message_map.email_id AND o.message_id=email_message_map.message_id AND o.stage=0)`, + Owner: `email_id IN ` + orgMailboxes, Note: "Maps provider message ids to internal ones, so replies still thread after the move.", }, { @@ -994,6 +995,7 @@ var Tables = []Table{ // with the reason. Kept as data so the docs page and the coverage test both // read from one list instead of restating it. var ExcludedTables = map[string]string{ + "sync_arrival_outbox": "Encrypted, org-key-owned arrival delivery queue bound to this instance's provider handles and body storage. Pending maps are excluded too so destination sync retries them; no queue secrets or blob references travel. Full instance backups retain queue, org keys and bodies together.", "send_result_effects": "Source-instance reconciliation delivery queue. Pending webhook and notification effects must drain on the source; imports must not replay customer notifications. Full instance backups retain the durable outbox.", "warmup_received": "Raw receiving-mailbox observations and provider thread handles are source-instance evidence, not portable send authority. Aggregated warmup placement history travels; imported threads cannot continue without new verified observations.", "diagnostic_auth_verifications": "Bounded DKIM observations and grants bind source-instance workers, provider messages and cross-workspace diagnostic parents. They cannot authorize imported mail; destination verification starts unknown. Full instance database backups retain them.", diff --git a/internal/app/orgtransfer/sync_arrival_scope_live_test.go b/internal/app/orgtransfer/sync_arrival_scope_live_test.go new file mode 100644 index 000000000..02823c07e --- /dev/null +++ b/internal/app/orgtransfer/sync_arrival_scope_live_test.go @@ -0,0 +1,63 @@ +package orgtransfer + +import ( + "testing" + + "github.com/google/uuid" +) + +func TestLiveSyncArrivalQueueAndPendingMapsStayInstanceLocal(t *testing.T) { + pool := specLivePool(t) + f := newTenantFixture(t, pool) + ctx := t.Context() + mailbox := uuid.New() + if _, err := pool.Exec(ctx, `INSERT INTO email_accounts(id,user_id,organization_id,email,name,signature_plain,signature_html,provider) VALUES($1,$2,$3,$4,'Arrival','','','smtp_imap')`, mailbox, f.owner, f.org, mailbox.String()+"@test.local"); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM email_accounts WHERE id=$1`, mailbox) }) + for i, key := range []string{"legacy", "pending", "handled"} { + id := uuid.New() + if _, err := pool.Exec(ctx, `INSERT INTO email_message_map(user_id,email_id,message_id,id) VALUES($1,$2,$3,$4)`, f.owner, mailbox, key, id); err != nil { + t.Fatal(err) + } + if i > 0 { + if _, err := pool.Exec(ctx, `INSERT INTO sync_arrival_outbox(user_id,email_id,organization_id,message_id,id,payload,stage) VALUES($1,$2,$3,$4,$5,'fixture',$6)`, f.owner, mailbox, f.org, key, id, i-1); err != nil { + t.Fatal(err) + } + } + } + if ExcludedTables["sync_arrival_outbox"] == "" { + t.Fatal("queue is not explicitly excluded") + } + var scope string + for _, spec := range Tables { + if spec.Name == "sync_arrival_outbox" { + t.Fatal("queue secrets or body references transferred") + } + if spec.Name == "email_message_map" { + scope = spec.Scope + } + } + if scope == "" { + t.Fatal("map scope missing") + } + rows, err := pool.Query(ctx, `SELECT message_id FROM email_message_map WHERE `+scope+` ORDER BY message_id`, f.org) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var keys []string + for rows.Next() { + var key string + if err := rows.Scan(&key); err != nil { + t.Fatal(err) + } + keys = append(keys, key) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + if len(keys) != 2 || keys[0] != "handled" || keys[1] != "legacy" { + t.Fatalf("pending maps suppressed destination resync: %v", keys) + } +} diff --git a/internal/app/worker/event_add_email.go b/internal/app/worker/event_add_email.go index 55a91d0ac..4051bb4b1 100644 --- a/internal/app/worker/event_add_email.go +++ b/internal/app/worker/event_add_email.go @@ -18,6 +18,7 @@ import ( type mailboxLoad struct { removed atomic.Bool done chan struct{} + prev *mailboxLoad } // HandleAddEmail loads a mailbox off the bus loop. Loading dials the mail @@ -28,13 +29,13 @@ func (w *WorkerService) HandleAddEmail(ctx context.Context, e *models.AddWorkerE return nil } - if w.mailManager.Has(e.ID) { + if _, loading := w.loads.Load(e.ID); !loading { // Already loaded: keep the handler idempotent, but take a changed sync // budget so an operator's settings change reaches this mailbox. - if mail := w.mailManager.Get(e.ID); mail != nil { + if mail := w.mailManager.Get(e.ID); mail != nil && mail.SameExecution(e) { mail.ApplySyncPolicy(e.Sync) + return nil } - return nil } load := &mailboxLoad{done: make(chan struct{})} @@ -45,10 +46,8 @@ func (w *WorkerService) HandleAddEmail(ctx context.Context, e *models.AddWorkerE break } old := v.(*mailboxLoad) - if !old.removed.Load() { - return nil - } - // A removal is pending on the load in flight: this add runs after it rather than being dropped. + // Queue after in-flight loads; AddWMail keeps unchanged adds idempotent. + load.prev = old if w.loads.CompareAndSwap(e.ID, old, load) { prev = old break @@ -73,10 +72,18 @@ func (w *WorkerService) HandleAddEmail(ctx context.Context, e *models.AddWorkerE // loadedMailbox returns a mailbox, waiting on a load of it still in flight so a // command queued behind its ADD_EMAIL finds it as it did when loads were inline. func (w *WorkerService) loadedMailbox(ctx context.Context, id uuid.UUID) (*wmail.WMail, bool) { - if v, ok := w.loads.Load(id); ok { + for { + v, ok := w.loads.Load(id) + if !ok { + break + } select { case <-v.(*mailboxLoad).done: case <-ctx.Done(): + return nil, false + } + if current, exists := w.loads.Load(id); !exists || current == v { + break } } mail := w.mailManager.Get(id) @@ -85,6 +92,10 @@ func (w *WorkerService) loadedMailbox(ctx context.Context, id uuid.UUID) (*wmail // loadMailbox connects one mailbox and starts its sync, or reports why it could not. func (w *WorkerService) loadMailbox(ctx context.Context, e *models.AddWorkerEmail, load *mailboxLoad) { + if load.removed.Load() { + return + } + previous := w.mailManager.Get(e.ID) if err := w.mailManager.AddWMail(ctx, e); err != nil { log.Error().Err(err).Str("email_id", e.ID.String()).Msg("failed to add email account to worker") w.reportLoadFailure(e, err) @@ -95,8 +106,10 @@ func (w *WorkerService) loadMailbox(ctx context.Context, e *models.AddWorkerEmai return } if load.removed.Load() { - mail.Discard() - w.mailManager.Terminate(e.ID) + w.mailManager.TerminateMailbox(e.ID, mail) + return + } + if previous == mail { return } @@ -146,8 +159,5 @@ func (w *WorkerService) reportLoadFailure(e *models.AddWorkerEmail, err error) { // dropMailbox removes a loaded mailbox and stops its work. func (w *WorkerService) dropMailbox(id uuid.UUID, mail *wmail.WMail) { - if mail.Cancel != nil { - mail.Cancel() - } - w.mailManager.Terminate(id) + w.mailManager.TerminateMailbox(id, mail) } diff --git a/internal/app/worker/event_mailbox_identity.go b/internal/app/worker/event_mailbox_identity.go index 80562aacb..68c3013df 100644 --- a/internal/app/worker/event_mailbox_identity.go +++ b/internal/app/worker/event_mailbox_identity.go @@ -58,6 +58,13 @@ func (w *WorkerService) HandleMailboxIdentity(ctx context.Context, data models.E if mail.GoogleData == nil || mail.GoogleData.Client == nil { return reply(models.MailboxIdentityResult{Error: "provider has no send-as list"}) } + done, ok := mail.BeginExecution() + if !ok { + return reply(models.MailboxIdentityResult{Error: "mailbox is not loaded on this worker"}) + } + defer done() + ctx, cancel = mail.ExecutionContext(ctx) + defer cancel() rows, err := mail.GoogleData.Client.ListSendAs(ctx) if err != nil { diff --git a/internal/app/worker/event_message_folder.go b/internal/app/worker/event_message_folder.go index ed882950e..6a0624640 100644 --- a/internal/app/worker/event_message_folder.go +++ b/internal/app/worker/event_message_folder.go @@ -29,6 +29,13 @@ func (w *WorkerService) HandleMessageFolder(ctx context.Context, action models.M log.Warn().Str("email_id", action.EmailID.String()).Msg("Mailbox not loaded; filing not relayed to the provider") return nil } + done, ok := mail.BeginExecution() + if !ok { + return nil + } + defer done() + ctx, cancel := mail.ExecutionContext(ctx) + defer cancel() log.Info(). Str("email_id", action.EmailID.String()). diff --git a/internal/app/worker/event_message_seen.go b/internal/app/worker/event_message_seen.go index fa14e8efd..5eb7073e4 100644 --- a/internal/app/worker/event_message_seen.go +++ b/internal/app/worker/event_message_seen.go @@ -28,6 +28,13 @@ func (w *WorkerService) HandleMessageSeen(ctx context.Context, action models.Mes log.Warn().Str("email_id", action.EmailID.String()).Msg("Mailbox not loaded; read state not relayed to the provider") return nil } + done, ok := mail.BeginExecution() + if !ok { + return nil + } + defer done() + ctx, cancel := mail.ExecutionContext(ctx) + defer cancel() log.Info(). Str("email_id", action.EmailID.String()). diff --git a/internal/app/worker/event_remove_email.go b/internal/app/worker/event_remove_email.go index 57a1bd08f..fab2f7f0d 100644 --- a/internal/app/worker/event_remove_email.go +++ b/internal/app/worker/event_remove_email.go @@ -22,7 +22,9 @@ func (w *WorkerService) HandleRemoveEmail(ctx context.Context, e *models.RemoveW // A load still dialing drops the mailbox when it finishes. if v, ok := w.loads.Load(id); ok { - v.(*mailboxLoad).removed.Store(true) + for load := v.(*mailboxLoad); load != nil; load = load.prev { + load.removed.Store(true) + } } mail := w.mailManager.Get(id) diff --git a/internal/app/worker/event_warmup_action.go b/internal/app/worker/event_warmup_action.go index 99c1c5feb..8cc7ba62f 100644 --- a/internal/app/worker/event_warmup_action.go +++ b/internal/app/worker/event_warmup_action.go @@ -52,6 +52,16 @@ func (w *WorkerService) HandleWarmupAction(ctx context.Context, action models.Wa } mail, exists := w.loadedMailbox(ctx, action.EmailID) + if exists { + if done, ok := mail.BeginExecution(); ok { + defer done() + var cancel func() + ctx, cancel = mail.ExecutionContext(ctx) + defer cancel() + } else { + exists = false + } + } var err error switch { case !exists: diff --git a/internal/app/worker/mailbox_lifecycle_test.go b/internal/app/worker/mailbox_lifecycle_test.go new file mode 100644 index 000000000..20dfcea65 --- /dev/null +++ b/internal/app/worker/mailbox_lifecycle_test.go @@ -0,0 +1,87 @@ +package worker + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/app/worker/wmail" + "github.com/warmbly/warmbly/internal/models" +) + +type closedWorkerIMAP struct { + wmail.ImapConn + closes atomic.Int32 +} + +func (c *closedWorkerIMAP) Close() error { c.closes.Add(1); return nil } + +func TestAddEmailReplacesChangedCredentialsAndRemoveClosesInstalledSession(t *testing.T) { + w := newLoadingWorker(&capturedEvents{}) + id := uuid.New() + d := &models.AddWorkerEmail{ID: id, UserID: uuid.New(), Type: models.InboxProviderSMTPIMAP, Email: "a@example.test", SmtpImap: &models.AddWorkerEmailSmtpImapData{Credentials: &models.SmtpImap{SMTP: &models.Service{Host: "smtp.example.test", Port: 587, Password: "old"}, IMAP: &models.Service{Host: "imap.example.test", Port: 993, Password: "old"}}}, Sync: &models.AddWorkerEmailSyncData{State: &models.SyncState{BackfillStatus: models.SyncBackfillComplete, BackfillSynced: 12}}} + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if err := w.HandleAddEmail(ctx, d); err != nil { + t.Fatal(err) + } + old, ok := w.loadedMailbox(ctx, id) + if !ok { + t.Fatal("initial load failed") + } + conn := &closedWorkerIMAP{} + old.SmtpImapData.ImapClient = conn + for i := 0; i < 3; i++ { + if err := w.HandleAddEmail(ctx, d); err != nil { + t.Fatal(err) + } + } + copyData := *d + copyData.Sync = &models.AddWorkerEmailSyncData{Policy: models.SyncPolicy{DailyMessages: 88}} + if err := w.HandleAddEmail(ctx, ©Data); err != nil { + t.Fatal(err) + } + if got, _ := w.loadedMailbox(ctx, id); got != old || conn.closes.Load() != 0 { + t.Fatal("policy/duplicate ADD_EMAIL retired healthy client") + } + s := *d.SmtpImap + creds := *s.Credentials + smtp := *creds.SMTP + smtp.Password = "new" + creds.SMTP = &smtp + s.Credentials = &creds + copyData.SmtpImap = &s + if err := w.HandleAddEmail(ctx, ©Data); err != nil { + t.Fatal(err) + } + current, ok := w.loadedMailbox(ctx, id) + if !ok || current == old || current.SmtpImapData.SmtpClient.Credentials.Password != "new" || conn.closes.Load() != 1 { + t.Fatal("still-loaded credentials were not replaced and retired") + } + currentConn := &closedWorkerIMAP{} + current.SmtpImapData.ImapClient = currentConn + old.Terminate() + if w.mailManager.Get(id) != current || currentConn.closes.Load() != 0 { + t.Fatal("stale old termination closed replacement") + } + rem := &models.RemoveWorkerEmail{EmailID: id.String()} + for i := 0; i < 2; i++ { + if err := w.HandleRemoveEmail(ctx, rem); err != nil { + t.Fatal(err) + } + } + if w.mailManager.Get(id) != nil || currentConn.closes.Load() != 1 { + t.Fatal("installed removal did not close exactly once") + } + if err := w.HandleAddEmail(ctx, ©Data); err != nil { + t.Fatal(err) + } + if mail, ok := w.loadedMailbox(ctx, id); !ok || mail == current { + t.Fatal("mailbox could not reload after removal") + } + if err := w.HandleRemoveEmail(ctx, rem); err != nil { + t.Fatal(err) + } +} diff --git a/internal/app/worker/mailmanager/add_email.go b/internal/app/worker/mailmanager/add_email.go index cafcd680c..ecdb0b323 100644 --- a/internal/app/worker/mailmanager/add_email.go +++ b/internal/app/worker/mailmanager/add_email.go @@ -15,6 +15,18 @@ func (m *MailManager) AddWMail( ctx context.Context, data *models.AddWorkerEmail, ) error { + dataCopy := *data + data = &dataCopy + old := m.Get(data.ID) + if old != nil { + if old.SameExecution(data) { + old.ApplySyncPolicy(data.Sync) + return nil + } + old.Stop() + old.Drain() + old.ResumeInto(data) + } // Cfg is avro-excluded from the payload, so rebuild it from the worker's // local oauth config for token refresh (no-op for smtp_imap). data.Cfg = m.cfgFor(data.Type) @@ -25,11 +37,12 @@ func (m *MailManager) AddWMail( data.TokenSource = m.tokenBroker.Source(data.ID) } + var newMail *wmail.WMail newMail, err := wmail.NewWMail( data, m.OnEvent, func() { - m.Terminate(data.ID) + m.TerminateMailbox(data.ID, newMail) }, m.cache, m.storage, @@ -47,13 +60,15 @@ func (m *MailManager) AddWMail( // Built before the lock: NewWMail dials the server, and holding the lock // through that stalled every send and lookup on this worker. m.Lock() - defer m.Unlock() - if _, loaded := m.Emails[data.ID]; loaded { + if current := m.Emails[data.ID]; current != old { + m.Unlock() newMail.Discard() return nil } + newMail.PreservePending(old) m.Emails[data.ID] = newMail newMail.ExecutorID = m.ExecutorID + m.Unlock() return nil } diff --git a/internal/app/worker/mailmanager/lifecycle_test.go b/internal/app/worker/mailmanager/lifecycle_test.go new file mode 100644 index 000000000..509dba95b --- /dev/null +++ b/internal/app/worker/mailmanager/lifecycle_test.go @@ -0,0 +1,172 @@ +package mailmanager + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/app/worker/wmail" + "github.com/warmbly/warmbly/internal/models" + "golang.org/x/oauth2" +) + +type countedClient struct { + wmail.ImapConn + closes atomic.Int32 +} + +func (c *countedClient) Close() error { c.closes.Add(1); return nil } + +func smtpPayload() *models.AddWorkerEmail { + return &models.AddWorkerEmail{ID: uuid.New(), UserID: uuid.New(), Type: models.InboxProviderSMTPIMAP, Email: "a@example.test", SmtpImap: &models.AddWorkerEmailSmtpImapData{Credentials: &models.SmtpImap{SMTP: &models.Service{Host: "smtp.example.test", Port: 587, Password: "old"}, IMAP: &models.Service{Host: "imap.example.test", Port: 993, Password: "old"}}, Mailboxes: []models.Mailbox{{Name: "Inbox", UIDValidity: 5, UIDNext: 25}}}, Sync: &models.AddWorkerEmailSyncData{State: &models.SyncState{BackfillStatus: models.SyncBackfillComplete, BackfillSynced: 7}}} +} + +func TestMailboxReplacementDrainsClosesAndPreservesCursors(t *testing.T) { + m := NewMailManager(func(models.JobEventType, string, any) error { return nil }, nil, nil, nil, nil, nil, nil) + d := smtpPayload() + if err := m.AddWMail(t.Context(), d); err != nil { + t.Fatal(err) + } + old := m.Get(d.ID) + c := &countedClient{} + old.SmtpImapData.ImapClient = c + for i := 0; i < 3; i++ { + if err := m.AddWMail(t.Context(), d); err != nil { + t.Fatal(err) + } + } + if m.Get(d.ID) != old || c.closes.Load() != 0 { + t.Fatal("unchanged add restarted client") + } + copyData := *d + data := ©Data + s := *d.SmtpImap + creds := *s.Credentials + smtp := *creds.SMTP + smtp.Password = "replacement" + creds.SMTP = &smtp + s.Credentials = &creds + data.SmtpImap = &s + done, ok := old.BeginExecution() + if !ok { + t.Fatal("live executor not admitted") + } + result := make(chan error, 1) + go func() { result <- m.AddWMail(context.Background(), data) }() + select { + case <-old.Ctx.Done(): + case <-time.After(time.Second): + t.Fatal("replacement did not cancel old executor") + } + if m.Get(d.ID) != old { + t.Fatal("new executor installed before old operation drained") + } + done() + select { + case err := <-result: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("replacement stuck") + } + next := m.Get(d.ID) + if next == nil || next == old || next.SmtpImapData.SmtpClient.Credentials.Password != "replacement" || c.closes.Load() != 1 { + t.Fatal("replacement credentials/cleanup not installed") + } + if len(next.SmtpImapData.Mailboxes) != 1 || next.SmtpImapData.Mailboxes[0].UIDNext != 25 { + t.Fatal("disabled sync lost saved cursor") + } + old.Terminate() + if m.Get(d.ID) != next { + t.Fatal("old auth callback retired replacement") + } + m.Terminate(d.ID) + m.Terminate(d.ID) + if next.Ctx.Err() == nil || c.closes.Load() != 1 { + t.Fatal("removal did not stop session idempotently") + } +} + +func TestMailboxReplacementLocalOAuthAndPolicy(t *testing.T) { + for _, provider := range []models.InboxProvider{models.InboxProviderGoogle, models.InboxProviderOutlook} { + t.Run(string(provider), func(t *testing.T) { + m := NewMailManager(func(models.JobEventType, string, any) error { return nil }, nil, nil, nil, nil, nil, nil) + d := &models.AddWorkerEmail{ID: uuid.New(), UserID: uuid.New(), Type: provider, Google: &models.AddWorkerEmailGoogleData{Token: &oauth2.Token{AccessToken: "old", RefreshToken: "old-refresh"}, LastHistoryID: 19}, Graph: &models.AddWorkerEmailGraphData{Token: &oauth2.Token{AccessToken: "old", RefreshToken: "old-refresh"}, DeltaLinks: map[string]string{"Inbox": "live"}}} + if err := m.AddWMail(t.Context(), d); err != nil { + t.Fatal(err) + } + old := m.Get(d.ID) + copyData := *d + copyData.Sync = &models.AddWorkerEmailSyncData{Policy: models.SyncPolicy{DailyMessages: 90}} + if err := m.AddWMail(t.Context(), ©Data); err != nil { + t.Fatal(err) + } + if m.Get(d.ID) != old { + t.Fatal("budget change rebuilt provider") + } + g := *d.Google + g.Token = &oauth2.Token{AccessToken: "new", RefreshToken: "new-refresh"} + copyData.Google = &g + graph := *d.Graph + graph.Token = g.Token + copyData.Graph = &graph + if err := m.AddWMail(t.Context(), ©Data); err != nil { + t.Fatal(err) + } + next := m.Get(d.ID) + if next == old || old.Ctx.Err() == nil || !next.SameExecution(©Data) { + t.Fatal("reauthorization kept local OAuth client") + } + if next.GoogleData != nil && next.GoogleData.LastHistoryID != 19 { + t.Fatal("history cursor lost") + } + if next.GraphData != nil && next.GraphData.Client.DeltaLinks["Inbox"] != "live" { + t.Fatal("delta cursor lost") + } + m.Terminate(d.ID) + if err := m.AddWMail(t.Context(), ©Data); err != nil { + t.Fatal(err) + } + if m.Get(d.ID) == nil || m.Get(d.ID) == next { + t.Fatal("reload after removal failed") + } + m.Terminate(d.ID) + }) + } +} + +func TestMailboxReplacementEndpointSkipAndSentSettings(t *testing.T) { + for _, field := range []string{"imap-password", "smtp-host", "smtp-security", "skip-folders", "sent-copy"} { + t.Run(field, func(t *testing.T) { + m := NewMailManager(func(models.JobEventType, string, any) error { return nil }, nil, nil, nil, nil, nil, nil) + d := smtpPayload() + if err := m.AddWMail(t.Context(), d); err != nil { + t.Fatal(err) + } + old := m.Get(d.ID) + switch field { + case "imap-password": + d.SmtpImap.Credentials.IMAP.Password = "new" + case "smtp-host": + d.SmtpImap.Credentials.SMTP.Host = "new.example.test" + case "smtp-security": + d.SmtpImap.Credentials.SMTP.Security = models.MailSecurityTLS + case "skip-folders": + d.Sync.Policy.SkipFolders = []string{"Custom"} + case "sent-copy": + v := false + d.SaveToSent = &v + } + if err := m.AddWMail(t.Context(), d); err != nil { + t.Fatal(err) + } + if m.Get(d.ID) == old { + t.Fatalf("changed %s ignored", field) + } + m.Terminate(d.ID) + }) + } +} diff --git a/internal/app/worker/mailmanager/terminate.go b/internal/app/worker/mailmanager/terminate.go index dbce9d67d..aaa67d726 100644 --- a/internal/app/worker/mailmanager/terminate.go +++ b/internal/app/worker/mailmanager/terminate.go @@ -1,10 +1,28 @@ package mailmanager -import "github.com/google/uuid" +import ( + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/app/worker/wmail" +) func (m *MailManager) Terminate(id uuid.UUID) { m.Lock() - defer m.Unlock() - + mail := m.Emails[id] delete(m.Emails, id) + m.Unlock() + if mail != nil { + mail.Stop() + } +} + +// TerminateMailbox cannot retire a newer generation through a stale callback. +func (m *MailManager) TerminateMailbox(id uuid.UUID, mail *wmail.WMail) { + m.Lock() + if m.Emails[id] == mail { + delete(m.Emails, id) + } + m.Unlock() + if mail != nil { + mail.Stop() + } } diff --git a/internal/app/worker/wmail/admit.go b/internal/app/worker/wmail/admit.go index 745511263..e6dba5690 100644 --- a/internal/app/worker/wmail/admit.go +++ b/internal/app/worker/wmail/admit.go @@ -2,6 +2,7 @@ package wmail import ( "context" + "errors" "time" "github.com/google/uuid" @@ -133,15 +134,21 @@ func (w *WMail) laneOf(ctx context.Context, key string, msg *models.EmailMessage // beginTick releases an expired hold so the state reports "within budget" // before the pass looks at anything. func (w *WMail) beginTick() { + w.tracker.tickComplete = false w.tracker.release(time.Now()) } -// endTick folds the pass into the relayed state. deferred is the number of -// live messages still waiting on the server; it is reported as-is (not -// accumulated) so a pass that admits everything zeroes it. -func (w *WMail) endTick(stats *tickStats) { - w.tracker.setDeferred(stats.deferred) - w.tracker.touch(time.Now()) +// endTick retains unmeasured backlog until a complete pass confirms catch-up. +func (w *WMail) endTick(ctx context.Context, stats *tickStats, complete bool) { + complete = complete && ctx.Err() == nil && !stats.aborted && stats.deferred == 0 && w.tracker.state.BackfillCursor.GoogleRecovery == nil + w.tracker.tickComplete = complete + if complete { + w.tracker.setDeferred(0) + w.tracker.touch(time.Now()) + return + } + w.tracker.setDeferred(max(w.tracker.state.Deferred, stats.deferred)) + w.tracker.flush(time.Now()) } // storeNew is the shared tail of every provider's new-message path: record @@ -164,14 +171,12 @@ func (w *WMail) storeNew(ctx context.Context, msg *models.EmailMessageData, data return err } - if err := w.EmailMessageMapRepository.Add(ctx, repository.EmailMessageData{ + mapping := repository.EmailMessageData{ UserID: w.UserID.String(), EmailID: w.ID.String(), MessageID: mapKey, ID: data.ID.String(), ThreadID: data.ThreadID, - }); err != nil { - return err } // Both report the send they are about, if any. The id goes on the arrival @@ -191,11 +196,22 @@ func (w *WMail) storeNew(ctx context.Context, msg *models.EmailMessageData, data } // The consumer decodes NEW_EMAIL as JobEventNewEmail{user_id, message}. - err := w.onEvent(models.JobEventTypeNewEmail, &models.JobEventNewEmail{ + arrival := &models.JobEventNewEmail{ UserID: w.UserID, Message: data, ReportOriginalMessageID: reportAbout, - }) + } + if durable, ok := w.EmailMessageMapRepository.(repository.ArrivalAdmission); ok { + err := durable.AdmitArrival(ctx, mapping, &repository.PendingArrival{Arrival: arrival, Bounce: bounce, Complaint: complaint}) + if !errors.Is(err, repository.ErrArrivalOutboxUnsupported) { + return err + } + log.Warn().Str("email_id", w.ID.String()).Msg("sync: legacy arrival publication; upgrade backend and consumer before workers for crash durability") + } + if err := w.EmailMessageMapRepository.Add(ctx, mapping); err != nil { + return err + } + err := w.onEvent(models.JobEventTypeNewEmail, arrival) if err != nil { // The entry would mark a message that never reached the unibox as // known, and every later pass would skip it; drop it so it is re-offered. diff --git a/internal/app/worker/wmail/arrival_durability_test.go b/internal/app/worker/wmail/arrival_durability_test.go new file mode 100644 index 000000000..e2e5bf935 --- /dev/null +++ b/internal/app/worker/wmail/arrival_durability_test.go @@ -0,0 +1,97 @@ +package wmail + +import ( + "context" + "errors" + "testing" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +type durableTestMap struct { + recoveryMessageMap + pending map[string]*repository.PendingArrival + failAfterCommit bool + unsupported bool +} + +func (m *durableTestMap) AdmitArrival(ctx context.Context, data repository.EmailMessageData, p *repository.PendingArrival) error { + if m.unsupported { + return repository.ErrArrivalOutboxUnsupported + } + if _, exists := m.data[data.MessageID]; !exists { + m.data[data.MessageID] = data + m.pending[data.MessageID] = p + } + if m.failAfterCommit { + return errors.New("admission response lost") + } + return nil +} + +func TestArrivalDurabilityRestartPreservesIDAcrossProviders(t *testing.T) { + for _, tc := range []struct { + name string + provider models.InboxProvider + store func(*WMail, context.Context, *models.EmailMessageData) error + key string + }{ + {"google", models.InboxProviderGoogle, (*WMail).googleStore, "provider-id"}, + {"graph", models.InboxProviderOutlook, (*WMail).graphStore, "provider-id"}, + {"imap", models.InboxProviderSMTPIMAP, (*WMail).imapStore, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + m := &durableTestMap{recoveryMessageMap: recoveryMessageMap{data: map[string]repository.EmailMessageData{}}, pending: map[string]*repository.PendingArrival{}, failAfterCommit: true} + var events []captured + w := newTestWMail(t, tc.provider, &events) + w.EmailMessageMapRepository = m + msg := &models.EmailMessageData{MessageID: "", GmailID: "provider-id", Subject: "arrival", BodyPlain: "complete body", InReplyTo: []string{""}} + if err := tc.store(w, ctx, msg); err == nil { + t.Fatal("expected ambiguous response") + } + canonical, err := m.Get(ctx, w.UserID, w.ID, tc.key) + if err != nil || canonical == nil { + t.Fatalf("durable mapping missing: %v", err) + } + original := canonical.ID + if len(events) != 0 || len(m.pending) != 1 { + t.Fatal("durable arrival must not depend on worker publication") + } + fresh := newTestWMail(t, tc.provider, &events) + fresh.UserID = w.UserID + fresh.ID = w.ID + fresh.EmailMessageMapRepository = m + m.failAfterCommit = false + if !fresh.retryUnmap(ctx) { + t.Fatal("fresh worker failed") + } + // Re-offering after a lost response cannot replace the queued canonical ID. + if err := tc.store(fresh, ctx, msg); err != nil { + t.Fatal(err) + } + p := m.pending[tc.key] + if len(m.pending) != 1 || p.Arrival.Message.ID.String() != original || m.data[tc.key].ID != original || len(events) != 0 { + t.Fatal("restart reminted the arrival or published outside durable delivery") + } + if p.Arrival.UserID != w.UserID || p.Arrival.Message.EmailID != w.ID || p.Arrival.Message.ID == uuid.Nil { + t.Fatal("ownership not retained") + } + }) + } +} + +func TestArrivalOldBackendKeepsLegacyPublication(t *testing.T) { + var events []captured + w := newTestWMail(t, models.InboxProviderGoogle, &events) + m := &durableTestMap{recoveryMessageMap: recoveryMessageMap{data: map[string]repository.EmailMessageData{}}, unsupported: true} + w.EmailMessageMapRepository = m + if err := w.googleStore(context.Background(), &models.EmailMessageData{MessageID: "legacy", GmailID: "legacy"}); err != nil { + t.Fatal(err) + } + if len(newEmails(events)) != 1 || len(m.data) != 1 { + t.Fatal("old backend lost existing sync functionality") + } +} diff --git a/internal/app/worker/wmail/err.go b/internal/app/worker/wmail/err.go index 2b01514e6..428e89ab7 100644 --- a/internal/app/worker/wmail/err.go +++ b/internal/app/worker/wmail/err.go @@ -37,6 +37,12 @@ func reportable(err error) bool { } func (w *WMail) CaptureError(err error) { + w.lifecycleMu.Lock() + stopped := w.stopped + w.lifecycleMu.Unlock() + if stopped { + return + } if reportable(err) { errs.CaptureException(err, errs.Tag("user_id", w.UserID.String()), @@ -71,18 +77,19 @@ func (w *WMail) CaptureError(err error) { Timestamp: time.Now().Unix(), } + w.lifecycleMu.Lock() + if w.stopped { + w.lifecycleMu.Unlock() + return + } _ = w.onEvent(eventType, errorEvent) + w.lifecycleMu.Unlock() // Critical errors should stop the sync loop and remove the account from the // worker's local state until the user re-authenticates. if eventType == models.JobEventTypeEmailAuthError || eventType == models.JobEventTypeEmailDisabled { - if w.Cancel != nil { - w.Cancel() - } - if w.TerminateFunc != nil { - w.TerminateFunc() - } + w.Terminate() } } diff --git a/internal/app/worker/wmail/err_mail.go b/internal/app/worker/wmail/err_mail.go index b304d3442..5dfd4e895 100644 --- a/internal/app/worker/wmail/err_mail.go +++ b/internal/app/worker/wmail/err_mail.go @@ -35,6 +35,8 @@ func (w *WMail) RunMailError(f func() *errx.MailError) { } func (w *WMail) Terminate() { - w.Cancel() - w.TerminateFunc() + w.Stop() + if w.TerminateFunc != nil { + w.TerminateFunc() + } } diff --git a/internal/app/worker/wmail/event_graph.go b/internal/app/worker/wmail/event_graph.go index 20f990e5d..e051330b3 100644 --- a/internal/app/worker/wmail/event_graph.go +++ b/internal/app/worker/wmail/event_graph.go @@ -10,13 +10,16 @@ import ( // onGraphMessageRemove emits REMOVE_EMAIL for a message deleted or moved out of a // tracked folder. providerID is the Graph message id. func (w *WMail) onGraphMessageRemove(ctx context.Context, providerID string) error { - internalMessage, err := w.EmailMessageMapRepository.Get(ctx, w.UserID, w.ID, providerID) + internalMessage, err := w.graphMessageMap(ctx, providerID) if err != nil { return err } if internalMessage == nil { return nil } + if w.GraphData.Client.ImmutableIDMode() { + return w.graphReconcileMessage(ctx, internalMessage.ID, providerID, "") + } internalID, err := uuid.Parse(internalMessage.ID) if err != nil { @@ -64,6 +67,11 @@ func (w *WMail) onGraphFlagsChange(ctx context.Context, providerID string, seen // durable persistence (the worker is disposable and must not be the source of // truth for the cursor). func (w *WMail) onGraphDelta(_ context.Context, folder, deltaLink string) error { + if w.graphRecovering(folder) { + if err := w.tracker.emit(w.tracker.state); err != nil { + return err + } + } return w.onEvent(models.JobEventTypeGraphDeltaUpdate, &models.JobEventGraphDeltaUpdate{ UserID: w.UserID, EmailID: w.ID, diff --git a/internal/app/worker/wmail/event_token.go b/internal/app/worker/wmail/event_token.go index 642240e9f..8e699d08f 100644 --- a/internal/app/worker/wmail/event_token.go +++ b/internal/app/worker/wmail/event_token.go @@ -6,6 +6,21 @@ import ( ) func (w *WMail) onTokenUpdate(token *oauth2.Token) error { + w.lifecycleMu.Lock() + defer w.lifecycleMu.Unlock() + if w.stopped { + return nil + } + if data := w.executionData; data != nil && !data.Brokered { + previous := ExecutionKey(data) + if data.Google != nil { + data.Google.Token = token + } + if data.Graph != nil { + data.Graph.Token = token + } + w.executionKeys = map[[32]byte]struct{}{w.initialExecutionKey: {}, previous: {}, ExecutionKey(data): {}} + } return w.onEvent(models.JobEventTypeTokenUpdate, &models.JobEventTokenUpdate{ UserID: w.UserID, EmailID: w.ID, diff --git a/internal/app/worker/wmail/folder_identity_test.go b/internal/app/worker/wmail/folder_identity_test.go index 318604d3c..547afffc2 100644 --- a/internal/app/worker/wmail/folder_identity_test.go +++ b/internal/app/worker/wmail/folder_identity_test.go @@ -1,9 +1,16 @@ package wmail import ( + "context" "testing" + goimap "github.com/emersion/go-imap/v2" + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/client/smtpimap/imap" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" ) // mailboxUpdates is every folder MAILBOX_UPDATE relayed, by name. @@ -163,16 +170,30 @@ func TestSyncDoesNotGuessARenameWhenTwoFoldersCouldBeIt(t *testing.T) { } } -// A changed UIDVALIDITY is the server saying every UID we hold for the folder -// is void. The folder is re-baselined rather than followed from a cursor that -// now addresses nothing, and an unfinished import walks it again. -func TestSyncRebaselinesAFolderWhoseUIDValidityChanged(t *testing.T) { - conn := &fakeImapConn{ - folders: []models.Mailbox{{Name: "INBOX", UIDValidity: 900, HighestModSeq: 5}}, - changed: uidRange(3), +type folderIdentityRecoveryConn struct{ *fakeImapConn } + +func (c *folderIdentityRecoveryConn) FetchEnvelopes(ctx context.Context, uids []goimap.UID) ([]*imap.Fetched, *errx.MailError) { + fetched, err := c.fakeImapConn.FetchEnvelopes(ctx, uids) + for _, f := range fetched { + f.Email.ModSeq = 5 + f.Email.Flags = []string{models.FlagSeen} } + return fetched, err +} + +// Voided UIDs require bounded generation recovery, even with completed history and unchanged MODSEQ. +func TestSyncRebaselinesAFolderWhoseUIDValidityChanged(t *testing.T) { + conn := &folderIdentityRecoveryConn{fakeImapConn: &fakeImapConn{ + folders: []models.Mailbox{{Name: "INBOX", UIDValidity: 900, UIDNext: 4, HighestModSeq: 5}}, + changed: uidRange(3), + all: uidRange(3), + }} w, events := newIMAPTestMail(conn, &fixedBudget{allow: 10}, - &models.Mailbox{Name: "INBOX", UIDValidity: 7, HighestModSeq: 100}) + &models.Mailbox{Name: "INBOX", UIDValidity: 7, UIDNext: 501, HighestModSeq: 5}) + knownID := uuid.NewString() + maps := &recoveryMessageMap{data: map[string]repository.EmailMessageData{"<2@fake.test>": {ID: knownID, MessageID: "<2@fake.test>"}}} + w.EmailMessageMapRepository = maps + w.tracker.state.BackfillSynced = 17 w.tracker.setFolder("INBOX", models.SyncFolderCursor{UID: 500, Done: true}) w.flagScan = map[string]*folderFlagScan{"INBOX": {}} @@ -187,11 +208,25 @@ func TestSyncRebaselinesAFolderWhoseUIDValidityChanged(t *testing.T) { if box.UIDValidity != 900 || box.HighestModSeq != 5 { t.Errorf("relayed %+v, want the server's current cursor", box) } - if hasEvent(*events, models.JobEventTypeNewEmail) { - t.Error("the folder was walked from a cursor its server had already voided") + arrivals := mailboxEvents(*events, models.JobEventTypeNewEmail) + if len(arrivals) != 2 || len(maps.data) != 3 || maps.data["<2@fake.test>"].ID != knownID { + t.Fatal("bounded recovery lost unseen mail or duplicated the known Message-ID") } - if cur := w.tracker.folder("INBOX"); cur.UID != 0 || cur.Done { - t.Errorf("backfill floor = %+v, want it cleared so the folder is walked again", cur) + updates := mailboxEvents(*events, models.JobEventTypeEmailUpdate) + if len(updates) != 1 { + t.Fatalf("known recovery updates=%d", len(updates)) + } + update := updates[0].body.(*models.JobEventEmailUpdate) + if update.ID.String() != knownID || update.UID != 2 || update.Mailbox != 900 || update.ModSeq != 5 || !models.SeenFromFlags(update.Flags) { + t.Fatalf("known message kept voided handles or stale read state: %+v", update) + } + cur := w.tracker.folder("INBOX") + recovery := imapRecoveryCursor(cur) + if !cur.Done || cur.UID != 1 || recovery == nil || recovery.Generation != 900 || recovery.Synced != 2 { + t.Fatalf("folder did not finish bounded replay from its new UIDs: %+v / %+v", cur, recovery) + } + if w.tracker.state.BackfillStatus != models.SyncBackfillComplete || w.tracker.state.BackfillSynced != 17 { + t.Fatal("generation recovery reopened completed global history") } if _, held := w.flagScan["INBOX"]; held { t.Error("the flag snapshot survived; it describes UIDs that no longer mean anything") @@ -199,6 +234,14 @@ func TestSyncRebaselinesAFolderWhoseUIDValidityChanged(t *testing.T) { if got := mailboxDeletes(*events); len(got) != 0 { t.Errorf("retired %v; the folder is still there", got) } + searches := conn.searches + next, later := imapReload(t, w, conn, &fixedBudget{allow: 10}) + if err := next.Sync(t.Context()); err != nil { + t.Fatal(err) + } + if conn.searches != searches || len(mailboxEvents(*later, models.JobEventTypeNewEmail)) != 0 || len(mailboxEvents(*later, models.JobEventTypeEmailUpdate)) != 0 || len(maps.data) != 3 { + t.Fatal("completed generation recovery repeated after reload") + } } // A folder's backfill floor goes with the folder. A name is reusable, so a diff --git a/internal/app/worker/wmail/governor.go b/internal/app/worker/wmail/governor.go index 11d754fe1..817703078 100644 --- a/internal/app/worker/wmail/governor.go +++ b/internal/app/worker/wmail/governor.go @@ -304,10 +304,5 @@ func (w *WMail) escalate(mailErr *errx.MailError) { ActionRequired: userInfo.ActionRequired, Timestamp: time.Now().Unix(), }) - if w.Cancel != nil { - w.Cancel() - } - if w.TerminateFunc != nil { - w.TerminateFunc() - } + w.Terminate() } diff --git a/internal/app/worker/wmail/graph_recovery_test.go b/internal/app/worker/wmail/graph_recovery_test.go new file mode 100644 index 000000000..9c453ce86 --- /dev/null +++ b/internal/app/worker/wmail/graph_recovery_test.go @@ -0,0 +1,442 @@ +package wmail + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/client/msgraph" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +type graphRecoveryProvider struct { + snapshot map[string][]any + page2 map[string][]any + messages map[string]map[string]any + expired string + seen bool + requests []string +} + +func (g *graphRecoveryProvider) serve(t *testing.T) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if !strings.Contains(r.Header.Get("Prefer"), `IdType="ImmutableId"`) { + t.Error("sync requested move-unstable ids") + } + path := strings.TrimPrefix(r.URL.Path, "/v1.0/me/") + g.requests = append(g.requests, path+"?"+r.URL.RawQuery) + switch { + case path == "translateExchangeIds": + var body struct { + IDs []string `json:"inputIds"` + Target string `json:"targetIdType"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Error(err) + } + value := []any{} + for _, id := range body.IDs { + target := "stable-" + strings.TrimPrefix(id, "regular-") + if body.Target == "restId" { + target = "regular-" + strings.TrimPrefix(id, "stable-") + } + value = append(value, map[string]any{"sourceId": id, "targetId": target}) + } + _ = json.NewEncoder(w).Encode(map[string]any{"value": value}) + case strings.HasPrefix(path, "messages/"): + id := strings.TrimPrefix(path, "messages/") + msg := g.messages[id] + if msg == nil { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"error":{"code":"ErrorItemNotFound"}}`)) + return + } + copy := map[string]any{} + for key, value := range msg { + copy[key] = value + } + copy["isRead"] = g.seen + _ = json.NewEncoder(w).Encode(copy) + case strings.HasPrefix(path, "mailFolders/"): + folder, tail, _ := strings.Cut(strings.TrimPrefix(path, "mailFolders/"), "/") + if tail == "" { + _ = json.NewEncoder(w).Encode(map[string]any{"id": "folder-" + folder}) + return + } + if tail != "messages/delta" { + t.Errorf("unexpected path %s", path) + w.WriteHeader(500) + return + } + if folder == g.expired && r.URL.Query().Get("$deltatoken") == "seed" { + w.WriteHeader(http.StatusGone) + _, _ = w.Write([]byte(`{"error":{"code":"syncStateNotFound"}}`)) + return + } + value := g.snapshot[folder] + out := map[string]any{"value": value, "@odata.deltaLink": "https://graph.microsoft.com" + r.URL.Path + "?$deltatoken=fresh"} + if g.page2[folder] != nil { + if r.URL.Query().Get("$skiptoken") == "page2" { + out["value"] = g.page2[folder] + } else { + delete(out, "@odata.deltaLink") + out["@odata.nextLink"] = "https://graph.microsoft.com" + r.URL.Path + "?$skiptoken=page2" + } + } + _ = json.NewEncoder(w).Encode(out) + default: + t.Errorf("unexpected Graph path %s", path) + w.WriteHeader(500) + } + })) + t.Cleanup(srv.Close) + return srv +} + +type graphRecoveryContext struct{ recoveryRows } + +func (*graphRecoveryContext) IsOwnConversation(_ context.Context, _, _ uuid.UUID, _ []string, thread string) (bool, error) { + return thread == "conv-stable-reply", nil +} + +type graphRecoveryBudget struct { + backfill int + lanes []SyncLane + observed int +} + +func (*graphRecoveryBudget) Policy() models.SyncPolicy { + return normalizePolicy(models.SyncPolicy{BackfillDays: 7, BackfillMessages: 2}) +} +func (*graphRecoveryBudget) SetPolicy(models.SyncPolicy) {} +func (*graphRecoveryBudget) RecordThrottledDay(context.Context) bool { return false } +func (b *graphRecoveryBudget) ObserveLive(_ context.Context, n int) bool { + b.observed += n + return false +} +func (b *graphRecoveryBudget) Admit(_ context.Context, lane SyncLane) Admission { + b.lanes = append(b.lanes, lane) + if lane == LaneBackfill { + if b.backfill == 0 { + return Admission{Until: time.Now().Add(time.Second)} + } + b.backfill-- + } + return Admission{OK: true} +} + +func graphReloadState(t *testing.T, state models.SyncState) *models.SyncState { + t.Helper() + data, err := json.Marshal(state) + if err != nil { + t.Fatal(err) + } + var out models.SyncState + if err := json.Unmarshal(data, &out); err != nil { + t.Fatal(err) + } + return &out +} + +func TestGraphRecoveryBudgetedReplyAndNativeContinuationSurviveReload(t *testing.T) { + for _, expired := range []bool{false, true} { + t.Run(map[bool]string{false: "missing completed Archive", true: "expired Inbox"}[expired], func(t *testing.T) { + folder := msgraph.FolderArchive + if expired { + folder = msgraph.FolderInbox + } + g := &graphRecoveryProvider{snapshot: map[string][]any{folder: {map[string]any{"id": "stable-one"}}}, page2: map[string][]any{folder: {map[string]any{"id": "stable-two"}, map[string]any{"id": "stable-capped"}, map[string]any{"id": "stable-reply"}}}, messages: map[string]map[string]any{}} + for _, id := range []string{"stable-one", "stable-two", "stable-capped", "stable-reply"} { + g.messages[id] = graphMessageJSON(id) + } + g.messages["stable-reply"]["receivedDateTime"] = time.Now().Add(-90 * 24 * time.Hour).UTC().Format(time.RFC3339) + if expired { + g.expired = folder + } + srv := g.serve(t) + var events []captured + var states []models.SyncState + w := newGraphTestMail(t, srv, &events, &states) + w.GraphData.Client.SetImmutableIDMode(true) + if !expired { + delete(w.GraphData.Client.DeltaLinks, folder) + } + maps := &recoveryMessageMap{data: map[string]repository.EmailMessageData{}} + w.EmailMessageMapRepository = maps + w.SyncContext = &graphRecoveryContext{} + w.tracker.state.BackfillStatus = models.SyncBackfillComplete + budget := &graphRecoveryBudget{backfill: 1} + w.gov = budget + if err := w.SyncGraph(t.Context()); err != nil { + t.Fatal(err) + } + ids := importedIDs(events) + if !hasID(ids, "stable-one") || !hasID(ids, "stable-reply") || hasID(ids, "stable-two") || budget.observed != 0 { + t.Fatalf("first replay imports=%v live-count=%d", ids, budget.observed) + } + if !strings.Contains(w.GraphData.Client.DeltaLinks[folder], "$skiptoken=page2") || w.tracker.state.LastSyncedAt != nil { + t.Fatalf("deferred replay advanced freshness or lost native continuation: %v", w.tracker.state) + } + before, err := w.graphRecovery(folder) + if err != nil || before.Count != 1 { + t.Fatalf("saved recovery=%+v err=%v", before, err) + } + reloaded := newGraphTestMail(t, srv, &events, &states) + reloaded.ID, reloaded.UserID = w.ID, w.UserID + reloaded.EmailMessageMapRepository, reloaded.SyncContext = maps, w.SyncContext + reloaded.GraphData.Client.SetImmutableIDMode(true) + reloaded.GraphData.Client.DeltaLinks = cloneStringMap(w.GraphData.Client.DeltaLinks) + reloaded.tracker = newSyncTracker(graphReloadState(t, w.tracker.state), func(st models.SyncState) error { states = append(states, st); return nil }) + reloaded.gov = &graphRecoveryBudget{backfill: 1} + if err := reloaded.SyncGraph(t.Context()); err != nil { + t.Fatal(err) + } + after, err := reloaded.graphRecovery(folder) + if err != nil || !before.Since.Equal(after.Since) || after.Count != 2 || reloaded.graphRecovering(folder) { + t.Fatalf("restarted recovery changed fixed window/count: before=%+v after=%+v err=%v", before, after, err) + } + ids = importedIDs(events) + if len(ids) != 3 || !hasID(ids, "stable-two") || !strings.Contains(reloaded.GraphData.Client.DeltaLinks[folder], "$deltatoken=fresh") { + t.Fatalf("eventual deduplicated replay imports=%v links=%v", ids, reloaded.GraphData.Client.DeltaLinks) + } + g.snapshot[folder], g.page2[folder] = nil, nil + if err := reloaded.SyncGraph(t.Context()); err != nil || len(importedIDs(events)) != 3 { + t.Fatalf("repeat created duplicate arrivals: %v %v", err, importedIDs(events)) + } + }) + } +} + +func TestGraphArchiveUpgradePreservesLegacyIdentityMovesAndFlags(t *testing.T) { + id := uuid.New() + g := &graphRecoveryProvider{snapshot: map[string][]any{ + msgraph.FolderInbox: {map[string]any{"id": "stable-known", "@removed": map[string]any{"reason": "deleted"}}}, + msgraph.FolderArchive: {map[string]any{"id": "stable-known"}, map[string]any{"id": "stable-new"}}, + }, messages: map[string]map[string]any{"stable-known": graphMessageJSON("stable-known"), "stable-new": graphMessageJSON("stable-new")}} + g.messages["stable-known"]["parentFolderId"] = "folder-archive" + var events []captured + var states []models.SyncState + w := newGraphTestMail(t, g.serve(t), &events, &states) + w.GraphData.Client.SetImmutableIDMode(true) + delete(w.GraphData.Client.DeltaLinks, msgraph.FolderArchive) + maps := &recoveryMessageMap{data: map[string]repository.EmailMessageData{"regular-known": {UserID: w.UserID.String(), EmailID: w.ID.String(), MessageID: "regular-known", ID: id.String(), ThreadID: "owned"}}} + w.EmailMessageMapRepository = maps + rows := &graphRecoveryContext{recoveryRows: recoveryRows{rows: []repository.ProviderFolderMessage{{ID: id, ProviderID: "regular-known", ProviderFolder: models.FolderInbox}}}} + w.SyncContext = rows + w.tracker.state.BackfillStatus = models.SyncBackfillComplete + baseEmit := w.onEvent + w.onEvent = func(kind models.JobEventType, body any) error { + if update, ok := body.(*models.JobEventFolderUpdate); ok && update.ID == id { + if update.Relayed { + rows.rows[0].ProviderID = update.ProviderID + } else { + rows.rows[0].ProviderFolder = update.Folder + } + } + return baseEmit(kind, body) + } + if err := w.SyncGraph(t.Context()); err != nil { + t.Fatal(err) + } + if len(importedIDs(events)) != 1 || maps.data["stable-known"].ID != id.String() || maps.data["regular-known"].ID != id.String() || rows.rows[0].ProviderFolder != models.FolderArchive { + t.Fatalf("move changed identity or lost Archive: ids=%v maps=%v rows=%v", importedIDs(events), maps.data, rows.rows) + } + for _, ev := range events { + if ev.eventType == models.JobEventTypeRemoveEmail { + t.Fatal("move-out deleted the platform conversation") + } + } + g.seen = true + before := len(events) + reloaded := newGraphTestMail(t, g.serve(t), &events, &states) + reloaded.ID, reloaded.UserID = w.ID, w.UserID + reloaded.GraphData.Client.SetImmutableIDMode(true) + reloaded.GraphData.Client.DeltaLinks = cloneStringMap(w.GraphData.Client.DeltaLinks) + reloaded.EmailMessageMapRepository, reloaded.SyncContext = maps, rows + reloaded.tracker = newSyncTracker(graphReloadState(t, w.tracker.state), func(models.SyncState) error { return nil }) + if err := reloaded.SyncGraph(t.Context()); err != nil { + t.Fatal(err) + } + flags := false + for _, ev := range events[before:] { + if ev.eventType == models.JobEventTypeFlagsAdd { + update := ev.body.(*models.JobEventFlags) + flags = flags || update.ID == id + } + } + if !flags || len(importedIDs(events)) != 1 { + t.Fatalf("Archive read-state update lost or reimported: flags=%v arrivals=%v", flags, importedIDs(events)) + } +} + +func TestGraphRecoveryPendingBaselineAndStoredRemovalScanResumeAfterReload(t *testing.T) { + g := &graphRecoveryProvider{snapshot: map[string][]any{}, expired: msgraph.FolderInbox, messages: map[string]map[string]any{}} + srv := g.serve(t) + var events []captured + var states []models.SyncState + w := newGraphTestMail(t, srv, &events, &states) + w.GraphData.Client.SetImmutableIDMode(true) + w.tracker.state.BackfillStatus = models.SyncBackfillComplete + w.tracker.setFolder(graphIdentityCursor, models.SyncFolderCursor{Done: true}) + rows := &graphRecoveryContext{} + maps := &recoveryMessageMap{data: map[string]repository.EmailMessageData{}} + for n := 1; n <= graphBackfillPage+1; n++ { + id := uuid.UUID{} + id[15] = byte(n) + providerID := "stable-" + id.String() + rows.rows = append(rows.rows, repository.ProviderFolderMessage{ID: id, ProviderID: providerID, ProviderFolder: models.FolderInbox}) + maps.data[providerID] = repository.EmailMessageData{ID: id.String(), MessageID: providerID} + } + w.EmailMessageMapRepository, w.SyncContext = maps, rows + if err := w.SyncGraph(t.Context()); err != nil { + t.Fatal(err) + } + removals := func() int { + n := 0 + for _, ev := range events { + if ev.eventType == models.JobEventTypeRemoveEmail { + n++ + } + } + return n + } + if removals() != graphBackfillPage || !strings.Contains(w.GraphData.Client.DeltaLinks[msgraph.FolderInbox], "seed") { + t.Fatalf("unreconciled baseline installed: events=%d links=%v", len(events), w.GraphData.Client.DeltaLinks) + } + state, err := w.graphRecovery(msgraph.FolderInbox) + if err != nil || !strings.Contains(state.PendingLink, "fresh") { + t.Fatalf("captured baseline missing: %+v %v", state, err) + } + countInbox := func() int { + n := 0 + for _, req := range g.requests { + if strings.HasPrefix(req, "mailFolders/inbox/messages/delta") { + n++ + } + } + return n + } + before := countInbox() + reloaded := newGraphTestMail(t, srv, &events, &states) + reloaded.ID, reloaded.UserID = w.ID, w.UserID + reloaded.GraphData.Client.SetImmutableIDMode(true) + reloaded.GraphData.Client.DeltaLinks = cloneStringMap(w.GraphData.Client.DeltaLinks) + reloaded.EmailMessageMapRepository, reloaded.SyncContext = maps, rows + reloaded.tracker = newSyncTracker(graphReloadState(t, w.tracker.state), func(models.SyncState) error { return nil }) + if err := reloaded.SyncGraph(t.Context()); err != nil { + t.Fatal(err) + } + if removals() != graphBackfillPage+1 || countInbox() != before || !strings.Contains(reloaded.GraphData.Client.DeltaLinks[msgraph.FolderInbox], "fresh") || reloaded.tracker.state.LastSyncedAt != nil { + t.Fatalf("stored scan did not resume/freshness claimed before live catch-up: events=%d inbox=%d/%d state=%v", len(events), countInbox(), before, reloaded.tracker.state) + } + if err := reloaded.SyncGraph(t.Context()); err != nil || reloaded.tracker.state.LastSyncedAt == nil { + t.Fatalf("fresh baseline did not catch up: err=%v", err) + } +} + +func TestGraphRecoveryCompletionSurvivesLaterLiveCursorAdvancement(t *testing.T) { + var events []captured + var states []models.SyncState + w := newGraphTestMail(t, newFakeGraph().serve(t), &events, &states) + if err := w.graphRecoveryStart(t.Context(), msgraph.FolderInbox); err != nil { + t.Fatal(err) + } + if err := w.graphRecoveryCheckpoint(t.Context(), msgraph.FolderInbox, "baseline"); err != nil { + t.Fatal(err) + } + w.GraphData.Client.DeltaLinks[msgraph.FolderInbox] = "baseline" + if w.graphRecovering(msgraph.FolderInbox) { + t.Fatal("installed baseline still recovering") + } + w.GraphData.Client.DeltaLinks[msgraph.FolderInbox] = "later-live-checkpoint" + if w.graphRecovering(msgraph.FolderInbox) { + t.Fatal("later live cursor reopened completed recovery") + } + legacy := graphReloadState(t, w.tracker.state) + if legacy.BackfillCursor.Folders["graph:delta-recovery:inbox"].Next == "" || !legacy.BackfillCursor.Folders["graph:delta-recovery:inbox"].Done { + t.Fatal("existing cursor protocol lost opaque recovery metadata") + } +} + +func TestGraphImmutableAliasRetainsHiddenLegacyMappingAndRejectsCollision(t *testing.T) { + g := &graphRecoveryProvider{snapshot: map[string][]any{}, messages: map[string]map[string]any{}} + var events []captured + var states []models.SyncState + w := newGraphTestMail(t, g.serve(t), &events, &states) + w.GraphData.Client.SetImmutableIDMode(true) + id := uuid.NewString() + maps := &recoveryMessageMap{data: map[string]repository.EmailMessageData{"regular-hidden": {ID: id, MessageID: "regular-hidden"}}} + w.EmailMessageMapRepository = maps + known, err := w.graphMessageMap(t.Context(), "stable-hidden") + if err != nil || known == nil || known.ID != id || maps.data["regular-hidden"].ID != id || len(importedIDs(events)) != 0 { + t.Fatalf("hidden legacy map became a new arrival: known=%v err=%v", known, err) + } + other := uuid.NewString() + maps.data["stable-conflict"] = repository.EmailMessageData{ID: other} + if err := w.graphAddAlias(t.Context(), repository.EmailMessageData{MessageID: "stable-conflict", ID: id}); err == nil || maps.data["stable-conflict"].ID != other { + t.Fatal("alias overwrote an unrelated provider mapping") + } +} + +func TestGraphRecoveryArrivalPublicationFailureRetriesAfterReload(t *testing.T) { + g := newFakeGraph() + g.live = []string{"reply"} + var events []captured + var states []models.SyncState + srv := g.serve(t) + w := newGraphTestMail(t, srv, &events, &states) + delete(w.GraphData.Client.DeltaLinks, msgraph.FolderInbox) + w.tracker.state.BackfillStatus = models.SyncBackfillComplete + maps := &recoveryMessageMap{data: map[string]repository.EmailMessageData{}} + w.EmailMessageMapRepository = maps + base := w.onEvent + w.onEvent = func(kind models.JobEventType, body any) error { + if kind == models.JobEventTypeNewEmail { + return errors.New("publication unavailable") + } + return base(kind, body) + } + if err := w.SyncGraph(t.Context()); err != nil { + t.Fatal(err) + } + if w.GraphData.Client.DeltaLinks[msgraph.FolderInbox] != "" || len(maps.data) != 0 { + t.Fatalf("failed publication advanced cursor or retained suppressing map: %v %v", w.GraphData.Client.DeltaLinks, maps.data) + } + g.live = []string{"reply"} + reloaded := newGraphTestMail(t, srv, &events, &states) + reloaded.EmailMessageMapRepository = maps + reloaded.GraphData.Client.DeltaLinks = cloneStringMap(w.GraphData.Client.DeltaLinks) + reloaded.tracker = newSyncTracker(graphReloadState(t, w.tracker.state), func(models.SyncState) error { return nil }) + if err := reloaded.SyncGraph(t.Context()); err != nil { + t.Fatal(err) + } + if len(importedIDs(events)) != 1 || reloaded.GraphData.Client.DeltaLinks[msgraph.FolderInbox] == "" { + t.Fatalf("arrival not replayed: ids=%v links=%v", importedIDs(events), reloaded.GraphData.Client.DeltaLinks) + } +} + +func TestGraphOldBackendRetainsLegacySyncWithoutImmutableClaim(t *testing.T) { + g := newFakeGraph() + g.live = []string{"live"} + var events []captured + var states []models.SyncState + w := newGraphTestMail(t, g.serve(t), &events, &states) + w.GraphData.Client.SetImmutableIDMode(true) + w.SyncContext = &recoveryRows{err: repository.ErrSyncContextUnsupported} + w.tracker.state.BackfillStatus = models.SyncBackfillComplete + if err := w.SyncGraph(t.Context()); err != nil { + t.Fatal(err) + } + if w.GraphData.Client.ImmutableIDMode() || len(importedIDs(events)) != 1 { + t.Fatalf("old backend stalled sync or claimed immutable reconciliation: immutable=%v imports=%v", w.GraphData.Client.ImmutableIDMode(), importedIDs(events)) + } +} diff --git a/internal/app/worker/wmail/imap_conn.go b/internal/app/worker/wmail/imap_conn.go index bbf0dee7f..3e36c4a24 100644 --- a/internal/app/worker/wmail/imap_conn.go +++ b/internal/app/worker/wmail/imap_conn.go @@ -40,7 +40,7 @@ type ImapConn interface { FetchFlags(ctx context.Context, uidFrom uint32) (map[uint32]imap.FlagState, *errx.MailError) SearchSince(since time.Time) ([]goimap.UID, *errx.MailError) FetchEnvelopes(ctx context.Context, uids []goimap.UID) ([]*imap.Fetched, *errx.MailError) - FetchBody(f *imap.Fetched) + FetchBody(f *imap.Fetched) *errx.MailError // Send path. AppendToSent(ctx context.Context, raw []byte, sentAt time.Time) error diff --git a/internal/app/worker/wmail/lifecycle.go b/internal/app/worker/wmail/lifecycle.go new file mode 100644 index 000000000..acfa8eaf6 --- /dev/null +++ b/internal/app/worker/wmail/lifecycle.go @@ -0,0 +1,189 @@ +package wmail + +import ( + "context" + "crypto/sha256" + "encoding/json" + "slices" + "strings" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/models" + "golang.org/x/oauth2" +) + +// ExecutionKey excludes checkpoints and policy budgets from client identity. +func ExecutionKey(data *models.AddWorkerEmail) [32]byte { + type credential struct{ Access, Refresh string } + tokenKey := func(t *oauth2.Token) credential { + if t == nil { + return credential{} + } + return credential{t.AccessToken, t.RefreshToken} + } + key := struct { + ID, User uuid.UUID + Org *uuid.UUID + Provider models.InboxProvider + Email, First, Last, GraphUser string + Brokered, ImapSync, SaveToSent bool + Token credential + SMTP, IMAP *models.Service + Skip []string + }{ID: data.ID, User: data.UserID, Org: data.OrganizationID, Provider: data.Type, + Email: data.Email, First: data.FirstName, Last: data.LastName, Brokered: data.Brokered, + ImapSync: data.Type == models.InboxProviderSMTPIMAP && data.ImapSync, SaveToSent: data.Type == models.InboxProviderSMTPIMAP && data.SavesSentCopy()} + if data.Type == models.InboxProviderGoogle && data.Google != nil && !data.Brokered { + key.Token = tokenKey(data.Google.Token) + } + if data.Type == models.InboxProviderOutlook && data.Graph != nil { + key.GraphUser = data.Graph.User + if !data.Brokered { + key.Token = tokenKey(data.Graph.Token) + } + } + if data.Type == models.InboxProviderSMTPIMAP && data.SmtpImap != nil && data.SmtpImap.Credentials != nil { + key.SMTP, key.IMAP = data.SmtpImap.Credentials.SMTP, data.SmtpImap.Credentials.IMAP + } + if data.Sync != nil && data.Type == models.InboxProviderSMTPIMAP { + for _, folder := range data.Sync.Policy.SkipFolders { + if folder = strings.ToLower(strings.TrimSpace(folder)); folder != "" { + key.Skip = append(key.Skip, folder) + } + } + slices.Sort(key.Skip) + key.Skip = slices.Compact(key.Skip) + } + b, _ := json.Marshal(key) + return sha256.Sum256(b) +} + +func (w *WMail) rememberExecution(data *models.AddWorkerEmail) { + // Own the token pointers; ADD_EMAIL payloads may be reused by a publisher. + copyData := *data + if data.Google != nil { + g := *data.Google + copyData.Google = &g + } + if data.Graph != nil { + g := *data.Graph + copyData.Graph = &g + } + w.executionData = ©Data + w.initialExecutionKey = ExecutionKey(data) + w.executionKeys = map[[32]byte]struct{}{w.initialExecutionKey: {}} +} + +func (w *WMail) SameExecution(data *models.AddWorkerEmail) bool { + w.lifecycleMu.Lock() + defer w.lifecycleMu.Unlock() + _, ok := w.executionKeys[ExecutionKey(data)] + return ok && !w.stopped +} + +// BeginExecution rejects stale references and lets replacements drain admitted work. +func (w *WMail) BeginExecution() (func(), bool) { + w.lifecycleMu.Lock() + defer w.lifecycleMu.Unlock() + if w.stopped { + return nil, false + } + w.active.Add(1) + return w.active.Done, true +} + +// Stop cancels scheduling and closes the provider without waiting on its caller. +func (w *WMail) Stop() { + w.lifecycleMu.Lock() + if w.stopped { + w.lifecycleMu.Unlock() + return + } + w.stopped = true + w.stopDone = make(chan struct{}) + done := w.stopDone + w.lifecycleMu.Unlock() + defer close(done) + if w.Cancel != nil { + w.Cancel() + } + if w.SmtpImapData != nil && w.SmtpImapData.ImapClient != nil { + if c, ok := w.SmtpImapData.ImapClient.(interface{ Close() error }); ok { + _ = c.Close() + } + } +} + +// Drain is called outside mailbox execution, after Stop prevents new work. +func (w *WMail) Drain() { + w.lifecycleMu.Lock() + done := w.stopDone + w.lifecycleMu.Unlock() + if done != nil { + <-done + } + w.active.Wait() +} + +// ResumeInto uses the drained live state, which can be newer than ADD_EMAIL. +func (w *WMail) ResumeInto(data *models.AddWorkerEmail) { + if w.UserID != data.UserID || w.EmailType != data.Type { + return + } + if w.tracker != nil { + if data.Sync == nil { + data.Sync = &models.AddWorkerEmailSyncData{} + } else { + s := *data.Sync + data.Sync = &s + } + state := w.tracker.state + state.BackfillCursor.Folders = make(map[string]models.SyncFolderCursor, len(w.tracker.state.BackfillCursor.Folders)) + for k, v := range w.tracker.state.BackfillCursor.Folders { + state.BackfillCursor.Folders[k] = v + } + data.Sync.State = &state + } + if w.GoogleData != nil && data.Google != nil { + g := *data.Google + g.LastHistoryID = w.GoogleData.LastHistoryID + data.Google = &g + } + if w.GraphData != nil && data.Graph != nil { + g := *data.Graph + g.DeltaLinks = cloneStringMap(w.GraphData.Client.DeltaLinks) + data.Graph = &g + } + if w.SmtpImapData != nil && data.SmtpImap != nil { + s := *data.SmtpImap + s.Mailboxes = nil + for _, box := range w.SmtpImapData.Mailboxes { + if box != nil { + s.Mailboxes = append(s.Mailboxes, *box) + } + } + data.SmtpImap = &s + } +} + +// PreservePending retains session-local retry work during same-worker replacement. +func (w *WMail) PreservePending(old *WMail) { + if old == nil || old.UserID != w.UserID || old.EmailType != w.EmailType { + return + } + w.unmapPending = old.unmapPending + w.diagnosticRetries = old.diagnosticRetries + if old.tracker != nil && w.tracker != nil { + w.tracker.dirty = old.tracker.dirty + w.tracker.lastSent = old.tracker.lastSent + } +} + +func (w *WMail) ExecutionContext(ctx context.Context) (context.Context, func()) { + ctx, cancel := context.WithCancel(ctx) + if w.Ctx == nil { + return ctx, cancel + } + stop := context.AfterFunc(w.Ctx, cancel) + return ctx, func() { stop(); cancel() } +} diff --git a/internal/app/worker/wmail/lifecycle_test.go b/internal/app/worker/wmail/lifecycle_test.go new file mode 100644 index 000000000..798740844 --- /dev/null +++ b/internal/app/worker/wmail/lifecycle_test.go @@ -0,0 +1,221 @@ +package wmail + +import ( + "context" + "net" + "sync/atomic" + "testing" + "time" + + goimap "github.com/emersion/go-imap/v2" + "github.com/emersion/go-imap/v2/imapserver" + "github.com/emersion/go-imap/v2/imapserver/imapmemserver" + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/client/msgraph" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" + "golang.org/x/oauth2" +) + +type closingIMAP struct { + ImapConn + closes atomic.Int32 +} + +type observedIMAPSession struct { + imapserver.Session + closed chan struct{} +} + +func (s *observedIMAPSession) Close() error { defer close(s.closed); return s.Session.Close() } + +func TestMailboxLifecycleStopClosesAuthenticatedIdleSocket(t *testing.T) { + t.Setenv("DEPLOYMENT_MODE", "self_hosted") + mem := imapmemserver.New() + user := imapmemserver.NewUser("mailbox@example.test", "local-test") + if err := user.Create("INBOX", nil); err != nil { + t.Fatal(err) + } + mem.AddUser(user) + closed := make(chan struct{}) + srv := imapserver.New(&imapserver.Options{Caps: goimap.CapSet{goimap.CapIMAP4rev1: {}}, InsecureAuth: true, NewSession: func(*imapserver.Conn) (imapserver.Session, *imapserver.GreetingData, error) { + return &observedIMAPSession{Session: mem.NewSession(), closed: closed}, nil, nil + }}) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + go func() { _ = srv.Serve(ln) }() + defer srv.Close() + d := &models.AddWorkerEmail{ID: uuid.New(), UserID: uuid.New(), Type: models.InboxProviderSMTPIMAP, Email: "mailbox@example.test", ImapSync: true, SmtpImap: &models.AddWorkerEmailSmtpImapData{Credentials: &models.SmtpImap{IMAP: &models.Service{Host: "127.0.0.1", Port: ln.Addr().(*net.TCPAddr).Port, Username: "mailbox@example.test", Password: "local-test", Security: models.MailSecurityNone}}}} + w, merr := NewWMail(d, func(models.JobEventType, string, any) error { return nil }, nil, nil, nil, nil, nil, nil) + if merr != nil { + t.Fatal(merr) + } + w.Stop() + w.Stop() + select { + case <-closed: + case <-time.After(time.Second): + t.Fatal("authenticated idle server session retained after stop") + } +} + +func (c *closingIMAP) Close() error { c.closes.Add(1); return nil } + +func TestMailboxLifecycleStopDrainsAndRejectsStaleWork(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + c := &closingIMAP{} + w := &WMail{Ctx: ctx, Cancel: cancel, SmtpImapData: &SmtpImapData{ImapClient: c}} + done, ok := w.BeginExecution() + if !ok { + t.Fatal("live execution rejected") + } + w.Stop() + w.Stop() + if c.closes.Load() != 1 || ctx.Err() == nil { + t.Fatal("stop must cancel and close once") + } + if _, ok := w.BeginExecution(); ok { + t.Fatal("retired execution admitted") + } + drained := make(chan struct{}) + go func() { w.Drain(); close(drained) }() + select { + case <-drained: + t.Fatal("drained before admitted work returned") + default: + } + done() + select { + case <-drained: + case <-time.After(time.Second): + t.Fatal("drain did not finish") + } +} + +func TestMailboxLifecycleExecutionKeyAndTokenRefresh(t *testing.T) { + token := &oauth2.Token{AccessToken: "old", RefreshToken: "refresh", Expiry: time.Now().Add(time.Hour)} + d := &models.AddWorkerEmail{ID: uuid.New(), UserID: uuid.New(), Type: models.InboxProviderGoogle, Google: &models.AddWorkerEmailGoogleData{Token: token}} + w := &WMail{onEvent: func(models.JobEventType, any) error { return nil }} + w.rememberExecution(d) + changed := *d + changed.Sync = &models.AddWorkerEmailSyncData{Policy: models.SyncPolicy{DailyMessages: 99}, State: &models.SyncState{BackfillStatus: models.SyncBackfillComplete}} + if !w.SameExecution(&changed) { + t.Fatal("policy/checkpoint publication restarted provider") + } + g := *d.Google + changed.Google = &g + g.Token = &oauth2.Token{AccessToken: "fresh", RefreshToken: "rotated", Expiry: time.Now().Add(time.Hour)} + if w.SameExecution(&changed) { + t.Fatal("local reauthorization ignored") + } + if err := w.onTokenUpdate(g.Token); err != nil { + t.Fatal(err) + } + if !w.SameExecution(&changed) || !w.SameExecution(d) { + t.Fatal("routine token refresh/repeated original publication restarted provider") + } + for i := 0; i < 10; i++ { + if err := w.onTokenUpdate(&oauth2.Token{AccessToken: string(rune('a' + i)), RefreshToken: "rotated"}); err != nil { + t.Fatal(err) + } + } + if len(w.executionKeys) > 3 { + t.Fatal("refresh aliases grew without bound") + } + brokered := *d + brokered.Brokered = true + w.rememberExecution(&brokered) + changed.Brokered = true + if !w.SameExecution(&changed) { + t.Fatal("broker-owned token rotation restarted provider") + } + w.Stop() + w.onEvent = func(models.JobEventType, any) error { t.Fatal("stale token callback published"); return nil } + if err := w.onTokenUpdate(token); err != nil { + t.Fatal(err) + } +} + +func TestMailboxLifecycleLegacyDefaultsAndNormalizedSkipSettings(t *testing.T) { + d := &models.AddWorkerEmail{ID: uuid.New(), UserID: uuid.New(), Type: models.InboxProviderSMTPIMAP} + w := &WMail{} + w.rememberExecution(d) + copyData := *d + v := true + copyData.SaveToSent = &v + copyData.Sync = &models.AddWorkerEmailSyncData{Policy: models.SyncPolicy{DailyMessages: 99}} + if !w.SameExecution(©Data) { + t.Fatal("old backend missing sync/sent fields differs from effective defaults") + } + copyData.Sync.Policy.SkipFolders = []string{" Custom ", "CUSTOM", ""} + w.rememberExecution(©Data) + copyData.Sync = &models.AddWorkerEmailSyncData{Policy: models.SyncPolicy{SkipFolders: []string{"custom"}, DailyMessages: 120}} + if !w.SameExecution(©Data) { + t.Fatal("same effective folder skips restarted client") + } +} + +func TestMailboxLifecycleResumePreservesCompletedStateAndProviderCursors(t *testing.T) { + id, user := uuid.New(), uuid.New() + stamp := time.Now().Add(-time.Hour) + state := models.SyncState{BackfillStatus: models.SyncBackfillComplete, BackfillSynced: 12, BackfillCompletedAt: &stamp, LastSyncedAt: &stamp, Deferred: 5, BackfillCursor: models.SyncCursor{PageToken: "page", Folders: map[string]models.SyncFolderCursor{"Inbox": {UID: 8, Done: true}}}} + for _, provider := range []models.InboxProvider{models.InboxProviderGoogle, models.InboxProviderOutlook, models.InboxProviderSMTPIMAP} { + t.Run(string(provider), func(t *testing.T) { + old := &WMail{ID: id, UserID: user, EmailType: provider, tracker: newSyncTracker(&state, func(models.SyncState) error { return nil }), unmapPending: map[string]uuid.UUID{"pending": uuid.New()}} + d := &models.AddWorkerEmail{ID: id, UserID: user, Type: provider} + switch provider { + case models.InboxProviderGoogle: + old.GoogleData = &GoogleData{LastHistoryID: 91} + d.Google = &models.AddWorkerEmailGoogleData{LastHistoryID: 1} + case models.InboxProviderOutlook: + old.GraphData = &GraphData{Client: &msgraph.Client{DeltaLinks: map[string]string{"Inbox": "live"}}} + d.Graph = &models.AddWorkerEmailGraphData{DeltaLinks: map[string]string{"Inbox": "stale"}} + case models.InboxProviderSMTPIMAP: + old.SmtpImapData = &SmtpImapData{Mailboxes: []*models.Mailbox{{Name: "Inbox", UIDValidity: 4, UIDNext: 17}}} + d.SmtpImap = &models.AddWorkerEmailSmtpImapData{} + } + old.Stop() + old.Drain() + old.ResumeInto(d) + if d.Sync.State.BackfillStatus != models.SyncBackfillComplete || d.Sync.State.BackfillSynced != 12 || d.Sync.State.Deferred != 5 || d.Sync.State.LastSyncedAt != &stamp || d.Sync.State.BackfillCursor.PageToken != "page" { + t.Fatal("replacement reset completed/live state") + } + d.Sync.State.BackfillCursor.Folders["Inbox"] = models.SyncFolderCursor{} + if old.tracker.state.BackfillCursor.Folders["Inbox"].UID != 8 { + t.Fatal("replacement aliases retired mutable state") + } + if d.Google != nil && d.Google.LastHistoryID != 91 { + t.Fatal("Gmail cursor lost") + } + if d.Graph != nil && d.Graph.DeltaLinks["Inbox"] != "live" { + t.Fatal("Graph cursor lost") + } + if d.SmtpImap != nil && (len(d.SmtpImap.Mailboxes) != 1 || d.SmtpImap.Mailboxes[0].UIDNext != 17) { + t.Fatal("IMAP cursor lost") + } + next := &WMail{UserID: user, EmailType: provider, tracker: newSyncTracker(d.Sync.State, func(models.SyncState) error { return nil })} + old.tracker.dirty = true + next.PreservePending(old) + if next.unmapPending["pending"] != old.unmapPending["pending"] || !next.tracker.dirty { + t.Fatal("pending retries/state flush lost") + } + }) + } +} + +func TestMailboxLifecycleAuthTerminationClosesOnceAndIgnoresRetiredErrors(t *testing.T) { + c := &closingIMAP{} + events := 0 + w := &WMail{SmtpImapData: &SmtpImapData{ImapClient: c}, onEvent: func(models.JobEventType, any) error { events++; return nil }} + w.CaptureError(errx.ErrMailServerUnreachable) + if c.closes.Load() != 0 { + t.Fatal("transport timeout permanently retired mailbox") + } + w.CaptureError(&errx.MailError{Type: errx.MailErrorCritical, Code: errx.MailErrorCodeInvalidCredentials}) + w.CaptureError(&errx.MailError{Type: errx.MailErrorCritical, Code: errx.MailErrorCodeInvalidCredentials}) + if c.closes.Load() != 1 || events != 2 { + t.Fatalf("closes=%d events=%d", c.closes.Load(), events) + } +} diff --git a/internal/app/worker/wmail/provider_deadline_test.go b/internal/app/worker/wmail/provider_deadline_test.go new file mode 100644 index 000000000..50ec75bdc --- /dev/null +++ b/internal/app/worker/wmail/provider_deadline_test.go @@ -0,0 +1,91 @@ +package wmail + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" + "golang.org/x/oauth2" +) + +func TestMailboxProviderDeadlineReleasesPassAndAllowsRetry(t *testing.T) { + for _, provider := range []models.InboxProvider{models.InboxProviderGoogle, models.InboxProviderOutlook} { + for _, body := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/body=%v", provider, body), func(t *testing.T) { + var stalled atomic.Bool + stalled.Store(true) + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + rw.Header().Set("Content-Type", "application/json") + if stalled.Load() { + if body { + rw.WriteHeader(http.StatusOK) + _, _ = rw.Write([]byte(`{"`)) + rw.(http.Flusher).Flush() + } + select { + case <-release: + case <-r.Context().Done(): + } + return + } + if provider == models.InboxProviderGoogle { + _, _ = rw.Write([]byte(`{"historyId":"91","history":[]}`)) + } else { + _, _ = fmt.Fprintf(rw, `{"value":[],"@odata.deltaLink":%q}`, "https://graph.microsoft.com"+r.URL.Path+"?next=live") + } + })) + defer srv.Close() + defer close(release) + var events []captured + var w *WMail + ctx := context.WithValue(context.Background(), oauth2.HTTPClient, &http.Client{Timeout: 40 * time.Millisecond, Transport: rewriteToTestServer(srv.URL)}) + token := &oauth2.Token{AccessToken: "test", Expiry: time.Now().Add(time.Hour)} + if provider == models.InboxProviderGoogle { + w = newGoogleTestMail(t, srv, &events) + w.GoogleData.LastHistoryID = 91 + if err := w.GoogleData.Client.Init(ctx, token, oauth2.Config{}); err != nil { + t.Fatal(err) + } + w.googleReconciledAt = time.Now() + } else { + var states []models.SyncState + w = newGraphTestMail(t, srv, &events, &states) + if err := w.GraphData.Client.Init(ctx, token, oauth2.Config{}); err != nil { + t.Fatal(err) + } + } + w.tracker.state.BackfillStatus = models.SyncBackfillComplete + mailCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + w.Ctx, w.Cancel = mailCtx, cancel + result := make(chan *errx.MailError, 1) + go func() { result <- w.SyncMail(context.Background()) }() + select { + case err := <-result: + if err == nil { + t.Fatal("stalled provider pass reported success") + } + case <-time.After(time.Second): + stalled.Store(false) + srv.CloseClientConnections() + <-result + t.Fatal("provider request deadline did not release sync pass") + } + if ctx.Err() != nil || w.Ctx.Err() != nil { + t.Fatal("request timeout cancelled mailbox lifetime") + } + stalled.Store(false) + if err := w.SyncMail(context.Background()); err != nil { + t.Fatalf("later retry without reconnect failed: %v", err) + } + }) + } + } +} diff --git a/internal/app/worker/wmail/send.go b/internal/app/worker/wmail/send.go index 3a1eed281..c34b6a0a7 100644 --- a/internal/app/worker/wmail/send.go +++ b/internal/app/worker/wmail/send.go @@ -182,6 +182,13 @@ func providerThrottle(err *errx.MailError) bool { // Send attempts to send an email with retry for transient failures func (w *WMail) Send(ctx context.Context, req *SendRequest) *SendResult { + done, ok := w.BeginExecution() + if !ok { + return &SendResult{Error: errx.ErrMailServerUnreachable} + } + defer done() + ctx, cancel := w.ExecutionContext(ctx) + defer cancel() // For warmup emails, ensure HTML is empty bodyHTML := req.BodyHTML if req.IsWarmup { diff --git a/internal/app/worker/wmail/start_sync.go b/internal/app/worker/wmail/start_sync.go index 3c7755021..c4e6b452a 100644 --- a/internal/app/worker/wmail/start_sync.go +++ b/internal/app/worker/wmail/start_sync.go @@ -78,20 +78,26 @@ func (w *WMail) nextSyncDelay(base time.Duration, last *errx.MailError) time.Dur // so one mailbox's bad server response must not take down every other // account's sync and send loops. func (w *WMail) syncOnce(ctx context.Context) (result *errx.MailError) { + if w.tracker != nil { + w.tracker.tickComplete = false + } defer func() { if r := recover(); r != nil { + if w.tracker != nil { + w.tracker.tickComplete = false + } err := fmt.Errorf("mail sync panic: %v", r) w.CaptureError(err) log.Error().Err(err).Str("email_id", w.ID.String()).Msg("mail sync panicked") + result = &errx.MailError{Code: errx.MailErrorCodeImapUnknown, Type: errx.MailErrorWarning, Message: "mail sync pass interrupted"} } }() if err := w.SyncMail(ctx); err != nil { - // A server that is down answers every pass the same way. Report the - // first one and then stay quiet until it comes back, so one outage is - // one warning in the drawer rather than one a minute. + // Keep observations fresh; the consumer deduplicates user notifications. if isTransportError(err) { w.transportFailures++ if w.transportFailures > 1 { + w.CaptureError(err) log.Debug().Err(err).Str("email_id", w.ID.String()).Int("consecutive", w.transportFailures).Msg("mail server still unreachable") return err } @@ -100,7 +106,9 @@ func (w *WMail) syncOnce(ctx context.Context) (result *errx.MailError) { log.Warn().Err(err).Str("email_id", w.ID.String()).Msg("mail sync error") return err } - w.transportFailures = 0 + if w.tracker != nil && w.tracker.tickComplete { + w.transportFailures = 0 + } return nil } diff --git a/internal/app/worker/wmail/sync_google.go b/internal/app/worker/wmail/sync_google.go index 8b6b40fda..6db4d9061 100644 --- a/internal/app/worker/wmail/sync_google.go +++ b/internal/app/worker/wmail/sync_google.go @@ -29,7 +29,8 @@ func (w *WMail) SyncGoogle(ctx context.Context) *errx.MailError { stats := &tickStats{} w.googleTick = stats w.googleFolders = nil - defer w.endTick(stats) + complete := false + defer func() { w.endTick(ctx, stats, complete) }() if !w.retryUnmap(ctx) { return nil } @@ -40,7 +41,7 @@ func (w *WMail) SyncGoogle(ctx context.Context) *errx.MailError { return nil } - newHistoryID, err := w.GoogleData.Client.FetchHistory(ctx, w.GoogleData.LastHistoryID) + newHistoryID, caughtUp, err := w.GoogleData.Client.FetchHistoryPass(ctx, w.GoogleData.LastHistoryID) if errors.Is(err, goog.ErrHistoryExpired) { baseline, berr := w.GoogleData.Client.HistoryBaseline(ctx) if berr != nil { @@ -81,10 +82,11 @@ func (w *WMail) SyncGoogle(ctx context.Context) *errx.MailError { if !stats.aborted { // The pass's own work is done; only a refusal the owner has to act // on is worth failing it for. - if merr := w.googleReconcileFolders(ctx, time.Now(), stats); merr != nil && merr.Type == errx.MailErrorCritical { + if merr := w.googleReconcileFolders(ctx, time.Now(), stats); merr != nil { return merr } } + complete = caughtUp return nil } @@ -337,7 +339,13 @@ func (w *WMail) googleBackfill(ctx context.Context, stats *tickStats) *errx.Mail return nil } ids, next, err := w.GoogleData.Client.ListMessages(ctx, q, st.BackfillCursor.PageToken, googleBackfillPage) + if errors.Is(err, goog.ErrPageTokenExpired) { + st.BackfillCursor.PageToken = "" + w.tracker.mark() + return nil + } if err != nil { + stats.aborted = true var errMail *errx.MailError if errors.As(err, &errMail) { return errMail @@ -364,6 +372,7 @@ func (w *WMail) googleBackfill(ctx context.Context, stats *tickStats) *errx.Mail } msg, err := w.GoogleData.Client.GetMessage(ctx, id) if err != nil { + stats.aborted = true var errMail *errx.MailError if errors.As(err, &errMail) { return errMail @@ -436,6 +445,7 @@ func (w *WMail) googleReconcileFolders(ctx context.Context, now time.Time, stats for page := 0; page < config.GmailFolderReconcilePages; page++ { ids, next, err := w.GoogleData.Client.ListLabelMessages(ctx, goog.Inbox, q, token, 500) if err != nil { + stats.aborted = true return w.googleReconcileError(err) } for _, id := range ids { @@ -479,6 +489,7 @@ func (w *WMail) googleReconcileFolders(ctx context.Context, now time.Time, stats labels, found, err := w.GoogleData.Client.MessageLabels(ctx, m.ProviderID) if err != nil { if gmailRetryable(err) { + stats.aborted = true return w.googleReconcileError(err) } // One message Gmail refuses must not hold up every row after it. diff --git a/internal/app/worker/wmail/sync_google_test.go b/internal/app/worker/wmail/sync_google_test.go index 8aba918b4..d9205c0ae 100644 --- a/internal/app/worker/wmail/sync_google_test.go +++ b/internal/app/worker/wmail/sync_google_test.go @@ -2,9 +2,12 @@ package wmail import ( "context" + "encoding/json" "fmt" "net/http" "net/http/httptest" + "reflect" + "slices" "strings" "testing" "time" @@ -12,6 +15,7 @@ import ( "github.com/google/uuid" "github.com/warmbly/warmbly/internal/client/goog" "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" "golang.org/x/oauth2" ) @@ -154,3 +158,202 @@ func TestGoogleBackfillRetriesAfterATransientFailure(t *testing.T) { t.Errorf("listed %d times, want the one call that succeeded", g.listCalls) } } + +type googleBackfillBudget struct { + fixedBudget + policy models.SyncPolicy + lanes []SyncLane +} + +func (b *googleBackfillBudget) Policy() models.SyncPolicy { return normalizePolicy(b.policy) } + +func (b *googleBackfillBudget) Admit(ctx context.Context, lane SyncLane) Admission { + b.lanes = append(b.lanes, lane) + return b.fixedBudget.Admit(ctx, lane) +} + +func TestGoogleBackfillExpiredPageResumesAfterReloadWithinWindowAndBudgets(t *testing.T) { + var tokens, gets []string + since := time.Date(2026, 9, 1, 12, 0, 0, 0, time.UTC) + started := since.Add(30 * 24 * time.Hour) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if strings.HasSuffix(r.URL.Path, "/messages") { + token := r.URL.Query().Get("pageToken") + tokens = append(tokens, token) + if q := r.URL.Query().Get("q"); q != fmt.Sprintf("after:%d -in:trash -in:spam -in:chats", since.Unix()) { + t.Errorf("import window changed: q=%q", q) + } + if r.URL.Query().Get("includeSpamTrash") != "false" || r.URL.Query().Get("maxResults") != "100" { + t.Errorf("import listing changed: %s", r.URL) + } + switch token { + case "expired": + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":{"code":400,"message":"Invalid page token"}}`)) + case "": + _, _ = w.Write([]byte(`{"messages":[{"id":"known"},{"id":"new-1"},{"id":"new-2"}],"nextPageToken":"page-2"}`)) + case "page-2": + _, _ = w.Write([]byte(`{"messages":[{"id":"new-3"},{"id":"over-cap"}]}`)) + default: + t.Errorf("unexpected token %q", token) + } + return + } + id := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:] + gets = append(gets, id) + _, _ = fmt.Fprintf(w, `{"id":%q,"threadId":"t-%s","payload":{"headers":[{"name":"Message-Id","value":"<%s@gmail.test>"}]}}`, id, id, id) + })) + t.Cleanup(srv.Close) + var events []captured + w := newGoogleTestMail(t, srv, &events) + budget := &googleBackfillBudget{fixedBudget: fixedBudget{allow: 1}, policy: models.SyncPolicy{BackfillDays: 2, BackfillMessages: 4}} + w.gov = budget + mapping := &recoveryMessageMap{data: map[string]repository.EmailMessageData{"known": {ID: uuid.NewString(), MessageID: "known"}}} + w.EmailMessageMapRepository = mapping + w.tracker.state = models.SyncState{ + BackfillStatus: models.SyncBackfillRunning, BackfillSynced: 1, + BackfillSince: &since, BackfillStartedAt: &started, + BackfillCursor: models.SyncCursor{PageToken: "expired", Folders: map[string]models.SyncFolderCursor{}}, + Deferred: 5, LastSyncedAt: &started, + } + reload := func(state models.SyncState) { + t.Helper() + encoded, err := json.Marshal(state) + if err != nil { + t.Fatal(err) + } + var seed models.SyncState + if err := json.Unmarshal(encoded, &seed); err != nil { + t.Fatal(err) + } + id, userID := w.ID, w.UserID + w = newGoogleTestMail(t, srv, &events) + w.ID, w.UserID = id, userID + w.gov, w.EmailMessageMapRepository = budget, mapping + w.tracker = newSyncTracker(&seed, func(models.SyncState) error { return nil }) + } + reload(w.tracker.state) + want := w.tracker.state + want.BackfillCursor.PageToken = "" + if err := w.googleBackfill(t.Context(), &tickStats{}); err != nil { + t.Fatalf("expired continuation must reset for retry: %v", err) + } + if !reflect.DeepEqual(w.tracker.state, want) || !w.tracker.dirty || budget.admitted != 0 || len(events) != 0 || !slices.Equal(tokens, []string{"expired"}) { + t.Fatalf("reset changed more than the continuation: state=%+v tokens=%v budget=%+v", w.tracker.state, tokens, budget) + } + var relayed models.SyncState + w.tracker.emit = func(state models.SyncState) error { relayed = state; return nil } + w.tracker.flush(time.Now()) + reload(relayed) + if err := w.googleBackfill(t.Context(), &tickStats{}); err != nil { + t.Fatal(err) + } + if w.tracker.state.BackfillSynced != 2 || w.tracker.state.BackfillCursor.PageToken != "" || w.tracker.state.BackfillStatus != models.SyncBackfillRunning { + t.Fatalf("pacing must pin the restarted page: %+v", w.tracker.state) + } + reload(w.tracker.state) + if err := w.googleBackfill(t.Context(), &tickStats{}); err != nil { + t.Fatal(err) + } + if len(newEmails(events)) != 1 || budget.admitted != 1 { + t.Fatal("reload bypassed the admission budget or duplicated an arrival") + } + budget.allow = 3 + if err := w.googleBackfill(t.Context(), &tickStats{}); err != nil { + t.Fatal(err) + } + if w.tracker.state.BackfillStatus != models.SyncBackfillComplete || w.tracker.state.BackfillSynced != 4 || !w.tracker.state.BackfillSince.Equal(since) || !w.tracker.state.BackfillStartedAt.Equal(started) { + t.Fatalf("import changed the fixed window or count: %+v", w.tracker.state) + } + if !slices.Equal(gets, []string{"new-1", "new-2", "new-3"}) || len(newEmails(events)) != 3 || len(mapping.data) != 4 || budget.admitted != 3 || budget.observed != 0 { + t.Fatalf("known mail or capped mail was imported: gets=%v arrivals=%d budget=%+v", gets, len(newEmails(events)), budget) + } + for _, lane := range budget.lanes { + if lane != LaneBackfill { + t.Fatalf("import charged %s instead of backfill", lane) + } + } + reload(w.tracker.state) + if err := w.googleBackfill(t.Context(), &tickStats{}); err != nil { + t.Fatal(err) + } + if !slices.Equal(tokens, []string{"expired", "", "", "", "page-2"}) || len(newEmails(events)) != 3 { + t.Fatalf("completed reload repeated the import: tokens=%v arrivals=%d", tokens, len(newEmails(events))) + } +} + +func TestGoogleBackfillRetainsContinuationOnOtherProviderErrors(t *testing.T) { + for _, tc := range []struct { + name, message string + code int + }{ + {"invalid query", "Invalid query", 400}, + {"quota", "Invalid page token", 429}, + {"auth", "Invalid page token", 401}, + {"forbidden", "Invalid page token", 403}, + {"unavailable", "Invalid page token", 503}, + } { + t.Run(tc.name, func(t *testing.T) { + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if r.URL.Query().Get("pageToken") != "keep" { + t.Error("provider error reset the continuation") + } + w.WriteHeader(tc.code) + _, _ = fmt.Fprintf(w, `{"error":{"code":%d,"message":%q}}`, tc.code, tc.message) + })) + t.Cleanup(srv.Close) + var events []captured + w := newGoogleTestMail(t, srv, &events) + w.tracker.startBackfill(time.Now(), 30) + w.tracker.state.BackfillCursor.PageToken = "keep" + w.tracker.state.BackfillSynced = 7 + want := w.tracker.state + for range 2 { + stats := &tickStats{} + if err := w.googleBackfill(t.Context(), stats); err == nil || !stats.aborted { + t.Fatalf("provider error was swallowed: err=%v aborted=%t", err, stats.aborted) + } + if !reflect.DeepEqual(w.tracker.state, want) || len(events) != 0 { + t.Fatalf("provider failure changed import progress: %+v", w.tracker.state) + } + } + if calls != 2 { + t.Fatalf("requests=%d, want one per attempt", calls) + } + }) + } +} + +func TestGoogleBackfillCompletedOrCappedImportDoesNotRestartExpiredPage(t *testing.T) { + for _, tc := range []struct { + name string + status models.SyncBackfillStatus + synced int + }{ + {"completed below cap", models.SyncBackfillComplete, 1}, + {"running at cap", models.SyncBackfillRunning, 4}, + } { + t.Run(tc.name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { + t.Error("completed or capped import must not re-list an expired page") + })) + t.Cleanup(srv.Close) + var events []captured + w := newGoogleTestMail(t, srv, &events) + w.gov = &googleBackfillBudget{fixedBudget: fixedBudget{allow: 10}, policy: models.SyncPolicy{BackfillMessages: 4}} + w.tracker.startBackfill(time.Now(), 30) + w.tracker.state.BackfillStatus = tc.status + w.tracker.state.BackfillSynced = tc.synced + w.tracker.state.BackfillCursor.PageToken = "expired" + if err := w.googleBackfill(t.Context(), &tickStats{}); err != nil { + t.Fatal(err) + } + if w.tracker.state.BackfillStatus != models.SyncBackfillComplete || w.tracker.state.BackfillSynced != tc.synced || w.tracker.state.BackfillCursor.PageToken != "expired" || len(events) != 0 { + t.Fatalf("completed import changed: %+v", w.tracker.state) + } + }) + } +} diff --git a/internal/app/worker/wmail/sync_graph.go b/internal/app/worker/wmail/sync_graph.go index d34558db7..743a533ac 100644 --- a/internal/app/worker/wmail/sync_graph.go +++ b/internal/app/worker/wmail/sync_graph.go @@ -10,6 +10,7 @@ import ( "github.com/warmbly/warmbly/internal/client/msgraph" "github.com/warmbly/warmbly/internal/errx" "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" ) // graphBackfillPage is $top for one backfill listing. Messages come back @@ -25,11 +26,37 @@ func (w *WMail) SyncGraph(ctx context.Context) *errx.MailError { w.beginTick() stats := &tickStats{} w.graphTick = stats + complete := false + defer func() { w.endTick(ctx, stats, complete) }() if !w.retryUnmap(ctx) { return nil } + w.tracker.startBackfill(time.Now(), w.gov.Policy().BackfillDays) + w.GraphData.Client.OnRecoveryStart = w.graphRecoveryStart + w.GraphData.Client.OnRecoveryCheckpoint = w.graphRecoveryCheckpoint + w.GraphData.Client.OnRecoveryBaseline = w.graphRecoveryBaseline + w.GraphData.Client.PendingRecoveryLink = w.graphPendingRecoveryLink + w.GraphData.Client.Recovering = w.graphRecovering + if w.SyncContext != nil && w.GraphData.Client.ImmutableIDMode() { + done, err := w.graphUpgradeIDs(ctx) + if errors.Is(err, repository.ErrSyncContextUnsupported) { + w.GraphData.Client.SetImmutableIDMode(false) + w.CaptureError(err) + done, err = true, nil + } + if err != nil { + return w.graphSyncError(err) + } + if !done { + return nil + } + if w.GraphData.Client.ImmutableIDMode() { + w.GraphData.Client.OnFolderReconcile = w.graphReconcileFolder + } + } - if err := w.GraphData.Client.Sync(ctx); err != nil { + caughtUp, err := w.GraphData.Client.SyncPass(ctx) + if err != nil { var mailErr *errx.MailError if errors.As(err, &mailErr) { return mailErr @@ -42,7 +69,7 @@ func (w *WMail) SyncGraph(ctx context.Context) *errx.MailError { return merr } } - w.endTick(stats) + complete = caughtUp return nil } @@ -58,19 +85,29 @@ func (w *WMail) onGraphMessageSeen(ctx context.Context, folder, providerID strin if stats.aborted { return false, msgraph.ErrStop } - known, err := w.EmailMessageMapRepository.Get(ctx, w.UserID, w.ID, providerID) + known, err := w.graphMessageMap(ctx, providerID) if err != nil { w.controlPlaneError(err, stats) return false, msgraph.ErrStop } if known != nil { - return true, w.onGraphFlagsChange(ctx, providerID, seen) - } - if w.observeLive(ctx, []string{providerID}, stats) { - return false, msgraph.ErrStop + if !w.GraphData.Client.ImmutableIDMode() { + if err := w.onGraphFlagsChange(ctx, providerID, seen); err != nil { + return false, err + } + id, err := uuid.Parse(known.ID) + if err != nil { + return false, err + } + return true, w.emitFolder(id, (&msgraph.GraphMessage{}).ToEmailData(folder).Folder) + } + return true, w.graphReconcileMessage(ctx, known.ID, providerID, folder) } lane, cached := w.laneCache.get(providerID) + if (msgraph.IsRecovery(ctx) && lane == LaneLive) || (!msgraph.IsRecovery(ctx) && lane == LaneBackfill) { + cached = false + } var msg *models.EmailMessageData if !cached { full, err := w.GraphData.Client.FetchMessage(ctx, folder, providerID) @@ -81,8 +118,34 @@ func (w *WMail) onGraphMessageSeen(ctx context.Context, folder, providerID strin return true, nil // gone between the delta item and now } msg = full.ToEmailData(folder) - lane = w.laneOf(ctx, providerID, msg, false) + lane = w.laneFor(ctx, msg, false) + if msgraph.IsRecovery(ctx) && lane != LanePriority { + recovery, err := w.graphRecovery(folder) + if err != nil { + return false, err + } + if (!msg.InternalDate.IsZero() && msg.InternalDate.Before(recovery.Since)) || recovery.Count >= w.gov.Policy().BackfillMessages { + return true, nil + } + lane = LaneBackfill + } } + if lane == LaneBackfill && w.tracker.state.BackfillStatus != models.SyncBackfillComplete && w.tracker.state.BackfillSynced >= w.gov.Policy().BackfillMessages { + return true, nil + } + if lane == LaneBackfill && msgraph.IsRecovery(ctx) { + recovery, err := w.graphRecovery(folder) + if err != nil { + return false, err + } + if recovery.Count >= w.gov.Policy().BackfillMessages { + return true, nil + } + } + if lane == LaneLive && w.observeLive(ctx, []string{providerID}, stats) { + return false, msgraph.ErrStop + } + w.laneCache.put(providerID, lane) if !w.admit(ctx, lane, stats) { return false, nil } @@ -101,6 +164,18 @@ func (w *WMail) onGraphMessageSeen(ctx context.Context, folder, providerID strin w.controlPlaneError(err, stats) return false, msgraph.ErrStop } + if lane == LaneBackfill { + if w.tracker.state.BackfillStatus != models.SyncBackfillComplete { + w.tracker.state.BackfillSynced++ + w.tracker.mark() + } + recovery, err := w.graphRecovery(folder) + if err != nil { + return false, err + } + recovery.Count++ + w.setGraphRecovery(folder, recovery) + } return true, nil } @@ -186,6 +261,7 @@ func (w *WMail) graphBackfill(ctx context.Context, stats *tickStats) *errx.MailE return mailErr } w.CaptureError(err) + stats.aborted = true return nil } for _, full := range msgs { diff --git a/internal/app/worker/wmail/sync_graph_reconcile.go b/internal/app/worker/wmail/sync_graph_reconcile.go new file mode 100644 index 000000000..45d4af032 --- /dev/null +++ b/internal/app/worker/wmail/sync_graph_reconcile.go @@ -0,0 +1,275 @@ +package wmail + +import ( + "context" + "encoding/json" + "errors" + "time" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/client/msgraph" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +const graphIdentityCursor = "graph:immutable-ids" + +type graphDeltaRecovery struct { + Version int `json:"version"` + Since time.Time `json:"since"` + Count int `json:"count"` + Baseline string `json:"baseline,omitempty"` + PendingLink string `json:"pending_link,omitempty"` +} + +func (w *WMail) graphRecovery(folder string) (graphDeltaRecovery, error) { + var recovery graphDeltaRecovery + data := w.tracker.folder("graph:delta-recovery:" + folder).Next + if data == "" { + return recovery, nil + } + if err := json.Unmarshal([]byte(data), &recovery); err != nil { + return recovery, err + } + if recovery.Version != 1 || recovery.Since.IsZero() || recovery.Count < 0 { + return recovery, errors.New("graph: unsupported recovery state") + } + return recovery, nil +} + +func (w *WMail) setGraphRecovery(folder string, recovery graphDeltaRecovery) { + data, _ := json.Marshal(recovery) + w.tracker.setFolder("graph:delta-recovery:"+folder, models.SyncFolderCursor{Next: string(data)}) +} + +func (w *WMail) graphRecovering(folder string) bool { + recovery, err := w.graphRecovery(folder) + if err != nil { + return true + } + key := "graph:delta-recovery:" + folder + cur := w.tracker.folder(key) + if recovery.Version == 0 || cur.Done { + return false + } + if recovery.Baseline != "" && recovery.Baseline == w.GraphData.Client.DeltaLinks[folder] { + cur.Done = true + w.tracker.setFolder(key, cur) + return false + } + return true +} + +func (w *WMail) graphPendingRecoveryLink(folder string) (string, error) { + recovery, err := w.graphRecovery(folder) + if err != nil { + return "", err + } + if w.graphRecovering(folder) { + return recovery.PendingLink, nil + } + return "", nil +} + +func (w *WMail) graphRecoveryBaseline(ctx context.Context, folder, link string) error { + recovery, err := w.graphRecovery(folder) + if err != nil { + return err + } + recovery.PendingLink = link + w.setGraphRecovery(folder, recovery) + return w.tracker.emit(w.tracker.state) +} + +func (w *WMail) graphRecoveryStart(ctx context.Context, folder string) error { + recovery, err := w.graphRecovery(folder) + if err != nil { + return err + } + if recovery.Version == 0 || !w.graphRecovering(folder) { + since := time.Now().Add(-time.Duration(w.gov.Policy().BackfillDays) * 24 * time.Hour) + if w.tracker.state.BackfillStatus != models.SyncBackfillComplete && w.tracker.state.BackfillSince != nil { + since = *w.tracker.state.BackfillSince + } + recovery = graphDeltaRecovery{Version: 1, Since: since} + } + recovery.Baseline = "" + w.setGraphRecovery(folder, recovery) + w.tracker.clearFolder("graph:reconcile:" + folder) + return w.tracker.emit(w.tracker.state) +} + +func (w *WMail) graphRecoveryCheckpoint(ctx context.Context, folder, link string) error { + recovery, err := w.graphRecovery(folder) + if err != nil { + return err + } + recovery.Baseline = link + w.setGraphRecovery(folder, recovery) + return w.tracker.emit(w.tracker.state) +} + +func (w *WMail) graphMessageMap(ctx context.Context, providerID string) (*repository.EmailMessageData, error) { + known, err := w.EmailMessageMapRepository.Get(ctx, w.UserID, w.ID, providerID) + if err != nil || known != nil || !w.GraphData.Client.ImmutableIDMode() { + return known, err + } + ids, err := w.GraphData.Client.RegularMessageIDs(ctx, []string{providerID}) + if err != nil { + return nil, err + } + known, err = w.EmailMessageMapRepository.Get(ctx, w.UserID, w.ID, ids[providerID]) + if err != nil || known == nil { + return known, err + } + alias := *known + alias.MessageID = providerID + if err := w.graphAddAlias(ctx, alias); err != nil { + return nil, err + } + return &alias, nil +} + +func (w *WMail) graphAddAlias(ctx context.Context, alias repository.EmailMessageData) error { + existing, err := w.EmailMessageMapRepository.Get(ctx, w.UserID, w.ID, alias.MessageID) + if err != nil { + return err + } + if existing != nil { + if existing.ID != alias.ID { + return errors.New("graph: conflicting immutable message mapping") + } + return nil + } + if err := w.EmailMessageMapRepository.Add(ctx, alias); err != nil { + return err + } + stored, err := w.EmailMessageMapRepository.Get(ctx, w.UserID, w.ID, alias.MessageID) + if err != nil { + return err + } + if stored == nil || stored.ID != alias.ID { + return errors.New("graph: immutable message mapping not retained") + } + return nil +} + +func (w *WMail) graphSyncError(err error) *errx.MailError { + var merr *errx.MailError + if errors.As(err, &merr) { + return merr + } + w.CaptureError(err) + return nil +} + +func (w *WMail) graphStoredPage(ctx context.Context, key string) ([]repository.ProviderFolderMessage, error) { + var after *uuid.UUID + if next := w.tracker.folder(key).Next; next != "" { + id, err := uuid.Parse(next) + if err != nil { + return nil, err + } + after = &id + } + return w.SyncContext.ListProviderMessages(ctx, w.UserID, w.ID, after, graphBackfillPage) +} + +func (w *WMail) graphUpgradeIDs(ctx context.Context) (bool, error) { + if w.tracker.folder(graphIdentityCursor).Done { + return true, nil + } + rows, err := w.graphStoredPage(ctx, graphIdentityCursor) + if err != nil { + return false, err + } + ids := make([]string, 0, len(rows)) + for _, row := range rows { + ids = append(ids, row.ProviderID) + } + var converted map[string]string + if len(ids) > 0 { + converted, err = w.GraphData.Client.ImmutableMessageIDs(ctx, ids) + if err != nil { + return false, err + } + } + cur := w.tracker.folder(graphIdentityCursor) + for _, row := range rows { + id := converted[row.ProviderID] + known, err := w.EmailMessageMapRepository.Get(ctx, w.UserID, w.ID, row.ProviderID) + if err != nil { + return false, err + } + if known == nil { + return false, errors.New("graph: saved message mapping missing during id upgrade") + } + alias := *known + alias.MessageID = id + if err := w.graphAddAlias(ctx, alias); err != nil { + return false, err + } + // Relayed placement updates only the provider id, preserving customer filing. + if err := w.onEvent(models.JobEventTypeFolderUpdate, &models.JobEventFolderUpdate{ + UserID: w.UserID, EmailID: w.ID, ID: row.ID, ProviderID: id, + Folder: row.ProviderFolder, Relayed: true, + }); err != nil { + return false, err + } + cur.Next = row.ID.String() + w.tracker.setFolder(graphIdentityCursor, cur) + } + cur.Done = len(rows) < graphBackfillPage + w.tracker.setFolder(graphIdentityCursor, cur) + w.tracker.flush(time.Now()) + return cur.Done, nil +} + +func (w *WMail) graphReconcileMessage(ctx context.Context, internalID, providerID, fallback string) error { + id, err := uuid.Parse(internalID) + if err != nil { + return err + } + full, err := w.GraphData.Client.FetchMessage(ctx, fallback, providerID) + if err != nil { + return err + } + if full == nil { + return w.onEvent(models.JobEventTypeRemoveEmail, &models.JobEventRemoveEmail{UserID: w.UserID, EmailID: w.ID, ID: id}) + } + folder, err := w.GraphData.Client.MessageFolder(ctx, full, fallback) + if err != nil { + return err + } + if folder != "" { + if err := w.emitFolder(id, folder); err != nil { + return err + } + } + return w.onGraphFlagsChange(ctx, providerID, full.IsRead) +} + +func (w *WMail) graphReconcileFolder(ctx context.Context, folder string) (bool, error) { + key := "graph:reconcile:" + folder + rows, err := w.graphStoredPage(ctx, key) + if err != nil { + return false, err + } + canonical := (&msgraph.GraphMessage{}).ToEmailData(folder).Folder + cur := w.tracker.folder(key) + for _, row := range rows { + if row.ProviderFolder == canonical { + if err := w.graphReconcileMessage(ctx, row.ID.String(), row.ProviderID, folder); err != nil { + return false, err + } + } + cur.Next = row.ID.String() + w.tracker.setFolder(key, cur) + } + done := len(rows) < graphBackfillPage + if done { + w.tracker.clearFolder(key) + } + return done, nil +} diff --git a/internal/app/worker/wmail/sync_graph_test.go b/internal/app/worker/wmail/sync_graph_test.go index c4f7e6507..eb026e8bc 100644 --- a/internal/app/worker/wmail/sync_graph_test.go +++ b/internal/app/worker/wmail/sync_graph_test.go @@ -156,8 +156,7 @@ func newGraphTestMail(t *testing.T, srv *httptest.Server, events *[]captured, re return nil }) - // Seeded cursors: a folder with no delta link is primed, not imported, so - // without these the live half of the pass would never run. + // Seeded cursors model a mailbox already following live deltas. deltaLinks := map[string]string{} for _, folder := range msgraph.TrackedFolders { deltaLinks[folder] = "https://graph.microsoft.com/v1.0/me/mailFolders/" + folder + "/messages/delta?$deltatoken=seed" diff --git a/internal/app/worker/wmail/sync_imap.go b/internal/app/worker/wmail/sync_imap.go index c775013ff..0510e5772 100644 --- a/internal/app/worker/wmail/sync_imap.go +++ b/internal/app/worker/wmail/sync_imap.go @@ -32,6 +32,8 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError { } w.beginTick() stats := &tickStats{} + complete := false + defer func() { w.endTick(ctx, stats, complete) }() if !w.retryUnmap(ctx) { return nil } @@ -74,13 +76,12 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError { return nil } - // condStore decides the incremental strategy for the whole account: - // mod-sequences where the server has CONDSTORE, UIDNEXT where it does not - // (Outlook.com, Microsoft 365 over IMAP, Yahoo, many hosted servers). - condStore := client.HasCondStore() + // CONDSTORE is effective per folder; NOMODSEQ uses UIDNEXT and periodic flags. + caughtUp := true for i := range folders { box := &folders[i] + condStore := client.HasCondStore() && box.HighestModSeq > 0 // The listing is remembered before anything is decided from it, so // the departure check below reads the previous pass, never this one. prevListing, listedBefore := w.listed[box.Name] @@ -90,6 +91,10 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError { // First sight: baseline. Live sync starts from this cursor; the // backfill owns everything before it. saved := *box + if !w.imapRecoverFolder(box) { + stats.aborted = true + return nil + } if err := w.mboxEvent(&saved); err != nil { return nil } @@ -102,20 +107,35 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError { // snapshot is about messages that may no longer be there. Re-baseline // it exactly like a first sighting, and drop its backfill floor so an // import still running walks it again (stored messages are matched by - // Message-ID, so nothing is stored twice). A finished import stays - // finished; the mail that is already here keeps its rows, and its - // stale UIDs are what the warmup path checks the generation against. + // Message-ID, so nothing is stored twice). Completed accounts schedule + // a separate bounded folder recovery without resetting global history. if befBox.UIDValidity != box.UIDValidity { saved := *box + if !w.imapRecoverFolder(box) { + stats.aborted = true + return nil + } if err := w.mboxEvent(&saved); err != nil { return nil } *befBox = saved delete(w.flagScan, box.Name) - w.tracker.setFolder(box.Name, models.SyncFolderCursor{}) continue } + var selected *imap.Selected + if condStore && !stats.aborted { + view, err := client.SelectForSyncState(box.Name) + if err != nil { + return err + } + selected = &view + condStore = view.HighestModSeq > 0 + if view.UIDValidity != 0 && view.UIDValidity != box.UIDValidity { + caughtUp = false + continue + } + } changed := imapFolderChanged(befBox, box, condStore) // A pass cut short by the search cap left rows unexamined, so the // next pass looks again whether or not the count moved. @@ -125,17 +145,21 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError { var view imap.Selected if changed && !stats.aborted { w.setWalking(box) - done, sel, ids, err := w.imapIncremental(ctx, box, befBox, condStore, stats) + done, sel, ids, err := w.imapIncremental(ctx, box, befBox, condStore, stats, selected) if err != nil { return err } fullyProcessed = done view = sel + condStore = condStore && view.HighestModSeq > 0 touched = ids } else if changed { // The pass was aborted before this folder; hold its cursor too. fullyProcessed = false } + if !fullyProcessed { + caughtUp = false + } if changed || !slices.Equal(befBox.Attrs, box.Attrs) { // The stored cursor only moves once every change up to it was @@ -233,12 +257,15 @@ outer: } if !stats.aborted { + if err := w.imapRecoverFolders(ctx, folders, stats); err != nil { + return err + } if err := w.imapBackfill(ctx, folders, stats); err != nil { return err } } - w.endTick(stats) + complete = caughtUp && !w.imapRecoveryPending(folders) return nil } @@ -455,7 +482,7 @@ const imapSkipSearchesPerPass = 50 // AND flag changes; without it only arrivals are visible here, and flag // changes are picked up by the periodic scan in imapIncremental. func imapFolderChanged(before, now *models.Mailbox, condStore bool) bool { - if condStore { + if condStore && before.HighestModSeq > 0 && now.HighestModSeq > 0 { return before.HighestModSeq != now.HighestModSeq } return before.UIDNext != now.UIDNext @@ -467,11 +494,17 @@ func imapFolderChanged(before, now *models.Mailbox, condStore bool) bool { // folder's cursor advance, the selected view the search ran against, which is // where it advances to, and the Message-IDs it fetched so the drafts // reconciliation can tell a re-appended draft from an expunged one. -func (w *WMail) imapIncremental(ctx context.Context, box, before *models.Mailbox, condStore bool, stats *tickStats) (bool, imap.Selected, map[string]struct{}, *errx.MailError) { +func (w *WMail) imapIncremental(ctx context.Context, box, before *models.Mailbox, condStore bool, stats *tickStats, selected *imap.Selected) (bool, imap.Selected, map[string]struct{}, *errx.MailError) { client := w.SmtpImapData.ImapClient - view, err := client.SelectForSyncState(box.Name) - if err != nil { - return false, view, nil, err + var view imap.Selected + var err *errx.MailError + if selected != nil { + view = *selected + } else { + view, err = client.SelectForSyncState(box.Name) + if err != nil { + return false, view, nil, err + } } // The listing and this view name different generations, so the search // would answer about UIDs the cursor does not; the next pass re-baselines. @@ -482,7 +515,7 @@ func (w *WMail) imapIncremental(ctx context.Context, box, before *models.Mailbox return true, view, nil, nil } var uids []goimap.UID - if condStore { + if condStore && before.HighestModSeq > 0 && view.HighestModSeq > 0 { uids, err = client.SearchChangedSince(before.HighestModSeq) } else { uids, err = client.SearchNewSince(before.UIDNext) @@ -509,7 +542,7 @@ func (w *WMail) imapIncremental(ctx context.Context, box, before *models.Mailbox if err != nil { return false, view, touched, err } - done, err := w.imapApply(ctx, fetched, false, stats) + done, err := w.imapApply(ctx, fetched, false, stats, nil) if err != nil { return false, view, touched, err } @@ -608,22 +641,37 @@ func (w *WMail) imapReconcileDrafts(ctx context.Context, box *models.Mailbox, to // unknown ones are stored if their lane admits them. backfill selects the // backfill lane and skips flood accounting. Returns whether every unknown // message in the batch was stored. -func (w *WMail) imapApply(ctx context.Context, fetched []*imap.Fetched, backfill bool, stats *tickStats) (bool, *errx.MailError) { +func (w *WMail) imapApply(ctx context.Context, fetched []*imap.Fetched, backfill bool, stats *tickStats, recovery *imapFolderRecovery) (bool, *errx.MailError) { sort.Slice(fetched, func(i, j int) bool { return fetched[i].Email.UID > fetched[j].Email.UID }) var fresh []*imap.Fetched for _, f := range fetched { + if ctx.Err() != nil || stats.aborted { + return false, nil + } w.ensureMessageKey(f.Email) internal, err := w.EmailMessageMapRepository.Get(ctx, w.UserID, w.ID, f.Email.MessageID) if err != nil { return false, w.controlPlaneError(err, stats) } if internal == nil { - fresh = append(fresh, f) + if recovery != nil && recovery.KnownOnly { + w.imapImportCursor(w.SmtpImapData.folderPath, f.Email.UID, false, recovery) + continue + } + if recovery == nil { + fresh = append(fresh, f) + continue + } + done, err := w.imapStoreFetched(ctx, f, backfill, stats, recovery) + if err != nil || !done { + return false, err + } + w.imapImportCursor(w.SmtpImapData.folderPath, f.Email.UID, false, recovery) continue } - if backfill { - // The backfill only cares about what it has not stored. + if backfill && recovery == nil { + // Initial history skips known maps; generation recovery also reconciles them. continue } internalID, perr := uuid.Parse(internal.ID) @@ -643,6 +691,9 @@ func (w *WMail) imapApply(ctx context.Context, fetched []*imap.Fetched, backfill }); err != nil { return false, w.controlPlaneError(err, stats) } + if recovery != nil { + w.imapImportCursor(w.SmtpImapData.folderPath, f.Email.UID, false, recovery) + } } if !backfill && len(fresh) > 0 { @@ -655,36 +706,58 @@ func (w *WMail) imapApply(ctx context.Context, fetched []*imap.Fetched, backfill } } - policy := w.gov.Policy() all := true for _, f := range fresh { - if stats.aborted { - return false, nil + done, err := w.imapStoreFetched(ctx, f, backfill, stats, recovery) + if err != nil { + return false, err } - if backfill && w.tracker.state.BackfillSynced >= policy.BackfillMessages { - return false, nil - } - if !w.admit(ctx, w.laneOf(ctx, f.Email.MessageID, f.Email, backfill), stats) { + if !done { all = false - if backfill { + if backfill || stats.aborted || ctx.Err() != nil { return false, nil } continue } - w.SmtpImapData.ImapClient.FetchBody(f) - if err := w.imapStore(ctx, f.Email); err != nil { - return false, w.controlPlaneError(err, stats) - } - w.laneCache.forget(f.Email.MessageID) if backfill { - w.tracker.state.BackfillSynced++ - w.tracker.mark() - w.tracker.setFolder(w.SmtpImapData.folderPath, models.SyncFolderCursor{UID: f.Email.UID}) + w.imapImportCursor(w.SmtpImapData.folderPath, f.Email.UID, false, recovery) } } return all, nil } +func (w *WMail) imapStoreFetched(ctx context.Context, f *imap.Fetched, backfill bool, stats *tickStats, recovery *imapFolderRecovery) (bool, *errx.MailError) { + if stats.aborted || ctx.Err() != nil { + return false, nil + } + if backfill && w.imapImportCount(recovery) >= w.gov.Policy().BackfillMessages { + // A capped initial import still reconciles known handles, without admitting more history. + return recovery != nil && recovery.Initial, nil + } + if !w.admit(ctx, w.laneOf(ctx, f.Email.MessageID, f.Email, backfill), stats) { + return false, nil + } + if err := w.SmtpImapData.ImapClient.FetchBody(f); err != nil { + if err.Code == errx.MailErrorCodeNotFound { + return true, nil + } + return false, err + } + if err := w.imapStore(ctx, f.Email); err != nil { + return false, w.controlPlaneError(err, stats) + } + w.laneCache.forget(f.Email.MessageID) + if backfill { + if recovery != nil && !recovery.Initial { + recovery.Synced++ + } else { + w.tracker.state.BackfillSynced++ + w.tracker.mark() + } + } + return true, nil +} + // ensureMessageKey gives a message without a Message-ID header one that is // stable for this mailbox, because the empty string is not a key: the map // endpoint refuses it with 400 and the failed lookup ends the whole sync pass @@ -824,6 +897,10 @@ func (w *WMail) imapBackfill(ctx context.Context, folders []models.Mailbox, stat } key := box.Name cur := w.tracker.folder(key) + if imapRecoveryCursor(cur) != nil { + allDone = allDone && cur.Done + continue + } if cur.Done { continue } @@ -864,7 +941,7 @@ func (w *WMail) imapBackfill(ctx context.Context, folders []models.Mailbox, stat if err != nil { return err } - done, err := w.imapApply(ctx, fetched, true, stats) + done, err := w.imapApply(ctx, fetched, true, stats, nil) if err != nil { return err } diff --git a/internal/app/worker/wmail/sync_imap_flags.go b/internal/app/worker/wmail/sync_imap_flags.go index d870fbfa2..3145fcaba 100644 --- a/internal/app/worker/wmail/sync_imap_flags.go +++ b/internal/app/worker/wmail/sync_imap_flags.go @@ -73,7 +73,8 @@ func (w *WMail) imapScanFlags(ctx context.Context, box *models.Mailbox, stats *t return err } if stats.aborted { - break + scan.at = now + return nil } } } diff --git a/internal/app/worker/wmail/sync_imap_recovery.go b/internal/app/worker/wmail/sync_imap_recovery.go new file mode 100644 index 000000000..e98f47778 --- /dev/null +++ b/internal/app/worker/wmail/sync_imap_recovery.go @@ -0,0 +1,166 @@ +package wmail + +import ( + "context" + "encoding/json" + "sort" + "strings" + "time" + + goimap "github.com/emersion/go-imap/v2" + "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" +) + +const imapRecoveryPrefix = "imap-recovery-v1:" +const imapInitialRecoveryPrefix = "imap-initial-generation-v1:" + +// Recovery metadata lives in the existing opaque folder continuation, not a new wire field. +type imapFolderRecovery struct { + Generation uint32 `json:"generation"` + Since time.Time `json:"since"` + Synced int `json:"synced"` + Initial bool `json:"-"` + KnownOnly bool `json:"-"` +} + +func imapRecoveryCursor(cur models.SyncFolderCursor) *imapFolderRecovery { + raw, ok := strings.CutPrefix(cur.Next, imapRecoveryPrefix) + initial := false + if !ok { + raw, ok = strings.CutPrefix(cur.Next, imapInitialRecoveryPrefix) + initial = ok + } + if !ok { + return nil + } + var recovery imapFolderRecovery + if json.Unmarshal([]byte(raw), &recovery) != nil || recovery.Generation == 0 || recovery.Since.IsZero() || recovery.Synced < 0 { + return nil + } + recovery.Initial = initial + return &recovery +} + +func (w *WMail) imapRecoverFolder(box *models.Mailbox) bool { + cur := w.tracker.folder(box.Name) + if recovery := imapRecoveryCursor(cur); recovery == nil || recovery.Generation != box.UIDValidity { + since := time.Now().Add(-time.Duration(w.gov.Policy().BackfillDays) * 24 * time.Hour) + initial := w.tracker.state.BackfillStatus != models.SyncBackfillComplete + if initial { + w.tracker.startBackfill(time.Now(), w.gov.Policy().BackfillDays) + if w.tracker.state.BackfillSince == nil { + return false + } + since = *w.tracker.state.BackfillSince + } + w.imapImportCursor(box.Name, 0, false, &imapFolderRecovery{Generation: box.UIDValidity, Since: since, Initial: initial}) + } + // Publish replay state before a live baseline can make the folder look caught up on reload. + w.tracker.flush(time.Now()) + return !w.tracker.dirty +} + +func (w *WMail) imapRecoveryPending(folders []models.Mailbox) bool { + for _, box := range folders { + cur := w.tracker.folder(box.Name) + if !cur.Done && imapRecoveryCursor(cur) != nil { + return true + } + } + return false +} + +func (w *WMail) imapImportCount(recovery *imapFolderRecovery) int { + if recovery != nil && !recovery.Initial { + return recovery.Synced + } + return w.tracker.state.BackfillSynced +} + +func (w *WMail) imapImportCursor(folder string, uid uint32, done bool, recovery *imapFolderRecovery) { + cur := models.SyncFolderCursor{UID: uid, Done: done} + if recovery != nil { + raw, _ := json.Marshal(recovery) + prefix := imapRecoveryPrefix + if recovery.Initial { + // Older workers must not interpret the global import cap as a per-folder recovery cap. + prefix = imapInitialRecoveryPrefix + } + cur.Next = prefix + string(raw) + } + w.tracker.setFolder(folder, cur) +} + +// Generation recovery uses its original window and the owning import's counters. +func (w *WMail) imapRecoverFolders(ctx context.Context, folders []models.Mailbox, stats *tickStats) *errx.MailError { + client := w.SmtpImapData.ImapClient + for i := range folders { + box := &folders[i] + cur := w.tracker.folder(box.Name) + recovery := imapRecoveryCursor(cur) + if recovery == nil { + continue + } + if recovery.Generation != box.UIDValidity { + if !w.imapRecoverFolder(box) { + stats.aborted = true + return nil + } + cur = w.tracker.folder(box.Name) + recovery = imapRecoveryCursor(cur) + if recovery == nil { + continue + } + } + if cur.Done { + continue + } + recovery.KnownOnly = !imapBackfillEligible(box) + if ctx.Err() != nil || stats.aborted || stats.laneDenied(LaneBackfill) { + return nil + } + if !recovery.Initial && !recovery.KnownOnly && recovery.Synced >= w.gov.Policy().BackfillMessages { + w.imapImportCursor(box.Name, cur.UID, true, recovery) + continue + } + w.setWalking(box) + view, err := client.SelectForSyncState(box.Name) + if err != nil { + return err + } + if view.UIDValidity != 0 && view.UIDValidity != recovery.Generation { + return errx.ErrMailResourceNotFound + } + uids, err := client.SearchSince(recovery.Since) + if err != nil { + return err + } + remaining := uids[:0:0] + for _, uid := range uids { + if cur.UID == 0 || uid < goimap.UID(cur.UID) { + remaining = append(remaining, uid) + } + } + sort.Slice(remaining, func(i, j int) bool { return remaining[i] > remaining[j] }) + for lo := 0; lo < len(remaining); lo += config.ImapFetchBatchSize { + hi := min(lo+config.ImapFetchBatchSize, len(remaining)) + fetched, err := client.FetchEnvelopes(ctx, remaining[lo:hi]) + if err != nil { + return err + } + done, err := w.imapApply(ctx, fetched, true, stats, recovery) + if err != nil { + return err + } + if !done || ctx.Err() != nil { + return nil + } + cur.UID = uint32(remaining[hi-1]) + w.imapImportCursor(box.Name, cur.UID, false, recovery) + } + w.imapImportCursor(box.Name, cur.UID, true, recovery) + } + return nil +} diff --git a/internal/app/worker/wmail/sync_imap_recovery_test.go b/internal/app/worker/wmail/sync_imap_recovery_test.go new file mode 100644 index 000000000..9b9e128e6 --- /dev/null +++ b/internal/app/worker/wmail/sync_imap_recovery_test.go @@ -0,0 +1,529 @@ +package wmail + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + "time" + + goimap "github.com/emersion/go-imap/v2" + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/client/smtpimap/imap" + "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +type knownRecoveryConn struct { + *fakeImapConn + message models.EmailMessageData +} + +func TestImapNonHistoryGenerationReconcilesOnlyKnownMessagesAcrossReload(t *testing.T) { + for _, attr := range []string{"\\Junk", "\\Trash"} { + for _, initial := range []bool{false, true} { + for _, fail := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/initial=%t/relayFailure=%t", attr, initial, fail), func(t *testing.T) { + box := models.Mailbox{Name: attr, Attrs: []string{attr}, UIDValidity: 9, UIDNext: 4, HighestModSeq: 5} + conn := &initialRecoveryConn{folderIdentityRecoveryConn: &folderIdentityRecoveryConn{fakeImapConn: &fakeImapConn{ + folders: []models.Mailbox{box}, all: uidRange(3), changed: uidRange(3), + }}} + budget := &initialRecoveryBudget{fixedBudget: fixedBudget{allow: 0}, limit: 2} + w, events := newIMAPTestMail(conn, budget, &models.Mailbox{Name: attr, Attrs: box.Attrs, UIDValidity: 8, UIDNext: 91, HighestModSeq: 5}) + knownID := uuid.NewString() + maps := &recoveryMessageMap{data: map[string]repository.EmailMessageData{ + "<2@fake.test>": {ID: knownID, MessageID: "<2@fake.test>"}, + "<3@fake.test>": {MessageID: "<3@fake.test>"}, + }} + w.EmailMessageMapRepository = maps + since, success := time.Now().Add(-time.Hour), time.Now().Add(-time.Minute) + w.tracker.state.BackfillSynced = 2 + w.tracker.state.LastSyncedAt = &success + if initial { + w.tracker.state.BackfillStatus = models.SyncBackfillRunning + w.tracker.state.BackfillSince = &since + } + if fail { + emit := w.onEvent + w.onEvent = func(kind models.JobEventType, body any) error { + if kind == models.JobEventTypeEmailUpdate { + return errors.New("known metadata relay failed") + } + return emit(kind, body) + } + } + if err := w.Sync(t.Context()); err != nil { + t.Fatal(err) + } + all := append([]captured{}, (*events)...) + if fail && (w.tracker.folder(attr).Done || !w.tracker.state.LastSyncedAt.Equal(success)) { + t.Fatal("failed metadata relay advanced recovery or last successful check") + } + w, events = imapReload(t, w, conn, budget) + for pass := 0; pass < 3; pass++ { + if err := w.Sync(t.Context()); err != nil { + t.Fatal(err) + } + } + all = append(all, (*events)...) + updates := mailboxEvents(all, models.JobEventTypeEmailUpdate) + if len(updates) != 1 { + t.Fatalf("known reconciliation count: %d", len(updates)) + } + update := updates[0].body.(*models.JobEventEmailUpdate) + if update.ID.String() != knownID || update.UID != 2 || update.Mailbox != 9 || !models.SeenFromFlags(update.Flags) { + t.Fatal("known message identity, UID, generation or read state lost") + } + if len(mailboxEvents(all, models.JobEventTypeNewEmail)) != 0 || len(conn.bodies) != 0 || len(maps.data) != 2 || w.tracker.state.BackfillSynced != 2 || !w.tracker.folder(attr).Done { + t.Fatal("non-history recovery imported unseen/hidden mail, changed counters or did not complete") + } + }) + } + } + } +} + +type initialRecoveryConn struct { + *folderIdentityRecoveryConn + since []time.Time + bodies []string +} + +func (c *initialRecoveryConn) SearchSince(since time.Time) ([]goimap.UID, *errx.MailError) { + c.since = append(c.since, since) + return c.fakeImapConn.SearchSince(since) +} + +func (c *initialRecoveryConn) FetchBody(f *imap.Fetched) *errx.MailError { + c.bodies = append(c.bodies, f.Email.MessageID) + return c.fakeImapConn.FetchBody(f) +} + +type initialRecoveryBudget struct { + fixedBudget + limit int +} + +func (b *initialRecoveryBudget) Policy() models.SyncPolicy { + policy := b.fixedBudget.Policy() + policy.BackfillMessages = b.limit + return policy +} + +func newRunningRecoveryMail(t *testing.T, budget syncBudget) (*WMail, *[]captured, *initialRecoveryConn, *recoveryMessageMap) { + t.Helper() + conn := &initialRecoveryConn{folderIdentityRecoveryConn: &folderIdentityRecoveryConn{fakeImapConn: &fakeImapConn{ + folders: []models.Mailbox{{Name: "INBOX", UIDValidity: 9, UIDNext: 4, HighestModSeq: 5}}, + all: uidRange(3), changed: uidRange(3), + }}} + w, events := newIMAPTestMail(conn, budget, &models.Mailbox{Name: "INBOX", UIDValidity: 8, UIDNext: 500, HighestModSeq: 5}) + since, started, success := time.Now().Add(-30*24*time.Hour), time.Now().Add(-16*24*time.Hour), time.Now().Add(-time.Hour) + w.tracker = newSyncTracker(&models.SyncState{BackfillStatus: models.SyncBackfillRunning, BackfillSince: &since, BackfillStartedAt: &started, BackfillSynced: 2, LastSyncedAt: &success}, func(models.SyncState) error { return nil }) + w.tracker.setFolder("INBOX", models.SyncFolderCursor{UID: 50, Done: true}) + maps := &recoveryMessageMap{data: map[string]repository.EmailMessageData{"<2@fake.test>": {ID: uuid.NewString(), MessageID: "<2@fake.test>"}}} + w.EmailMessageMapRepository = maps + return w, events, conn, maps +} + +func TestImapRunningImportGenerationRecoveryPreservesWindowCountersAndReload(t *testing.T) { + for _, outcome := range []string{"complete", "deferred", "relay failure", "global cap", "cap already reached"} { + t.Run(outcome, func(t *testing.T) { + budget := &initialRecoveryBudget{fixedBudget: fixedBudget{allow: 10}, limit: 4} + if outcome == "deferred" { + budget.allow = 1 + } + if outcome == "global cap" { + budget.limit = 3 + } + if outcome == "cap already reached" { + budget.limit = 2 + budget.allow = 0 + } + w, events, conn, maps := newRunningRecoveryMail(t, budget) + since, started, success := *w.tracker.state.BackfillSince, *w.tracker.state.BackfillStartedAt, *w.tracker.state.LastSyncedAt + knownID := maps.data["<2@fake.test>"].ID + if outcome == "relay failure" { + emit := w.onEvent + w.onEvent = func(kind models.JobEventType, body any) error { + if kind == models.JobEventTypeEmailUpdate { + return errors.New("known generation relay failed") + } + return emit(kind, body) + } + } + if err := w.Sync(t.Context()); err != nil { + t.Fatal(err) + } + allEvents := append([]captured{}, (*events)...) + if outcome == "deferred" || outcome == "relay failure" { + cur := w.tracker.folder("INBOX") + recovery := imapRecoveryCursor(cur) + wantUID := uint32(2) + if outcome == "relay failure" { + wantUID = 3 + } + if recovery == nil || !recovery.Initial || recovery.Generation != 9 || !recovery.Since.Equal(since) || cur.Done || cur.UID != wantUID || w.tracker.state.BackfillSynced != 3 || w.tracker.state.BackfillStatus != models.SyncBackfillRunning || !w.tracker.state.LastSyncedAt.Equal(success) { + t.Fatalf("unfinished recovery lost progress/window: %+v / %+v / %+v", cur, recovery, w.tracker.state) + } + w, events = imapReload(t, w, conn, &initialRecoveryBudget{fixedBudget: fixedBudget{allow: 10}, limit: 4}) + if err := w.Sync(t.Context()); err != nil { + t.Fatal(err) + } + allEvents = append(allEvents, (*events)...) + } + if !w.tracker.state.BackfillSince.Equal(since) || !w.tracker.state.BackfillStartedAt.Equal(started) { + t.Fatal("generation changed original import window/timer") + } + updates := mailboxEvents(allEvents, models.JobEventTypeEmailUpdate) + if len(updates) != 1 { + t.Fatalf("known metadata relayed %d times, want once across reload", len(updates)) + } + update := updates[0].body.(*models.JobEventEmailUpdate) + if update.ID.String() != knownID || update.UID != 2 || update.Mailbox != 9 || update.ModSeq != 5 || !models.SeenFromFlags(update.Flags) { + t.Fatalf("known handles/read state stale: %+v", update) + } + wantCount := budget.limit + if w.tracker.state.BackfillStatus != models.SyncBackfillComplete || w.tracker.state.BackfillSynced != wantCount || len(maps.data) != 1+wantCount-2 || len(mailboxEvents(allEvents, models.JobEventTypeNewEmail)) != wantCount-2 || len(conn.bodies) != wantCount-2 { + t.Fatal("recovery duplicated known mail, bypassed cap or lost cumulative admission count") + } + if maps.data["<2@fake.test>"].ID != knownID { + t.Fatal("known identity changed") + } + for _, cutoff := range conn.since { + if !cutoff.Equal(since) { + t.Fatal("provider search escaped original import window") + } + } + cur := w.tracker.folder("INBOX") + if !cur.Done || cur.UID != 1 || imapRecoveryCursor(cur) == nil || !imapRecoveryCursor(cur).Initial { + t.Fatalf("running-import recovery continuation lost: %+v", cur) + } + if strings.HasPrefix(cur.Next, imapRecoveryPrefix) { + t.Fatal("an older worker could apply a separate cap to the initial import") + } + searches := conn.searches + again, later := imapReload(t, w, conn, &initialRecoveryBudget{fixedBudget: fixedBudget{allow: 10}, limit: 4}) + if err := again.Sync(t.Context()); err != nil { + t.Fatal(err) + } + if conn.searches != searches || len(mailboxEvents(*later, models.JobEventTypeEmailUpdate)) != 0 || len(mailboxEvents(*later, models.JobEventTypeNewEmail)) != 0 { + t.Fatal("completed initial-generation replay repeated") + } + }) + } +} + +func TestImapRunningImportGenerationMarkerPrecedesBaseline(t *testing.T) { + w, events, conn, _ := newRunningRecoveryMail(t, &initialRecoveryBudget{fixedBudget: fixedBudget{allow: 10}, limit: 4}) + w.tracker.emit = func(models.SyncState) error { return errors.New("generation state relay failed") } + if err := w.Sync(t.Context()); err != nil { + t.Fatal(err) + } + if conn.fetches != 0 || len(mailboxEvents(*events, models.JobEventTypeMailboxUpdate)) != 0 || w.SmtpImapData.Mailboxes[0].UIDValidity != 8 || w.tracker.state.BackfillSynced != 2 { + t.Fatal("baseline or admissions outran initial-generation marker") + } + w.tracker.emit = func(models.SyncState) error { return nil } + if err := w.Sync(t.Context()); err != nil { + t.Fatal(err) + } + if w.tracker.state.BackfillStatus != models.SyncBackfillComplete || w.tracker.state.BackfillSynced != 4 || len(mailboxEvents(*events, models.JobEventTypeEmailUpdate)) != 1 { + t.Fatal("state relay retry did not recover initial generation") + } +} + +func (c *knownRecoveryConn) FetchEnvelopes(_ context.Context, _ []goimap.UID) ([]*imap.Fetched, *errx.MailError) { + c.fetches++ + message := c.message + message.Flags = append([]string{}, c.message.Flags...) + return []*imap.Fetched{{Email: &message}}, nil +} + +func TestImapRecoveryReconcilesKnownMetadataBeforeCheckpoint(t *testing.T) { + for _, newFolder := range []bool{false, true} { + for _, relayFailure := range []bool{false, true} { + t.Run(fmt.Sprintf("new folder=%t relay failure=%t", newFolder, relayFailure), func(t *testing.T) { + box := models.Mailbox{Name: "Archive", UIDValidity: 9, UIDNext: 43, HighestModSeq: 100} + conn := &knownRecoveryConn{fakeImapConn: &fakeImapConn{folders: []models.Mailbox{box}, all: []goimap.UID{42}, bodyErr: errx.ErrMailServerUnreachable}, message: models.EmailMessageData{UID: 42, MessageID: "", ModSeq: 100, Flags: []string{models.FlagSeen}}} + w, events := newIMAPTestMail(conn, &fixedBudget{allow: 0}, &models.Mailbox{Name: box.Name, UIDValidity: 8, UIDNext: 200, HighestModSeq: 100}) + if newFolder { + w.SmtpImapData.Mailboxes = nil + } + id := uuid.NewString() + maps := &recoveryMessageMap{data: map[string]repository.EmailMessageData{conn.message.MessageID: {ID: id, MessageID: conn.message.MessageID}}} + w.EmailMessageMapRepository = maps + w.tracker.state.BackfillSynced = 23 + w.tracker.setFolder(box.Name, models.SyncFolderCursor{UID: 200, Done: true}) + oldSuccess := time.Now().Add(-time.Hour) + w.tracker.state.LastSyncedAt = &oldSuccess + if relayFailure { + emit := w.onEvent + w.onEvent = func(kind models.JobEventType, body any) error { + if kind == models.JobEventTypeEmailUpdate { + return errors.New("known metadata relay unavailable") + } + return emit(kind, body) + } + } + if err := w.Sync(t.Context()); err != nil { + t.Fatal(err) + } + if relayFailure { + cur := w.tracker.folder(box.Name) + if cur.Done || cur.UID != 0 || !w.tracker.state.LastSyncedAt.Equal(oldSuccess) || len(mailboxEvents(*events, models.JobEventTypeEmailUpdate)) != 0 { + t.Fatal("failed metadata relay completed recovery or refreshed success") + } + w, events = imapReload(t, w, conn, &fixedBudget{allow: 0}) + if err := w.Sync(t.Context()); err != nil { + t.Fatal(err) + } + } + updates := mailboxEvents(*events, models.JobEventTypeEmailUpdate) + if len(updates) != 1 { + t.Fatalf("recovery known updates=%d", len(updates)) + } + update := updates[0].body.(*models.JobEventEmailUpdate) + if update.ID.String() != id || update.UID != 42 || update.Mailbox != 9 || update.ModSeq != 100 || update.FolderPath != box.Name || !models.SeenFromFlags(update.Flags) { + t.Fatalf("stale recovery metadata: %+v", update) + } + cur := w.tracker.folder(box.Name) + if !cur.Done || cur.UID != 42 || imapRecoveryCursor(cur).Synced != 0 || w.tracker.state.BackfillSynced != 23 || w.tracker.state.BackfillStatus != models.SyncBackfillComplete { + t.Fatal("known reconciliation reset or consumed global history") + } + if len(mailboxEvents(*events, models.JobEventTypeNewEmail)) != 0 || len(maps.data) != 1 || maps.data[conn.message.MessageID].ID != id { + t.Fatal("known recovery reimported a mapped message") + } + searches := conn.searches + again, later := imapReload(t, w, conn, &fixedBudget{allow: 0}) + if err := again.Sync(t.Context()); err != nil { + t.Fatal(err) + } + if conn.searches != searches || len(mailboxEvents(*later, models.JobEventTypeEmailUpdate)) != 0 || len(mailboxEvents(*later, models.JobEventTypeNewEmail)) != 0 { + t.Fatal("unchanged HIGHESTMODSEQ repeated completed recovery") + } + }) + } + } +} + +func imapReload(t *testing.T, w *WMail, conn ImapConn, budget syncBudget) (*WMail, *[]captured) { + t.Helper() + // Use the existing JSON wire shape, not shared in-memory state, to model restart. + raw, err := json.Marshal(w.tracker.state) + if err != nil { + t.Fatal(err) + } + var state models.SyncState + if err := json.Unmarshal(raw, &state); err != nil { + t.Fatal(err) + } + saved := *w.SmtpImapData.Mailboxes[0] + next, events := newIMAPTestMail(conn, budget, &saved) + next.EmailMessageMapRepository = w.EmailMessageMapRepository + next.tracker = newSyncTracker(&state, func(models.SyncState) error { return nil }) + return next, events +} + +func TestImapCompletedFolderRecoveryResumesAfterDeferralAndReload(t *testing.T) { + for _, generation := range []bool{false, true} { + t.Run(map[bool]string{false: "new populated folder", true: "new generation"}[generation], func(t *testing.T) { + box := models.Mailbox{Name: "Archive", UIDValidity: 9, UIDNext: 4, HighestModSeq: 100} + conn := &fakeImapConn{folders: []models.Mailbox{box}, all: []goimap.UID{1, 2, 3}} + saved := &models.Mailbox{Name: "Archive", UIDValidity: 8, UIDNext: 90, HighestModSeq: 100} + w, events := newIMAPTestMail(conn, &fixedBudget{allow: 1}, saved) + if !generation { + w.SmtpImapData.Mailboxes = nil + } + maps := &recoveryMessageMap{data: map[string]repository.EmailMessageData{}} + knownID := uuid.NewString() + maps.data["<2@fake.test>"] = repository.EmailMessageData{ID: knownID, MessageID: "<2@fake.test>"} + w.EmailMessageMapRepository = maps + w.tracker.setFolder("Archive", models.SyncFolderCursor{UID: 80, Done: true}) + oldSuccess := time.Now().Add(-time.Hour) + w.tracker.state.LastSyncedAt = &oldSuccess + if err := w.Sync(t.Context()); err != nil { + t.Fatal(err) + } + cur := w.tracker.folder("Archive") + recovery := imapRecoveryCursor(cur) + if cur.Done || cur.UID != 2 || recovery == nil || recovery.Generation != 9 || recovery.Synced != 1 { + t.Fatalf("deferred recovery = %+v / %+v", cur, recovery) + } + if len(mailboxEvents(*events, models.JobEventTypeNewEmail)) != 1 { + t.Fatal("newest recovery arrival missing") + } + if len(mailboxEvents(*events, models.JobEventTypeEmailUpdate)) != 1 { + t.Fatal("known metadata was not reconciled before the saved floor") + } + if !w.tracker.state.LastSyncedAt.Equal(oldSuccess) { + t.Fatal("deferred recovery refreshed success") + } + cutoff := recovery.Since + next, resumed := imapReload(t, w, conn, &fixedBudget{allow: 10}) + if err := next.Sync(t.Context()); err != nil { + t.Fatal(err) + } + cur = next.tracker.folder("Archive") + if !cur.Done || cur.UID != 1 || !imapRecoveryCursor(cur).Since.Equal(cutoff) { + t.Fatalf("restarted recovery = %+v", cur) + } + if len(maps.data) != 3 || len(mailboxEvents(*resumed, models.JobEventTypeNewEmail)) != 1 || maps.data["<2@fake.test>"].ID != knownID { + t.Fatal("recovery lost or duplicated arrivals") + } + if len(mailboxEvents(*resumed, models.JobEventTypeEmailUpdate)) != 0 { + t.Fatal("checkpointed known metadata was relayed again after reload") + } + if next.tracker.state.BackfillStatus != models.SyncBackfillComplete || next.tracker.state.BackfillSynced != 0 { + t.Fatal("global history reset") + } + before := conn.searches + again, later := imapReload(t, next, conn, &fixedBudget{allow: 10}) + if err := again.Sync(t.Context()); err != nil { + t.Fatal(err) + } + if conn.searches != before || len(mailboxEvents(*later, models.JobEventTypeNewEmail)) != 0 { + t.Fatal("completed recovery reimported on restart") + } + }) + } +} + +func TestImapRecoveryDoesNotBaselineBeforeReplayStateRelays(t *testing.T) { + conn := &fakeImapConn{folders: []models.Mailbox{{Name: "Archive", UIDValidity: 9, UIDNext: 3, HighestModSeq: 100}}, all: []goimap.UID{1, 2}} + w, events := newIMAPTestMail(conn, &fixedBudget{allow: 10}, &models.Mailbox{Name: "Archive", UIDValidity: 8}) + w.tracker.emit = func(models.SyncState) error { return errors.New("state unavailable") } + if err := w.Sync(t.Context()); err != nil { + t.Fatal(err) + } + if w.SmtpImapData.Mailboxes[0].UIDValidity != 8 || len(mailboxEvents(*events, models.JobEventTypeMailboxUpdate)) != 0 || conn.fetches != 0 { + t.Fatal("baseline outran replay checkpoint") + } + w.tracker.emit = func(models.SyncState) error { return nil } + if err := w.Sync(t.Context()); err != nil { + t.Fatal(err) + } + if !w.tracker.folder("Archive").Done || len(mailboxEvents(*events, models.JobEventTypeNewEmail)) != 2 { + t.Fatal("checkpoint retry did not recover folder") + } +} + +func TestImapBodyFailurePinsLiveAndRecoveryUntilReload(t *testing.T) { + for _, recovery := range []bool{false, true} { + t.Run(map[bool]string{false: "live", true: "recovery"}[recovery], func(t *testing.T) { + box := models.Mailbox{Name: "INBOX", UIDValidity: 7, UIDNext: 2, HighestModSeq: 101} + conn := &fakeImapConn{folders: []models.Mailbox{box}, changed: []goimap.UID{1}, all: []goimap.UID{1}, bodyErr: errx.ErrMailServerUnreachable} + w, events := newIMAPTestMail(conn, &fixedBudget{allow: 10}, &models.Mailbox{Name: "INBOX", UIDValidity: 7, UIDNext: 1, HighestModSeq: 100}) + if recovery { + w.SmtpImapData.Mailboxes[0].UIDValidity = 6 + } + maps := &recoveryMessageMap{data: map[string]repository.EmailMessageData{}} + w.EmailMessageMapRepository = maps + if err := w.Sync(t.Context()); err == nil { + t.Fatal("body failure swallowed") + } + if len(maps.data) != 0 || len(mailboxEvents(*events, models.JobEventTypeNewEmail)) != 0 || w.tracker.state.BackfillSynced != 0 || w.tracker.folder("INBOX").UID != 0 || w.tracker.folder("INBOX").Done { + t.Fatal("body failure imported partial message or advanced recovery") + } + if !recovery && w.SmtpImapData.Mailboxes[0].UIDNext != 1 { + t.Fatal("body failure advanced live cursor") + } + conn.bodyErr = nil + next, resumed := imapReload(t, w, conn, &fixedBudget{allow: 10}) + if err := next.Sync(t.Context()); err != nil { + t.Fatal(err) + } + arrivals := mailboxEvents(*resumed, models.JobEventTypeNewEmail) + if len(arrivals) != 1 || len(maps.data) != 1 { + t.Fatal("hydration was not retried exactly once") + } + if arrivals[0].body.(*models.JobEventNewEmail).Message.BodyText != "complete body" { + t.Fatal("arrival did not contain hydrated body") + } + if next.SmtpImapData.Mailboxes[0].UIDNext != 2 { + t.Fatal("live cursor did not advance after hydration") + } + }) + } +} + +func TestImapNomodseqUsesUIDNextAndPeriodicFlags(t *testing.T) { + for _, listedModseq := range []uint64{0, 100} { + t.Run(map[uint64]string{0: "status NOMODSEQ", 100: "select NOMODSEQ"}[listedModseq], func(t *testing.T) { + conn := &fakeImapConn{folders: []models.Mailbox{{Name: "INBOX", UIDValidity: 7, UIDNext: 3, HighestModSeq: listedModseq}}, changed: []goimap.UID{2}, view: &imap.Selected{UIDNext: 3}, flags: map[uint32]imap.FlagState{1: {MessageID: ""}}} + w, events := newIMAPTestMail(conn, &fixedBudget{allow: 10}, &models.Mailbox{Name: "INBOX", UIDValidity: 7, UIDNext: 2, HighestModSeq: listedModseq}) + if err := w.Sync(t.Context()); err != nil { + t.Fatal(err) + } + if len(mailboxEvents(*events, models.JobEventTypeNewEmail)) != 1 || conn.flagScans != 1 || conn.uidSearches != 1 || conn.modseqSearches != 0 { + t.Fatal("NOMODSEQ ignored arrival or fallback flags") + } + w.EmailMessageMapRepository = knownMessageMap{id: uuid.NewString()} + conn.flags[1] = imap.FlagState{MessageID: "", Flags: []string{`\Seen`}} + w.flagScan["INBOX"].at = time.Now().Add(-2 * config.ImapFlagScanInterval) + if err := w.Sync(t.Context()); err != nil { + t.Fatal(err) + } + if len(mailboxEvents(*events, models.JobEventTypeEmailUpdate)) != 1 { + t.Fatal("NOMODSEQ dropped periodic read update") + } + }) + } +} + +func TestImapBodyFailurePinsInitialHistoryUntilReload(t *testing.T) { + box := models.Mailbox{Name: "INBOX", UIDValidity: 7, UIDNext: 2, HighestModSeq: 100, Messages: 1} + conn := &fakeImapConn{folders: []models.Mailbox{box}, changed: []goimap.UID{1}, all: []goimap.UID{1}, bodyErr: errx.ErrMailServerUnreachable} + w, events := newIMAPTestMail(conn, &fixedBudget{allow: 10}, &box) + w.tracker = newSyncTracker(nil, func(models.SyncState) error { return nil }) + maps := &recoveryMessageMap{data: map[string]repository.EmailMessageData{}} + w.EmailMessageMapRepository = maps + if err := w.Sync(t.Context()); err == nil { + t.Fatal("history body failure swallowed") + } + if len(maps.data) != 0 || len(mailboxEvents(*events, models.JobEventTypeNewEmail)) != 0 || w.tracker.state.BackfillSynced != 0 || w.tracker.folder("INBOX").UID != 0 || w.tracker.folder("INBOX").Done { + t.Fatal("history advanced after body failure") + } + conn.bodyErr = nil + next, resumed := imapReload(t, w, conn, &fixedBudget{allow: 10}) + if err := next.Sync(t.Context()); err != nil { + t.Fatal(err) + } + if len(maps.data) != 1 || len(mailboxEvents(*resumed, models.JobEventTypeNewEmail)) != 1 || next.tracker.state.BackfillSynced != 1 || !next.tracker.folder("INBOX").Done { + t.Fatal("history hydration did not resume") + } +} + +func TestImapFlagScanRetriesBothChangesAfterPublicationAbort(t *testing.T) { + conn := &fakeImapConn{folders: []models.Mailbox{{Name: "INBOX", UIDValidity: 7, UIDNext: 3}}, noCondStore: true, flags: map[uint32]imap.FlagState{1: {MessageID: ""}, 2: {MessageID: ""}}} + w, events := newIMAPTestMail(conn, &fixedBudget{allow: 10}, &models.Mailbox{Name: "INBOX", UIDValidity: 7, UIDNext: 3}) + w.EmailMessageMapRepository = knownMessageMap{id: uuid.NewString()} + if err := w.Sync(t.Context()); err != nil { + t.Fatal(err) + } + for uid, st := range conn.flags { + st.Flags = []string{`\Seen`} + conn.flags[uid] = st + } + emit := w.onEvent + w.onEvent = func(kind models.JobEventType, body any) error { + if kind == models.JobEventTypeEmailUpdate { + return errors.New("publish unavailable") + } + return emit(kind, body) + } + w.flagScan["INBOX"].at = time.Now().Add(-2 * config.ImapFlagScanInterval) + if err := w.Sync(t.Context()); err != nil { + t.Fatal(err) + } + w.onEvent = emit + w.flagScan["INBOX"].at = time.Now().Add(-2 * config.ImapFlagScanInterval) + if err := w.Sync(t.Context()); err != nil { + t.Fatal(err) + } + if len(mailboxEvents(*events, models.JobEventTypeEmailUpdate)) != 2 { + t.Fatal("failed or unvisited flag changes were absorbed into baseline") + } +} diff --git a/internal/app/worker/wmail/sync_imap_test.go b/internal/app/worker/wmail/sync_imap_test.go index 752986310..5cc493ee7 100644 --- a/internal/app/worker/wmail/sync_imap_test.go +++ b/internal/app/worker/wmail/sync_imap_test.go @@ -43,7 +43,11 @@ type fakeImapConn struct { failFinds bool // view, when set, is the cursors SELECT reports in place of the listing's: // a server whose selected view lags or leads its STATUS. - view *imap.Selected + view *imap.Selected + searches int + bodyErr *errx.MailError + modseqSearches int + uidSearches int } // Folders hands out a copy, like a real listing: the pass filters the slice @@ -61,6 +65,7 @@ func (c *fakeImapConn) FolderConflicts() int { return c.conflicts } func (c *fakeImapConn) HasCondStore() bool { return !c.noCondStore } func (c *fakeImapConn) SearchNewSince(uidNext uint32) ([]goimap.UID, *errx.MailError) { + c.uidSearches++ var out []goimap.UID for _, uid := range c.changed { if uint32(uid) >= uidNext { @@ -115,6 +120,7 @@ func (c *fakeImapConn) SelectForSyncGen(name string) (uint32, uint32, *errx.Mail } func (c *fakeImapConn) SearchChangedSince(uint64) ([]goimap.UID, *errx.MailError) { + c.modseqSearches++ return append([]goimap.UID(nil), c.changed...), nil } @@ -122,6 +128,11 @@ func (c *fakeImapConn) SearchAll() ([]goimap.UID, *errx.MailError) { return append([]goimap.UID(nil), c.all...), nil } +func (c *fakeImapConn) SearchSince(time.Time) ([]goimap.UID, *errx.MailError) { + c.searches++ + return append([]goimap.UID(nil), c.all...), nil +} + func (c *fakeImapConn) FetchEnvelopes(_ context.Context, uids []goimap.UID) ([]*imap.Fetched, *errx.MailError) { c.fetches++ out := make([]*imap.Fetched, 0, len(uids)) @@ -135,7 +146,13 @@ func (c *fakeImapConn) FetchEnvelopes(_ context.Context, uids []goimap.UID) ([]* return out, nil } -func (c *fakeImapConn) FetchBody(*imap.Fetched) {} +func (c *fakeImapConn) FetchBody(f *imap.Fetched) *errx.MailError { + if c.bodyErr != nil { + return c.bodyErr + } + f.Email.BodyPlain = "complete body" + return nil +} // fixedBudget is a syncBudget that admits a fixed number of messages and then // denies on the daily window, which is how a real governor answers once the @@ -339,6 +356,17 @@ func (c *backfillImapConn) FolderConflicts() int { retur func (c *backfillImapConn) HasCondStore() bool { return true } func (c *backfillImapConn) ReleaseMailbox() {} +func (c *backfillImapConn) SelectForSyncState(folder string) (imap.Selected, *errx.MailError) { + count, err := c.SelectForSync(folder) + view := imap.Selected{Count: count} + for _, box := range c.folders { + if box.Name == folder { + view.UIDValidity, view.UIDNext, view.HighestModSeq = box.UIDValidity, box.UIDNext, box.HighestModSeq + } + } + return view, err +} + func (c *backfillImapConn) SelectForSync(name string) (uint32, *errx.MailError) { c.selected = name return uint32(len(c.uids[name])), nil @@ -375,7 +403,7 @@ func (c *backfillImapConn) FetchEnvelopes(_ context.Context, uids []goimap.UID) return out, nil } -func (c *backfillImapConn) FetchBody(*imap.Fetched) {} +func (c *backfillImapConn) FetchBody(*imap.Fetched) *errx.MailError { return nil } // The Graph defect's shape, checked on the IMAP import: a folder whose search // fails holds its cursor and is walked again on the next pass. Nothing about a diff --git a/internal/app/worker/wmail/sync_mail.go b/internal/app/worker/wmail/sync_mail.go index 7badcb353..e2b9b9ee3 100644 --- a/internal/app/worker/wmail/sync_mail.go +++ b/internal/app/worker/wmail/sync_mail.go @@ -8,6 +8,18 @@ import ( ) func (w *WMail) SyncMail(ctx context.Context) *errx.MailError { + done, ok := w.BeginExecution() + if !ok { + return errx.ErrMailServerUnreachable + } + defer done() + w.syncMu.Lock() + defer w.syncMu.Unlock() + ctx, cancel := w.ExecutionContext(ctx) + defer cancel() + if ctx.Err() != nil { + return errx.ErrMailServerUnreachable + } w.retryDiagnostics(ctx) defer w.retryDiagnostics(ctx) switch w.EmailType { diff --git a/internal/app/worker/wmail/sync_state.go b/internal/app/worker/wmail/sync_state.go index 32c3d0693..d896d05df 100644 --- a/internal/app/worker/wmail/sync_state.go +++ b/internal/app/worker/wmail/sync_state.go @@ -16,10 +16,11 @@ const syncStateHeartbeat = 10 * time.Minute // of the tick; the consumer persists it and the loader hands it back on the // next assignment. type syncTracker struct { - state models.SyncState - dirty bool - lastSent time.Time - emit func(models.SyncState) error + state models.SyncState + dirty bool + lastSent time.Time + emit func(models.SyncState) error + tickComplete bool } func newSyncTracker(seed *models.SyncState, emit func(models.SyncState) error) *syncTracker { @@ -85,6 +86,11 @@ func (t *syncTracker) setFoldersSkipped(cap, conflict int) { func (t *syncTracker) touch(now time.Time) { stamp := now t.state.LastSyncedAt = &stamp + t.flush(now) +} + +// flush publishes safe progress without implying a successful provider check. +func (t *syncTracker) flush(now time.Time) { if !t.dirty && now.Sub(t.lastSent) < syncStateHeartbeat { return } diff --git a/internal/app/worker/wmail/sync_truth_test.go b/internal/app/worker/wmail/sync_truth_test.go new file mode 100644 index 000000000..306f2fe48 --- /dev/null +++ b/internal/app/worker/wmail/sync_truth_test.go @@ -0,0 +1,376 @@ +package wmail + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/client/goog" + "github.com/warmbly/warmbly/internal/client/msgraph" + "github.com/warmbly/warmbly/internal/client/smtpimap/imap" + "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +func seedSyncTruth(w *WMail, relayed *[]models.SyncState) time.Time { + previous := time.Now().Add(-time.Hour) + w.tracker = newSyncTracker(&models.SyncState{BackfillStatus: models.SyncBackfillComplete, LastSyncedAt: &previous, Deferred: 5}, func(st models.SyncState) error { + *relayed = append(*relayed, st) + return nil + }) + return previous +} + +func TestSyncTruthRepeatedTransportFailuresRelayFreshObservations(t *testing.T) { + srv := httptest.NewServer(http.NotFoundHandler()) + srv.Close() + var events []captured + w := newGoogleTestMail(t, srv, &events) + for i := 0; i < 2; i++ { + if err := w.syncOnce(t.Context()); err == nil { + t.Fatal("provider failure hidden") + } + } + errors := 0 + for _, event := range events { + if event.eventType == models.JobEventTypeEmailServerError { + errors++ + } + } + if errors != 2 || w.transportFailures != 2 { + t.Fatalf("observations=%d failures=%d", errors, w.transportFailures) + } +} + +func assertNoSyncSuccess(t *testing.T, w *WMail, previous time.Time) { + t.Helper() + if w.tracker.state.LastSyncedAt == nil || !w.tracker.state.LastSyncedAt.Equal(previous) || w.tracker.state.Deferred != 5 { + t.Fatalf("incomplete pass replaced successful evidence/backlog: %+v", w.tracker.state) + } +} + +func assertSyncCatchUp(t *testing.T, w *WMail, previous time.Time) { + t.Helper() + if w.tracker.state.LastSyncedAt == nil || !w.tracker.state.LastSyncedAt.After(previous) || w.tracker.state.Deferred != 0 { + t.Fatalf("successful catch-up did not replace old evidence: %+v", w.tracker.state) + } +} + +func TestSyncTruthGoogleFailureHeartbeatAndRecovery(t *testing.T) { + failing := true + srv := httptest.NewServer(http.HandlerFunc(func(out http.ResponseWriter, _ *http.Request) { + out.Header().Set("Content-Type", "application/json") + if failing { + out.WriteHeader(http.StatusServiceUnavailable) + _, _ = out.Write([]byte(`{"error":{"code":503,"message":"retry later"}}`)) + return + } + _, _ = out.Write([]byte(`{"historyId":"200"}`)) + })) + defer srv.Close() + var events []captured + w := newGoogleTestMail(t, srv, &events) + w.GoogleData.LastHistoryID = 100 + var relayed []models.SyncState + previous := seedSyncTruth(w, &relayed) + for range 2 { + w.tracker.lastSent = time.Now().Add(-2 * syncStateHeartbeat) + if err := w.SyncGoogle(t.Context()); err == nil { + t.Fatal("expected provider error") + } + assertNoSyncSuccess(t, w, previous) + } + if len(relayed) != 2 { + t.Fatalf("failed passes must relay safe heartbeat state: %d", len(relayed)) + } + for _, st := range relayed { + if !st.LastSyncedAt.Equal(previous) || st.Deferred != 5 { + t.Fatalf("false success escaped via heartbeat: %+v", st) + } + } + failing = false + if err := w.SyncGoogle(t.Context()); err != nil { + t.Fatal(err) + } + assertSyncCatchUp(t, w, previous) +} + +type syncTruthMap struct { + repository.EmailMessageMapRepository + failGet bool + failDel bool +} + +func (r *syncTruthMap) Get(context.Context, uuid.UUID, uuid.UUID, string) (*repository.EmailMessageData, error) { + if r.failGet { + return nil, errors.New("map unavailable") + } + return nil, nil +} + +func (r *syncTruthMap) Del(context.Context, uuid.UUID, uuid.UUID, string, uuid.UUID) error { + if r.failDel { + return errors.New("unmap unavailable") + } + return nil +} + +func TestSyncTruthGraphAbortedAdmissionAndCheckpoint(t *testing.T) { + for _, failure := range []string{"map", "checkpoint"} { + t.Run(failure, func(t *testing.T) { + g := newFakeGraph() + var events []captured + var relayed []models.SyncState + w := newGraphTestMail(t, g.serve(t), &events, &relayed) + previous := seedSyncTruth(w, &relayed) + before := w.GraphData.Client.DeltaLinks[msgraph.FolderInbox] + r := &syncTruthMap{failGet: failure == "map"} + w.EmailMessageMapRepository = r + failing := true + w.onEvent = func(kind models.JobEventType, _ any) error { + if failing && failure == "checkpoint" && kind == models.JobEventTypeGraphDeltaUpdate { + return errors.New("checkpoint unavailable") + } + return nil + } + for range 2 { + if failure == "map" { + g.live = []string{"new-arrival"} + } + w.tracker.lastSent = time.Now().Add(-2 * syncStateHeartbeat) + _ = w.SyncGraph(t.Context()) + assertNoSyncSuccess(t, w, previous) + if w.GraphData.Client.DeltaLinks[msgraph.FolderInbox] != before { + t.Fatal("failed/aborted page advanced local checkpoint") + } + } + failing, r.failGet = false, false + if err := w.SyncGraph(t.Context()); err != nil { + t.Fatal(err) + } + assertSyncCatchUp(t, w, previous) + }) + } +} + +func TestSyncTruthRetryUnmapAndCancellation(t *testing.T) { + for _, provider := range []string{"google", "graph", "imap"} { + t.Run(provider, func(t *testing.T) { + w, _ := newIMAPTestMail(&fakeImapConn{}, &fixedBudget{}, &models.Mailbox{}) + var relayed []models.SyncState + previous := seedSyncTruth(w, &relayed) + w.unmapPending = map[string]uuid.UUID{"failed-arrival": uuid.New()} + w.EmailMessageMapRepository = &syncTruthMap{failDel: true} + switch provider { + case "google": + _ = w.SyncGoogle(t.Context()) + case "graph": + _ = w.SyncGraph(t.Context()) + case "imap": + _ = w.Sync(t.Context()) + } + assertNoSyncSuccess(t, w, previous) + if len(relayed) != 1 { + t.Fatal("retry-unmap blockage did not relay safe state") + } + }) + } + w, _ := newIMAPTestMail(&fakeImapConn{}, &fixedBudget{}, &models.Mailbox{}) + var relayed []models.SyncState + previous := seedSyncTruth(w, &relayed) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _ = w.Sync(ctx) + assertNoSyncSuccess(t, w, previous) +} + +func TestSyncTruthImapFlagFailureRetainsBothUpdatesAndSuccessEvidence(t *testing.T) { + box := &models.Mailbox{Name: "INBOX", UIDValidity: 7, UIDNext: 101} + conn := &fakeImapConn{noCondStore: true, folders: []models.Mailbox{*box}, flags: map[uint32]imap.FlagState{ + 1: {MessageID: "one", Flags: nil}, 2: {MessageID: "two", Flags: nil}, + }} + w, events := newIMAPTestMail(conn, &fixedBudget{allow: 10}, box) + w.EmailMessageMapRepository = knownMessageMap{id: uuid.NewString()} + if err := w.Sync(t.Context()); err != nil { + t.Fatal(err) + } + var relayed []models.SyncState + previous := seedSyncTruth(w, &relayed) + conn.flags[1], conn.flags[2] = imap.FlagState{MessageID: "one", Flags: []string{models.FlagSeen}}, imap.FlagState{MessageID: "two", Flags: []string{models.FlagSeen}} + w.flagScan[box.Name].at = time.Now().Add(-2 * config.ImapFlagScanInterval) + failing := true + w.onEvent = func(kind models.JobEventType, body any) error { + if kind == models.JobEventTypeEmailUpdate && failing { + return errors.New("publisher unavailable") + } + *events = append(*events, captured{eventType: kind, body: body}) + return nil + } + if err := w.Sync(t.Context()); err != nil { + t.Fatal(err) + } + assertNoSyncSuccess(t, w, previous) + for uid := range conn.flags { + if w.flagScan[box.Name].flags[uid] != flagFingerprint(nil) { + t.Fatal("failed/unvisited flags were absorbed into baseline") + } + } + failing = false + w.flagScan[box.Name].at = time.Now().Add(-2 * config.ImapFlagScanInterval) + if err := w.Sync(t.Context()); err != nil { + t.Fatal(err) + } + updates := 0 + for _, e := range *events { + if e.eventType == models.JobEventTypeEmailUpdate { + updates++ + } + } + if updates != 2 { + t.Fatalf("unchanged provider flags did not retry both updates: %d", updates) + } + assertSyncCatchUp(t, w, previous) +} + +func TestSyncTruthPanicDoesNotResetTransportFailure(t *testing.T) { + w, _ := newIMAPTestMail(&fakeImapConn{}, &fixedBudget{}, &models.Mailbox{}) + w.EmailType = models.InboxProviderGoogle + w.GoogleData = nil // Fault inside the provider pass, after beginTick. + w.transportFailures = 2 + var relayed []models.SyncState + previous := seedSyncTruth(w, &relayed) + if err := w.syncOnce(t.Context()); err == nil { + t.Fatal("recovered panic returned successful outcome") + } + assertNoSyncSuccess(t, w, previous) + if w.transportFailures != 2 { + t.Fatal("panic reset existing transport warning/backoff") + } +} + +func TestSyncTruthBoundedAndPinnedProviderPasses(t *testing.T) { + for _, provider := range []string{"google", "graph"} { + for _, outcome := range []string{"bounded", "pinned", "stop"} { + t.Run(provider+"/"+outcome, func(t *testing.T) { + caughtUp := false + srv := httptest.NewServer(http.HandlerFunc(func(out http.ResponseWriter, r *http.Request) { + out.Header().Set("Content-Type", "application/json") + if provider == "google" { + if caughtUp { + _, _ = out.Write([]byte(`{"historyId":"200"}`)) + return + } + if outcome == "bounded" { + _, _ = out.Write([]byte(`{"historyId":"200","nextPageToken":"next"}`)) + return + } + _, _ = out.Write([]byte(`{"historyId":"200","history":[{"id":"150","messagesAdded":[{"message":{"id":"new-arrival","threadId":"thread"}}]}]}`)) + return + } + folder := msgraph.FolderInbox + if caughtUp { + _, _ = fmt.Fprintf(out, `{"value":[],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/me/mailFolders/%s/messages/delta?final=1"}`, folder) + return + } + if outcome == "bounded" { + _, _ = fmt.Fprintf(out, `{"value":[],"@odata.nextLink":"https://graph.microsoft.com%s?next=1"}`, r.URL.Path) + return + } + _, _ = fmt.Fprintf(out, `{"value":[{"id":"new-arrival","isRead":false}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/me/mailFolders/%s/messages/delta?final=1"}`, folder) + })) + defer srv.Close() + var events []captured + var relayed []models.SyncState + var w *WMail + var sync func(context.Context) *errx.MailError + if provider == "google" { + w = newGoogleTestMail(t, srv, &events) + w.GoogleData.LastHistoryID = 100 + w.GoogleData.Client.OnMessageAdded = func(context.Context, string, string) (bool, error) { + if outcome == "stop" { + return false, goog.ErrStop + } + return false, nil + } + sync = w.SyncGoogle + } else { + w = newGraphTestMail(t, srv, &events, &relayed) + w.GraphData.Client.OnMessageSeen = func(context.Context, string, string, bool) (bool, error) { + if outcome == "stop" { + return false, msgraph.ErrStop + } + return false, nil + } + sync = w.SyncGraph + } + previous := seedSyncTruth(w, &relayed) + w.transportFailures = 2 + if err := sync(t.Context()); err != nil { + t.Fatal(err) + } + assertNoSyncSuccess(t, w, previous) + caughtUp = true + if err := sync(t.Context()); err != nil { + t.Fatal(err) + } + assertSyncCatchUp(t, w, previous) + }) + } + } +} + +func TestSyncTruthGoogleCheckpointFailureAndDirtyRelayRetry(t *testing.T) { + failingProvider := false + srv := httptest.NewServer(http.HandlerFunc(func(out http.ResponseWriter, _ *http.Request) { + out.Header().Set("Content-Type", "application/json") + if failingProvider { + out.WriteHeader(503) + _, _ = out.Write([]byte(`{"error":{"code":503,"message":"injected"}}`)) + return + } + _, _ = out.Write([]byte(`{"historyId":"200"}`)) + })) + defer srv.Close() + var events []captured + w := newGoogleTestMail(t, srv, &events) + w.GoogleData.LastHistoryID = 100 + var relayed []models.SyncState + previous := seedSyncTruth(w, &relayed) + w.transportFailures = 2 + w.onEvent = func(kind models.JobEventType, _ any) error { + if kind == models.JobEventTypeHistoryIDUpdate { + return errors.New("checkpoint unavailable") + } + return nil + } + if err := w.syncOnce(t.Context()); err != nil { + t.Fatal(err) + } + assertNoSyncSuccess(t, w, previous) + if w.GoogleData.LastHistoryID != 100 || w.transportFailures != 2 { + t.Fatal("failed checkpoint was installed or claimed recovery") + } + failingProvider = true + expired := time.Now().Add(-time.Minute) + w.tracker.state.ThrottledUntil = &expired + w.tracker.lastSent = time.Now().Add(-2 * syncStateHeartbeat) + w.tracker.emit = func(st models.SyncState) error { return errors.New("state relay unavailable") } + _ = w.SyncGoogle(t.Context()) + assertNoSyncSuccess(t, w, previous) + if !w.tracker.dirty || w.tracker.state.ThrottledUntil != nil { + t.Fatal("failed relay forgot dirty throttle progress") + } + w.tracker.emit = func(st models.SyncState) error { relayed = append(relayed, st); return nil } + _ = w.SyncGoogle(t.Context()) + assertNoSyncSuccess(t, w, previous) + if w.tracker.dirty || relayed[len(relayed)-1].ThrottledUntil != nil { + t.Fatal("safe dirty-state relay did not retry") + } +} diff --git a/internal/app/worker/wmail/wmail.go b/internal/app/worker/wmail/wmail.go index 8c5931760..260bf4584 100644 --- a/internal/app/worker/wmail/wmail.go +++ b/internal/app/worker/wmail/wmail.go @@ -2,6 +2,7 @@ package wmail import ( "context" + "sync" "time" "github.com/google/uuid" @@ -58,10 +59,18 @@ type SmtpImapData struct { } type WMail struct { - diagnosticRetries map[uuid.UUID]diagnosticRetry - ExecutorID uuid.UUID - UserID uuid.UUID - ID uuid.UUID + lifecycleMu sync.Mutex + stopped bool + stopDone chan struct{} + active sync.WaitGroup + syncMu sync.Mutex + executionKeys map[[32]byte]struct{} + initialExecutionKey [32]byte + executionData *models.AddWorkerEmail + diagnosticRetries map[uuid.UUID]diagnosticRetry + ExecutorID uuid.UUID + UserID uuid.UUID + ID uuid.UUID // OrgID scopes the organization-wide sync budget; nil for a legacy // personal mailbox. OrgID *uuid.UUID @@ -169,6 +178,13 @@ func NewWMail( SyncContext: syncContext, CipherService: cipherService, } + mail.rememberExecution(data) + ready := false + defer func() { + if !ready { + mail.Discard() + } + }() // A publisher older than the sync policy sends no Sync block; compiled // defaults then apply and the mailbox starts its backfill from scratch. @@ -251,9 +267,10 @@ func NewWMail( FirstName: data.FirstName, LastName: data.LastName, - User: data.Graph.User, - Cache: mail.Cache, - DeltaLinks: cloneStringMap(deltaLinks), + User: data.Graph.User, + Cache: mail.Cache, + DeltaLinks: cloneStringMap(deltaLinks), + ImmutableIDs: mail.SyncContext != nil, OnMessageSeen: mail.onGraphMessageSeen, OnMessageRemove: mail.onGraphMessageRemove, @@ -286,6 +303,7 @@ func NewWMail( if data.ImapSync { conn := &imap.Client{ + Context: mailCtx, Email: data.Email, AuthType: models.AuthPlain, Credentials: data.SmtpImap.Credentials.IMAP, @@ -294,13 +312,11 @@ func NewWMail( return nil, err } mail.SmtpImapData.ImapClient = conn - // Saved folder cursors: live sync resumes from each folder's stored - // HIGHESTMODSEQ instead of re-baselining (and, before this, instead - // of re-walking every folder on every worker restart). - for i := range data.SmtpImap.Mailboxes { - box := data.SmtpImap.Mailboxes[i] - mail.SmtpImapData.Mailboxes = append(mail.SmtpImapData.Mailboxes, &box) - } + } + // Keep saved cursors even while IMAP sync is temporarily disabled. + for i := range data.SmtpImap.Mailboxes { + box := data.SmtpImap.Mailboxes[i] + mail.SmtpImapData.Mailboxes = append(mail.SmtpImapData.Mailboxes, &box) } mail.SmtpImapData.SmtpClient = &smtp.Client{ @@ -319,6 +335,7 @@ func NewWMail( ) } + ready = true return mail, nil } @@ -335,13 +352,5 @@ func (w *WMail) ApplySyncPolicy(data *models.AddWorkerEmailSyncData) { // Discard releases a WMail that was built but never put to work. func (w *WMail) Discard() { - if w.Cancel != nil { - w.Cancel() - } - if w.SmtpImapData == nil || w.SmtpImapData.ImapClient == nil { - return - } - if c, ok := w.SmtpImapData.ImapClient.(interface{ Close() error }); ok { - _ = c.Close() - } + w.Stop() } diff --git a/internal/client/goog/deadline_test.go b/internal/client/goog/deadline_test.go new file mode 100644 index 000000000..8b4152fec --- /dev/null +++ b/internal/client/goog/deadline_test.go @@ -0,0 +1,32 @@ +package goog + +import ( + "context" + "testing" + + "github.com/warmbly/warmbly/internal/pkg/stoken" + "golang.org/x/oauth2" +) + +func TestProviderDefaultRequestDeadline(t *testing.T) { + for _, brokered := range []bool{false, true} { + c := &Client{} + token := &oauth2.Token{AccessToken: "local-test"} + var err error + if brokered { + if merr := c.InitWithSource(context.Background(), oauth2.StaticTokenSource(token)); merr != nil { + err = merr + } + } else { + if merr := c.Init(context.Background(), token, oauth2.Config{}); merr != nil { + err = merr + } + } + if err != nil { + t.Fatal(err) + } + if c.rawClient.Timeout != stoken.ProviderRequestTimeout { + t.Fatalf("brokered=%v default timeout=%v", brokered, c.rawClient.Timeout) + } + } +} diff --git a/internal/client/goog/goog.go b/internal/client/goog/goog.go index 618b14d51..a37436894 100644 --- a/internal/client/goog/goog.go +++ b/internal/client/goog/goog.go @@ -47,6 +47,7 @@ type Client struct { } func (c *Client) Init(ctx context.Context, token *oauth2.Token, cfg oauth2.Config) *errx.MailError { + ctx = stoken.BoundedContext(ctx) ts := cfg.TokenSource(ctx, token) ts = oauth2.ReuseTokenSource(token, ts) // Only wrap when there is somewhere to persist to. stoken calls onUpdate on @@ -63,7 +64,7 @@ func (c *Client) Init(ctx context.Context, token *oauth2.Token, cfg oauth2.Confi // InitWithSource builds the client on a caller-owned token source (brokered // tokens from Warmbly Cloud); nothing is persisted from it. func (c *Client) InitWithSource(ctx context.Context, ts oauth2.TokenSource) *errx.MailError { - httpClient := oauth2.NewClient(ctx, ts) + httpClient := stoken.HTTPClient(ctx, ts) c.rawClient = httpClient var err error c.srv, err = gmail.NewService(ctx, option.WithHTTPClient(httpClient)) diff --git a/internal/client/goog/history.go b/internal/client/goog/history.go index 7865baabc..f51cf1583 100644 --- a/internal/client/goog/history.go +++ b/internal/client/goog/history.go @@ -14,9 +14,13 @@ import ( // returns the checkpoint reached so far and a nil error. var ErrStop = errors.New("goog: stop") +var errHistoryIncomplete = errors.New("goog: history pass incomplete") + var ErrHistoryExpired = errors.New("goog: history expired; resynchronization required") -var ErrRecoveryPageExpired = errors.New("goog: recovery page token expired") +var ErrPageTokenExpired = errors.New("goog: messages.list page token expired") + +var ErrRecoveryPageExpired = ErrPageTokenExpired // historyPagesPerPass bounds one walk. A mailbox that has fallen far behind // (or is held by fair use, so the checkpoint cannot advance) is caught up @@ -35,6 +39,20 @@ const historyPagesPerPass = 10 // continues, so later records that are admitted (a reply, say) still land and // the deferred ones are re-offered next tick. func (c *Client) FetchHistory(ctx context.Context, lastHistoryID uint64) (uint64, error) { + checkpoint, _, err := c.FetchHistoryPass(ctx, lastHistoryID) + return checkpoint, err +} + +// FetchHistoryPass distinguishes a bounded or pinned walk from full live catch-up. +func (c *Client) FetchHistoryPass(ctx context.Context, lastHistoryID uint64) (uint64, bool, error) { + checkpoint, err := c.fetchHistory(ctx, lastHistoryID) + if errors.Is(err, errHistoryIncomplete) { + return checkpoint, false, nil + } + return checkpoint, err == nil, err +} + +func (c *Client) fetchHistory(ctx context.Context, lastHistoryID uint64) (uint64, error) { if lastHistoryID == 0 { return c.HistoryBaseline(ctx) } @@ -62,7 +80,7 @@ func (c *Client) FetchHistory(ctx context.Context, lastHistoryID uint64) (uint64 } added, err := c.OnMessageAdded(ctx, m.Message.Id, m.Message.ThreadId) if errors.Is(err, ErrStop) { - return checkpoint, nil + return checkpoint, errHistoryIncomplete } if err != nil { return checkpoint, err @@ -96,13 +114,14 @@ func (c *Client) FetchHistory(ctx context.Context, lastHistoryID uint64) (uint64 if resp.NextPageToken == "" { if !pinned { checkpoint = resp.HistoryId + return checkpoint, nil } - break + return checkpoint, errHistoryIncomplete } call.PageToken(resp.NextPageToken) } - return checkpoint, nil + return checkpoint, errHistoryIncomplete } func (c *Client) HistoryBaseline(ctx context.Context) (uint64, error) { @@ -185,8 +204,8 @@ func (c *Client) listMessages(ctx context.Context, q, pageToken string, max int6 resp, err := call.Do() if err != nil { var apiErr *googleapi.Error - if includeSpamTrash && pageToken != "" && errors.As(err, &apiErr) && apiErr.Code == 400 && strings.Contains(strings.ReplaceAll(strings.ToLower(apiErr.Message), " ", ""), "invalidpagetoken") { - return nil, "", ErrRecoveryPageExpired + if pageToken != "" && errors.As(err, &apiErr) && apiErr.Code == 400 && strings.Contains(strings.ReplaceAll(strings.ToLower(apiErr.Message), " ", ""), "invalidpagetoken") { + return nil, "", ErrPageTokenExpired } return nil, "", fmt.Errorf("gmail messages.list: %w", HandleError(err)) } diff --git a/internal/client/goog/history_test.go b/internal/client/goog/history_test.go index 002f85a55..5c50bcf18 100644 --- a/internal/client/goog/history_test.go +++ b/internal/client/goog/history_test.go @@ -76,7 +76,15 @@ func TestRecoveryPageExpiryIsNarrow(t *testing.T) { {"other invalid input", "old", "Invalid query", 400, true, false}, {"first page invalid input", "", "Invalid pageToken", 400, true, false}, {"transient", "old", "Invalid pageToken", 429, true, false}, - {"ordinary backfill", "old", "Invalid pageToken", 400, false, false}, + {"ordinary backfill", "old", "Invalid pageToken", 400, false, true}, + {"ordinary spaced token", "old", "Invalid page token", 400, false, true}, + {"ordinary other invalid input", "old", "Invalid query", 400, false, false}, + {"ordinary invalid page size", "old", "Invalid page size", 400, false, false}, + {"ordinary first page", "", "Invalid pageToken", 400, false, false}, + {"ordinary quota", "old", "Invalid pageToken", 429, false, false}, + {"ordinary auth", "old", "Invalid pageToken", 401, false, false}, + {"ordinary forbidden", "old", "Invalid pageToken", 403, false, false}, + {"ordinary transport", "old", "Invalid pageToken", 503, false, false}, } { t.Run(tc.name, func(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -89,7 +97,7 @@ func TestRecoveryPageExpiryIsNarrow(t *testing.T) { t.Fatal(err) } _, _, err = (&Client{srv: svc}).listMessages(t.Context(), "after:2026/09/01", tc.token, 50, tc.recovery) - if err == nil || errors.Is(err, ErrRecoveryPageExpired) != tc.expired { + if err == nil || errors.Is(err, ErrPageTokenExpired) != tc.expired || errors.Is(err, ErrRecoveryPageExpired) != tc.expired { t.Fatalf("err=%v; want expired=%t", err, tc.expired) } }) diff --git a/internal/client/msgraph/deadline_test.go b/internal/client/msgraph/deadline_test.go new file mode 100644 index 000000000..3a679d400 --- /dev/null +++ b/internal/client/msgraph/deadline_test.go @@ -0,0 +1,32 @@ +package msgraph + +import ( + "context" + "testing" + + "github.com/warmbly/warmbly/internal/pkg/stoken" + "golang.org/x/oauth2" +) + +func TestProviderDefaultRequestDeadline(t *testing.T) { + for _, brokered := range []bool{false, true} { + c := &Client{} + token := &oauth2.Token{AccessToken: "local-test"} + var err error + if brokered { + if merr := c.InitWithSource(context.Background(), oauth2.StaticTokenSource(token)); merr != nil { + err = merr + } + } else { + if merr := c.Init(context.Background(), token, oauth2.Config{}); merr != nil { + err = merr + } + } + if err != nil { + t.Fatal(err) + } + if c.hc.Timeout != stoken.ProviderRequestTimeout { + t.Fatalf("brokered=%v default timeout=%v", brokered, c.hc.Timeout) + } + } +} diff --git a/internal/client/msgraph/delta.go b/internal/client/msgraph/delta.go index df1baf3ba..b1c4e7155 100644 --- a/internal/client/msgraph/delta.go +++ b/internal/client/msgraph/delta.go @@ -10,6 +10,7 @@ import ( "strconv" "time" + "github.com/warmbly/warmbly/internal/errx" "github.com/warmbly/warmbly/internal/models" ) @@ -19,6 +20,15 @@ func itoa(n int) string { return strconv.Itoa(n) } // where it was last persisted. var ErrStop = errors.New("msgraph: stop") +var errDeltaIncomplete = errors.New("msgraph: delta pass incomplete") + +type recoveryContextKey struct{} + +func IsRecovery(ctx context.Context) bool { + recovery, _ := ctx.Value(recoveryContextKey{}).(bool) + return recovery +} + // deltaSelect keeps delta pages light: we only need the id, read state, and the // @removed marker to decide add/update vs remove. Full envelope + body + headers // are hydrated per admitted message via FetchMessage. @@ -27,7 +37,7 @@ const deltaSelect = "id,isRead" // msgSelect is the property set fetched for each new message: enough to build a // complete EmailMessageData (envelope, threading, body) plus internetMessageHeaders // so the warmup verification token survives into sync. -const msgSelect = "id,internetMessageId,conversationId,subject,bodyPreview,isRead," + +const msgSelect = "id,parentFolderId,internetMessageId,conversationId,subject,bodyPreview,isRead," + "receivedDateTime,from,sender,toRecipients,ccRecipients,bccRecipients,replyTo,body,internetMessageHeaders" // deltaPagesPerPass bounds one folder's walk per tick, like Gmail's @@ -52,7 +62,7 @@ type listPage struct { // for placement, sent so a conversation shows both sides (as the Gmail and // IMAP paths already do), and drafts so the Drafts scope is populated on // Outlook the way it is on the other two providers. -var TrackedFolders = []string{FolderInbox, FolderJunk, FolderSent, FolderDrafts} +var TrackedFolders = []string{FolderInbox, FolderJunk, FolderSent, FolderDrafts, FolderArchive} // BackfillFolders are the folders the initial import walks. Junk is followed // live for placement signals but its history is not worth importing, and would @@ -64,42 +74,69 @@ var BackfillFolders = []string{FolderInbox, FolderSent, FolderArchive, FolderDra // fits the disposable worker natively: no webhook endpoint, no subscription // lifecycle, just a cursor the control plane persists via OnDelta. func (c *Client) Sync(ctx context.Context) error { + _, err := c.SyncPass(ctx) + return err +} + +// SyncPass reports catch-up only when every tracked folder reaches an unpinned delta link. +func (c *Client) SyncPass(ctx context.Context) (bool, error) { + complete := true + var firstErr error for _, folder := range TrackedFolders { if err := c.syncFolder(ctx, folder); err != nil { if errors.Is(err, ErrStop) { - return nil + return false, nil + } + if errors.Is(err, errDeltaIncomplete) { + complete = false + continue + } + var mailErr *errx.MailError + if folder == FolderArchive && errors.As(err, &mailErr) && mailErr.Code == errx.MailErrorCodeNotFound { + continue + } + if errors.As(err, &mailErr) && (mailErr.Type == errx.MailErrorCritical || mailErr.Code == errx.MailErrorCodeSendingTooFast) { + return false, err + } + complete = false + if firstErr == nil { + firstErr = err } - return err } } - return nil + return complete, firstErr +} + +func (c *Client) initialDelta(folder string) string { + q := url.Values{} + q.Set("$select", deltaSelect) + // Do not filter the stream: a fixed date filter would eventually hide new moves of older mail. + return c.root() + "/mailFolders/" + folder + "/messages/delta?" + q.Encode() } func (c *Client) syncFolder(ctx context.Context, folder string) error { - next := c.DeltaLinks[folder] - - // First run for this folder: there is no cursor yet. We page to the end - // purely to capture a deltaLink representing "now" and do NOT import the - // mailbox's existing history; the backfill does that under its budget. - // The walk is uncapped and nothing is persisted before the deltaLink: a - // saved intermediate nextLink would make the next pass treat the rest of - // the priming walk as live mail. - if next == "" { - next = c.root() + "/mailFolders/" + folder + "/messages/delta?$select=" + url.QueryEscape(deltaSelect) - for { - var pg deltaPage - if err := c.doJSON(ctx, "GET", next, nil, &pg); err != nil { + if c.PendingRecoveryLink != nil { + link, err := c.PendingRecoveryLink(folder) + if err != nil { + return err + } + if link != "" { + if err := c.finishDelta(ctx, folder, link, true); err != nil { return err } - switch { - case pg.NextLink != "": - next = pg.NextLink - case pg.DeltaLink != "": - return c.saveCursor(ctx, folder, pg.DeltaLink) - default: - return nil + // The captured baseline may be old; catch up its changes on the next pass. + return errDeltaIncomplete + } + } + next := c.DeltaLinks[folder] + + if next == "" { + if c.OnRecoveryStart != nil { + if err := c.OnRecoveryStart(ctx, folder); err != nil { + return err } } + next = c.initialDelta(folder) } // The persisted cursor only moves past a page once everything on it was @@ -107,15 +144,28 @@ func (c *Client) syncFolder(ctx context.Context, folder string) error { // while the walk continues, so admitted messages further on still land // and the deferred ones are re-offered next tick from the pinned link. pinned := false + reset := false for page := 0; page < deltaPagesPerPass; page++ { var pg deltaPage if err := c.doJSON(ctx, "GET", next, nil, &pg); err != nil { + if !reset && deltaExpired(err) { + if c.OnRecoveryStart != nil { + if err := c.OnRecoveryStart(ctx, folder); err != nil { + return err + } + } + reset = true + next = c.initialDelta(folder) + continue + } return err } + recovering := next == c.initialDelta(folder) || reset || (c.Recovering != nil && c.Recovering(folder)) + pageCtx := context.WithValue(ctx, recoveryContextKey{}, recovering) complete := true for i := range pg.Value { - stored, err := c.applyDelta(ctx, folder, &pg.Value[i]) + stored, err := c.applyDelta(pageCtx, folder, &pg.Value[i]) if err != nil { return err } @@ -137,21 +187,46 @@ func (c *Client) syncFolder(ctx context.Context, folder string) error { next = pg.NextLink case pg.DeltaLink != "": if !pinned { - return c.saveCursor(ctx, folder, pg.DeltaLink) + if recovering && c.OnRecoveryBaseline != nil { + if err := c.OnRecoveryBaseline(ctx, folder, pg.DeltaLink); err != nil { + return err + } + } + return c.finishDelta(ctx, folder, pg.DeltaLink, recovering) } - return nil + return errDeltaIncomplete default: - return nil + return errDeltaIncomplete } } - return nil + return errDeltaIncomplete +} + +func (c *Client) finishDelta(ctx context.Context, folder, link string, recovering bool) error { + if recovering && c.OnFolderReconcile != nil { + complete, err := c.OnFolderReconcile(ctx, folder) + if err != nil { + return err + } + if !complete { + return errDeltaIncomplete + } + } + if recovering && c.OnRecoveryCheckpoint != nil { + if err := c.OnRecoveryCheckpoint(ctx, folder, link); err != nil { + return err + } + } + return c.saveCursor(ctx, folder, link) } func (c *Client) saveCursor(ctx context.Context, folder, link string) error { - c.DeltaLinks[folder] = link if c.OnDelta != nil { - return c.OnDelta(ctx, folder, link) + if err := c.OnDelta(ctx, folder, link); err != nil { + return err + } } + c.DeltaLinks[folder] = link return nil } diff --git a/internal/client/msgraph/delta_test.go b/internal/client/msgraph/delta_test.go new file mode 100644 index 000000000..2b4307526 --- /dev/null +++ b/internal/client/msgraph/delta_test.go @@ -0,0 +1,141 @@ +package msgraph + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "golang.org/x/oauth2" +) + +func deltaTestClient(t *testing.T, handler http.HandlerFunc, links map[string]string) *Client { + t.Helper() + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + c := &Client{Email: "box@outlook.test", DeltaLinks: links} + ctx := context.WithValue(t.Context(), oauth2.HTTPClient, &http.Client{Transport: rewriteTo(srv.URL)}) + if err := c.Init(ctx, &oauth2.Token{AccessToken: "live", Expiry: time.Now().Add(time.Hour)}, oauth2.Config{}); err != nil { + t.Fatal(err) + } + return c +} + +func TestDeltaExpiryClassificationIsNarrow(t *testing.T) { + for _, tc := range []struct { + status int + code string + want bool + }{ + {410, "syncStateNotFound", true}, {410, "", true}, {400, "syncStateNotFound", true}, + {400, "InvalidRequest", false}, {401, "syncStateNotFound", false}, + {403, "syncStateNotFound", false}, {429, "syncStateNotFound", false}, {503, "syncStateNotFound", false}, + } { + resp := &http.Response{StatusCode: tc.status, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(`{"error":{"code":"` + tc.code + `"}}`))} + if got := deltaExpired(HandleError(resp)); got != tc.want { + t.Errorf("status=%d code=%s: expiry=%v, want %v", tc.status, tc.code, got, tc.want) + } + } + if deltaExpired(context.DeadlineExceeded) { + t.Fatal("transport failure reset a checkpoint") + } +} + +func TestDeltaExpiredRecoveryReplaysBeforeCheckpointAndResumesOnReload(t *testing.T) { + base := "https://graph.microsoft.com/v1.0/me/mailFolders/inbox/messages/delta" + saved := base + "?$deltatoken=expired" + page2 := base + "?$skiptoken=recovery-2" + fresh := base + "?$deltatoken=fresh" + requests := []string{} + handler := func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + requests = append(requests, r.URL.RawQuery) + switch { + case r.URL.Query().Get("$deltatoken") == "expired": + w.WriteHeader(http.StatusGone) + _, _ = w.Write([]byte(`{"error":{"code":"syncStateNotFound"}}`)) + case r.URL.Query().Get("$skiptoken") == "recovery-2": + _ = json.NewEncoder(w).Encode(map[string]any{"value": []any{map[string]any{"id": "later", "isRead": false}}, "@odata.deltaLink": fresh}) + default: + _ = json.NewEncoder(w).Encode(map[string]any{"value": []any{map[string]any{"id": "unseen", "isRead": false}, map[string]any{"id": "known", "isRead": true}}, "@odata.nextLink": page2}) + } + } + delivered := map[string]bool{} + c := deltaTestClient(t, handler, map[string]string{FolderInbox: saved}) + denyLater := true + c.OnMessageSeen = func(_ context.Context, _, id string, seen bool) (bool, error) { + if id == "later" && denyLater { + return false, ErrStop + } + if id == "known" && !seen { + t.Fatal("lost changed read state") + } + delivered[id] = true + return true, nil + } + c.OnDelta = func(_ context.Context, _, link string) error { + if link == fresh && !delivered["later"] { + t.Fatal("fresh baseline published before replay") + } + saved = link + return nil + } + if err := c.syncFolder(t.Context(), FolderInbox); !errors.Is(err, ErrStop) { + t.Fatalf("first pass = %v, want budget stop", err) + } + if saved != page2 || !delivered["unseen"] || !delivered["known"] { + t.Fatalf("checkpoint=%s delivered=%v", saved, delivered) + } + denyLater = false + reloaded := deltaTestClient(t, handler, map[string]string{FolderInbox: saved}) + reloaded.OnMessageSeen, reloaded.OnDelta = c.OnMessageSeen, c.OnDelta + before := len(requests) + if err := reloaded.syncFolder(t.Context(), FolderInbox); err != nil { + t.Fatal(err) + } + if saved != fresh || len(delivered) != 3 || len(requests) != before+1 || !strings.Contains(requests[before], "recovery-2") { + t.Fatalf("restart lost progress: saved=%s delivered=%v requests=%v", saved, delivered, requests) + } +} + +func TestDeltaCheckpointPublicationFailurePinsReplay(t *testing.T) { + fresh := "https://graph.microsoft.com/v1.0/me/mailFolders/archive/messages/delta?$deltatoken=fresh" + c := deltaTestClient(t, func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{"value": []any{map[string]any{"id": "unseen"}}, "@odata.deltaLink": fresh}) + }, nil) + delivered := 0 + c.OnMessageSeen = func(context.Context, string, string, bool) (bool, error) { delivered++; return true, nil } + c.OnDelta = func(context.Context, string, string) error { return errors.New("publication failed") } + if err := c.syncFolder(t.Context(), FolderArchive); err == nil || c.DeltaLinks[FolderArchive] != "" || delivered != 1 { + t.Fatalf("publication failure: err=%v links=%v delivered=%d", err, c.DeltaLinks, delivered) + } +} + +func TestDeltaContinuesLaterFoldersAndSkipsOnlyAbsentArchive(t *testing.T) { + for _, earlyFailure := range []bool{false, true} { + seen := map[string]bool{} + c := deltaTestClient(t, func(w http.ResponseWriter, r *http.Request) { + folder := strings.Split(strings.TrimPrefix(r.URL.Path, "/v1.0/me/mailFolders/"), "/")[0] + seen[folder] = true + if folder == FolderArchive || (folder == FolderInbox && earlyFailure) { + status := http.StatusNotFound + if folder == FolderInbox { + status = http.StatusServiceUnavailable + } + w.WriteHeader(status) + _, _ = w.Write([]byte(`{"error":{"code":"ErrorItemNotFound"}}`)) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{"value": []any{}, "@odata.deltaLink": "https://graph.microsoft.com" + r.URL.Path + "?$deltatoken=fresh"}) + }, nil) + complete, err := c.SyncPass(t.Context()) + if (err != nil) != earlyFailure || complete == earlyFailure || len(seen) != len(TrackedFolders) { + t.Fatalf("earlyFailure=%v complete=%v err=%v seen=%v", earlyFailure, complete, err, seen) + } + } +} diff --git a/internal/client/msgraph/error.go b/internal/client/msgraph/error.go index 0d3374310..56d09b731 100644 --- a/internal/client/msgraph/error.go +++ b/internal/client/msgraph/error.go @@ -2,13 +2,25 @@ package msgraph import ( "encoding/json" + "errors" "io" "net/http" + "strings" "time" "github.com/warmbly/warmbly/internal/errx" ) +func deltaExpired(err error) bool { + var mailErr *errx.MailError + if !errors.As(err, &mailErr) || mailErr.Failure == nil { + return false + } + failure := mailErr.Failure + return failure.Status == http.StatusGone || + (failure.Status == http.StatusBadRequest && strings.EqualFold(failure.Cause, "syncStateNotFound")) +} + // graphErrorEnvelope is Graph's standard error shape. type graphErrorEnvelope struct { Error struct { diff --git a/internal/client/msgraph/message.go b/internal/client/msgraph/message.go index 764f5f25b..150739c44 100644 --- a/internal/client/msgraph/message.go +++ b/internal/client/msgraph/message.go @@ -15,6 +15,7 @@ import ( // the envelope, body, and headers. type GraphMessage struct { ID string `json:"id"` + ParentFolderID string `json:"parentFolderId"` InternetMessageID string `json:"internetMessageId"` ConversationID string `json:"conversationId"` Subject string `json:"subject"` diff --git a/internal/client/msgraph/msgraph.go b/internal/client/msgraph/msgraph.go index e66e979b5..0a3876687 100644 --- a/internal/client/msgraph/msgraph.go +++ b/internal/client/msgraph/msgraph.go @@ -11,7 +11,9 @@ import ( "bytes" "context" "encoding/json" + "errors" "io" + "net" "net/http" "net/url" "sync" @@ -55,7 +57,9 @@ type Client struct { // DeltaLinks holds the opaque per-folder delta cursor (well-known folder // name -> deltaLink URL). Seeded from persisted state on init and advanced // as sync runs; OnDelta persists each new value off the disposable worker. - DeltaLinks map[string]string + DeltaLinks map[string]string + ImmutableIDs bool + idMu sync.RWMutex // folderIDs caches resolved folder ids (e.g. the created "Warmbly" folder) // so we don't re-list on every warmup action. @@ -66,10 +70,16 @@ type Client struct { // read state: the caller dedupes, hydrates (FetchMessage) and stores as // budget allows, and returns false for a message it left on the server, // which pins the folder cursor before that page. - OnMessageSeen func(ctx context.Context, folder, providerID string, seen bool) (stored bool, err error) - OnMessageRemove func(ctx context.Context, providerID string) error - OnDelta func(ctx context.Context, folder, deltaLink string) error - OnTokenRefresh func(ctx context.Context, token *oauth2.Token) error + OnMessageSeen func(ctx context.Context, folder, providerID string, seen bool) (stored bool, err error) + OnMessageRemove func(ctx context.Context, providerID string) error + OnDelta func(ctx context.Context, folder, deltaLink string) error + OnTokenRefresh func(ctx context.Context, token *oauth2.Token) error + OnFolderReconcile func(ctx context.Context, folder string) (bool, error) + OnRecoveryStart func(ctx context.Context, folder string) error + OnRecoveryCheckpoint func(ctx context.Context, folder, link string) error + OnRecoveryBaseline func(ctx context.Context, folder, link string) error + PendingRecoveryLink func(folder string) (string, error) + Recovering func(folder string) bool } // Init builds the auto-refreshing OAuth2 HTTP client. It mirrors goog.Client.Init: @@ -77,6 +87,7 @@ type Client struct { // cfg and persists the new token via OnTokenRefresh (the worker relays it back to // the control plane, which owns the encrypted credential store). func (c *Client) Init(ctx context.Context, token *oauth2.Token, cfg oauth2.Config) *errx.MailError { + ctx = stoken.BoundedContext(ctx) ts := cfg.TokenSource(ctx, token) ts = oauth2.ReuseTokenSource(token, ts) // Same guard as goog.Init: a nil OnTokenRefresh would panic per request. @@ -91,7 +102,7 @@ func (c *Client) Init(ctx context.Context, token *oauth2.Token, cfg oauth2.Confi // InitWithSource builds the client on a caller-owned token source (brokered // tokens from Warmbly Cloud); nothing is persisted from it. func (c *Client) InitWithSource(ctx context.Context, ts oauth2.TokenSource) *errx.MailError { - c.hc = oauth2.NewClient(ctx, ts) + c.hc = stoken.HTTPClient(ctx, ts) if c.DeltaLinks == nil { c.DeltaLinks = map[string]string{} } @@ -120,6 +131,9 @@ func (c *Client) do(ctx context.Context, method, url, contentType string, body [ if contentType != "" { req.Header.Set("Content-Type", contentType) } + if c.ImmutableIDMode() { + req.Header.Set("Prefer", `IdType="ImmutableId", odata.maxpagesize=50`) + } return c.hc.Do(req) } @@ -147,9 +161,16 @@ func (c *Client) doJSON(ctx context.Context, method, url string, in, out any) er return HandleError(resp) } if out != nil { - return json.NewDecoder(resp.Body).Decode(out) + err := json.NewDecoder(resp.Body).Decode(out) + var networkError net.Error + if errors.As(err, &networkError) || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { + return transportError(err) + } + return err + } + if _, err := io.Copy(io.Discard, resp.Body); err != nil { + return transportError(err) } - _, _ = io.Copy(io.Discard, resp.Body) return nil } diff --git a/internal/client/msgraph/sync_identity.go b/internal/client/msgraph/sync_identity.go new file mode 100644 index 000000000..7c156797a --- /dev/null +++ b/internal/client/msgraph/sync_identity.go @@ -0,0 +1,80 @@ +package msgraph + +import ( + "context" + "fmt" + "net/http" + + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" +) + +func (c *Client) ImmutableIDMode() bool { + c.idMu.RLock() + defer c.idMu.RUnlock() + return c.ImmutableIDs +} + +func (c *Client) SetImmutableIDMode(enabled bool) { + c.idMu.Lock() + defer c.idMu.Unlock() + c.ImmutableIDs = enabled +} + +// ImmutableMessageIDs converts saved regular ids without changing mailbox contents. +func (c *Client) ImmutableMessageIDs(ctx context.Context, ids []string) (map[string]string, error) { + return c.translateIDs(ctx, ids, "restId", "restImmutableEntryId") +} + +func (c *Client) RegularMessageIDs(ctx context.Context, ids []string) (map[string]string, error) { + return c.translateIDs(ctx, ids, "restImmutableEntryId", "restId") +} + +func (c *Client) translateIDs(ctx context.Context, ids []string, source, target string) (map[string]string, error) { + var out struct { + Value []struct { + SourceID string `json:"sourceId"` + TargetID string `json:"targetId"` + } `json:"value"` + } + if err := c.doJSON(ctx, http.MethodPost, c.root()+"/translateExchangeIds", map[string]any{ + "inputIds": ids, "sourceIdType": source, "targetIdType": target, + }, &out); err != nil { + return nil, err + } + converted := make(map[string]string, len(out.Value)) + for _, id := range out.Value { + if id.SourceID != "" && id.TargetID != "" { + converted[id.SourceID] = id.TargetID + } + } + for _, id := range ids { + if converted[id] == "" { + return nil, fmt.Errorf("graph: incomplete immutable id conversion") + } + } + return converted, nil +} + +// MessageFolder resolves current placement, not the source of a stale delta item. +func (c *Client) MessageFolder(ctx context.Context, msg *GraphMessage, fallback string) (string, error) { + if msg.ParentFolderID == "" { + return msg.ToEmailData(fallback).Folder, nil + } + for _, folder := range append(append([]string{}, TrackedFolders...), FolderDeletedItems) { + id, err := c.wellKnownFolderID(ctx, folder) + if err != nil { + if merr, ok := err.(*errx.MailError); ok && folder == FolderArchive && merr.Code == errx.MailErrorCodeNotFound { + continue + } + return "", err + } + if msg.ParentFolderID == id { + if folder == FolderDeletedItems { + return models.FolderTrash, nil + } + return msg.ToEmailData(folder).Folder, nil + } + } + return "", nil +} diff --git a/internal/client/smtpimap/imap/body_read.go b/internal/client/smtpimap/imap/body_read.go index 3d198416b..eb8b75640 100644 --- a/internal/client/smtpimap/imap/body_read.go +++ b/internal/client/smtpimap/imap/body_read.go @@ -3,6 +3,7 @@ package imap import ( "bytes" "encoding/base64" + "errors" "io" "mime/quotedprintable" "strconv" @@ -35,14 +36,14 @@ const maxTextParts = 5 // bytes back as the HTML body too, so plain text was rendered as markup: line // breaks collapsed, "&" showed as an entity, and anything inside angle // brackets vanished. -func fetchTextParts(c *imapclient.Client, uid imap.UID, bs imap.BodyStructure) (plain, html string) { +func fetchTextParts(c *imapclient.Client, uid imap.UID, bs imap.BodyStructure) (plain, html string, err error) { if bs == nil { - return "", "" + return "", "", nil } wanted := selectTextParts(bs) if len(wanted) == 0 { - return "", "" + return "", "", nil } sections := make([]*imap.FetchItemBodySection, 0, len(wanted)) @@ -56,12 +57,15 @@ func fetchTextParts(c *imapclient.Client, uid imap.UID, bs imap.BodyStructure) ( }) } - raw := fetchSections(c, uid, sections) + raw, err := fetchSections(c, uid, sections) + if err != nil { + return "", "", err + } var plainParts, htmlParts []string for _, p := range wanted { body, ok := raw[partKey(p.path)] if !ok { - continue + return "", "", errors.New("imap: required body section missing") } decoded := decodePart(body, p.encoding, p.charset) if p.html { @@ -72,7 +76,7 @@ func fetchTextParts(c *imapclient.Client, uid imap.UID, bs imap.BodyStructure) ( } // Sibling inline parts (a body followed by a disclaimer, say) are pieces of // one message, so they are joined rather than one replacing the other. - return strings.Join(plainParts, "\n\n"), strings.Join(htmlParts, "") + return strings.Join(plainParts, "\n\n"), strings.Join(htmlParts, ""), nil } // reportPartTypes are the machine-readable parts of a multipart/report. They @@ -159,16 +163,20 @@ func selectTextParts(bs imap.BodyStructure) []textPart { // fetchSections runs one FETCH for the given sections and returns each part's // raw (still transfer-encoded) bytes, keyed by part path. -func fetchSections(c *imapclient.Client, uid imap.UID, sections []*imap.FetchItemBodySection) map[string][]byte { +var errBodyMessageGone = errors.New("imap: body message no longer exists") + +func fetchSections(c *imapclient.Client, uid imap.UID, sections []*imap.FetchItemBodySection) (map[string][]byte, error) { out := make(map[string][]byte, len(sections)) cmd := c.Fetch(imap.UIDSetNum(uid), &imap.FetchOptions{ UID: true, BodySection: sections, }) - defer cmd.Close() + seen := false + var readErr error for msg := cmd.Next(); msg != nil; msg = cmd.Next() { + seen = true for item := msg.Next(); item != nil; item = msg.Next() { v, ok := item.(imapclient.FetchItemDataBodySection) if !ok || v.Literal == nil || v.Section == nil { @@ -178,13 +186,29 @@ func fetchSections(c *imapclient.Client, uid imap.UID, sections []*imap.FetchIte // Belt and braces alongside the server-side Partial: a server that // ignores the partial range must not be able to stream us an // unbounded body. - if _, err := io.Copy(buf, io.LimitReader(v.Literal, int64(config.MaxEmailBodySize))); err != nil { + n, err := io.Copy(buf, io.LimitReader(v.Literal, int64(config.MaxEmailBodySize))) + if err == nil && n < min(v.Literal.Size(), int64(config.MaxEmailBodySize)) { + err = io.ErrUnexpectedEOF + } + if err != nil { + // Stop the decoder before go-imap tries discarding a short literal a second time. + _ = c.Close() + readErr = err continue } out[partKey(v.Section.Part)] = buf.Bytes() } } - return out + if err := cmd.Close(); err != nil { + return nil, err + } + if readErr != nil { + return nil, readErr + } + if !seen { + return nil, errBodyMessageGone + } + return out, nil } // partKey renders an IMAP part path ("1", "1.2") so a fetched section can be diff --git a/internal/client/smtpimap/imap/client.go b/internal/client/smtpimap/imap/client.go index ec9ea19e9..c023f57de 100644 --- a/internal/client/smtpimap/imap/client.go +++ b/internal/client/smtpimap/imap/client.go @@ -35,6 +35,8 @@ var headerFetchFields = append([]string{config.WarmupVerifyHeader}, config.Inbou var evidenceFetchFields = append(append([]string{}, headerFetchFields...), "Authentication-Results", "From", "Return-Path", "DKIM-Signature", "List-Unsubscribe", "List-Unsubscribe-Post") type Client struct { + // Context optionally binds connections and setup to the installed mailbox lifetime. + Context context.Context Email string AuthType models.AuthType Credentials *models.Service @@ -79,6 +81,12 @@ type Client struct { // like client itself. conn *idleConn + setupMu sync.Mutex + setupCancel context.CancelFunc + transport net.Conn + lifetimeStop func() bool + closed atomic.Bool + // IdleTimeout bounds how long a command waits for the server to say // anything before the session is declared dead. Zero means // config.ImapCommandIdleTimeout. @@ -130,6 +138,11 @@ func (c *Client) HasCondStore() bool { return c.condStore.Load() } +func (c *Client) selectedCondStore() bool { + sel := c.selection.Load() + return c.condStore.Load() && sel != nil && sel.modSeq > 0 +} + // ensureConnected re-dials after the server has dropped the session. go-imap // parks a dead client in the Logout state and fails every later command with // net.ErrClosed; nothing re-dialed, so one drop (Gmail closes sessions after a @@ -139,6 +152,9 @@ func (c *Client) HasCondStore() bool { func (c *Client) ensureConnected() *errx.MailError { c.lifecycle.Lock() defer c.lifecycle.Unlock() + if c.closed.Load() || c.Context != nil && c.Context.Err() != nil { + return errx.ErrMailServerUnreachable + } // Only a session that got past auth is worth keeping: a failed Login // leaves go-imap in NotAuthenticated, which is just as unusable as Logout. @@ -160,6 +176,18 @@ func (c *Client) Connect() *errx.MailError { // connectLocked dials and authenticates a fresh session. lifecycle must be // held for writing. func (c *Client) connectLocked() *errx.MailError { + if c.closed.Load() || c.Context != nil && c.Context.Err() != nil { + return errx.ErrMailServerUnreachable + } + c.setupMu.Lock() + if c.lifetimeStop != nil { + c.lifetimeStop() + } + if c.transport != nil { + _ = c.transport.Close() + c.transport = nil + } + c.setupMu.Unlock() var addr, host, security string var port int switch c.AuthType { @@ -193,7 +221,26 @@ func (c *Client) connectLocked() *errx.MailError { if resolved == models.MailSecurityNone && !models.CleartextMailAllowed(host) { return errx.ErrMailInsecureRemoteHost } - raw, err := netbind.Dialer(c.BindIP).DialContext(context.Background(), "tcp", addr) + lifetime := c.Context + if lifetime == nil { + lifetime = context.Background() + } + setupCtx, cancel := context.WithTimeout(lifetime, timeout) + c.setupMu.Lock() + if c.closed.Load() { + c.setupMu.Unlock() + cancel() + return errx.ErrMailServerUnreachable + } + c.setupCancel = cancel + c.setupMu.Unlock() + defer func() { + c.setupMu.Lock() + c.setupCancel = nil + c.setupMu.Unlock() + cancel() + }() + raw, err := netbind.Dialer(c.BindIP).DialContext(setupCtx, "tcp", addr) if err != nil { return errx.ErrMailServerUnreachable } @@ -201,7 +248,21 @@ func (c *Client) connectLocked() *errx.MailError { _ = raw.Close() return errx.ErrMailInsecureRemoteHost } + c.setupMu.Lock() + if c.closed.Load() { + c.setupMu.Unlock() + _ = raw.Close() + return errx.ErrMailServerUnreachable + } + c.transport = raw + c.lifetimeStop = context.AfterFunc(lifetime, func() { _ = raw.Close() }) + c.setupMu.Unlock() conn := &idleConn{Conn: raw, timeout: timeout} + // Library greeting/STARTTLS waits precede client installation; close the pending socket too. + stopSetup := context.AfterFunc(setupCtx, func() { _ = raw.Close() }) + defer stopSetup() + done := conn.arm() + defer done() var client *imapclient.Client switch { @@ -216,9 +277,7 @@ func (c *Client) connectLocked() *errx.MailError { } default: tconn := tls.Client(conn, tlsConf) - hctx, cancel := context.WithTimeout(context.Background(), timeout) - err = tconn.HandshakeContext(hctx) - cancel() + err = tconn.HandshakeContext(setupCtx) if err != nil { _ = tconn.Close() return errx.ErrMailServerUnreachable @@ -231,9 +290,6 @@ func (c *Client) connectLocked() *errx.MailError { c.selected.Store(false) c.selection.Store(nil) c.condStore.Store(false) - done := conn.arm() - defer done() - var xerr *errx.MailError switch c.AuthType { @@ -254,17 +310,34 @@ func (c *Client) connectLocked() *errx.MailError { // check must run post-auth. Without it (Outlook.com, Microsoft 365 over // IMAP, Yahoo, many hosted servers) the sync loop keys on UIDNEXT instead. c.condStore.Store(c.client.Caps().Has(imap.CapCondStore)) + if !stopSetup() || setupCtx.Err() != nil { + _ = client.Close() + return errx.ErrMailServerUnreachable + } return nil } func (c *Client) Close() error { - c.lifecycle.RLock() - defer c.lifecycle.RUnlock() - if c.client == nil { + if !c.closed.CompareAndSwap(false, true) { return nil } - return c.client.Close() + c.setupMu.Lock() + defer c.setupMu.Unlock() + if c.setupCancel != nil { + c.setupCancel() + } + if c.lifetimeStop != nil { + c.lifetimeStop() + } + if c.transport == nil { + return nil + } + err := c.transport.Close() + if errors.Is(err, net.ErrClosed) { + return nil + } + return err } func (c *Client) plainAuth() *errx.MailError { @@ -312,7 +385,7 @@ func (c *Client) selectMailbox(mailbox string, opts *imap.SelectOptions) (*imap. data, err := c.client.Select(mailbox, opts).Wait() c.selected.Store(err == nil) if err == nil { - c.selection.Store(&selection{name: mailbox, uidValidity: data.UIDValidity, count: data.NumMessages}) + c.selection.Store(&selection{name: mailbox, uidValidity: data.UIDValidity, count: data.NumMessages, modSeq: data.HighestModSeq}) } return data, err } @@ -649,7 +722,7 @@ func (c *Client) FetchEnvelopes(ctx context.Context, uids []imap.UID) ([]*Fetche // never advertised it for one is a malformed fetch-att, and a strict // parser answers BAD and syncs nothing: issue #405, where IONOS said // `BAD expected fetch-att instead of "MODSEQ BODY.P"` on every pass. - ModSeq: c.condStore.Load(), + ModSeq: c.selectedCondStore(), InternalDate: true, RFC822Size: true, BodySection: []*imap.FetchItemBodySection{{ @@ -720,9 +793,9 @@ func (c *Client) FetchEnvelopes(ctx context.Context, uids []imap.UID) ([]*Fetche // the FetchEnvelopes command that produced it is closed: a nested FETCH on // the same connection blocks until the outer one finishes, and the outer one // cannot finish while we wait, which deadlocks the sync on the first message. -func (c *Client) FetchBody(f *Fetched) { +func (c *Client) FetchBody(f *Fetched) *errx.MailError { if f == nil || f.Email == nil { - return + return nil } c.mu.Lock() defer c.mu.Unlock() @@ -730,10 +803,20 @@ func (c *Client) FetchBody(f *Fetched) { defer c.lifecycle.RUnlock() defer c.begin()() if err := c.resumeSyncLocked(); err != nil { - log.Debug().Err(err).Uint32("uid", uint32(f.uid)).Msg("imap: could not re-open the folder to read a body") - return + return errx.ErrMailServerUnreachable } - f.Email.BodyPlain, f.Email.BodyHTML = fetchTextParts(c.client, f.uid, f.body) + plain, html, err := fetchTextParts(c.client, f.uid, f.body) + if errors.Is(err, errBodyMessageGone) { + return errx.ErrMailResourceNotFound + } + if err != nil { + if mailErr := c.handleError(err); mailErr.Code != errx.MailErrorCodeNotFound { + return mailErr + } + return errx.ErrMailServerUnreachable + } + f.Email.BodyPlain, f.Email.BodyHTML = plain, html + return nil } // parseHeaderFlags reads a HEADER.FIELDS literal and renders the fetched diff --git a/internal/client/smtpimap/imap/fetch_caps_test.go b/internal/client/smtpimap/imap/fetch_caps_test.go index e92cfb9d0..0fbe15b6c 100644 --- a/internal/client/smtpimap/imap/fetch_caps_test.go +++ b/internal/client/smtpimap/imap/fetch_caps_test.go @@ -87,6 +87,10 @@ func TestFetchEnvelopesAsksModSeqWithCondStore(t *testing.T) { t.Fatalf("SelectForSync: %v", err) } c.condStore.Store(true) + selected := *c.selection.Load() + selected.modSeq = 1 + c.selection.Store(&selected) + c.syncView = &selected // The result is deliberately ignored: this server refuses MODSEQ when // CONDSTORE was not enabled, which is #405's failure reproduced in diff --git a/internal/client/smtpimap/imap/search_date.go b/internal/client/smtpimap/imap/search_date.go index 45b6efc73..468244a0d 100644 --- a/internal/client/smtpimap/imap/search_date.go +++ b/internal/client/smtpimap/imap/search_date.go @@ -22,6 +22,7 @@ type selection struct { name string uidValidity uint32 count uint32 + modSeq uint64 } // dateScan is the INTERNALDATE read of one folder for one window. UIDs below diff --git a/internal/client/smtpimap/imap/sync_repair_wire_test.go b/internal/client/smtpimap/imap/sync_repair_wire_test.go new file mode 100644 index 000000000..f8a3c264e --- /dev/null +++ b/internal/client/smtpimap/imap/sync_repair_wire_test.go @@ -0,0 +1,525 @@ +package imap + +import ( + "bufio" + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "io" + "net" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + goimap "github.com/emersion/go-imap/v2" + "github.com/emersion/go-imap/v2/imapclient" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" +) + +func repairWireServer(t *testing.T, serve func(net.Conn)) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = ln.Close() }) + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + go func() { defer func() { _ = conn.Close() }(); serve(conn) }() + } + }() + return ln.Addr().String() +} + +// This fixture advertises global CONDSTORE but selects a NOMODSEQ folder. +func bodyRepairFixture(t *testing.T, bodyReply func(net.Conn, string) bool) (*Client, *wireLog) { + t.Helper() + wire := &wireLog{} + addr := repairWireServer(t, func(conn net.Conn) { + _, _ = io.WriteString(conn, "* OK [CAPABILITY IMAP4rev1 CONDSTORE] ready\r\n") + r := bufio.NewReader(conn) + for { + line, err := r.ReadString('\n') + if err != nil { + return + } + wire.add(strings.TrimSpace(line)) + fields := strings.Fields(line) + if len(fields) < 2 { + return + } + tag := fields[0] + switch strings.ToUpper(fields[1]) { + case "LOGIN": + _, _ = fmt.Fprintf(conn, "%s OK [CAPABILITY IMAP4rev1 CONDSTORE] authenticated\r\n", tag) + case "CAPABILITY": + _, _ = fmt.Fprintf(conn, "* CAPABILITY IMAP4rev1 CONDSTORE\r\n%s OK capability\r\n", tag) + case "EXAMINE": + _, _ = fmt.Fprintf(conn, "* 1 EXISTS\r\n* OK [UIDVALIDITY 7] generation\r\n* OK [UIDNEXT 2] next\r\n* OK [NOMODSEQ] no modseq\r\n%s OK [READ-ONLY] selected\r\n", tag) + case "UID": + if len(fields) > 2 && strings.EqualFold(fields[2], "SEARCH") { + _, _ = fmt.Fprintf(conn, "* SEARCH 1\r\n%s OK search\r\n", tag) + } else if strings.Contains(strings.ToUpper(line), "BODYSTRUCTURE") { + _, _ = fmt.Fprintf(conn, "* 1 FETCH (UID 1 FLAGS () ENVELOPE (NIL \"hello\" NIL NIL NIL NIL NIL NIL NIL \"\") BODYSTRUCTURE (\"TEXT\" \"PLAIN\" (\"CHARSET\" \"UTF-8\") NIL NIL \"7BIT\" 13 1))\r\n%s OK envelope\r\n", tag) + } else if bodyReply != nil { + if !bodyReply(conn, tag) { + return + } + } else { + _, _ = fmt.Fprintf(conn, "* 1 FETCH (UID 1 FLAGS (\\Seen) ENVELOPE (NIL NIL NIL NIL NIL NIL NIL NIL NIL \"\"))\r\n%s OK flags\r\n", tag) + } + default: + _, _ = fmt.Fprintf(conn, "%s BAD unexpected command\r\n", tag) + } + } + }) + return clientTo(t, addr), wire +} + +func TestFetchBodyWireErrorsRemainRetryable(t *testing.T) { + for _, failure := range []string{"tagged refusal", "short literal", "reset literal", "missing section", "failed completion"} { + t.Run(failure, func(t *testing.T) { + var attempts atomic.Int32 + c, _ := bodyRepairFixture(t, func(conn net.Conn, tag string) bool { + if attempts.Add(1) == 1 { + switch failure { + case "tagged refusal": + _, _ = fmt.Fprintf(conn, "%s NO [NONEXISTENT] body refused\r\n", tag) + case "short literal": + _, _ = io.WriteString(conn, "* 1 FETCH (UID 1 BODY[1] {13}\r\nshort") + return false + case "reset literal": + _, _ = io.WriteString(conn, "* 1 FETCH (UID 1 BODY[1] {13}\r\nshort") + _ = conn.(*net.TCPConn).SetLinger(0) + return false + case "missing section": + _, _ = fmt.Fprintf(conn, "* 1 FETCH (UID 1)\r\n%s OK completed\r\n", tag) + case "failed completion": + _, _ = fmt.Fprintf(conn, "* 1 FETCH (UID 1 BODY[1] {13}\r\ncomplete body)\r\n%s NO incomplete\r\n", tag) + } + return true + } + _, _ = fmt.Fprintf(conn, "* 1 FETCH (UID 1 BODY[1] {13}\r\ncomplete body)\r\n%s OK body\r\n", tag) + return true + }) + c.IdleTimeout = time.Second + if err := c.Connect(); err != nil { + t.Fatal(err) + } + if _, err := c.SelectForSync("INBOX"); err != nil { + t.Fatal(err) + } + fetched, err := c.FetchEnvelopes(t.Context(), []goimap.UID{1}) + if err != nil || len(fetched) != 1 { + t.Fatalf("envelope: %v %v", fetched, err) + } + if err := c.FetchBody(fetched[0]); err == nil || err.Code == errx.MailErrorCodeNotFound { + t.Fatalf("hydration failure treated as bodyless or gone: %v", err) + } + if fetched[0].Email.BodyPlain != "" || fetched[0].Email.BodyHTML != "" { + t.Fatal("failed body installed partial text") + } + if err := c.ensureConnected(); err != nil { + t.Fatal(err) + } + if _, err := c.SelectForSync("INBOX"); err != nil { + t.Fatal(err) + } + fetched, err = c.FetchEnvelopes(t.Context(), []goimap.UID{1}) + if err != nil || len(fetched) != 1 { + t.Fatalf("retry envelope: %v %v", fetched, err) + } + if err := c.FetchBody(fetched[0]); err != nil { + t.Fatal(err) + } + if fetched[0].Email.BodyPlain != "complete body" { + t.Fatalf("retry body = %q", fetched[0].Email.BodyPlain) + } + }) + } +} + +func TestFetchBodyFolderResumeErrorAndValidEmptyBodies(t *testing.T) { + var refuse atomic.Bool + c, _ := interceptingServer(t, nil, func(line string) string { + if refuse.Load() && strings.Contains(strings.ToUpper(line), "EXAMINE") { + return strings.Fields(line)[0] + " NO folder unavailable" + } + return "" + }, "Archive") + if err := c.Connect(); err != nil { + t.Fatal(err) + } + uid := putMessage(t, c) + if _, err := c.SelectForSync("INBOX"); err != nil { + t.Fatal(err) + } + fetched, err := c.FetchEnvelopes(t.Context(), []goimap.UID{uid}) + if err != nil || len(fetched) != 1 { + t.Fatal("envelope failed") + } + if _, err := c.selectMailbox("Archive", nil); err != nil { + t.Fatal(err) + } + refuse.Store(true) + if err := c.FetchBody(fetched[0]); err == nil { + t.Fatal("failed folder resume swallowed") + } + refuse.Store(false) + if err := c.FetchBody(fetched[0]); err != nil { + t.Fatal(err) + } + for _, bs := range []goimap.BodyStructure{nil, &goimap.BodyStructureSinglePart{Type: "application", Subtype: "octet-stream"}, &goimap.BodyStructureSinglePart{Type: "text", Subtype: "plain", Extended: &goimap.BodyStructureSinglePartExt{Disposition: &goimap.BodyStructureDisposition{Value: "attachment"}}}} { + f := &Fetched{Email: &models.EmailMessageData{}, uid: uid, body: bs} + if err := c.FetchBody(f); err != nil || f.Email.BodyPlain != "" || f.Email.BodyHTML != "" { + t.Fatalf("valid no-text message refused: %v", err) + } + } +} + +func TestNomodseqWireOmitsModseqAndChangedSince(t *testing.T) { + c, wire := bodyRepairFixture(t, nil) + if err := c.Connect(); err != nil { + t.Fatal(err) + } + if !c.HasCondStore() { + t.Fatal("fixture did not advertise CONDSTORE") + } + view, err := c.SelectForSyncState("INBOX") + if err != nil || view.HighestModSeq != 0 { + t.Fatalf("NOMODSEQ SELECT: %+v %v", view, err) + } + if _, err := c.FetchEnvelopes(t.Context(), []goimap.UID{1}); err != nil { + t.Fatal(err) + } + if uids, err := c.SearchNewSince(1); err != nil || len(uids) != 1 { + t.Fatalf("UIDNEXT fallback: %v %v", uids, err) + } + flags, err := c.FetchFlags(t.Context(), 1) + if err != nil || len(flags) != 1 { + t.Fatalf("flag fallback: %v %v", flags, err) + } + for _, line := range wire.commands("FETCH") { + if strings.Contains(line, "MODSEQ") || strings.Contains(line, "CHANGEDSINCE") { + t.Fatalf("NOMODSEQ command: %s", line) + } + } + for _, line := range wire.commands("SEARCH") { + if strings.Contains(line, "MODSEQ") || strings.Contains(line, "CHANGEDSINCE") { + t.Fatalf("NOMODSEQ search: %s", line) + } + } +} + +func TestSetupBlackholesAreBoundedAndCanRedial(t *testing.T) { + for _, phase := range []string{"greeting", "STARTTLS response", "STARTTLS handshake", "implicit TLS handshake", "authentication"} { + t.Run(phase, func(t *testing.T) { + var attempts atomic.Int32 + wire := &wireLog{} + addr := repairWireServer(t, func(conn net.Conn) { + attempt := attempts.Add(1) + if attempt == 1 && (phase == "greeting" || phase == "implicit TLS handshake") { + _, _ = io.Copy(io.Discard, conn) + return + } + _, _ = io.WriteString(conn, "* OK [CAPABILITY IMAP4rev1 STARTTLS] ready\r\n") + r := bufio.NewReader(conn) + for { + line, err := r.ReadString('\n') + if err != nil { + return + } + wire.add(strings.TrimSpace(line)) + fields := strings.Fields(line) + if len(fields) < 2 { + return + } + tag := fields[0] + if attempt == 1 { + if phase == "STARTTLS handshake" { + _, _ = fmt.Fprintf(conn, "%s OK begin TLS\r\n", tag) + } + _, _ = io.Copy(io.Discard, conn) + return + } + switch strings.ToUpper(fields[1]) { + case "LOGIN": + _, _ = fmt.Fprintf(conn, "%s OK [CAPABILITY IMAP4rev1] logged in\r\n", tag) + case "CAPABILITY": + _, _ = fmt.Fprintf(conn, "* CAPABILITY IMAP4rev1\r\n%s OK capability\r\n", tag) + default: + _, _ = fmt.Fprintf(conn, "%s BAD unexpected command\r\n", tag) + } + } + }) + c := clientTo(t, addr) + c.plaintext = phase == "authentication" + c.Credentials.Security = models.MailSecurityStartTLS + if phase == "implicit TLS handshake" { + c.Credentials.Security = models.MailSecurityTLS + } + c.IdleTimeout = 100 * time.Millisecond + start := time.Now() + if err := c.Connect(); err == nil { + t.Fatal("blackhole succeeded") + } + if time.Since(start) > 2*time.Second { + t.Fatal("setup budget did not bound silent peer") + } + if phase != "authentication" && len(wire.commands("LOGIN")) != 0 { + t.Fatal("TLS failure fell back to plaintext authentication") + } + // The second local connection uses test-only plaintext to isolate redial from certificate trust. + c.plaintext = true + if err := c.ensureConnected(); err != nil { + t.Fatalf("redial: %v", err) + } + if attempts.Load() != 2 { + t.Fatalf("connections = %d", attempts.Load()) + } + }) + } +} + +func TestCloseAndLifetimeCancellationInterruptPendingSetup(t *testing.T) { + for _, explicit := range []bool{false, true} { + t.Run(fmt.Sprintf("explicit close=%t", explicit), func(t *testing.T) { + accepted := make(chan struct{}) + ended := make(chan struct{}) + addr := repairWireServer(t, func(conn net.Conn) { close(accepted); _, _ = io.Copy(io.Discard, conn); close(ended) }) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + c := clientTo(t, addr) + c.plaintext = false + c.Context = ctx + c.IdleTimeout = 10 * time.Second + c.Credentials.Security = models.MailSecurityStartTLS + result := make(chan *errx.MailError, 1) + go func() { result <- c.Connect() }() + select { + case <-accepted: + case <-time.After(time.Second): + t.Fatal("setup did not connect") + } + if explicit { + if err := c.Close(); err != nil { + t.Fatal(err) + } + } else { + cancel() + } + select { + case err := <-result: + if err == nil { + t.Fatal("cancelled setup succeeded") + } + case <-time.After(time.Second): + t.Fatal("pending setup did not cancel") + } + select { + case <-ended: + case <-time.After(time.Second): + t.Fatal("pending socket did not close") + } + if err := c.ensureConnected(); err == nil { + t.Fatal("retired lifetime redialed") + } + }) + } +} + +func TestCloseAndLifetimeCancellationCloseIdleSessionWithoutRedial(t *testing.T) { + for _, explicit := range []bool{false, true} { + t.Run(fmt.Sprintf("explicit close=%t", explicit), func(t *testing.T) { + ended := make(chan struct{}) + var attempts atomic.Int32 + addr := repairWireServer(t, func(conn net.Conn) { + attempts.Add(1) + _, _ = io.WriteString(conn, "* OK [CAPABILITY IMAP4rev1] ready\r\n") + r := bufio.NewReader(conn) + for { + line, err := r.ReadString('\n') + if err != nil { + close(ended) + return + } + _, _ = fmt.Fprintf(conn, "%s OK [CAPABILITY IMAP4rev1] authenticated\r\n", strings.Fields(line)[0]) + } + }) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + c := clientTo(t, addr) + c.Context = ctx + if err := c.Connect(); err != nil { + t.Fatal(err) + } + if explicit { + if err := c.Close(); err != nil { + t.Fatal(err) + } + if err := c.Close(); err != nil { + t.Fatal(err) + } + } else { + cancel() + } + select { + case <-ended: + case <-time.After(time.Second): + t.Fatal("idle server did not observe socket EOF") + } + if err := c.ensureConnected(); err == nil || attempts.Load() != 1 { + t.Fatal("retired installed session reconnected") + } + }) + } +} + +func TestStartTLSSetupStillRejectsUntrustedCertificate(t *testing.T) { + t.Setenv("MAIL_TLS_INSECURE", "false") + certServer := httptest.NewTLSServer(nil) + defer certServer.Close() + wire := &wireLog{} + addr := repairWireServer(t, func(conn net.Conn) { + _, _ = io.WriteString(conn, "* OK [CAPABILITY IMAP4rev1 STARTTLS] ready\r\n") + line, err := bufio.NewReader(conn).ReadString('\n') + if err != nil { + return + } + wire.add(strings.TrimSpace(line)) + _, _ = fmt.Fprintf(conn, "%s OK begin TLS\r\n", strings.Fields(line)[0]) + secure := tls.Server(conn, certServer.TLS) + if err := secure.Handshake(); err == nil { + line, _ := bufio.NewReader(secure).ReadString('\n') + wire.add(strings.TrimSpace(line)) + } + }) + c := clientTo(t, addr) + c.plaintext = false + c.Credentials.Security = models.MailSecurityStartTLS + c.IdleTimeout = time.Second + if err := c.Connect(); err == nil { + t.Fatal("untrusted fixture certificate was accepted") + } + if len(wire.commands("LOGIN")) != 0 { + t.Fatal("certificate failure sent plaintext credentials") + } +} + +type literalReadSignalConn struct { + net.Conn + ready chan struct{} + sawShort atomic.Bool + signaled atomic.Bool +} + +func (c *literalReadSignalConn) Read(p []byte) (int, error) { + if c.sawShort.Load() && c.signaled.CompareAndSwap(false, true) { + c.ready <- struct{}{} + } + n, err := c.Conn.Read(p) + if strings.Contains(string(p[:n]), "short") { + c.sawShort.Store(true) + } + return n, err +} + +func TestFetchBodyTLSLiteralFailureReleasesDecoderAndRetries(t *testing.T) { + var attempts atomic.Int32 + bodyStarted := make(chan struct{}, 1) + literalReadReady := make(chan struct{}, 1) + unblockServer := make(chan struct{}) + defer close(unblockServer) + c, _ := bodyRepairFixture(t, func(conn net.Conn, tag string) bool { + if attempts.Add(1) == 1 { + _, _ = io.WriteString(conn, "* 1 FETCH (UID 1 BODY[1] {13}\r\nshort") + bodyStarted <- struct{}{} + <-unblockServer + return false + } + _, _ = fmt.Fprintf(conn, "* 1 FETCH (UID 1 BODY[1] {13}\r\ncomplete body)\r\n%s OK body\r\n", tag) + return true + }) + certServer := httptest.NewTLSServer(nil) + defer certServer.Close() + backendAddr := net.JoinHostPort(c.Credentials.Host, fmt.Sprint(c.Credentials.Port)) + addr := repairWireServer(t, func(raw net.Conn) { + secure := tls.Server(raw, certServer.TLS) + upstream, err := net.DialTimeout("tcp", backendAddr, time.Second) + if err != nil { + return + } + defer func() { _ = upstream.Close() }() + go func() { _, _ = io.Copy(upstream, secure); _ = upstream.Close() }() + _, _ = io.Copy(secure, upstream) + // Close the raw transport without TLS close_notify to produce TLS unexpected EOF. + }) + roots := x509.NewCertPool() + roots.AddCert(certServer.Certificate()) + connect := func() { + raw, err := net.DialTimeout("tcp", addr, time.Second) + if err != nil { + t.Fatal(err) + } + conn := &idleConn{Conn: raw, timeout: time.Second} + secure := tls.Client(conn, &tls.Config{RootCAs: roots, ServerName: certServer.Certificate().DNSNames[0], MinVersion: tls.VersionTLS12}) + if err := secure.HandshakeContext(t.Context()); err != nil { + _ = raw.Close() + t.Fatal(err) + } + c.client = imapclient.New(&literalReadSignalConn{Conn: secure, ready: literalReadReady}, nil) + c.conn = conn + c.transport = raw + c.selected.Store(false) + c.selection.Store(nil) + if err := c.plainAuth(); err != nil { + t.Fatal(err) + } + c.condStore.Store(c.client.Caps().Has(goimap.CapCondStore)) + if _, err := c.SelectForSync("INBOX"); err != nil { + t.Fatal(err) + } + } + connect() + fetched, err := c.FetchEnvelopes(t.Context(), []goimap.UID{1}) + if err != nil || len(fetched) != 1 { + t.Fatalf("TLS envelope: %v %v", fetched, err) + } + result := make(chan *errx.MailError, 1) + go func() { result <- c.FetchBody(fetched[0]) }() + select { + case <-bodyStarted: + case <-time.After(time.Second): + t.Fatal("TLS literal did not start") + } + select { + case <-literalReadReady: + case <-time.After(time.Second): + t.Fatal("TLS literal reader did not block on remaining bytes") + } + _ = c.transport.Close() + select { + case err := <-result: + if err == nil { + t.Fatal("TLS literal failure swallowed") + } + case <-time.After(time.Second): + t.Fatal("TLS literal error left decoder blocked") + } + connect() + fetched, err = c.FetchEnvelopes(t.Context(), []goimap.UID{1}) + if err != nil || len(fetched) != 1 { + t.Fatalf("TLS retry envelope: %v %v", fetched, err) + } + if err := c.FetchBody(fetched[0]); err != nil || fetched[0].Email.BodyPlain != "complete body" { + t.Fatalf("TLS retry body: %q %v", fetched[0].Email.BodyPlain, err) + } +} diff --git a/internal/infrastructure/db/migrations/000278_sync_arrival_outbox.down.sql b/internal/infrastructure/db/migrations/000278_sync_arrival_outbox.down.sql new file mode 100644 index 000000000..f3f03ceac --- /dev/null +++ b/internal/infrastructure/db/migrations/000278_sync_arrival_outbox.down.sql @@ -0,0 +1,8 @@ +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM sync_arrival_outbox) THEN + RAISE EXCEPTION 'Cannot downgrade with unresolved sync arrival delivery'; + END IF; +END $$; + +DROP TABLE sync_arrival_outbox; diff --git a/internal/infrastructure/db/migrations/000278_sync_arrival_outbox.up.sql b/internal/infrastructure/db/migrations/000278_sync_arrival_outbox.up.sql new file mode 100644 index 000000000..ff3ddb279 --- /dev/null +++ b/internal/infrastructure/db/migrations/000278_sync_arrival_outbox.up.sql @@ -0,0 +1,18 @@ +CREATE TABLE sync_arrival_outbox ( + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + email_id uuid NOT NULL REFERENCES email_accounts(id) ON DELETE CASCADE, + organization_id uuid NOT NULL REFERENCES organizations(id) ON DELETE CASCADE, + message_id text NOT NULL, + id uuid NOT NULL, + payload text NOT NULL, + stage smallint NOT NULL DEFAULT 0 CHECK (stage BETWEEN 0 AND 3), + lease uuid, + locked_until timestamptz, + retry_at timestamptz NOT NULL DEFAULT now(), + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, email_id, message_id), + FOREIGN KEY (user_id, email_id, message_id) + REFERENCES email_message_map(user_id, email_id, message_id) ON DELETE CASCADE +); +CREATE INDEX sync_arrival_outbox_due ON sync_arrival_outbox(retry_at, created_at); +CREATE INDEX sync_arrival_outbox_pending ON sync_arrival_outbox(user_id, email_id, id) WHERE stage = 0; diff --git a/internal/infrastructure/db/sync_arrival_upgrade_test.go b/internal/infrastructure/db/sync_arrival_upgrade_test.go new file mode 100644 index 000000000..62f287c31 --- /dev/null +++ b/internal/infrastructure/db/sync_arrival_upgrade_test.go @@ -0,0 +1,106 @@ +package db + +import ( + "context" + "net/url" + "os" + "strings" + "testing" + + "github.com/golang-migrate/migrate/v4/source/iofs" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +func TestLiveSyncArrivalUpgradePreservesLegacyMaps(t *testing.T) { + dsn := os.Getenv("WARMBLY_TEST_DB") + if dsn == "" { + t.Skip("WARMBLY_TEST_DB not set") + } + u, err := url.Parse(dsn) + if err != nil || (u.Hostname() != "127.0.0.1" && u.Hostname() != "localhost") { + t.Fatal("upgrade fixture requires local PostgreSQL") + } + ctx := t.Context() + admin, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer admin.Close(context.Background()) + name := "arrival_upgrade_" + uuid.NewString()[:8] + quoted := pgx.Identifier{name}.Sanitize() + if _, err := admin.Exec(ctx, "CREATE DATABASE "+quoted); err != nil { + t.Fatal(err) + } + defer func() { + if _, err := admin.Exec(context.Background(), "DROP DATABASE "+quoted); err != nil { + t.Error(err) + } + }() + u.Path = "/" + name + source, err := iofs.New(migrationsFS, "migrations") + if err != nil { + t.Fatal(err) + } + m, err := openMigrate(source, u.String()) + if err != nil { + t.Fatal(err) + } + defer m.Close() + if err := m.Migrate(277); err != nil { + t.Fatal(err) + } + conn, err := pgx.Connect(ctx, u.String()) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + user, org, mailbox, id := uuid.New(), uuid.New(), uuid.New(), uuid.New() + for _, row := range []struct { + sql string + args []any + }{ + {`INSERT INTO users(id,email,first_name,last_name)VALUES($1,$2,'Legacy','Fixture')`, []any{user, user.String() + "@test.local"}}, + {`INSERT INTO organizations(id,name,slug,owner_user_id)VALUES($1,'Legacy',$2,$3)`, []any{org, org.String(), user}}, + {`INSERT INTO email_accounts(id,user_id,organization_id,email,name,signature_plain,signature_html,provider)VALUES($1,$2,$3,$4,'Legacy','','','smtp_imap')`, []any{mailbox, user, org, mailbox.String() + "@test.local"}}, + {`INSERT INTO email_sync_state(email_id,user_id,backfill_status,backfill_synced)VALUES($1,$2,'complete',42)`, []any{mailbox, user}}, + {`INSERT INTO email_message_map(user_id,email_id,message_id,id,thread_id)VALUES($1,$2,'legacy',$3,'thread')`, []any{user, mailbox, id}}, + } { + if _, err := conn.Exec(ctx, row.sql, row.args...); err != nil { + t.Fatal(err) + } + } + if err := m.Up(); err != nil { + t.Fatal(err) + } + var retained bool + if err := conn.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM email_message_map m JOIN email_sync_state s ON s.email_id=m.email_id WHERE m.id=$1 AND s.backfill_status='complete' AND s.backfill_synced=42 AND m.thread_id='thread')`, id).Scan(&retained); err != nil || !retained { + t.Fatal("upgrade changed legacy map or completed import", err) + } + var count int + if err := conn.QueryRow(ctx, `SELECT count(*) FROM sync_arrival_outbox`).Scan(&count); err != nil || count != 0 { + t.Fatal("upgrade invented pending arrivals", err) + } + if _, err := conn.Exec(ctx, `INSERT INTO sync_arrival_outbox(user_id,email_id,organization_id,message_id,id,payload)VALUES($1,$2,$3,'legacy',$4,'fixture')`, user, mailbox, org, id); err != nil { + t.Fatal(err) + } + down, err := migrationsFS.ReadFile("migrations/000278_sync_arrival_outbox.down.sql") + if err != nil { + t.Fatal(err) + } + if _, err := conn.Exec(ctx, string(down)); err == nil || !strings.Contains(err.Error(), "unresolved sync arrival") { + t.Fatalf("downgrade discarded pending delivery: %v", err) + } + if _, err := conn.Exec(ctx, `DELETE FROM sync_arrival_outbox`); err != nil { + t.Fatal(err) + } + if err := m.Steps(-1); err != nil { + t.Fatal(err) + } + if err := conn.QueryRow(ctx, `SELECT count(*) FROM email_message_map WHERE id=$1`, id).Scan(&count); err != nil || count != 1 { + t.Fatal("downgrade removed canonical map", err) + } + if err := m.Up(); err != nil { + t.Fatal(err) + } +} diff --git a/internal/infrastructure/eventbus/bus.go b/internal/infrastructure/eventbus/bus.go index 4ff66e0cd..e8c63d68a 100644 --- a/internal/infrastructure/eventbus/bus.go +++ b/internal/infrastructure/eventbus/bus.go @@ -72,6 +72,29 @@ type EventBus interface { // the message; returning an error leaves it for redelivery. type Handler func(ctx context.Context, msg Message) error +func retrySubscription(ctx context.Context, subscribe func(context.Context) error) error { + for { + if err := ctx.Err(); err != nil { + return err + } + err := subscribe(ctx) + if ctx.Err() != nil { + return ctx.Err() + } + if errors.Is(err, ErrBusClosed) { + return err + } + log.Warn().Msg("eventbus subscription interrupted; reopening for replay") + timer := time.NewTimer(time.Second) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } +} + // Message is the broker-neutral envelope passed to a Handler. Payload bytes // are owned by the bus and must not be retained past the handler call. // @@ -87,9 +110,7 @@ type Message struct { // handler that retries by returning an error can read it to know when the // broker is about to stop redelivering and give up cleanly instead. Attempt int - // Redelivers is true when a handler error leaves the message for another - // delivery (NATS). Kafka commits regardless, so a handler must not count - // on a retry there and should finish what it can on this delivery. + // Redelivers is true when a handler error leaves the message for retry. Redelivers bool } diff --git a/internal/infrastructure/eventbus/kafka.go b/internal/infrastructure/eventbus/kafka.go index 7a16e1000..427d14ed4 100644 --- a/internal/infrastructure/eventbus/kafka.go +++ b/internal/infrastructure/eventbus/kafka.go @@ -106,8 +106,7 @@ func (b *KafkaBus) Publish(ctx context.Context, topic, key string, payload []byt // Subscribe creates a fresh consumer in the given group, subscribes to all // topics, and blocks reading messages until ctx is cancelled or a fatal error -// occurs. Handler errors are logged but do not abort the loop; the message's -// offset is still stored, to match the existing kafka.Consumer.Consume behaviour. +// occurs. Failed handlers never store offsets; exhausted retries reopen for replay. func (b *KafkaBus) Subscribe(ctx context.Context, topics []string, group string, handler Handler) error { if len(topics) == 0 { return errors.New("eventbus kafka: at least one topic required") @@ -118,6 +117,12 @@ func (b *KafkaBus) Subscribe(ctx context.Context, topics []string, group string, if handler == nil { return errors.New("eventbus kafka: handler required") } + return retrySubscription(ctx, func(ctx context.Context) error { + return b.subscribeOnce(ctx, topics, group, handler) + }) +} + +func (b *KafkaBus) subscribeOnce(ctx context.Context, topics []string, group string, handler Handler) error { // Subscribing to a topic that does not exist yet returns no messages and // no error, so a worker would sit silent rather than fail. @@ -153,6 +158,17 @@ func (b *KafkaBus) Subscribe(ctx context.Context, topics []string, group string, } b.consumers = append(b.consumers, cons) b.mu.Unlock() + defer func() { + b.mu.Lock() + for i, c := range b.consumers { + if c == cons { + b.consumers = append(b.consumers[:i], b.consumers[i+1:]...) + break + } + } + b.mu.Unlock() + cons.Close() + }() err = cons.Consume(ctx, func(msg *ckf.Message) error { topic := "" @@ -163,10 +179,11 @@ func (b *KafkaBus) Subscribe(ctx context.Context, topics []string, group string, hctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), handlerTimeout()) defer cancel() if err := invokeHandler(hctx, handler, Message{ - Topic: topic, - Key: string(msg.Key), - Payload: msg.Value, - Attempt: 1, + Topic: topic, + Key: string(msg.Key), + Payload: msg.Value, + Attempt: 1, + Redelivers: true, }); err != nil { log.Error().Err(err).Str("topic", topic).Msg("eventbus kafka handler error") return err diff --git a/internal/infrastructure/eventbus/kafka_test.go b/internal/infrastructure/eventbus/kafka_test.go index c7da95b73..23752240b 100644 --- a/internal/infrastructure/eventbus/kafka_test.go +++ b/internal/infrastructure/eventbus/kafka_test.go @@ -4,11 +4,55 @@ package eventbus import ( "context" + "errors" "testing" "github.com/warmbly/warmbly/internal/infrastructure/kafka" ) +func TestKafkaSubscriptionReopensAfterFailureOrUnexpectedExit(t *testing.T) { + for _, failure := range []error{errors.New("offset assignment lost"), nil} { + ctx, cancel := context.WithCancel(t.Context()) + calls := 0 + err := retrySubscription(ctx, func(context.Context) error { + calls++ + if calls == 2 { + cancel() + } + return failure + }) + cancel() + if !errors.Is(err, context.Canceled) || calls != 2 { + t.Fatalf("err=%v calls=%d", err, calls) + } + } + calls := 0 + err := retrySubscription(t.Context(), func(context.Context) error { calls++; return ErrBusClosed }) + if !errors.Is(err, ErrBusClosed) || calls != 1 { + t.Fatal("closed bus reopened") + } +} + +func TestKafkaSubscriptionRetiresClientsBeforeReopening(t *testing.T) { + b := &KafkaBus{bootstrap: "127.0.0.1:1"} + ctx, cancel := context.WithCancel(t.Context()) + cancel() + for i := 0; i < 3; i++ { + if err := b.subscribeOnce(ctx, []string{"sync-test"}, "sync-test", func(context.Context, Message) error { return nil }); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + b.mu.Lock() + retained := len(b.consumers) + b.mu.Unlock() + if retained != 0 { + t.Fatalf("exited subscription retained %d clients", retained) + } + } + if err := b.Close(); err != nil { + t.Fatal(err) + } +} + // TestKafkaBus_InterfaceSatisfaction is a compile-time check that KafkaBus // satisfies the EventBus interface and that the constructors enforce their // minimum required arguments. A live Kafka broker is intentionally not used diff --git a/internal/infrastructure/kafka/consumer.go b/internal/infrastructure/kafka/consumer.go index 90054151d..c1e85883a 100644 --- a/internal/infrastructure/kafka/consumer.go +++ b/internal/infrastructure/kafka/consumer.go @@ -14,6 +14,7 @@ import ( ) var ErrClientClosed = errors.New("kafka: client closed") +var ErrAssignmentLost = errors.New("kafka: assignment lost before offset storage") type consumerClient interface { ReadMessage(time.Duration) (*ckf.Message, error) @@ -23,10 +24,11 @@ type consumerClient interface { } type Consumer struct { - c consumerClient - Avrov2 *Avrov2 - mu sync.Mutex - closed bool + c consumerClient + Avrov2 *Avrov2 + mu sync.Mutex + closed bool + retryLimit int } type ConsumerConfig struct { @@ -110,6 +112,9 @@ func (cons *Consumer) storeMessage(msg *ckf.Message) error { if cons.closed { return ErrClientClosed } + if native, ok := cons.c.(interface{ AssignmentLost() bool }); ok && native.AssignmentLost() { + return ErrAssignmentLost + } _, err := cons.c.StoreMessage(msg) return err } @@ -126,16 +131,43 @@ func (cons *Consumer) Consume(ctx context.Context, handler func(msg *ckf.Message if kafkaErr.Code() == ckf.ErrTimedOut { continue } + if kafkaErr.IsFatal() || kafkaErr.Code() == ckf.ErrMaxPollExceeded { + return fmt.Errorf("kafka: consumer must reopen: %w", err) + } // Log transient Kafka errors and retry after a brief delay log.Warn().Str("code", fmt.Sprintf("%d", kafkaErr.Code())).Err(kafkaErr).Msg("kafka consumer error") - time.Sleep(time.Second) + if err := consumerBackoff(ctx, time.Second); err != nil { + return err + } continue } return fmt.Errorf("error reading message: %w", err) } - if err := handler(msg); err != nil { - log.Error().Err(err).Msg("kafka message handler error") + backoff := 100 * time.Millisecond + limit := cons.retryLimit + if limit <= 0 { + limit = 4 + } + for attempt := 1; ; attempt++ { + if err := handler(msg); err == nil { + break + } else { + log.Error().Err(err).Msg("kafka message handler error; retrying before advancing offsets") + if attempt >= limit { + return fmt.Errorf("kafka: handler retry budget exhausted before offset storage: %w", err) + } + } + if err := consumerBackoff(ctx, backoff); err != nil { + return err + } + cons.mu.Lock() + closed := cons.closed + cons.mu.Unlock() + if closed { + return ErrClientClosed + } + backoff = min(2*backoff, 5*time.Second) } // Needs enable.auto.offset.store=false; the background commit sends it. @@ -143,8 +175,19 @@ func (cons *Consumer) Consume(ctx context.Context, handler func(msg *ckf.Message if errors.Is(err, ErrClientClosed) { return err } - log.Warn().Err(err).Msg("kafka: storing a handled offset failed") + return fmt.Errorf("kafka: storing a handled offset failed: %w", err) } } } } + +func consumerBackoff(ctx context.Context, delay time.Duration) error { + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} diff --git a/internal/infrastructure/kafka/consumer_retry_test.go b/internal/infrastructure/kafka/consumer_retry_test.go new file mode 100644 index 000000000..1ec234de3 --- /dev/null +++ b/internal/infrastructure/kafka/consumer_retry_test.go @@ -0,0 +1,120 @@ +//go:build kafka + +package kafka + +import ( + "context" + "errors" + "testing" + "time" + + ckf "github.com/confluentinc/confluent-kafka-go/v2/kafka" +) + +type orderedConsumer struct { + shutdownConsumer + queue []*ckf.Message + stored []ckf.Offset + reads int + storeErr error + lost bool + readErr error +} + +func (c *orderedConsumer) ReadMessage(time.Duration) (*ckf.Message, error) { + c.reads++ + if c.readErr != nil { + return nil, c.readErr + } + if len(c.queue) == 0 { + return nil, errors.New("unexpected poll") + } + m := c.queue[0] + c.queue = c.queue[1:] + return m, nil +} + +func (c *orderedConsumer) AssignmentLost() bool { return c.lost } + +func TestConsumerAssignmentLossRetainsReplayAndPollFailureReopens(t *testing.T) { + c := &orderedConsumer{queue: []*ckf.Message{{}, {}}} + err := (&Consumer{c: c}).Consume(t.Context(), func(*ckf.Message) error { c.lost = true; return nil }) + if !errors.Is(err, ErrAssignmentLost) || c.reads != 1 || len(c.stored) != 0 { + t.Fatalf("err=%v reads=%d stored=%v", err, c.reads, c.stored) + } + for _, readErr := range []ckf.Error{ckf.NewError(ckf.ErrMaxPollExceeded, "poll expired", false), ckf.NewError(ckf.ErrFatal, "fatal consumer failure", true)} { + c = &orderedConsumer{readErr: readErr} + err := (&Consumer{c: c}).Consume(t.Context(), func(*ckf.Message) error { t.Fatal("no record to handle"); return nil }) + if err == nil || c.reads != 1 || len(c.stored) != 0 { + t.Fatalf("err=%v reads=%d stored=%v", err, c.reads, c.stored) + } + } +} +func (c *orderedConsumer) StoreMessage(m *ckf.Message) ([]ckf.TopicPartition, error) { + c.stored = append(c.stored, m.TopicPartition.Offset) + return nil, c.storeErr +} + +func TestConsumerRetriesFailedRecordBeforeLaterOffsets(t *testing.T) { + first := &ckf.Message{TopicPartition: ckf.TopicPartition{Partition: 0, Offset: 10}} + second := &ckf.Message{TopicPartition: ckf.TopicPartition{Partition: 0, Offset: 11}} + c := &orderedConsumer{queue: []*ckf.Message{first, second}} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var handled []ckf.Offset + err := (&Consumer{c: c}).Consume(ctx, func(m *ckf.Message) error { + handled = append(handled, m.TopicPartition.Offset) + if len(handled) == 1 { + return errors.New("transient database failure") + } + if len(handled) == 2 && (c.reads != 1 || len(c.stored) != 0 || m != first) { + t.Fatal("failed record was skipped or acked") + } + if m == second { + cancel() + } + return nil + }) + if !errors.Is(err, context.Canceled) || len(handled) != 3 || handled[0] != 10 || handled[1] != 10 || handled[2] != 11 || len(c.stored) != 2 || c.stored[0] != 10 || c.stored[1] != 11 { + t.Fatalf("err=%v handled=%v stored=%v", err, handled, c.stored) + } +} + +func TestConsumerFailureCancellationNeverStoresOffset(t *testing.T) { + c := &orderedConsumer{queue: []*ckf.Message{{}}} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + err := (&Consumer{c: c}).Consume(ctx, func(*ckf.Message) error { cancel(); return errors.New("failed") }) + if !errors.Is(err, context.Canceled) || len(c.stored) != 0 || c.reads != 1 { + t.Fatalf("err=%v stored=%v reads=%d", err, c.stored, c.reads) + } +} + +func TestConsumerOffsetStorageFailureStopsBeforeNextRecord(t *testing.T) { + c := &orderedConsumer{queue: []*ckf.Message{{}, {}}, storeErr: errors.New("store unavailable")} + err := (&Consumer{c: c}).Consume(context.Background(), func(*ckf.Message) error { return nil }) + if !errors.Is(err, c.storeErr) || c.reads != 1 { + t.Fatalf("err=%v reads=%d", err, c.reads) + } +} + +func TestConsumerRetryBackoffCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := consumerBackoff(ctx, 5*time.Second); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } +} + +func TestConsumerPersistentFailureReturnsForReplayWithoutStoring(t *testing.T) { + c := &orderedConsumer{queue: []*ckf.Message{{}, {}}} + failure := errors.New("database still unavailable") + calls := 0 + err := (&Consumer{c: c, retryLimit: 2}).Consume(t.Context(), func(*ckf.Message) error { + calls++ + return failure + }) + if !errors.Is(err, failure) || calls != 2 || c.reads != 1 || len(c.stored) != 0 { + t.Fatalf("err=%v calls=%d reads=%d stored=%v", err, calls, c.reads, c.stored) + } +} diff --git a/internal/pkg/stoken/http.go b/internal/pkg/stoken/http.go new file mode 100644 index 000000000..6753364d5 --- /dev/null +++ b/internal/pkg/stoken/http.go @@ -0,0 +1,31 @@ +package stoken + +import ( + "context" + "net/http" + "time" + + "golang.org/x/oauth2" +) + +const ProviderRequestTimeout = 45 * time.Second + +// BoundedContext also bounds refresh I/O, which uses the token source's context. +func BoundedContext(ctx context.Context) context.Context { + base := http.DefaultClient + if client, ok := ctx.Value(oauth2.HTTPClient).(*http.Client); ok && client != nil { + base = client + } + client := *base + if client.Timeout <= 0 || client.Timeout > ProviderRequestTimeout { + client.Timeout = ProviderRequestTimeout + } + return context.WithValue(ctx, oauth2.HTTPClient, &client) +} + +func HTTPClient(ctx context.Context, source oauth2.TokenSource) *http.Client { + ctx = BoundedContext(ctx) + client := oauth2.NewClient(ctx, source) + client.Timeout = ctx.Value(oauth2.HTTPClient).(*http.Client).Timeout + return client +} diff --git a/internal/pkg/stoken/http_test.go b/internal/pkg/stoken/http_test.go new file mode 100644 index 000000000..1d87cb68e --- /dev/null +++ b/internal/pkg/stoken/http_test.go @@ -0,0 +1,47 @@ +package stoken + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "golang.org/x/oauth2" +) + +func TestBoundedProviderAndRefreshHTTPClient(t *testing.T) { + for _, timeout := range []time.Duration{0, time.Hour, 20 * time.Millisecond} { + base := &http.Client{Timeout: timeout} + ctx := BoundedContext(context.WithValue(context.Background(), oauth2.HTTPClient, base)) + bounded := ctx.Value(oauth2.HTTPClient).(*http.Client) + want := ProviderRequestTimeout + if timeout > 0 && timeout < want { + want = timeout + } + if base.Timeout != timeout || bounded.Timeout != want { + t.Fatalf("base=%v bounded=%v want=%v", base.Timeout, bounded.Timeout, want) + } + if c := HTTPClient(ctx, oauth2.StaticTokenSource(&oauth2.Token{AccessToken: "test"})); c.Timeout != want { + t.Fatalf("provider timeout=%v want=%v", c.Timeout, want) + } + } + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + select { + case <-r.Context().Done(): + case <-release: + } + })) + defer srv.Close() + defer close(release) + ctx := BoundedContext(context.WithValue(t.Context(), oauth2.HTTPClient, &http.Client{Timeout: 30 * time.Millisecond})) + cfg := oauth2.Config{Endpoint: oauth2.Endpoint{TokenURL: srv.URL, AuthStyle: oauth2.AuthStyleInParams}} + start := time.Now() + if _, err := cfg.TokenSource(ctx, &oauth2.Token{RefreshToken: "test"}).Token(); err == nil { + t.Fatal("stalled refresh succeeded") + } + if time.Since(start) > time.Second || ctx.Err() != nil { + t.Fatal("refresh was unbounded or cancelled mailbox context") + } +} diff --git a/internal/repository/email_error_dedupe_live_test.go b/internal/repository/email_error_dedupe_live_test.go index de88d20ae..217e00e7b 100644 --- a/internal/repository/email_error_dedupe_live_test.go +++ b/internal/repository/email_error_dedupe_live_test.go @@ -4,6 +4,7 @@ import ( "context" "sync" "testing" + "time" "github.com/google/uuid" ) @@ -85,6 +86,8 @@ func TestLiveEmailErrorDedupeKeepsOneUnresolvedRowPerCode(t *testing.T) { if first == nil { t.Fatal("the first occurrence of an error must be recorded") } + exec(`UPDATE email_account_errors SET created_at=NOW()-interval '3 days' WHERE id=$1`, first.ID) + before := time.Now() // The sync loop relaying the same refusal for the next hour. for i := 0; i < 5; i++ { @@ -99,6 +102,16 @@ func TestLiveEmailErrorDedupeKeepsOneUnresolvedRowPerCode(t *testing.T) { if n := unresolved("IMAP_UNKNOWN"); n != 1 { t.Errorf("unresolved IMAP_UNKNOWN rows = %d, want 1", n) } + rows, xerr := repo.GetByAccountID(ctx, account, true) + if xerr != nil || len(rows) != 1 || rows[0].CreatedAt.Before(before) { + t.Fatalf("ongoing failure observation expired: %+v %v", rows, xerr) + } + if xerr := repo.ResolveByCodesBefore(ctx, account, []string{"IMAP_UNKNOWN"}, before, "stale success"); xerr != nil { + t.Fatal(xerr) + } + if n := unresolved("IMAP_UNKNOWN"); n != 1 { + t.Fatal("old success cleared a newly observed failure") + } // A different failure is a different row: dedupe must not swallow it. other, xerr := repo.CreateOnce(ctx, row("SERVER_UNREACHABLE")) diff --git a/internal/repository/email_message_map.go b/internal/repository/email_message_map.go index 863d2128b..2d2de8591 100644 --- a/internal/repository/email_message_map.go +++ b/internal/repository/email_message_map.go @@ -2,8 +2,11 @@ package repository import ( "context" + "errors" + "time" "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/models" ) // EmailMessageMapRepository maps a provider messageId to the internal email @@ -20,6 +23,34 @@ type EmailMessageMapRepository interface { Del(ctx context.Context, userID, emailID uuid.UUID, messageID string, id uuid.UUID) error } +var ErrArrivalOutboxUnsupported = errors.New("durable arrival admission unavailable; upgrade backend and consumer first") +var ErrArrivalAdmissionUnconfirmed = errors.New("existing provider mapping has no verifiable pending admission") + +// ArrivalAdmission is an optional internal protocol, separate from legacy map writes. +type ArrivalAdmission interface { + AdmitArrival(context.Context, EmailMessageData, *PendingArrival) error +} + +type PendingArrival struct { + Arrival *models.JobEventNewEmail `json:"arrival"` + Bounce *models.JobEventInboundBounce `json:"bounce,omitempty"` + Complaint *models.JobEventInboundComplaint `json:"complaint,omitempty"` +} + +type ArrivalOutbox interface { + HasPendingArrival(context.Context, uuid.UUID, uuid.UUID, uuid.UUID) (bool, error) + DeliverArrivals(context.Context, func(context.Context, models.JobEventType, any) error) error +} + +type ArrivalBacklog struct { + Pending int + Oldest *time.Time +} + +type ArrivalBacklogRepository interface { + ArrivalBacklog(context.Context) (ArrivalBacklog, error) +} + // EmailMessageData is one (user, mailbox, providerMessageID) -> internal // (id, threadID) mapping row. The UUID-typed values travel as strings to match // the worker call sites, which pass uuid.String(). diff --git a/internal/repository/http_email_message_map.go b/internal/repository/http_email_message_map.go index e9739fde2..0c9e6f2d1 100644 --- a/internal/repository/http_email_message_map.go +++ b/internal/repository/http_email_message_map.go @@ -93,6 +93,34 @@ func (r *httpEmailMessageMapRepository) Add(ctx context.Context, data EmailMessa } } +func (r *httpEmailMessageMapRepository) AdmitArrival(ctx context.Context, data EmailMessageData, pending *PendingArrival) error { + body, err := json.Marshal(struct { + Map emailMessageMapPayload `json:"map"` + Pending *PendingArrival `json:"pending"` + }{emailMessageMapPayload(data), pending}) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.endpoint()+"/arrival", bytes.NewReader(body)) + if err != nil { + return err + } + r.authed(req) + req.Header.Set("Content-Type", "application/json") + resp, err := r.client.Do(req) + if err != nil { + return fmt.Errorf("arrival admission: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusMethodNotAllowed { + return ErrArrivalOutboxUnsupported + } + if resp.StatusCode != http.StatusNoContent || resp.Header.Get("X-Warmbly-Arrival-Durable") != "1" { + return fmt.Errorf("arrival admission not confirmed: status %d", resp.StatusCode) + } + return nil +} + func (r *httpEmailMessageMapRepository) Get(ctx context.Context, userID, emailID uuid.UUID, messageID string) (*EmailMessageData, error) { q := url.Values{} q.Set("user_id", userID.String()) diff --git a/internal/repository/http_graph_upgrade_test.go b/internal/repository/http_graph_upgrade_test.go new file mode 100644 index 000000000..27b0efca4 --- /dev/null +++ b/internal/repository/http_graph_upgrade_test.go @@ -0,0 +1,22 @@ +package repository + +import ( + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/google/uuid" +) + +func TestProviderMessagesOldBackendIsTypedUnsupported(t *testing.T) { + for _, status := range []int{http.StatusNotFound, http.StatusMethodNotAllowed, http.StatusServiceUnavailable, http.StatusForbidden} { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(status) })) + repo := &httpSyncContextRepository{baseURL: srv.URL, client: srv.Client()} + _, err := repo.ListProviderMessages(t.Context(), uuid.New(), uuid.New(), nil, 10) + if errors.Is(err, ErrSyncContextUnsupported) != (status == http.StatusNotFound || status == http.StatusMethodNotAllowed) { + t.Errorf("status=%d err=%v", status, err) + } + srv.Close() + } +} diff --git a/internal/repository/http_sync_arrival_test.go b/internal/repository/http_sync_arrival_test.go new file mode 100644 index 000000000..4853fcae5 --- /dev/null +++ b/internal/repository/http_sync_arrival_test.go @@ -0,0 +1,67 @@ +package repository + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" +) + +func TestHTTPArrivalAdmissionUpgradeAcknowledgement(t *testing.T) { + for _, tc := range []struct { + name string + status int + ack string + unsupported, success bool + }{ + {"old-backend", 404, "", true, false}, {"old-method", 405, "", true, false}, + {"durable", 204, "1", false, true}, {"unconfirmed", 204, "", false, false}, + {"backend-failure", 503, "", false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/internal/email-message-map/arrival" || r.Method != http.MethodPost || r.Header.Get("Authorization") != "Bearer fixture-token" { + t.Error("incorrect internal protocol") + } + w.Header().Set("X-Warmbly-Arrival-Durable", tc.ack) + w.WriteHeader(tc.status) + })) + defer srv.Close() + repo, err := NewHTTPEmailMessageMapRepository(srv.URL, "fixture-token") + if err != nil { + t.Fatal(err) + } + err = repo.(ArrivalAdmission).AdmitArrival(context.Background(), EmailMessageData{}, &PendingArrival{}) + if errors.Is(err, ErrArrivalOutboxUnsupported) != tc.unsupported || (err == nil) != tc.success { + t.Fatalf("err=%v", err) + } + }) + } +} + +func TestHTTPArrivalAdmissionResponseLossDoesNotDowngrade(t *testing.T) { + var committed atomic.Bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + t.Error("ambiguous admission fell back to legacy write") + } + committed.Store(true) + conn, _, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + _ = conn.Close() + })) + defer srv.Close() + r, err := NewHTTPEmailMessageMapRepository(srv.URL, "fixture-token") + if err != nil { + t.Fatal(err) + } + err = r.(ArrivalAdmission).AdmitArrival(context.Background(), EmailMessageData{}, &PendingArrival{}) + if err == nil || errors.Is(err, ErrArrivalOutboxUnsupported) || !committed.Load() { + t.Fatalf("response loss treated as unsupported: committed=%v err=%v", committed.Load(), err) + } +} diff --git a/internal/repository/http_sync_context.go b/internal/repository/http_sync_context.go index e23fcc72a..4434fda40 100644 --- a/internal/repository/http_sync_context.go +++ b/internal/repository/http_sync_context.go @@ -54,6 +54,8 @@ type SyncContextRepository interface { ListProviderMessages(ctx context.Context, userID, emailID uuid.UUID, after *uuid.UUID, limit int) ([]ProviderFolderMessage, error) } +var ErrSyncContextUnsupported = errors.New("provider message enumeration unavailable; upgrade backend first") + func (r *httpSyncContextRepository) ListProviderMessages(ctx context.Context, userID, emailID uuid.UUID, after *uuid.UUID, limit int) ([]ProviderFolderMessage, error) { q := url.Values{} q.Set("user_id", userID.String()) @@ -73,6 +75,9 @@ func (r *httpSyncContextRepository) ListProviderMessages(ctx context.Context, us return nil, err } defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusMethodNotAllowed { + return nil, ErrSyncContextUnsupported + } if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("sync_context.http: provider messages: unexpected status %d", resp.StatusCode) } diff --git a/internal/repository/pg_email_error.go b/internal/repository/pg_email_error.go index e6bee85fc..c9a4ca2c5 100644 --- a/internal/repository/pg_email_error.go +++ b/internal/repository/pg_email_error.go @@ -81,8 +81,7 @@ func NewEmailAccountErrorRepository(database *db.DB) EmailAccountErrorRepository } // Create stores a new email account error -// CreateOnce records the error unless the account already has an unresolved -// one carrying the same code, and returns (nil, nil) when it declined. +// CreateOnce refreshes unresolved transport observations without creating or notifying twice. // // A mail server that refuses the same command every pass is not a new problem // every pass. The sync loop retries about once a minute and reports what it @@ -96,7 +95,14 @@ func NewEmailAccountErrorRepository(database *db.DB) EmailAccountErrorRepository // 000145), so resolved history is untouched and a problem that returns after // it was fixed is recorded again. func (r *emailAccountErrorRepository) CreateOnce(ctx context.Context, data *CreateEmailAccountError) (*EmailAccountError, *errx.Error) { + // Unresolved transport timestamps represent the latest failure observation. query := ` + WITH observed AS ( + UPDATE email_account_errors SET created_at=NOW() + WHERE email_account_id=$1 AND user_id=$2 AND error_code=$3 AND resolved_at IS NULL + AND error_code IN ('SERVER_UNREACHABLE','CONNECTION_LOST','RESOURCE_NOT_FOUND','IMAP_UNKNOWN') + RETURNING id + ) INSERT INTO email_account_errors ( email_account_id, user_id, error_code, severity, resolve_method, title, message, user_message, action_required, task_id diff --git a/internal/repository/pg_email_message_map.go b/internal/repository/pg_email_message_map.go index 0225a6cb5..74e726657 100644 --- a/internal/repository/pg_email_message_map.go +++ b/internal/repository/pg_email_message_map.go @@ -7,6 +7,7 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5" + "github.com/warmbly/warmbly/internal/app/cipher" "github.com/warmbly/warmbly/internal/infrastructure/db" ) @@ -14,7 +15,8 @@ import ( // backend, which serves it to workers over the internal API. The composite key // (user_id, email_id, message_id) is canonical — see the 000003 migration. type pgEmailMessageMapRepository struct { - db *db.DB + db *db.DB + cipher cipher.CipherService } // NewEmailMessageMapRepository returns the Postgres-backed implementation. @@ -42,6 +44,8 @@ func (r *pgEmailMessageMapRepository) Add(ctx context.Context, data EmailMessage VALUES ($1, $2, $3, $4, $5) ON CONFLICT (user_id, email_id, message_id) DO UPDATE SET id = EXCLUDED.id, thread_id = EXCLUDED.thread_id + WHERE NOT EXISTS (SELECT 1 FROM sync_arrival_outbox o + WHERE o.user_id = EXCLUDED.user_id AND o.email_id = EXCLUDED.email_id AND o.message_id = EXCLUDED.message_id) ` if _, err := r.db.Exec(ctx, q, userID, emailID, data.MessageID, id, data.ThreadID); err != nil { return fmt.Errorf("email_message_map: add: %w", err) @@ -73,11 +77,13 @@ func (r *pgEmailMessageMapRepository) Get(ctx context.Context, userID, emailID u }, nil } -func (r *pgEmailMessageMapRepository) Del(ctx context.Context, userID, emailID uuid.UUID, messageID string, _ uuid.UUID) error { - // (user_id, email_id, message_id) is the primary key, so the internal id - // is not needed to identify the row. - const q = `DELETE FROM email_message_map WHERE user_id = $1 AND email_id = $2 AND message_id = $3` - if _, err := r.db.Exec(ctx, q, userID, emailID, messageID); err != nil { +func (r *pgEmailMessageMapRepository) Del(ctx context.Context, userID, emailID uuid.UUID, messageID string, id uuid.UUID) error { + // Legacy publication rollback must not remove an already admitted arrival. + const q = `DELETE FROM email_message_map m WHERE user_id=$1 AND email_id=$2 AND message_id=$3 + AND ($4::uuid='00000000-0000-0000-0000-000000000000' OR id=$4) + AND NOT EXISTS (SELECT 1 FROM sync_arrival_outbox o + WHERE o.user_id=m.user_id AND o.email_id=m.email_id AND o.message_id=m.message_id)` + if _, err := r.db.Exec(ctx, q, userID, emailID, messageID, id); err != nil { return fmt.Errorf("email_message_map: del: %w", err) } return nil diff --git a/internal/repository/pg_sync_arrival.go b/internal/repository/pg_sync_arrival.go new file mode 100644 index 000000000..f67d6b34f --- /dev/null +++ b/internal/repository/pg_sync_arrival.go @@ -0,0 +1,194 @@ +package repository + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/warmbly/warmbly/internal/app/cipher" + "github.com/warmbly/warmbly/internal/infrastructure/db" + "github.com/warmbly/warmbly/internal/models" +) + +func NewDurableEmailMessageMapRepository(d *db.DB, c cipher.CipherService) *pgEmailMessageMapRepository { + return &pgEmailMessageMapRepository{db: d, cipher: c} +} + +func validateArrival(data EmailMessageData, pending *PendingArrival) (uuid.UUID, uuid.UUID, uuid.UUID, error) { + user, uerr := uuid.Parse(data.UserID) + email, eerr := uuid.Parse(data.EmailID) + id, ierr := uuid.Parse(data.ID) + if uerr != nil || eerr != nil || ierr != nil || user == uuid.Nil || email == uuid.Nil || id == uuid.Nil || data.MessageID == "" { + return user, email, id, errors.New("invalid arrival mapping") + } + if pending == nil || pending.Arrival == nil || pending.Arrival.Message == nil || + pending.Arrival.UserID != user || pending.Arrival.Message.EmailID != email || + pending.Arrival.Message.ID != id || pending.Arrival.Message.ThreadID != data.ThreadID { + return user, email, id, errors.New("arrival does not match mapping") + } + if pending.Bounce != nil && (pending.Bounce.UserID != user || pending.Bounce.EmailID != email) || + pending.Complaint != nil && (pending.Complaint.UserID != user || pending.Complaint.EmailID != email) { + return user, email, id, errors.New("arrival report does not match mailbox") + } + return user, email, id, nil +} + +func (r *pgEmailMessageMapRepository) AdmitArrival(ctx context.Context, data EmailMessageData, pending *PendingArrival) error { + user, email, id, err := validateArrival(data, pending) + if err != nil { + return err + } + if r.cipher == nil { + return ErrArrivalOutboxUnsupported + } + var org uuid.UUID + err = r.db.QueryRow(ctx, `SELECT organization_id FROM email_accounts WHERE id=$1 AND user_id=$2`, email, user).Scan(&org) + if err != nil { + return fmt.Errorf("arrival mailbox ownership: %w", err) + } + sealed, err := r.cipher.Cipher(ctx, org) + if err != nil { + return err + } + raw, err := json.Marshal(pending) + if err != nil { + return err + } + payload, err := sealed.Encrypt(ctx, string(raw)) + if err != nil { + return err + } + tx, err := r.db.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + // Recheck and lock ownership after crypto, without nesting key-store I/O in the transaction. + var owned uuid.UUID + if err = tx.QueryRow(ctx, `SELECT id FROM email_accounts WHERE id=$1 AND user_id=$2 AND organization_id=$3 FOR SHARE`, email, user, org).Scan(&owned); err != nil { + return fmt.Errorf("arrival mailbox ownership: %w", err) + } + // A retry after an ambiguous response adopts the first canonical identity. + inserted, err := tx.Exec(ctx, `INSERT INTO email_message_map(user_id,email_id,message_id,id,thread_id) + VALUES($1,$2,$3,$4,$5) ON CONFLICT DO NOTHING`, user, email, data.MessageID, id, data.ThreadID) + if err != nil { + return err + } + if inserted.RowsAffected() != 0 { + _, err = tx.Exec(ctx, `INSERT INTO sync_arrival_outbox(user_id,email_id,organization_id,message_id,id,payload) + VALUES($1,$2,$3,$4,$5,$6)`, user, email, org, data.MessageID, id, payload) + if err != nil { + return err + } + } else { + var pending bool + err = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM email_message_map m + JOIN sync_arrival_outbox o USING(user_id,email_id,message_id,id) + WHERE m.user_id=$1 AND m.email_id=$2 AND m.message_id=$3 AND o.organization_id=$4)`, user, email, data.MessageID, org).Scan(&pending) + if err != nil { + return err + } + // Legacy, deleted and already-delivered maps have no durable admission proof. + if !pending { + return ErrArrivalAdmissionUnconfirmed + } + } + return tx.Commit(ctx) +} + +func (r *pgEmailMessageMapRepository) ArrivalBacklog(ctx context.Context) (ArrivalBacklog, error) { + var backlog ArrivalBacklog + err := r.db.QueryRow(ctx, `SELECT count(*),min(created_at) FROM sync_arrival_outbox`).Scan(&backlog.Pending, &backlog.Oldest) + return backlog, err +} + +func (r *pgEmailMessageMapRepository) HasPendingArrival(ctx context.Context, user, email, id uuid.UUID) (bool, error) { + var pending bool + err := r.db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM sync_arrival_outbox o + JOIN email_accounts a ON a.id=o.email_id AND a.user_id=o.user_id AND a.organization_id=o.organization_id + WHERE o.user_id=$1 AND o.email_id=$2 AND o.id=$3 AND o.stage=0)`, user, email, id).Scan(&pending) + return pending, err +} + +// DeliverArrivals invokes the existing consumer handlers, retaining every failed stage. +func (r *pgEmailMessageMapRepository) DeliverArrivals(ctx context.Context, deliver func(context.Context, models.JobEventType, any) error) error { + var first error + for i := 0; i < 32; i++ { + found, err := r.deliverArrival(ctx, deliver) + if err != nil && first == nil { + first = err + } + if !found || ctx.Err() != nil { + break + } + } + return first +} + +func (r *pgEmailMessageMapRepository) deliverArrival(ctx context.Context, deliver func(context.Context, models.JobEventType, any) error) (bool, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + var user, email, org, lease uuid.UUID + var key, payload string + var stage int + // Claim without retaining a connection while handlers use their repositories. + err := r.db.QueryRow(ctx, `WITH candidate AS (SELECT o.user_id,o.email_id,o.message_id + FROM sync_arrival_outbox o JOIN email_accounts a ON a.id=o.email_id AND a.user_id=o.user_id AND a.organization_id=o.organization_id + WHERE o.retry_at<=now() AND (o.locked_until IS NULL OR o.locked_until"} + p.Complaint = &models.JobEventInboundComplaint{UserID: f.user, EmailID: f.mailbox, OriginalMessageID: ""} + wrong := uuid.New() + data.UserID = wrong.String() + p.Arrival.UserID = wrong + p.Bounce.UserID = wrong + p.Complaint.UserID = wrong + if err := r.AdmitArrival(ctx, data, p); err == nil { + t.Fatal("another user admitted mail") + } + data.UserID = f.user.String() + p.Arrival.UserID = f.user + p.Bounce.UserID = f.user + p.Complaint.UserID = f.user + p.Bounce.EmailID = uuid.New() + if err := r.AdmitArrival(ctx, data, p); err == nil { + t.Fatal("cross-mailbox report accepted") + } + p.Bounce.EmailID = f.mailbox + if err := r.AdmitArrival(ctx, data, p); err != nil { + t.Fatal(err) + } + var seen []models.JobEventType + failed := errors.New("report handler unavailable") + handle := func(_ context.Context, kind models.JobEventType, _ any) error { + seen = append(seen, kind) + if kind == models.JobEventTypeInboundBounce && len(seen) == 2 { + return failed + } + return nil + } + if err := r.DeliverArrivals(ctx, handle); !errors.Is(err, failed) { + t.Fatal(err) + } + if pending, err := r.HasPendingArrival(ctx, f.user, f.mailbox, p.Arrival.Message.ID); err != nil || pending { + t.Fatal("completed arrival confused with pending report") + } + dueArrival(t, d, f) + if err := NewDurableEmailMessageMapRepository(d, arrivalLiveCipher(t, d)).DeliverArrivals(ctx, handle); err != nil { + t.Fatal(err) + } + if len(seen) != 4 || seen[0] != models.JobEventTypeNewEmail || seen[1] != models.JobEventTypeInboundBounce || seen[2] != models.JobEventTypeInboundBounce || seen[3] != models.JobEventTypeInboundComplaint { + t.Fatalf("stages=%v", seen) + } + // Completed and legacy mappings are not guessed to be missing arrivals. + legacy, lp := arrivalLivePayload(f, "legacy") + if err := r.Add(ctx, legacy); err != nil { + t.Fatal(err) + } + lp.Arrival.Message.ID = uuid.New() + retry := legacy + retry.ID = lp.Arrival.Message.ID.String() + if err := r.AdmitArrival(ctx, retry, lp); !errors.Is(err, ErrArrivalAdmissionUnconfirmed) { + t.Fatalf("legacy mapping claimed durable admission: %v", err) + } + known, err := r.Get(ctx, f.user, f.mailbox, legacy.MessageID) + if err != nil || known.ID != legacy.ID { + t.Fatal("legacy mapping replaced") + } + if err := r.DeliverArrivals(ctx, func(context.Context, models.JobEventType, any) error { + t.Fatal("legacy or completed mapping replayed") + return nil + }); err != nil { + t.Fatal(err) + } +} + +func TestLiveSyncArrivalAtomicRollbackDeletedMailboxAndLease(t *testing.T) { + d := liveUniboxFolderDB(t) + f := newUniboxFolderFixture(t, d.Pool) + ctx := context.Background() + r := NewDurableEmailMessageMapRepository(d, arrivalLiveCipher(t, d)) + data, p := arrivalLivePayload(f, "atomic") + // Force only the outbox insert to fail, without disturbing ordinary mappings. + if _, err := d.Exec(ctx, `ALTER TABLE sync_arrival_outbox ADD CONSTRAINT sync_arrival_test_reject CHECK (message_id<>'atomic')`); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _, _ = d.Exec(ctx, `ALTER TABLE sync_arrival_outbox DROP CONSTRAINT IF EXISTS sync_arrival_test_reject`) + }) + if err := r.AdmitArrival(ctx, data, p); err == nil { + t.Fatal("expected outbox insertion failure") + } + if known, err := r.Get(ctx, f.user, f.mailbox, data.MessageID); err != nil || known != nil { + t.Fatal("map escaped rolled-back admission") + } + if _, err := d.Exec(ctx, `ALTER TABLE sync_arrival_outbox DROP CONSTRAINT sync_arrival_test_reject`); err != nil { + t.Fatal(err) + } + if err := r.AdmitArrival(ctx, data, p); err != nil { + t.Fatal(err) + } + if _, err := d.Exec(ctx, `UPDATE sync_arrival_outbox SET locked_until=now()+interval '1 minute' WHERE email_id=$1`, f.mailbox); err != nil { + t.Fatal(err) + } + if err := r.DeliverArrivals(ctx, func(context.Context, models.JobEventType, any) error { t.Fatal("active lease replayed"); return nil }); err != nil { + t.Fatal(err) + } + dueArrival(t, d, f) + // A process lost while leased becomes replayable, including on a one-connection self-host. + config := d.Pool.Config() + config.MaxConns = 1 + config.MinConns = 0 + pool, err := pgxpool.NewWithConfig(ctx, config) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + single := &db.DB{Pool: pool} + sr := NewDurableEmailMessageMapRepository(single, arrivalLiveCipher(t, single)) + if err := sr.DeliverArrivals(ctx, func(ctx context.Context, _ models.JobEventType, _ any) error { + var n int + return single.QueryRow(ctx, "SELECT 1").Scan(&n) + }); err != nil { + t.Fatal(err) + } + data, p = arrivalLivePayload(f, "single-connection") + if err := sr.AdmitArrival(ctx, data, p); err != nil { + t.Fatal(err) + } + if err := sr.DeliverArrivals(ctx, func(context.Context, models.JobEventType, any) error { return nil }); err != nil { + t.Fatal(err) + } + data, p = arrivalLivePayload(f, "deleted") + if err := r.AdmitArrival(ctx, data, p); err != nil { + t.Fatal(err) + } + if _, err := d.Exec(ctx, `DELETE FROM email_accounts WHERE id=$1`, f.mailbox); err != nil { + t.Fatal(err) + } + if pending, err := r.HasPendingArrival(ctx, f.user, f.mailbox, p.Arrival.Message.ID); err != nil || pending { + t.Fatal("deleted mailbox retained pending arrival") + } + var n int + if err := d.QueryRow(ctx, `SELECT count(*) FROM sync_arrival_outbox WHERE email_id=$1`, f.mailbox).Scan(&n); err != nil || n != 0 { + t.Fatal("deleted mailbox retained queue") + } +} diff --git a/internal/repository/pg_unibox.go b/internal/repository/pg_unibox.go index 724c92636..55f269aba 100644 --- a/internal/repository/pg_unibox.go +++ b/internal/repository/pg_unibox.go @@ -39,6 +39,7 @@ type UniboxRepository interface { UpdateEntry(ctx context.Context, userID, emailID, id uuid.UUID, e *UpdateUniboxEntry) error GetIncoming(ctx context.Context, userID uuid.UUID, limit int, cursor string) (*models.MailSearchResult, error) GetByID(ctx context.Context, userID, id uuid.UUID) (*models.EmailMessageStoreData, error) + GetForSync(ctx context.Context, userID, emailID, id uuid.UUID) (*models.EmailMessageStoreData, error) // GetByIDForOrg is the org-scoped read for the unibox detail view: any // member with unibox access can open a message in the org-wide list. It // returns the row plus the mailbox OWNER's user_id, which the S3 body key @@ -455,14 +456,35 @@ func (r *uniboxRepository) GetIncoming(ctx context.Context, userID uuid.UUID, li var ErrEmailNotFound = errors.New("email not found") func (r *uniboxRepository) GetByID(ctx context.Context, userID, id uuid.UUID) (*models.EmailMessageStoreData, error) { + e, err := r.getMessage(ctx, userID, uuid.Nil, id) + if err != nil { + return nil, err + } + if !e.Seen { + _ = r.MarkSeen(ctx, userID, id, true) + e.Seen = true + } + return e, nil +} + +// GetForSync reads provider state without treating background work as a user view. +func (r *uniboxRepository) GetForSync(ctx context.Context, userID, emailID, id uuid.UUID) (*models.EmailMessageStoreData, error) { + if emailID == uuid.Nil { + return nil, ErrEmailNotFound + } + return r.getMessage(ctx, userID, emailID, id) +} + +func (r *uniboxRepository) getMessage(ctx context.Context, userID, emailID, id uuid.UUID) (*models.EmailMessageStoreData, error) { query := fmt.Sprintf(` SELECT %s FROM unibox_emails WHERE user_id = $1 AND id = $2 + AND ($3::uuid = '00000000-0000-0000-0000-000000000000' OR email_id = $3) `, strings.Join(mailFieldsFull, ", ")) var e models.EmailMessageStoreData - err := r.db.QueryRow(ctx, query, userID, id).Scan( + err := r.db.QueryRow(ctx, query, userID, id, emailID).Scan( &e.ID, &e.EmailID, &e.Mailbox, &e.FolderPath, &e.ThreadID, &e.MessageID, &e.GmailID, &e.ParentID, &e.UID, &e.ModSeq, &e.Flags, &e.BCC, &e.CC, &e.FromAddr, &e.InReplyTo, &e.ReplyTo, @@ -476,12 +498,6 @@ func (r *uniboxRepository) GetByID(ctx context.Context, userID, id uuid.UUID) (* return nil, err } - // Auto-mark as seen - if !e.Seen { - _ = r.MarkSeen(ctx, userID, id, true) - e.Seen = true - } - return &e, nil } diff --git a/web/src/components/app/emails/SyncStatusCard.test.tsx b/web/src/components/app/emails/SyncStatusCard.test.tsx new file mode 100644 index 000000000..79f3a68b3 --- /dev/null +++ b/web/src/components/app/emails/SyncStatusCard.test.tsx @@ -0,0 +1,70 @@ +import { act, cleanup, render, screen } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type EmailSync from "@/lib/api/models/app/emails/SyncState"; +import SyncStatusCard from "./SyncStatusCard"; + +const fixtures = vi.hoisted(() => ({ data: undefined as EmailSync | undefined })); +vi.mock("@/lib/api/hooks/app/emails/useSync", () => ({ default: () => ({ data: fixtures.data, isPending: false }) })); +vi.mock("@/lib/api/hooks/app/emails/useUpdateSyncSkipFolders", () => ({ default: () => ({}) })); +vi.mock("@/hooks/context/confirm", () => ({ useConfirm: () => ({}) })); + +describe("truthful mailbox sync status", () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-10-09T12:00:00Z")); + fixtures.data = { + state: { backfill_status: "complete", backfill_synced: 42, deferred: 0 }, + policy: { backfill_days: 30, backfill_messages: 2000, daily_messages: 500, org_daily_messages: 2000 }, + folders: [], skip_folders: [], + }; + }); + afterEach(() => { cleanup(); vi.useRealTimers(); }); + const draw = () => render(); + + it("does not treat a completed historical import as proof of live health", () => { + draw(); + expect(screen.getByText("Recent-mail import complete")).toBeInTheDocument(); + expect(screen.getByText("No successful sync check reported yet.")).toBeInTheDocument(); + expect(screen.getByText(/Import completion does not confirm live sync is current/)).toBeInTheDocument(); + expect(screen.queryByText(/Up to date/i)).not.toBeInTheDocument(); + }); + + it("ages a recent successful check into stale evidence without a new server payload", () => { + fixtures.data!.state!.last_synced_at = new Date("2026-10-09T11:59:00Z"); + draw(); + expect(screen.getByText("Last successful sync 1 min ago.")).toBeInTheDocument(); + act(() => { vi.advanceTimersByTime(21 * 60_000); }); + expect(screen.getByText("No recent successful sync check")).toBeInTheDocument(); + expect(screen.getByText(/Recovery is not yet confirmed by a recent successful check/)).toBeInTheDocument(); + expect(screen.queryByText(/Up to date/i)).not.toBeInTheDocument(); + }); + + it("shows ongoing recovery from the existing cursor instead of historical completion", () => { + fixtures.data!.state!.backfill_cursor = { google_recovery: { history_id: 100, since: new Date(), messages_done: false } }; + draw(); + expect(screen.getByText("Recovering mailbox sync")).toBeInTheDocument(); + expect(screen.queryByText("Recent-mail import complete")).not.toBeInTheDocument(); + }); + + it("keeps deferred evidence visible after the budget hold expires", () => { + fixtures.data!.state!.deferred = 5; + fixtures.data!.state!.throttled_until = new Date("2026-10-09T12:01:00Z"); + const view = draw(); + expect(screen.getByText(/Waiting on the sync budget until/)).toBeInTheDocument(); + act(() => { vi.advanceTimersByTime(2 * 60_000); }); + expect(screen.getByText("New mail was waiting on sync at the last check")).toBeInTheDocument(); + expect(screen.getByText(/Last observed backlog: 5 messages/)).toBeInTheDocument(); + fixtures.data!.state!.deferred = 0; + fixtures.data!.state!.last_synced_at = new Date(); + view.rerender(); + expect(screen.getByText("Last successful sync just now.")).toBeInTheDocument(); + expect(screen.queryByText(/Last observed backlog/)).not.toBeInTheDocument(); + }); + + it("describes import progress separately from the last successful live check", () => { + fixtures.data!.state!.backfill_status = "running"; + draw(); + expect(screen.getByText("Importing recent mail: 42 messages so far")).toBeInTheDocument(); + expect(screen.getByText("No successful sync check reported yet.")).toBeInTheDocument(); + }); +}); diff --git a/web/src/components/app/emails/SyncStatusCard.tsx b/web/src/components/app/emails/SyncStatusCard.tsx index 2a925bd17..2fb55bce0 100644 --- a/web/src/components/app/emails/SyncStatusCard.tsx +++ b/web/src/components/app/emails/SyncStatusCard.tsx @@ -1,4 +1,4 @@ -import { useState } from "react"; +import { useEffect, useState } from "react"; import { motion } from "framer-motion"; import { CheckCircle2Icon, DownloadIcon, HourglassIcon, PlusIcon, RefreshCwIcon } from "lucide-react"; import toast from "react-hot-toast/headless"; @@ -25,8 +25,8 @@ const REASON_COPY: Record = { priority_daily: "the daily limit for replies was reached", }; -function relative(iso: string | Date): string { - const diff = Date.now() - new Date(iso).getTime(); +function relative(iso: string | Date, now: number): string { + const diff = now - new Date(iso).getTime(); const m = Math.round(diff / 60_000); if (m < 1) return "just now"; if (m < 60) return `${m} min ago`; @@ -148,6 +148,11 @@ function SkipFoldersSection({ mailboxId, listed, skipped }: { mailboxId: string; } export default function SyncStatusCard({ mailboxId, provider }: { mailboxId: string; provider?: string }) { + const [now, setNow] = useState(Date.now); + useEffect(() => { + const timer = setInterval(() => setNow(Date.now()), 60_000); + return () => clearInterval(timer); + }, []); const sync = useSync(mailboxId); const state = sync.data?.state ?? null; const policy = sync.data?.policy; @@ -162,15 +167,19 @@ export default function SyncStatusCard({ mailboxId, provider }: { mailboxId: str } if (!sync.data) return null; - const throttled = !!state?.throttled_until && new Date(state.throttled_until).getTime() > Date.now(); + const throttled = !!state?.throttled_until && new Date(state.throttled_until).getTime() > now; + const lastSuccess = state?.last_synced_at && new Date(state.last_synced_at).getTime(); + // The worker relays successful checks at least every ten minutes. + const stale = !!lastSuccess && now - lastSuccess > 20 * 60_000; + const recovering = !!state?.backfill_cursor?.google_recovery; const status = state?.backfill_status ?? "pending"; const cap = policy?.backfill_messages ?? 0; const synced = state?.backfill_synced ?? 0; const pct = cap > 0 ? Math.min(100, Math.round((synced / cap) * 100)) : 0; let headline: React.ReactNode; - let Icon = CheckCircle2Icon; - let tone = "text-emerald-600"; + let Icon = RefreshCwIcon; + let tone = "text-slate-500"; if (throttled && state?.throttled_until) { Icon = HourglassIcon; tone = "text-amber-600"; @@ -181,8 +190,19 @@ export default function SyncStatusCard({ mailboxId, provider }: { mailboxId: str {reason ? ({reason}) : null} ); + } else if (recovering) { + tone = "text-sky-600"; + headline = "Recovering mailbox sync"; + } else if ((state?.deferred ?? 0) > 0) { + Icon = HourglassIcon; + tone = "text-amber-600"; + headline = "New mail was waiting on sync at the last check"; + } else if (stale) { + tone = "text-amber-600"; + headline = "No recent successful sync check"; } else if (status === "complete") { - headline = state?.last_synced_at ? `Up to date, last checked ${relative(state.last_synced_at)}` : "Up to date"; + Icon = CheckCircle2Icon; + headline = "Recent-mail import complete"; } else if (status === "running") { Icon = DownloadIcon; tone = "text-sky-600"; @@ -214,16 +234,23 @@ export default function SyncStatusCard({ mailboxId, provider }: { mailboxId: str

{status === "complete" && policy - ? `Imported ${synced.toLocaleString()} message${synced === 1 ? "" : "s"} from the last ${policy.backfill_days} days. New mail syncs as it arrives.` + ? `Imported ${synced.toLocaleString()} message${synced === 1 ? "" : "s"} from the configured ${policy.backfill_days}-day window. Import completion does not confirm live sync is current.` : policy ? `The last ${policy.backfill_days} days come in newest first, up to ${cap.toLocaleString()} messages. New mail syncs alongside.` : null} - {throttled ? " Replies to your outreach keep syncing; the rest resumes automatically." : null} + {throttled ? " Mail is retried automatically within its sync budget." : null} +

+ +

+ {state?.last_synced_at + ? `Last successful sync ${relative(state.last_synced_at, now)}.` + : "No successful sync check reported yet."} + {stale ? " Recovery is not yet confirmed by a recent successful check." : null}

{(state?.deferred ?? 0) > 0 && (

- {state!.deferred.toLocaleString()} message{state!.deferred === 1 ? "" : "s"} waiting on the server. + Last observed backlog: {state!.deferred.toLocaleString()} message{state!.deferred === 1 ? "" : "s"} waiting on the server. The current total may have changed.

)} diff --git a/web/src/lib/api/hooks/app/emails/useSync.ts b/web/src/lib/api/hooks/app/emails/useSync.ts index ece52e272..acc031410 100644 --- a/web/src/lib/api/hooks/app/emails/useSync.ts +++ b/web/src/lib/api/hooks/app/emails/useSync.ts @@ -12,8 +12,8 @@ export default function useSync(id: string, enabled = true) { enabled: !!id && enabled, staleTime: 15_000, refetchInterval: (query) => { - const status = query.state.data?.state?.backfill_status; - return status === "running" || status === "pending" ? 20_000 : false; + const state = query.state.data?.state; + return state?.backfill_status === "running" || state?.backfill_status === "pending" || state?.backfill_cursor?.google_recovery ? 20_000 : 60_000; }, }); } diff --git a/web/src/lib/api/models/app/emails/SyncState.ts b/web/src/lib/api/models/app/emails/SyncState.ts index 9c7e76b70..7078a989d 100644 --- a/web/src/lib/api/models/app/emails/SyncState.ts +++ b/web/src/lib/api/models/app/emails/SyncState.ts @@ -12,6 +12,17 @@ export type SyncThrottleReason = export interface SyncState { backfill_status: SyncBackfillStatus; + backfill_cursor?: { + page_token?: string; + folders?: Record; + google_recovery?: { + history_id: number; + since: Date | string; + page_token?: string; + messages_done: boolean; + stored_after?: string; + }; + }; backfill_synced: number; backfill_since?: Date; backfill_started_at?: Date;