starred/warmbly
1
0
Fork 0
mirror of https://github.com/warmbly/warmbly.git synced 2026-10-13 00:02:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: recover mailbox synchronization across Gmail Outlook and IMAP with encrypted arrival replay and safe worker lifecycle

Browse Source
This commit is contained in:
Devin AI
2026-10-09 14:24:10 +00:00
parent 62017e71dd
commit 445e014b5e
109 changed files with 7053 additions and 306 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yml
+7 -1
View File
@@ -205,7 +205,13 @@ jobs:
go test -race ./internal/repository -run '^TestLive(LastWarmupSendFailure|PersistentMailboxLoadingFailure|DispatchRetry|LineageQueueRevisions|AdminPlatformOverview|WarmupRecoversOldPreparation|WarmupPreparationRecoveryRetains)' -count=1
go test -race ./internal/tasks -run '^Test(LiveWarmup(SafetyHoldReschedules|Recovery|TaskDoesNotApplyResults)|WarmupSenderDistinguishes)' -count=1
- name: Test the Kafka codec and client lifecycle
- name: Mailbox sync admission, replay, truthful state and upgrade regressions (PostgreSQL)
env:
WARMBLY_TEST_DB: postgres://warmbly:warmbly@127.0.0.1:5432/warmbly_test?sslmode=disable
run: |
go test -race -p 1 ./internal/repository ./internal/app/consumer ./internal/app/orgtransfer ./internal/infrastructure/db -run '^TestLive(SyncArrival|SyncTruth|EmailErrorDedupe)' -count=1
- name: Test the Kafka codec, failed-handler replay and client lifecycle
run: go test -race -tags kafka ./internal/infrastructure/codec/ ./internal/infrastructure/kafka/ ./internal/infrastructure/eventbus/
# fail_ci_if_error does not cover the uploader crashing before it runs
cmd/backend/main.go
+1
View File
@@ -1137,6 +1137,7 @@ func main() {
}
cipherService = cipher.NewService(kms, cache, encryptedKeys)
emailMessageMapForHandler = repository.NewDurableEmailMessageMapRepository(primaryDB, cipherService)
// Third-party integrations: OAuth connect flows + encrypted token
// storage (sealed with the connecting user's DEK) + event-driven actions.
cmd/consumer/main.go
+1
View File
@@ -526,6 +526,7 @@ func main() {
EmailHistoryIDRepository: emailHistoryIDRepo,
EmailGraphDeltaRepository: emailGraphDeltaRepo,
EmailSyncStateRepository: repository.NewEmailSyncStateRepository(primaryDB),
ArrivalOutbox: repository.NewDurableEmailMessageMapRepository(primaryDB, cipherService),
EmailAccountErrorRepository: emailAccountErrorRepo,
WarmupRepo: warmupRepo,
PoolLinkRepo: repository.NewPoolLinkRepository(primaryDB.Pool),
docs/content/docs/development/data-control.mdx
+10
View File
@@ -281,6 +281,16 @@ The `rsync` alternative works too and is sometimes simpler, as long as the stack
[Workspace export and import](/guides/workspace-export-import/) moves a single organization's data between two running instances, re-sealing its secrets for the destination's keys. That is the per-customer tool; the bundle here is the instance-level one. They are not interchangeable: a bundle cannot be applied to one workspace, and a workspace archive cannot restore an instance.
### Mailbox sync recovery data
New durable admissions retain an organization-encrypted arrival payload in `sync_arrival_outbox` alongside the provider map. The queue tracks delivery stages and retries consumer failures under the original message identity. A delivered arrival is removed; an unresolved arrival is retained for retry until it completes or its owning mailbox or organization is deleted. Deleting the mailbox intentionally removes that mailbox's pending work. This queue does not schedule new sends or authorize repeating a send.
A full-instance backup must preserve the database, wrapped organization keys, the root key needed to unwrap them, and referenced message-body storage together. Restoring a queue without its keys or body objects cannot restore delivery. Use the normal consistent instance backup, not a separate copy of just the queue. A workspace archive excludes the queue and provider mappings whose arrivals are still pending. Drain pending work on the source before moving a workspace; importing an archive does not resume source-instance delivery.
The consumer logs aggregate pending counts and oldest-arrival age. Arrivals older than five minutes, or delivery/inspection failures continuing for one minute, raise a deduplicated operator alert through the existing operator-notification integration when configured. Logs and notifications contain counts and age, not message bodies or addresses. Operator notifications repeat at most once per fifteen minutes; without an operator channel, monitor consumer logs for `sync arrival queue needs attention`. An empty queue is not proof of provider success or historical completeness.
See [upgrade order and downgrade limits](/development/install/#mailbox-sync-upgrade-order). Legacy maps without stored arrival payloads cannot reconstruct historical lost mail.
## Outbound calls
### Diagnostic and managed-consent portability
docs/content/docs/development/install.mdx
+10
View File
@@ -294,6 +294,16 @@ Put your own compose changes in a `docker-compose.override.yml`, which compose m
Upgrade the backend first and confirm its database migrations complete before upgrading consumers. Consumers wait up to two minutes for the clean migration version required by their embedded code before starting database-dependent work. If the backend is still old or a migration failed, the consumer exits with an actionable schema-readiness error and can retry through the container restart policy. It does not run migrations or ignore missing tables.
### Mailbox sync upgrade order
For durable mailbox arrivals, upgrade the backend, let its migrations complete, then upgrade and start the consumer before upgrading workers. The backend admits each new provider map and its organization-encrypted arrival payload in one database transaction; the consumer delivers pending stages after restart. Workers continue using internal HTTP endpoints and do not need database access. Preserve the instance's organization keys and body storage with the database, as described in [data control](/development/data-control/#mailbox-sync-recovery-data).
An old worker with a new backend still writes legacy map-only records. A new worker with an old backend falls back only when the durable internal endpoint is absent (HTTP 404 or 405). Neither mixed-version path has the new durable-admission guarantee. A backend failure or an acknowledgement without the durable marker is not treated as success. When a conflicting map has no retained pending payload, the backend fails closed instead of claiming a durable admission; this includes a retry after the pending row has already been delivered and removed. Ordinary map lookup still deduplicates that message.
Existing completed imports and maps are preserved. Pre-upgrade maps contain no replay payload, so an old lost arrival cannot be reconstructed from the map alone. Bounded provider recovery covers supported recent history, not an unlimited historical repair. [Mailbox sync](/guides/mailboxes/#what-gets-synced) explains the provider limits.
Keep the upgraded consumer running until all pending arrivals finish before downgrading. The migration refuses to remove the arrival table while any pending row remains. Do not delete queued rows or force a migration version to bypass that refusal. An older consumer cannot drain this queue. Pending arrivals are instance-local and do not move in a workspace export.
Fleet auto-updates select releases no newer than the running backend. Existing fleet and node pins are checked again when answering heartbeats, so an incompatible stored target is not sent to nodes; new incompatible pins are rejected. Newly joined machines fall back to the backend's exact release, not the moving `prod` tag. Main builds use their published commit-SHA image tag. Unstamped custom builds must set `WARMBLY_VERSION` to their published image tag; unknown versions cannot be ordered safely. Upgrade the backend before moving the fleet to a newer release. This protection only takes effect after installing the compatibility release; it cannot change the behavior of an older backend already running.
The dashboard discovers a separate realtime origin from the backend's existing public configuration at container startup or static build time. Bring the backend up first; if discovery warns that it is unavailable, restart the dashboard after backend readiness or rebuild the static deployment. See [configuration](/development/configuration/).
docs/content/docs/guides/mailboxes.mdx
+11 -3
View File
@@ -207,7 +207,15 @@ Connecting a mailbox does two things: it imports the mailbox's recent history, a
**The import** brings in the last `90` days, newest first, up to `5,000` messages per mailbox (the defaults; a self-hosted instance can change both under Instance settings). It reads the inbox, sent mail and archive (plus your own folders on Gmail and IMAP), and skips trash, drafts, spam and Gmail's "All Mail". You can watch it in the mailbox drawer's **Sync** card: a progress bar with a running count while it works, then "Up to date" with what was imported. Because it runs newest first, the mail you would look for is there within the first minute; a large mailbox finishes over the next while at a steady pace. A few IMAP servers (Seznam.cz, for one) refuse to search a folder by date; on those the import reads each message's arrival date instead, which makes the first pass over a large folder a little slower and changes nothing else.
**Live sync** then follows every folder the provider exposes, including junk (so placement problems are visible) and sent mail (so a conversation shows both sides). Read state, flags, deletions and moves are mirrored too.
**Live sync** follows the provider's supported folders, including junk (so placement problems are visible) and sent mail (so a conversation shows both sides). Outlook follows Inbox, Sent, Junk, Drafts and Archive where Archive is available; it does not follow every custom Graph folder. Read state, flags, deletions and moves are mirrored too.
The Sync card distinguishes a completed history import from a recent successful live check. An unfinished pass, provider failure, recovery scan or budget hold does not refresh the last successful check or erase the last observed deferred count. That count is an observation from sync, not a measurement of all mail still on the provider. A worker heartbeat alone is not evidence that a mailbox is current. Repeated connection failures refresh the warning's observation time without generating a new notification each pass; no later successful check means recovery is unverified.
If Gmail rejects an ordinary import continuation token, the import restarts inside its original fixed date window. Already mapped mail is deduplicated, and the existing message cap and sync budgets still apply. Outlook recovers an expired delta checkpoint by replaying the configured recent-history window, reconciling known messages and following changes from a fresh checkpoint only after recovery completes. Older messages never imported are outside that recovery window. Authentication failures, throttles and transport errors are not treated as expired checkpoints.
On IMAP, discovering a new folder or a changed UIDVALIDITY generation during or after the initial import schedules a bounded recent-history scan. Generation recovery reconciles known message handles and read state without restarting the global import count. In folders excluded from history imports, such as Junk and Trash, recovery updates only known messages and does not import unseen history. An unreadable or incomplete message body remains retryable rather than being stored as a successful empty message. A folder without usable modification sequences falls back to UIDNEXT checks and periodic flags even if the server advertises CONDSTORE. Failed flag publication keeps the previous baseline so it can retry. Background lookups do not mark unread messages as seen.
On upgraded instances, a newly admitted provider message is recorded together with an encrypted pending arrival before sync advances. The consumer replays that arrival under its original identity after a restart, including related bounce or complaint processing, and provider updates wait for an unresolved arrival. This is an at-least-once recovery path, not a promise that all historical mail can be reconstructed. Old map-only records have no replay payload, and mixed-version instances retain their legacy behavior. See the [self-host upgrade order and limits](/development/install/#mailbox-sync-upgrade-order).
If Gmail expires its history checkpoint after a mailbox has been offline, Warmbly automatically scans the configured recent-history window for missed messages and checks every stored Gmail message for deletions, moves and read-state changes. Recovery resumes across worker restarts and respects the normal sync budget. Messages older than the configured import window that were never stored are not imported by recovery. Once the scan completes, changes made during recovery are followed from a fresh history checkpoint; no reconnection is needed.
@@ -221,12 +229,12 @@ A folder the sync does not recognize files as inbox, so its mail shows up in the
On an IMAP mailbox, the drawer's **Sync** card lists the folders the worker has seen under **Folders not synced**. Tick one and the sync stops opening it on its next pass, within a minute; mail already imported from it is removed from Warmbly, and mail that later moves into it from a synced folder is removed too. The mail itself is never touched at the provider. A folder that is not listed yet, because the mailbox has not synced or the folder was just created, can be typed in by name, exactly as your mail server lists it (`Warmer`, or `INBOX/Warmer` on a server that nests everything under the inbox). Case does not matter, and a skipped folder covers its subfolders.
The inbox, sent, drafts, spam, trash and archive folders cannot be skipped: a sync without them is a broken mailbox, not a quieter one. Un-ticking a folder brings it back the way a new folder arrives, syncing what lands in it from then on; the mail that accumulated while it was skipped is not imported. Up to `50` folders can be skipped per mailbox. Gmail and Outlook mailboxes have no such list.
The inbox, sent, drafts, spam, trash and archive folders cannot be skipped: a sync without them is a broken mailbox, not a quieter one. Un-ticking a folder brings it back with a bounded scan of the configured recent-history window; older mail outside that window is not imported. Up to `50` folders can be skipped per mailbox. Gmail and Outlook mailboxes have no such list.
The same setting is `PUT /emails/:id/sync` in the [API](/api/endpoints/) and `warmbly mailbox skip-folders` in the CLI.
<Callout type="info" title="Read state on older IMAP servers">
Some IMAP servers, including Outlook.com, Microsoft 365 over IMAP and Yahoo, cannot tell a client what changed since it last looked. New mail from those servers still arrives within a minute. Reading or flagging a message in another mail client shows up in Warmbly within about ten minutes rather than immediately. Nothing is lost either way, and Gmail, Outlook over OAuth, Fastmail and most self-hosted servers are immediate.
Some IMAP servers, including Outlook.com, Microsoft 365 over IMAP and Yahoo, cannot tell a client what changed since it last looked. Warmbly checks for new mail on each sync pass and scans read state and flags about every ten minutes. Those changes may take longer to appear than on a provider with usable modification sequences. Provider availability and sync budgets still apply.
</Callout>
<Callout type="info" title="Fair use, not a hard cap">
docs/content/docs/guides/warmup.mdx
+2
View File
@@ -208,6 +208,8 @@ At the volumes warmup runs, a few plaintext messages of a couple of kilobytes ea
### In the unibox
Mailbox-sync recovery replays stored arrivals, not sends. Recognized warmup remains hidden, and replay does not create another warmup send or count sync progress as proof that a provider sent mail. Receipt verification and duplicate-send protections remain separate from the mailbox's live-sync status. See [mailbox sync recovery and its limits](/guides/mailboxes/#what-gets-synced).
Warmup never appears in the unified inbox, in either direction. Warmbly recognizes it by its token, including for recipient-only mailboxes. A consumed or expired token still identifies warmup for filtering, but cannot trigger engagement again. When the header is missing, a recorded message identifier can identify older warmup during history sync; the sender-and-subject fallback stays limited to recent inbound sends so an ordinary conversation is not hidden just because it shares a subject. Existing verified warmup entries are removed from the unibox automatically in background batches, without deleting mail at the provider or changing warmup standing.
Manual replies to verified diagnostic parents can stay hidden from the unibox and follow housekeeping. They do not authorize another automated turn or count as real campaign replies.
docs/content/docs/guides/workspace-export-import.mdx
+2
View File
@@ -150,6 +150,8 @@ If you have shell access to the source, `warmblyctl org export` writes the same
## What does not travel
Pending mailbox arrivals and their delivery queue are instance-local. Provider mappings whose arrivals are still pending are excluded too. Let the source consumer finish its pending arrivals before exporting; importing the archive does not replay that source queue. A [full-instance backup](/development/data-control/#mailbox-sync-recovery-data) instead preserves the queue together with its keys and message-body storage.
The Test workspace classification is instance-local. Importing an archive does not classify the destination as Test or grant a managed Test plan or testing credits.
A few things belong to the instance rather than the workspace and are left out on purpose: the instance's link to Warmbly Cloud and which mailboxes it enrolled in the hosted pool ([Warmbly Cloud](/guides/warmbly-cloud/)), linked self-hosted instances on a cloud workspace, Google Workspace and Microsoft 365 admin grants, live sessions, in-flight OAuth handshakes and the organization's wrapped data key. Reconnect to Warmbly Cloud and make each admin grant again after the move.
go.mod
+1 -1
View File
@@ -24,7 +24,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/ssm v1.67.7
github.com/aws/smithy-go v1.28.1
github.com/confluentinc/confluent-kafka-go/v2 v2.12.0
github.com/emersion/go-imap/v2 v2.0.0-beta.5
github.com/emersion/go-imap/v2 v2.0.0-beta.7
github.com/emersion/go-message v0.18.1
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6
github.com/getsentry/sentry-go v0.36.2
go.sum
+2
View File
@@ -269,6 +269,8 @@ github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203 h1:XBBHcIb256gUJ
github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203/go.mod h1:E1jcSv8FaEny+OP/5k9UxZVw9YFWGj7eI4KR/iOBqCg=
github.com/emersion/go-imap/v2 v2.0.0-beta.5 h1:H3858DNmBuXyMK1++YrQIRdpKE1MwBc+ywBtg3n+0wA=
github.com/emersion/go-imap/v2 v2.0.0-beta.5/go.mod h1:BZTFHsS1hmgBkFlHqbxGLXk2hnRqTItUgwjSSCsYNAk=
github.com/emersion/go-imap/v2 v2.0.0-beta.7 h1:lNznYWa5uhMrngnSYEklzCeye4DBq9TEJ+pr0K593+8=
github.com/emersion/go-imap/v2 v2.0.0-beta.7/go.mod h1:BZTFHsS1hmgBkFlHqbxGLXk2hnRqTItUgwjSSCsYNAk=
github.com/emersion/go-message v0.18.1 h1:tfTxIoXFSFRwWaZsgnqS1DSZuGpYGzSmCZD8SK3QA2E=
github.com/emersion/go-message v0.18.1/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA=
github.com/emersion/go-msgauth v0.7.0 h1:vj2hMn6KhFtW41kshIBTXvp6KgYSqpA/ZN9Pv4g1INc=
internal/api/handler/internal_arrival_test.go
+70
View File
@@ -0,0 +1,70 @@
package handler
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/repository"
)
type internalArrivalMap struct {
calls, legacyCalls int
err error
}
func (m *internalArrivalMap) Add(context.Context, repository.EmailMessageData) error {
m.legacyCalls++
return nil
}
func (*internalArrivalMap) Get(context.Context, uuid.UUID, uuid.UUID, string) (*repository.EmailMessageData, error) {
return nil, nil
}
func (*internalArrivalMap) Del(context.Context, uuid.UUID, uuid.UUID, string, uuid.UUID) error {
return nil
}
func (m *internalArrivalMap) AdmitArrival(context.Context, repository.EmailMessageData, *repository.PendingArrival) error {
m.calls++
return m.err
}
func TestInternalArrivalAcknowledgesOnlyCompletedAdmission(t *testing.T) {
for _, tc := range []struct {
name string
err error
status int
ack string
}{
{"success", nil, 204, "1"}, {"failure", errors.New("private payload or cipher failure"), 503, ""},
{"unconfirmed collision", repository.ErrArrivalAdmissionUnconfirmed, 503, ""},
} {
t.Run(tc.name, func(t *testing.T) {
m := &internalArrivalMap{err: tc.err}
h := &Handler{EmailMessageMap: m}
r := gin.New()
r.POST("/arrival", h.InternalAdmitEmailArrival)
w := httptest.NewRecorder()
r.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/arrival", strings.NewReader(`{"map":{},"pending":{}}`)))
if m.calls != 1 || w.Code != tc.status || w.Header().Get("X-Warmbly-Arrival-Durable") != tc.ack || w.Body.Len() != 0 {
t.Fatalf("calls=%d status=%d body=%q", m.calls, w.Code, w.Body.String())
}
})
}
}
func TestInternalArrivalPreservesLegacyWorkerProtocol(t *testing.T) {
m := &internalArrivalMap{}
h := &Handler{EmailMessageMap: m}
r := gin.New()
r.PUT("/map", h.InternalPutEmailMessageMap)
w := httptest.NewRecorder()
r.ServeHTTP(w, httptest.NewRequest(http.MethodPut, "/map", strings.NewReader(`{"user_id":"fixture","email_id":"fixture","message_id":"fixture","id":"fixture","thread_id":"fixture"}`)))
if w.Code != 204 || m.legacyCalls != 1 || m.calls != 0 || w.Header().Get("X-Warmbly-Arrival-Durable") != "" {
t.Fatalf("legacy contract changed: %d", w.Code)
}
}
internal/api/handler/internal_email_message_map.go
+23
View File
@@ -11,6 +11,29 @@ import (
"github.com/warmbly/warmbly/internal/repository"
)
func (h *Handler) InternalAdmitEmailArrival(c *gin.Context) {
var p struct {
Map emailMessageMapPayload `json:"map"`
Pending *repository.PendingArrival `json:"pending"`
}
if err := json.NewDecoder(http.MaxBytesReader(c.Writer, c.Request.Body, 4<<20)).Decode(&p); err != nil {
c.Status(http.StatusBadRequest)
return
}
repo, ok := h.EmailMessageMap.(repository.ArrivalAdmission)
if !ok {
c.Status(http.StatusServiceUnavailable)
return
}
if err := repo.AdmitArrival(c.Request.Context(), repository.EmailMessageData(p.Map), p.Pending); err != nil {
// Do not expose message content or org encryption failures over the protocol.
c.Status(http.StatusServiceUnavailable)
return
}
c.Header("X-Warmbly-Arrival-Durable", "1")
c.Status(http.StatusNoContent)
}
// Internal email-message-map endpoints. Workers call these instead of touching
// Postgres directly (per CLAUDE.md), mirroring the DEK endpoints. Auth via
// middleware.NodeBrokerAuthMiddleware (the node token).
internal/api/routes.go
+1
View File
@@ -191,6 +191,7 @@ func Run(
// Worker mailbox-sync messageId -> internal email map.
node.GET("/email-message-map", h.InternalGetEmailMessageMap)
node.PUT("/email-message-map", h.InternalPutEmailMessageMap)
node.POST("/email-message-map/arrival", h.InternalAdmitEmailArrival)
node.DELETE("/email-message-map", h.InternalDeleteEmailMessageMap)
// Sync governor priority lane: "is this new message a reply to
internal/app/consumer/arrival_outbox.go
+98
View File
@@ -0,0 +1,98 @@
package jobs
import (
"context"
"fmt"
"strconv"
"time"
"github.com/rs/zerolog/log"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/repository"
)
func (s *JobsService) runArrivalOutbox(ctx context.Context) {
if s.ArrivalOutbox == nil {
return
}
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
var failedSince, lastAlert time.Time
for {
pass, cancel := context.WithTimeout(ctx, 30*time.Second)
err := s.ArrivalOutbox.DeliverArrivals(pass, func(ctx context.Context, kind models.JobEventType, body any) error {
return s.deliverArrivalEvent(ctx, kind, body)
})
if err != nil {
log.Warn().Msg("sync arrival delivery deferred")
}
s.observeArrivalBacklog(pass, err, time.Now(), &failedSince, &lastAlert)
cancel()
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
func (s *JobsService) observeArrivalBacklog(ctx context.Context, deliveryErr error, now time.Time, failedSince, lastAlert *time.Time) {
backlog := repository.ArrivalBacklog{}
known := false
if stats, ok := s.ArrivalOutbox.(repository.ArrivalBacklogRepository); ok {
var err error
backlog, err = stats.ArrivalBacklog(ctx)
if err != nil {
deliveryErr = err
} else {
known = true
}
}
if deliveryErr != nil {
if failedSince.IsZero() {
*failedSince = now
}
} else {
*failedSince = time.Time{}
}
age := time.Duration(0)
if backlog.Oldest != nil {
age = max(0, now.Sub(*backlog.Oldest))
}
if age < 5*time.Minute && (failedSince.IsZero() || now.Sub(*failedSince) < time.Minute) {
return
}
if !lastAlert.IsZero() && now.Sub(*lastAlert) < 15*time.Minute {
return
}
*lastAlert = now
entry := log.Warn().Bool("backlog_known", known)
if known {
entry.Int("pending", backlog.Pending).Int64("oldest_age_seconds", int64(age.Seconds()))
}
entry.Msg("sync arrival queue needs attention")
if s.OpsNotifier == nil {
return
}
if s.Cache != nil {
ok, err := s.Cache.SetNX(ctx, "sync:arrival:opsnotify", "1", 15*time.Minute).Result()
if err == nil && !ok {
return
}
}
fields := map[string]string{"Pending": "unavailable", "Oldest age seconds": "unavailable"}
if known {
fields["Pending"] = strconv.Itoa(backlog.Pending)
fields["Oldest age seconds"] = strconv.FormatInt(int64(age.Seconds()), 10)
}
s.OpsNotifier.NotifyOperator("sync.arrival.backlog", "Mailbox arrival delivery delayed",
"Stored arrivals are waiting for delivery, or delivery checks are failing. Check the consumer and database before assuming sync is current.",
fields)
}
func (s *JobsService) deliverArrivalEvent(ctx context.Context, kind models.JobEventType, body any) error {
if _, ok := s.eventHandlers[kind]; !ok {
return fmt.Errorf("durable arrival handler unavailable: %s", kind)
}
return s.HandleEvent(ctx, &models.JobEvent{Type: kind, Body: body})
}
internal/app/consumer/arrival_outbox_live_test.go
+151
View File
@@ -0,0 +1,151 @@
package jobs
import (
"context"
"errors"
"testing"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/app/advanced"
"github.com/warmbly/warmbly/internal/app/cipher"
"github.com/warmbly/warmbly/internal/app/warmup"
"github.com/warmbly/warmbly/internal/infrastructure/encryptedkeys"
"github.com/warmbly/warmbly/internal/infrastructure/kms"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/repository"
)
func TestLiveSyncArrivalReplayPreservesHiddenWarmupAndReceiptAuthority(t *testing.T) {
s, d := liveWarmupService(t)
f := newWarmupFixture(t, d)
ctx := t.Context()
f.mintToken(t, s.WarmupRepo)
k, err := kms.NewLocal(make([]byte, 32))
if err != nil {
t.Fatal(err)
}
r := repository.NewDurableEmailMessageMapRepository(d, cipher.NewService(k, nil, encryptedkeys.NewPostgres(d)))
e := f.arrival(f.sentMsgID, nil)
e.UserID = f.senderUser
e.Message.EmailID = f.sender
e.Message.Folder = models.FolderSent
data := repository.EmailMessageData{UserID: e.UserID.String(), EmailID: e.Message.EmailID.String(), MessageID: e.Message.MessageID, ID: e.Message.ID.String(), ThreadID: e.Message.ThreadID}
if err := r.AdmitArrival(ctx, data, &repository.PendingArrival{Arrival: e}); err != nil {
t.Fatal(err)
}
s.InitEvents()
lost := errors.New("consumer terminated after handling")
if err := r.DeliverArrivals(ctx, func(ctx context.Context, kind models.JobEventType, body any) error {
if err := s.deliverArrivalEvent(ctx, kind, body); err != nil {
return err
}
return lost
}); !errors.Is(err, lost) {
t.Fatal(err)
}
if _, err := d.Exec(ctx, `UPDATE sync_arrival_outbox SET retry_at=now() WHERE email_id=$1`, f.sender); err != nil {
t.Fatal(err)
}
fresh, _ := liveWarmupService(t)
fresh.InitEvents()
if err := r.DeliverArrivals(ctx, fresh.deliverArrivalEvent); err != nil {
t.Fatal(err)
}
var visible, pending, receipts int
if err := d.QueryRow(ctx, `SELECT count(*) FROM unibox_emails WHERE id=$1`, e.Message.ID).Scan(&visible); err != nil || visible != 0 {
t.Fatalf("unconfirmed sent copy exposed: %d %v", visible, err)
}
if err := d.QueryRow(ctx, `SELECT count(*) FROM unibox_pending_emails WHERE id=$1`, e.Message.ID).Scan(&pending); err != nil || pending != 1 {
t.Fatalf("verification deferral lost original ID: %d %v", pending, err)
}
if err := d.QueryRow(ctx, `SELECT count(*) FROM warmup_received WHERE sender_account_id=$1`, f.sender).Scan(&receipts); err != nil || receipts != 0 {
t.Fatalf("sync invented receipt: %d %v", receipts, err)
}
if err := fresh.HandleEmailSent(ctx, models.SendEmailResult{TaskID: f.task, Success: true, MessageID: f.sentMsgID}); err != nil {
t.Fatal(err)
}
if _, err := d.Exec(ctx, `UPDATE unibox_pending_emails SET retry_at=now() WHERE email_account_id=$1`, f.sender); err != nil {
t.Fatal(err)
}
if err := fresh.retryPendingWarmupVerification(ctx); err != nil {
t.Fatal(err)
}
if err := d.QueryRow(ctx, `SELECT count(*) FROM unibox_emails WHERE id=$1`, e.Message.ID).Scan(&visible); err != nil || visible != 0 {
t.Fatalf("confirmed warmup exposed: %d %v", visible, err)
}
if err := d.QueryRow(ctx, `SELECT count(*) FROM warmup_received WHERE sender_account_id=$1`, f.sender).Scan(&receipts); err != nil || receipts != 0 {
t.Fatalf("sent copy invented recipient receipt: %d %v", receipts, err)
}
other := f.arrival("<"+uuid.NewString()+"@test.local>", nil)
other.Message.Subject = "ordinary customer question"
data = repository.EmailMessageData{UserID: other.UserID.String(), EmailID: other.Message.EmailID.String(), MessageID: other.Message.MessageID, ID: other.Message.ID.String()}
if err := r.AdmitArrival(ctx, data, &repository.PendingArrival{Arrival: other}); err != nil {
t.Fatal(err)
}
if err := r.DeliverArrivals(ctx, fresh.deliverArrivalEvent); err != nil {
t.Fatal(err)
}
if err := d.QueryRow(ctx, `SELECT count(*) FROM unibox_emails WHERE id=$1`, other.Message.ID).Scan(&visible); err != nil || visible != 1 {
t.Fatalf("ordinary arrival missing: %d %v", visible, err)
}
}
func TestLiveSyncArrivalReportReplayIsIdempotent(t *testing.T) {
d := liveDB(t)
f := newSendResultFixture(t, d)
s := liveJobsService(d)
ctx := t.Context()
task := f.stampSend(t, s)
messageID := "<" + task.String() + "@test.local>"
if _, err := d.Exec(ctx, `UPDATE tasks SET message_id=$2 WHERE id=$1`, task, messageID); err != nil {
t.Fatal(err)
}
s.UniboxRepository = repository.NewUniboxRepository(d)
s.EmailRepository = repository.NewEmailRepostory(d, nil)
s.WarmupRepo = repository.NewWarmupRepository(d.Pool)
s.AdvancedService = advanced.NewService(repository.NewAdvancedOutreachRepository(d.Pool), s.CampaignRepo, s.EmailRepository, s.TaskRepo, repository.NewContactRepostory(d), s.CampaignProgressRepo, nil, nil, nil, nil, warmup.NewService(s.WarmupRepo))
s.InitEvents()
k, err := kms.NewLocal(make([]byte, 32))
if err != nil {
t.Fatal(err)
}
cipherService := cipher.NewService(k, nil, encryptedkeys.NewPostgres(d))
r := repository.NewDurableEmailMessageMapRepository(d, cipherService)
id := uuid.New()
recipient := "lead-" + f.contact.String()[:8] + "@test.local"
p := &repository.PendingArrival{Arrival: &models.JobEventNewEmail{UserID: f.user, ReportOriginalMessageID: messageID, Message: &models.EmailMessageStoreData{ID: id, EmailID: f.mailbox, MessageID: "report", Folder: models.FolderInbox}}, Bounce: &models.JobEventInboundBounce{UserID: f.user, EmailID: f.mailbox, OriginalMessageID: messageID, FailedRecipient: recipient, Reason: "recipient refused"}, Complaint: &models.JobEventInboundComplaint{UserID: f.user, EmailID: f.mailbox, OriginalMessageID: messageID, ComplainedRecipient: recipient, Provider: "fixture"}}
data := repository.EmailMessageData{UserID: f.user.String(), EmailID: f.mailbox.String(), MessageID: "report", ID: id.String()}
if err := r.AdmitArrival(ctx, data, p); err != nil {
t.Fatal(err)
}
lost := errors.New("result lost after report was recorded")
for _, target := range []models.JobEventType{models.JobEventTypeInboundBounce, models.JobEventTypeInboundComplaint} {
if err := r.DeliverArrivals(ctx, func(ctx context.Context, kind models.JobEventType, body any) error {
if err := s.deliverArrivalEvent(ctx, kind, body); err != nil {
return err
}
if kind == target {
return lost
}
return nil
}); !errors.Is(err, lost) {
t.Fatal(err)
}
if _, err := d.Exec(ctx, `UPDATE sync_arrival_outbox SET retry_at=now() WHERE email_id=$1`, f.mailbox); err != nil {
t.Fatal(err)
}
}
if err := repository.NewDurableEmailMessageMapRepository(d, cipherService).DeliverArrivals(ctx, s.deliverArrivalEvent); err != nil {
t.Fatal(err)
}
var events, suppressed int
if err := d.QueryRow(ctx, `SELECT count(*) FROM deliverability_events WHERE organization_id=$1`, f.org).Scan(&events); err != nil {
t.Fatal(err)
}
if err := d.QueryRow(ctx, `SELECT count(*) FROM suppressed_recipients WHERE organization_id=$1`, f.org).Scan(&suppressed); err != nil {
t.Fatal(err)
}
if events != 2 || suppressed != 1 {
t.Fatalf("report replay duplicated effects: events=%d suppressed=%d", events, suppressed)
}
}
internal/app/consumer/arrival_outbox_test.go
+81
View File
@@ -0,0 +1,81 @@
package jobs
import (
"context"
"errors"
"strings"
"testing"
"time"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/repository"
)
func TestDurableArrivalRequiresRegisteredHandler(t *testing.T) {
s := &JobsService{}
if err := s.deliverArrivalEvent(context.Background(), models.JobEventTypeNewEmail, &models.JobEventNewEmail{}); err == nil {
t.Fatal("unregistered durable event dropped")
}
called := false
s.InitEvents()
Register(s, models.JobEventTypeNewEmail, func(context.Context, *models.JobEventNewEmail) error { called = true; return nil })
if err := s.deliverArrivalEvent(context.Background(), models.JobEventTypeNewEmail, &models.JobEventNewEmail{}); err != nil || !called {
t.Fatalf("called=%v err=%v", called, err)
}
}
type arrivalBacklogStub struct {
repository.ArrivalOutbox
backlog repository.ArrivalBacklog
err error
}
func (s *arrivalBacklogStub) ArrivalBacklog(context.Context) (repository.ArrivalBacklog, error) {
return s.backlog, s.err
}
type arrivalOperatorStub struct {
calls int
content string
fields map[string]string
}
func (s *arrivalOperatorStub) NotifyOperator(key, title, summary string, fields map[string]string) {
s.calls++
s.content = key + title + summary
s.fields = fields
if len(fields) != 2 {
panic("unexpected private fields")
}
}
func TestDurableArrivalBacklogAlertsAreAgedAndDeduplicated(t *testing.T) {
now := time.Now()
old := now.Add(-6 * time.Minute)
outbox := &arrivalBacklogStub{backlog: repository.ArrivalBacklog{Pending: 8, Oldest: &old}}
ops := &arrivalOperatorStub{}
s := &JobsService{ArrivalOutbox: outbox, OpsNotifier: ops}
var failed, alerted time.Time
s.observeArrivalBacklog(t.Context(), nil, now, &failed, &alerted)
s.observeArrivalBacklog(t.Context(), nil, now.Add(time.Minute), &failed, &alerted)
if ops.calls != 1 {
t.Fatalf("repeat alerts=%d", ops.calls)
}
outbox.backlog = repository.ArrivalBacklog{}
s.observeArrivalBacklog(t.Context(), nil, now.Add(16*time.Minute), &failed, &alerted)
if ops.calls != 1 {
t.Fatal("empty queue claimed ongoing backlog")
}
outbox.err = errors.New("private database error")
s.observeArrivalBacklog(t.Context(), nil, now.Add(17*time.Minute), &failed, &alerted)
if ops.calls != 1 {
t.Fatal("transient read failure alerted too soon")
}
s.observeArrivalBacklog(t.Context(), nil, now.Add(18*time.Minute), &failed, &alerted)
if ops.calls != 2 {
t.Fatal("persistent queue inspection failure was silent")
}
if strings.Contains(ops.content, "private database error") || ops.fields["Pending"] != "unavailable" || ops.fields["Oldest age seconds"] != "unavailable" {
t.Fatal("operator notice exposed an error or fabricated unavailable counts")
}
}
internal/app/consumer/event_flags_update.go
+2 -2
View File
@@ -38,7 +38,7 @@ func (s *JobsService) HandleFlagsAdd(ctx context.Context, e *models.JobEventFlag
}
}
email, err := s.emailForSyncUpdate(ctx, e.UserID, e.ID, func(message *models.EmailMessageStoreData) {
email, err := s.emailForSyncUpdate(ctx, e.UserID, e.EmailID, e.ID, func(message *models.EmailMessageStoreData) {
for _, flag := range e.Flags {
if !slices.Contains(message.Flags, flag) {
message.Flags = append(message.Flags, flag)
@@ -155,7 +155,7 @@ func warmupTokenFromFlags(flags []string) string {
}
func (s *JobsService) HandleFlagsRemove(ctx context.Context, e *models.JobEventFlags) error {
email, err := s.emailForSyncUpdate(ctx, e.UserID, e.ID, func(message *models.EmailMessageStoreData) {
email, err := s.emailForSyncUpdate(ctx, e.UserID, e.EmailID, e.ID, func(message *models.EmailMessageStoreData) {
message.Flags = slices.DeleteFunc(message.Flags, func(flag string) bool { return slices.Contains(e.Flags, flag) })
if models.SeenFromFlags(e.Flags) {
message.Seen = false
internal/app/consumer/event_remove_email.go
+14 -2
View File
@@ -2,6 +2,7 @@ package jobs
import (
"context"
"errors"
"time"
"github.com/rs/zerolog/log"
@@ -20,8 +21,17 @@ import (
// mailbox. A removal later than the window is housekeeping (see
// warmupDeletionCounts) and is not held against anyone.
//
// It also drops the local unibox entry for the removed message (best-effort).
// It drops the local unibox entry only after durable arrival replay is complete.
func (s *JobsService) HandleRemoveEmail(ctx context.Context, e *models.JobEventRemoveEmail) error {
if s.ArrivalOutbox != nil {
pending, err := s.ArrivalOutbox.HasPendingArrival(ctx, e.UserID, e.EmailID, e.ID)
if err != nil {
return err
}
if pending {
return ErrSyncArrivalPending
}
}
var checkErr error
// A message the sync found in a folder the owner excluded is filed, not
// deleted: it is still in the mailbox, so nothing is held against anyone.
@@ -48,7 +58,9 @@ func (s *JobsService) HandleRemoveEmail(ctx context.Context, e *models.JobEventR
}
if s.UniboxRepository != nil {
_ = s.UniboxRepository.Delete(ctx, e.UserID, e.ID)
if err := s.UniboxRepository.Delete(ctx, e.UserID, e.ID); err != nil {
return errors.Join(checkErr, err)
}
}
s.publishInboxDeleted(ctx, e.UserID, e.EmailID, e.ID.String())
internal/app/consumer/event_sync_state.go
+5 -6
View File
@@ -34,12 +34,11 @@ func (s *JobsService) HandleSyncState(ctx context.Context, e *models.JobEventSyn
return err
}
// A relayed state means a pass completed: the worker reached the server,
// listed its folders and finished the tick. That is the only signal we
// get that an outage is over, and without it a five-minute blip left a
// red "needs attention" on the mailbox for good, because nothing but a
// credential reconnect ever resolved an error row.
s.resolveTransientMailErrors(ctx, e.EmailID, e.State.LastSyncedAt)
// Progress relays are not recovery evidence; require a newer successful check.
if e.State.LastSyncedAt != nil && e.State.Deferred == 0 && e.State.BackfillCursor.GoogleRecovery == nil &&
(prev == nil || prev.LastSyncedAt == nil || e.State.LastSyncedAt.After(*prev.LastSyncedAt)) {
s.resolveTransientMailErrors(ctx, e.EmailID, e.State.LastSyncedAt)
}
if s.StreamingPublisher == nil {
return nil
internal/app/consumer/event_sync_state_test.go
+3 -2
View File
@@ -14,11 +14,12 @@ import (
// stubSyncStateRepo accepts what the worker relays.
type stubSyncStateRepo struct {
repository.EmailSyncStateRepository
put *models.SyncState
put *models.SyncState
saved *models.SyncState
}
func (s *stubSyncStateRepo) Get(context.Context, uuid.UUID) (*models.SyncState, error) {
return nil, nil
return s.saved, nil
}
func (s *stubSyncStateRepo) Put(_ context.Context, _, _ uuid.UUID, st *models.SyncState) error {
internal/app/consumer/event_update_email.go
+23 -6
View File
@@ -13,7 +13,7 @@ import (
)
func (s *JobsService) HandleUpdateEmail(ctx context.Context, e *models.JobEventEmailUpdate) error {
email, err := s.emailForSyncUpdate(ctx, e.UserID, e.ID, func(message *models.EmailMessageStoreData) {
email, err := s.emailForSyncUpdate(ctx, e.UserID, e.EmailID, e.ID, func(message *models.EmailMessageStoreData) {
token := warmupTokenFromMessage(message)
message.Flags = append([]string{}, e.Flags...)
if token != "" {
@@ -98,7 +98,7 @@ func (s *JobsService) HandleFolderUpdate(ctx context.Context, e *models.JobEvent
if e.Relayed {
return s.applyRelayedFolder(ctx, e)
}
email, err := s.emailForSyncUpdate(ctx, e.UserID, e.ID, func(message *models.EmailMessageStoreData) {
email, err := s.emailForSyncUpdate(ctx, e.UserID, e.EmailID, e.ID, func(message *models.EmailMessageStoreData) {
// A pending row is not visible yet, so nothing has filed it locally.
message.Folder = e.Folder
})
@@ -144,16 +144,33 @@ func (s *JobsService) applyRelayedFolder(ctx context.Context, e *models.JobEvent
}
// emailForSyncUpdate rechecks visible mail if verification won the pending-row lock.
func (s *JobsService) emailForSyncUpdate(ctx context.Context, userID, id uuid.UUID, updatePending func(*models.EmailMessageStoreData)) (*models.EmailMessageStoreData, error) {
message, err := s.UniboxRepository.GetByID(ctx, userID, id)
var ErrSyncArrivalPending = errors.New("sync arrival pending delivery")
func (s *JobsService) emailForSyncUpdate(ctx context.Context, userID, emailID, id uuid.UUID, updatePending func(*models.EmailMessageStoreData)) (*models.EmailMessageStoreData, error) {
message, err := s.UniboxRepository.GetForSync(ctx, userID, emailID, id)
if !errors.Is(err, repository.ErrEmailNotFound) {
return message, err
}
updated, err := s.UniboxRepository.UpdatePendingEmail(ctx, userID, id, updatePending)
updated, err := s.UniboxRepository.UpdatePendingEmail(ctx, userID, id, func(message *models.EmailMessageStoreData) {
if message.EmailID == emailID {
updatePending(message)
}
})
if err != nil || updated {
return nil, err
}
message, err = s.UniboxRepository.GetByID(ctx, userID, id)
message, err = s.UniboxRepository.GetForSync(ctx, userID, emailID, id)
if errors.Is(err, repository.ErrEmailNotFound) && s.ArrivalOutbox != nil {
pending, lookupErr := s.ArrivalOutbox.HasPendingArrival(ctx, userID, emailID, id)
if lookupErr != nil {
return nil, lookupErr
}
if pending {
return nil, ErrSyncArrivalPending
}
// Delivery may have finished between the visible-row read and the marker read.
message, err = s.UniboxRepository.GetForSync(ctx, userID, emailID, id)
}
if errors.Is(err, repository.ErrEmailNotFound) {
return nil, nil
}
internal/app/consumer/service.go
+2
View File
@@ -41,6 +41,7 @@ type JobsService struct {
EmailHistoryIDRepository repository.EmailHistoryIDRepository
EmailGraphDeltaRepository repository.EmailGraphDeltaRepository
EmailSyncStateRepository repository.EmailSyncStateRepository
ArrivalOutbox repository.ArrivalOutbox
EmailAccountErrorRepository repository.EmailAccountErrorRepository
WarmupRepo repository.WarmupRepository
// PoolLinkRepo marks warmup-only mailboxes of linked instances; nil when unused.
@@ -126,6 +127,7 @@ func (s *JobsService) Start(ctx context.Context) {
// timestamps.
go s.sweepFollowUps(ctx)
go s.runSendResultEffects(ctx)
go s.runArrivalOutbox(ctx)
if err := s.Bus.Subscribe(ctx, []string{kafka.TopicWorkerEvents}, "consumer-group", s.receive); err != nil {
log.Error().Err(err).Msg("consumer: worker-events subscription ended")
internal/app/consumer/sync_truth_live_test.go
+317
View File
@@ -0,0 +1,317 @@
package jobs
import (
"context"
"errors"
"testing"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/app/cipher"
"github.com/warmbly/warmbly/internal/config"
"github.com/warmbly/warmbly/internal/infrastructure/encryptedkeys"
"github.com/warmbly/warmbly/internal/infrastructure/kms"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/repository"
)
func TestLiveSyncTruthRemovalWaitsForArrivalAndLostStageAcknowledgement(t *testing.T) {
for _, lostAck := range []bool{false, true} {
for _, kind := range []string{"ordinary", "confirmed warmup", "unconfirmed warmup"} {
name := kind + "/before ingestion"
if lostAck {
name = kind + "/lost stage acknowledgement"
}
t.Run(name, func(t *testing.T) {
s, d := liveWarmupService(t)
f := newWarmupFixture(t, d)
ctx := t.Context()
key, err := kms.NewLocal(make([]byte, 32))
if err != nil {
t.Fatal(err)
}
r := repository.NewDurableEmailMessageMapRepository(d, cipher.NewService(key, nil, encryptedkeys.NewPostgres(d)))
s.ArrivalOutbox = r
s.InitEvents()
e := f.arrival("<"+uuid.NewString()+"@test.local>", nil)
e.Message.Subject = "ordinary customer question"
var token uuid.UUID
if kind != "ordinary" {
token = f.mintToken(t, s.WarmupRepo)
if _, err := d.Exec(ctx, `INSERT INTO warmup_tasks(task_id,lineage_version,subject,max_turns) VALUES($1,1,$2,3)`, f.task, f.subject); err != nil {
t.Fatal(err)
}
e = f.arrival(f.sentMsgID, []string{config.WarmupVerifyHeader + ":" + token.String()})
if kind == "confirmed warmup" {
if err := s.HandleEmailSent(ctx, models.SendEmailResult{TaskID: f.task, Success: true, MessageID: f.sentMsgID}); err != nil {
t.Fatal(err)
}
}
}
mapping := repository.EmailMessageData{UserID: e.UserID.String(), EmailID: e.Message.EmailID.String(), ID: e.Message.ID.String(), MessageID: e.Message.MessageID}
if err := r.AdmitArrival(ctx, mapping, &repository.PendingArrival{Arrival: e}); err != nil {
t.Fatal(err)
}
if lostAck {
lost := errors.New("arrival handled but stage acknowledgement lost")
if err := r.DeliverArrivals(ctx, func(ctx context.Context, kind models.JobEventType, body any) error {
if err := s.deliverArrivalEvent(ctx, kind, body); err != nil {
return err
}
return lost
}); !errors.Is(err, lost) {
t.Fatalf("stage loss fixture: %v", err)
}
}
remove := &models.JobEventRemoveEmail{UserID: e.UserID, EmailID: e.Message.EmailID, ID: e.Message.ID}
if err := s.HandleRemoveEmail(ctx, remove); !errors.Is(err, ErrSyncArrivalPending) {
t.Fatalf("removal overtook pending original arrival: %v", err)
}
var visible int
if err := d.QueryRow(ctx, `SELECT count(*) FROM unibox_emails WHERE id=$1`, e.Message.ID).Scan(&visible); err != nil {
t.Fatal(err)
}
if lostAck && kind == "ordinary" && visible != 1 {
t.Fatal("blocked removal deleted already-ingested mail")
}
if _, err := d.Exec(ctx, `UPDATE sync_arrival_outbox SET retry_at=now() WHERE id=$1`, e.Message.ID); err != nil {
t.Fatal(err)
}
fresh, _ := liveWarmupService(t)
fresh.ArrivalOutbox = r
fresh.InitEvents()
if err := r.DeliverArrivals(ctx, fresh.deliverArrivalEvent); err != nil {
t.Fatal(err)
}
if pending, err := r.HasPendingArrival(ctx, e.UserID, e.Message.EmailID, e.Message.ID); err != nil || pending {
t.Fatalf("arrival did not finish: %v %v", pending, err)
}
unowned := *remove
unowned.UserID = f.senderUser
if err := fresh.HandleRemoveEmail(ctx, &unowned); err != nil {
t.Fatalf("unowned removal: %v", err)
}
if kind == "ordinary" {
if _, err := fresh.UniboxRepository.GetForSync(ctx, e.UserID, e.Message.EmailID, e.Message.ID); err != nil {
t.Fatalf("unowned removal deleted owned row: %v", err)
}
}
if err := fresh.HandleRemoveEmail(ctx, remove); err != nil {
t.Fatal(err)
}
if err := r.DeliverArrivals(ctx, fresh.deliverArrivalEvent); err != nil {
t.Fatal(err)
}
if err := fresh.HandleRemoveEmail(ctx, remove); err != nil {
t.Fatalf("removal retry: %v", err)
}
// Recovery updates to a retained map must not resurrect hidden or deleted mail.
if err := fresh.HandleUpdateEmail(ctx, &models.JobEventEmailUpdate{UserID: e.UserID, EmailID: e.Message.EmailID, ID: e.Message.ID, UID: 42, Mailbox: 9, ModSeq: 100, Flags: []string{models.FlagSeen}}); err != nil {
t.Fatal(err)
}
for _, table := range []string{"unibox_emails", "unibox_pending_emails", "sync_arrival_outbox"} {
var rows int
if err := d.QueryRow(ctx, "SELECT count(*) FROM "+table+" WHERE id=$1", e.Message.ID).Scan(&rows); err != nil || rows != 0 {
t.Fatalf("%s retained or recreated mail: %d %v", table, rows, err)
}
}
known, err := r.Get(ctx, e.UserID, e.Message.EmailID, e.Message.MessageID)
if err != nil || known == nil || known.ID != e.Message.ID.String() {
t.Fatalf("original mapping lost: %+v %v", known, err)
}
if kind != "ordinary" {
var receipts int
want := 0
if kind == "confirmed warmup" {
want = 1
}
if err := d.QueryRow(ctx, `SELECT count(*) FROM warmup_received WHERE sender_account_id=$1 AND email_account_id=$2`, f.sender, f.partner).Scan(&receipts); err != nil || receipts != want {
t.Fatalf("warmup authority changed: %d want=%d err=%v", receipts, want, err)
}
stored, err := fresh.WarmupRepo.FindWarmupToken(ctx, token)
if err != nil || stored == nil || (stored.ConsumedAt != nil) != (want == 1) {
t.Fatalf("warmup token authority changed: %+v %v", stored, err)
}
}
})
}
}
}
type failSyncArrivalCreate struct {
repository.UniboxRepository
fail bool
}
func (r *failSyncArrivalCreate) CreateEntry(ctx context.Context, user uuid.UUID, message *models.EmailMessageStoreData) error {
if r.fail {
r.fail = false
return errors.New("transient arrival write failure")
}
return r.UniboxRepository.CreateEntry(ctx, user, message)
}
func TestLiveSyncTruthPendingArrivalRetainsProviderUpdatesAcrossReload(t *testing.T) {
for _, kind := range []string{"seen", "folder", "full update"} {
t.Run(kind, func(t *testing.T) {
s, d := liveWarmupService(t)
f := newWarmupFixture(t, d)
ctx := t.Context()
key, err := kms.NewLocal(make([]byte, 32))
if err != nil {
t.Fatal(err)
}
cipherService := cipher.NewService(key, nil, encryptedkeys.NewPostgres(d))
r := repository.NewDurableEmailMessageMapRepository(d, cipherService)
s.ArrivalOutbox = r
s.UniboxRepository = &failSyncArrivalCreate{UniboxRepository: s.UniboxRepository, fail: true}
e := f.arrival("<"+uuid.NewString()+"@test.local>", nil)
e.Message.Subject = "ordinary customer question"
mapping := repository.EmailMessageData{UserID: e.UserID.String(), EmailID: e.Message.EmailID.String(), ID: e.Message.ID.String(), MessageID: e.Message.MessageID}
if err := r.AdmitArrival(ctx, mapping, &repository.PendingArrival{Arrival: e}); err != nil {
t.Fatal(err)
}
s.InitEvents()
if err := r.DeliverArrivals(ctx, s.deliverArrivalEvent); err == nil {
t.Fatal("expected transient arrival write failure")
}
apply := func(service *JobsService) error {
switch kind {
case "seen":
return service.HandleFlagsAdd(ctx, &models.JobEventFlags{UserID: e.UserID, EmailID: e.Message.EmailID, ID: e.Message.ID, Flags: []string{models.FlagSeen}})
case "folder":
return service.HandleFolderUpdate(ctx, &models.JobEventFolderUpdate{UserID: e.UserID, EmailID: e.Message.EmailID, ID: e.Message.ID, Folder: models.FolderArchive})
default:
return service.HandleUpdateEmail(ctx, &models.JobEventEmailUpdate{UserID: e.UserID, EmailID: e.Message.EmailID, ID: e.Message.ID, Flags: []string{models.FlagSeen}, Folder: models.FolderArchive, FolderPath: "Archive", UID: 42, Mailbox: 7})
}
}
if err := apply(s); !errors.Is(err, ErrSyncArrivalPending) {
t.Fatalf("update was acknowledged ahead of failed arrival: %v", err)
}
if _, err := d.Exec(ctx, `UPDATE sync_arrival_outbox SET retry_at=now() WHERE id=$1`, e.Message.ID); err != nil {
t.Fatal(err)
}
fresh, _ := liveWarmupService(t)
fresh.ArrivalOutbox = repository.NewDurableEmailMessageMapRepository(d, cipherService)
fresh.InitEvents()
if err := fresh.ArrivalOutbox.DeliverArrivals(ctx, fresh.deliverArrivalEvent); err != nil {
t.Fatal(err)
}
if err := apply(fresh); err != nil {
t.Fatal(err)
}
message, err := fresh.UniboxRepository.GetForSync(ctx, e.UserID, e.Message.EmailID, e.Message.ID)
if err != nil {
t.Fatal(err)
}
if kind != "folder" && !message.Seen {
t.Fatal("provider Seen update lost after original-ID arrival replay")
}
if kind != "seen" && message.Folder != models.FolderArchive {
t.Fatalf("provider folder update lost: %q", message.Folder)
}
if kind == "full update" && (message.UID != 42 || message.Mailbox != 7 || message.FolderPath != "Archive") {
t.Fatalf("provider handles lost: %+v", message)
}
if _, err := d.Exec(ctx, `DELETE FROM email_accounts WHERE id=$1`, e.Message.EmailID); err != nil {
t.Fatal(err)
}
if err := apply(fresh); err != nil {
t.Fatalf("deleted mailbox update retried forever: %v", err)
}
})
}
}
func TestLiveSyncTruthBackgroundReadsKeepUnreadAndStayOwnerScoped(t *testing.T) {
s, d := liveWarmupService(t)
f := newWarmupFixture(t, d)
ctx := t.Context()
e := f.arrival("<"+uuid.NewString()+"@test.local>", nil)
if err := s.UniboxRepository.CreateEntry(ctx, e.UserID, e.Message); err != nil {
t.Fatal(err)
}
assertUnread := func() {
t.Helper()
var seen bool
if err := d.QueryRow(ctx, `SELECT seen FROM unibox_emails WHERE id=$1`, e.Message.ID).Scan(&seen); err != nil || seen {
t.Fatalf("background work marked unread mail seen: %v %v", seen, err)
}
}
for _, owner := range []uuid.UUID{f.senderUser, uuid.New()} {
if _, err := s.UniboxRepository.GetForSync(ctx, owner, e.Message.EmailID, e.Message.ID); !errors.Is(err, repository.ErrEmailNotFound) {
t.Fatalf("unowned sync lookup returned a row: %v", err)
}
}
for _, mailbox := range []uuid.UUID{f.sender, uuid.New(), uuid.Nil} {
if _, err := s.UniboxRepository.GetForSync(ctx, e.UserID, mailbox, e.Message.ID); !errors.Is(err, repository.ErrEmailNotFound) {
t.Fatalf("wrong-mailbox sync lookup returned a row: %v", err)
}
}
assertUnread()
if err := s.HandleFlagsAdd(ctx, &models.JobEventFlags{UserID: e.UserID, EmailID: e.Message.EmailID, ID: e.Message.ID, Flags: []string{models.FlagFlagged}}); err != nil {
t.Fatal(err)
}
assertUnread()
for _, folder := range []string{models.FolderInbox, models.FolderArchive, models.FolderArchive} {
if err := s.HandleFolderUpdate(ctx, &models.JobEventFolderUpdate{UserID: e.UserID, EmailID: e.Message.EmailID, ID: e.Message.ID, Folder: folder}); err != nil {
t.Fatal(err)
}
assertUnread()
}
if err := s.HandleFlagsAdd(ctx, &models.JobEventFlags{UserID: e.UserID, EmailID: e.Message.EmailID, ID: e.Message.ID, Flags: []string{models.FlagSeen}}); err != nil {
t.Fatal(err)
}
message, err := s.UniboxRepository.GetForSync(ctx, e.UserID, e.Message.EmailID, e.Message.ID)
if err != nil || message == nil || !message.Seen {
t.Fatalf("real provider Seen flag did not mark read: %v", err)
}
if err := s.UniboxRepository.MarkSeen(ctx, e.UserID, e.Message.ID, false); err != nil {
t.Fatal(err)
}
assertUnread()
message, err = s.UniboxRepository.GetByID(ctx, e.UserID, e.Message.ID)
if err != nil || message == nil || !message.Seen {
t.Fatalf("explicit user view no longer marks read: %v", err)
}
}
func TestLiveSyncTruthLegacyArrivalHasNoPendingGuard(t *testing.T) {
s, d := liveWarmupService(t)
f := newWarmupFixture(t, d)
ctx := t.Context()
key, err := kms.NewLocal(make([]byte, 32))
if err != nil {
t.Fatal(err)
}
r := repository.NewDurableEmailMessageMapRepository(d, cipher.NewService(key, nil, encryptedkeys.NewPostgres(d)))
s.ArrivalOutbox = r
s.UniboxRepository = &failSyncArrivalCreate{UniboxRepository: s.UniboxRepository, fail: true}
e := f.arrival("<"+uuid.NewString()+"@test.local>", nil)
e.Message.Subject = "ordinary legacy customer question"
// Legacy workers admit only the map, then publish NEW_EMAIL separately.
if err := r.Add(ctx, repository.EmailMessageData{UserID: e.UserID.String(), EmailID: e.Message.EmailID.String(), ID: e.Message.ID.String(), MessageID: e.Message.MessageID}); err != nil {
t.Fatal(err)
}
if err := s.HandleNewEmail(ctx, e); err == nil {
t.Fatal("expected legacy arrival creation failure")
}
if pending, err := r.HasPendingArrival(ctx, e.UserID, e.Message.EmailID, e.Message.ID); err != nil || pending {
t.Fatalf("legacy map unexpectedly acquired a durable arrival marker: %v %v", pending, err)
}
if err := s.HandleFlagsAdd(ctx, &models.JobEventFlags{UserID: e.UserID, EmailID: e.Message.EmailID, ID: e.Message.ID, Flags: []string{models.FlagSeen}}); err != nil {
t.Fatalf("legacy map alone inferred a pending arrival: %v", err)
}
if err := s.HandleFolderUpdate(ctx, &models.JobEventFolderUpdate{UserID: e.UserID, EmailID: e.Message.EmailID, ID: e.Message.ID, Folder: models.FolderArchive}); err != nil {
t.Fatalf("legacy map alone inferred a pending arrival: %v", err)
}
if err := s.HandleNewEmail(ctx, e); err != nil {
t.Fatal(err)
}
message, err := s.UniboxRepository.GetForSync(ctx, e.UserID, e.Message.EmailID, e.Message.ID)
if err != nil {
t.Fatal(err)
}
if message.Seen || message.Folder != models.FolderInbox {
t.Fatalf("legacy update ordering must not be reported as protected: %+v", message)
}
}
internal/app/consumer/sync_truth_test.go
+253
View File
@@ -0,0 +1,253 @@
package jobs
import (
"context"
"errors"
"testing"
"time"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/infrastructure/pubsub"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/repository"
)
func TestSyncTruthProgressRelaysDoNotResolveTransportWarnings(t *testing.T) {
previous := time.Now().Add(-time.Hour)
fresh := previous.Add(time.Minute)
for _, tc := range []struct {
name string
stamp *time.Time
deferred int
recovery *models.GoogleHistoryRecovery
resolved int
}{
{"heartbeat", &previous, 0, nil, 0},
{"older redelivery", nil, 0, nil, 0},
{"deferred", &fresh, 5, nil, 0},
{"recovering", &fresh, 0, &models.GoogleHistoryRecovery{HistoryID: 100}, 0},
{"successful catch-up", &fresh, 0, nil, 1},
} {
t.Run(tc.name, func(t *testing.T) {
errRepo := &stubErrorRepo{}
repo := &stubSyncStateRepo{saved: &models.SyncState{LastSyncedAt: &previous}}
s := &JobsService{EmailSyncStateRepository: repo, EmailAccountErrorRepository: errRepo}
if err := s.HandleSyncState(t.Context(), &models.JobEventSyncState{UserID: uuid.New(), EmailID: uuid.New(), State: models.SyncState{BackfillStatus: models.SyncBackfillComplete, LastSyncedAt: tc.stamp, Deferred: tc.deferred, BackfillCursor: models.SyncCursor{GoogleRecovery: tc.recovery}}}); err != nil {
t.Fatal(err)
}
if repo.put == nil || errRepo.calls != tc.resolved {
t.Fatalf("state publication/recovery outcome: saved=%v resolved=%d", repo.put != nil, errRepo.calls)
}
})
}
}
type removeSyncInbox struct {
repository.UniboxRepository
deletes int
err error
}
func (r *removeSyncInbox) Delete(context.Context, uuid.UUID, uuid.UUID) error {
r.deletes++
return r.err
}
type removeSyncBus struct{ deleted int }
type removeSyncWarmup struct {
repository.WarmupRepository
lookups int
}
func (r *removeSyncWarmup) GetWarmupReceived(_ context.Context, mailbox, id uuid.UUID) (*repository.WarmupReceived, error) {
r.lookups++
return &repository.WarmupReceived{EmailAccountID: mailbox, InternalID: id, MessageID: "<warmup@fake.test>", CreatedAt: time.Now()}, nil
}
func (b *removeSyncBus) Publish(_ context.Context, _ string, data any, _ map[string]string) error {
if event, ok := data.(*pubsub.EmailInboxEvent); ok && event.EventType == pubsub.EventEmailDeleted {
b.deleted++
}
return nil
}
func TestSyncTruthRemovalWaitsForScopedPendingArrivals(t *testing.T) {
user, mailbox, id := uuid.New(), uuid.New(), uuid.New()
lookupErr := errors.New("pending marker unavailable")
for _, tc := range []struct {
name string
owner, account, message uuid.UUID
pending bool
err, want error
}{
{"pending", user, mailbox, id, true, nil, ErrSyncArrivalPending},
{"marker error", user, mailbox, id, false, lookupErr, lookupErr},
{"finished or legacy", user, mailbox, id, false, nil, nil},
{"other user", uuid.New(), mailbox, id, true, nil, nil},
{"other mailbox", user, uuid.New(), id, true, nil, nil},
{"other message", user, mailbox, uuid.New(), true, nil, nil},
} {
t.Run(tc.name, func(t *testing.T) {
inbox, bus := &removeSyncInbox{}, &removeSyncBus{}
s := &JobsService{UniboxRepository: inbox, ArrivalOutbox: &syncTruthOutbox{user: user, mailbox: mailbox, id: id, pending: tc.pending, err: tc.err}, EmailRepository: newEmailAccountRepo{}, StreamingPublisher: pubsub.NewStreamingPublisher(bus)}
err := s.HandleRemoveEmail(t.Context(), &models.JobEventRemoveEmail{UserID: tc.owner, EmailID: tc.account, ID: tc.message})
if !errors.Is(err, tc.want) {
t.Fatalf("removal error=%v want=%v", err, tc.want)
}
want := 1
if tc.want != nil {
want = 0
}
if inbox.deletes != want || bus.deleted != want {
t.Fatalf("deletes=%d notifications=%d want=%d", inbox.deletes, bus.deleted, want)
}
})
}
}
func TestSyncTruthRemovalDeleteFailureRetriesBeforeNotification(t *testing.T) {
failure := errors.New("unibox delete unavailable")
inbox, bus := &removeSyncInbox{err: failure}, &removeSyncBus{}
warmup := &removeSyncWarmup{}
s := &JobsService{UniboxRepository: inbox, WarmupRepo: warmup, EmailRepository: newEmailAccountRepo{}, StreamingPublisher: pubsub.NewStreamingPublisher(bus)}
e := &models.JobEventRemoveEmail{UserID: uuid.New(), EmailID: uuid.New(), ID: uuid.New()}
if err := s.HandleRemoveEmail(t.Context(), e); !errors.Is(err, failure) {
t.Fatalf("delete failure acknowledged: %v", err)
}
if bus.deleted != 0 {
t.Fatal("failed deletion published success")
}
inbox.err = nil
if err := s.HandleRemoveEmail(t.Context(), e); err != nil {
t.Fatal(err)
}
if inbox.deletes != 2 || bus.deleted != 1 || warmup.lookups != 2 {
t.Fatalf("deletes=%d notifications=%d", inbox.deletes, bus.deleted)
}
}
type syncTruthLookup struct {
repository.UniboxRepository
message *models.EmailMessageStoreData
reads int
lookups int
appearOn int
}
func (r *syncTruthLookup) GetByID(context.Context, uuid.UUID, uuid.UUID) (*models.EmailMessageStoreData, error) {
r.reads++
if r.message == nil {
return nil, repository.ErrEmailNotFound
}
r.message.Seen = true
return r.message, nil
}
func (r *syncTruthLookup) GetForSync(context.Context, uuid.UUID, uuid.UUID, uuid.UUID) (*models.EmailMessageStoreData, error) {
r.lookups++
if r.message == nil || (r.appearOn > 0 && r.lookups < r.appearOn) {
return nil, repository.ErrEmailNotFound
}
return r.message, nil
}
func (*syncTruthLookup) UpdatePendingEmail(context.Context, uuid.UUID, uuid.UUID, func(*models.EmailMessageStoreData)) (bool, error) {
return false, nil
}
func (r *syncTruthLookup) UpdateEntry(_ context.Context, _, _, _ uuid.UUID, update *repository.UpdateUniboxEntry) error {
if update.Seen != nil {
r.message.Seen = *update.Seen
}
return nil
}
type syncTruthOutbox struct {
repository.ArrivalOutbox
user, mailbox, id uuid.UUID
pending bool
err error
}
func (r *syncTruthOutbox) HasPendingArrival(_ context.Context, user, mailbox, id uuid.UUID) (bool, error) {
if user != r.user || mailbox != r.mailbox || id != r.id {
return false, nil
}
return r.pending, r.err
}
func TestSyncTruthBackgroundUpdatesNeverViewUnreadMail(t *testing.T) {
for _, kind := range []string{"flagged", "folder", "same folder"} {
t.Run(kind, func(t *testing.T) {
r := &syncTruthLookup{message: &models.EmailMessageStoreData{Seen: false, Folder: models.FolderInbox, ProviderFolder: models.FolderInbox}}
s := &JobsService{UniboxRepository: r, EmailRepository: warmupInboxEmailRepo{}}
user, mailbox, id := uuid.New(), uuid.New(), uuid.New()
var err error
if kind == "flagged" {
err = s.HandleFlagsAdd(t.Context(), &models.JobEventFlags{UserID: user, EmailID: mailbox, ID: id, Flags: []string{models.FlagFlagged}})
} else {
folder := models.FolderInbox
if kind == "folder" {
folder = models.FolderArchive
}
err = s.HandleFolderUpdate(t.Context(), &models.JobEventFolderUpdate{UserID: user, EmailID: mailbox, ID: id, Folder: folder})
}
if err != nil || r.message.Seen || r.reads != 0 {
t.Fatalf("background update treated as view: seen=%v reads=%d err=%v", r.message.Seen, r.reads, err)
}
})
}
}
func TestSyncTruthPendingArrivalUpdatesRetryOnlyForOwnedUndeliveredMail(t *testing.T) {
user, mailbox, id := uuid.New(), uuid.New(), uuid.New()
for _, kind := range []string{"flags add", "flags remove", "folder", "full update"} {
t.Run(kind, func(t *testing.T) {
r := &syncTruthLookup{}
outbox := &syncTruthOutbox{user: user, mailbox: mailbox, id: id, pending: true}
s := &JobsService{UniboxRepository: r, ArrivalOutbox: outbox}
apply := func(owner, box uuid.UUID) error {
switch kind {
case "flags add":
return s.HandleFlagsAdd(t.Context(), &models.JobEventFlags{UserID: owner, EmailID: box, ID: id, Flags: []string{models.FlagSeen}})
case "flags remove":
return s.HandleFlagsRemove(t.Context(), &models.JobEventFlags{UserID: owner, EmailID: box, ID: id, Flags: []string{models.FlagSeen}})
case "folder":
return s.HandleFolderUpdate(t.Context(), &models.JobEventFolderUpdate{UserID: owner, EmailID: box, ID: id, Folder: models.FolderArchive})
default:
return s.HandleUpdateEmail(t.Context(), &models.JobEventEmailUpdate{UserID: owner, EmailID: box, ID: id, Flags: []string{models.FlagSeen}})
}
}
if err := apply(user, mailbox); !errors.Is(err, ErrSyncArrivalPending) {
t.Fatalf("pending arrival was acknowledged: %v", err)
}
if err := apply(uuid.New(), mailbox); err != nil {
t.Fatalf("unowned update retried: %v", err)
}
if err := apply(user, uuid.New()); err != nil {
t.Fatalf("other-mailbox update retried: %v", err)
}
outbox.pending = false
if err := apply(user, mailbox); err != nil {
t.Fatalf("deliberately hidden/deleted mail retried: %v", err)
}
outbox.err = errors.New("marker unavailable")
if err := apply(user, mailbox); !errors.Is(err, outbox.err) {
t.Fatalf("marker failure swallowed: %v", err)
}
})
}
}
func TestSyncTruthArrivalFinishingDuringLookupStillGetsUpdate(t *testing.T) {
user, mailbox, id := uuid.New(), uuid.New(), uuid.New()
r := &syncTruthLookup{message: &models.EmailMessageStoreData{EmailID: mailbox}, appearOn: 3}
s := &JobsService{UniboxRepository: r, EmailRepository: warmupInboxEmailRepo{}, ArrivalOutbox: &syncTruthOutbox{user: user, mailbox: mailbox, id: id}}
if err := s.HandleFlagsAdd(t.Context(), &models.JobEventFlags{UserID: user, EmailID: mailbox, ID: id, Flags: []string{models.FlagSeen}}); err != nil {
t.Fatal(err)
}
if !r.message.Seen {
t.Fatal("arrival committed between reads lost the later Seen update")
}
}
internal/app/consumer/unibox_seen_sync_test.go
+4
View File
@@ -21,6 +21,10 @@ func (r *seenSyncRepo) GetByID(context.Context, uuid.UUID, uuid.UUID) (*models.E
return r.stored, nil
}
func (r *seenSyncRepo) GetForSync(context.Context, uuid.UUID, uuid.UUID, uuid.UUID) (*models.EmailMessageStoreData, error) {
return r.stored, nil
}
func (r *seenSyncRepo) UpdateEntry(_ context.Context, _, _, _ uuid.UUID, e *repository.UpdateUniboxEntry) error {
r.updates = append(r.updates, *e)
return nil
internal/app/email/graph_checkpoint_test.go
+66
View File
@@ -0,0 +1,66 @@
package email
import (
"context"
"errors"
"testing"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/repository"
)
type graphCheckpointRepo struct {
links map[string]string
err error
}
type graphLoaderState struct {
repository.EmailSyncStateRepository
state *models.SyncState
err error
}
func (s *graphLoaderState) Get(context.Context, uuid.UUID) (*models.SyncState, error) {
return s.state, s.err
}
func (r *graphCheckpointRepo) Get(context.Context, uuid.UUID, uuid.UUID) (map[string]string, error) {
return r.links, r.err
}
func (r *graphCheckpointRepo) Put(context.Context, uuid.UUID, uuid.UUID, string, string) error {
return nil
}
func TestGraphCheckpointLoadFailureDoesNotPublishCursorlessWorker(t *testing.T) {
org, worker := uuid.New(), uuid.New()
acc := &models.Email{ID: uuid.New(), UserID: uuid.NewString(), Email: "box@outlook.test", Provider: string(models.InboxProviderOutlook), Status: "active", OrganizationID: &org, WorkerID: &worker}
repo := &stubReauthRepo{account: acc}
lookupErr := errors.New("checkpoint read failed")
checkpoints := &graphCheckpointRepo{links: map[string]string{"inbox": "saved-delta"}, err: lookupErr}
pub := &countingPublisher{}
s := &emailService{emailRepository: repo, workerAssignment: &countingAssignment{}, publisher: pub, graphDelta: checkpoints}
state := &graphLoaderState{state: &models.SyncState{BackfillStatus: models.SyncBackfillComplete}}
s.syncState = state
if err := s.LoadAccountOntoWorker(t.Context(), acc.ID); !errors.Is(err, lookupErr) || pub.added != 0 {
t.Fatalf("err=%v published=%d", err, pub.added)
}
checkpoints.err = nil
state.err = lookupErr
if err := s.LoadAccountOntoWorker(t.Context(), acc.ID); !errors.Is(err, lookupErr) || pub.added != 0 {
t.Fatalf("recovery state failure published unsafe worker: err=%v published=%d", err, pub.added)
}
state.err = nil
payload, err := s.buildAddWorkerEmail(t.Context(), acc)
if err != nil || payload.Graph.DeltaLinks["inbox"] != "saved-delta" || payload.Sync.State.BackfillStatus != models.SyncBackfillComplete {
t.Fatalf("reload did not recover saved checkpoint: %v %v", payload, err)
}
if err := s.LoadAccountOntoWorker(t.Context(), acc.ID); err != nil || pub.added != 1 {
t.Fatalf("retry err=%v published=%d", err, pub.added)
}
checkpoints.links = nil
payload, err = s.buildAddWorkerEmail(t.Context(), acc)
if err != nil || len(payload.Graph.DeltaLinks) != 0 {
t.Fatalf("genuinely missing checkpoint: %v %v", payload, err)
}
}
internal/app/email/loader.go
+19 -15
View File
@@ -19,7 +19,7 @@ import (
// WireGraphDelta attaches the Graph delta-cursor repository so the reconciler can
// seed a Graph mailbox's saved per-folder cursors when (re)loading it. Optional;
// when unset, Graph mailboxes prime from empty on load.
// when unset, Graph mailboxes recover from empty on load.
func (s *emailService) WireGraphDelta(repo repository.EmailGraphDeltaRepository) {
s.graphDelta = repo
}
@@ -352,6 +352,13 @@ func (s *emailService) buildAddWorkerEmail(ctx context.Context, acc *models.Emai
// Only SMTP/IMAP acts on this; Gmail and Graph file their own copy.
SaveToSent: &saveToSent,
}
if provider == models.InboxProviderOutlook {
links, err := s.deltaLinksFor(ctx, userID, acc.ID)
if err != nil {
return nil, err
}
out.Graph = &models.AddWorkerEmailGraphData{DeltaLinks: links}
}
// A mailbox under an administrator's grant has no stored credential
// either; the worker draws tokens the control plane mints per use.
@@ -368,7 +375,7 @@ func (s *emailService) buildAddWorkerEmail(ctx context.Context, acc *models.Emai
case models.InboxProviderGoogle:
out.Google = &models.AddWorkerEmailGoogleData{LastHistoryID: s.lastHistoryFor(ctx, userID, acc.ID, acc.LastID)}
case models.InboxProviderOutlook:
out.Graph = &models.AddWorkerEmailGraphData{DeltaLinks: s.deltaLinksFor(ctx, userID, acc.ID), User: d.Subject}
out.Graph.User = d.Subject
default:
return nil, nil
}
@@ -383,7 +390,7 @@ func (s *emailService) buildAddWorkerEmail(ctx context.Context, acc *models.Emai
case models.InboxProviderGoogle:
out.Google = &models.AddWorkerEmailGoogleData{LastHistoryID: s.lastHistoryFor(ctx, userID, acc.ID, acc.LastID)}
case models.InboxProviderOutlook:
out.Graph = &models.AddWorkerEmailGraphData{DeltaLinks: s.deltaLinksFor(ctx, userID, acc.ID)}
// The checkpoint was loaded before selecting the token owner.
default:
return nil, nil
}
@@ -406,10 +413,7 @@ func (s *emailService) buildAddWorkerEmail(ctx context.Context, acc *models.Emai
if cerr != nil {
return nil, cerr
}
out.Graph = &models.AddWorkerEmailGraphData{
Token: oauthToken(creds),
DeltaLinks: s.deltaLinksFor(ctx, userID, acc.ID),
}
out.Graph.Token = oauthToken(creds)
case models.InboxProviderSMTPIMAP:
creds, cerr := s.emailRepository.GetSMTPCredentials(ctx, acc.ID)
if cerr != nil {
@@ -498,11 +502,11 @@ func (s *emailService) syncDataFor(ctx context.Context, emailID uuid.UUID) (*mod
}
}
if s.syncState != nil {
if saved, err := s.syncState.Get(ctx, emailID); err == nil {
data.State = saved
} else {
log.Warn().Err(err).Str("email_id", emailID.String()).Msg("sync state lookup failed; worker starts fresh")
saved, err := s.syncState.Get(ctx, emailID)
if err != nil {
return nil, fmt.Errorf("sync state lookup: %w", err)
}
data.State = saved
}
return data, nil
}
@@ -522,15 +526,15 @@ func (s *emailService) mailboxesFor(ctx context.Context, userID, emailID uuid.UU
return saved
}
func (s *emailService) deltaLinksFor(ctx context.Context, userID, emailID uuid.UUID) map[string]string {
func (s *emailService) deltaLinksFor(ctx context.Context, userID, emailID uuid.UUID) (map[string]string, error) {
if s.graphDelta == nil {
return nil
return nil, nil
}
links, err := s.graphDelta.Get(ctx, userID, emailID)
if err != nil {
return nil
return nil, fmt.Errorf("graph checkpoint lookup: %w", err)
}
return links
return links, nil
}
func oauthToken(c *repository.OAuthCredentials) *oauth2.Token {
internal/app/orgtransfer/spec.go
+3 -1
View File
@@ -745,7 +745,8 @@ var Tables = []Table{
},
{
Name: "email_message_map", Group: models.OrgDataGroupInbox,
Scope: `email_id IN ` + orgMailboxes,
Scope: `email_id IN ` + orgMailboxes + ` AND NOT EXISTS (SELECT 1 FROM sync_arrival_outbox o WHERE o.user_id=email_message_map.user_id AND o.email_id=email_message_map.email_id AND o.message_id=email_message_map.message_id AND o.stage=0)`,
Owner: `email_id IN ` + orgMailboxes,
Note: "Maps provider message ids to internal ones, so replies still thread after the move.",
},
{
@@ -994,6 +995,7 @@ var Tables = []Table{
// with the reason. Kept as data so the docs page and the coverage test both
// read from one list instead of restating it.
var ExcludedTables = map[string]string{
"sync_arrival_outbox": "Encrypted, org-key-owned arrival delivery queue bound to this instance's provider handles and body storage. Pending maps are excluded too so destination sync retries them; no queue secrets or blob references travel. Full instance backups retain queue, org keys and bodies together.",
"send_result_effects": "Source-instance reconciliation delivery queue. Pending webhook and notification effects must drain on the source; imports must not replay customer notifications. Full instance backups retain the durable outbox.",
"warmup_received": "Raw receiving-mailbox observations and provider thread handles are source-instance evidence, not portable send authority. Aggregated warmup placement history travels; imported threads cannot continue without new verified observations.",
"diagnostic_auth_verifications": "Bounded DKIM observations and grants bind source-instance workers, provider messages and cross-workspace diagnostic parents. They cannot authorize imported mail; destination verification starts unknown. Full instance database backups retain them.",
internal/app/orgtransfer/sync_arrival_scope_live_test.go
+63
View File
@@ -0,0 +1,63 @@
package orgtransfer
import (
"testing"
"github.com/google/uuid"
)
func TestLiveSyncArrivalQueueAndPendingMapsStayInstanceLocal(t *testing.T) {
pool := specLivePool(t)
f := newTenantFixture(t, pool)
ctx := t.Context()
mailbox := uuid.New()
if _, err := pool.Exec(ctx, `INSERT INTO email_accounts(id,user_id,organization_id,email,name,signature_plain,signature_html,provider) VALUES($1,$2,$3,$4,'Arrival','','','smtp_imap')`, mailbox, f.owner, f.org, mailbox.String()+"@test.local"); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM email_accounts WHERE id=$1`, mailbox) })
for i, key := range []string{"legacy", "pending", "handled"} {
id := uuid.New()
if _, err := pool.Exec(ctx, `INSERT INTO email_message_map(user_id,email_id,message_id,id) VALUES($1,$2,$3,$4)`, f.owner, mailbox, key, id); err != nil {
t.Fatal(err)
}
if i > 0 {
if _, err := pool.Exec(ctx, `INSERT INTO sync_arrival_outbox(user_id,email_id,organization_id,message_id,id,payload,stage) VALUES($1,$2,$3,$4,$5,'fixture',$6)`, f.owner, mailbox, f.org, key, id, i-1); err != nil {
t.Fatal(err)
}
}
}
if ExcludedTables["sync_arrival_outbox"] == "" {
t.Fatal("queue is not explicitly excluded")
}
var scope string
for _, spec := range Tables {
if spec.Name == "sync_arrival_outbox" {
t.Fatal("queue secrets or body references transferred")
}
if spec.Name == "email_message_map" {
scope = spec.Scope
}
}
if scope == "" {
t.Fatal("map scope missing")
}
rows, err := pool.Query(ctx, `SELECT message_id FROM email_message_map WHERE `+scope+` ORDER BY message_id`, f.org)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
var keys []string
for rows.Next() {
var key string
if err := rows.Scan(&key); err != nil {
t.Fatal(err)
}
keys = append(keys, key)
}
if err := rows.Err(); err != nil {
t.Fatal(err)
}
if len(keys) != 2 || keys[0] != "handled" || keys[1] != "legacy" {
t.Fatalf("pending maps suppressed destination resync: %v", keys)
}
}
internal/app/worker/event_add_email.go
+24 -14
View File
@@ -18,6 +18,7 @@ import (
type mailboxLoad struct {
removed atomic.Bool
done chan struct{}
prev *mailboxLoad
}
// HandleAddEmail loads a mailbox off the bus loop. Loading dials the mail
@@ -28,13 +29,13 @@ func (w *WorkerService) HandleAddEmail(ctx context.Context, e *models.AddWorkerE
return nil
}
if w.mailManager.Has(e.ID) {
if _, loading := w.loads.Load(e.ID); !loading {
// Already loaded: keep the handler idempotent, but take a changed sync
// budget so an operator's settings change reaches this mailbox.
if mail := w.mailManager.Get(e.ID); mail != nil {
if mail := w.mailManager.Get(e.ID); mail != nil && mail.SameExecution(e) {
mail.ApplySyncPolicy(e.Sync)
return nil
}
return nil
}
load := &mailboxLoad{done: make(chan struct{})}
@@ -45,10 +46,8 @@ func (w *WorkerService) HandleAddEmail(ctx context.Context, e *models.AddWorkerE
break
}
old := v.(*mailboxLoad)
if !old.removed.Load() {
return nil
}
// A removal is pending on the load in flight: this add runs after it rather than being dropped.
// Queue after in-flight loads; AddWMail keeps unchanged adds idempotent.
load.prev = old
if w.loads.CompareAndSwap(e.ID, old, load) {
prev = old
break
@@ -73,10 +72,18 @@ func (w *WorkerService) HandleAddEmail(ctx context.Context, e *models.AddWorkerE
// loadedMailbox returns a mailbox, waiting on a load of it still in flight so a
// command queued behind its ADD_EMAIL finds it as it did when loads were inline.
func (w *WorkerService) loadedMailbox(ctx context.Context, id uuid.UUID) (*wmail.WMail, bool) {
if v, ok := w.loads.Load(id); ok {
for {
v, ok := w.loads.Load(id)
if !ok {
break
}
select {
case <-v.(*mailboxLoad).done:
case <-ctx.Done():
return nil, false
}
if current, exists := w.loads.Load(id); !exists || current == v {
break
}
}
mail := w.mailManager.Get(id)
@@ -85,6 +92,10 @@ func (w *WorkerService) loadedMailbox(ctx context.Context, id uuid.UUID) (*wmail
// loadMailbox connects one mailbox and starts its sync, or reports why it could not.
func (w *WorkerService) loadMailbox(ctx context.Context, e *models.AddWorkerEmail, load *mailboxLoad) {
if load.removed.Load() {
return
}
previous := w.mailManager.Get(e.ID)
if err := w.mailManager.AddWMail(ctx, e); err != nil {
log.Error().Err(err).Str("email_id", e.ID.String()).Msg("failed to add email account to worker")
w.reportLoadFailure(e, err)
@@ -95,8 +106,10 @@ func (w *WorkerService) loadMailbox(ctx context.Context, e *models.AddWorkerEmai
return
}
if load.removed.Load() {
mail.Discard()
w.mailManager.Terminate(e.ID)
w.mailManager.TerminateMailbox(e.ID, mail)
return
}
if previous == mail {
return
}
@@ -146,8 +159,5 @@ func (w *WorkerService) reportLoadFailure(e *models.AddWorkerEmail, err error) {
// dropMailbox removes a loaded mailbox and stops its work.
func (w *WorkerService) dropMailbox(id uuid.UUID, mail *wmail.WMail) {
if mail.Cancel != nil {
mail.Cancel()
}
w.mailManager.Terminate(id)
w.mailManager.TerminateMailbox(id, mail)
}
internal/app/worker/event_mailbox_identity.go
+7
View File
@@ -58,6 +58,13 @@ func (w *WorkerService) HandleMailboxIdentity(ctx context.Context, data models.E
if mail.GoogleData == nil || mail.GoogleData.Client == nil {
return reply(models.MailboxIdentityResult{Error: "provider has no send-as list"})
}
done, ok := mail.BeginExecution()
if !ok {
return reply(models.MailboxIdentityResult{Error: "mailbox is not loaded on this worker"})
}
defer done()
ctx, cancel = mail.ExecutionContext(ctx)
defer cancel()
rows, err := mail.GoogleData.Client.ListSendAs(ctx)
if err != nil {
internal/app/worker/event_message_folder.go
+7
View File
@@ -29,6 +29,13 @@ func (w *WorkerService) HandleMessageFolder(ctx context.Context, action models.M
log.Warn().Str("email_id", action.EmailID.String()).Msg("Mailbox not loaded; filing not relayed to the provider")
return nil
}
done, ok := mail.BeginExecution()
if !ok {
return nil
}
defer done()
ctx, cancel := mail.ExecutionContext(ctx)
defer cancel()
log.Info().
Str("email_id", action.EmailID.String()).
internal/app/worker/event_message_seen.go
+7
View File
@@ -28,6 +28,13 @@ func (w *WorkerService) HandleMessageSeen(ctx context.Context, action models.Mes
log.Warn().Str("email_id", action.EmailID.String()).Msg("Mailbox not loaded; read state not relayed to the provider")
return nil
}
done, ok := mail.BeginExecution()
if !ok {
return nil
}
defer done()
ctx, cancel := mail.ExecutionContext(ctx)
defer cancel()
log.Info().
Str("email_id", action.EmailID.String()).
internal/app/worker/event_remove_email.go
+3 -1
View File
@@ -22,7 +22,9 @@ func (w *WorkerService) HandleRemoveEmail(ctx context.Context, e *models.RemoveW
// A load still dialing drops the mailbox when it finishes.
if v, ok := w.loads.Load(id); ok {
v.(*mailboxLoad).removed.Store(true)
for load := v.(*mailboxLoad); load != nil; load = load.prev {
load.removed.Store(true)
}
}
mail := w.mailManager.Get(id)
internal/app/worker/event_warmup_action.go
+10
View File
@@ -52,6 +52,16 @@ func (w *WorkerService) HandleWarmupAction(ctx context.Context, action models.Wa
}
mail, exists := w.loadedMailbox(ctx, action.EmailID)
if exists {
if done, ok := mail.BeginExecution(); ok {
defer done()
var cancel func()
ctx, cancel = mail.ExecutionContext(ctx)
defer cancel()
} else {
exists = false
}
}
var err error
switch {
case !exists:
internal/app/worker/mailbox_lifecycle_test.go
+87
View File
@@ -0,0 +1,87 @@
package worker
import (
"context"
"sync/atomic"
"testing"
"time"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/app/worker/wmail"
"github.com/warmbly/warmbly/internal/models"
)
type closedWorkerIMAP struct {
wmail.ImapConn
closes atomic.Int32
}
func (c *closedWorkerIMAP) Close() error { c.closes.Add(1); return nil }
func TestAddEmailReplacesChangedCredentialsAndRemoveClosesInstalledSession(t *testing.T) {
w := newLoadingWorker(&capturedEvents{})
id := uuid.New()
d := &models.AddWorkerEmail{ID: id, UserID: uuid.New(), Type: models.InboxProviderSMTPIMAP, Email: "a@example.test", SmtpImap: &models.AddWorkerEmailSmtpImapData{Credentials: &models.SmtpImap{SMTP: &models.Service{Host: "smtp.example.test", Port: 587, Password: "old"}, IMAP: &models.Service{Host: "imap.example.test", Port: 993, Password: "old"}}}, Sync: &models.AddWorkerEmailSyncData{State: &models.SyncState{BackfillStatus: models.SyncBackfillComplete, BackfillSynced: 12}}}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
if err := w.HandleAddEmail(ctx, d); err != nil {
t.Fatal(err)
}
old, ok := w.loadedMailbox(ctx, id)
if !ok {
t.Fatal("initial load failed")
}
conn := &closedWorkerIMAP{}
old.SmtpImapData.ImapClient = conn
for i := 0; i < 3; i++ {
if err := w.HandleAddEmail(ctx, d); err != nil {
t.Fatal(err)
}
}
copyData := *d
copyData.Sync = &models.AddWorkerEmailSyncData{Policy: models.SyncPolicy{DailyMessages: 88}}
if err := w.HandleAddEmail(ctx, &copyData); err != nil {
t.Fatal(err)
}
if got, _ := w.loadedMailbox(ctx, id); got != old || conn.closes.Load() != 0 {
t.Fatal("policy/duplicate ADD_EMAIL retired healthy client")
}
s := *d.SmtpImap
creds := *s.Credentials
smtp := *creds.SMTP
smtp.Password = "new"
creds.SMTP = &smtp
s.Credentials = &creds
copyData.SmtpImap = &s
if err := w.HandleAddEmail(ctx, &copyData); err != nil {
t.Fatal(err)
}
current, ok := w.loadedMailbox(ctx, id)
if !ok || current == old || current.SmtpImapData.SmtpClient.Credentials.Password != "new" || conn.closes.Load() != 1 {
t.Fatal("still-loaded credentials were not replaced and retired")
}
currentConn := &closedWorkerIMAP{}
current.SmtpImapData.ImapClient = currentConn
old.Terminate()
if w.mailManager.Get(id) != current || currentConn.closes.Load() != 0 {
t.Fatal("stale old termination closed replacement")
}
rem := &models.RemoveWorkerEmail{EmailID: id.String()}
for i := 0; i < 2; i++ {
if err := w.HandleRemoveEmail(ctx, rem); err != nil {
t.Fatal(err)
}
}
if w.mailManager.Get(id) != nil || currentConn.closes.Load() != 1 {
t.Fatal("installed removal did not close exactly once")
}
if err := w.HandleAddEmail(ctx, &copyData); err != nil {
t.Fatal(err)
}
if mail, ok := w.loadedMailbox(ctx, id); !ok || mail == current {
t.Fatal("mailbox could not reload after removal")
}
if err := w.HandleRemoveEmail(ctx, rem); err != nil {
t.Fatal(err)
}
}
internal/app/worker/mailmanager/add_email.go
+18 -3
View File
@@ -15,6 +15,18 @@ func (m *MailManager) AddWMail(
ctx context.Context,
data *models.AddWorkerEmail,
) error {
dataCopy := *data
data = &dataCopy
old := m.Get(data.ID)
if old != nil {
if old.SameExecution(data) {
old.ApplySyncPolicy(data.Sync)
return nil
}
old.Stop()
old.Drain()
old.ResumeInto(data)
}
// Cfg is avro-excluded from the payload, so rebuild it from the worker's
// local oauth config for token refresh (no-op for smtp_imap).
data.Cfg = m.cfgFor(data.Type)
@@ -25,11 +37,12 @@ func (m *MailManager) AddWMail(
data.TokenSource = m.tokenBroker.Source(data.ID)
}
var newMail *wmail.WMail
newMail, err := wmail.NewWMail(
data,
m.OnEvent,
func() {
m.Terminate(data.ID)
m.TerminateMailbox(data.ID, newMail)
},
m.cache,
m.storage,
@@ -47,13 +60,15 @@ func (m *MailManager) AddWMail(
// Built before the lock: NewWMail dials the server, and holding the lock
// through that stalled every send and lookup on this worker.
m.Lock()
defer m.Unlock()
if _, loaded := m.Emails[data.ID]; loaded {
if current := m.Emails[data.ID]; current != old {
m.Unlock()
newMail.Discard()
return nil
}
newMail.PreservePending(old)
m.Emails[data.ID] = newMail
newMail.ExecutorID = m.ExecutorID
m.Unlock()
return nil
}
internal/app/worker/mailmanager/lifecycle_test.go
+172
View File
@@ -0,0 +1,172 @@
package mailmanager
import (
"context"
"sync/atomic"
"testing"
"time"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/app/worker/wmail"
"github.com/warmbly/warmbly/internal/models"
"golang.org/x/oauth2"
)
type countedClient struct {
wmail.ImapConn
closes atomic.Int32
}
func (c *countedClient) Close() error { c.closes.Add(1); return nil }
func smtpPayload() *models.AddWorkerEmail {
return &models.AddWorkerEmail{ID: uuid.New(), UserID: uuid.New(), Type: models.InboxProviderSMTPIMAP, Email: "a@example.test", SmtpImap: &models.AddWorkerEmailSmtpImapData{Credentials: &models.SmtpImap{SMTP: &models.Service{Host: "smtp.example.test", Port: 587, Password: "old"}, IMAP: &models.Service{Host: "imap.example.test", Port: 993, Password: "old"}}, Mailboxes: []models.Mailbox{{Name: "Inbox", UIDValidity: 5, UIDNext: 25}}}, Sync: &models.AddWorkerEmailSyncData{State: &models.SyncState{BackfillStatus: models.SyncBackfillComplete, BackfillSynced: 7}}}
}
func TestMailboxReplacementDrainsClosesAndPreservesCursors(t *testing.T) {
m := NewMailManager(func(models.JobEventType, string, any) error { return nil }, nil, nil, nil, nil, nil, nil)
d := smtpPayload()
if err := m.AddWMail(t.Context(), d); err != nil {
t.Fatal(err)
}
old := m.Get(d.ID)
c := &countedClient{}
old.SmtpImapData.ImapClient = c
for i := 0; i < 3; i++ {
if err := m.AddWMail(t.Context(), d); err != nil {
t.Fatal(err)
}
}
if m.Get(d.ID) != old || c.closes.Load() != 0 {
t.Fatal("unchanged add restarted client")
}
copyData := *d
data := &copyData
s := *d.SmtpImap
creds := *s.Credentials
smtp := *creds.SMTP
smtp.Password = "replacement"
creds.SMTP = &smtp
s.Credentials = &creds
data.SmtpImap = &s
done, ok := old.BeginExecution()
if !ok {
t.Fatal("live executor not admitted")
}
result := make(chan error, 1)
go func() { result <- m.AddWMail(context.Background(), data) }()
select {
case <-old.Ctx.Done():
case <-time.After(time.Second):
t.Fatal("replacement did not cancel old executor")
}
if m.Get(d.ID) != old {
t.Fatal("new executor installed before old operation drained")
}
done()
select {
case err := <-result:
if err != nil {
t.Fatal(err)
}
case <-time.After(time.Second):
t.Fatal("replacement stuck")
}
next := m.Get(d.ID)
if next == nil || next == old || next.SmtpImapData.SmtpClient.Credentials.Password != "replacement" || c.closes.Load() != 1 {
t.Fatal("replacement credentials/cleanup not installed")
}
if len(next.SmtpImapData.Mailboxes) != 1 || next.SmtpImapData.Mailboxes[0].UIDNext != 25 {
t.Fatal("disabled sync lost saved cursor")
}
old.Terminate()
if m.Get(d.ID) != next {
t.Fatal("old auth callback retired replacement")
}
m.Terminate(d.ID)
m.Terminate(d.ID)
if next.Ctx.Err() == nil || c.closes.Load() != 1 {
t.Fatal("removal did not stop session idempotently")
}
}
func TestMailboxReplacementLocalOAuthAndPolicy(t *testing.T) {
for _, provider := range []models.InboxProvider{models.InboxProviderGoogle, models.InboxProviderOutlook} {
t.Run(string(provider), func(t *testing.T) {
m := NewMailManager(func(models.JobEventType, string, any) error { return nil }, nil, nil, nil, nil, nil, nil)
d := &models.AddWorkerEmail{ID: uuid.New(), UserID: uuid.New(), Type: provider, Google: &models.AddWorkerEmailGoogleData{Token: &oauth2.Token{AccessToken: "old", RefreshToken: "old-refresh"}, LastHistoryID: 19}, Graph: &models.AddWorkerEmailGraphData{Token: &oauth2.Token{AccessToken: "old", RefreshToken: "old-refresh"}, DeltaLinks: map[string]string{"Inbox": "live"}}}
if err := m.AddWMail(t.Context(), d); err != nil {
t.Fatal(err)
}
old := m.Get(d.ID)
copyData := *d
copyData.Sync = &models.AddWorkerEmailSyncData{Policy: models.SyncPolicy{DailyMessages: 90}}
if err := m.AddWMail(t.Context(), &copyData); err != nil {
t.Fatal(err)
}
if m.Get(d.ID) != old {
t.Fatal("budget change rebuilt provider")
}
g := *d.Google
g.Token = &oauth2.Token{AccessToken: "new", RefreshToken: "new-refresh"}
copyData.Google = &g
graph := *d.Graph
graph.Token = g.Token
copyData.Graph = &graph
if err := m.AddWMail(t.Context(), &copyData); err != nil {
t.Fatal(err)
}
next := m.Get(d.ID)
if next == old || old.Ctx.Err() == nil || !next.SameExecution(&copyData) {
t.Fatal("reauthorization kept local OAuth client")
}
if next.GoogleData != nil && next.GoogleData.LastHistoryID != 19 {
t.Fatal("history cursor lost")
}
if next.GraphData != nil && next.GraphData.Client.DeltaLinks["Inbox"] != "live" {
t.Fatal("delta cursor lost")
}
m.Terminate(d.ID)
if err := m.AddWMail(t.Context(), &copyData); err != nil {
t.Fatal(err)
}
if m.Get(d.ID) == nil || m.Get(d.ID) == next {
t.Fatal("reload after removal failed")
}
m.Terminate(d.ID)
})
}
}
func TestMailboxReplacementEndpointSkipAndSentSettings(t *testing.T) {
for _, field := range []string{"imap-password", "smtp-host", "smtp-security", "skip-folders", "sent-copy"} {
t.Run(field, func(t *testing.T) {
m := NewMailManager(func(models.JobEventType, string, any) error { return nil }, nil, nil, nil, nil, nil, nil)
d := smtpPayload()
if err := m.AddWMail(t.Context(), d); err != nil {
t.Fatal(err)
}
old := m.Get(d.ID)
switch field {
case "imap-password":
d.SmtpImap.Credentials.IMAP.Password = "new"
case "smtp-host":
d.SmtpImap.Credentials.SMTP.Host = "new.example.test"
case "smtp-security":
d.SmtpImap.Credentials.SMTP.Security = models.MailSecurityTLS
case "skip-folders":
d.Sync.Policy.SkipFolders = []string{"Custom"}
case "sent-copy":
v := false
d.SaveToSent = &v
}
if err := m.AddWMail(t.Context(), d); err != nil {
t.Fatal(err)
}
if m.Get(d.ID) == old {
t.Fatalf("changed %s ignored", field)
}
m.Terminate(d.ID)
})
}
}
internal/app/worker/mailmanager/terminate.go
+21 -3
View File
@@ -1,10 +1,28 @@
package mailmanager
import "github.com/google/uuid"
import (
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/app/worker/wmail"
)
func (m *MailManager) Terminate(id uuid.UUID) {
m.Lock()
defer m.Unlock()
mail := m.Emails[id]
delete(m.Emails, id)
m.Unlock()
if mail != nil {
mail.Stop()
}
}
// TerminateMailbox cannot retire a newer generation through a stale callback.
func (m *MailManager) TerminateMailbox(id uuid.UUID, mail *wmail.WMail) {
m.Lock()
if m.Emails[id] == mail {
delete(m.Emails, id)
}
m.Unlock()
if mail != nil {
mail.Stop()
}
}
internal/app/worker/wmail/admit.go
+27 -11
View File
@@ -2,6 +2,7 @@ package wmail
import (
"context"
"errors"
"time"
"github.com/google/uuid"
@@ -133,15 +134,21 @@ func (w *WMail) laneOf(ctx context.Context, key string, msg *models.EmailMessage
// beginTick releases an expired hold so the state reports "within budget"
// before the pass looks at anything.
func (w *WMail) beginTick() {
w.tracker.tickComplete = false
w.tracker.release(time.Now())
}
// endTick folds the pass into the relayed state. deferred is the number of
// live messages still waiting on the server; it is reported as-is (not
// accumulated) so a pass that admits everything zeroes it.
func (w *WMail) endTick(stats *tickStats) {
w.tracker.setDeferred(stats.deferred)
w.tracker.touch(time.Now())
// endTick retains unmeasured backlog until a complete pass confirms catch-up.
func (w *WMail) endTick(ctx context.Context, stats *tickStats, complete bool) {
complete = complete && ctx.Err() == nil && !stats.aborted && stats.deferred == 0 && w.tracker.state.BackfillCursor.GoogleRecovery == nil
w.tracker.tickComplete = complete
if complete {
w.tracker.setDeferred(0)
w.tracker.touch(time.Now())
return
}
w.tracker.setDeferred(max(w.tracker.state.Deferred, stats.deferred))
w.tracker.flush(time.Now())
}
// storeNew is the shared tail of every provider's new-message path: record
@@ -164,14 +171,12 @@ func (w *WMail) storeNew(ctx context.Context, msg *models.EmailMessageData, data
return err
}
if err := w.EmailMessageMapRepository.Add(ctx, repository.EmailMessageData{
mapping := repository.EmailMessageData{
UserID: w.UserID.String(),
EmailID: w.ID.String(),
MessageID: mapKey,
ID: data.ID.String(),
ThreadID: data.ThreadID,
}); err != nil {
return err
}
// Both report the send they are about, if any. The id goes on the arrival
@@ -191,11 +196,22 @@ func (w *WMail) storeNew(ctx context.Context, msg *models.EmailMessageData, data
}
// The consumer decodes NEW_EMAIL as JobEventNewEmail{user_id, message}.
err := w.onEvent(models.JobEventTypeNewEmail, &models.JobEventNewEmail{
arrival := &models.JobEventNewEmail{
UserID: w.UserID,
Message: data,
ReportOriginalMessageID: reportAbout,
})
}
if durable, ok := w.EmailMessageMapRepository.(repository.ArrivalAdmission); ok {
err := durable.AdmitArrival(ctx, mapping, &repository.PendingArrival{Arrival: arrival, Bounce: bounce, Complaint: complaint})
if !errors.Is(err, repository.ErrArrivalOutboxUnsupported) {
return err
}
log.Warn().Str("email_id", w.ID.String()).Msg("sync: legacy arrival publication; upgrade backend and consumer before workers for crash durability")
}
if err := w.EmailMessageMapRepository.Add(ctx, mapping); err != nil {
return err
}
err := w.onEvent(models.JobEventTypeNewEmail, arrival)
if err != nil {
// The entry would mark a message that never reached the unibox as
// known, and every later pass would skip it; drop it so it is re-offered.
internal/app/worker/wmail/arrival_durability_test.go
+97
View File
@@ -0,0 +1,97 @@
package wmail
import (
"context"
"errors"
"testing"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/repository"
)
type durableTestMap struct {
recoveryMessageMap
pending map[string]*repository.PendingArrival
failAfterCommit bool
unsupported bool
}
func (m *durableTestMap) AdmitArrival(ctx context.Context, data repository.EmailMessageData, p *repository.PendingArrival) error {
if m.unsupported {
return repository.ErrArrivalOutboxUnsupported
}
if _, exists := m.data[data.MessageID]; !exists {
m.data[data.MessageID] = data
m.pending[data.MessageID] = p
}
if m.failAfterCommit {
return errors.New("admission response lost")
}
return nil
}
func TestArrivalDurabilityRestartPreservesIDAcrossProviders(t *testing.T) {
for _, tc := range []struct {
name string
provider models.InboxProvider
store func(*WMail, context.Context, *models.EmailMessageData) error
key string
}{
{"google", models.InboxProviderGoogle, (*WMail).googleStore, "provider-id"},
{"graph", models.InboxProviderOutlook, (*WMail).graphStore, "provider-id"},
{"imap", models.InboxProviderSMTPIMAP, (*WMail).imapStore, "<arrival@test.local>"},
} {
t.Run(tc.name, func(t *testing.T) {
ctx := context.Background()
m := &durableTestMap{recoveryMessageMap: recoveryMessageMap{data: map[string]repository.EmailMessageData{}}, pending: map[string]*repository.PendingArrival{}, failAfterCommit: true}
var events []captured
w := newTestWMail(t, tc.provider, &events)
w.EmailMessageMapRepository = m
msg := &models.EmailMessageData{MessageID: "<arrival@test.local>", GmailID: "provider-id", Subject: "arrival", BodyPlain: "complete body", InReplyTo: []string{"<send@test.local>"}}
if err := tc.store(w, ctx, msg); err == nil {
t.Fatal("expected ambiguous response")
}
canonical, err := m.Get(ctx, w.UserID, w.ID, tc.key)
if err != nil || canonical == nil {
t.Fatalf("durable mapping missing: %v", err)
}
original := canonical.ID
if len(events) != 0 || len(m.pending) != 1 {
t.Fatal("durable arrival must not depend on worker publication")
}
fresh := newTestWMail(t, tc.provider, &events)
fresh.UserID = w.UserID
fresh.ID = w.ID
fresh.EmailMessageMapRepository = m
m.failAfterCommit = false
if !fresh.retryUnmap(ctx) {
t.Fatal("fresh worker failed")
}
// Re-offering after a lost response cannot replace the queued canonical ID.
if err := tc.store(fresh, ctx, msg); err != nil {
t.Fatal(err)
}
p := m.pending[tc.key]
if len(m.pending) != 1 || p.Arrival.Message.ID.String() != original || m.data[tc.key].ID != original || len(events) != 0 {
t.Fatal("restart reminted the arrival or published outside durable delivery")
}
if p.Arrival.UserID != w.UserID || p.Arrival.Message.EmailID != w.ID || p.Arrival.Message.ID == uuid.Nil {
t.Fatal("ownership not retained")
}
})
}
}
func TestArrivalOldBackendKeepsLegacyPublication(t *testing.T) {
var events []captured
w := newTestWMail(t, models.InboxProviderGoogle, &events)
m := &durableTestMap{recoveryMessageMap: recoveryMessageMap{data: map[string]repository.EmailMessageData{}}, unsupported: true}
w.EmailMessageMapRepository = m
if err := w.googleStore(context.Background(), &models.EmailMessageData{MessageID: "legacy", GmailID: "legacy"}); err != nil {
t.Fatal(err)
}
if len(newEmails(events)) != 1 || len(m.data) != 1 {
t.Fatal("old backend lost existing sync functionality")
}
}
internal/app/worker/wmail/err.go
+13 -6
View File
@@ -37,6 +37,12 @@ func reportable(err error) bool {
}
func (w *WMail) CaptureError(err error) {
w.lifecycleMu.Lock()
stopped := w.stopped
w.lifecycleMu.Unlock()
if stopped {
return
}
if reportable(err) {
errs.CaptureException(err,
errs.Tag("user_id", w.UserID.String()),
@@ -71,18 +77,19 @@ func (w *WMail) CaptureError(err error) {
Timestamp: time.Now().Unix(),
}
w.lifecycleMu.Lock()
if w.stopped {
w.lifecycleMu.Unlock()
return
}
_ = w.onEvent(eventType, errorEvent)
w.lifecycleMu.Unlock()
// Critical errors should stop the sync loop and remove the account from the
// worker's local state until the user re-authenticates.
if eventType == models.JobEventTypeEmailAuthError ||
eventType == models.JobEventTypeEmailDisabled {
if w.Cancel != nil {
w.Cancel()
}
if w.TerminateFunc != nil {
w.TerminateFunc()
}
w.Terminate()
}
}
internal/app/worker/wmail/err_mail.go
+4 -2
View File
@@ -35,6 +35,8 @@ func (w *WMail) RunMailError(f func() *errx.MailError) {
}
func (w *WMail) Terminate() {
w.Cancel()
w.TerminateFunc()
w.Stop()
if w.TerminateFunc != nil {
w.TerminateFunc()
}
}
internal/app/worker/wmail/event_graph.go
+9 -1
View File
@@ -10,13 +10,16 @@ import (
// onGraphMessageRemove emits REMOVE_EMAIL for a message deleted or moved out of a
// tracked folder. providerID is the Graph message id.
func (w *WMail) onGraphMessageRemove(ctx context.Context, providerID string) error {
internalMessage, err := w.EmailMessageMapRepository.Get(ctx, w.UserID, w.ID, providerID)
internalMessage, err := w.graphMessageMap(ctx, providerID)
if err != nil {
return err
}
if internalMessage == nil {
return nil
}
if w.GraphData.Client.ImmutableIDMode() {
return w.graphReconcileMessage(ctx, internalMessage.ID, providerID, "")
}
internalID, err := uuid.Parse(internalMessage.ID)
if err != nil {
@@ -64,6 +67,11 @@ func (w *WMail) onGraphFlagsChange(ctx context.Context, providerID string, seen
// durable persistence (the worker is disposable and must not be the source of
// truth for the cursor).
func (w *WMail) onGraphDelta(_ context.Context, folder, deltaLink string) error {
if w.graphRecovering(folder) {
if err := w.tracker.emit(w.tracker.state); err != nil {
return err
}
}
return w.onEvent(models.JobEventTypeGraphDeltaUpdate, &models.JobEventGraphDeltaUpdate{
UserID: w.UserID,
EmailID: w.ID,
internal/app/worker/wmail/event_token.go
+15
View File
@@ -6,6 +6,21 @@ import (
)
func (w *WMail) onTokenUpdate(token *oauth2.Token) error {
w.lifecycleMu.Lock()
defer w.lifecycleMu.Unlock()
if w.stopped {
return nil
}
if data := w.executionData; data != nil && !data.Brokered {
previous := ExecutionKey(data)
if data.Google != nil {
data.Google.Token = token
}
if data.Graph != nil {
data.Graph.Token = token
}
w.executionKeys = map[[32]byte]struct{}{w.initialExecutionKey: {}, previous: {}, ExecutionKey(data): {}}
}
return w.onEvent(models.JobEventTypeTokenUpdate, &models.JobEventTokenUpdate{
UserID: w.UserID,
EmailID: w.ID,
internal/app/worker/wmail/folder_identity_test.go
+55 -12
View File
@@ -1,9 +1,16 @@
package wmail
import (
"context"
"testing"
goimap "github.com/emersion/go-imap/v2"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/client/smtpimap/imap"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/repository"
)
// mailboxUpdates is every folder MAILBOX_UPDATE relayed, by name.
@@ -163,16 +170,30 @@ func TestSyncDoesNotGuessARenameWhenTwoFoldersCouldBeIt(t *testing.T) {
}
}
// A changed UIDVALIDITY is the server saying every UID we hold for the folder
// is void. The folder is re-baselined rather than followed from a cursor that
// now addresses nothing, and an unfinished import walks it again.
func TestSyncRebaselinesAFolderWhoseUIDValidityChanged(t *testing.T) {
conn := &fakeImapConn{
folders: []models.Mailbox{{Name: "INBOX", UIDValidity: 900, HighestModSeq: 5}},
changed: uidRange(3),
type folderIdentityRecoveryConn struct{ *fakeImapConn }
func (c *folderIdentityRecoveryConn) FetchEnvelopes(ctx context.Context, uids []goimap.UID) ([]*imap.Fetched, *errx.MailError) {
fetched, err := c.fakeImapConn.FetchEnvelopes(ctx, uids)
for _, f := range fetched {
f.Email.ModSeq = 5
f.Email.Flags = []string{models.FlagSeen}
}
return fetched, err
}
// Voided UIDs require bounded generation recovery, even with completed history and unchanged MODSEQ.
func TestSyncRebaselinesAFolderWhoseUIDValidityChanged(t *testing.T) {
conn := &folderIdentityRecoveryConn{fakeImapConn: &fakeImapConn{
folders: []models.Mailbox{{Name: "INBOX", UIDValidity: 900, UIDNext: 4, HighestModSeq: 5}},
changed: uidRange(3),
all: uidRange(3),
}}
w, events := newIMAPTestMail(conn, &fixedBudget{allow: 10},
&models.Mailbox{Name: "INBOX", UIDValidity: 7, HighestModSeq: 100})
&models.Mailbox{Name: "INBOX", UIDValidity: 7, UIDNext: 501, HighestModSeq: 5})
knownID := uuid.NewString()
maps := &recoveryMessageMap{data: map[string]repository.EmailMessageData{"<2@fake.test>": {ID: knownID, MessageID: "<2@fake.test>"}}}
w.EmailMessageMapRepository = maps
w.tracker.state.BackfillSynced = 17
w.tracker.setFolder("INBOX", models.SyncFolderCursor{UID: 500, Done: true})
w.flagScan = map[string]*folderFlagScan{"INBOX": {}}
@@ -187,11 +208,25 @@ func TestSyncRebaselinesAFolderWhoseUIDValidityChanged(t *testing.T) {
if box.UIDValidity != 900 || box.HighestModSeq != 5 {
t.Errorf("relayed %+v, want the server's current cursor", box)
}
if hasEvent(*events, models.JobEventTypeNewEmail) {
t.Error("the folder was walked from a cursor its server had already voided")
arrivals := mailboxEvents(*events, models.JobEventTypeNewEmail)
if len(arrivals) != 2 || len(maps.data) != 3 || maps.data["<2@fake.test>"].ID != knownID {
t.Fatal("bounded recovery lost unseen mail or duplicated the known Message-ID")
}
if cur := w.tracker.folder("INBOX"); cur.UID != 0 || cur.Done {
t.Errorf("backfill floor = %+v, want it cleared so the folder is walked again", cur)
updates := mailboxEvents(*events, models.JobEventTypeEmailUpdate)
if len(updates) != 1 {
t.Fatalf("known recovery updates=%d", len(updates))
}
update := updates[0].body.(*models.JobEventEmailUpdate)
if update.ID.String() != knownID || update.UID != 2 || update.Mailbox != 900 || update.ModSeq != 5 || !models.SeenFromFlags(update.Flags) {
t.Fatalf("known message kept voided handles or stale read state: %+v", update)
}
cur := w.tracker.folder("INBOX")
recovery := imapRecoveryCursor(cur)
if !cur.Done || cur.UID != 1 || recovery == nil || recovery.Generation != 900 || recovery.Synced != 2 {
t.Fatalf("folder did not finish bounded replay from its new UIDs: %+v / %+v", cur, recovery)
}
if w.tracker.state.BackfillStatus != models.SyncBackfillComplete || w.tracker.state.BackfillSynced != 17 {
t.Fatal("generation recovery reopened completed global history")
}
if _, held := w.flagScan["INBOX"]; held {
t.Error("the flag snapshot survived; it describes UIDs that no longer mean anything")
@@ -199,6 +234,14 @@ func TestSyncRebaselinesAFolderWhoseUIDValidityChanged(t *testing.T) {
if got := mailboxDeletes(*events); len(got) != 0 {
t.Errorf("retired %v; the folder is still there", got)
}
searches := conn.searches
next, later := imapReload(t, w, conn, &fixedBudget{allow: 10})
if err := next.Sync(t.Context()); err != nil {
t.Fatal(err)
}
if conn.searches != searches || len(mailboxEvents(*later, models.JobEventTypeNewEmail)) != 0 || len(mailboxEvents(*later, models.JobEventTypeEmailUpdate)) != 0 || len(maps.data) != 3 {
t.Fatal("completed generation recovery repeated after reload")
}
}
// A folder's backfill floor goes with the folder. A name is reusable, so a
internal/app/worker/wmail/governor.go
+1 -6
View File
@@ -304,10 +304,5 @@ func (w *WMail) escalate(mailErr *errx.MailError) {
ActionRequired: userInfo.ActionRequired,
Timestamp: time.Now().Unix(),
})
if w.Cancel != nil {
w.Cancel()
}
if w.TerminateFunc != nil {
w.TerminateFunc()
}
w.Terminate()
}
internal/app/worker/wmail/graph_recovery_test.go
+442
View File
@@ -0,0 +1,442 @@
package wmail
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/client/msgraph"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/repository"
)
type graphRecoveryProvider struct {
snapshot map[string][]any
page2 map[string][]any
messages map[string]map[string]any
expired string
seen bool
requests []string
}
func (g *graphRecoveryProvider) serve(t *testing.T) *httptest.Server {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if !strings.Contains(r.Header.Get("Prefer"), `IdType="ImmutableId"`) {
t.Error("sync requested move-unstable ids")
}
path := strings.TrimPrefix(r.URL.Path, "/v1.0/me/")
g.requests = append(g.requests, path+"?"+r.URL.RawQuery)
switch {
case path == "translateExchangeIds":
var body struct {
IDs []string `json:"inputIds"`
Target string `json:"targetIdType"`
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Error(err)
}
value := []any{}
for _, id := range body.IDs {
target := "stable-" + strings.TrimPrefix(id, "regular-")
if body.Target == "restId" {
target = "regular-" + strings.TrimPrefix(id, "stable-")
}
value = append(value, map[string]any{"sourceId": id, "targetId": target})
}
_ = json.NewEncoder(w).Encode(map[string]any{"value": value})
case strings.HasPrefix(path, "messages/"):
id := strings.TrimPrefix(path, "messages/")
msg := g.messages[id]
if msg == nil {
w.WriteHeader(http.StatusNotFound)
_, _ = w.Write([]byte(`{"error":{"code":"ErrorItemNotFound"}}`))
return
}
copy := map[string]any{}
for key, value := range msg {
copy[key] = value
}
copy["isRead"] = g.seen
_ = json.NewEncoder(w).Encode(copy)
case strings.HasPrefix(path, "mailFolders/"):
folder, tail, _ := strings.Cut(strings.TrimPrefix(path, "mailFolders/"), "/")
if tail == "" {
_ = json.NewEncoder(w).Encode(map[string]any{"id": "folder-" + folder})
return
}
if tail != "messages/delta" {
t.Errorf("unexpected path %s", path)
w.WriteHeader(500)
return
}
if folder == g.expired && r.URL.Query().Get("$deltatoken") == "seed" {
w.WriteHeader(http.StatusGone)
_, _ = w.Write([]byte(`{"error":{"code":"syncStateNotFound"}}`))
return
}
value := g.snapshot[folder]
out := map[string]any{"value": value, "@odata.deltaLink": "https://graph.microsoft.com" + r.URL.Path + "?$deltatoken=fresh"}
if g.page2[folder] != nil {
if r.URL.Query().Get("$skiptoken") == "page2" {
out["value"] = g.page2[folder]
} else {
delete(out, "@odata.deltaLink")
out["@odata.nextLink"] = "https://graph.microsoft.com" + r.URL.Path + "?$skiptoken=page2"
}
}
_ = json.NewEncoder(w).Encode(out)
default:
t.Errorf("unexpected Graph path %s", path)
w.WriteHeader(500)
}
}))
t.Cleanup(srv.Close)
return srv
}
type graphRecoveryContext struct{ recoveryRows }
func (*graphRecoveryContext) IsOwnConversation(_ context.Context, _, _ uuid.UUID, _ []string, thread string) (bool, error) {
return thread == "conv-stable-reply", nil
}
type graphRecoveryBudget struct {
backfill int
lanes []SyncLane
observed int
}
func (*graphRecoveryBudget) Policy() models.SyncPolicy {
return normalizePolicy(models.SyncPolicy{BackfillDays: 7, BackfillMessages: 2})
}
func (*graphRecoveryBudget) SetPolicy(models.SyncPolicy) {}
func (*graphRecoveryBudget) RecordThrottledDay(context.Context) bool { return false }
func (b *graphRecoveryBudget) ObserveLive(_ context.Context, n int) bool {
b.observed += n
return false
}
func (b *graphRecoveryBudget) Admit(_ context.Context, lane SyncLane) Admission {
b.lanes = append(b.lanes, lane)
if lane == LaneBackfill {
if b.backfill == 0 {
return Admission{Until: time.Now().Add(time.Second)}
}
b.backfill--
}
return Admission{OK: true}
}
func graphReloadState(t *testing.T, state models.SyncState) *models.SyncState {
t.Helper()
data, err := json.Marshal(state)
if err != nil {
t.Fatal(err)
}
var out models.SyncState
if err := json.Unmarshal(data, &out); err != nil {
t.Fatal(err)
}
return &out
}
func TestGraphRecoveryBudgetedReplyAndNativeContinuationSurviveReload(t *testing.T) {
for _, expired := range []bool{false, true} {
t.Run(map[bool]string{false: "missing completed Archive", true: "expired Inbox"}[expired], func(t *testing.T) {
folder := msgraph.FolderArchive
if expired {
folder = msgraph.FolderInbox
}
g := &graphRecoveryProvider{snapshot: map[string][]any{folder: {map[string]any{"id": "stable-one"}}}, page2: map[string][]any{folder: {map[string]any{"id": "stable-two"}, map[string]any{"id": "stable-capped"}, map[string]any{"id": "stable-reply"}}}, messages: map[string]map[string]any{}}
for _, id := range []string{"stable-one", "stable-two", "stable-capped", "stable-reply"} {
g.messages[id] = graphMessageJSON(id)
}
g.messages["stable-reply"]["receivedDateTime"] = time.Now().Add(-90 * 24 * time.Hour).UTC().Format(time.RFC3339)
if expired {
g.expired = folder
}
srv := g.serve(t)
var events []captured
var states []models.SyncState
w := newGraphTestMail(t, srv, &events, &states)
w.GraphData.Client.SetImmutableIDMode(true)
if !expired {
delete(w.GraphData.Client.DeltaLinks, folder)
}
maps := &recoveryMessageMap{data: map[string]repository.EmailMessageData{}}
w.EmailMessageMapRepository = maps
w.SyncContext = &graphRecoveryContext{}
w.tracker.state.BackfillStatus = models.SyncBackfillComplete
budget := &graphRecoveryBudget{backfill: 1}
w.gov = budget
if err := w.SyncGraph(t.Context()); err != nil {
t.Fatal(err)
}
ids := importedIDs(events)
if !hasID(ids, "stable-one") || !hasID(ids, "stable-reply") || hasID(ids, "stable-two") || budget.observed != 0 {
t.Fatalf("first replay imports=%v live-count=%d", ids, budget.observed)
}
if !strings.Contains(w.GraphData.Client.DeltaLinks[folder], "$skiptoken=page2") || w.tracker.state.LastSyncedAt != nil {
t.Fatalf("deferred replay advanced freshness or lost native continuation: %v", w.tracker.state)
}
before, err := w.graphRecovery(folder)
if err != nil || before.Count != 1 {
t.Fatalf("saved recovery=%+v err=%v", before, err)
}
reloaded := newGraphTestMail(t, srv, &events, &states)
reloaded.ID, reloaded.UserID = w.ID, w.UserID
reloaded.EmailMessageMapRepository, reloaded.SyncContext = maps, w.SyncContext
reloaded.GraphData.Client.SetImmutableIDMode(true)
reloaded.GraphData.Client.DeltaLinks = cloneStringMap(w.GraphData.Client.DeltaLinks)
reloaded.tracker = newSyncTracker(graphReloadState(t, w.tracker.state), func(st models.SyncState) error { states = append(states, st); return nil })
reloaded.gov = &graphRecoveryBudget{backfill: 1}
if err := reloaded.SyncGraph(t.Context()); err != nil {
t.Fatal(err)
}
after, err := reloaded.graphRecovery(folder)
if err != nil || !before.Since.Equal(after.Since) || after.Count != 2 || reloaded.graphRecovering(folder) {
t.Fatalf("restarted recovery changed fixed window/count: before=%+v after=%+v err=%v", before, after, err)
}
ids = importedIDs(events)
if len(ids) != 3 || !hasID(ids, "stable-two") || !strings.Contains(reloaded.GraphData.Client.DeltaLinks[folder], "$deltatoken=fresh") {
t.Fatalf("eventual deduplicated replay imports=%v links=%v", ids, reloaded.GraphData.Client.DeltaLinks)
}
g.snapshot[folder], g.page2[folder] = nil, nil
if err := reloaded.SyncGraph(t.Context()); err != nil || len(importedIDs(events)) != 3 {
t.Fatalf("repeat created duplicate arrivals: %v %v", err, importedIDs(events))
}
})
}
}
func TestGraphArchiveUpgradePreservesLegacyIdentityMovesAndFlags(t *testing.T) {
id := uuid.New()
g := &graphRecoveryProvider{snapshot: map[string][]any{
msgraph.FolderInbox: {map[string]any{"id": "stable-known", "@removed": map[string]any{"reason": "deleted"}}},
msgraph.FolderArchive: {map[string]any{"id": "stable-known"}, map[string]any{"id": "stable-new"}},
}, messages: map[string]map[string]any{"stable-known": graphMessageJSON("stable-known"), "stable-new": graphMessageJSON("stable-new")}}
g.messages["stable-known"]["parentFolderId"] = "folder-archive"
var events []captured
var states []models.SyncState
w := newGraphTestMail(t, g.serve(t), &events, &states)
w.GraphData.Client.SetImmutableIDMode(true)
delete(w.GraphData.Client.DeltaLinks, msgraph.FolderArchive)
maps := &recoveryMessageMap{data: map[string]repository.EmailMessageData{"regular-known": {UserID: w.UserID.String(), EmailID: w.ID.String(), MessageID: "regular-known", ID: id.String(), ThreadID: "owned"}}}
w.EmailMessageMapRepository = maps
rows := &graphRecoveryContext{recoveryRows: recoveryRows{rows: []repository.ProviderFolderMessage{{ID: id, ProviderID: "regular-known", ProviderFolder: models.FolderInbox}}}}
w.SyncContext = rows
w.tracker.state.BackfillStatus = models.SyncBackfillComplete
baseEmit := w.onEvent
w.onEvent = func(kind models.JobEventType, body any) error {
if update, ok := body.(*models.JobEventFolderUpdate); ok && update.ID == id {
if update.Relayed {
rows.rows[0].ProviderID = update.ProviderID
} else {
rows.rows[0].ProviderFolder = update.Folder
}
}
return baseEmit(kind, body)
}
if err := w.SyncGraph(t.Context()); err != nil {
t.Fatal(err)
}
if len(importedIDs(events)) != 1 || maps.data["stable-known"].ID != id.String() || maps.data["regular-known"].ID != id.String() || rows.rows[0].ProviderFolder != models.FolderArchive {
t.Fatalf("move changed identity or lost Archive: ids=%v maps=%v rows=%v", importedIDs(events), maps.data, rows.rows)
}
for _, ev := range events {
if ev.eventType == models.JobEventTypeRemoveEmail {
t.Fatal("move-out deleted the platform conversation")
}
}
g.seen = true
before := len(events)
reloaded := newGraphTestMail(t, g.serve(t), &events, &states)
reloaded.ID, reloaded.UserID = w.ID, w.UserID
reloaded.GraphData.Client.SetImmutableIDMode(true)
reloaded.GraphData.Client.DeltaLinks = cloneStringMap(w.GraphData.Client.DeltaLinks)
reloaded.EmailMessageMapRepository, reloaded.SyncContext = maps, rows
reloaded.tracker = newSyncTracker(graphReloadState(t, w.tracker.state), func(models.SyncState) error { return nil })
if err := reloaded.SyncGraph(t.Context()); err != nil {
t.Fatal(err)
}
flags := false
for _, ev := range events[before:] {
if ev.eventType == models.JobEventTypeFlagsAdd {
update := ev.body.(*models.JobEventFlags)
flags = flags || update.ID == id
}
}
if !flags || len(importedIDs(events)) != 1 {
t.Fatalf("Archive read-state update lost or reimported: flags=%v arrivals=%v", flags, importedIDs(events))
}
}
func TestGraphRecoveryPendingBaselineAndStoredRemovalScanResumeAfterReload(t *testing.T) {
g := &graphRecoveryProvider{snapshot: map[string][]any{}, expired: msgraph.FolderInbox, messages: map[string]map[string]any{}}
srv := g.serve(t)
var events []captured
var states []models.SyncState
w := newGraphTestMail(t, srv, &events, &states)
w.GraphData.Client.SetImmutableIDMode(true)
w.tracker.state.BackfillStatus = models.SyncBackfillComplete
w.tracker.setFolder(graphIdentityCursor, models.SyncFolderCursor{Done: true})
rows := &graphRecoveryContext{}
maps := &recoveryMessageMap{data: map[string]repository.EmailMessageData{}}
for n := 1; n <= graphBackfillPage+1; n++ {
id := uuid.UUID{}
id[15] = byte(n)
providerID := "stable-" + id.String()
rows.rows = append(rows.rows, repository.ProviderFolderMessage{ID: id, ProviderID: providerID, ProviderFolder: models.FolderInbox})
maps.data[providerID] = repository.EmailMessageData{ID: id.String(), MessageID: providerID}
}
w.EmailMessageMapRepository, w.SyncContext = maps, rows
if err := w.SyncGraph(t.Context()); err != nil {
t.Fatal(err)
}
removals := func() int {
n := 0
for _, ev := range events {
if ev.eventType == models.JobEventTypeRemoveEmail {
n++
}
}
return n
}
if removals() != graphBackfillPage || !strings.Contains(w.GraphData.Client.DeltaLinks[msgraph.FolderInbox], "seed") {
t.Fatalf("unreconciled baseline installed: events=%d links=%v", len(events), w.GraphData.Client.DeltaLinks)
}
state, err := w.graphRecovery(msgraph.FolderInbox)
if err != nil || !strings.Contains(state.PendingLink, "fresh") {
t.Fatalf("captured baseline missing: %+v %v", state, err)
}
countInbox := func() int {
n := 0
for _, req := range g.requests {
if strings.HasPrefix(req, "mailFolders/inbox/messages/delta") {
n++
}
}
return n
}
before := countInbox()
reloaded := newGraphTestMail(t, srv, &events, &states)
reloaded.ID, reloaded.UserID = w.ID, w.UserID
reloaded.GraphData.Client.SetImmutableIDMode(true)
reloaded.GraphData.Client.DeltaLinks = cloneStringMap(w.GraphData.Client.DeltaLinks)
reloaded.EmailMessageMapRepository, reloaded.SyncContext = maps, rows
reloaded.tracker = newSyncTracker(graphReloadState(t, w.tracker.state), func(models.SyncState) error { return nil })
if err := reloaded.SyncGraph(t.Context()); err != nil {
t.Fatal(err)
}
if removals() != graphBackfillPage+1 || countInbox() != before || !strings.Contains(reloaded.GraphData.Client.DeltaLinks[msgraph.FolderInbox], "fresh") || reloaded.tracker.state.LastSyncedAt != nil {
t.Fatalf("stored scan did not resume/freshness claimed before live catch-up: events=%d inbox=%d/%d state=%v", len(events), countInbox(), before, reloaded.tracker.state)
}
if err := reloaded.SyncGraph(t.Context()); err != nil || reloaded.tracker.state.LastSyncedAt == nil {
t.Fatalf("fresh baseline did not catch up: err=%v", err)
}
}
func TestGraphRecoveryCompletionSurvivesLaterLiveCursorAdvancement(t *testing.T) {
var events []captured
var states []models.SyncState
w := newGraphTestMail(t, newFakeGraph().serve(t), &events, &states)
if err := w.graphRecoveryStart(t.Context(), msgraph.FolderInbox); err != nil {
t.Fatal(err)
}
if err := w.graphRecoveryCheckpoint(t.Context(), msgraph.FolderInbox, "baseline"); err != nil {
t.Fatal(err)
}
w.GraphData.Client.DeltaLinks[msgraph.FolderInbox] = "baseline"
if w.graphRecovering(msgraph.FolderInbox) {
t.Fatal("installed baseline still recovering")
}
w.GraphData.Client.DeltaLinks[msgraph.FolderInbox] = "later-live-checkpoint"
if w.graphRecovering(msgraph.FolderInbox) {
t.Fatal("later live cursor reopened completed recovery")
}
legacy := graphReloadState(t, w.tracker.state)
if legacy.BackfillCursor.Folders["graph:delta-recovery:inbox"].Next == "" || !legacy.BackfillCursor.Folders["graph:delta-recovery:inbox"].Done {
t.Fatal("existing cursor protocol lost opaque recovery metadata")
}
}
func TestGraphImmutableAliasRetainsHiddenLegacyMappingAndRejectsCollision(t *testing.T) {
g := &graphRecoveryProvider{snapshot: map[string][]any{}, messages: map[string]map[string]any{}}
var events []captured
var states []models.SyncState
w := newGraphTestMail(t, g.serve(t), &events, &states)
w.GraphData.Client.SetImmutableIDMode(true)
id := uuid.NewString()
maps := &recoveryMessageMap{data: map[string]repository.EmailMessageData{"regular-hidden": {ID: id, MessageID: "regular-hidden"}}}
w.EmailMessageMapRepository = maps
known, err := w.graphMessageMap(t.Context(), "stable-hidden")
if err != nil || known == nil || known.ID != id || maps.data["regular-hidden"].ID != id || len(importedIDs(events)) != 0 {
t.Fatalf("hidden legacy map became a new arrival: known=%v err=%v", known, err)
}
other := uuid.NewString()
maps.data["stable-conflict"] = repository.EmailMessageData{ID: other}
if err := w.graphAddAlias(t.Context(), repository.EmailMessageData{MessageID: "stable-conflict", ID: id}); err == nil || maps.data["stable-conflict"].ID != other {
t.Fatal("alias overwrote an unrelated provider mapping")
}
}
func TestGraphRecoveryArrivalPublicationFailureRetriesAfterReload(t *testing.T) {
g := newFakeGraph()
g.live = []string{"reply"}
var events []captured
var states []models.SyncState
srv := g.serve(t)
w := newGraphTestMail(t, srv, &events, &states)
delete(w.GraphData.Client.DeltaLinks, msgraph.FolderInbox)
w.tracker.state.BackfillStatus = models.SyncBackfillComplete
maps := &recoveryMessageMap{data: map[string]repository.EmailMessageData{}}
w.EmailMessageMapRepository = maps
base := w.onEvent
w.onEvent = func(kind models.JobEventType, body any) error {
if kind == models.JobEventTypeNewEmail {
return errors.New("publication unavailable")
}
return base(kind, body)
}
if err := w.SyncGraph(t.Context()); err != nil {
t.Fatal(err)
}
if w.GraphData.Client.DeltaLinks[msgraph.FolderInbox] != "" || len(maps.data) != 0 {
t.Fatalf("failed publication advanced cursor or retained suppressing map: %v %v", w.GraphData.Client.DeltaLinks, maps.data)
}
g.live = []string{"reply"}
reloaded := newGraphTestMail(t, srv, &events, &states)
reloaded.EmailMessageMapRepository = maps
reloaded.GraphData.Client.DeltaLinks = cloneStringMap(w.GraphData.Client.DeltaLinks)
reloaded.tracker = newSyncTracker(graphReloadState(t, w.tracker.state), func(models.SyncState) error { return nil })
if err := reloaded.SyncGraph(t.Context()); err != nil {
t.Fatal(err)
}
if len(importedIDs(events)) != 1 || reloaded.GraphData.Client.DeltaLinks[msgraph.FolderInbox] == "" {
t.Fatalf("arrival not replayed: ids=%v links=%v", importedIDs(events), reloaded.GraphData.Client.DeltaLinks)
}
}
func TestGraphOldBackendRetainsLegacySyncWithoutImmutableClaim(t *testing.T) {
g := newFakeGraph()
g.live = []string{"live"}
var events []captured
var states []models.SyncState
w := newGraphTestMail(t, g.serve(t), &events, &states)
w.GraphData.Client.SetImmutableIDMode(true)
w.SyncContext = &recoveryRows{err: repository.ErrSyncContextUnsupported}
w.tracker.state.BackfillStatus = models.SyncBackfillComplete
if err := w.SyncGraph(t.Context()); err != nil {
t.Fatal(err)
}
if w.GraphData.Client.ImmutableIDMode() || len(importedIDs(events)) != 1 {
t.Fatalf("old backend stalled sync or claimed immutable reconciliation: immutable=%v imports=%v", w.GraphData.Client.ImmutableIDMode(), importedIDs(events))
}
}
internal/app/worker/wmail/imap_conn.go
+1 -1
View File
@@ -40,7 +40,7 @@ type ImapConn interface {
FetchFlags(ctx context.Context, uidFrom uint32) (map[uint32]imap.FlagState, *errx.MailError)
SearchSince(since time.Time) ([]goimap.UID, *errx.MailError)
FetchEnvelopes(ctx context.Context, uids []goimap.UID) ([]*imap.Fetched, *errx.MailError)
FetchBody(f *imap.Fetched)
FetchBody(f *imap.Fetched) *errx.MailError
// Send path.
AppendToSent(ctx context.Context, raw []byte, sentAt time.Time) error
internal/app/worker/wmail/lifecycle.go
+189
View File
@@ -0,0 +1,189 @@
package wmail
import (
"context"
"crypto/sha256"
"encoding/json"
"slices"
"strings"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/models"
"golang.org/x/oauth2"
)
// ExecutionKey excludes checkpoints and policy budgets from client identity.
func ExecutionKey(data *models.AddWorkerEmail) [32]byte {
type credential struct{ Access, Refresh string }
tokenKey := func(t *oauth2.Token) credential {
if t == nil {
return credential{}
}
return credential{t.AccessToken, t.RefreshToken}
}
key := struct {
ID, User uuid.UUID
Org *uuid.UUID
Provider models.InboxProvider
Email, First, Last, GraphUser string
Brokered, ImapSync, SaveToSent bool
Token credential
SMTP, IMAP *models.Service
Skip []string
}{ID: data.ID, User: data.UserID, Org: data.OrganizationID, Provider: data.Type,
Email: data.Email, First: data.FirstName, Last: data.LastName, Brokered: data.Brokered,
ImapSync: data.Type == models.InboxProviderSMTPIMAP && data.ImapSync, SaveToSent: data.Type == models.InboxProviderSMTPIMAP && data.SavesSentCopy()}
if data.Type == models.InboxProviderGoogle && data.Google != nil && !data.Brokered {
key.Token = tokenKey(data.Google.Token)
}
if data.Type == models.InboxProviderOutlook && data.Graph != nil {
key.GraphUser = data.Graph.User
if !data.Brokered {
key.Token = tokenKey(data.Graph.Token)
}
}
if data.Type == models.InboxProviderSMTPIMAP && data.SmtpImap != nil && data.SmtpImap.Credentials != nil {
key.SMTP, key.IMAP = data.SmtpImap.Credentials.SMTP, data.SmtpImap.Credentials.IMAP
}
if data.Sync != nil && data.Type == models.InboxProviderSMTPIMAP {
for _, folder := range data.Sync.Policy.SkipFolders {
if folder = strings.ToLower(strings.TrimSpace(folder)); folder != "" {
key.Skip = append(key.Skip, folder)
}
}
slices.Sort(key.Skip)
key.Skip = slices.Compact(key.Skip)
}
b, _ := json.Marshal(key)
return sha256.Sum256(b)
}
func (w *WMail) rememberExecution(data *models.AddWorkerEmail) {
// Own the token pointers; ADD_EMAIL payloads may be reused by a publisher.
copyData := *data
if data.Google != nil {
g := *data.Google
copyData.Google = &g
}
if data.Graph != nil {
g := *data.Graph
copyData.Graph = &g
}
w.executionData = &copyData
w.initialExecutionKey = ExecutionKey(data)
w.executionKeys = map[[32]byte]struct{}{w.initialExecutionKey: {}}
}
func (w *WMail) SameExecution(data *models.AddWorkerEmail) bool {
w.lifecycleMu.Lock()
defer w.lifecycleMu.Unlock()
_, ok := w.executionKeys[ExecutionKey(data)]
return ok && !w.stopped
}
// BeginExecution rejects stale references and lets replacements drain admitted work.
func (w *WMail) BeginExecution() (func(), bool) {
w.lifecycleMu.Lock()
defer w.lifecycleMu.Unlock()
if w.stopped {
return nil, false
}
w.active.Add(1)
return w.active.Done, true
}
// Stop cancels scheduling and closes the provider without waiting on its caller.
func (w *WMail) Stop() {
w.lifecycleMu.Lock()
if w.stopped {
w.lifecycleMu.Unlock()
return
}
w.stopped = true
w.stopDone = make(chan struct{})
done := w.stopDone
w.lifecycleMu.Unlock()
defer close(done)
if w.Cancel != nil {
w.Cancel()
}
if w.SmtpImapData != nil && w.SmtpImapData.ImapClient != nil {
if c, ok := w.SmtpImapData.ImapClient.(interface{ Close() error }); ok {
_ = c.Close()
}
}
}
// Drain is called outside mailbox execution, after Stop prevents new work.
func (w *WMail) Drain() {
w.lifecycleMu.Lock()
done := w.stopDone
w.lifecycleMu.Unlock()
if done != nil {
<-done
}
w.active.Wait()
}
// ResumeInto uses the drained live state, which can be newer than ADD_EMAIL.
func (w *WMail) ResumeInto(data *models.AddWorkerEmail) {
if w.UserID != data.UserID || w.EmailType != data.Type {
return
}
if w.tracker != nil {
if data.Sync == nil {
data.Sync = &models.AddWorkerEmailSyncData{}
} else {
s := *data.Sync
data.Sync = &s
}
state := w.tracker.state
state.BackfillCursor.Folders = make(map[string]models.SyncFolderCursor, len(w.tracker.state.BackfillCursor.Folders))
for k, v := range w.tracker.state.BackfillCursor.Folders {
state.BackfillCursor.Folders[k] = v
}
data.Sync.State = &state
}
if w.GoogleData != nil && data.Google != nil {
g := *data.Google
g.LastHistoryID = w.GoogleData.LastHistoryID
data.Google = &g
}
if w.GraphData != nil && data.Graph != nil {
g := *data.Graph
g.DeltaLinks = cloneStringMap(w.GraphData.Client.DeltaLinks)
data.Graph = &g
}
if w.SmtpImapData != nil && data.SmtpImap != nil {
s := *data.SmtpImap
s.Mailboxes = nil
for _, box := range w.SmtpImapData.Mailboxes {
if box != nil {
s.Mailboxes = append(s.Mailboxes, *box)
}
}
data.SmtpImap = &s
}
}
// PreservePending retains session-local retry work during same-worker replacement.
func (w *WMail) PreservePending(old *WMail) {
if old == nil || old.UserID != w.UserID || old.EmailType != w.EmailType {
return
}
w.unmapPending = old.unmapPending
w.diagnosticRetries = old.diagnosticRetries
if old.tracker != nil && w.tracker != nil {
w.tracker.dirty = old.tracker.dirty
w.tracker.lastSent = old.tracker.lastSent
}
}
func (w *WMail) ExecutionContext(ctx context.Context) (context.Context, func()) {
ctx, cancel := context.WithCancel(ctx)
if w.Ctx == nil {
return ctx, cancel
}
stop := context.AfterFunc(w.Ctx, cancel)
return ctx, func() { stop(); cancel() }
}
internal/app/worker/wmail/lifecycle_test.go
+221
View File
@@ -0,0 +1,221 @@
package wmail
import (
"context"
"net"
"sync/atomic"
"testing"
"time"
goimap "github.com/emersion/go-imap/v2"
"github.com/emersion/go-imap/v2/imapserver"
"github.com/emersion/go-imap/v2/imapserver/imapmemserver"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/client/msgraph"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/models"
"golang.org/x/oauth2"
)
type closingIMAP struct {
ImapConn
closes atomic.Int32
}
type observedIMAPSession struct {
imapserver.Session
closed chan struct{}
}
func (s *observedIMAPSession) Close() error { defer close(s.closed); return s.Session.Close() }
func TestMailboxLifecycleStopClosesAuthenticatedIdleSocket(t *testing.T) {
t.Setenv("DEPLOYMENT_MODE", "self_hosted")
mem := imapmemserver.New()
user := imapmemserver.NewUser("mailbox@example.test", "local-test")
if err := user.Create("INBOX", nil); err != nil {
t.Fatal(err)
}
mem.AddUser(user)
closed := make(chan struct{})
srv := imapserver.New(&imapserver.Options{Caps: goimap.CapSet{goimap.CapIMAP4rev1: {}}, InsecureAuth: true, NewSession: func(*imapserver.Conn) (imapserver.Session, *imapserver.GreetingData, error) {
return &observedIMAPSession{Session: mem.NewSession(), closed: closed}, nil, nil
}})
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
go func() { _ = srv.Serve(ln) }()
defer srv.Close()
d := &models.AddWorkerEmail{ID: uuid.New(), UserID: uuid.New(), Type: models.InboxProviderSMTPIMAP, Email: "mailbox@example.test", ImapSync: true, SmtpImap: &models.AddWorkerEmailSmtpImapData{Credentials: &models.SmtpImap{IMAP: &models.Service{Host: "127.0.0.1", Port: ln.Addr().(*net.TCPAddr).Port, Username: "mailbox@example.test", Password: "local-test", Security: models.MailSecurityNone}}}}
w, merr := NewWMail(d, func(models.JobEventType, string, any) error { return nil }, nil, nil, nil, nil, nil, nil)
if merr != nil {
t.Fatal(merr)
}
w.Stop()
w.Stop()
select {
case <-closed:
case <-time.After(time.Second):
t.Fatal("authenticated idle server session retained after stop")
}
}
func (c *closingIMAP) Close() error { c.closes.Add(1); return nil }
func TestMailboxLifecycleStopDrainsAndRejectsStaleWork(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
c := &closingIMAP{}
w := &WMail{Ctx: ctx, Cancel: cancel, SmtpImapData: &SmtpImapData{ImapClient: c}}
done, ok := w.BeginExecution()
if !ok {
t.Fatal("live execution rejected")
}
w.Stop()
w.Stop()
if c.closes.Load() != 1 || ctx.Err() == nil {
t.Fatal("stop must cancel and close once")
}
if _, ok := w.BeginExecution(); ok {
t.Fatal("retired execution admitted")
}
drained := make(chan struct{})
go func() { w.Drain(); close(drained) }()
select {
case <-drained:
t.Fatal("drained before admitted work returned")
default:
}
done()
select {
case <-drained:
case <-time.After(time.Second):
t.Fatal("drain did not finish")
}
}
func TestMailboxLifecycleExecutionKeyAndTokenRefresh(t *testing.T) {
token := &oauth2.Token{AccessToken: "old", RefreshToken: "refresh", Expiry: time.Now().Add(time.Hour)}
d := &models.AddWorkerEmail{ID: uuid.New(), UserID: uuid.New(), Type: models.InboxProviderGoogle, Google: &models.AddWorkerEmailGoogleData{Token: token}}
w := &WMail{onEvent: func(models.JobEventType, any) error { return nil }}
w.rememberExecution(d)
changed := *d
changed.Sync = &models.AddWorkerEmailSyncData{Policy: models.SyncPolicy{DailyMessages: 99}, State: &models.SyncState{BackfillStatus: models.SyncBackfillComplete}}
if !w.SameExecution(&changed) {
t.Fatal("policy/checkpoint publication restarted provider")
}
g := *d.Google
changed.Google = &g
g.Token = &oauth2.Token{AccessToken: "fresh", RefreshToken: "rotated", Expiry: time.Now().Add(time.Hour)}
if w.SameExecution(&changed) {
t.Fatal("local reauthorization ignored")
}
if err := w.onTokenUpdate(g.Token); err != nil {
t.Fatal(err)
}
if !w.SameExecution(&changed) || !w.SameExecution(d) {
t.Fatal("routine token refresh/repeated original publication restarted provider")
}
for i := 0; i < 10; i++ {
if err := w.onTokenUpdate(&oauth2.Token{AccessToken: string(rune('a' + i)), RefreshToken: "rotated"}); err != nil {
t.Fatal(err)
}
}
if len(w.executionKeys) > 3 {
t.Fatal("refresh aliases grew without bound")
}
brokered := *d
brokered.Brokered = true
w.rememberExecution(&brokered)
changed.Brokered = true
if !w.SameExecution(&changed) {
t.Fatal("broker-owned token rotation restarted provider")
}
w.Stop()
w.onEvent = func(models.JobEventType, any) error { t.Fatal("stale token callback published"); return nil }
if err := w.onTokenUpdate(token); err != nil {
t.Fatal(err)
}
}
func TestMailboxLifecycleLegacyDefaultsAndNormalizedSkipSettings(t *testing.T) {
d := &models.AddWorkerEmail{ID: uuid.New(), UserID: uuid.New(), Type: models.InboxProviderSMTPIMAP}
w := &WMail{}
w.rememberExecution(d)
copyData := *d
v := true
copyData.SaveToSent = &v
copyData.Sync = &models.AddWorkerEmailSyncData{Policy: models.SyncPolicy{DailyMessages: 99}}
if !w.SameExecution(&copyData) {
t.Fatal("old backend missing sync/sent fields differs from effective defaults")
}
copyData.Sync.Policy.SkipFolders = []string{" Custom ", "CUSTOM", ""}
w.rememberExecution(&copyData)
copyData.Sync = &models.AddWorkerEmailSyncData{Policy: models.SyncPolicy{SkipFolders: []string{"custom"}, DailyMessages: 120}}
if !w.SameExecution(&copyData) {
t.Fatal("same effective folder skips restarted client")
}
}
func TestMailboxLifecycleResumePreservesCompletedStateAndProviderCursors(t *testing.T) {
id, user := uuid.New(), uuid.New()
stamp := time.Now().Add(-time.Hour)
state := models.SyncState{BackfillStatus: models.SyncBackfillComplete, BackfillSynced: 12, BackfillCompletedAt: &stamp, LastSyncedAt: &stamp, Deferred: 5, BackfillCursor: models.SyncCursor{PageToken: "page", Folders: map[string]models.SyncFolderCursor{"Inbox": {UID: 8, Done: true}}}}
for _, provider := range []models.InboxProvider{models.InboxProviderGoogle, models.InboxProviderOutlook, models.InboxProviderSMTPIMAP} {
t.Run(string(provider), func(t *testing.T) {
old := &WMail{ID: id, UserID: user, EmailType: provider, tracker: newSyncTracker(&state, func(models.SyncState) error { return nil }), unmapPending: map[string]uuid.UUID{"pending": uuid.New()}}
d := &models.AddWorkerEmail{ID: id, UserID: user, Type: provider}
switch provider {
case models.InboxProviderGoogle:
old.GoogleData = &GoogleData{LastHistoryID: 91}
d.Google = &models.AddWorkerEmailGoogleData{LastHistoryID: 1}
case models.InboxProviderOutlook:
old.GraphData = &GraphData{Client: &msgraph.Client{DeltaLinks: map[string]string{"Inbox": "live"}}}
d.Graph = &models.AddWorkerEmailGraphData{DeltaLinks: map[string]string{"Inbox": "stale"}}
case models.InboxProviderSMTPIMAP:
old.SmtpImapData = &SmtpImapData{Mailboxes: []*models.Mailbox{{Name: "Inbox", UIDValidity: 4, UIDNext: 17}}}
d.SmtpImap = &models.AddWorkerEmailSmtpImapData{}
}
old.Stop()
old.Drain()
old.ResumeInto(d)
if d.Sync.State.BackfillStatus != models.SyncBackfillComplete || d.Sync.State.BackfillSynced != 12 || d.Sync.State.Deferred != 5 || d.Sync.State.LastSyncedAt != &stamp || d.Sync.State.BackfillCursor.PageToken != "page" {
t.Fatal("replacement reset completed/live state")
}
d.Sync.State.BackfillCursor.Folders["Inbox"] = models.SyncFolderCursor{}
if old.tracker.state.BackfillCursor.Folders["Inbox"].UID != 8 {
t.Fatal("replacement aliases retired mutable state")
}
if d.Google != nil && d.Google.LastHistoryID != 91 {
t.Fatal("Gmail cursor lost")
}
if d.Graph != nil && d.Graph.DeltaLinks["Inbox"] != "live" {
t.Fatal("Graph cursor lost")
}
if d.SmtpImap != nil && (len(d.SmtpImap.Mailboxes) != 1 || d.SmtpImap.Mailboxes[0].UIDNext != 17) {
t.Fatal("IMAP cursor lost")
}
next := &WMail{UserID: user, EmailType: provider, tracker: newSyncTracker(d.Sync.State, func(models.SyncState) error { return nil })}
old.tracker.dirty = true
next.PreservePending(old)
if next.unmapPending["pending"] != old.unmapPending["pending"] || !next.tracker.dirty {
t.Fatal("pending retries/state flush lost")
}
})
}
}
func TestMailboxLifecycleAuthTerminationClosesOnceAndIgnoresRetiredErrors(t *testing.T) {
c := &closingIMAP{}
events := 0
w := &WMail{SmtpImapData: &SmtpImapData{ImapClient: c}, onEvent: func(models.JobEventType, any) error { events++; return nil }}
w.CaptureError(errx.ErrMailServerUnreachable)
if c.closes.Load() != 0 {
t.Fatal("transport timeout permanently retired mailbox")
}
w.CaptureError(&errx.MailError{Type: errx.MailErrorCritical, Code: errx.MailErrorCodeInvalidCredentials})
w.CaptureError(&errx.MailError{Type: errx.MailErrorCritical, Code: errx.MailErrorCodeInvalidCredentials})
if c.closes.Load() != 1 || events != 2 {
t.Fatalf("closes=%d events=%d", c.closes.Load(), events)
}
}
internal/app/worker/wmail/provider_deadline_test.go
+91
View File
@@ -0,0 +1,91 @@
package wmail
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"time"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/models"
"golang.org/x/oauth2"
)
func TestMailboxProviderDeadlineReleasesPassAndAllowsRetry(t *testing.T) {
for _, provider := range []models.InboxProvider{models.InboxProviderGoogle, models.InboxProviderOutlook} {
for _, body := range []bool{false, true} {
t.Run(fmt.Sprintf("%s/body=%v", provider, body), func(t *testing.T) {
var stalled atomic.Bool
stalled.Store(true)
release := make(chan struct{})
srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
rw.Header().Set("Content-Type", "application/json")
if stalled.Load() {
if body {
rw.WriteHeader(http.StatusOK)
_, _ = rw.Write([]byte(`{"`))
rw.(http.Flusher).Flush()
}
select {
case <-release:
case <-r.Context().Done():
}
return
}
if provider == models.InboxProviderGoogle {
_, _ = rw.Write([]byte(`{"historyId":"91","history":[]}`))
} else {
_, _ = fmt.Fprintf(rw, `{"value":[],"@odata.deltaLink":%q}`, "https://graph.microsoft.com"+r.URL.Path+"?next=live")
}
}))
defer srv.Close()
defer close(release)
var events []captured
var w *WMail
ctx := context.WithValue(context.Background(), oauth2.HTTPClient, &http.Client{Timeout: 40 * time.Millisecond, Transport: rewriteToTestServer(srv.URL)})
token := &oauth2.Token{AccessToken: "test", Expiry: time.Now().Add(time.Hour)}
if provider == models.InboxProviderGoogle {
w = newGoogleTestMail(t, srv, &events)
w.GoogleData.LastHistoryID = 91
if err := w.GoogleData.Client.Init(ctx, token, oauth2.Config{}); err != nil {
t.Fatal(err)
}
w.googleReconciledAt = time.Now()
} else {
var states []models.SyncState
w = newGraphTestMail(t, srv, &events, &states)
if err := w.GraphData.Client.Init(ctx, token, oauth2.Config{}); err != nil {
t.Fatal(err)
}
}
w.tracker.state.BackfillStatus = models.SyncBackfillComplete
mailCtx, cancel := context.WithCancel(context.Background())
defer cancel()
w.Ctx, w.Cancel = mailCtx, cancel
result := make(chan *errx.MailError, 1)
go func() { result <- w.SyncMail(context.Background()) }()
select {
case err := <-result:
if err == nil {
t.Fatal("stalled provider pass reported success")
}
case <-time.After(time.Second):
stalled.Store(false)
srv.CloseClientConnections()
<-result
t.Fatal("provider request deadline did not release sync pass")
}
if ctx.Err() != nil || w.Ctx.Err() != nil {
t.Fatal("request timeout cancelled mailbox lifetime")
}
stalled.Store(false)
if err := w.SyncMail(context.Background()); err != nil {
t.Fatalf("later retry without reconnect failed: %v", err)
}
})
}
}
}
internal/app/worker/wmail/send.go
+7
View File
@@ -182,6 +182,13 @@ func providerThrottle(err *errx.MailError) bool {
// Send attempts to send an email with retry for transient failures
func (w *WMail) Send(ctx context.Context, req *SendRequest) *SendResult {
done, ok := w.BeginExecution()
if !ok {
return &SendResult{Error: errx.ErrMailServerUnreachable}
}
defer done()
ctx, cancel := w.ExecutionContext(ctx)
defer cancel()
// For warmup emails, ensure HTML is empty
bodyHTML := req.BodyHTML
if req.IsWarmup {
internal/app/worker/wmail/start_sync.go
+12 -4
View File
@@ -78,20 +78,26 @@ func (w *WMail) nextSyncDelay(base time.Duration, last *errx.MailError) time.Dur
// so one mailbox's bad server response must not take down every other
// account's sync and send loops.
func (w *WMail) syncOnce(ctx context.Context) (result *errx.MailError) {
if w.tracker != nil {
w.tracker.tickComplete = false
}
defer func() {
if r := recover(); r != nil {
if w.tracker != nil {
w.tracker.tickComplete = false
}
err := fmt.Errorf("mail sync panic: %v", r)
w.CaptureError(err)
log.Error().Err(err).Str("email_id", w.ID.String()).Msg("mail sync panicked")
result = &errx.MailError{Code: errx.MailErrorCodeImapUnknown, Type: errx.MailErrorWarning, Message: "mail sync pass interrupted"}
}
}()
if err := w.SyncMail(ctx); err != nil {
// A server that is down answers every pass the same way. Report the
// first one and then stay quiet until it comes back, so one outage is
// one warning in the drawer rather than one a minute.
// Keep observations fresh; the consumer deduplicates user notifications.
if isTransportError(err) {
w.transportFailures++
if w.transportFailures > 1 {
w.CaptureError(err)
log.Debug().Err(err).Str("email_id", w.ID.String()).Int("consecutive", w.transportFailures).Msg("mail server still unreachable")
return err
}
@@ -100,7 +106,9 @@ func (w *WMail) syncOnce(ctx context.Context) (result *errx.MailError) {
log.Warn().Err(err).Str("email_id", w.ID.String()).Msg("mail sync error")
return err
}
w.transportFailures = 0
if w.tracker != nil && w.tracker.tickComplete {
w.transportFailures = 0
}
return nil
}
internal/app/worker/wmail/sync_google.go
+14 -3
View File
@@ -29,7 +29,8 @@ func (w *WMail) SyncGoogle(ctx context.Context) *errx.MailError {
stats := &tickStats{}
w.googleTick = stats
w.googleFolders = nil
defer w.endTick(stats)
complete := false
defer func() { w.endTick(ctx, stats, complete) }()
if !w.retryUnmap(ctx) {
return nil
}
@@ -40,7 +41,7 @@ func (w *WMail) SyncGoogle(ctx context.Context) *errx.MailError {
return nil
}
newHistoryID, err := w.GoogleData.Client.FetchHistory(ctx, w.GoogleData.LastHistoryID)
newHistoryID, caughtUp, err := w.GoogleData.Client.FetchHistoryPass(ctx, w.GoogleData.LastHistoryID)
if errors.Is(err, goog.ErrHistoryExpired) {
baseline, berr := w.GoogleData.Client.HistoryBaseline(ctx)
if berr != nil {
@@ -81,10 +82,11 @@ func (w *WMail) SyncGoogle(ctx context.Context) *errx.MailError {
if !stats.aborted {
// The pass's own work is done; only a refusal the owner has to act
// on is worth failing it for.
if merr := w.googleReconcileFolders(ctx, time.Now(), stats); merr != nil && merr.Type == errx.MailErrorCritical {
if merr := w.googleReconcileFolders(ctx, time.Now(), stats); merr != nil {
return merr
}
}
complete = caughtUp
return nil
}
@@ -337,7 +339,13 @@ func (w *WMail) googleBackfill(ctx context.Context, stats *tickStats) *errx.Mail
return nil
}
ids, next, err := w.GoogleData.Client.ListMessages(ctx, q, st.BackfillCursor.PageToken, googleBackfillPage)
if errors.Is(err, goog.ErrPageTokenExpired) {
st.BackfillCursor.PageToken = ""
w.tracker.mark()
return nil
}
if err != nil {
stats.aborted = true
var errMail *errx.MailError
if errors.As(err, &errMail) {
return errMail
@@ -364,6 +372,7 @@ func (w *WMail) googleBackfill(ctx context.Context, stats *tickStats) *errx.Mail
}
msg, err := w.GoogleData.Client.GetMessage(ctx, id)
if err != nil {
stats.aborted = true
var errMail *errx.MailError
if errors.As(err, &errMail) {
return errMail
@@ -436,6 +445,7 @@ func (w *WMail) googleReconcileFolders(ctx context.Context, now time.Time, stats
for page := 0; page < config.GmailFolderReconcilePages; page++ {
ids, next, err := w.GoogleData.Client.ListLabelMessages(ctx, goog.Inbox, q, token, 500)
if err != nil {
stats.aborted = true
return w.googleReconcileError(err)
}
for _, id := range ids {
@@ -479,6 +489,7 @@ func (w *WMail) googleReconcileFolders(ctx context.Context, now time.Time, stats
labels, found, err := w.GoogleData.Client.MessageLabels(ctx, m.ProviderID)
if err != nil {
if gmailRetryable(err) {
stats.aborted = true
return w.googleReconcileError(err)
}
// One message Gmail refuses must not hold up every row after it.
internal/app/worker/wmail/sync_google_test.go
+203
View File
@@ -2,9 +2,12 @@ package wmail
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"reflect"
"slices"
"strings"
"testing"
"time"
@@ -12,6 +15,7 @@ import (
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/client/goog"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/repository"
"golang.org/x/oauth2"
)
@@ -154,3 +158,202 @@ func TestGoogleBackfillRetriesAfterATransientFailure(t *testing.T) {
t.Errorf("listed %d times, want the one call that succeeded", g.listCalls)
}
}
type googleBackfillBudget struct {
fixedBudget
policy models.SyncPolicy
lanes []SyncLane
}
func (b *googleBackfillBudget) Policy() models.SyncPolicy { return normalizePolicy(b.policy) }
func (b *googleBackfillBudget) Admit(ctx context.Context, lane SyncLane) Admission {
b.lanes = append(b.lanes, lane)
return b.fixedBudget.Admit(ctx, lane)
}
func TestGoogleBackfillExpiredPageResumesAfterReloadWithinWindowAndBudgets(t *testing.T) {
var tokens, gets []string
since := time.Date(2026, 9, 1, 12, 0, 0, 0, time.UTC)
started := since.Add(30 * 24 * time.Hour)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if strings.HasSuffix(r.URL.Path, "/messages") {
token := r.URL.Query().Get("pageToken")
tokens = append(tokens, token)
if q := r.URL.Query().Get("q"); q != fmt.Sprintf("after:%d -in:trash -in:spam -in:chats", since.Unix()) {
t.Errorf("import window changed: q=%q", q)
}
if r.URL.Query().Get("includeSpamTrash") != "false" || r.URL.Query().Get("maxResults") != "100" {
t.Errorf("import listing changed: %s", r.URL)
}
switch token {
case "expired":
w.WriteHeader(http.StatusBadRequest)
_, _ = w.Write([]byte(`{"error":{"code":400,"message":"Invalid page token"}}`))
case "":
_, _ = w.Write([]byte(`{"messages":[{"id":"known"},{"id":"new-1"},{"id":"new-2"}],"nextPageToken":"page-2"}`))
case "page-2":
_, _ = w.Write([]byte(`{"messages":[{"id":"new-3"},{"id":"over-cap"}]}`))
default:
t.Errorf("unexpected token %q", token)
}
return
}
id := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]
gets = append(gets, id)
_, _ = fmt.Fprintf(w, `{"id":%q,"threadId":"t-%s","payload":{"headers":[{"name":"Message-Id","value":"<%s@gmail.test>"}]}}`, id, id, id)
}))
t.Cleanup(srv.Close)
var events []captured
w := newGoogleTestMail(t, srv, &events)
budget := &googleBackfillBudget{fixedBudget: fixedBudget{allow: 1}, policy: models.SyncPolicy{BackfillDays: 2, BackfillMessages: 4}}
w.gov = budget
mapping := &recoveryMessageMap{data: map[string]repository.EmailMessageData{"known": {ID: uuid.NewString(), MessageID: "known"}}}
w.EmailMessageMapRepository = mapping
w.tracker.state = models.SyncState{
BackfillStatus: models.SyncBackfillRunning, BackfillSynced: 1,
BackfillSince: &since, BackfillStartedAt: &started,
BackfillCursor: models.SyncCursor{PageToken: "expired", Folders: map[string]models.SyncFolderCursor{}},
Deferred: 5, LastSyncedAt: &started,
}
reload := func(state models.SyncState) {
t.Helper()
encoded, err := json.Marshal(state)
if err != nil {
t.Fatal(err)
}
var seed models.SyncState
if err := json.Unmarshal(encoded, &seed); err != nil {
t.Fatal(err)
}
id, userID := w.ID, w.UserID
w = newGoogleTestMail(t, srv, &events)
w.ID, w.UserID = id, userID
w.gov, w.EmailMessageMapRepository = budget, mapping
w.tracker = newSyncTracker(&seed, func(models.SyncState) error { return nil })
}
reload(w.tracker.state)
want := w.tracker.state
want.BackfillCursor.PageToken = ""
if err := w.googleBackfill(t.Context(), &tickStats{}); err != nil {
t.Fatalf("expired continuation must reset for retry: %v", err)
}
if !reflect.DeepEqual(w.tracker.state, want) || !w.tracker.dirty || budget.admitted != 0 || len(events) != 0 || !slices.Equal(tokens, []string{"expired"}) {
t.Fatalf("reset changed more than the continuation: state=%+v tokens=%v budget=%+v", w.tracker.state, tokens, budget)
}
var relayed models.SyncState
w.tracker.emit = func(state models.SyncState) error { relayed = state; return nil }
w.tracker.flush(time.Now())
reload(relayed)
if err := w.googleBackfill(t.Context(), &tickStats{}); err != nil {
t.Fatal(err)
}
if w.tracker.state.BackfillSynced != 2 || w.tracker.state.BackfillCursor.PageToken != "" || w.tracker.state.BackfillStatus != models.SyncBackfillRunning {
t.Fatalf("pacing must pin the restarted page: %+v", w.tracker.state)
}
reload(w.tracker.state)
if err := w.googleBackfill(t.Context(), &tickStats{}); err != nil {
t.Fatal(err)
}
if len(newEmails(events)) != 1 || budget.admitted != 1 {
t.Fatal("reload bypassed the admission budget or duplicated an arrival")
}
budget.allow = 3
if err := w.googleBackfill(t.Context(), &tickStats{}); err != nil {
t.Fatal(err)
}
if w.tracker.state.BackfillStatus != models.SyncBackfillComplete || w.tracker.state.BackfillSynced != 4 || !w.tracker.state.BackfillSince.Equal(since) || !w.tracker.state.BackfillStartedAt.Equal(started) {
t.Fatalf("import changed the fixed window or count: %+v", w.tracker.state)
}
if !slices.Equal(gets, []string{"new-1", "new-2", "new-3"}) || len(newEmails(events)) != 3 || len(mapping.data) != 4 || budget.admitted != 3 || budget.observed != 0 {
t.Fatalf("known mail or capped mail was imported: gets=%v arrivals=%d budget=%+v", gets, len(newEmails(events)), budget)
}
for _, lane := range budget.lanes {
if lane != LaneBackfill {
t.Fatalf("import charged %s instead of backfill", lane)
}
}
reload(w.tracker.state)
if err := w.googleBackfill(t.Context(), &tickStats{}); err != nil {
t.Fatal(err)
}
if !slices.Equal(tokens, []string{"expired", "", "", "", "page-2"}) || len(newEmails(events)) != 3 {
t.Fatalf("completed reload repeated the import: tokens=%v arrivals=%d", tokens, len(newEmails(events)))
}
}
func TestGoogleBackfillRetainsContinuationOnOtherProviderErrors(t *testing.T) {
for _, tc := range []struct {
name, message string
code int
}{
{"invalid query", "Invalid query", 400},
{"quota", "Invalid page token", 429},
{"auth", "Invalid page token", 401},
{"forbidden", "Invalid page token", 403},
{"unavailable", "Invalid page token", 503},
} {
t.Run(tc.name, func(t *testing.T) {
calls := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
if r.URL.Query().Get("pageToken") != "keep" {
t.Error("provider error reset the continuation")
}
w.WriteHeader(tc.code)
_, _ = fmt.Fprintf(w, `{"error":{"code":%d,"message":%q}}`, tc.code, tc.message)
}))
t.Cleanup(srv.Close)
var events []captured
w := newGoogleTestMail(t, srv, &events)
w.tracker.startBackfill(time.Now(), 30)
w.tracker.state.BackfillCursor.PageToken = "keep"
w.tracker.state.BackfillSynced = 7
want := w.tracker.state
for range 2 {
stats := &tickStats{}
if err := w.googleBackfill(t.Context(), stats); err == nil || !stats.aborted {
t.Fatalf("provider error was swallowed: err=%v aborted=%t", err, stats.aborted)
}
if !reflect.DeepEqual(w.tracker.state, want) || len(events) != 0 {
t.Fatalf("provider failure changed import progress: %+v", w.tracker.state)
}
}
if calls != 2 {
t.Fatalf("requests=%d, want one per attempt", calls)
}
})
}
}
func TestGoogleBackfillCompletedOrCappedImportDoesNotRestartExpiredPage(t *testing.T) {
for _, tc := range []struct {
name string
status models.SyncBackfillStatus
synced int
}{
{"completed below cap", models.SyncBackfillComplete, 1},
{"running at cap", models.SyncBackfillRunning, 4},
} {
t.Run(tc.name, func(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {
t.Error("completed or capped import must not re-list an expired page")
}))
t.Cleanup(srv.Close)
var events []captured
w := newGoogleTestMail(t, srv, &events)
w.gov = &googleBackfillBudget{fixedBudget: fixedBudget{allow: 10}, policy: models.SyncPolicy{BackfillMessages: 4}}
w.tracker.startBackfill(time.Now(), 30)
w.tracker.state.BackfillStatus = tc.status
w.tracker.state.BackfillSynced = tc.synced
w.tracker.state.BackfillCursor.PageToken = "expired"
if err := w.googleBackfill(t.Context(), &tickStats{}); err != nil {
t.Fatal(err)
}
if w.tracker.state.BackfillStatus != models.SyncBackfillComplete || w.tracker.state.BackfillSynced != tc.synced || w.tracker.state.BackfillCursor.PageToken != "expired" || len(events) != 0 {
t.Fatalf("completed import changed: %+v", w.tracker.state)
}
})
}
}
internal/app/worker/wmail/sync_graph.go
+84 -8
View File
@@ -10,6 +10,7 @@ import (
"github.com/warmbly/warmbly/internal/client/msgraph"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/repository"
)
// graphBackfillPage is $top for one backfill listing. Messages come back
@@ -25,11 +26,37 @@ func (w *WMail) SyncGraph(ctx context.Context) *errx.MailError {
w.beginTick()
stats := &tickStats{}
w.graphTick = stats
complete := false
defer func() { w.endTick(ctx, stats, complete) }()
if !w.retryUnmap(ctx) {
return nil
}
w.tracker.startBackfill(time.Now(), w.gov.Policy().BackfillDays)
w.GraphData.Client.OnRecoveryStart = w.graphRecoveryStart
w.GraphData.Client.OnRecoveryCheckpoint = w.graphRecoveryCheckpoint
w.GraphData.Client.OnRecoveryBaseline = w.graphRecoveryBaseline
w.GraphData.Client.PendingRecoveryLink = w.graphPendingRecoveryLink
w.GraphData.Client.Recovering = w.graphRecovering
if w.SyncContext != nil && w.GraphData.Client.ImmutableIDMode() {
done, err := w.graphUpgradeIDs(ctx)
if errors.Is(err, repository.ErrSyncContextUnsupported) {
w.GraphData.Client.SetImmutableIDMode(false)
w.CaptureError(err)
done, err = true, nil
}
if err != nil {
return w.graphSyncError(err)
}
if !done {
return nil
}
if w.GraphData.Client.ImmutableIDMode() {
w.GraphData.Client.OnFolderReconcile = w.graphReconcileFolder
}
}
if err := w.GraphData.Client.Sync(ctx); err != nil {
caughtUp, err := w.GraphData.Client.SyncPass(ctx)
if err != nil {
var mailErr *errx.MailError
if errors.As(err, &mailErr) {
return mailErr
@@ -42,7 +69,7 @@ func (w *WMail) SyncGraph(ctx context.Context) *errx.MailError {
return merr
}
}
w.endTick(stats)
complete = caughtUp
return nil
}
@@ -58,19 +85,29 @@ func (w *WMail) onGraphMessageSeen(ctx context.Context, folder, providerID strin
if stats.aborted {
return false, msgraph.ErrStop
}
known, err := w.EmailMessageMapRepository.Get(ctx, w.UserID, w.ID, providerID)
known, err := w.graphMessageMap(ctx, providerID)
if err != nil {
w.controlPlaneError(err, stats)
return false, msgraph.ErrStop
}
if known != nil {
return true, w.onGraphFlagsChange(ctx, providerID, seen)
}
if w.observeLive(ctx, []string{providerID}, stats) {
return false, msgraph.ErrStop
if !w.GraphData.Client.ImmutableIDMode() {
if err := w.onGraphFlagsChange(ctx, providerID, seen); err != nil {
return false, err
}
id, err := uuid.Parse(known.ID)
if err != nil {
return false, err
}
return true, w.emitFolder(id, (&msgraph.GraphMessage{}).ToEmailData(folder).Folder)
}
return true, w.graphReconcileMessage(ctx, known.ID, providerID, folder)
}
lane, cached := w.laneCache.get(providerID)
if (msgraph.IsRecovery(ctx) && lane == LaneLive) || (!msgraph.IsRecovery(ctx) && lane == LaneBackfill) {
cached = false
}
var msg *models.EmailMessageData
if !cached {
full, err := w.GraphData.Client.FetchMessage(ctx, folder, providerID)
@@ -81,8 +118,34 @@ func (w *WMail) onGraphMessageSeen(ctx context.Context, folder, providerID strin
return true, nil // gone between the delta item and now
}
msg = full.ToEmailData(folder)
lane = w.laneOf(ctx, providerID, msg, false)
lane = w.laneFor(ctx, msg, false)
if msgraph.IsRecovery(ctx) && lane != LanePriority {
recovery, err := w.graphRecovery(folder)
if err != nil {
return false, err
}
if (!msg.InternalDate.IsZero() && msg.InternalDate.Before(recovery.Since)) || recovery.Count >= w.gov.Policy().BackfillMessages {
return true, nil
}
lane = LaneBackfill
}
}
if lane == LaneBackfill && w.tracker.state.BackfillStatus != models.SyncBackfillComplete && w.tracker.state.BackfillSynced >= w.gov.Policy().BackfillMessages {
return true, nil
}
if lane == LaneBackfill && msgraph.IsRecovery(ctx) {
recovery, err := w.graphRecovery(folder)
if err != nil {
return false, err
}
if recovery.Count >= w.gov.Policy().BackfillMessages {
return true, nil
}
}
if lane == LaneLive && w.observeLive(ctx, []string{providerID}, stats) {
return false, msgraph.ErrStop
}
w.laneCache.put(providerID, lane)
if !w.admit(ctx, lane, stats) {
return false, nil
}
@@ -101,6 +164,18 @@ func (w *WMail) onGraphMessageSeen(ctx context.Context, folder, providerID strin
w.controlPlaneError(err, stats)
return false, msgraph.ErrStop
}
if lane == LaneBackfill {
if w.tracker.state.BackfillStatus != models.SyncBackfillComplete {
w.tracker.state.BackfillSynced++
w.tracker.mark()
}
recovery, err := w.graphRecovery(folder)
if err != nil {
return false, err
}
recovery.Count++
w.setGraphRecovery(folder, recovery)
}
return true, nil
}
@@ -186,6 +261,7 @@ func (w *WMail) graphBackfill(ctx context.Context, stats *tickStats) *errx.MailE
return mailErr
}
w.CaptureError(err)
stats.aborted = true
return nil
}
for _, full := range msgs {
internal/app/worker/wmail/sync_graph_reconcile.go
+275
View File
@@ -0,0 +1,275 @@
package wmail
import (
"context"
"encoding/json"
"errors"
"time"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/client/msgraph"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/repository"
)
const graphIdentityCursor = "graph:immutable-ids"
type graphDeltaRecovery struct {
Version int `json:"version"`
Since time.Time `json:"since"`
Count int `json:"count"`
Baseline string `json:"baseline,omitempty"`
PendingLink string `json:"pending_link,omitempty"`
}
func (w *WMail) graphRecovery(folder string) (graphDeltaRecovery, error) {
var recovery graphDeltaRecovery
data := w.tracker.folder("graph:delta-recovery:" + folder).Next
if data == "" {
return recovery, nil
}
if err := json.Unmarshal([]byte(data), &recovery); err != nil {
return recovery, err
}
if recovery.Version != 1 || recovery.Since.IsZero() || recovery.Count < 0 {
return recovery, errors.New("graph: unsupported recovery state")
}
return recovery, nil
}
func (w *WMail) setGraphRecovery(folder string, recovery graphDeltaRecovery) {
data, _ := json.Marshal(recovery)
w.tracker.setFolder("graph:delta-recovery:"+folder, models.SyncFolderCursor{Next: string(data)})
}
func (w *WMail) graphRecovering(folder string) bool {
recovery, err := w.graphRecovery(folder)
if err != nil {
return true
}
key := "graph:delta-recovery:" + folder
cur := w.tracker.folder(key)
if recovery.Version == 0 || cur.Done {
return false
}
if recovery.Baseline != "" && recovery.Baseline == w.GraphData.Client.DeltaLinks[folder] {
cur.Done = true
w.tracker.setFolder(key, cur)
return false
}
return true
}
func (w *WMail) graphPendingRecoveryLink(folder string) (string, error) {
recovery, err := w.graphRecovery(folder)
if err != nil {
return "", err
}
if w.graphRecovering(folder) {
return recovery.PendingLink, nil
}
return "", nil
}
func (w *WMail) graphRecoveryBaseline(ctx context.Context, folder, link string) error {
recovery, err := w.graphRecovery(folder)
if err != nil {
return err
}
recovery.PendingLink = link
w.setGraphRecovery(folder, recovery)
return w.tracker.emit(w.tracker.state)
}
func (w *WMail) graphRecoveryStart(ctx context.Context, folder string) error {
recovery, err := w.graphRecovery(folder)
if err != nil {
return err
}
if recovery.Version == 0 || !w.graphRecovering(folder) {
since := time.Now().Add(-time.Duration(w.gov.Policy().BackfillDays) * 24 * time.Hour)
if w.tracker.state.BackfillStatus != models.SyncBackfillComplete && w.tracker.state.BackfillSince != nil {
since = *w.tracker.state.BackfillSince
}
recovery = graphDeltaRecovery{Version: 1, Since: since}
}
recovery.Baseline = ""
w.setGraphRecovery(folder, recovery)
w.tracker.clearFolder("graph:reconcile:" + folder)
return w.tracker.emit(w.tracker.state)
}
func (w *WMail) graphRecoveryCheckpoint(ctx context.Context, folder, link string) error {
recovery, err := w.graphRecovery(folder)
if err != nil {
return err
}
recovery.Baseline = link
w.setGraphRecovery(folder, recovery)
return w.tracker.emit(w.tracker.state)
}
func (w *WMail) graphMessageMap(ctx context.Context, providerID string) (*repository.EmailMessageData, error) {
known, err := w.EmailMessageMapRepository.Get(ctx, w.UserID, w.ID, providerID)
if err != nil || known != nil || !w.GraphData.Client.ImmutableIDMode() {
return known, err
}
ids, err := w.GraphData.Client.RegularMessageIDs(ctx, []string{providerID})
if err != nil {
return nil, err
}
known, err = w.EmailMessageMapRepository.Get(ctx, w.UserID, w.ID, ids[providerID])
if err != nil || known == nil {
return known, err
}
alias := *known
alias.MessageID = providerID
if err := w.graphAddAlias(ctx, alias); err != nil {
return nil, err
}
return &alias, nil
}
func (w *WMail) graphAddAlias(ctx context.Context, alias repository.EmailMessageData) error {
existing, err := w.EmailMessageMapRepository.Get(ctx, w.UserID, w.ID, alias.MessageID)
if err != nil {
return err
}
if existing != nil {
if existing.ID != alias.ID {
return errors.New("graph: conflicting immutable message mapping")
}
return nil
}
if err := w.EmailMessageMapRepository.Add(ctx, alias); err != nil {
return err
}
stored, err := w.EmailMessageMapRepository.Get(ctx, w.UserID, w.ID, alias.MessageID)
if err != nil {
return err
}
if stored == nil || stored.ID != alias.ID {
return errors.New("graph: immutable message mapping not retained")
}
return nil
}
func (w *WMail) graphSyncError(err error) *errx.MailError {
var merr *errx.MailError
if errors.As(err, &merr) {
return merr
}
w.CaptureError(err)
return nil
}
func (w *WMail) graphStoredPage(ctx context.Context, key string) ([]repository.ProviderFolderMessage, error) {
var after *uuid.UUID
if next := w.tracker.folder(key).Next; next != "" {
id, err := uuid.Parse(next)
if err != nil {
return nil, err
}
after = &id
}
return w.SyncContext.ListProviderMessages(ctx, w.UserID, w.ID, after, graphBackfillPage)
}
func (w *WMail) graphUpgradeIDs(ctx context.Context) (bool, error) {
if w.tracker.folder(graphIdentityCursor).Done {
return true, nil
}
rows, err := w.graphStoredPage(ctx, graphIdentityCursor)
if err != nil {
return false, err
}
ids := make([]string, 0, len(rows))
for _, row := range rows {
ids = append(ids, row.ProviderID)
}
var converted map[string]string
if len(ids) > 0 {
converted, err = w.GraphData.Client.ImmutableMessageIDs(ctx, ids)
if err != nil {
return false, err
}
}
cur := w.tracker.folder(graphIdentityCursor)
for _, row := range rows {
id := converted[row.ProviderID]
known, err := w.EmailMessageMapRepository.Get(ctx, w.UserID, w.ID, row.ProviderID)
if err != nil {
return false, err
}
if known == nil {
return false, errors.New("graph: saved message mapping missing during id upgrade")
}
alias := *known
alias.MessageID = id
if err := w.graphAddAlias(ctx, alias); err != nil {
return false, err
}
// Relayed placement updates only the provider id, preserving customer filing.
if err := w.onEvent(models.JobEventTypeFolderUpdate, &models.JobEventFolderUpdate{
UserID: w.UserID, EmailID: w.ID, ID: row.ID, ProviderID: id,
Folder: row.ProviderFolder, Relayed: true,
}); err != nil {
return false, err
}
cur.Next = row.ID.String()
w.tracker.setFolder(graphIdentityCursor, cur)
}
cur.Done = len(rows) < graphBackfillPage
w.tracker.setFolder(graphIdentityCursor, cur)
w.tracker.flush(time.Now())
return cur.Done, nil
}
func (w *WMail) graphReconcileMessage(ctx context.Context, internalID, providerID, fallback string) error {
id, err := uuid.Parse(internalID)
if err != nil {
return err
}
full, err := w.GraphData.Client.FetchMessage(ctx, fallback, providerID)
if err != nil {
return err
}
if full == nil {
return w.onEvent(models.JobEventTypeRemoveEmail, &models.JobEventRemoveEmail{UserID: w.UserID, EmailID: w.ID, ID: id})
}
folder, err := w.GraphData.Client.MessageFolder(ctx, full, fallback)
if err != nil {
return err
}
if folder != "" {
if err := w.emitFolder(id, folder); err != nil {
return err
}
}
return w.onGraphFlagsChange(ctx, providerID, full.IsRead)
}
func (w *WMail) graphReconcileFolder(ctx context.Context, folder string) (bool, error) {
key := "graph:reconcile:" + folder
rows, err := w.graphStoredPage(ctx, key)
if err != nil {
return false, err
}
canonical := (&msgraph.GraphMessage{}).ToEmailData(folder).Folder
cur := w.tracker.folder(key)
for _, row := range rows {
if row.ProviderFolder == canonical {
if err := w.graphReconcileMessage(ctx, row.ID.String(), row.ProviderID, folder); err != nil {
return false, err
}
}
cur.Next = row.ID.String()
w.tracker.setFolder(key, cur)
}
done := len(rows) < graphBackfillPage
if done {
w.tracker.clearFolder(key)
}
return done, nil
}
internal/app/worker/wmail/sync_graph_test.go
+1 -2
View File
@@ -156,8 +156,7 @@ func newGraphTestMail(t *testing.T, srv *httptest.Server, events *[]captured, re
return nil
})
// Seeded cursors: a folder with no delta link is primed, not imported, so
// without these the live half of the pass would never run.
// Seeded cursors model a mailbox already following live deltas.
deltaLinks := map[string]string{}
for _, folder := range msgraph.TrackedFolders {
deltaLinks[folder] = "https://graph.microsoft.com/v1.0/me/mailFolders/" + folder + "/messages/delta?$deltatoken=seed"
internal/app/worker/wmail/sync_imap.go
+115 -38
View File
@@ -32,6 +32,8 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError {
}
w.beginTick()
stats := &tickStats{}
complete := false
defer func() { w.endTick(ctx, stats, complete) }()
if !w.retryUnmap(ctx) {
return nil
}
@@ -74,13 +76,12 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError {
return nil
}
// condStore decides the incremental strategy for the whole account:
// mod-sequences where the server has CONDSTORE, UIDNEXT where it does not
// (Outlook.com, Microsoft 365 over IMAP, Yahoo, many hosted servers).
condStore := client.HasCondStore()
// CONDSTORE is effective per folder; NOMODSEQ uses UIDNEXT and periodic flags.
caughtUp := true
for i := range folders {
box := &folders[i]
condStore := client.HasCondStore() && box.HighestModSeq > 0
// The listing is remembered before anything is decided from it, so
// the departure check below reads the previous pass, never this one.
prevListing, listedBefore := w.listed[box.Name]
@@ -90,6 +91,10 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError {
// First sight: baseline. Live sync starts from this cursor; the
// backfill owns everything before it.
saved := *box
if !w.imapRecoverFolder(box) {
stats.aborted = true
return nil
}
if err := w.mboxEvent(&saved); err != nil {
return nil
}
@@ -102,20 +107,35 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError {
// snapshot is about messages that may no longer be there. Re-baseline
// it exactly like a first sighting, and drop its backfill floor so an
// import still running walks it again (stored messages are matched by
// Message-ID, so nothing is stored twice). A finished import stays
// finished; the mail that is already here keeps its rows, and its
// stale UIDs are what the warmup path checks the generation against.
// Message-ID, so nothing is stored twice). Completed accounts schedule
// a separate bounded folder recovery without resetting global history.
if befBox.UIDValidity != box.UIDValidity {
saved := *box
if !w.imapRecoverFolder(box) {
stats.aborted = true
return nil
}
if err := w.mboxEvent(&saved); err != nil {
return nil
}
*befBox = saved
delete(w.flagScan, box.Name)
w.tracker.setFolder(box.Name, models.SyncFolderCursor{})
continue
}
var selected *imap.Selected
if condStore && !stats.aborted {
view, err := client.SelectForSyncState(box.Name)
if err != nil {
return err
}
selected = &view
condStore = view.HighestModSeq > 0
if view.UIDValidity != 0 && view.UIDValidity != box.UIDValidity {
caughtUp = false
continue
}
}
changed := imapFolderChanged(befBox, box, condStore)
// A pass cut short by the search cap left rows unexamined, so the
// next pass looks again whether or not the count moved.
@@ -125,17 +145,21 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError {
var view imap.Selected
if changed && !stats.aborted {
w.setWalking(box)
done, sel, ids, err := w.imapIncremental(ctx, box, befBox, condStore, stats)
done, sel, ids, err := w.imapIncremental(ctx, box, befBox, condStore, stats, selected)
if err != nil {
return err
}
fullyProcessed = done
view = sel
condStore = condStore && view.HighestModSeq > 0
touched = ids
} else if changed {
// The pass was aborted before this folder; hold its cursor too.
fullyProcessed = false
}
if !fullyProcessed {
caughtUp = false
}
if changed || !slices.Equal(befBox.Attrs, box.Attrs) {
// The stored cursor only moves once every change up to it was
@@ -233,12 +257,15 @@ outer:
}
if !stats.aborted {
if err := w.imapRecoverFolders(ctx, folders, stats); err != nil {
return err
}
if err := w.imapBackfill(ctx, folders, stats); err != nil {
return err
}
}
w.endTick(stats)
complete = caughtUp && !w.imapRecoveryPending(folders)
return nil
}
@@ -455,7 +482,7 @@ const imapSkipSearchesPerPass = 50
// AND flag changes; without it only arrivals are visible here, and flag
// changes are picked up by the periodic scan in imapIncremental.
func imapFolderChanged(before, now *models.Mailbox, condStore bool) bool {
if condStore {
if condStore && before.HighestModSeq > 0 && now.HighestModSeq > 0 {
return before.HighestModSeq != now.HighestModSeq
}
return before.UIDNext != now.UIDNext
@@ -467,11 +494,17 @@ func imapFolderChanged(before, now *models.Mailbox, condStore bool) bool {
// folder's cursor advance, the selected view the search ran against, which is
// where it advances to, and the Message-IDs it fetched so the drafts
// reconciliation can tell a re-appended draft from an expunged one.
func (w *WMail) imapIncremental(ctx context.Context, box, before *models.Mailbox, condStore bool, stats *tickStats) (bool, imap.Selected, map[string]struct{}, *errx.MailError) {
func (w *WMail) imapIncremental(ctx context.Context, box, before *models.Mailbox, condStore bool, stats *tickStats, selected *imap.Selected) (bool, imap.Selected, map[string]struct{}, *errx.MailError) {
client := w.SmtpImapData.ImapClient
view, err := client.SelectForSyncState(box.Name)
if err != nil {
return false, view, nil, err
var view imap.Selected
var err *errx.MailError
if selected != nil {
view = *selected
} else {
view, err = client.SelectForSyncState(box.Name)
if err != nil {
return false, view, nil, err
}
}
// The listing and this view name different generations, so the search
// would answer about UIDs the cursor does not; the next pass re-baselines.
@@ -482,7 +515,7 @@ func (w *WMail) imapIncremental(ctx context.Context, box, before *models.Mailbox
return true, view, nil, nil
}
var uids []goimap.UID
if condStore {
if condStore && before.HighestModSeq > 0 && view.HighestModSeq > 0 {
uids, err = client.SearchChangedSince(before.HighestModSeq)
} else {
uids, err = client.SearchNewSince(before.UIDNext)
@@ -509,7 +542,7 @@ func (w *WMail) imapIncremental(ctx context.Context, box, before *models.Mailbox
if err != nil {
return false, view, touched, err
}
done, err := w.imapApply(ctx, fetched, false, stats)
done, err := w.imapApply(ctx, fetched, false, stats, nil)
if err != nil {
return false, view, touched, err
}
@@ -608,22 +641,37 @@ func (w *WMail) imapReconcileDrafts(ctx context.Context, box *models.Mailbox, to
// unknown ones are stored if their lane admits them. backfill selects the
// backfill lane and skips flood accounting. Returns whether every unknown
// message in the batch was stored.
func (w *WMail) imapApply(ctx context.Context, fetched []*imap.Fetched, backfill bool, stats *tickStats) (bool, *errx.MailError) {
func (w *WMail) imapApply(ctx context.Context, fetched []*imap.Fetched, backfill bool, stats *tickStats, recovery *imapFolderRecovery) (bool, *errx.MailError) {
sort.Slice(fetched, func(i, j int) bool { return fetched[i].Email.UID > fetched[j].Email.UID })
var fresh []*imap.Fetched
for _, f := range fetched {
if ctx.Err() != nil || stats.aborted {
return false, nil
}
w.ensureMessageKey(f.Email)
internal, err := w.EmailMessageMapRepository.Get(ctx, w.UserID, w.ID, f.Email.MessageID)
if err != nil {
return false, w.controlPlaneError(err, stats)
}
if internal == nil {
fresh = append(fresh, f)
if recovery != nil && recovery.KnownOnly {
w.imapImportCursor(w.SmtpImapData.folderPath, f.Email.UID, false, recovery)
continue
}
if recovery == nil {
fresh = append(fresh, f)
continue
}
done, err := w.imapStoreFetched(ctx, f, backfill, stats, recovery)
if err != nil || !done {
return false, err
}
w.imapImportCursor(w.SmtpImapData.folderPath, f.Email.UID, false, recovery)
continue
}
if backfill {
// The backfill only cares about what it has not stored.
if backfill && recovery == nil {
// Initial history skips known maps; generation recovery also reconciles them.
continue
}
internalID, perr := uuid.Parse(internal.ID)
@@ -643,6 +691,9 @@ func (w *WMail) imapApply(ctx context.Context, fetched []*imap.Fetched, backfill
}); err != nil {
return false, w.controlPlaneError(err, stats)
}
if recovery != nil {
w.imapImportCursor(w.SmtpImapData.folderPath, f.Email.UID, false, recovery)
}
}
if !backfill && len(fresh) > 0 {
@@ -655,36 +706,58 @@ func (w *WMail) imapApply(ctx context.Context, fetched []*imap.Fetched, backfill
}
}
policy := w.gov.Policy()
all := true
for _, f := range fresh {
if stats.aborted {
return false, nil
done, err := w.imapStoreFetched(ctx, f, backfill, stats, recovery)
if err != nil {
return false, err
}
if backfill && w.tracker.state.BackfillSynced >= policy.BackfillMessages {
return false, nil
}
if !w.admit(ctx, w.laneOf(ctx, f.Email.MessageID, f.Email, backfill), stats) {
if !done {
all = false
if backfill {
if backfill || stats.aborted || ctx.Err() != nil {
return false, nil
}
continue
}
w.SmtpImapData.ImapClient.FetchBody(f)
if err := w.imapStore(ctx, f.Email); err != nil {
return false, w.controlPlaneError(err, stats)
}
w.laneCache.forget(f.Email.MessageID)
if backfill {
w.tracker.state.BackfillSynced++
w.tracker.mark()
w.tracker.setFolder(w.SmtpImapData.folderPath, models.SyncFolderCursor{UID: f.Email.UID})
w.imapImportCursor(w.SmtpImapData.folderPath, f.Email.UID, false, recovery)
}
}
return all, nil
}
func (w *WMail) imapStoreFetched(ctx context.Context, f *imap.Fetched, backfill bool, stats *tickStats, recovery *imapFolderRecovery) (bool, *errx.MailError) {
if stats.aborted || ctx.Err() != nil {
return false, nil
}
if backfill && w.imapImportCount(recovery) >= w.gov.Policy().BackfillMessages {
// A capped initial import still reconciles known handles, without admitting more history.
return recovery != nil && recovery.Initial, nil
}
if !w.admit(ctx, w.laneOf(ctx, f.Email.MessageID, f.Email, backfill), stats) {
return false, nil
}
if err := w.SmtpImapData.ImapClient.FetchBody(f); err != nil {
if err.Code == errx.MailErrorCodeNotFound {
return true, nil
}
return false, err
}
if err := w.imapStore(ctx, f.Email); err != nil {
return false, w.controlPlaneError(err, stats)
}
w.laneCache.forget(f.Email.MessageID)
if backfill {
if recovery != nil && !recovery.Initial {
recovery.Synced++
} else {
w.tracker.state.BackfillSynced++
w.tracker.mark()
}
}
return true, nil
}
// ensureMessageKey gives a message without a Message-ID header one that is
// stable for this mailbox, because the empty string is not a key: the map
// endpoint refuses it with 400 and the failed lookup ends the whole sync pass
@@ -824,6 +897,10 @@ func (w *WMail) imapBackfill(ctx context.Context, folders []models.Mailbox, stat
}
key := box.Name
cur := w.tracker.folder(key)
if imapRecoveryCursor(cur) != nil {
allDone = allDone && cur.Done
continue
}
if cur.Done {
continue
}
@@ -864,7 +941,7 @@ func (w *WMail) imapBackfill(ctx context.Context, folders []models.Mailbox, stat
if err != nil {
return err
}
done, err := w.imapApply(ctx, fetched, true, stats)
done, err := w.imapApply(ctx, fetched, true, stats, nil)
if err != nil {
return err
}
internal/app/worker/wmail/sync_imap_flags.go
+2 -1
View File
@@ -73,7 +73,8 @@ func (w *WMail) imapScanFlags(ctx context.Context, box *models.Mailbox, stats *t
return err
}
if stats.aborted {
break
scan.at = now
return nil
}
}
}
internal/app/worker/wmail/sync_imap_recovery.go
+166
View File
@@ -0,0 +1,166 @@
package wmail
import (
"context"
"encoding/json"
"sort"
"strings"
"time"
goimap "github.com/emersion/go-imap/v2"
"github.com/warmbly/warmbly/internal/config"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/models"
)
const imapRecoveryPrefix = "imap-recovery-v1:"
const imapInitialRecoveryPrefix = "imap-initial-generation-v1:"
// Recovery metadata lives in the existing opaque folder continuation, not a new wire field.
type imapFolderRecovery struct {
Generation uint32 `json:"generation"`
Since time.Time `json:"since"`
Synced int `json:"synced"`
Initial bool `json:"-"`
KnownOnly bool `json:"-"`
}
func imapRecoveryCursor(cur models.SyncFolderCursor) *imapFolderRecovery {
raw, ok := strings.CutPrefix(cur.Next, imapRecoveryPrefix)
initial := false
if !ok {
raw, ok = strings.CutPrefix(cur.Next, imapInitialRecoveryPrefix)
initial = ok
}
if !ok {
return nil
}
var recovery imapFolderRecovery
if json.Unmarshal([]byte(raw), &recovery) != nil || recovery.Generation == 0 || recovery.Since.IsZero() || recovery.Synced < 0 {
return nil
}
recovery.Initial = initial
return &recovery
}
func (w *WMail) imapRecoverFolder(box *models.Mailbox) bool {
cur := w.tracker.folder(box.Name)
if recovery := imapRecoveryCursor(cur); recovery == nil || recovery.Generation != box.UIDValidity {
since := time.Now().Add(-time.Duration(w.gov.Policy().BackfillDays) * 24 * time.Hour)
initial := w.tracker.state.BackfillStatus != models.SyncBackfillComplete
if initial {
w.tracker.startBackfill(time.Now(), w.gov.Policy().BackfillDays)
if w.tracker.state.BackfillSince == nil {
return false
}
since = *w.tracker.state.BackfillSince
}
w.imapImportCursor(box.Name, 0, false, &imapFolderRecovery{Generation: box.UIDValidity, Since: since, Initial: initial})
}
// Publish replay state before a live baseline can make the folder look caught up on reload.
w.tracker.flush(time.Now())
return !w.tracker.dirty
}
func (w *WMail) imapRecoveryPending(folders []models.Mailbox) bool {
for _, box := range folders {
cur := w.tracker.folder(box.Name)
if !cur.Done && imapRecoveryCursor(cur) != nil {
return true
}
}
return false
}
func (w *WMail) imapImportCount(recovery *imapFolderRecovery) int {
if recovery != nil && !recovery.Initial {
return recovery.Synced
}
return w.tracker.state.BackfillSynced
}
func (w *WMail) imapImportCursor(folder string, uid uint32, done bool, recovery *imapFolderRecovery) {
cur := models.SyncFolderCursor{UID: uid, Done: done}
if recovery != nil {
raw, _ := json.Marshal(recovery)
prefix := imapRecoveryPrefix
if recovery.Initial {
// Older workers must not interpret the global import cap as a per-folder recovery cap.
prefix = imapInitialRecoveryPrefix
}
cur.Next = prefix + string(raw)
}
w.tracker.setFolder(folder, cur)
}
// Generation recovery uses its original window and the owning import's counters.
func (w *WMail) imapRecoverFolders(ctx context.Context, folders []models.Mailbox, stats *tickStats) *errx.MailError {
client := w.SmtpImapData.ImapClient
for i := range folders {
box := &folders[i]
cur := w.tracker.folder(box.Name)
recovery := imapRecoveryCursor(cur)
if recovery == nil {
continue
}
if recovery.Generation != box.UIDValidity {
if !w.imapRecoverFolder(box) {
stats.aborted = true
return nil
}
cur = w.tracker.folder(box.Name)
recovery = imapRecoveryCursor(cur)
if recovery == nil {
continue
}
}
if cur.Done {
continue
}
recovery.KnownOnly = !imapBackfillEligible(box)
if ctx.Err() != nil || stats.aborted || stats.laneDenied(LaneBackfill) {
return nil
}
if !recovery.Initial && !recovery.KnownOnly && recovery.Synced >= w.gov.Policy().BackfillMessages {
w.imapImportCursor(box.Name, cur.UID, true, recovery)
continue
}
w.setWalking(box)
view, err := client.SelectForSyncState(box.Name)
if err != nil {
return err
}
if view.UIDValidity != 0 && view.UIDValidity != recovery.Generation {
return errx.ErrMailResourceNotFound
}
uids, err := client.SearchSince(recovery.Since)
if err != nil {
return err
}
remaining := uids[:0:0]
for _, uid := range uids {
if cur.UID == 0 || uid < goimap.UID(cur.UID) {
remaining = append(remaining, uid)
}
}
sort.Slice(remaining, func(i, j int) bool { return remaining[i] > remaining[j] })
for lo := 0; lo < len(remaining); lo += config.ImapFetchBatchSize {
hi := min(lo+config.ImapFetchBatchSize, len(remaining))
fetched, err := client.FetchEnvelopes(ctx, remaining[lo:hi])
if err != nil {
return err
}
done, err := w.imapApply(ctx, fetched, true, stats, recovery)
if err != nil {
return err
}
if !done || ctx.Err() != nil {
return nil
}
cur.UID = uint32(remaining[hi-1])
w.imapImportCursor(box.Name, cur.UID, false, recovery)
}
w.imapImportCursor(box.Name, cur.UID, true, recovery)
}
return nil
}
internal/app/worker/wmail/sync_imap_recovery_test.go
+529
View File
@@ -0,0 +1,529 @@
package wmail
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
"time"
goimap "github.com/emersion/go-imap/v2"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/client/smtpimap/imap"
"github.com/warmbly/warmbly/internal/config"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/repository"
)
type knownRecoveryConn struct {
*fakeImapConn
message models.EmailMessageData
}
func TestImapNonHistoryGenerationReconcilesOnlyKnownMessagesAcrossReload(t *testing.T) {
for _, attr := range []string{"\\Junk", "\\Trash"} {
for _, initial := range []bool{false, true} {
for _, fail := range []bool{false, true} {
t.Run(fmt.Sprintf("%s/initial=%t/relayFailure=%t", attr, initial, fail), func(t *testing.T) {
box := models.Mailbox{Name: attr, Attrs: []string{attr}, UIDValidity: 9, UIDNext: 4, HighestModSeq: 5}
conn := &initialRecoveryConn{folderIdentityRecoveryConn: &folderIdentityRecoveryConn{fakeImapConn: &fakeImapConn{
folders: []models.Mailbox{box}, all: uidRange(3), changed: uidRange(3),
}}}
budget := &initialRecoveryBudget{fixedBudget: fixedBudget{allow: 0}, limit: 2}
w, events := newIMAPTestMail(conn, budget, &models.Mailbox{Name: attr, Attrs: box.Attrs, UIDValidity: 8, UIDNext: 91, HighestModSeq: 5})
knownID := uuid.NewString()
maps := &recoveryMessageMap{data: map[string]repository.EmailMessageData{
"<2@fake.test>": {ID: knownID, MessageID: "<2@fake.test>"},
"<3@fake.test>": {MessageID: "<3@fake.test>"},
}}
w.EmailMessageMapRepository = maps
since, success := time.Now().Add(-time.Hour), time.Now().Add(-time.Minute)
w.tracker.state.BackfillSynced = 2
w.tracker.state.LastSyncedAt = &success
if initial {
w.tracker.state.BackfillStatus = models.SyncBackfillRunning
w.tracker.state.BackfillSince = &since
}
if fail {
emit := w.onEvent
w.onEvent = func(kind models.JobEventType, body any) error {
if kind == models.JobEventTypeEmailUpdate {
return errors.New("known metadata relay failed")
}
return emit(kind, body)
}
}
if err := w.Sync(t.Context()); err != nil {
t.Fatal(err)
}
all := append([]captured{}, (*events)...)
if fail && (w.tracker.folder(attr).Done || !w.tracker.state.LastSyncedAt.Equal(success)) {
t.Fatal("failed metadata relay advanced recovery or last successful check")
}
w, events = imapReload(t, w, conn, budget)
for pass := 0; pass < 3; pass++ {
if err := w.Sync(t.Context()); err != nil {
t.Fatal(err)
}
}
all = append(all, (*events)...)
updates := mailboxEvents(all, models.JobEventTypeEmailUpdate)
if len(updates) != 1 {
t.Fatalf("known reconciliation count: %d", len(updates))
}
update := updates[0].body.(*models.JobEventEmailUpdate)
if update.ID.String() != knownID || update.UID != 2 || update.Mailbox != 9 || !models.SeenFromFlags(update.Flags) {
t.Fatal("known message identity, UID, generation or read state lost")
}
if len(mailboxEvents(all, models.JobEventTypeNewEmail)) != 0 || len(conn.bodies) != 0 || len(maps.data) != 2 || w.tracker.state.BackfillSynced != 2 || !w.tracker.folder(attr).Done {
t.Fatal("non-history recovery imported unseen/hidden mail, changed counters or did not complete")
}
})
}
}
}
}
type initialRecoveryConn struct {
*folderIdentityRecoveryConn
since []time.Time
bodies []string
}
func (c *initialRecoveryConn) SearchSince(since time.Time) ([]goimap.UID, *errx.MailError) {
c.since = append(c.since, since)
return c.fakeImapConn.SearchSince(since)
}
func (c *initialRecoveryConn) FetchBody(f *imap.Fetched) *errx.MailError {
c.bodies = append(c.bodies, f.Email.MessageID)
return c.fakeImapConn.FetchBody(f)
}
type initialRecoveryBudget struct {
fixedBudget
limit int
}
func (b *initialRecoveryBudget) Policy() models.SyncPolicy {
policy := b.fixedBudget.Policy()
policy.BackfillMessages = b.limit
return policy
}
func newRunningRecoveryMail(t *testing.T, budget syncBudget) (*WMail, *[]captured, *initialRecoveryConn, *recoveryMessageMap) {
t.Helper()
conn := &initialRecoveryConn{folderIdentityRecoveryConn: &folderIdentityRecoveryConn{fakeImapConn: &fakeImapConn{
folders: []models.Mailbox{{Name: "INBOX", UIDValidity: 9, UIDNext: 4, HighestModSeq: 5}},
all: uidRange(3), changed: uidRange(3),
}}}
w, events := newIMAPTestMail(conn, budget, &models.Mailbox{Name: "INBOX", UIDValidity: 8, UIDNext: 500, HighestModSeq: 5})
since, started, success := time.Now().Add(-30*24*time.Hour), time.Now().Add(-16*24*time.Hour), time.Now().Add(-time.Hour)
w.tracker = newSyncTracker(&models.SyncState{BackfillStatus: models.SyncBackfillRunning, BackfillSince: &since, BackfillStartedAt: &started, BackfillSynced: 2, LastSyncedAt: &success}, func(models.SyncState) error { return nil })
w.tracker.setFolder("INBOX", models.SyncFolderCursor{UID: 50, Done: true})
maps := &recoveryMessageMap{data: map[string]repository.EmailMessageData{"<2@fake.test>": {ID: uuid.NewString(), MessageID: "<2@fake.test>"}}}
w.EmailMessageMapRepository = maps
return w, events, conn, maps
}
func TestImapRunningImportGenerationRecoveryPreservesWindowCountersAndReload(t *testing.T) {
for _, outcome := range []string{"complete", "deferred", "relay failure", "global cap", "cap already reached"} {
t.Run(outcome, func(t *testing.T) {
budget := &initialRecoveryBudget{fixedBudget: fixedBudget{allow: 10}, limit: 4}
if outcome == "deferred" {
budget.allow = 1
}
if outcome == "global cap" {
budget.limit = 3
}
if outcome == "cap already reached" {
budget.limit = 2
budget.allow = 0
}
w, events, conn, maps := newRunningRecoveryMail(t, budget)
since, started, success := *w.tracker.state.BackfillSince, *w.tracker.state.BackfillStartedAt, *w.tracker.state.LastSyncedAt
knownID := maps.data["<2@fake.test>"].ID
if outcome == "relay failure" {
emit := w.onEvent
w.onEvent = func(kind models.JobEventType, body any) error {
if kind == models.JobEventTypeEmailUpdate {
return errors.New("known generation relay failed")
}
return emit(kind, body)
}
}
if err := w.Sync(t.Context()); err != nil {
t.Fatal(err)
}
allEvents := append([]captured{}, (*events)...)
if outcome == "deferred" || outcome == "relay failure" {
cur := w.tracker.folder("INBOX")
recovery := imapRecoveryCursor(cur)
wantUID := uint32(2)
if outcome == "relay failure" {
wantUID = 3
}
if recovery == nil || !recovery.Initial || recovery.Generation != 9 || !recovery.Since.Equal(since) || cur.Done || cur.UID != wantUID || w.tracker.state.BackfillSynced != 3 || w.tracker.state.BackfillStatus != models.SyncBackfillRunning || !w.tracker.state.LastSyncedAt.Equal(success) {
t.Fatalf("unfinished recovery lost progress/window: %+v / %+v / %+v", cur, recovery, w.tracker.state)
}
w, events = imapReload(t, w, conn, &initialRecoveryBudget{fixedBudget: fixedBudget{allow: 10}, limit: 4})
if err := w.Sync(t.Context()); err != nil {
t.Fatal(err)
}
allEvents = append(allEvents, (*events)...)
}
if !w.tracker.state.BackfillSince.Equal(since) || !w.tracker.state.BackfillStartedAt.Equal(started) {
t.Fatal("generation changed original import window/timer")
}
updates := mailboxEvents(allEvents, models.JobEventTypeEmailUpdate)
if len(updates) != 1 {
t.Fatalf("known metadata relayed %d times, want once across reload", len(updates))
}
update := updates[0].body.(*models.JobEventEmailUpdate)
if update.ID.String() != knownID || update.UID != 2 || update.Mailbox != 9 || update.ModSeq != 5 || !models.SeenFromFlags(update.Flags) {
t.Fatalf("known handles/read state stale: %+v", update)
}
wantCount := budget.limit
if w.tracker.state.BackfillStatus != models.SyncBackfillComplete || w.tracker.state.BackfillSynced != wantCount || len(maps.data) != 1+wantCount-2 || len(mailboxEvents(allEvents, models.JobEventTypeNewEmail)) != wantCount-2 || len(conn.bodies) != wantCount-2 {
t.Fatal("recovery duplicated known mail, bypassed cap or lost cumulative admission count")
}
if maps.data["<2@fake.test>"].ID != knownID {
t.Fatal("known identity changed")
}
for _, cutoff := range conn.since {
if !cutoff.Equal(since) {
t.Fatal("provider search escaped original import window")
}
}
cur := w.tracker.folder("INBOX")
if !cur.Done || cur.UID != 1 || imapRecoveryCursor(cur) == nil || !imapRecoveryCursor(cur).Initial {
t.Fatalf("running-import recovery continuation lost: %+v", cur)
}
if strings.HasPrefix(cur.Next, imapRecoveryPrefix) {
t.Fatal("an older worker could apply a separate cap to the initial import")
}
searches := conn.searches
again, later := imapReload(t, w, conn, &initialRecoveryBudget{fixedBudget: fixedBudget{allow: 10}, limit: 4})
if err := again.Sync(t.Context()); err != nil {
t.Fatal(err)
}
if conn.searches != searches || len(mailboxEvents(*later, models.JobEventTypeEmailUpdate)) != 0 || len(mailboxEvents(*later, models.JobEventTypeNewEmail)) != 0 {
t.Fatal("completed initial-generation replay repeated")
}
})
}
}
func TestImapRunningImportGenerationMarkerPrecedesBaseline(t *testing.T) {
w, events, conn, _ := newRunningRecoveryMail(t, &initialRecoveryBudget{fixedBudget: fixedBudget{allow: 10}, limit: 4})
w.tracker.emit = func(models.SyncState) error { return errors.New("generation state relay failed") }
if err := w.Sync(t.Context()); err != nil {
t.Fatal(err)
}
if conn.fetches != 0 || len(mailboxEvents(*events, models.JobEventTypeMailboxUpdate)) != 0 || w.SmtpImapData.Mailboxes[0].UIDValidity != 8 || w.tracker.state.BackfillSynced != 2 {
t.Fatal("baseline or admissions outran initial-generation marker")
}
w.tracker.emit = func(models.SyncState) error { return nil }
if err := w.Sync(t.Context()); err != nil {
t.Fatal(err)
}
if w.tracker.state.BackfillStatus != models.SyncBackfillComplete || w.tracker.state.BackfillSynced != 4 || len(mailboxEvents(*events, models.JobEventTypeEmailUpdate)) != 1 {
t.Fatal("state relay retry did not recover initial generation")
}
}
func (c *knownRecoveryConn) FetchEnvelopes(_ context.Context, _ []goimap.UID) ([]*imap.Fetched, *errx.MailError) {
c.fetches++
message := c.message
message.Flags = append([]string{}, c.message.Flags...)
return []*imap.Fetched{{Email: &message}}, nil
}
func TestImapRecoveryReconcilesKnownMetadataBeforeCheckpoint(t *testing.T) {
for _, newFolder := range []bool{false, true} {
for _, relayFailure := range []bool{false, true} {
t.Run(fmt.Sprintf("new folder=%t relay failure=%t", newFolder, relayFailure), func(t *testing.T) {
box := models.Mailbox{Name: "Archive", UIDValidity: 9, UIDNext: 43, HighestModSeq: 100}
conn := &knownRecoveryConn{fakeImapConn: &fakeImapConn{folders: []models.Mailbox{box}, all: []goimap.UID{42}, bodyErr: errx.ErrMailServerUnreachable}, message: models.EmailMessageData{UID: 42, MessageID: "<same@fake.test>", ModSeq: 100, Flags: []string{models.FlagSeen}}}
w, events := newIMAPTestMail(conn, &fixedBudget{allow: 0}, &models.Mailbox{Name: box.Name, UIDValidity: 8, UIDNext: 200, HighestModSeq: 100})
if newFolder {
w.SmtpImapData.Mailboxes = nil
}
id := uuid.NewString()
maps := &recoveryMessageMap{data: map[string]repository.EmailMessageData{conn.message.MessageID: {ID: id, MessageID: conn.message.MessageID}}}
w.EmailMessageMapRepository = maps
w.tracker.state.BackfillSynced = 23
w.tracker.setFolder(box.Name, models.SyncFolderCursor{UID: 200, Done: true})
oldSuccess := time.Now().Add(-time.Hour)
w.tracker.state.LastSyncedAt = &oldSuccess
if relayFailure {
emit := w.onEvent
w.onEvent = func(kind models.JobEventType, body any) error {
if kind == models.JobEventTypeEmailUpdate {
return errors.New("known metadata relay unavailable")
}
return emit(kind, body)
}
}
if err := w.Sync(t.Context()); err != nil {
t.Fatal(err)
}
if relayFailure {
cur := w.tracker.folder(box.Name)
if cur.Done || cur.UID != 0 || !w.tracker.state.LastSyncedAt.Equal(oldSuccess) || len(mailboxEvents(*events, models.JobEventTypeEmailUpdate)) != 0 {
t.Fatal("failed metadata relay completed recovery or refreshed success")
}
w, events = imapReload(t, w, conn, &fixedBudget{allow: 0})
if err := w.Sync(t.Context()); err != nil {
t.Fatal(err)
}
}
updates := mailboxEvents(*events, models.JobEventTypeEmailUpdate)
if len(updates) != 1 {
t.Fatalf("recovery known updates=%d", len(updates))
}
update := updates[0].body.(*models.JobEventEmailUpdate)
if update.ID.String() != id || update.UID != 42 || update.Mailbox != 9 || update.ModSeq != 100 || update.FolderPath != box.Name || !models.SeenFromFlags(update.Flags) {
t.Fatalf("stale recovery metadata: %+v", update)
}
cur := w.tracker.folder(box.Name)
if !cur.Done || cur.UID != 42 || imapRecoveryCursor(cur).Synced != 0 || w.tracker.state.BackfillSynced != 23 || w.tracker.state.BackfillStatus != models.SyncBackfillComplete {
t.Fatal("known reconciliation reset or consumed global history")
}
if len(mailboxEvents(*events, models.JobEventTypeNewEmail)) != 0 || len(maps.data) != 1 || maps.data[conn.message.MessageID].ID != id {
t.Fatal("known recovery reimported a mapped message")
}
searches := conn.searches
again, later := imapReload(t, w, conn, &fixedBudget{allow: 0})
if err := again.Sync(t.Context()); err != nil {
t.Fatal(err)
}
if conn.searches != searches || len(mailboxEvents(*later, models.JobEventTypeEmailUpdate)) != 0 || len(mailboxEvents(*later, models.JobEventTypeNewEmail)) != 0 {
t.Fatal("unchanged HIGHESTMODSEQ repeated completed recovery")
}
})
}
}
}
func imapReload(t *testing.T, w *WMail, conn ImapConn, budget syncBudget) (*WMail, *[]captured) {
t.Helper()
// Use the existing JSON wire shape, not shared in-memory state, to model restart.
raw, err := json.Marshal(w.tracker.state)
if err != nil {
t.Fatal(err)
}
var state models.SyncState
if err := json.Unmarshal(raw, &state); err != nil {
t.Fatal(err)
}
saved := *w.SmtpImapData.Mailboxes[0]
next, events := newIMAPTestMail(conn, budget, &saved)
next.EmailMessageMapRepository = w.EmailMessageMapRepository
next.tracker = newSyncTracker(&state, func(models.SyncState) error { return nil })
return next, events
}
func TestImapCompletedFolderRecoveryResumesAfterDeferralAndReload(t *testing.T) {
for _, generation := range []bool{false, true} {
t.Run(map[bool]string{false: "new populated folder", true: "new generation"}[generation], func(t *testing.T) {
box := models.Mailbox{Name: "Archive", UIDValidity: 9, UIDNext: 4, HighestModSeq: 100}
conn := &fakeImapConn{folders: []models.Mailbox{box}, all: []goimap.UID{1, 2, 3}}
saved := &models.Mailbox{Name: "Archive", UIDValidity: 8, UIDNext: 90, HighestModSeq: 100}
w, events := newIMAPTestMail(conn, &fixedBudget{allow: 1}, saved)
if !generation {
w.SmtpImapData.Mailboxes = nil
}
maps := &recoveryMessageMap{data: map[string]repository.EmailMessageData{}}
knownID := uuid.NewString()
maps.data["<2@fake.test>"] = repository.EmailMessageData{ID: knownID, MessageID: "<2@fake.test>"}
w.EmailMessageMapRepository = maps
w.tracker.setFolder("Archive", models.SyncFolderCursor{UID: 80, Done: true})
oldSuccess := time.Now().Add(-time.Hour)
w.tracker.state.LastSyncedAt = &oldSuccess
if err := w.Sync(t.Context()); err != nil {
t.Fatal(err)
}
cur := w.tracker.folder("Archive")
recovery := imapRecoveryCursor(cur)
if cur.Done || cur.UID != 2 || recovery == nil || recovery.Generation != 9 || recovery.Synced != 1 {
t.Fatalf("deferred recovery = %+v / %+v", cur, recovery)
}
if len(mailboxEvents(*events, models.JobEventTypeNewEmail)) != 1 {
t.Fatal("newest recovery arrival missing")
}
if len(mailboxEvents(*events, models.JobEventTypeEmailUpdate)) != 1 {
t.Fatal("known metadata was not reconciled before the saved floor")
}
if !w.tracker.state.LastSyncedAt.Equal(oldSuccess) {
t.Fatal("deferred recovery refreshed success")
}
cutoff := recovery.Since
next, resumed := imapReload(t, w, conn, &fixedBudget{allow: 10})
if err := next.Sync(t.Context()); err != nil {
t.Fatal(err)
}
cur = next.tracker.folder("Archive")
if !cur.Done || cur.UID != 1 || !imapRecoveryCursor(cur).Since.Equal(cutoff) {
t.Fatalf("restarted recovery = %+v", cur)
}
if len(maps.data) != 3 || len(mailboxEvents(*resumed, models.JobEventTypeNewEmail)) != 1 || maps.data["<2@fake.test>"].ID != knownID {
t.Fatal("recovery lost or duplicated arrivals")
}
if len(mailboxEvents(*resumed, models.JobEventTypeEmailUpdate)) != 0 {
t.Fatal("checkpointed known metadata was relayed again after reload")
}
if next.tracker.state.BackfillStatus != models.SyncBackfillComplete || next.tracker.state.BackfillSynced != 0 {
t.Fatal("global history reset")
}
before := conn.searches
again, later := imapReload(t, next, conn, &fixedBudget{allow: 10})
if err := again.Sync(t.Context()); err != nil {
t.Fatal(err)
}
if conn.searches != before || len(mailboxEvents(*later, models.JobEventTypeNewEmail)) != 0 {
t.Fatal("completed recovery reimported on restart")
}
})
}
}
func TestImapRecoveryDoesNotBaselineBeforeReplayStateRelays(t *testing.T) {
conn := &fakeImapConn{folders: []models.Mailbox{{Name: "Archive", UIDValidity: 9, UIDNext: 3, HighestModSeq: 100}}, all: []goimap.UID{1, 2}}
w, events := newIMAPTestMail(conn, &fixedBudget{allow: 10}, &models.Mailbox{Name: "Archive", UIDValidity: 8})
w.tracker.emit = func(models.SyncState) error { return errors.New("state unavailable") }
if err := w.Sync(t.Context()); err != nil {
t.Fatal(err)
}
if w.SmtpImapData.Mailboxes[0].UIDValidity != 8 || len(mailboxEvents(*events, models.JobEventTypeMailboxUpdate)) != 0 || conn.fetches != 0 {
t.Fatal("baseline outran replay checkpoint")
}
w.tracker.emit = func(models.SyncState) error { return nil }
if err := w.Sync(t.Context()); err != nil {
t.Fatal(err)
}
if !w.tracker.folder("Archive").Done || len(mailboxEvents(*events, models.JobEventTypeNewEmail)) != 2 {
t.Fatal("checkpoint retry did not recover folder")
}
}
func TestImapBodyFailurePinsLiveAndRecoveryUntilReload(t *testing.T) {
for _, recovery := range []bool{false, true} {
t.Run(map[bool]string{false: "live", true: "recovery"}[recovery], func(t *testing.T) {
box := models.Mailbox{Name: "INBOX", UIDValidity: 7, UIDNext: 2, HighestModSeq: 101}
conn := &fakeImapConn{folders: []models.Mailbox{box}, changed: []goimap.UID{1}, all: []goimap.UID{1}, bodyErr: errx.ErrMailServerUnreachable}
w, events := newIMAPTestMail(conn, &fixedBudget{allow: 10}, &models.Mailbox{Name: "INBOX", UIDValidity: 7, UIDNext: 1, HighestModSeq: 100})
if recovery {
w.SmtpImapData.Mailboxes[0].UIDValidity = 6
}
maps := &recoveryMessageMap{data: map[string]repository.EmailMessageData{}}
w.EmailMessageMapRepository = maps
if err := w.Sync(t.Context()); err == nil {
t.Fatal("body failure swallowed")
}
if len(maps.data) != 0 || len(mailboxEvents(*events, models.JobEventTypeNewEmail)) != 0 || w.tracker.state.BackfillSynced != 0 || w.tracker.folder("INBOX").UID != 0 || w.tracker.folder("INBOX").Done {
t.Fatal("body failure imported partial message or advanced recovery")
}
if !recovery && w.SmtpImapData.Mailboxes[0].UIDNext != 1 {
t.Fatal("body failure advanced live cursor")
}
conn.bodyErr = nil
next, resumed := imapReload(t, w, conn, &fixedBudget{allow: 10})
if err := next.Sync(t.Context()); err != nil {
t.Fatal(err)
}
arrivals := mailboxEvents(*resumed, models.JobEventTypeNewEmail)
if len(arrivals) != 1 || len(maps.data) != 1 {
t.Fatal("hydration was not retried exactly once")
}
if arrivals[0].body.(*models.JobEventNewEmail).Message.BodyText != "complete body" {
t.Fatal("arrival did not contain hydrated body")
}
if next.SmtpImapData.Mailboxes[0].UIDNext != 2 {
t.Fatal("live cursor did not advance after hydration")
}
})
}
}
func TestImapNomodseqUsesUIDNextAndPeriodicFlags(t *testing.T) {
for _, listedModseq := range []uint64{0, 100} {
t.Run(map[uint64]string{0: "status NOMODSEQ", 100: "select NOMODSEQ"}[listedModseq], func(t *testing.T) {
conn := &fakeImapConn{folders: []models.Mailbox{{Name: "INBOX", UIDValidity: 7, UIDNext: 3, HighestModSeq: listedModseq}}, changed: []goimap.UID{2}, view: &imap.Selected{UIDNext: 3}, flags: map[uint32]imap.FlagState{1: {MessageID: "<known@test>"}}}
w, events := newIMAPTestMail(conn, &fixedBudget{allow: 10}, &models.Mailbox{Name: "INBOX", UIDValidity: 7, UIDNext: 2, HighestModSeq: listedModseq})
if err := w.Sync(t.Context()); err != nil {
t.Fatal(err)
}
if len(mailboxEvents(*events, models.JobEventTypeNewEmail)) != 1 || conn.flagScans != 1 || conn.uidSearches != 1 || conn.modseqSearches != 0 {
t.Fatal("NOMODSEQ ignored arrival or fallback flags")
}
w.EmailMessageMapRepository = knownMessageMap{id: uuid.NewString()}
conn.flags[1] = imap.FlagState{MessageID: "<known@test>", Flags: []string{`\Seen`}}
w.flagScan["INBOX"].at = time.Now().Add(-2 * config.ImapFlagScanInterval)
if err := w.Sync(t.Context()); err != nil {
t.Fatal(err)
}
if len(mailboxEvents(*events, models.JobEventTypeEmailUpdate)) != 1 {
t.Fatal("NOMODSEQ dropped periodic read update")
}
})
}
}
func TestImapBodyFailurePinsInitialHistoryUntilReload(t *testing.T) {
box := models.Mailbox{Name: "INBOX", UIDValidity: 7, UIDNext: 2, HighestModSeq: 100, Messages: 1}
conn := &fakeImapConn{folders: []models.Mailbox{box}, changed: []goimap.UID{1}, all: []goimap.UID{1}, bodyErr: errx.ErrMailServerUnreachable}
w, events := newIMAPTestMail(conn, &fixedBudget{allow: 10}, &box)
w.tracker = newSyncTracker(nil, func(models.SyncState) error { return nil })
maps := &recoveryMessageMap{data: map[string]repository.EmailMessageData{}}
w.EmailMessageMapRepository = maps
if err := w.Sync(t.Context()); err == nil {
t.Fatal("history body failure swallowed")
}
if len(maps.data) != 0 || len(mailboxEvents(*events, models.JobEventTypeNewEmail)) != 0 || w.tracker.state.BackfillSynced != 0 || w.tracker.folder("INBOX").UID != 0 || w.tracker.folder("INBOX").Done {
t.Fatal("history advanced after body failure")
}
conn.bodyErr = nil
next, resumed := imapReload(t, w, conn, &fixedBudget{allow: 10})
if err := next.Sync(t.Context()); err != nil {
t.Fatal(err)
}
if len(maps.data) != 1 || len(mailboxEvents(*resumed, models.JobEventTypeNewEmail)) != 1 || next.tracker.state.BackfillSynced != 1 || !next.tracker.folder("INBOX").Done {
t.Fatal("history hydration did not resume")
}
}
func TestImapFlagScanRetriesBothChangesAfterPublicationAbort(t *testing.T) {
conn := &fakeImapConn{folders: []models.Mailbox{{Name: "INBOX", UIDValidity: 7, UIDNext: 3}}, noCondStore: true, flags: map[uint32]imap.FlagState{1: {MessageID: "<one@test>"}, 2: {MessageID: "<two@test>"}}}
w, events := newIMAPTestMail(conn, &fixedBudget{allow: 10}, &models.Mailbox{Name: "INBOX", UIDValidity: 7, UIDNext: 3})
w.EmailMessageMapRepository = knownMessageMap{id: uuid.NewString()}
if err := w.Sync(t.Context()); err != nil {
t.Fatal(err)
}
for uid, st := range conn.flags {
st.Flags = []string{`\Seen`}
conn.flags[uid] = st
}
emit := w.onEvent
w.onEvent = func(kind models.JobEventType, body any) error {
if kind == models.JobEventTypeEmailUpdate {
return errors.New("publish unavailable")
}
return emit(kind, body)
}
w.flagScan["INBOX"].at = time.Now().Add(-2 * config.ImapFlagScanInterval)
if err := w.Sync(t.Context()); err != nil {
t.Fatal(err)
}
w.onEvent = emit
w.flagScan["INBOX"].at = time.Now().Add(-2 * config.ImapFlagScanInterval)
if err := w.Sync(t.Context()); err != nil {
t.Fatal(err)
}
if len(mailboxEvents(*events, models.JobEventTypeEmailUpdate)) != 2 {
t.Fatal("failed or unvisited flag changes were absorbed into baseline")
}
}
internal/app/worker/wmail/sync_imap_test.go
+31 -3
View File
@@ -43,7 +43,11 @@ type fakeImapConn struct {
failFinds bool
// view, when set, is the cursors SELECT reports in place of the listing's:
// a server whose selected view lags or leads its STATUS.
view *imap.Selected
view *imap.Selected
searches int
bodyErr *errx.MailError
modseqSearches int
uidSearches int
}
// Folders hands out a copy, like a real listing: the pass filters the slice
@@ -61,6 +65,7 @@ func (c *fakeImapConn) FolderConflicts() int { return c.conflicts }
func (c *fakeImapConn) HasCondStore() bool { return !c.noCondStore }
func (c *fakeImapConn) SearchNewSince(uidNext uint32) ([]goimap.UID, *errx.MailError) {
c.uidSearches++
var out []goimap.UID
for _, uid := range c.changed {
if uint32(uid) >= uidNext {
@@ -115,6 +120,7 @@ func (c *fakeImapConn) SelectForSyncGen(name string) (uint32, uint32, *errx.Mail
}
func (c *fakeImapConn) SearchChangedSince(uint64) ([]goimap.UID, *errx.MailError) {
c.modseqSearches++
return append([]goimap.UID(nil), c.changed...), nil
}
@@ -122,6 +128,11 @@ func (c *fakeImapConn) SearchAll() ([]goimap.UID, *errx.MailError) {
return append([]goimap.UID(nil), c.all...), nil
}
func (c *fakeImapConn) SearchSince(time.Time) ([]goimap.UID, *errx.MailError) {
c.searches++
return append([]goimap.UID(nil), c.all...), nil
}
func (c *fakeImapConn) FetchEnvelopes(_ context.Context, uids []goimap.UID) ([]*imap.Fetched, *errx.MailError) {
c.fetches++
out := make([]*imap.Fetched, 0, len(uids))
@@ -135,7 +146,13 @@ func (c *fakeImapConn) FetchEnvelopes(_ context.Context, uids []goimap.UID) ([]*
return out, nil
}
func (c *fakeImapConn) FetchBody(*imap.Fetched) {}
func (c *fakeImapConn) FetchBody(f *imap.Fetched) *errx.MailError {
if c.bodyErr != nil {
return c.bodyErr
}
f.Email.BodyPlain = "complete body"
return nil
}
// fixedBudget is a syncBudget that admits a fixed number of messages and then
// denies on the daily window, which is how a real governor answers once the
@@ -339,6 +356,17 @@ func (c *backfillImapConn) FolderConflicts() int { retur
func (c *backfillImapConn) HasCondStore() bool { return true }
func (c *backfillImapConn) ReleaseMailbox() {}
func (c *backfillImapConn) SelectForSyncState(folder string) (imap.Selected, *errx.MailError) {
count, err := c.SelectForSync(folder)
view := imap.Selected{Count: count}
for _, box := range c.folders {
if box.Name == folder {
view.UIDValidity, view.UIDNext, view.HighestModSeq = box.UIDValidity, box.UIDNext, box.HighestModSeq
}
}
return view, err
}
func (c *backfillImapConn) SelectForSync(name string) (uint32, *errx.MailError) {
c.selected = name
return uint32(len(c.uids[name])), nil
@@ -375,7 +403,7 @@ func (c *backfillImapConn) FetchEnvelopes(_ context.Context, uids []goimap.UID)
return out, nil
}
func (c *backfillImapConn) FetchBody(*imap.Fetched) {}
func (c *backfillImapConn) FetchBody(*imap.Fetched) *errx.MailError { return nil }
// The Graph defect's shape, checked on the IMAP import: a folder whose search
// fails holds its cursor and is walked again on the next pass. Nothing about a
internal/app/worker/wmail/sync_mail.go
+12
View File
@@ -8,6 +8,18 @@ import (
)
func (w *WMail) SyncMail(ctx context.Context) *errx.MailError {
done, ok := w.BeginExecution()
if !ok {
return errx.ErrMailServerUnreachable
}
defer done()
w.syncMu.Lock()
defer w.syncMu.Unlock()
ctx, cancel := w.ExecutionContext(ctx)
defer cancel()
if ctx.Err() != nil {
return errx.ErrMailServerUnreachable
}
w.retryDiagnostics(ctx)
defer w.retryDiagnostics(ctx)
switch w.EmailType {
internal/app/worker/wmail/sync_state.go
+10 -4
View File
@@ -16,10 +16,11 @@ const syncStateHeartbeat = 10 * time.Minute
// of the tick; the consumer persists it and the loader hands it back on the
// next assignment.
type syncTracker struct {
state models.SyncState
dirty bool
lastSent time.Time
emit func(models.SyncState) error
state models.SyncState
dirty bool
lastSent time.Time
emit func(models.SyncState) error
tickComplete bool
}
func newSyncTracker(seed *models.SyncState, emit func(models.SyncState) error) *syncTracker {
@@ -85,6 +86,11 @@ func (t *syncTracker) setFoldersSkipped(cap, conflict int) {
func (t *syncTracker) touch(now time.Time) {
stamp := now
t.state.LastSyncedAt = &stamp
t.flush(now)
}
// flush publishes safe progress without implying a successful provider check.
func (t *syncTracker) flush(now time.Time) {
if !t.dirty && now.Sub(t.lastSent) < syncStateHeartbeat {
return
}
internal/app/worker/wmail/sync_truth_test.go
+376
View File
@@ -0,0 +1,376 @@
package wmail
import (
"context"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/client/goog"
"github.com/warmbly/warmbly/internal/client/msgraph"
"github.com/warmbly/warmbly/internal/client/smtpimap/imap"
"github.com/warmbly/warmbly/internal/config"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/repository"
)
func seedSyncTruth(w *WMail, relayed *[]models.SyncState) time.Time {
previous := time.Now().Add(-time.Hour)
w.tracker = newSyncTracker(&models.SyncState{BackfillStatus: models.SyncBackfillComplete, LastSyncedAt: &previous, Deferred: 5}, func(st models.SyncState) error {
*relayed = append(*relayed, st)
return nil
})
return previous
}
func TestSyncTruthRepeatedTransportFailuresRelayFreshObservations(t *testing.T) {
srv := httptest.NewServer(http.NotFoundHandler())
srv.Close()
var events []captured
w := newGoogleTestMail(t, srv, &events)
for i := 0; i < 2; i++ {
if err := w.syncOnce(t.Context()); err == nil {
t.Fatal("provider failure hidden")
}
}
errors := 0
for _, event := range events {
if event.eventType == models.JobEventTypeEmailServerError {
errors++
}
}
if errors != 2 || w.transportFailures != 2 {
t.Fatalf("observations=%d failures=%d", errors, w.transportFailures)
}
}
func assertNoSyncSuccess(t *testing.T, w *WMail, previous time.Time) {
t.Helper()
if w.tracker.state.LastSyncedAt == nil || !w.tracker.state.LastSyncedAt.Equal(previous) || w.tracker.state.Deferred != 5 {
t.Fatalf("incomplete pass replaced successful evidence/backlog: %+v", w.tracker.state)
}
}
func assertSyncCatchUp(t *testing.T, w *WMail, previous time.Time) {
t.Helper()
if w.tracker.state.LastSyncedAt == nil || !w.tracker.state.LastSyncedAt.After(previous) || w.tracker.state.Deferred != 0 {
t.Fatalf("successful catch-up did not replace old evidence: %+v", w.tracker.state)
}
}
func TestSyncTruthGoogleFailureHeartbeatAndRecovery(t *testing.T) {
failing := true
srv := httptest.NewServer(http.HandlerFunc(func(out http.ResponseWriter, _ *http.Request) {
out.Header().Set("Content-Type", "application/json")
if failing {
out.WriteHeader(http.StatusServiceUnavailable)
_, _ = out.Write([]byte(`{"error":{"code":503,"message":"retry later"}}`))
return
}
_, _ = out.Write([]byte(`{"historyId":"200"}`))
}))
defer srv.Close()
var events []captured
w := newGoogleTestMail(t, srv, &events)
w.GoogleData.LastHistoryID = 100
var relayed []models.SyncState
previous := seedSyncTruth(w, &relayed)
for range 2 {
w.tracker.lastSent = time.Now().Add(-2 * syncStateHeartbeat)
if err := w.SyncGoogle(t.Context()); err == nil {
t.Fatal("expected provider error")
}
assertNoSyncSuccess(t, w, previous)
}
if len(relayed) != 2 {
t.Fatalf("failed passes must relay safe heartbeat state: %d", len(relayed))
}
for _, st := range relayed {
if !st.LastSyncedAt.Equal(previous) || st.Deferred != 5 {
t.Fatalf("false success escaped via heartbeat: %+v", st)
}
}
failing = false
if err := w.SyncGoogle(t.Context()); err != nil {
t.Fatal(err)
}
assertSyncCatchUp(t, w, previous)
}
type syncTruthMap struct {
repository.EmailMessageMapRepository
failGet bool
failDel bool
}
func (r *syncTruthMap) Get(context.Context, uuid.UUID, uuid.UUID, string) (*repository.EmailMessageData, error) {
if r.failGet {
return nil, errors.New("map unavailable")
}
return nil, nil
}
func (r *syncTruthMap) Del(context.Context, uuid.UUID, uuid.UUID, string, uuid.UUID) error {
if r.failDel {
return errors.New("unmap unavailable")
}
return nil
}
func TestSyncTruthGraphAbortedAdmissionAndCheckpoint(t *testing.T) {
for _, failure := range []string{"map", "checkpoint"} {
t.Run(failure, func(t *testing.T) {
g := newFakeGraph()
var events []captured
var relayed []models.SyncState
w := newGraphTestMail(t, g.serve(t), &events, &relayed)
previous := seedSyncTruth(w, &relayed)
before := w.GraphData.Client.DeltaLinks[msgraph.FolderInbox]
r := &syncTruthMap{failGet: failure == "map"}
w.EmailMessageMapRepository = r
failing := true
w.onEvent = func(kind models.JobEventType, _ any) error {
if failing && failure == "checkpoint" && kind == models.JobEventTypeGraphDeltaUpdate {
return errors.New("checkpoint unavailable")
}
return nil
}
for range 2 {
if failure == "map" {
g.live = []string{"new-arrival"}
}
w.tracker.lastSent = time.Now().Add(-2 * syncStateHeartbeat)
_ = w.SyncGraph(t.Context())
assertNoSyncSuccess(t, w, previous)
if w.GraphData.Client.DeltaLinks[msgraph.FolderInbox] != before {
t.Fatal("failed/aborted page advanced local checkpoint")
}
}
failing, r.failGet = false, false
if err := w.SyncGraph(t.Context()); err != nil {
t.Fatal(err)
}
assertSyncCatchUp(t, w, previous)
})
}
}
func TestSyncTruthRetryUnmapAndCancellation(t *testing.T) {
for _, provider := range []string{"google", "graph", "imap"} {
t.Run(provider, func(t *testing.T) {
w, _ := newIMAPTestMail(&fakeImapConn{}, &fixedBudget{}, &models.Mailbox{})
var relayed []models.SyncState
previous := seedSyncTruth(w, &relayed)
w.unmapPending = map[string]uuid.UUID{"failed-arrival": uuid.New()}
w.EmailMessageMapRepository = &syncTruthMap{failDel: true}
switch provider {
case "google":
_ = w.SyncGoogle(t.Context())
case "graph":
_ = w.SyncGraph(t.Context())
case "imap":
_ = w.Sync(t.Context())
}
assertNoSyncSuccess(t, w, previous)
if len(relayed) != 1 {
t.Fatal("retry-unmap blockage did not relay safe state")
}
})
}
w, _ := newIMAPTestMail(&fakeImapConn{}, &fixedBudget{}, &models.Mailbox{})
var relayed []models.SyncState
previous := seedSyncTruth(w, &relayed)
ctx, cancel := context.WithCancel(t.Context())
cancel()
_ = w.Sync(ctx)
assertNoSyncSuccess(t, w, previous)
}
func TestSyncTruthImapFlagFailureRetainsBothUpdatesAndSuccessEvidence(t *testing.T) {
box := &models.Mailbox{Name: "INBOX", UIDValidity: 7, UIDNext: 101}
conn := &fakeImapConn{noCondStore: true, folders: []models.Mailbox{*box}, flags: map[uint32]imap.FlagState{
1: {MessageID: "one", Flags: nil}, 2: {MessageID: "two", Flags: nil},
}}
w, events := newIMAPTestMail(conn, &fixedBudget{allow: 10}, box)
w.EmailMessageMapRepository = knownMessageMap{id: uuid.NewString()}
if err := w.Sync(t.Context()); err != nil {
t.Fatal(err)
}
var relayed []models.SyncState
previous := seedSyncTruth(w, &relayed)
conn.flags[1], conn.flags[2] = imap.FlagState{MessageID: "one", Flags: []string{models.FlagSeen}}, imap.FlagState{MessageID: "two", Flags: []string{models.FlagSeen}}
w.flagScan[box.Name].at = time.Now().Add(-2 * config.ImapFlagScanInterval)
failing := true
w.onEvent = func(kind models.JobEventType, body any) error {
if kind == models.JobEventTypeEmailUpdate && failing {
return errors.New("publisher unavailable")
}
*events = append(*events, captured{eventType: kind, body: body})
return nil
}
if err := w.Sync(t.Context()); err != nil {
t.Fatal(err)
}
assertNoSyncSuccess(t, w, previous)
for uid := range conn.flags {
if w.flagScan[box.Name].flags[uid] != flagFingerprint(nil) {
t.Fatal("failed/unvisited flags were absorbed into baseline")
}
}
failing = false
w.flagScan[box.Name].at = time.Now().Add(-2 * config.ImapFlagScanInterval)
if err := w.Sync(t.Context()); err != nil {
t.Fatal(err)
}
updates := 0
for _, e := range *events {
if e.eventType == models.JobEventTypeEmailUpdate {
updates++
}
}
if updates != 2 {
t.Fatalf("unchanged provider flags did not retry both updates: %d", updates)
}
assertSyncCatchUp(t, w, previous)
}
func TestSyncTruthPanicDoesNotResetTransportFailure(t *testing.T) {
w, _ := newIMAPTestMail(&fakeImapConn{}, &fixedBudget{}, &models.Mailbox{})
w.EmailType = models.InboxProviderGoogle
w.GoogleData = nil // Fault inside the provider pass, after beginTick.
w.transportFailures = 2
var relayed []models.SyncState
previous := seedSyncTruth(w, &relayed)
if err := w.syncOnce(t.Context()); err == nil {
t.Fatal("recovered panic returned successful outcome")
}
assertNoSyncSuccess(t, w, previous)
if w.transportFailures != 2 {
t.Fatal("panic reset existing transport warning/backoff")
}
}
func TestSyncTruthBoundedAndPinnedProviderPasses(t *testing.T) {
for _, provider := range []string{"google", "graph"} {
for _, outcome := range []string{"bounded", "pinned", "stop"} {
t.Run(provider+"/"+outcome, func(t *testing.T) {
caughtUp := false
srv := httptest.NewServer(http.HandlerFunc(func(out http.ResponseWriter, r *http.Request) {
out.Header().Set("Content-Type", "application/json")
if provider == "google" {
if caughtUp {
_, _ = out.Write([]byte(`{"historyId":"200"}`))
return
}
if outcome == "bounded" {
_, _ = out.Write([]byte(`{"historyId":"200","nextPageToken":"next"}`))
return
}
_, _ = out.Write([]byte(`{"historyId":"200","history":[{"id":"150","messagesAdded":[{"message":{"id":"new-arrival","threadId":"thread"}}]}]}`))
return
}
folder := msgraph.FolderInbox
if caughtUp {
_, _ = fmt.Fprintf(out, `{"value":[],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/me/mailFolders/%s/messages/delta?final=1"}`, folder)
return
}
if outcome == "bounded" {
_, _ = fmt.Fprintf(out, `{"value":[],"@odata.nextLink":"https://graph.microsoft.com%s?next=1"}`, r.URL.Path)
return
}
_, _ = fmt.Fprintf(out, `{"value":[{"id":"new-arrival","isRead":false}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/me/mailFolders/%s/messages/delta?final=1"}`, folder)
}))
defer srv.Close()
var events []captured
var relayed []models.SyncState
var w *WMail
var sync func(context.Context) *errx.MailError
if provider == "google" {
w = newGoogleTestMail(t, srv, &events)
w.GoogleData.LastHistoryID = 100
w.GoogleData.Client.OnMessageAdded = func(context.Context, string, string) (bool, error) {
if outcome == "stop" {
return false, goog.ErrStop
}
return false, nil
}
sync = w.SyncGoogle
} else {
w = newGraphTestMail(t, srv, &events, &relayed)
w.GraphData.Client.OnMessageSeen = func(context.Context, string, string, bool) (bool, error) {
if outcome == "stop" {
return false, msgraph.ErrStop
}
return false, nil
}
sync = w.SyncGraph
}
previous := seedSyncTruth(w, &relayed)
w.transportFailures = 2
if err := sync(t.Context()); err != nil {
t.Fatal(err)
}
assertNoSyncSuccess(t, w, previous)
caughtUp = true
if err := sync(t.Context()); err != nil {
t.Fatal(err)
}
assertSyncCatchUp(t, w, previous)
})
}
}
}
func TestSyncTruthGoogleCheckpointFailureAndDirtyRelayRetry(t *testing.T) {
failingProvider := false
srv := httptest.NewServer(http.HandlerFunc(func(out http.ResponseWriter, _ *http.Request) {
out.Header().Set("Content-Type", "application/json")
if failingProvider {
out.WriteHeader(503)
_, _ = out.Write([]byte(`{"error":{"code":503,"message":"injected"}}`))
return
}
_, _ = out.Write([]byte(`{"historyId":"200"}`))
}))
defer srv.Close()
var events []captured
w := newGoogleTestMail(t, srv, &events)
w.GoogleData.LastHistoryID = 100
var relayed []models.SyncState
previous := seedSyncTruth(w, &relayed)
w.transportFailures = 2
w.onEvent = func(kind models.JobEventType, _ any) error {
if kind == models.JobEventTypeHistoryIDUpdate {
return errors.New("checkpoint unavailable")
}
return nil
}
if err := w.syncOnce(t.Context()); err != nil {
t.Fatal(err)
}
assertNoSyncSuccess(t, w, previous)
if w.GoogleData.LastHistoryID != 100 || w.transportFailures != 2 {
t.Fatal("failed checkpoint was installed or claimed recovery")
}
failingProvider = true
expired := time.Now().Add(-time.Minute)
w.tracker.state.ThrottledUntil = &expired
w.tracker.lastSent = time.Now().Add(-2 * syncStateHeartbeat)
w.tracker.emit = func(st models.SyncState) error { return errors.New("state relay unavailable") }
_ = w.SyncGoogle(t.Context())
assertNoSyncSuccess(t, w, previous)
if !w.tracker.dirty || w.tracker.state.ThrottledUntil != nil {
t.Fatal("failed relay forgot dirty throttle progress")
}
w.tracker.emit = func(st models.SyncState) error { relayed = append(relayed, st); return nil }
_ = w.SyncGoogle(t.Context())
assertNoSyncSuccess(t, w, previous)
if w.tracker.dirty || relayed[len(relayed)-1].ThrottledUntil != nil {
t.Fatal("safe dirty-state relay did not retry")
}
}
internal/app/worker/wmail/wmail.go
+32 -23
View File
@@ -2,6 +2,7 @@ package wmail
import (
"context"
"sync"
"time"
"github.com/google/uuid"
@@ -58,10 +59,18 @@ type SmtpImapData struct {
}
type WMail struct {
diagnosticRetries map[uuid.UUID]diagnosticRetry
ExecutorID uuid.UUID
UserID uuid.UUID
ID uuid.UUID
lifecycleMu sync.Mutex
stopped bool
stopDone chan struct{}
active sync.WaitGroup
syncMu sync.Mutex
executionKeys map[[32]byte]struct{}
initialExecutionKey [32]byte
executionData *models.AddWorkerEmail
diagnosticRetries map[uuid.UUID]diagnosticRetry
ExecutorID uuid.UUID
UserID uuid.UUID
ID uuid.UUID
// OrgID scopes the organization-wide sync budget; nil for a legacy
// personal mailbox.
OrgID *uuid.UUID
@@ -169,6 +178,13 @@ func NewWMail(
SyncContext: syncContext,
CipherService: cipherService,
}
mail.rememberExecution(data)
ready := false
defer func() {
if !ready {
mail.Discard()
}
}()
// A publisher older than the sync policy sends no Sync block; compiled
// defaults then apply and the mailbox starts its backfill from scratch.
@@ -251,9 +267,10 @@ func NewWMail(
FirstName: data.FirstName,
LastName: data.LastName,
User: data.Graph.User,
Cache: mail.Cache,
DeltaLinks: cloneStringMap(deltaLinks),
User: data.Graph.User,
Cache: mail.Cache,
DeltaLinks: cloneStringMap(deltaLinks),
ImmutableIDs: mail.SyncContext != nil,
OnMessageSeen: mail.onGraphMessageSeen,
OnMessageRemove: mail.onGraphMessageRemove,
@@ -286,6 +303,7 @@ func NewWMail(
if data.ImapSync {
conn := &imap.Client{
Context: mailCtx,
Email: data.Email,
AuthType: models.AuthPlain,
Credentials: data.SmtpImap.Credentials.IMAP,
@@ -294,13 +312,11 @@ func NewWMail(
return nil, err
}
mail.SmtpImapData.ImapClient = conn
// Saved folder cursors: live sync resumes from each folder's stored
// HIGHESTMODSEQ instead of re-baselining (and, before this, instead
// of re-walking every folder on every worker restart).
for i := range data.SmtpImap.Mailboxes {
box := data.SmtpImap.Mailboxes[i]
mail.SmtpImapData.Mailboxes = append(mail.SmtpImapData.Mailboxes, &box)
}
}
// Keep saved cursors even while IMAP sync is temporarily disabled.
for i := range data.SmtpImap.Mailboxes {
box := data.SmtpImap.Mailboxes[i]
mail.SmtpImapData.Mailboxes = append(mail.SmtpImapData.Mailboxes, &box)
}
mail.SmtpImapData.SmtpClient = &smtp.Client{
@@ -319,6 +335,7 @@ func NewWMail(
)
}
ready = true
return mail, nil
}
@@ -335,13 +352,5 @@ func (w *WMail) ApplySyncPolicy(data *models.AddWorkerEmailSyncData) {
// Discard releases a WMail that was built but never put to work.
func (w *WMail) Discard() {
if w.Cancel != nil {
w.Cancel()
}
if w.SmtpImapData == nil || w.SmtpImapData.ImapClient == nil {
return
}
if c, ok := w.SmtpImapData.ImapClient.(interface{ Close() error }); ok {
_ = c.Close()
}
w.Stop()
}
internal/client/goog/deadline_test.go
+32
View File
@@ -0,0 +1,32 @@
package goog
import (
"context"
"testing"
"github.com/warmbly/warmbly/internal/pkg/stoken"
"golang.org/x/oauth2"
)
func TestProviderDefaultRequestDeadline(t *testing.T) {
for _, brokered := range []bool{false, true} {
c := &Client{}
token := &oauth2.Token{AccessToken: "local-test"}
var err error
if brokered {
if merr := c.InitWithSource(context.Background(), oauth2.StaticTokenSource(token)); merr != nil {
err = merr
}
} else {
if merr := c.Init(context.Background(), token, oauth2.Config{}); merr != nil {
err = merr
}
}
if err != nil {
t.Fatal(err)
}
if c.rawClient.Timeout != stoken.ProviderRequestTimeout {
t.Fatalf("brokered=%v default timeout=%v", brokered, c.rawClient.Timeout)
}
}
}
internal/client/goog/goog.go
+2 -1
View File
@@ -47,6 +47,7 @@ type Client struct {
}
func (c *Client) Init(ctx context.Context, token *oauth2.Token, cfg oauth2.Config) *errx.MailError {
ctx = stoken.BoundedContext(ctx)
ts := cfg.TokenSource(ctx, token)
ts = oauth2.ReuseTokenSource(token, ts)
// Only wrap when there is somewhere to persist to. stoken calls onUpdate on
@@ -63,7 +64,7 @@ func (c *Client) Init(ctx context.Context, token *oauth2.Token, cfg oauth2.Confi
// InitWithSource builds the client on a caller-owned token source (brokered
// tokens from Warmbly Cloud); nothing is persisted from it.
func (c *Client) InitWithSource(ctx context.Context, ts oauth2.TokenSource) *errx.MailError {
httpClient := oauth2.NewClient(ctx, ts)
httpClient := stoken.HTTPClient(ctx, ts)
c.rawClient = httpClient
var err error
c.srv, err = gmail.NewService(ctx, option.WithHTTPClient(httpClient))
internal/client/goog/history.go
+25 -6
View File
@@ -14,9 +14,13 @@ import (
// returns the checkpoint reached so far and a nil error.
var ErrStop = errors.New("goog: stop")
var errHistoryIncomplete = errors.New("goog: history pass incomplete")
var ErrHistoryExpired = errors.New("goog: history expired; resynchronization required")
var ErrRecoveryPageExpired = errors.New("goog: recovery page token expired")
var ErrPageTokenExpired = errors.New("goog: messages.list page token expired")
var ErrRecoveryPageExpired = ErrPageTokenExpired
// historyPagesPerPass bounds one walk. A mailbox that has fallen far behind
// (or is held by fair use, so the checkpoint cannot advance) is caught up
@@ -35,6 +39,20 @@ const historyPagesPerPass = 10
// continues, so later records that are admitted (a reply, say) still land and
// the deferred ones are re-offered next tick.
func (c *Client) FetchHistory(ctx context.Context, lastHistoryID uint64) (uint64, error) {
checkpoint, _, err := c.FetchHistoryPass(ctx, lastHistoryID)
return checkpoint, err
}
// FetchHistoryPass distinguishes a bounded or pinned walk from full live catch-up.
func (c *Client) FetchHistoryPass(ctx context.Context, lastHistoryID uint64) (uint64, bool, error) {
checkpoint, err := c.fetchHistory(ctx, lastHistoryID)
if errors.Is(err, errHistoryIncomplete) {
return checkpoint, false, nil
}
return checkpoint, err == nil, err
}
func (c *Client) fetchHistory(ctx context.Context, lastHistoryID uint64) (uint64, error) {
if lastHistoryID == 0 {
return c.HistoryBaseline(ctx)
}
@@ -62,7 +80,7 @@ func (c *Client) FetchHistory(ctx context.Context, lastHistoryID uint64) (uint64
}
added, err := c.OnMessageAdded(ctx, m.Message.Id, m.Message.ThreadId)
if errors.Is(err, ErrStop) {
return checkpoint, nil
return checkpoint, errHistoryIncomplete
}
if err != nil {
return checkpoint, err
@@ -96,13 +114,14 @@ func (c *Client) FetchHistory(ctx context.Context, lastHistoryID uint64) (uint64
if resp.NextPageToken == "" {
if !pinned {
checkpoint = resp.HistoryId
return checkpoint, nil
}
break
return checkpoint, errHistoryIncomplete
}
call.PageToken(resp.NextPageToken)
}
return checkpoint, nil
return checkpoint, errHistoryIncomplete
}
func (c *Client) HistoryBaseline(ctx context.Context) (uint64, error) {
@@ -185,8 +204,8 @@ func (c *Client) listMessages(ctx context.Context, q, pageToken string, max int6
resp, err := call.Do()
if err != nil {
var apiErr *googleapi.Error
if includeSpamTrash && pageToken != "" && errors.As(err, &apiErr) && apiErr.Code == 400 && strings.Contains(strings.ReplaceAll(strings.ToLower(apiErr.Message), " ", ""), "invalidpagetoken") {
return nil, "", ErrRecoveryPageExpired
if pageToken != "" && errors.As(err, &apiErr) && apiErr.Code == 400 && strings.Contains(strings.ReplaceAll(strings.ToLower(apiErr.Message), " ", ""), "invalidpagetoken") {
return nil, "", ErrPageTokenExpired
}
return nil, "", fmt.Errorf("gmail messages.list: %w", HandleError(err))
}
internal/client/goog/history_test.go
+10 -2
View File
@@ -76,7 +76,15 @@ func TestRecoveryPageExpiryIsNarrow(t *testing.T) {
{"other invalid input", "old", "Invalid query", 400, true, false},
{"first page invalid input", "", "Invalid pageToken", 400, true, false},
{"transient", "old", "Invalid pageToken", 429, true, false},
{"ordinary backfill", "old", "Invalid pageToken", 400, false, false},
{"ordinary backfill", "old", "Invalid pageToken", 400, false, true},
{"ordinary spaced token", "old", "Invalid page token", 400, false, true},
{"ordinary other invalid input", "old", "Invalid query", 400, false, false},
{"ordinary invalid page size", "old", "Invalid page size", 400, false, false},
{"ordinary first page", "", "Invalid pageToken", 400, false, false},
{"ordinary quota", "old", "Invalid pageToken", 429, false, false},
{"ordinary auth", "old", "Invalid pageToken", 401, false, false},
{"ordinary forbidden", "old", "Invalid pageToken", 403, false, false},
{"ordinary transport", "old", "Invalid pageToken", 503, false, false},
} {
t.Run(tc.name, func(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
@@ -89,7 +97,7 @@ func TestRecoveryPageExpiryIsNarrow(t *testing.T) {
t.Fatal(err)
}
_, _, err = (&Client{srv: svc}).listMessages(t.Context(), "after:2026/09/01", tc.token, 50, tc.recovery)
if err == nil || errors.Is(err, ErrRecoveryPageExpired) != tc.expired {
if err == nil || errors.Is(err, ErrPageTokenExpired) != tc.expired || errors.Is(err, ErrRecoveryPageExpired) != tc.expired {
t.Fatalf("err=%v; want expired=%t", err, tc.expired)
}
})
internal/client/msgraph/deadline_test.go
+32
View File
@@ -0,0 +1,32 @@
package msgraph
import (
"context"
"testing"
"github.com/warmbly/warmbly/internal/pkg/stoken"
"golang.org/x/oauth2"
)
func TestProviderDefaultRequestDeadline(t *testing.T) {
for _, brokered := range []bool{false, true} {
c := &Client{}
token := &oauth2.Token{AccessToken: "local-test"}
var err error
if brokered {
if merr := c.InitWithSource(context.Background(), oauth2.StaticTokenSource(token)); merr != nil {
err = merr
}
} else {
if merr := c.Init(context.Background(), token, oauth2.Config{}); merr != nil {
err = merr
}
}
if err != nil {
t.Fatal(err)
}
if c.hc.Timeout != stoken.ProviderRequestTimeout {
t.Fatalf("brokered=%v default timeout=%v", brokered, c.hc.Timeout)
}
}
}
internal/client/msgraph/delta.go
+107 -32
View File
@@ -10,6 +10,7 @@ import (
"strconv"
"time"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/models"
)
@@ -19,6 +20,15 @@ func itoa(n int) string { return strconv.Itoa(n) }
// where it was last persisted.
var ErrStop = errors.New("msgraph: stop")
var errDeltaIncomplete = errors.New("msgraph: delta pass incomplete")
type recoveryContextKey struct{}
func IsRecovery(ctx context.Context) bool {
recovery, _ := ctx.Value(recoveryContextKey{}).(bool)
return recovery
}
// deltaSelect keeps delta pages light: we only need the id, read state, and the
// @removed marker to decide add/update vs remove. Full envelope + body + headers
// are hydrated per admitted message via FetchMessage.
@@ -27,7 +37,7 @@ const deltaSelect = "id,isRead"
// msgSelect is the property set fetched for each new message: enough to build a
// complete EmailMessageData (envelope, threading, body) plus internetMessageHeaders
// so the warmup verification token survives into sync.
const msgSelect = "id,internetMessageId,conversationId,subject,bodyPreview,isRead," +
const msgSelect = "id,parentFolderId,internetMessageId,conversationId,subject,bodyPreview,isRead," +
"receivedDateTime,from,sender,toRecipients,ccRecipients,bccRecipients,replyTo,body,internetMessageHeaders"
// deltaPagesPerPass bounds one folder's walk per tick, like Gmail's
@@ -52,7 +62,7 @@ type listPage struct {
// for placement, sent so a conversation shows both sides (as the Gmail and
// IMAP paths already do), and drafts so the Drafts scope is populated on
// Outlook the way it is on the other two providers.
var TrackedFolders = []string{FolderInbox, FolderJunk, FolderSent, FolderDrafts}
var TrackedFolders = []string{FolderInbox, FolderJunk, FolderSent, FolderDrafts, FolderArchive}
// BackfillFolders are the folders the initial import walks. Junk is followed
// live for placement signals but its history is not worth importing, and would
@@ -64,42 +74,69 @@ var BackfillFolders = []string{FolderInbox, FolderSent, FolderArchive, FolderDra
// fits the disposable worker natively: no webhook endpoint, no subscription
// lifecycle, just a cursor the control plane persists via OnDelta.
func (c *Client) Sync(ctx context.Context) error {
_, err := c.SyncPass(ctx)
return err
}
// SyncPass reports catch-up only when every tracked folder reaches an unpinned delta link.
func (c *Client) SyncPass(ctx context.Context) (bool, error) {
complete := true
var firstErr error
for _, folder := range TrackedFolders {
if err := c.syncFolder(ctx, folder); err != nil {
if errors.Is(err, ErrStop) {
return nil
return false, nil
}
if errors.Is(err, errDeltaIncomplete) {
complete = false
continue
}
var mailErr *errx.MailError
if folder == FolderArchive && errors.As(err, &mailErr) && mailErr.Code == errx.MailErrorCodeNotFound {
continue
}
if errors.As(err, &mailErr) && (mailErr.Type == errx.MailErrorCritical || mailErr.Code == errx.MailErrorCodeSendingTooFast) {
return false, err
}
complete = false
if firstErr == nil {
firstErr = err
}
return err
}
}
return nil
return complete, firstErr
}
func (c *Client) initialDelta(folder string) string {
q := url.Values{}
q.Set("$select", deltaSelect)
// Do not filter the stream: a fixed date filter would eventually hide new moves of older mail.
return c.root() + "/mailFolders/" + folder + "/messages/delta?" + q.Encode()
}
func (c *Client) syncFolder(ctx context.Context, folder string) error {
next := c.DeltaLinks[folder]
// First run for this folder: there is no cursor yet. We page to the end
// purely to capture a deltaLink representing "now" and do NOT import the
// mailbox's existing history; the backfill does that under its budget.
// The walk is uncapped and nothing is persisted before the deltaLink: a
// saved intermediate nextLink would make the next pass treat the rest of
// the priming walk as live mail.
if next == "" {
next = c.root() + "/mailFolders/" + folder + "/messages/delta?$select=" + url.QueryEscape(deltaSelect)
for {
var pg deltaPage
if err := c.doJSON(ctx, "GET", next, nil, &pg); err != nil {
if c.PendingRecoveryLink != nil {
link, err := c.PendingRecoveryLink(folder)
if err != nil {
return err
}
if link != "" {
if err := c.finishDelta(ctx, folder, link, true); err != nil {
return err
}
switch {
case pg.NextLink != "":
next = pg.NextLink
case pg.DeltaLink != "":
return c.saveCursor(ctx, folder, pg.DeltaLink)
default:
return nil
// The captured baseline may be old; catch up its changes on the next pass.
return errDeltaIncomplete
}
}
next := c.DeltaLinks[folder]
if next == "" {
if c.OnRecoveryStart != nil {
if err := c.OnRecoveryStart(ctx, folder); err != nil {
return err
}
}
next = c.initialDelta(folder)
}
// The persisted cursor only moves past a page once everything on it was
@@ -107,15 +144,28 @@ func (c *Client) syncFolder(ctx context.Context, folder string) error {
// while the walk continues, so admitted messages further on still land
// and the deferred ones are re-offered next tick from the pinned link.
pinned := false
reset := false
for page := 0; page < deltaPagesPerPass; page++ {
var pg deltaPage
if err := c.doJSON(ctx, "GET", next, nil, &pg); err != nil {
if !reset && deltaExpired(err) {
if c.OnRecoveryStart != nil {
if err := c.OnRecoveryStart(ctx, folder); err != nil {
return err
}
}
reset = true
next = c.initialDelta(folder)
continue
}
return err
}
recovering := next == c.initialDelta(folder) || reset || (c.Recovering != nil && c.Recovering(folder))
pageCtx := context.WithValue(ctx, recoveryContextKey{}, recovering)
complete := true
for i := range pg.Value {
stored, err := c.applyDelta(ctx, folder, &pg.Value[i])
stored, err := c.applyDelta(pageCtx, folder, &pg.Value[i])
if err != nil {
return err
}
@@ -137,21 +187,46 @@ func (c *Client) syncFolder(ctx context.Context, folder string) error {
next = pg.NextLink
case pg.DeltaLink != "":
if !pinned {
return c.saveCursor(ctx, folder, pg.DeltaLink)
if recovering && c.OnRecoveryBaseline != nil {
if err := c.OnRecoveryBaseline(ctx, folder, pg.DeltaLink); err != nil {
return err
}
}
return c.finishDelta(ctx, folder, pg.DeltaLink, recovering)
}
return nil
return errDeltaIncomplete
default:
return nil
return errDeltaIncomplete
}
}
return nil
return errDeltaIncomplete
}
func (c *Client) finishDelta(ctx context.Context, folder, link string, recovering bool) error {
if recovering && c.OnFolderReconcile != nil {
complete, err := c.OnFolderReconcile(ctx, folder)
if err != nil {
return err
}
if !complete {
return errDeltaIncomplete
}
}
if recovering && c.OnRecoveryCheckpoint != nil {
if err := c.OnRecoveryCheckpoint(ctx, folder, link); err != nil {
return err
}
}
return c.saveCursor(ctx, folder, link)
}
func (c *Client) saveCursor(ctx context.Context, folder, link string) error {
c.DeltaLinks[folder] = link
if c.OnDelta != nil {
return c.OnDelta(ctx, folder, link)
if err := c.OnDelta(ctx, folder, link); err != nil {
return err
}
}
c.DeltaLinks[folder] = link
return nil
}
internal/client/msgraph/delta_test.go
+141
View File
@@ -0,0 +1,141 @@
package msgraph
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"golang.org/x/oauth2"
)
func deltaTestClient(t *testing.T, handler http.HandlerFunc, links map[string]string) *Client {
t.Helper()
srv := httptest.NewServer(handler)
t.Cleanup(srv.Close)
c := &Client{Email: "box@outlook.test", DeltaLinks: links}
ctx := context.WithValue(t.Context(), oauth2.HTTPClient, &http.Client{Transport: rewriteTo(srv.URL)})
if err := c.Init(ctx, &oauth2.Token{AccessToken: "live", Expiry: time.Now().Add(time.Hour)}, oauth2.Config{}); err != nil {
t.Fatal(err)
}
return c
}
func TestDeltaExpiryClassificationIsNarrow(t *testing.T) {
for _, tc := range []struct {
status int
code string
want bool
}{
{410, "syncStateNotFound", true}, {410, "", true}, {400, "syncStateNotFound", true},
{400, "InvalidRequest", false}, {401, "syncStateNotFound", false},
{403, "syncStateNotFound", false}, {429, "syncStateNotFound", false}, {503, "syncStateNotFound", false},
} {
resp := &http.Response{StatusCode: tc.status, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(`{"error":{"code":"` + tc.code + `"}}`))}
if got := deltaExpired(HandleError(resp)); got != tc.want {
t.Errorf("status=%d code=%s: expiry=%v, want %v", tc.status, tc.code, got, tc.want)
}
}
if deltaExpired(context.DeadlineExceeded) {
t.Fatal("transport failure reset a checkpoint")
}
}
func TestDeltaExpiredRecoveryReplaysBeforeCheckpointAndResumesOnReload(t *testing.T) {
base := "https://graph.microsoft.com/v1.0/me/mailFolders/inbox/messages/delta"
saved := base + "?$deltatoken=expired"
page2 := base + "?$skiptoken=recovery-2"
fresh := base + "?$deltatoken=fresh"
requests := []string{}
handler := func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
requests = append(requests, r.URL.RawQuery)
switch {
case r.URL.Query().Get("$deltatoken") == "expired":
w.WriteHeader(http.StatusGone)
_, _ = w.Write([]byte(`{"error":{"code":"syncStateNotFound"}}`))
case r.URL.Query().Get("$skiptoken") == "recovery-2":
_ = json.NewEncoder(w).Encode(map[string]any{"value": []any{map[string]any{"id": "later", "isRead": false}}, "@odata.deltaLink": fresh})
default:
_ = json.NewEncoder(w).Encode(map[string]any{"value": []any{map[string]any{"id": "unseen", "isRead": false}, map[string]any{"id": "known", "isRead": true}}, "@odata.nextLink": page2})
}
}
delivered := map[string]bool{}
c := deltaTestClient(t, handler, map[string]string{FolderInbox: saved})
denyLater := true
c.OnMessageSeen = func(_ context.Context, _, id string, seen bool) (bool, error) {
if id == "later" && denyLater {
return false, ErrStop
}
if id == "known" && !seen {
t.Fatal("lost changed read state")
}
delivered[id] = true
return true, nil
}
c.OnDelta = func(_ context.Context, _, link string) error {
if link == fresh && !delivered["later"] {
t.Fatal("fresh baseline published before replay")
}
saved = link
return nil
}
if err := c.syncFolder(t.Context(), FolderInbox); !errors.Is(err, ErrStop) {
t.Fatalf("first pass = %v, want budget stop", err)
}
if saved != page2 || !delivered["unseen"] || !delivered["known"] {
t.Fatalf("checkpoint=%s delivered=%v", saved, delivered)
}
denyLater = false
reloaded := deltaTestClient(t, handler, map[string]string{FolderInbox: saved})
reloaded.OnMessageSeen, reloaded.OnDelta = c.OnMessageSeen, c.OnDelta
before := len(requests)
if err := reloaded.syncFolder(t.Context(), FolderInbox); err != nil {
t.Fatal(err)
}
if saved != fresh || len(delivered) != 3 || len(requests) != before+1 || !strings.Contains(requests[before], "recovery-2") {
t.Fatalf("restart lost progress: saved=%s delivered=%v requests=%v", saved, delivered, requests)
}
}
func TestDeltaCheckpointPublicationFailurePinsReplay(t *testing.T) {
fresh := "https://graph.microsoft.com/v1.0/me/mailFolders/archive/messages/delta?$deltatoken=fresh"
c := deltaTestClient(t, func(w http.ResponseWriter, r *http.Request) {
_ = json.NewEncoder(w).Encode(map[string]any{"value": []any{map[string]any{"id": "unseen"}}, "@odata.deltaLink": fresh})
}, nil)
delivered := 0
c.OnMessageSeen = func(context.Context, string, string, bool) (bool, error) { delivered++; return true, nil }
c.OnDelta = func(context.Context, string, string) error { return errors.New("publication failed") }
if err := c.syncFolder(t.Context(), FolderArchive); err == nil || c.DeltaLinks[FolderArchive] != "" || delivered != 1 {
t.Fatalf("publication failure: err=%v links=%v delivered=%d", err, c.DeltaLinks, delivered)
}
}
func TestDeltaContinuesLaterFoldersAndSkipsOnlyAbsentArchive(t *testing.T) {
for _, earlyFailure := range []bool{false, true} {
seen := map[string]bool{}
c := deltaTestClient(t, func(w http.ResponseWriter, r *http.Request) {
folder := strings.Split(strings.TrimPrefix(r.URL.Path, "/v1.0/me/mailFolders/"), "/")[0]
seen[folder] = true
if folder == FolderArchive || (folder == FolderInbox && earlyFailure) {
status := http.StatusNotFound
if folder == FolderInbox {
status = http.StatusServiceUnavailable
}
w.WriteHeader(status)
_, _ = w.Write([]byte(`{"error":{"code":"ErrorItemNotFound"}}`))
return
}
_ = json.NewEncoder(w).Encode(map[string]any{"value": []any{}, "@odata.deltaLink": "https://graph.microsoft.com" + r.URL.Path + "?$deltatoken=fresh"})
}, nil)
complete, err := c.SyncPass(t.Context())
if (err != nil) != earlyFailure || complete == earlyFailure || len(seen) != len(TrackedFolders) {
t.Fatalf("earlyFailure=%v complete=%v err=%v seen=%v", earlyFailure, complete, err, seen)
}
}
}
internal/client/msgraph/error.go
+12
View File
@@ -2,13 +2,25 @@ package msgraph
import (
"encoding/json"
"errors"
"io"
"net/http"
"strings"
"time"
"github.com/warmbly/warmbly/internal/errx"
)
func deltaExpired(err error) bool {
var mailErr *errx.MailError
if !errors.As(err, &mailErr) || mailErr.Failure == nil {
return false
}
failure := mailErr.Failure
return failure.Status == http.StatusGone ||
(failure.Status == http.StatusBadRequest && strings.EqualFold(failure.Cause, "syncStateNotFound"))
}
// graphErrorEnvelope is Graph's standard error shape.
type graphErrorEnvelope struct {
Error struct {
internal/client/msgraph/message.go
+1
View File
@@ -15,6 +15,7 @@ import (
// the envelope, body, and headers.
type GraphMessage struct {
ID string `json:"id"`
ParentFolderID string `json:"parentFolderId"`
InternetMessageID string `json:"internetMessageId"`
ConversationID string `json:"conversationId"`
Subject string `json:"subject"`
internal/client/msgraph/msgraph.go
+29 -8
View File
@@ -11,7 +11,9 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"net"
"net/http"
"net/url"
"sync"
@@ -55,7 +57,9 @@ type Client struct {
// DeltaLinks holds the opaque per-folder delta cursor (well-known folder
// name -> deltaLink URL). Seeded from persisted state on init and advanced
// as sync runs; OnDelta persists each new value off the disposable worker.
DeltaLinks map[string]string
DeltaLinks map[string]string
ImmutableIDs bool
idMu sync.RWMutex
// folderIDs caches resolved folder ids (e.g. the created "Warmbly" folder)
// so we don't re-list on every warmup action.
@@ -66,10 +70,16 @@ type Client struct {
// read state: the caller dedupes, hydrates (FetchMessage) and stores as
// budget allows, and returns false for a message it left on the server,
// which pins the folder cursor before that page.
OnMessageSeen func(ctx context.Context, folder, providerID string, seen bool) (stored bool, err error)
OnMessageRemove func(ctx context.Context, providerID string) error
OnDelta func(ctx context.Context, folder, deltaLink string) error
OnTokenRefresh func(ctx context.Context, token *oauth2.Token) error
OnMessageSeen func(ctx context.Context, folder, providerID string, seen bool) (stored bool, err error)
OnMessageRemove func(ctx context.Context, providerID string) error
OnDelta func(ctx context.Context, folder, deltaLink string) error
OnTokenRefresh func(ctx context.Context, token *oauth2.Token) error
OnFolderReconcile func(ctx context.Context, folder string) (bool, error)
OnRecoveryStart func(ctx context.Context, folder string) error
OnRecoveryCheckpoint func(ctx context.Context, folder, link string) error
OnRecoveryBaseline func(ctx context.Context, folder, link string) error
PendingRecoveryLink func(folder string) (string, error)
Recovering func(folder string) bool
}
// Init builds the auto-refreshing OAuth2 HTTP client. It mirrors goog.Client.Init:
@@ -77,6 +87,7 @@ type Client struct {
// cfg and persists the new token via OnTokenRefresh (the worker relays it back to
// the control plane, which owns the encrypted credential store).
func (c *Client) Init(ctx context.Context, token *oauth2.Token, cfg oauth2.Config) *errx.MailError {
ctx = stoken.BoundedContext(ctx)
ts := cfg.TokenSource(ctx, token)
ts = oauth2.ReuseTokenSource(token, ts)
// Same guard as goog.Init: a nil OnTokenRefresh would panic per request.
@@ -91,7 +102,7 @@ func (c *Client) Init(ctx context.Context, token *oauth2.Token, cfg oauth2.Confi
// InitWithSource builds the client on a caller-owned token source (brokered
// tokens from Warmbly Cloud); nothing is persisted from it.
func (c *Client) InitWithSource(ctx context.Context, ts oauth2.TokenSource) *errx.MailError {
c.hc = oauth2.NewClient(ctx, ts)
c.hc = stoken.HTTPClient(ctx, ts)
if c.DeltaLinks == nil {
c.DeltaLinks = map[string]string{}
}
@@ -120,6 +131,9 @@ func (c *Client) do(ctx context.Context, method, url, contentType string, body [
if contentType != "" {
req.Header.Set("Content-Type", contentType)
}
if c.ImmutableIDMode() {
req.Header.Set("Prefer", `IdType="ImmutableId", odata.maxpagesize=50`)
}
return c.hc.Do(req)
}
@@ -147,9 +161,16 @@ func (c *Client) doJSON(ctx context.Context, method, url string, in, out any) er
return HandleError(resp)
}
if out != nil {
return json.NewDecoder(resp.Body).Decode(out)
err := json.NewDecoder(resp.Body).Decode(out)
var networkError net.Error
if errors.As(err, &networkError) || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
return transportError(err)
}
return err
}
if _, err := io.Copy(io.Discard, resp.Body); err != nil {
return transportError(err)
}
_, _ = io.Copy(io.Discard, resp.Body)
return nil
}
internal/client/msgraph/sync_identity.go
+80
View File
@@ -0,0 +1,80 @@
package msgraph
import (
"context"
"fmt"
"net/http"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/models"
)
func (c *Client) ImmutableIDMode() bool {
c.idMu.RLock()
defer c.idMu.RUnlock()
return c.ImmutableIDs
}
func (c *Client) SetImmutableIDMode(enabled bool) {
c.idMu.Lock()
defer c.idMu.Unlock()
c.ImmutableIDs = enabled
}
// ImmutableMessageIDs converts saved regular ids without changing mailbox contents.
func (c *Client) ImmutableMessageIDs(ctx context.Context, ids []string) (map[string]string, error) {
return c.translateIDs(ctx, ids, "restId", "restImmutableEntryId")
}
func (c *Client) RegularMessageIDs(ctx context.Context, ids []string) (map[string]string, error) {
return c.translateIDs(ctx, ids, "restImmutableEntryId", "restId")
}
func (c *Client) translateIDs(ctx context.Context, ids []string, source, target string) (map[string]string, error) {
var out struct {
Value []struct {
SourceID string `json:"sourceId"`
TargetID string `json:"targetId"`
} `json:"value"`
}
if err := c.doJSON(ctx, http.MethodPost, c.root()+"/translateExchangeIds", map[string]any{
"inputIds": ids, "sourceIdType": source, "targetIdType": target,
}, &out); err != nil {
return nil, err
}
converted := make(map[string]string, len(out.Value))
for _, id := range out.Value {
if id.SourceID != "" && id.TargetID != "" {
converted[id.SourceID] = id.TargetID
}
}
for _, id := range ids {
if converted[id] == "" {
return nil, fmt.Errorf("graph: incomplete immutable id conversion")
}
}
return converted, nil
}
// MessageFolder resolves current placement, not the source of a stale delta item.
func (c *Client) MessageFolder(ctx context.Context, msg *GraphMessage, fallback string) (string, error) {
if msg.ParentFolderID == "" {
return msg.ToEmailData(fallback).Folder, nil
}
for _, folder := range append(append([]string{}, TrackedFolders...), FolderDeletedItems) {
id, err := c.wellKnownFolderID(ctx, folder)
if err != nil {
if merr, ok := err.(*errx.MailError); ok && folder == FolderArchive && merr.Code == errx.MailErrorCodeNotFound {
continue
}
return "", err
}
if msg.ParentFolderID == id {
if folder == FolderDeletedItems {
return models.FolderTrash, nil
}
return msg.ToEmailData(folder).Folder, nil
}
}
return "", nil
}
internal/client/smtpimap/imap/body_read.go
+34 -10
View File
@@ -3,6 +3,7 @@ package imap
import (
"bytes"
"encoding/base64"
"errors"
"io"
"mime/quotedprintable"
"strconv"
@@ -35,14 +36,14 @@ const maxTextParts = 5
// bytes back as the HTML body too, so plain text was rendered as markup: line
// breaks collapsed, "&" showed as an entity, and anything inside angle
// brackets vanished.
func fetchTextParts(c *imapclient.Client, uid imap.UID, bs imap.BodyStructure) (plain, html string) {
func fetchTextParts(c *imapclient.Client, uid imap.UID, bs imap.BodyStructure) (plain, html string, err error) {
if bs == nil {
return "", ""
return "", "", nil
}
wanted := selectTextParts(bs)
if len(wanted) == 0 {
return "", ""
return "", "", nil
}
sections := make([]*imap.FetchItemBodySection, 0, len(wanted))
@@ -56,12 +57,15 @@ func fetchTextParts(c *imapclient.Client, uid imap.UID, bs imap.BodyStructure) (
})
}
raw := fetchSections(c, uid, sections)
raw, err := fetchSections(c, uid, sections)
if err != nil {
return "", "", err
}
var plainParts, htmlParts []string
for _, p := range wanted {
body, ok := raw[partKey(p.path)]
if !ok {
continue
return "", "", errors.New("imap: required body section missing")
}
decoded := decodePart(body, p.encoding, p.charset)
if p.html {
@@ -72,7 +76,7 @@ func fetchTextParts(c *imapclient.Client, uid imap.UID, bs imap.BodyStructure) (
}
// Sibling inline parts (a body followed by a disclaimer, say) are pieces of
// one message, so they are joined rather than one replacing the other.
return strings.Join(plainParts, "\n\n"), strings.Join(htmlParts, "")
return strings.Join(plainParts, "\n\n"), strings.Join(htmlParts, ""), nil
}
// reportPartTypes are the machine-readable parts of a multipart/report. They
@@ -159,16 +163,20 @@ func selectTextParts(bs imap.BodyStructure) []textPart {
// fetchSections runs one FETCH for the given sections and returns each part's
// raw (still transfer-encoded) bytes, keyed by part path.
func fetchSections(c *imapclient.Client, uid imap.UID, sections []*imap.FetchItemBodySection) map[string][]byte {
var errBodyMessageGone = errors.New("imap: body message no longer exists")
func fetchSections(c *imapclient.Client, uid imap.UID, sections []*imap.FetchItemBodySection) (map[string][]byte, error) {
out := make(map[string][]byte, len(sections))
cmd := c.Fetch(imap.UIDSetNum(uid), &imap.FetchOptions{
UID: true,
BodySection: sections,
})
defer cmd.Close()
seen := false
var readErr error
for msg := cmd.Next(); msg != nil; msg = cmd.Next() {
seen = true
for item := msg.Next(); item != nil; item = msg.Next() {
v, ok := item.(imapclient.FetchItemDataBodySection)
if !ok || v.Literal == nil || v.Section == nil {
@@ -178,13 +186,29 @@ func fetchSections(c *imapclient.Client, uid imap.UID, sections []*imap.FetchIte
// Belt and braces alongside the server-side Partial: a server that
// ignores the partial range must not be able to stream us an
// unbounded body.
if _, err := io.Copy(buf, io.LimitReader(v.Literal, int64(config.MaxEmailBodySize))); err != nil {
n, err := io.Copy(buf, io.LimitReader(v.Literal, int64(config.MaxEmailBodySize)))
if err == nil && n < min(v.Literal.Size(), int64(config.MaxEmailBodySize)) {
err = io.ErrUnexpectedEOF
}
if err != nil {
// Stop the decoder before go-imap tries discarding a short literal a second time.
_ = c.Close()
readErr = err
continue
}
out[partKey(v.Section.Part)] = buf.Bytes()
}
}
return out
if err := cmd.Close(); err != nil {
return nil, err
}
if readErr != nil {
return nil, readErr
}
if !seen {
return nil, errBodyMessageGone
}
return out, nil
}
// partKey renders an IMAP part path ("1", "1.2") so a fetched section can be
internal/client/smtpimap/imap/client.go
+101 -18
View File
@@ -35,6 +35,8 @@ var headerFetchFields = append([]string{config.WarmupVerifyHeader}, config.Inbou
var evidenceFetchFields = append(append([]string{}, headerFetchFields...), "Authentication-Results", "From", "Return-Path", "DKIM-Signature", "List-Unsubscribe", "List-Unsubscribe-Post")
type Client struct {
// Context optionally binds connections and setup to the installed mailbox lifetime.
Context context.Context
Email string
AuthType models.AuthType
Credentials *models.Service
@@ -79,6 +81,12 @@ type Client struct {
// like client itself.
conn *idleConn
setupMu sync.Mutex
setupCancel context.CancelFunc
transport net.Conn
lifetimeStop func() bool
closed atomic.Bool
// IdleTimeout bounds how long a command waits for the server to say
// anything before the session is declared dead. Zero means
// config.ImapCommandIdleTimeout.
@@ -130,6 +138,11 @@ func (c *Client) HasCondStore() bool {
return c.condStore.Load()
}
func (c *Client) selectedCondStore() bool {
sel := c.selection.Load()
return c.condStore.Load() && sel != nil && sel.modSeq > 0
}
// ensureConnected re-dials after the server has dropped the session. go-imap
// parks a dead client in the Logout state and fails every later command with
// net.ErrClosed; nothing re-dialed, so one drop (Gmail closes sessions after a
@@ -139,6 +152,9 @@ func (c *Client) HasCondStore() bool {
func (c *Client) ensureConnected() *errx.MailError {
c.lifecycle.Lock()
defer c.lifecycle.Unlock()
if c.closed.Load() || c.Context != nil && c.Context.Err() != nil {
return errx.ErrMailServerUnreachable
}
// Only a session that got past auth is worth keeping: a failed Login
// leaves go-imap in NotAuthenticated, which is just as unusable as Logout.
@@ -160,6 +176,18 @@ func (c *Client) Connect() *errx.MailError {
// connectLocked dials and authenticates a fresh session. lifecycle must be
// held for writing.
func (c *Client) connectLocked() *errx.MailError {
if c.closed.Load() || c.Context != nil && c.Context.Err() != nil {
return errx.ErrMailServerUnreachable
}
c.setupMu.Lock()
if c.lifetimeStop != nil {
c.lifetimeStop()
}
if c.transport != nil {
_ = c.transport.Close()
c.transport = nil
}
c.setupMu.Unlock()
var addr, host, security string
var port int
switch c.AuthType {
@@ -193,7 +221,26 @@ func (c *Client) connectLocked() *errx.MailError {
if resolved == models.MailSecurityNone && !models.CleartextMailAllowed(host) {
return errx.ErrMailInsecureRemoteHost
}
raw, err := netbind.Dialer(c.BindIP).DialContext(context.Background(), "tcp", addr)
lifetime := c.Context
if lifetime == nil {
lifetime = context.Background()
}
setupCtx, cancel := context.WithTimeout(lifetime, timeout)
c.setupMu.Lock()
if c.closed.Load() {
c.setupMu.Unlock()
cancel()
return errx.ErrMailServerUnreachable
}
c.setupCancel = cancel
c.setupMu.Unlock()
defer func() {
c.setupMu.Lock()
c.setupCancel = nil
c.setupMu.Unlock()
cancel()
}()
raw, err := netbind.Dialer(c.BindIP).DialContext(setupCtx, "tcp", addr)
if err != nil {
return errx.ErrMailServerUnreachable
}
@@ -201,7 +248,21 @@ func (c *Client) connectLocked() *errx.MailError {
_ = raw.Close()
return errx.ErrMailInsecureRemoteHost
}
c.setupMu.Lock()
if c.closed.Load() {
c.setupMu.Unlock()
_ = raw.Close()
return errx.ErrMailServerUnreachable
}
c.transport = raw
c.lifetimeStop = context.AfterFunc(lifetime, func() { _ = raw.Close() })
c.setupMu.Unlock()
conn := &idleConn{Conn: raw, timeout: timeout}
// Library greeting/STARTTLS waits precede client installation; close the pending socket too.
stopSetup := context.AfterFunc(setupCtx, func() { _ = raw.Close() })
defer stopSetup()
done := conn.arm()
defer done()
var client *imapclient.Client
switch {
@@ -216,9 +277,7 @@ func (c *Client) connectLocked() *errx.MailError {
}
default:
tconn := tls.Client(conn, tlsConf)
hctx, cancel := context.WithTimeout(context.Background(), timeout)
err = tconn.HandshakeContext(hctx)
cancel()
err = tconn.HandshakeContext(setupCtx)
if err != nil {
_ = tconn.Close()
return errx.ErrMailServerUnreachable
@@ -231,9 +290,6 @@ func (c *Client) connectLocked() *errx.MailError {
c.selected.Store(false)
c.selection.Store(nil)
c.condStore.Store(false)
done := conn.arm()
defer done()
var xerr *errx.MailError
switch c.AuthType {
@@ -254,17 +310,34 @@ func (c *Client) connectLocked() *errx.MailError {
// check must run post-auth. Without it (Outlook.com, Microsoft 365 over
// IMAP, Yahoo, many hosted servers) the sync loop keys on UIDNEXT instead.
c.condStore.Store(c.client.Caps().Has(imap.CapCondStore))
if !stopSetup() || setupCtx.Err() != nil {
_ = client.Close()
return errx.ErrMailServerUnreachable
}
return nil
}
func (c *Client) Close() error {
c.lifecycle.RLock()
defer c.lifecycle.RUnlock()
if c.client == nil {
if !c.closed.CompareAndSwap(false, true) {
return nil
}
return c.client.Close()
c.setupMu.Lock()
defer c.setupMu.Unlock()
if c.setupCancel != nil {
c.setupCancel()
}
if c.lifetimeStop != nil {
c.lifetimeStop()
}
if c.transport == nil {
return nil
}
err := c.transport.Close()
if errors.Is(err, net.ErrClosed) {
return nil
}
return err
}
func (c *Client) plainAuth() *errx.MailError {
@@ -312,7 +385,7 @@ func (c *Client) selectMailbox(mailbox string, opts *imap.SelectOptions) (*imap.
data, err := c.client.Select(mailbox, opts).Wait()
c.selected.Store(err == nil)
if err == nil {
c.selection.Store(&selection{name: mailbox, uidValidity: data.UIDValidity, count: data.NumMessages})
c.selection.Store(&selection{name: mailbox, uidValidity: data.UIDValidity, count: data.NumMessages, modSeq: data.HighestModSeq})
}
return data, err
}
@@ -649,7 +722,7 @@ func (c *Client) FetchEnvelopes(ctx context.Context, uids []imap.UID) ([]*Fetche
// never advertised it for one is a malformed fetch-att, and a strict
// parser answers BAD and syncs nothing: issue #405, where IONOS said
// `BAD expected fetch-att instead of "MODSEQ BODY.P"` on every pass.
ModSeq: c.condStore.Load(),
ModSeq: c.selectedCondStore(),
InternalDate: true,
RFC822Size: true,
BodySection: []*imap.FetchItemBodySection{{
@@ -720,9 +793,9 @@ func (c *Client) FetchEnvelopes(ctx context.Context, uids []imap.UID) ([]*Fetche
// the FetchEnvelopes command that produced it is closed: a nested FETCH on
// the same connection blocks until the outer one finishes, and the outer one
// cannot finish while we wait, which deadlocks the sync on the first message.
func (c *Client) FetchBody(f *Fetched) {
func (c *Client) FetchBody(f *Fetched) *errx.MailError {
if f == nil || f.Email == nil {
return
return nil
}
c.mu.Lock()
defer c.mu.Unlock()
@@ -730,10 +803,20 @@ func (c *Client) FetchBody(f *Fetched) {
defer c.lifecycle.RUnlock()
defer c.begin()()
if err := c.resumeSyncLocked(); err != nil {
log.Debug().Err(err).Uint32("uid", uint32(f.uid)).Msg("imap: could not re-open the folder to read a body")
return
return errx.ErrMailServerUnreachable
}
f.Email.BodyPlain, f.Email.BodyHTML = fetchTextParts(c.client, f.uid, f.body)
plain, html, err := fetchTextParts(c.client, f.uid, f.body)
if errors.Is(err, errBodyMessageGone) {
return errx.ErrMailResourceNotFound
}
if err != nil {
if mailErr := c.handleError(err); mailErr.Code != errx.MailErrorCodeNotFound {
return mailErr
}
return errx.ErrMailServerUnreachable
}
f.Email.BodyPlain, f.Email.BodyHTML = plain, html
return nil
}
// parseHeaderFlags reads a HEADER.FIELDS literal and renders the fetched
internal/client/smtpimap/imap/fetch_caps_test.go
+4
View File
@@ -87,6 +87,10 @@ func TestFetchEnvelopesAsksModSeqWithCondStore(t *testing.T) {
t.Fatalf("SelectForSync: %v", err)
}
c.condStore.Store(true)
selected := *c.selection.Load()
selected.modSeq = 1
c.selection.Store(&selected)
c.syncView = &selected
// The result is deliberately ignored: this server refuses MODSEQ when
// CONDSTORE was not enabled, which is #405's failure reproduced in
internal/client/smtpimap/imap/search_date.go
+1
View File
@@ -22,6 +22,7 @@ type selection struct {
name string
uidValidity uint32
count uint32
modSeq uint64
}
// dateScan is the INTERNALDATE read of one folder for one window. UIDs below
internal/client/smtpimap/imap/sync_repair_wire_test.go
+525
View File
@@ -0,0 +1,525 @@
package imap
import (
"bufio"
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"io"
"net"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
goimap "github.com/emersion/go-imap/v2"
"github.com/emersion/go-imap/v2/imapclient"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/models"
)
func repairWireServer(t *testing.T, serve func(net.Conn)) string {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = ln.Close() })
go func() {
for {
conn, err := ln.Accept()
if err != nil {
return
}
go func() { defer func() { _ = conn.Close() }(); serve(conn) }()
}
}()
return ln.Addr().String()
}
// This fixture advertises global CONDSTORE but selects a NOMODSEQ folder.
func bodyRepairFixture(t *testing.T, bodyReply func(net.Conn, string) bool) (*Client, *wireLog) {
t.Helper()
wire := &wireLog{}
addr := repairWireServer(t, func(conn net.Conn) {
_, _ = io.WriteString(conn, "* OK [CAPABILITY IMAP4rev1 CONDSTORE] ready\r\n")
r := bufio.NewReader(conn)
for {
line, err := r.ReadString('\n')
if err != nil {
return
}
wire.add(strings.TrimSpace(line))
fields := strings.Fields(line)
if len(fields) < 2 {
return
}
tag := fields[0]
switch strings.ToUpper(fields[1]) {
case "LOGIN":
_, _ = fmt.Fprintf(conn, "%s OK [CAPABILITY IMAP4rev1 CONDSTORE] authenticated\r\n", tag)
case "CAPABILITY":
_, _ = fmt.Fprintf(conn, "* CAPABILITY IMAP4rev1 CONDSTORE\r\n%s OK capability\r\n", tag)
case "EXAMINE":
_, _ = fmt.Fprintf(conn, "* 1 EXISTS\r\n* OK [UIDVALIDITY 7] generation\r\n* OK [UIDNEXT 2] next\r\n* OK [NOMODSEQ] no modseq\r\n%s OK [READ-ONLY] selected\r\n", tag)
case "UID":
if len(fields) > 2 && strings.EqualFold(fields[2], "SEARCH") {
_, _ = fmt.Fprintf(conn, "* SEARCH 1\r\n%s OK search\r\n", tag)
} else if strings.Contains(strings.ToUpper(line), "BODYSTRUCTURE") {
_, _ = fmt.Fprintf(conn, "* 1 FETCH (UID 1 FLAGS () ENVELOPE (NIL \"hello\" NIL NIL NIL NIL NIL NIL NIL \"<wire@test>\") BODYSTRUCTURE (\"TEXT\" \"PLAIN\" (\"CHARSET\" \"UTF-8\") NIL NIL \"7BIT\" 13 1))\r\n%s OK envelope\r\n", tag)
} else if bodyReply != nil {
if !bodyReply(conn, tag) {
return
}
} else {
_, _ = fmt.Fprintf(conn, "* 1 FETCH (UID 1 FLAGS (\\Seen) ENVELOPE (NIL NIL NIL NIL NIL NIL NIL NIL NIL \"<wire@test>\"))\r\n%s OK flags\r\n", tag)
}
default:
_, _ = fmt.Fprintf(conn, "%s BAD unexpected command\r\n", tag)
}
}
})
return clientTo(t, addr), wire
}
func TestFetchBodyWireErrorsRemainRetryable(t *testing.T) {
for _, failure := range []string{"tagged refusal", "short literal", "reset literal", "missing section", "failed completion"} {
t.Run(failure, func(t *testing.T) {
var attempts atomic.Int32
c, _ := bodyRepairFixture(t, func(conn net.Conn, tag string) bool {
if attempts.Add(1) == 1 {
switch failure {
case "tagged refusal":
_, _ = fmt.Fprintf(conn, "%s NO [NONEXISTENT] body refused\r\n", tag)
case "short literal":
_, _ = io.WriteString(conn, "* 1 FETCH (UID 1 BODY[1] {13}\r\nshort")
return false
case "reset literal":
_, _ = io.WriteString(conn, "* 1 FETCH (UID 1 BODY[1] {13}\r\nshort")
_ = conn.(*net.TCPConn).SetLinger(0)
return false
case "missing section":
_, _ = fmt.Fprintf(conn, "* 1 FETCH (UID 1)\r\n%s OK completed\r\n", tag)
case "failed completion":
_, _ = fmt.Fprintf(conn, "* 1 FETCH (UID 1 BODY[1] {13}\r\ncomplete body)\r\n%s NO incomplete\r\n", tag)
}
return true
}
_, _ = fmt.Fprintf(conn, "* 1 FETCH (UID 1 BODY[1] {13}\r\ncomplete body)\r\n%s OK body\r\n", tag)
return true
})
c.IdleTimeout = time.Second
if err := c.Connect(); err != nil {
t.Fatal(err)
}
if _, err := c.SelectForSync("INBOX"); err != nil {
t.Fatal(err)
}
fetched, err := c.FetchEnvelopes(t.Context(), []goimap.UID{1})
if err != nil || len(fetched) != 1 {
t.Fatalf("envelope: %v %v", fetched, err)
}
if err := c.FetchBody(fetched[0]); err == nil || err.Code == errx.MailErrorCodeNotFound {
t.Fatalf("hydration failure treated as bodyless or gone: %v", err)
}
if fetched[0].Email.BodyPlain != "" || fetched[0].Email.BodyHTML != "" {
t.Fatal("failed body installed partial text")
}
if err := c.ensureConnected(); err != nil {
t.Fatal(err)
}
if _, err := c.SelectForSync("INBOX"); err != nil {
t.Fatal(err)
}
fetched, err = c.FetchEnvelopes(t.Context(), []goimap.UID{1})
if err != nil || len(fetched) != 1 {
t.Fatalf("retry envelope: %v %v", fetched, err)
}
if err := c.FetchBody(fetched[0]); err != nil {
t.Fatal(err)
}
if fetched[0].Email.BodyPlain != "complete body" {
t.Fatalf("retry body = %q", fetched[0].Email.BodyPlain)
}
})
}
}
func TestFetchBodyFolderResumeErrorAndValidEmptyBodies(t *testing.T) {
var refuse atomic.Bool
c, _ := interceptingServer(t, nil, func(line string) string {
if refuse.Load() && strings.Contains(strings.ToUpper(line), "EXAMINE") {
return strings.Fields(line)[0] + " NO folder unavailable"
}
return ""
}, "Archive")
if err := c.Connect(); err != nil {
t.Fatal(err)
}
uid := putMessage(t, c)
if _, err := c.SelectForSync("INBOX"); err != nil {
t.Fatal(err)
}
fetched, err := c.FetchEnvelopes(t.Context(), []goimap.UID{uid})
if err != nil || len(fetched) != 1 {
t.Fatal("envelope failed")
}
if _, err := c.selectMailbox("Archive", nil); err != nil {
t.Fatal(err)
}
refuse.Store(true)
if err := c.FetchBody(fetched[0]); err == nil {
t.Fatal("failed folder resume swallowed")
}
refuse.Store(false)
if err := c.FetchBody(fetched[0]); err != nil {
t.Fatal(err)
}
for _, bs := range []goimap.BodyStructure{nil, &goimap.BodyStructureSinglePart{Type: "application", Subtype: "octet-stream"}, &goimap.BodyStructureSinglePart{Type: "text", Subtype: "plain", Extended: &goimap.BodyStructureSinglePartExt{Disposition: &goimap.BodyStructureDisposition{Value: "attachment"}}}} {
f := &Fetched{Email: &models.EmailMessageData{}, uid: uid, body: bs}
if err := c.FetchBody(f); err != nil || f.Email.BodyPlain != "" || f.Email.BodyHTML != "" {
t.Fatalf("valid no-text message refused: %v", err)
}
}
}
func TestNomodseqWireOmitsModseqAndChangedSince(t *testing.T) {
c, wire := bodyRepairFixture(t, nil)
if err := c.Connect(); err != nil {
t.Fatal(err)
}
if !c.HasCondStore() {
t.Fatal("fixture did not advertise CONDSTORE")
}
view, err := c.SelectForSyncState("INBOX")
if err != nil || view.HighestModSeq != 0 {
t.Fatalf("NOMODSEQ SELECT: %+v %v", view, err)
}
if _, err := c.FetchEnvelopes(t.Context(), []goimap.UID{1}); err != nil {
t.Fatal(err)
}
if uids, err := c.SearchNewSince(1); err != nil || len(uids) != 1 {
t.Fatalf("UIDNEXT fallback: %v %v", uids, err)
}
flags, err := c.FetchFlags(t.Context(), 1)
if err != nil || len(flags) != 1 {
t.Fatalf("flag fallback: %v %v", flags, err)
}
for _, line := range wire.commands("FETCH") {
if strings.Contains(line, "MODSEQ") || strings.Contains(line, "CHANGEDSINCE") {
t.Fatalf("NOMODSEQ command: %s", line)
}
}
for _, line := range wire.commands("SEARCH") {
if strings.Contains(line, "MODSEQ") || strings.Contains(line, "CHANGEDSINCE") {
t.Fatalf("NOMODSEQ search: %s", line)
}
}
}
func TestSetupBlackholesAreBoundedAndCanRedial(t *testing.T) {
for _, phase := range []string{"greeting", "STARTTLS response", "STARTTLS handshake", "implicit TLS handshake", "authentication"} {
t.Run(phase, func(t *testing.T) {
var attempts atomic.Int32
wire := &wireLog{}
addr := repairWireServer(t, func(conn net.Conn) {
attempt := attempts.Add(1)
if attempt == 1 && (phase == "greeting" || phase == "implicit TLS handshake") {
_, _ = io.Copy(io.Discard, conn)
return
}
_, _ = io.WriteString(conn, "* OK [CAPABILITY IMAP4rev1 STARTTLS] ready\r\n")
r := bufio.NewReader(conn)
for {
line, err := r.ReadString('\n')
if err != nil {
return
}
wire.add(strings.TrimSpace(line))
fields := strings.Fields(line)
if len(fields) < 2 {
return
}
tag := fields[0]
if attempt == 1 {
if phase == "STARTTLS handshake" {
_, _ = fmt.Fprintf(conn, "%s OK begin TLS\r\n", tag)
}
_, _ = io.Copy(io.Discard, conn)
return
}
switch strings.ToUpper(fields[1]) {
case "LOGIN":
_, _ = fmt.Fprintf(conn, "%s OK [CAPABILITY IMAP4rev1] logged in\r\n", tag)
case "CAPABILITY":
_, _ = fmt.Fprintf(conn, "* CAPABILITY IMAP4rev1\r\n%s OK capability\r\n", tag)
default:
_, _ = fmt.Fprintf(conn, "%s BAD unexpected command\r\n", tag)
}
}
})
c := clientTo(t, addr)
c.plaintext = phase == "authentication"
c.Credentials.Security = models.MailSecurityStartTLS
if phase == "implicit TLS handshake" {
c.Credentials.Security = models.MailSecurityTLS
}
c.IdleTimeout = 100 * time.Millisecond
start := time.Now()
if err := c.Connect(); err == nil {
t.Fatal("blackhole succeeded")
}
if time.Since(start) > 2*time.Second {
t.Fatal("setup budget did not bound silent peer")
}
if phase != "authentication" && len(wire.commands("LOGIN")) != 0 {
t.Fatal("TLS failure fell back to plaintext authentication")
}
// The second local connection uses test-only plaintext to isolate redial from certificate trust.
c.plaintext = true
if err := c.ensureConnected(); err != nil {
t.Fatalf("redial: %v", err)
}
if attempts.Load() != 2 {
t.Fatalf("connections = %d", attempts.Load())
}
})
}
}
func TestCloseAndLifetimeCancellationInterruptPendingSetup(t *testing.T) {
for _, explicit := range []bool{false, true} {
t.Run(fmt.Sprintf("explicit close=%t", explicit), func(t *testing.T) {
accepted := make(chan struct{})
ended := make(chan struct{})
addr := repairWireServer(t, func(conn net.Conn) { close(accepted); _, _ = io.Copy(io.Discard, conn); close(ended) })
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
c := clientTo(t, addr)
c.plaintext = false
c.Context = ctx
c.IdleTimeout = 10 * time.Second
c.Credentials.Security = models.MailSecurityStartTLS
result := make(chan *errx.MailError, 1)
go func() { result <- c.Connect() }()
select {
case <-accepted:
case <-time.After(time.Second):
t.Fatal("setup did not connect")
}
if explicit {
if err := c.Close(); err != nil {
t.Fatal(err)
}
} else {
cancel()
}
select {
case err := <-result:
if err == nil {
t.Fatal("cancelled setup succeeded")
}
case <-time.After(time.Second):
t.Fatal("pending setup did not cancel")
}
select {
case <-ended:
case <-time.After(time.Second):
t.Fatal("pending socket did not close")
}
if err := c.ensureConnected(); err == nil {
t.Fatal("retired lifetime redialed")
}
})
}
}
func TestCloseAndLifetimeCancellationCloseIdleSessionWithoutRedial(t *testing.T) {
for _, explicit := range []bool{false, true} {
t.Run(fmt.Sprintf("explicit close=%t", explicit), func(t *testing.T) {
ended := make(chan struct{})
var attempts atomic.Int32
addr := repairWireServer(t, func(conn net.Conn) {
attempts.Add(1)
_, _ = io.WriteString(conn, "* OK [CAPABILITY IMAP4rev1] ready\r\n")
r := bufio.NewReader(conn)
for {
line, err := r.ReadString('\n')
if err != nil {
close(ended)
return
}
_, _ = fmt.Fprintf(conn, "%s OK [CAPABILITY IMAP4rev1] authenticated\r\n", strings.Fields(line)[0])
}
})
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
c := clientTo(t, addr)
c.Context = ctx
if err := c.Connect(); err != nil {
t.Fatal(err)
}
if explicit {
if err := c.Close(); err != nil {
t.Fatal(err)
}
if err := c.Close(); err != nil {
t.Fatal(err)
}
} else {
cancel()
}
select {
case <-ended:
case <-time.After(time.Second):
t.Fatal("idle server did not observe socket EOF")
}
if err := c.ensureConnected(); err == nil || attempts.Load() != 1 {
t.Fatal("retired installed session reconnected")
}
})
}
}
func TestStartTLSSetupStillRejectsUntrustedCertificate(t *testing.T) {
t.Setenv("MAIL_TLS_INSECURE", "false")
certServer := httptest.NewTLSServer(nil)
defer certServer.Close()
wire := &wireLog{}
addr := repairWireServer(t, func(conn net.Conn) {
_, _ = io.WriteString(conn, "* OK [CAPABILITY IMAP4rev1 STARTTLS] ready\r\n")
line, err := bufio.NewReader(conn).ReadString('\n')
if err != nil {
return
}
wire.add(strings.TrimSpace(line))
_, _ = fmt.Fprintf(conn, "%s OK begin TLS\r\n", strings.Fields(line)[0])
secure := tls.Server(conn, certServer.TLS)
if err := secure.Handshake(); err == nil {
line, _ := bufio.NewReader(secure).ReadString('\n')
wire.add(strings.TrimSpace(line))
}
})
c := clientTo(t, addr)
c.plaintext = false
c.Credentials.Security = models.MailSecurityStartTLS
c.IdleTimeout = time.Second
if err := c.Connect(); err == nil {
t.Fatal("untrusted fixture certificate was accepted")
}
if len(wire.commands("LOGIN")) != 0 {
t.Fatal("certificate failure sent plaintext credentials")
}
}
type literalReadSignalConn struct {
net.Conn
ready chan struct{}
sawShort atomic.Bool
signaled atomic.Bool
}
func (c *literalReadSignalConn) Read(p []byte) (int, error) {
if c.sawShort.Load() && c.signaled.CompareAndSwap(false, true) {
c.ready <- struct{}{}
}
n, err := c.Conn.Read(p)
if strings.Contains(string(p[:n]), "short") {
c.sawShort.Store(true)
}
return n, err
}
func TestFetchBodyTLSLiteralFailureReleasesDecoderAndRetries(t *testing.T) {
var attempts atomic.Int32
bodyStarted := make(chan struct{}, 1)
literalReadReady := make(chan struct{}, 1)
unblockServer := make(chan struct{})
defer close(unblockServer)
c, _ := bodyRepairFixture(t, func(conn net.Conn, tag string) bool {
if attempts.Add(1) == 1 {
_, _ = io.WriteString(conn, "* 1 FETCH (UID 1 BODY[1] {13}\r\nshort")
bodyStarted <- struct{}{}
<-unblockServer
return false
}
_, _ = fmt.Fprintf(conn, "* 1 FETCH (UID 1 BODY[1] {13}\r\ncomplete body)\r\n%s OK body\r\n", tag)
return true
})
certServer := httptest.NewTLSServer(nil)
defer certServer.Close()
backendAddr := net.JoinHostPort(c.Credentials.Host, fmt.Sprint(c.Credentials.Port))
addr := repairWireServer(t, func(raw net.Conn) {
secure := tls.Server(raw, certServer.TLS)
upstream, err := net.DialTimeout("tcp", backendAddr, time.Second)
if err != nil {
return
}
defer func() { _ = upstream.Close() }()
go func() { _, _ = io.Copy(upstream, secure); _ = upstream.Close() }()
_, _ = io.Copy(secure, upstream)
// Close the raw transport without TLS close_notify to produce TLS unexpected EOF.
})
roots := x509.NewCertPool()
roots.AddCert(certServer.Certificate())
connect := func() {
raw, err := net.DialTimeout("tcp", addr, time.Second)
if err != nil {
t.Fatal(err)
}
conn := &idleConn{Conn: raw, timeout: time.Second}
secure := tls.Client(conn, &tls.Config{RootCAs: roots, ServerName: certServer.Certificate().DNSNames[0], MinVersion: tls.VersionTLS12})
if err := secure.HandshakeContext(t.Context()); err != nil {
_ = raw.Close()
t.Fatal(err)
}
c.client = imapclient.New(&literalReadSignalConn{Conn: secure, ready: literalReadReady}, nil)
c.conn = conn
c.transport = raw
c.selected.Store(false)
c.selection.Store(nil)
if err := c.plainAuth(); err != nil {
t.Fatal(err)
}
c.condStore.Store(c.client.Caps().Has(goimap.CapCondStore))
if _, err := c.SelectForSync("INBOX"); err != nil {
t.Fatal(err)
}
}
connect()
fetched, err := c.FetchEnvelopes(t.Context(), []goimap.UID{1})
if err != nil || len(fetched) != 1 {
t.Fatalf("TLS envelope: %v %v", fetched, err)
}
result := make(chan *errx.MailError, 1)
go func() { result <- c.FetchBody(fetched[0]) }()
select {
case <-bodyStarted:
case <-time.After(time.Second):
t.Fatal("TLS literal did not start")
}
select {
case <-literalReadReady:
case <-time.After(time.Second):
t.Fatal("TLS literal reader did not block on remaining bytes")
}
_ = c.transport.Close()
select {
case err := <-result:
if err == nil {
t.Fatal("TLS literal failure swallowed")
}
case <-time.After(time.Second):
t.Fatal("TLS literal error left decoder blocked")
}
connect()
fetched, err = c.FetchEnvelopes(t.Context(), []goimap.UID{1})
if err != nil || len(fetched) != 1 {
t.Fatalf("TLS retry envelope: %v %v", fetched, err)
}
if err := c.FetchBody(fetched[0]); err != nil || fetched[0].Email.BodyPlain != "complete body" {
t.Fatalf("TLS retry body: %q %v", fetched[0].Email.BodyPlain, err)
}
}
internal/infrastructure/db/migrations/000278_sync_arrival_outbox.down.sql
+8
View File
@@ -0,0 +1,8 @@
DO $$
BEGIN
IF EXISTS (SELECT 1 FROM sync_arrival_outbox) THEN
RAISE EXCEPTION 'Cannot downgrade with unresolved sync arrival delivery';
END IF;
END $$;
DROP TABLE sync_arrival_outbox;
internal/infrastructure/db/migrations/000278_sync_arrival_outbox.up.sql
+18
View File
@@ -0,0 +1,18 @@
CREATE TABLE sync_arrival_outbox (
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
email_id uuid NOT NULL REFERENCES email_accounts(id) ON DELETE CASCADE,
organization_id uuid NOT NULL REFERENCES organizations(id) ON DELETE CASCADE,
message_id text NOT NULL,
id uuid NOT NULL,
payload text NOT NULL,
stage smallint NOT NULL DEFAULT 0 CHECK (stage BETWEEN 0 AND 3),
lease uuid,
locked_until timestamptz,
retry_at timestamptz NOT NULL DEFAULT now(),
created_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, email_id, message_id),
FOREIGN KEY (user_id, email_id, message_id)
REFERENCES email_message_map(user_id, email_id, message_id) ON DELETE CASCADE
);
CREATE INDEX sync_arrival_outbox_due ON sync_arrival_outbox(retry_at, created_at);
CREATE INDEX sync_arrival_outbox_pending ON sync_arrival_outbox(user_id, email_id, id) WHERE stage = 0;
internal/infrastructure/db/sync_arrival_upgrade_test.go
+106
View File
@@ -0,0 +1,106 @@
package db
import (
"context"
"net/url"
"os"
"strings"
"testing"
"github.com/golang-migrate/migrate/v4/source/iofs"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
)
func TestLiveSyncArrivalUpgradePreservesLegacyMaps(t *testing.T) {
dsn := os.Getenv("WARMBLY_TEST_DB")
if dsn == "" {
t.Skip("WARMBLY_TEST_DB not set")
}
u, err := url.Parse(dsn)
if err != nil || (u.Hostname() != "127.0.0.1" && u.Hostname() != "localhost") {
t.Fatal("upgrade fixture requires local PostgreSQL")
}
ctx := t.Context()
admin, err := pgx.Connect(ctx, dsn)
if err != nil {
t.Fatal(err)
}
defer admin.Close(context.Background())
name := "arrival_upgrade_" + uuid.NewString()[:8]
quoted := pgx.Identifier{name}.Sanitize()
if _, err := admin.Exec(ctx, "CREATE DATABASE "+quoted); err != nil {
t.Fatal(err)
}
defer func() {
if _, err := admin.Exec(context.Background(), "DROP DATABASE "+quoted); err != nil {
t.Error(err)
}
}()
u.Path = "/" + name
source, err := iofs.New(migrationsFS, "migrations")
if err != nil {
t.Fatal(err)
}
m, err := openMigrate(source, u.String())
if err != nil {
t.Fatal(err)
}
defer m.Close()
if err := m.Migrate(277); err != nil {
t.Fatal(err)
}
conn, err := pgx.Connect(ctx, u.String())
if err != nil {
t.Fatal(err)
}
defer conn.Close(context.Background())
user, org, mailbox, id := uuid.New(), uuid.New(), uuid.New(), uuid.New()
for _, row := range []struct {
sql string
args []any
}{
{`INSERT INTO users(id,email,first_name,last_name)VALUES($1,$2,'Legacy','Fixture')`, []any{user, user.String() + "@test.local"}},
{`INSERT INTO organizations(id,name,slug,owner_user_id)VALUES($1,'Legacy',$2,$3)`, []any{org, org.String(), user}},
{`INSERT INTO email_accounts(id,user_id,organization_id,email,name,signature_plain,signature_html,provider)VALUES($1,$2,$3,$4,'Legacy','','','smtp_imap')`, []any{mailbox, user, org, mailbox.String() + "@test.local"}},
{`INSERT INTO email_sync_state(email_id,user_id,backfill_status,backfill_synced)VALUES($1,$2,'complete',42)`, []any{mailbox, user}},
{`INSERT INTO email_message_map(user_id,email_id,message_id,id,thread_id)VALUES($1,$2,'legacy',$3,'thread')`, []any{user, mailbox, id}},
} {
if _, err := conn.Exec(ctx, row.sql, row.args...); err != nil {
t.Fatal(err)
}
}
if err := m.Up(); err != nil {
t.Fatal(err)
}
var retained bool
if err := conn.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM email_message_map m JOIN email_sync_state s ON s.email_id=m.email_id WHERE m.id=$1 AND s.backfill_status='complete' AND s.backfill_synced=42 AND m.thread_id='thread')`, id).Scan(&retained); err != nil || !retained {
t.Fatal("upgrade changed legacy map or completed import", err)
}
var count int
if err := conn.QueryRow(ctx, `SELECT count(*) FROM sync_arrival_outbox`).Scan(&count); err != nil || count != 0 {
t.Fatal("upgrade invented pending arrivals", err)
}
if _, err := conn.Exec(ctx, `INSERT INTO sync_arrival_outbox(user_id,email_id,organization_id,message_id,id,payload)VALUES($1,$2,$3,'legacy',$4,'fixture')`, user, mailbox, org, id); err != nil {
t.Fatal(err)
}
down, err := migrationsFS.ReadFile("migrations/000278_sync_arrival_outbox.down.sql")
if err != nil {
t.Fatal(err)
}
if _, err := conn.Exec(ctx, string(down)); err == nil || !strings.Contains(err.Error(), "unresolved sync arrival") {
t.Fatalf("downgrade discarded pending delivery: %v", err)
}
if _, err := conn.Exec(ctx, `DELETE FROM sync_arrival_outbox`); err != nil {
t.Fatal(err)
}
if err := m.Steps(-1); err != nil {
t.Fatal(err)
}
if err := conn.QueryRow(ctx, `SELECT count(*) FROM email_message_map WHERE id=$1`, id).Scan(&count); err != nil || count != 1 {
t.Fatal("downgrade removed canonical map", err)
}
if err := m.Up(); err != nil {
t.Fatal(err)
}
}
internal/infrastructure/eventbus/bus.go
+24 -3
View File
@@ -72,6 +72,29 @@ type EventBus interface {
// the message; returning an error leaves it for redelivery.
type Handler func(ctx context.Context, msg Message) error
func retrySubscription(ctx context.Context, subscribe func(context.Context) error) error {
for {
if err := ctx.Err(); err != nil {
return err
}
err := subscribe(ctx)
if ctx.Err() != nil {
return ctx.Err()
}
if errors.Is(err, ErrBusClosed) {
return err
}
log.Warn().Msg("eventbus subscription interrupted; reopening for replay")
timer := time.NewTimer(time.Second)
select {
case <-ctx.Done():
timer.Stop()
return ctx.Err()
case <-timer.C:
}
}
}
// Message is the broker-neutral envelope passed to a Handler. Payload bytes
// are owned by the bus and must not be retained past the handler call.
//
@@ -87,9 +110,7 @@ type Message struct {
// handler that retries by returning an error can read it to know when the
// broker is about to stop redelivering and give up cleanly instead.
Attempt int
// Redelivers is true when a handler error leaves the message for another
// delivery (NATS). Kafka commits regardless, so a handler must not count
// on a retry there and should finish what it can on this delivery.
// Redelivers is true when a handler error leaves the message for retry.
Redelivers bool
}
internal/infrastructure/eventbus/kafka.go
+23 -6
View File
@@ -106,8 +106,7 @@ func (b *KafkaBus) Publish(ctx context.Context, topic, key string, payload []byt
// Subscribe creates a fresh consumer in the given group, subscribes to all
// topics, and blocks reading messages until ctx is cancelled or a fatal error
// occurs. Handler errors are logged but do not abort the loop; the message's
// offset is still stored, to match the existing kafka.Consumer.Consume behaviour.
// occurs. Failed handlers never store offsets; exhausted retries reopen for replay.
func (b *KafkaBus) Subscribe(ctx context.Context, topics []string, group string, handler Handler) error {
if len(topics) == 0 {
return errors.New("eventbus kafka: at least one topic required")
@@ -118,6 +117,12 @@ func (b *KafkaBus) Subscribe(ctx context.Context, topics []string, group string,
if handler == nil {
return errors.New("eventbus kafka: handler required")
}
return retrySubscription(ctx, func(ctx context.Context) error {
return b.subscribeOnce(ctx, topics, group, handler)
})
}
func (b *KafkaBus) subscribeOnce(ctx context.Context, topics []string, group string, handler Handler) error {
// Subscribing to a topic that does not exist yet returns no messages and
// no error, so a worker would sit silent rather than fail.
@@ -153,6 +158,17 @@ func (b *KafkaBus) Subscribe(ctx context.Context, topics []string, group string,
}
b.consumers = append(b.consumers, cons)
b.mu.Unlock()
defer func() {
b.mu.Lock()
for i, c := range b.consumers {
if c == cons {
b.consumers = append(b.consumers[:i], b.consumers[i+1:]...)
break
}
}
b.mu.Unlock()
cons.Close()
}()
err = cons.Consume(ctx, func(msg *ckf.Message) error {
topic := ""
@@ -163,10 +179,11 @@ func (b *KafkaBus) Subscribe(ctx context.Context, topics []string, group string,
hctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), handlerTimeout())
defer cancel()
if err := invokeHandler(hctx, handler, Message{
Topic: topic,
Key: string(msg.Key),
Payload: msg.Value,
Attempt: 1,
Topic: topic,
Key: string(msg.Key),
Payload: msg.Value,
Attempt: 1,
Redelivers: true,
}); err != nil {
log.Error().Err(err).Str("topic", topic).Msg("eventbus kafka handler error")
return err
internal/infrastructure/eventbus/kafka_test.go
+44
View File
@@ -4,11 +4,55 @@ package eventbus
import (
"context"
"errors"
"testing"
"github.com/warmbly/warmbly/internal/infrastructure/kafka"
)
func TestKafkaSubscriptionReopensAfterFailureOrUnexpectedExit(t *testing.T) {
for _, failure := range []error{errors.New("offset assignment lost"), nil} {
ctx, cancel := context.WithCancel(t.Context())
calls := 0
err := retrySubscription(ctx, func(context.Context) error {
calls++
if calls == 2 {
cancel()
}
return failure
})
cancel()
if !errors.Is(err, context.Canceled) || calls != 2 {
t.Fatalf("err=%v calls=%d", err, calls)
}
}
calls := 0
err := retrySubscription(t.Context(), func(context.Context) error { calls++; return ErrBusClosed })
if !errors.Is(err, ErrBusClosed) || calls != 1 {
t.Fatal("closed bus reopened")
}
}
func TestKafkaSubscriptionRetiresClientsBeforeReopening(t *testing.T) {
b := &KafkaBus{bootstrap: "127.0.0.1:1"}
ctx, cancel := context.WithCancel(t.Context())
cancel()
for i := 0; i < 3; i++ {
if err := b.subscribeOnce(ctx, []string{"sync-test"}, "sync-test", func(context.Context, Message) error { return nil }); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
b.mu.Lock()
retained := len(b.consumers)
b.mu.Unlock()
if retained != 0 {
t.Fatalf("exited subscription retained %d clients", retained)
}
}
if err := b.Close(); err != nil {
t.Fatal(err)
}
}
// TestKafkaBus_InterfaceSatisfaction is a compile-time check that KafkaBus
// satisfies the EventBus interface and that the constructors enforce their
// minimum required arguments. A live Kafka broker is intentionally not used
internal/infrastructure/kafka/consumer.go
+51 -8
View File
@@ -14,6 +14,7 @@ import (
)
var ErrClientClosed = errors.New("kafka: client closed")
var ErrAssignmentLost = errors.New("kafka: assignment lost before offset storage")
type consumerClient interface {
ReadMessage(time.Duration) (*ckf.Message, error)
@@ -23,10 +24,11 @@ type consumerClient interface {
}
type Consumer struct {
c consumerClient
Avrov2 *Avrov2
mu sync.Mutex
closed bool
c consumerClient
Avrov2 *Avrov2
mu sync.Mutex
closed bool
retryLimit int
}
type ConsumerConfig struct {
@@ -110,6 +112,9 @@ func (cons *Consumer) storeMessage(msg *ckf.Message) error {
if cons.closed {
return ErrClientClosed
}
if native, ok := cons.c.(interface{ AssignmentLost() bool }); ok && native.AssignmentLost() {
return ErrAssignmentLost
}
_, err := cons.c.StoreMessage(msg)
return err
}
@@ -126,16 +131,43 @@ func (cons *Consumer) Consume(ctx context.Context, handler func(msg *ckf.Message
if kafkaErr.Code() == ckf.ErrTimedOut {
continue
}
if kafkaErr.IsFatal() || kafkaErr.Code() == ckf.ErrMaxPollExceeded {
return fmt.Errorf("kafka: consumer must reopen: %w", err)
}
// Log transient Kafka errors and retry after a brief delay
log.Warn().Str("code", fmt.Sprintf("%d", kafkaErr.Code())).Err(kafkaErr).Msg("kafka consumer error")
time.Sleep(time.Second)
if err := consumerBackoff(ctx, time.Second); err != nil {
return err
}
continue
}
return fmt.Errorf("error reading message: %w", err)
}
if err := handler(msg); err != nil {
log.Error().Err(err).Msg("kafka message handler error")
backoff := 100 * time.Millisecond
limit := cons.retryLimit
if limit <= 0 {
limit = 4
}
for attempt := 1; ; attempt++ {
if err := handler(msg); err == nil {
break
} else {
log.Error().Err(err).Msg("kafka message handler error; retrying before advancing offsets")
if attempt >= limit {
return fmt.Errorf("kafka: handler retry budget exhausted before offset storage: %w", err)
}
}
if err := consumerBackoff(ctx, backoff); err != nil {
return err
}
cons.mu.Lock()
closed := cons.closed
cons.mu.Unlock()
if closed {
return ErrClientClosed
}
backoff = min(2*backoff, 5*time.Second)
}
// Needs enable.auto.offset.store=false; the background commit sends it.
@@ -143,8 +175,19 @@ func (cons *Consumer) Consume(ctx context.Context, handler func(msg *ckf.Message
if errors.Is(err, ErrClientClosed) {
return err
}
log.Warn().Err(err).Msg("kafka: storing a handled offset failed")
return fmt.Errorf("kafka: storing a handled offset failed: %w", err)
}
}
}
}
func consumerBackoff(ctx context.Context, delay time.Duration) error {
timer := time.NewTimer(delay)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
}
}
internal/infrastructure/kafka/consumer_retry_test.go
+120
View File
@@ -0,0 +1,120 @@
//go:build kafka
package kafka
import (
"context"
"errors"
"testing"
"time"
ckf "github.com/confluentinc/confluent-kafka-go/v2/kafka"
)
type orderedConsumer struct {
shutdownConsumer
queue []*ckf.Message
stored []ckf.Offset
reads int
storeErr error
lost bool
readErr error
}
func (c *orderedConsumer) ReadMessage(time.Duration) (*ckf.Message, error) {
c.reads++
if c.readErr != nil {
return nil, c.readErr
}
if len(c.queue) == 0 {
return nil, errors.New("unexpected poll")
}
m := c.queue[0]
c.queue = c.queue[1:]
return m, nil
}
func (c *orderedConsumer) AssignmentLost() bool { return c.lost }
func TestConsumerAssignmentLossRetainsReplayAndPollFailureReopens(t *testing.T) {
c := &orderedConsumer{queue: []*ckf.Message{{}, {}}}
err := (&Consumer{c: c}).Consume(t.Context(), func(*ckf.Message) error { c.lost = true; return nil })
if !errors.Is(err, ErrAssignmentLost) || c.reads != 1 || len(c.stored) != 0 {
t.Fatalf("err=%v reads=%d stored=%v", err, c.reads, c.stored)
}
for _, readErr := range []ckf.Error{ckf.NewError(ckf.ErrMaxPollExceeded, "poll expired", false), ckf.NewError(ckf.ErrFatal, "fatal consumer failure", true)} {
c = &orderedConsumer{readErr: readErr}
err := (&Consumer{c: c}).Consume(t.Context(), func(*ckf.Message) error { t.Fatal("no record to handle"); return nil })
if err == nil || c.reads != 1 || len(c.stored) != 0 {
t.Fatalf("err=%v reads=%d stored=%v", err, c.reads, c.stored)
}
}
}
func (c *orderedConsumer) StoreMessage(m *ckf.Message) ([]ckf.TopicPartition, error) {
c.stored = append(c.stored, m.TopicPartition.Offset)
return nil, c.storeErr
}
func TestConsumerRetriesFailedRecordBeforeLaterOffsets(t *testing.T) {
first := &ckf.Message{TopicPartition: ckf.TopicPartition{Partition: 0, Offset: 10}}
second := &ckf.Message{TopicPartition: ckf.TopicPartition{Partition: 0, Offset: 11}}
c := &orderedConsumer{queue: []*ckf.Message{first, second}}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var handled []ckf.Offset
err := (&Consumer{c: c}).Consume(ctx, func(m *ckf.Message) error {
handled = append(handled, m.TopicPartition.Offset)
if len(handled) == 1 {
return errors.New("transient database failure")
}
if len(handled) == 2 && (c.reads != 1 || len(c.stored) != 0 || m != first) {
t.Fatal("failed record was skipped or acked")
}
if m == second {
cancel()
}
return nil
})
if !errors.Is(err, context.Canceled) || len(handled) != 3 || handled[0] != 10 || handled[1] != 10 || handled[2] != 11 || len(c.stored) != 2 || c.stored[0] != 10 || c.stored[1] != 11 {
t.Fatalf("err=%v handled=%v stored=%v", err, handled, c.stored)
}
}
func TestConsumerFailureCancellationNeverStoresOffset(t *testing.T) {
c := &orderedConsumer{queue: []*ckf.Message{{}}}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
err := (&Consumer{c: c}).Consume(ctx, func(*ckf.Message) error { cancel(); return errors.New("failed") })
if !errors.Is(err, context.Canceled) || len(c.stored) != 0 || c.reads != 1 {
t.Fatalf("err=%v stored=%v reads=%d", err, c.stored, c.reads)
}
}
func TestConsumerOffsetStorageFailureStopsBeforeNextRecord(t *testing.T) {
c := &orderedConsumer{queue: []*ckf.Message{{}, {}}, storeErr: errors.New("store unavailable")}
err := (&Consumer{c: c}).Consume(context.Background(), func(*ckf.Message) error { return nil })
if !errors.Is(err, c.storeErr) || c.reads != 1 {
t.Fatalf("err=%v reads=%d", err, c.reads)
}
}
func TestConsumerRetryBackoffCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
if err := consumerBackoff(ctx, 5*time.Second); !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
}
func TestConsumerPersistentFailureReturnsForReplayWithoutStoring(t *testing.T) {
c := &orderedConsumer{queue: []*ckf.Message{{}, {}}}
failure := errors.New("database still unavailable")
calls := 0
err := (&Consumer{c: c, retryLimit: 2}).Consume(t.Context(), func(*ckf.Message) error {
calls++
return failure
})
if !errors.Is(err, failure) || calls != 2 || c.reads != 1 || len(c.stored) != 0 {
t.Fatalf("err=%v calls=%d reads=%d stored=%v", err, calls, c.reads, c.stored)
}
}
internal/pkg/stoken/http.go
+31
View File
@@ -0,0 +1,31 @@
package stoken
import (
"context"
"net/http"
"time"
"golang.org/x/oauth2"
)
const ProviderRequestTimeout = 45 * time.Second
// BoundedContext also bounds refresh I/O, which uses the token source's context.
func BoundedContext(ctx context.Context) context.Context {
base := http.DefaultClient
if client, ok := ctx.Value(oauth2.HTTPClient).(*http.Client); ok && client != nil {
base = client
}
client := *base
if client.Timeout <= 0 || client.Timeout > ProviderRequestTimeout {
client.Timeout = ProviderRequestTimeout
}
return context.WithValue(ctx, oauth2.HTTPClient, &client)
}
func HTTPClient(ctx context.Context, source oauth2.TokenSource) *http.Client {
ctx = BoundedContext(ctx)
client := oauth2.NewClient(ctx, source)
client.Timeout = ctx.Value(oauth2.HTTPClient).(*http.Client).Timeout
return client
}
internal/pkg/stoken/http_test.go
+47
View File
@@ -0,0 +1,47 @@
package stoken
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"time"
"golang.org/x/oauth2"
)
func TestBoundedProviderAndRefreshHTTPClient(t *testing.T) {
for _, timeout := range []time.Duration{0, time.Hour, 20 * time.Millisecond} {
base := &http.Client{Timeout: timeout}
ctx := BoundedContext(context.WithValue(context.Background(), oauth2.HTTPClient, base))
bounded := ctx.Value(oauth2.HTTPClient).(*http.Client)
want := ProviderRequestTimeout
if timeout > 0 && timeout < want {
want = timeout
}
if base.Timeout != timeout || bounded.Timeout != want {
t.Fatalf("base=%v bounded=%v want=%v", base.Timeout, bounded.Timeout, want)
}
if c := HTTPClient(ctx, oauth2.StaticTokenSource(&oauth2.Token{AccessToken: "test"})); c.Timeout != want {
t.Fatalf("provider timeout=%v want=%v", c.Timeout, want)
}
}
release := make(chan struct{})
srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
select {
case <-r.Context().Done():
case <-release:
}
}))
defer srv.Close()
defer close(release)
ctx := BoundedContext(context.WithValue(t.Context(), oauth2.HTTPClient, &http.Client{Timeout: 30 * time.Millisecond}))
cfg := oauth2.Config{Endpoint: oauth2.Endpoint{TokenURL: srv.URL, AuthStyle: oauth2.AuthStyleInParams}}
start := time.Now()
if _, err := cfg.TokenSource(ctx, &oauth2.Token{RefreshToken: "test"}).Token(); err == nil {
t.Fatal("stalled refresh succeeded")
}
if time.Since(start) > time.Second || ctx.Err() != nil {
t.Fatal("refresh was unbounded or cancelled mailbox context")
}
}
internal/repository/email_error_dedupe_live_test.go
+13
View File
@@ -4,6 +4,7 @@ import (
"context"
"sync"
"testing"
"time"
"github.com/google/uuid"
)
@@ -85,6 +86,8 @@ func TestLiveEmailErrorDedupeKeepsOneUnresolvedRowPerCode(t *testing.T) {
if first == nil {
t.Fatal("the first occurrence of an error must be recorded")
}
exec(`UPDATE email_account_errors SET created_at=NOW()-interval '3 days' WHERE id=$1`, first.ID)
before := time.Now()
// The sync loop relaying the same refusal for the next hour.
for i := 0; i < 5; i++ {
@@ -99,6 +102,16 @@ func TestLiveEmailErrorDedupeKeepsOneUnresolvedRowPerCode(t *testing.T) {
if n := unresolved("IMAP_UNKNOWN"); n != 1 {
t.Errorf("unresolved IMAP_UNKNOWN rows = %d, want 1", n)
}
rows, xerr := repo.GetByAccountID(ctx, account, true)
if xerr != nil || len(rows) != 1 || rows[0].CreatedAt.Before(before) {
t.Fatalf("ongoing failure observation expired: %+v %v", rows, xerr)
}
if xerr := repo.ResolveByCodesBefore(ctx, account, []string{"IMAP_UNKNOWN"}, before, "stale success"); xerr != nil {
t.Fatal(xerr)
}
if n := unresolved("IMAP_UNKNOWN"); n != 1 {
t.Fatal("old success cleared a newly observed failure")
}
// A different failure is a different row: dedupe must not swallow it.
other, xerr := repo.CreateOnce(ctx, row("SERVER_UNREACHABLE"))
internal/repository/email_message_map.go
+31
View File
@@ -2,8 +2,11 @@ package repository
import (
"context"
"errors"
"time"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/models"
)
// EmailMessageMapRepository maps a provider messageId to the internal email
@@ -20,6 +23,34 @@ type EmailMessageMapRepository interface {
Del(ctx context.Context, userID, emailID uuid.UUID, messageID string, id uuid.UUID) error
}
var ErrArrivalOutboxUnsupported = errors.New("durable arrival admission unavailable; upgrade backend and consumer first")
var ErrArrivalAdmissionUnconfirmed = errors.New("existing provider mapping has no verifiable pending admission")
// ArrivalAdmission is an optional internal protocol, separate from legacy map writes.
type ArrivalAdmission interface {
AdmitArrival(context.Context, EmailMessageData, *PendingArrival) error
}
type PendingArrival struct {
Arrival *models.JobEventNewEmail `json:"arrival"`
Bounce *models.JobEventInboundBounce `json:"bounce,omitempty"`
Complaint *models.JobEventInboundComplaint `json:"complaint,omitempty"`
}
type ArrivalOutbox interface {
HasPendingArrival(context.Context, uuid.UUID, uuid.UUID, uuid.UUID) (bool, error)
DeliverArrivals(context.Context, func(context.Context, models.JobEventType, any) error) error
}
type ArrivalBacklog struct {
Pending int
Oldest *time.Time
}
type ArrivalBacklogRepository interface {
ArrivalBacklog(context.Context) (ArrivalBacklog, error)
}
// EmailMessageData is one (user, mailbox, providerMessageID) -> internal
// (id, threadID) mapping row. The UUID-typed values travel as strings to match
// the worker call sites, which pass uuid.String().
internal/repository/http_email_message_map.go
+28
View File
@@ -93,6 +93,34 @@ func (r *httpEmailMessageMapRepository) Add(ctx context.Context, data EmailMessa
}
}
func (r *httpEmailMessageMapRepository) AdmitArrival(ctx context.Context, data EmailMessageData, pending *PendingArrival) error {
body, err := json.Marshal(struct {
Map emailMessageMapPayload `json:"map"`
Pending *PendingArrival `json:"pending"`
}{emailMessageMapPayload(data), pending})
if err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.endpoint()+"/arrival", bytes.NewReader(body))
if err != nil {
return err
}
r.authed(req)
req.Header.Set("Content-Type", "application/json")
resp, err := r.client.Do(req)
if err != nil {
return fmt.Errorf("arrival admission: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusMethodNotAllowed {
return ErrArrivalOutboxUnsupported
}
if resp.StatusCode != http.StatusNoContent || resp.Header.Get("X-Warmbly-Arrival-Durable") != "1" {
return fmt.Errorf("arrival admission not confirmed: status %d", resp.StatusCode)
}
return nil
}
func (r *httpEmailMessageMapRepository) Get(ctx context.Context, userID, emailID uuid.UUID, messageID string) (*EmailMessageData, error) {
q := url.Values{}
q.Set("user_id", userID.String())
internal/repository/http_graph_upgrade_test.go
+22
View File
@@ -0,0 +1,22 @@
package repository
import (
"errors"
"net/http"
"net/http/httptest"
"testing"
"github.com/google/uuid"
)
func TestProviderMessagesOldBackendIsTypedUnsupported(t *testing.T) {
for _, status := range []int{http.StatusNotFound, http.StatusMethodNotAllowed, http.StatusServiceUnavailable, http.StatusForbidden} {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(status) }))
repo := &httpSyncContextRepository{baseURL: srv.URL, client: srv.Client()}
_, err := repo.ListProviderMessages(t.Context(), uuid.New(), uuid.New(), nil, 10)
if errors.Is(err, ErrSyncContextUnsupported) != (status == http.StatusNotFound || status == http.StatusMethodNotAllowed) {
t.Errorf("status=%d err=%v", status, err)
}
srv.Close()
}
}
internal/repository/http_sync_arrival_test.go
+67
View File
@@ -0,0 +1,67 @@
package repository
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
)
func TestHTTPArrivalAdmissionUpgradeAcknowledgement(t *testing.T) {
for _, tc := range []struct {
name string
status int
ack string
unsupported, success bool
}{
{"old-backend", 404, "", true, false}, {"old-method", 405, "", true, false},
{"durable", 204, "1", false, true}, {"unconfirmed", 204, "", false, false},
{"backend-failure", 503, "", false, false},
} {
t.Run(tc.name, func(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/v1/internal/email-message-map/arrival" || r.Method != http.MethodPost || r.Header.Get("Authorization") != "Bearer fixture-token" {
t.Error("incorrect internal protocol")
}
w.Header().Set("X-Warmbly-Arrival-Durable", tc.ack)
w.WriteHeader(tc.status)
}))
defer srv.Close()
repo, err := NewHTTPEmailMessageMapRepository(srv.URL, "fixture-token")
if err != nil {
t.Fatal(err)
}
err = repo.(ArrivalAdmission).AdmitArrival(context.Background(), EmailMessageData{}, &PendingArrival{})
if errors.Is(err, ErrArrivalOutboxUnsupported) != tc.unsupported || (err == nil) != tc.success {
t.Fatalf("err=%v", err)
}
})
}
}
func TestHTTPArrivalAdmissionResponseLossDoesNotDowngrade(t *testing.T) {
var committed atomic.Bool
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
t.Error("ambiguous admission fell back to legacy write")
}
committed.Store(true)
conn, _, err := w.(http.Hijacker).Hijack()
if err != nil {
t.Error(err)
return
}
_ = conn.Close()
}))
defer srv.Close()
r, err := NewHTTPEmailMessageMapRepository(srv.URL, "fixture-token")
if err != nil {
t.Fatal(err)
}
err = r.(ArrivalAdmission).AdmitArrival(context.Background(), EmailMessageData{}, &PendingArrival{})
if err == nil || errors.Is(err, ErrArrivalOutboxUnsupported) || !committed.Load() {
t.Fatalf("response loss treated as unsupported: committed=%v err=%v", committed.Load(), err)
}
}
internal/repository/http_sync_context.go
+5
View File
@@ -54,6 +54,8 @@ type SyncContextRepository interface {
ListProviderMessages(ctx context.Context, userID, emailID uuid.UUID, after *uuid.UUID, limit int) ([]ProviderFolderMessage, error)
}
var ErrSyncContextUnsupported = errors.New("provider message enumeration unavailable; upgrade backend first")
func (r *httpSyncContextRepository) ListProviderMessages(ctx context.Context, userID, emailID uuid.UUID, after *uuid.UUID, limit int) ([]ProviderFolderMessage, error) {
q := url.Values{}
q.Set("user_id", userID.String())
@@ -73,6 +75,9 @@ func (r *httpSyncContextRepository) ListProviderMessages(ctx context.Context, us
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusMethodNotAllowed {
return nil, ErrSyncContextUnsupported
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("sync_context.http: provider messages: unexpected status %d", resp.StatusCode)
}

Some files were not shown because too many files have changed in this diff Show More