diff --git a/docs/content/docs/development/mailbox-push.mdx b/docs/content/docs/development/mailbox-push.mdx index 1a62a1bd2..8d6bcf3b9 100644 --- a/docs/content/docs/development/mailbox-push.mdx +++ b/docs/content/docs/development/mailbox-push.mdx @@ -64,7 +64,7 @@ How the channel behaves: - The endpoint answers `202` well inside Graph's 3-second limit. When the wake cannot be sent it answers `503`, and Graph retries for up to 4 hours. - A subscription lasts just under 7 days and is renewed with 2 days left. A `reauthorizationRequired` or `subscriptionRemoved` lifecycle event renews or reopens it at once; a `missed` event triggers a sync pass. - Changing `AUTH_SECRET` changes every client state. Workers reopen their subscriptions the next time the mailbox is reloaded, within about 30 minutes, and the 10-minute fallback poll covers the gap. -- A subscription Microsoft refuses is retried every 30 minutes while the mailbox polls every minute. +- A subscription Microsoft could not create is retried after 30 minutes, or later when Microsoft sends `Retry-After`. One it refuses outright (a `4xx` other than throttling) is retried after 6 hours. The mailbox polls every minute meanwhile, and a changed notification URL or client state is tried at once. ## Gmail and Google Workspace @@ -129,13 +129,13 @@ Migration `000351` adds `fleet_nodes.sync_wake_protocol` (default `0`) and the ` ## Troubleshooting -When a worker cannot open a mailbox's Gmail watch or Graph subscription, it reports the first failure for that mailbox to error tracking (PostHog or Sentry, when configured), tagged `component=mailbox_push`, with the provider's own message. The mailbox keeps polling meanwhile. +When a worker cannot open a mailbox's Gmail watch or Graph subscription, it reports the failure to error tracking (PostHog or Sentry, when configured) at most once a day per mailbox, tagged `component=mailbox_push`. The report names the provider call and its HTTP status, with the provider's own message, for example `gmail users.watch: HTTP 400 Invalid topicName ...` or `graph subscription create: HTTP 429`. A throttle there limits opening the channel, not sending. The mailbox keeps polling meanwhile, and the worker asks again after 30 minutes, after the provider's `Retry-After`, or after 6 hours when the provider refused the request outright. | Symptom | Cause and fix | |---|---| | Outlook mail still takes about a minute | Check that `GRAPH_PUSH=true` and that `BACKEND_PUBLIC_URL` (or `API_PUBLIC_URL`) is an `https://` URL. Plain `http` leaves push off. From outside your network, `curl -X POST '/api/v1/mailbox-push/graph/x?validationToken=ok'` must answer `ok`; Microsoft sends the same request when it creates a subscription | -| Graph subscriptions are never created | Microsoft cannot reach the URL, or the proxy in front of the backend does not forward `/api/v1/mailbox-push/`. Workers retry every 30 minutes and log the refusal at debug level | -| Gmail watch fails with `Invalid topicName` | The topic is in a different project from `BOX_GOOGLE_CLIENT_ID`. Create it in that project. Whole-domain grant mailboxes are only asked to watch when the delegation key's `project_id` matches the topic's project | +| Graph subscriptions are never created | Microsoft cannot reach the URL, or the proxy in front of the backend does not forward `/api/v1/mailbox-push/`. Workers retry as described above and log each refusal at debug level | +| Gmail watch fails with `Invalid topicName` | The topic is in a different project from the OAuth client that issued the mailbox's grant, normally `BOX_GOOGLE_CLIENT_ID`. Create it in that project. A mailbox whose grant came from another Google project keeps polling. Whole-domain grant mailboxes are only asked to watch when the delegation key's `project_id` matches the topic's project | | Gmail watch fails with a permission error | Gmail's push service account `gmail-api-push@system.gserviceaccount.com` lacks **Pub/Sub Publisher** on the topic, often because domain restricted sharing refused the grant | | Pub/Sub shows push deliveries failing with `401` | The subscription signs with a different service account than `GMAIL_PUSH_SERVICE_ACCOUNT`, authentication is off, or its audience was set to something other than the endpoint URL | | Pub/Sub shows push deliveries failing with `404` | `GMAIL_PUSH_SERVICE_ACCOUNT` is unset on the backend, or `BACKEND_PUBLIC_URL` is not `https` | diff --git a/internal/app/worker/wmail/push.go b/internal/app/worker/wmail/push.go index 0464750be..0464d125a 100644 --- a/internal/app/worker/wmail/push.go +++ b/internal/app/worker/wmail/push.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "errors" "math/rand/v2" "sync" "sync/atomic" @@ -24,8 +25,14 @@ const ( graphRenewBefore = 48 * time.Hour // gmailRenewBefore renews a Gmail watch daily, as Google recommends. gmailRenewBefore = 6 * 24 * time.Hour - // pushRetryAfter spaces attempts to open a channel the provider refused. + // pushRetryAfter spaces attempts to open a channel the provider could not open just now. pushRetryAfter = 30 * time.Minute + // pushRefusedRetryAfter spaces attempts the provider refused outright, which wait on someone fixing it. + pushRefusedRetryAfter = 6 * time.Hour + // pushRetryAfterMax bounds how long a provider's Retry-After can hold a channel closed. + pushRetryAfterMax = 24 * time.Hour + // pushReportEvery is how often one mailbox's refused channel reaches error tracking. + pushReportEvery = 24 * time.Hour // imapWatchBackoffMax bounds the wait before re-opening a dropped IDLE session. imapWatchBackoffMax = 30 * time.Minute ) @@ -34,16 +41,23 @@ const ( // not sign every mailbox in at once. var imapWatchDials = rate.NewLimiter(rate.Limit(20), 20) +// pushRefusal is a mailbox's last refused channel: which target, when to ask again, when it was reported. +type pushRefusal struct { + target string + retryAt time.Time + reported time.Time +} + +// pushRefusals is kept per process, not per WMail, so a rebuilt mailbox neither asks early nor reports again. +var pushRefusals sync.Map // uuid.UUID -> pushRefusal + // pushChannel is the provider channel a mailbox holds open, and what the // control plane asked for. type pushChannel struct { - mu sync.Mutex - want *models.AddWorkerEmailPushData - state *models.MailboxPushState - retryAt time.Time - renew atomic.Bool - // failing keeps a refused channel to one warning until it opens again. - failing atomic.Bool + mu sync.Mutex + want *models.AddWorkerEmailPushData + state *models.MailboxPushState + renew atomic.Bool // unsent is set while the control plane has not taken the current state. unsent atomic.Bool // closed is the last channel this worker closed, so a stored copy a consumer never cleared is not adopted. @@ -91,6 +105,33 @@ func (w *WMail) pollInterval() time.Duration { return ImapCheckInterval } +// pushTarget names the channel the control plane asked for, so a changed topic or endpoint is tried at once. +func pushTarget(want *models.AddWorkerEmailPushData) string { + if want == nil { + return "" + } + return credentialPrint(want.GmailTopic + "\x00" + want.GraphNotificationURL + "\x00" + want.GraphClientState) +} + +// pushRetryAt is when to ask again: a refusal that waits on a fix waits longest, and a provider's Retry-After is never cut short. +func pushRetryAt(err error, now time.Time) time.Time { + at := now.Add(pushRetryAfter) + var mailErr *errx.MailError + if !errors.As(err, &mailErr) || mailErr.Failure == nil { + return at + } + if f := mailErr.Failure; f.Disposition == errx.SendPermanent || f.Disposition == errx.SendAuth { + at = now.Add(pushRefusedRetryAfter) + } + if r := mailErr.Failure.RetryAt; r != nil && r.After(at) { + at = *r + if limit := now.Add(pushRetryAfterMax); at.After(limit) { + at = limit + } + } + return at +} + func credentialPrint(v string) string { sum := sha256.Sum256([]byte(v)) return hex.EncodeToString(sum[:8]) @@ -108,10 +149,15 @@ func (w *WMail) maintainPush(ctx context.Context) { defer cancel() now := time.Now() w.push.mu.Lock() - want, state, retryAt := w.push.want, w.push.state, w.push.retryAt + want, state := w.push.want, w.push.state w.push.mu.Unlock() renew := w.push.renew.Swap(false) - if !renew && now.Before(retryAt) { + target := pushTarget(want) + var refusal pushRefusal + if v, ok := pushRefusals.Load(w.ID); ok { + refusal = v.(pushRefusal) + } + if !renew && refusal.target == target && now.Before(refusal.retryAt) { if w.push.unsent.Load() { w.reportPush(state) } @@ -134,18 +180,18 @@ func (w *WMail) maintainPush(ctx context.Context) { return } if err != nil { - w.push.mu.Lock() - w.push.retryAt = now.Add(pushRetryAfter) - w.push.mu.Unlock() + next := pushRefusal{target: target, retryAt: pushRetryAt(err, now), reported: refusal.reported} event := log.Debug() - if !w.push.failing.Swap(true) { + if refusal.target != target || now.Sub(refusal.reported) >= pushReportEvery { + next.reported = now event = log.Warn() - // Workers ship no logs, so the first failure per mailbox is how an operator sees why push is off. + // Workers ship no logs, so this daily report is how an operator sees why push is off. errs.CaptureException(err, errs.Tag("component", "mailbox_push"), errs.Tag("provider", string(w.EmailType)), errs.Tag("email_id", w.ID.String())) } + pushRefusals.Store(w.ID, next) event.Err(err).Str("email_id", w.ID.String()).Msg("mailbox push channel not opened; polling continues") - } else { - w.push.failing.Store(false) + } else if !refusal.reported.IsZero() { + pushRefusals.Delete(w.ID) } // A failed open still reports a channel it closed on the way, and a lost report is sent again next pass. if samePush(state, next) && !w.push.unsent.Load() { diff --git a/internal/app/worker/wmail/push_test.go b/internal/app/worker/wmail/push_test.go index 0b8fdcd98..7a4ca8433 100644 --- a/internal/app/worker/wmail/push_test.go +++ b/internal/app/worker/wmail/push_test.go @@ -5,7 +5,9 @@ import ( "testing" "time" + "github.com/google/uuid" "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/errx" "github.com/warmbly/warmbly/internal/models" ) @@ -135,7 +137,9 @@ func TestAPendingReportIsResentDuringTheRetryBackoff(t *testing.T) { return nil }} // A channel was closed, its report refused, and the provider is in backoff. - w.push.retryAt = time.Now().Add(time.Hour) + w.ID = uuid.New() + pushRefusals.Store(w.ID, pushRefusal{retryAt: time.Now().Add(time.Hour), reported: time.Now()}) + t.Cleanup(func() { pushRefusals.Delete(w.ID) }) w.push.unsent.Store(true) w.maintainPush(t.Context()) if reports != 1 || sent[0] != nil || w.push.unsent.Load() { @@ -146,3 +150,43 @@ func TestAPendingReportIsResentDuringTheRetryBackoff(t *testing.T) { t.Fatalf("a report already taken was sent again: %d", reports) } } + +func TestPushRetryWaitsLongerOnARefusalAndHonoursRetryAfter(t *testing.T) { + now := time.Now() + fail := func(f errx.SendFailure) error { + return errx.ProviderCall("op", "", errx.WithSendFailure(errx.ErrMailSendRejected("x"), f)) + } + if got := pushRetryAt(errors.New("dial tcp: timeout"), now); !got.Equal(now.Add(pushRetryAfter)) { + t.Fatalf("transport failure retries at %v", got.Sub(now)) + } + if got := pushRetryAt(fail(errx.SendFailure{Status: 400, Disposition: errx.SendPermanent}), now); !got.Equal(now.Add(pushRefusedRetryAfter)) { + t.Fatalf("a refused topic retries at %v", got.Sub(now)) + } + if got := pushRetryAt(fail(errx.SendFailure{Status: 503, Disposition: errx.SendRetry}), now); !got.Equal(now.Add(pushRetryAfter)) { + t.Fatalf("an outage retries at %v", got.Sub(now)) + } + later := now.Add(2 * time.Hour) + if got := pushRetryAt(fail(errx.SendFailure{Status: 429, Disposition: errx.SendThrottle, RetryAt: &later}), now); !got.Equal(later) { + t.Fatalf("Retry-After cut short: %v", got.Sub(now)) + } + far := now.Add(30 * 24 * time.Hour) + if got := pushRetryAt(fail(errx.SendFailure{Status: 429, Disposition: errx.SendThrottle, RetryAt: &far}), now); !got.Equal(now.Add(pushRetryAfterMax)) { + t.Fatalf("Retry-After not bounded: %v", got.Sub(now)) + } +} + +func TestARebuiltMailboxKeepsItsPushBackoff(t *testing.T) { + id := uuid.New() + t.Cleanup(func() { pushRefusals.Delete(id) }) + want := &models.AddWorkerEmailPushData{GmailTopic: "projects/p/topics/t"} + pushRefusals.Store(id, pushRefusal{target: pushTarget(want), retryAt: time.Now().Add(time.Hour), reported: time.Now()}) + // A fresh WMail with no Google client: it must not reach the provider while the backoff holds. + w := &WMail{ID: id, EmailType: models.InboxProviderGoogle, GoogleData: &GoogleData{}, wake: make(chan struct{}, 1)} + w.ApplyPush(want) + w.maintainPush(t.Context()) + + other := &models.AddWorkerEmailPushData{GmailTopic: "projects/p/topics/other"} + if pushTarget(other) == pushTarget(want) { + t.Fatal("a changed topic is not a new target") + } +} diff --git a/internal/client/goog/watch.go b/internal/client/goog/watch.go index 41d9df2d8..ec32135c1 100644 --- a/internal/client/goog/watch.go +++ b/internal/client/goog/watch.go @@ -3,9 +3,9 @@ package goog import ( "context" "errors" - "fmt" "time" + "github.com/warmbly/warmbly/internal/errx" "google.golang.org/api/gmail/v1" "google.golang.org/api/googleapi" ) @@ -17,11 +17,7 @@ func (c *Client) Watch(ctx context.Context, topic string) (time.Time, error) { resp, err := c.srv.Users.Watch("me", &gmail.WatchRequest{TopicName: topic}).Context(ctx).Do() if err != nil { // Keep Google's reason (an unknown topic, a missing publisher grant): it names the fix. - var gerr *googleapi.Error - if errors.As(err, &gerr) { - return time.Time{}, fmt.Errorf("gmail users.watch: %d %s: %w", gerr.Code, gerr.Message, HandleError(err)) - } - return time.Time{}, fmt.Errorf("gmail users.watch: %w", HandleError(err)) + return time.Time{}, errx.ProviderCall("gmail users.watch", googleReason(err), HandleError(err)) } if resp.Expiration <= 0 { return time.Time{}, errors.New("goog: users.watch returned no expiration") @@ -29,10 +25,18 @@ func (c *Client) Watch(ctx context.Context, topic string) (time.Time, error) { return time.UnixMilli(resp.Expiration), nil } +func googleReason(err error) string { + var gerr *googleapi.Error + if errors.As(err, &gerr) { + return gerr.Message + } + return "" +} + // StopWatch ends the mailbox's watch. func (c *Client) StopWatch(ctx context.Context) error { if err := c.srv.Users.Stop("me").Context(ctx).Do(); err != nil { - return fmt.Errorf("gmail users.stop: %w", HandleError(err)) + return errx.ProviderCall("gmail users.stop", googleReason(err), HandleError(err)) } return nil } diff --git a/internal/client/msgraph/subscription.go b/internal/client/msgraph/subscription.go index 26479a17d..83c871e09 100644 --- a/internal/client/msgraph/subscription.go +++ b/internal/client/msgraph/subscription.go @@ -44,7 +44,7 @@ func (c *Client) CreateSubscription(ctx context.Context, notificationURL, client } var out Subscription if err := c.doJSON(ctx, http.MethodPost, graphBase+"/subscriptions", in, &out); err != nil { - return nil, err + return nil, errx.ProviderCall("graph subscription create", "", err) } return &out, nil } @@ -54,7 +54,7 @@ func (c *Client) RenewSubscription(ctx context.Context, id string, now time.Time in := map[string]any{"expirationDateTime": now.Add(SubscriptionLifetime).UTC().Format(time.RFC3339)} var out Subscription if err := c.doJSON(ctx, http.MethodPatch, graphBase+"/subscriptions/"+url.PathEscape(id), in, &out); err != nil { - return nil, err + return nil, errx.ProviderCall("graph subscription renew", "", err) } return &out, nil } @@ -65,7 +65,7 @@ func (c *Client) DeleteSubscription(ctx context.Context, id string) error { if SubscriptionGone(err) { return nil } - return err + return errx.ProviderCall("graph subscription delete", "", err) } // subscriptionListMaxPages bounds how many continuations ListSubscriptions follows. @@ -81,7 +81,7 @@ func (c *Client) ListSubscriptions(ctx context.Context) ([]Subscription, error) NextLink json.RawMessage `json:"@odata.nextLink"` } if err := c.doJSON(ctx, http.MethodGet, next, nil, &page); err != nil { - return nil, err + return nil, errx.ProviderCall("graph subscription list", "", err) } subs = append(subs, page.Value...) if page.NextLink == nil { diff --git a/internal/errx/provider_call.go b/internal/errx/provider_call.go new file mode 100644 index 000000000..40b384c36 --- /dev/null +++ b/internal/errx/provider_call.go @@ -0,0 +1,51 @@ +package errx + +import ( + "errors" + "fmt" +) + +// ProviderCallError is a provider refusing a mailbox call that is not a send, such as +// opening a push channel. It reads as the call and its status, never as a refused send, +// and unwraps to the classified MailError so callers still read its disposition. +type ProviderCallError struct { + Op string + Detail string + Mail *MailError +} + +func (e *ProviderCallError) Error() string { + s := e.Op + if e.Mail != nil && e.Mail.Failure != nil && e.Mail.Failure.Status != 0 { + s += fmt.Sprintf(": HTTP %d", e.Mail.Failure.Status) + } else if e.Mail != nil { + s += ": " + e.Mail.Error() + } + if e.Detail != "" { + s += " " + e.Detail + } + return s +} + +func (e *ProviderCallError) Unwrap() error { + if e.Mail == nil { + return nil + } + return e.Mail +} + +// ProviderCall names op on a provider's refusal; detail is the provider's own reason when it has one. +// An error that is not a classified MailError keeps its text behind op. +func ProviderCall(op, detail string, err error) error { + if err == nil { + return nil + } + var mailErr *MailError + if !errors.As(err, &mailErr) { + return fmt.Errorf("%s: %w", op, err) + } + if detail == "" && mailErr.Failure != nil { + detail = mailErr.Failure.Cause + } + return &ProviderCallError{Op: op, Detail: detail, Mail: mailErr} +} diff --git a/internal/errx/provider_call_test.go b/internal/errx/provider_call_test.go new file mode 100644 index 000000000..60079f22e --- /dev/null +++ b/internal/errx/provider_call_test.go @@ -0,0 +1,35 @@ +package errx + +import ( + "errors" + "strings" + "testing" +) + +func TestProviderCallReadsAsTheCallNotASend(t *testing.T) { + throttled := WithSendFailure(ErrMailSendingTooFast, SendFailure{Status: 429, Cause: "TooManyRequests", Disposition: SendThrottle}) + err := ProviderCall("graph subscription create", "", throttled) + if got := err.Error(); got != "graph subscription create: HTTP 429 TooManyRequests" { + t.Fatalf("message %q", got) + } + if strings.Contains(err.Error(), "sent too quickly") { + t.Fatal("a push refusal reads as a send") + } + var mailErr *MailError + if !errors.As(err, &mailErr) || mailErr.Failure.Disposition != SendThrottle { + t.Fatal("the classified failure is not reachable") + } + + refused := WithSendFailure(ErrMailSendRejected("The provider rejected the request"), SendFailure{Status: 400, Disposition: SendPermanent}) + if got := ProviderCall("gmail users.watch", "Invalid topicName", refused).Error(); got != "gmail users.watch: HTTP 400 Invalid topicName" { + t.Fatalf("message %q", got) + } + + plain := errors.New("dial tcp: timeout") + if got := ProviderCall("gmail users.watch", "", plain); !errors.Is(got, plain) || got.Error() != "gmail users.watch: dial tcp: timeout" { + t.Fatalf("an unclassified error lost its text: %v", got) + } + if ProviderCall("x", "", nil) != nil { + t.Fatal("no error became one") + } +}