feat: let an IMAP mailbox exclude folders from sync (email_accounts.sync_skip_folders, migration 000196) so a folder another tool fills never reaches the unified inbox: the worker drops skipped folders and their subfolders before the walk, retires an already-synced one with its stored mail, and removes mail that later moves into one only when its Message-ID is found there; PUT /emails/:id/sync and the drawer's Sync card set the list, GET reports it with the server's folder list, warmbly mailbox skip-folders mirrors it, with docs, OpenAPI and error-code invalid_sync_folder

This commit is contained in:
Matthew Meszaros
2026-09-22 05:15:49 -07:00
parent cc898cc040
commit 5a967cc3ce
40 changed files with 1178 additions and 17 deletions
+3
View File
@@ -1185,6 +1185,9 @@ func main() {
emailSyncStateRepository = repository.NewEmailSyncStateRepository(primaryDB)
emailService.WireSyncState(emailSyncStateRepository)
emailService.WireMailboxes(repository.NewMailboxRepository(primaryDB))
// A folder excluded from sync has its already-stored mail dropped at
// the moment of the change, not a pass later.
emailService.WireUnibox(repository.NewUniboxRepository(primaryDB))
if instanceSettings != nil {
emailService.WireSyncBudget(instanceSettings)
}
+17 -1
View File
@@ -661,10 +661,26 @@ so a removed alias stops being used instead of failing every send.`,
Success: "Sending identity refreshed.",
},
{
Name: "sync", Short: "The mailbox's sync state and backfill progress",
Name: "sync", Short: "The mailbox's sync state, backfill progress and skipped folders",
Method: http.MethodGet, Path: "/emails/{id}/sync",
Args: []argSpec{{Name: "id", Help: "The mailbox's id"}},
},
{
Name: "skip-folders", Short: "Choose the folders an IMAP mailbox's sync leaves alone",
Long: `Replaces the list of folders the sync never opens, named as the mail
server lists them (see "mailbox sync" for the names). Each also covers its
subfolders. Mail already imported from a folder is removed from Warmbly
when it is skipped; the mail itself stays in the mailbox. Inbox, sent,
drafts, spam, trash and archive cannot be skipped. To sync everything
again, send an empty list with --input.`,
Example: " $ warmbly mailbox skip-folders MAILBOX_ID --folder Warmer\n $ warmbly mailbox skip-folders MAILBOX_ID --folder Warmer --folder \"Clients/Acme\"\n $ warmbly mailbox skip-folders MAILBOX_ID --input '{\"skip_folders\": []}'",
Method: http.MethodPut, Path: "/emails/{id}/sync", Body: bodyRequired,
Args: []argSpec{{Name: "id", Help: "The mailbox's id"}},
Flag: []flagSpec{
{Name: "folder", Help: "A folder to skip, as the server lists it (repeatable)", Kind: flagStrings, Key: "skip_folders"},
},
Success: "Skipped folders updated.",
},
{
Name: "behavior", Short: "The mailbox's human-sending ranges",
Method: http.MethodGet, Path: "/emails/{id}/behavior",
+2 -1
View File
@@ -100,7 +100,8 @@ var apiSpecs = []apiSpec{
{name: "mailbox update", summary: "Update mailbox settings (limits, tags, timezone, signature)", method: "PATCH", path: "/emails/{id}", body: bodyRequired},
{name: "mailbox delete", summary: "Disconnect a mailbox", method: "DELETE", path: "/emails/{id}"},
{name: "mailbox auth-check", summary: "Check the mailbox's SPF, DKIM and DMARC", method: "GET", path: "/emails/{id}/auth-check"},
{name: "mailbox sync", summary: "The mailbox's sync state and backfill progress", method: "GET", path: "/emails/{id}/sync"},
{name: "mailbox sync", summary: "The mailbox's sync state, backfill progress and skipped folders", method: "GET", path: "/emails/{id}/sync"},
{name: "mailbox skip-folders", summary: "Replace the folders an IMAP mailbox's sync leaves alone (body: {\"skip_folders\": [...]})", method: "PUT", path: "/emails/{id}/sync", body: bodyRequired},
{name: "mailbox identity", summary: "The addresses this mailbox may send as (Gmail only)", method: "GET", path: "/emails/{id}/identity"},
{name: "mailbox refresh-identity", summary: "Re-read the send-as addresses from the provider, optionally importing its signature", method: "POST", path: "/emails/{id}/identity/refresh", body: bodyOptional},
{name: "mailbox behavior", summary: "The mailbox's human-sending ranges", method: "GET", path: "/emails/{id}/behavior"},
+3
View File
@@ -29,6 +29,7 @@ All paths below are relative to the versioned base URL `https://api.warmbly.com/
| POST | `/emails/:id/track/verify` | `WRITE_EMAILS` |
| PATCH | `/emails/:id/direct-tracking` | `WRITE_EMAILS` |
| GET | `/emails/:id/sync` | `READ_EMAILS` |
| PUT | `/emails/:id/sync` | `WRITE_EMAILS` |
| GET | `/emails/:id/identity` | `READ_EMAILS` |
| POST | `/emails/:id/identity/refresh` | `WRITE_EMAILS` |
| GET | `/emails/:id/auth-check` | `READ_EMAILS` |
@@ -235,6 +236,8 @@ The `/unibox/drafts` endpoints hold autosaved compose drafts, scoped to the call
`GET /emails/:id/identity` reports which addresses the mailbox's provider will let it send as, which one it uses, and whether its signature was imported or written here; it contacts no provider. `POST /emails/:id/identity/refresh` re-reads that list from the provider and stores it, and with `import_signature` also replaces the stored signature with the one configured at the provider. Storing the list is what a `send_as_email` choice is validated against, so the refresh needs `WRITE_EMAILS` rather than `READ_EMAILS`; it takes no `Idempotency-Key` because it writes exactly what the provider currently says. Gmail only. See [sending identity](/guides/mailboxes/#sending-identity).
`GET /emails/:id/sync` reports, alongside the import state and budget, `skip_folders` (the folders the mailbox's sync leaves alone) and `folders` (what the worker last listed on the server, each with its `name` and the canonical `folder` it files under, INBOX first; empty for Gmail and Outlook). `PUT /emails/:id/sync` takes `{"skip_folders": [...]}` and replaces the list: names as the server lists them, matched without regard to case, each covering its subfolders. Mail already stored from a newly skipped folder is removed, the mailbox is re-shipped to its worker so the change applies on the next pass, and the response is the list as stored. It refuses `INBOX` and any sent, drafts, spam, trash or archive folder with `400 invalid_sync_folder`, as it does a list of more than `50` names or a mailbox that is not IMAP. The body is the desired state, so it takes no `Idempotency-Key`. See [folders you do not want synced](/guides/mailboxes/#folders-you-do-not-want-synced).
`PATCH /emails/:id` accepts `save_to_sent` (boolean) on SMTP/IMAP mailboxes: when true, which is the default, the worker files a copy of each outbound message in the mailbox's Sent folder. It has no effect on Gmail and Outlook mailboxes, whose APIs file their own copy. See [keeping a copy of sent mail](/guides/mailboxes/#keeping-a-copy-of-sent-mail).
`DELETE /emails/:id` releases the mailbox on Warmbly Cloud before removing the local mailbox. An enrolled mailbox's stored credential comes out of the pool, and a cloud-managed mirror's claim is released so the mailbox returns to the cloud workspace and can be adopted again. If the cloud cannot confirm the release, the delete returns `409 mailbox_cloud_unenroll_failed` and keeps the mailbox record so the request can be retried safely. Warmbly attempts to restore the mailbox onto its worker immediately, and the worker reconciler may restore it later if that attempt fails. See [mailboxes](/guides/mailboxes/).
+1
View File
@@ -94,6 +94,7 @@ Returned when the request cannot be processed due to invalid syntax.
| `invalid_filter` | A task filter carried an id that is not one: `assigned_to`, `contact_id` and `deal_id` name records, and are matched against id columns. Sent by `POST /crm/tasks/search`, `POST /crm/tasks/summary`, and the `filters` of a `"all": true` bulk selection |
| `invalid_setting` | `PATCH /outreach/settings` (or a campaign's advanced settings) carried a value outside the documented vocabulary, for example a `reply_intent.crm_task_intents` entry that is not a reply intent |
| `invalid_slug` | `PATCH /organizations/current` was given a `slug` that is not 2 to 80 lowercase letters, numbers or dashes starting and ending with a letter or number |
| `invalid_sync_folder` | `PUT /emails/:id/sync` was given a folder the sync always follows (`INBOX`, or a sent, drafts, spam, trash or archive folder by attribute or name), a name that is empty after trimming, longer than 255 characters or carrying a control character, more than 50 names, or a mailbox that is not IMAP. The `message` names the entry refused |
| `no_organization` | The request needs a workspace and the caller has none selected. Every entitlement, limit and suppression rule is scoped to a workspace, so a write that would run unscoped is refused rather than run without those checks. API keys always carry their workspace; a dashboard session picks one at sign-in, so this normally means the session predates the workspace being chosen. Select a workspace and retry |
#### Password refusals
+10
View File
@@ -183,6 +183,16 @@ Nested folders are followed as well, so mail in a subfolder of the inbox or unde
On IMAP, folders are identified by the standard attributes a server publishes, and by name when it publishes none. The common names are recognized in a dozen languages, so a mailbox whose Sent folder is called "Gesendete Elemente" or "Éléments envoyés" still files sent mail as sent rather than as inbox.
### Folders you do not want synced
A folder the sync does not recognize files as inbox, so its mail shows up in the [unified inbox](/guides/unibox/). That is right for a folder you sort real mail into and wrong for one another tool fills: a second warmup service running out of its own folder, for example, puts machine mail in your inbox, spends the mailbox's sync budget on it, and has it classified like a reply.
On an IMAP mailbox, the drawer's **Sync** card lists the folders the worker has seen under **Folders not synced**. Tick one and the sync stops opening it on its next pass, within a minute; mail already imported from it is removed from Warmbly, and mail that later moves into it from a synced folder is removed too. The mail itself is never touched at the provider. A folder that is not listed yet, because the mailbox has not synced or the folder was just created, can be typed in by name, exactly as your mail server lists it (`Warmer`, or `INBOX/Warmer` on a server that nests everything under the inbox). Case does not matter, and a skipped folder covers its subfolders.
The inbox, sent, drafts, spam, trash and archive folders cannot be skipped: a sync without them is a broken mailbox, not a quieter one. Un-ticking a folder brings it back the way a new folder arrives, syncing what lands in it from then on; the mail that accumulated while it was skipped is not imported. Up to `50` folders can be skipped per mailbox. Gmail and Outlook mailboxes have no such list.
The same setting is `PUT /emails/:id/sync` in the [API](/api/endpoints/) and `warmbly mailbox skip-folders` in the CLI.
<Callout type="info" title="Read state on older IMAP servers">
Some IMAP servers, including Outlook.com, Microsoft 365 over IMAP and Yahoo, cannot tell a client what changed since it last looked. New mail from those servers still arrives within a minute. Reading or flagging a message in another mail client shows up in Warmbly within about ten minutes rather than immediately. Nothing is lost either way, and Gmail, Outlook over OAuth, Fastmail and most self-hosted servers are immediate.
</Callout>
+1 -1
View File
@@ -44,7 +44,7 @@ The rail is two short groups and then your mailboxes. The first group is where y
| Folder | Group | Shows |
| --- | --- | --- |
| Inbox | First | Inbound mail, plus anything filed in a custom folder at the provider |
| Inbox | First | Inbound mail, plus anything filed in a custom folder at the provider, unless that folder is [excluded from sync](/guides/mailboxes/#folders-you-do-not-want-synced) |
| Drafts | Second | Messages sitting in a mailbox's drafts folder |
| Sent | Second | Outbound mail, campaign and manual |
| Archive | Second | Archived (Gmail's All Mail, the Archive folder elsewhere) |
+166 -2
View File
@@ -2810,7 +2810,7 @@
"get": {
"operationId": "mailboxes_sync_state",
"summary": "Get sync state",
"description": "Where the mailbox's initial import stands, whether fair use is holding new mail, and the budget the mailbox syncs under. `state` is null until the worker has reported once.",
"description": "Where the mailbox's initial import stands, whether fair use is holding new mail, the budget the mailbox syncs under, the folders its sync leaves alone, and the folders the worker has seen on the server. `state` is null until the worker has reported once.",
"tags": [
"mailboxes"
],
@@ -2893,6 +2893,103 @@
}
}
}
},
"put": {
"operationId": "mailboxes_sync_update",
"summary": "Set skipped folders",
"description": "Replaces the folders an IMAP mailbox's sync leaves alone. Names are matched as the mail server lists them, without regard to case, and each covers its subfolders. Mail already stored from a newly skipped folder is removed from Warmbly (never from the mailbox), and the mailbox is re-shipped to its worker so the change applies on the next pass. The body is the desired state, so the call is naturally idempotent. INBOX and the sent, drafts, spam, trash and archive folders cannot be skipped.",
"tags": [
"mailboxes"
],
"security": [
{
"bearerAuth": []
}
],
"parameters": [
{
"name": "id",
"in": "path",
"required": true,
"description": "The mailbox id.",
"schema": {
"type": "string",
"format": "uuid"
}
}
],
"responses": {
"200": {
"description": "The skip list as stored.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/MailboxSyncSettings"
}
}
}
},
"400": {
"description": "Invalid id or body, or `invalid_sync_folder`: a folder the sync always follows, a malformed or over-long name, more than 50 names, or a mailbox that is not IMAP.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/Error"
}
}
}
},
"401": {
"description": "Missing or invalid credentials.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/Error"
}
}
}
},
"403": {
"description": "Insufficient scope, permission, or mailbox not allowed for this key.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/Error"
}
}
}
},
"404": {
"description": "Mailbox not found.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/Error"
}
}
}
},
"429": {
"description": "Rate limited.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/Error"
}
}
}
}
},
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/MailboxSyncSettings"
}
}
}
}
}
},
"/emails/{id}/identity": {
@@ -21808,6 +21905,13 @@
"org_daily_messages": {
"type": "integer",
"description": "New plus imported messages stored across the organization per UTC day."
},
"skip_folders": {
"type": "array",
"items": {
"type": "string"
},
"description": "The same skip list the mailbox carries to its worker; present when non-empty."
}
},
"required": [
@@ -21894,11 +21998,71 @@
},
"policy": {
"$ref": "#/components/schemas/MailboxSyncPolicy"
},
"skip_folders": {
"type": "array",
"items": {
"type": "string"
},
"description": "The folders the mailbox's sync leaves alone, as the server lists them. Replaced with `PUT /emails/{id}/sync`."
},
"folders": {
"type": "array",
"items": {
"$ref": "#/components/schemas/MailboxSyncFolder"
},
"description": "What the worker last listed on the server, INBOX first. Empty for Gmail and Outlook mailboxes."
}
},
"required": [
"state",
"policy"
"policy",
"skip_folders",
"folders"
]
},
"MailboxSyncFolder": {
"type": "object",
"description": "One folder the sync has seen on the server.",
"properties": {
"name": {
"type": "string",
"description": "The folder's name as the server lists it: the value to use in `skip_folders`."
},
"folder": {
"type": "string",
"enum": [
"inbox",
"sent",
"drafts",
"spam",
"trash",
"archive"
],
"description": "The canonical folder it files under. Only `inbox` folders other than INBOX itself can be skipped."
}
},
"required": [
"name",
"folder"
]
},
"MailboxSyncSettings": {
"type": "object",
"description": "The folders an IMAP mailbox's sync leaves alone.",
"properties": {
"skip_folders": {
"type": "array",
"items": {
"type": "string",
"maxLength": 255
},
"maxItems": 50,
"description": "Folder names as the server lists them. Case does not matter; each covers its subfolders. An empty list syncs every folder."
}
},
"required": [
"skip_folders"
]
},
"MailboxVerifyRequest": {
+55 -3
View File
@@ -4,20 +4,30 @@ import (
"net/http"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/api/middleware"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/models"
)
// emailSyncResponse is GET /emails/:id/sync: where the mailbox's import
// stands, whether fair use is holding it, and the budget it runs under.
// stands, whether fair use is holding it, the budget it runs under, the
// folders the owner excluded, and the folders the sync has seen so a client
// can offer them by name.
type emailSyncResponse struct {
// State is null until the worker has reported once.
State *models.SyncState `json:"state"`
Policy models.SyncPolicy `json:"policy"`
// SkipFolders is the stored skip list; the same value the policy carries,
// surfaced where PUT writes it.
SkipFolders []string `json:"skip_folders"`
// Folders is what the worker last listed on the server, INBOX first.
// Empty for Gmail and Outlook, which have no IMAP folder list.
Folders []models.SyncFolder `json:"folders"`
}
// GetEmailSync reports a mailbox's sync progress and fair-use status.
// GetEmailSync reports a mailbox's sync progress, fair-use status and folder
// settings.
func (h *Handler) GetEmailSync(c *gin.Context) {
orgID := middleware.GetOrganizationID(c)
if orgID == nil {
@@ -29,5 +39,47 @@ func (h *Handler) GetEmailSync(c *gin.Context) {
errx.JSON(c, xerr)
return
}
c.JSON(http.StatusOK, emailSyncResponse{State: state, Policy: policy})
folders, xerr := h.EmailService.GetSyncFolders(c.Request.Context(), orgID.String(), c.Param("id"))
if xerr != nil {
errx.JSON(c, xerr)
return
}
skip := policy.SkipFolders
if skip == nil {
skip = []string{}
}
c.JSON(http.StatusOK, emailSyncResponse{State: state, Policy: policy, SkipFolders: skip, Folders: folders})
}
// UpdateEmailSync replaces the folders the mailbox's sync leaves alone.
//
// Naturally idempotent: the body is the desired list, so a retry converges
// on the same stored value and no Idempotency-Key is needed.
// PUT /emails/:id/sync
func (h *Handler) UpdateEmailSync(c *gin.Context) {
orgID := middleware.GetOrganizationID(c)
if orgID == nil {
errx.JSON(c, errx.ErrUnauthorized)
return
}
accountID, err := uuid.Parse(c.Param("id"))
if err != nil {
errx.Handle(c, errx.ErrUuid)
return
}
var body models.UpdateSyncSettings
if err := c.ShouldBindJSON(&body); err != nil {
errx.Handle(c, errx.ErrInvalid)
return
}
skip, xerr := h.EmailService.UpdateSyncSettings(c.Request.Context(), orgID.String(), accountID.String(), &body)
if xerr != nil {
errx.JSON(c, xerr)
return
}
h.auditOrg(c, models.AuditActionUpdate, models.AuditEntityEmailAccount, &accountID, nil, map[string]string{"sync_skip_folders": "updated"})
c.JSON(http.StatusOK, gin.H{"skip_folders": skip})
}
+1
View File
@@ -504,6 +504,7 @@ func Run(
// and warmup gate, so a read-only key must not reach it.
emails.POST("/:id/auth-check", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.RefreshEmailAuthCheck)
emails.GET("/:id/sync", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.GetEmailSync)
emails.PUT("/:id/sync", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.UpdateEmailSync)
// Which addresses the provider will let this mailbox send as,
// and where its signature came from. The refresh is the only
// half that calls the provider, and storing its answer is what
@@ -22,6 +22,20 @@ func (s *JobsService) HandleMailboxDelete(ctx context.Context, e *models.JobEven
CaptureError(e.UserID, e.EmailID, err)
return err
}
// A folder the owner excluded from sync is retired with the mail
// already stored from it; the mail itself stays at the provider.
if e.Skipped && s.UniboxRepository != nil {
n, err := s.UniboxRepository.DeleteByFolderPaths(ctx, e.EmailID, []string{e.Mailbox})
if err != nil {
CaptureError(e.UserID, e.EmailID, err)
return err
}
log.Info().
Str("email_id", e.EmailID.String()).
Str("folder", e.Mailbox).
Int64("messages", n).
Msg("folder excluded from sync: stored mail dropped")
}
return nil
}
+3 -1
View File
@@ -23,7 +23,9 @@ import (
//
// It also drops the local unibox entry for the removed message (best-effort).
func (s *JobsService) HandleRemoveEmail(ctx context.Context, e *models.JobEventRemoveEmail) error {
if s.WarmupRepo != nil {
// A message the sync found in a folder the owner excluded is filed, not
// deleted: it is still in the mailbox, so nothing is held against anyone.
if s.WarmupRepo != nil && e.SkippedFolder == "" {
if rec, _ := s.WarmupRepo.GetWarmupReceived(ctx, e.EmailID, e.ID); rec != nil {
switch {
case s.consumeSelfMove(ctx, e.EmailID, rec.MessageID):
@@ -126,6 +126,21 @@ func TestRemoveEmailNeverStrikesARetiredMessage(t *testing.T) {
}
}
// A fresh warmup message that the sync found in a folder the owner excluded
// from sync was filed, not deleted: it is still in the mailbox, so it is not
// a strike whatever its age.
func TestRemoveEmailNeverStrikesAMessageFiledIntoASkippedFolder(t *testing.T) {
s, svc := retentionService(receivedAgo(time.Hour))
if err := s.HandleRemoveEmail(context.Background(), &models.JobEventRemoveEmail{
UserID: uuid.New(), EmailID: uuid.New(), ID: uuid.New(), SkippedFolder: "Warmer",
}); err != nil {
t.Fatal(err)
}
if len(svc.strikes) != 0 {
t.Fatalf("a move into a skipped folder was recorded as tampering: %v", svc.strikes)
}
}
// Gmail reports Delete as gaining the TRASH label. That is the owner's act
// and is judged on the same freshness rule; a spam flag is still the graver
// strike and is never subject to the window.
+5
View File
@@ -368,6 +368,11 @@ func (s *emailService) syncDataFor(ctx context.Context, emailID uuid.UUID) *mode
OrgDailyMessages: budget.DailyMessagesPerOrg,
},
}
if skip, xerr := s.emailRepository.GetSyncSkipFolders(ctx, emailID); xerr == nil {
data.Policy.SkipFolders = skip
} else {
log.Warn().Str("email_id", emailID.String()).Msg("sync skip folders lookup failed; worker syncs every folder until the next republish")
}
// A pool-linked mailbox is a warmup-only mirror: no history import.
if s.poolLink != nil {
if linked, err := s.poolLink.GetMailboxByAccount(ctx, emailID); err == nil && linked != nil {
+5
View File
@@ -19,6 +19,11 @@ type stubLoaderRepo struct {
account *models.Email
}
// GetSyncSkipFolders answers the loader's skip-list read with an empty list.
func (s *stubLoaderRepo) GetSyncSkipFolders(context.Context, uuid.UUID) ([]string, *errx.Error) {
return nil, nil
}
func (s *stubLoaderRepo) GetByID(ctx context.Context, emailAccountID uuid.UUID) (*models.Email, *errx.Error) {
return s.account, nil
}
+5
View File
@@ -25,6 +25,11 @@ type stubReauthRepo struct {
updated *models.UpdateEmail
}
// GetSyncSkipFolders answers the loader's skip-list read with an empty list.
func (s *stubReauthRepo) GetSyncSkipFolders(context.Context, uuid.UUID) ([]string, *errx.Error) {
return nil, nil
}
func (s *stubReauthRepo) GetByID(ctx context.Context, emailAccountID uuid.UUID) (*models.Email, *errx.Error) {
return s.account, nil
}
+84 -2
View File
@@ -2,10 +2,15 @@ package email
import (
"context"
"github.com/warmbly/warmbly/internal/app/instancesettings"
"golang.org/x/oauth2"
"sort"
"strings"
"time"
"github.com/rs/zerolog/log"
"github.com/warmbly/warmbly/internal/app/instancesettings"
"github.com/warmbly/warmbly/internal/client/smtpimap/imap"
"golang.org/x/oauth2"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/app/cipher"
"github.com/warmbly/warmbly/internal/app/feature"
@@ -106,6 +111,7 @@ type EmailService interface {
// operator-editable budget the mailbox syncs under.
WireSyncState(repo repository.EmailSyncStateRepository)
WireMailboxes(repo repository.MailboxRepository)
WireUnibox(repo repository.UniboxRepository)
WireSyncBudget(src SyncBudgetSource)
WirePoolLink(repo repository.PoolLinkRepository)
// WireCloudLink marks managed mailboxes, which ship to the worker without a credential.
@@ -129,6 +135,13 @@ type EmailService interface {
// GetSyncState is the dashboard's view of a mailbox's sync: nil state when
// the worker has not reported yet.
GetSyncState(ctx context.Context, userID, emailID string) (*models.SyncState, models.SyncPolicy, *errx.Error)
// GetSyncFolders is the folders the sync has seen on the server, so a
// client can name one to skip. Empty for providers without folders.
GetSyncFolders(ctx context.Context, orgID, emailID string) ([]models.SyncFolder, *errx.Error)
// UpdateSyncSettings replaces the mailbox's skip list, drops the mail
// already stored from those folders, and re-ships the mailbox so the
// worker applies it on its next pass. Returns the list as stored.
UpdateSyncSettings(ctx context.Context, orgID, emailID string, body *models.UpdateSyncSettings) ([]string, *errx.Error)
// StartWorkerReconciler periodically ensures every active mailbox is
// assigned to a worker and loaded onto it (blocks until ctx is cancelled).
StartWorkerReconciler(ctx context.Context, interval time.Duration)
@@ -170,6 +183,15 @@ type emailService struct {
lifecycleRepo repository.SendLifecycleRepository
// accountErrors is resolved-on-reconnect error state. Optional/nil-safe.
accountErrors repository.EmailAccountErrorRepository
// unibox is where a skipped folder's already-stored mail is dropped from.
// Optional: without it the worker's retirement of the folder does it.
unibox repository.UniboxRepository
}
// WireUnibox attaches the unified inbox store, for the purge that follows a
// folder being excluded from sync.
func (s *emailService) WireUnibox(repo repository.UniboxRepository) {
s.unibox = repo
}
// WireAccountErrors attaches the mailbox error log so reconnects can resolve it.
@@ -352,3 +374,63 @@ func (s *emailService) GetSyncState(ctx context.Context, orgID, emailID string)
data := s.syncDataFor(ctx, acc.ID)
return data.State, data.Policy, nil
}
// GetSyncFolders lists the IMAP folders the worker has reported for this
// mailbox, INBOX first and then by name, each with the canonical folder it
// files under. Gmail and Outlook mailboxes have no folder list here.
func (s *emailService) GetSyncFolders(ctx context.Context, orgID, emailID string) ([]models.SyncFolder, *errx.Error) {
acc, xerr := s.Get(ctx, orgID, emailID)
if xerr != nil {
return nil, xerr
}
out := []models.SyncFolder{}
userID, err := uuid.Parse(acc.UserID)
if acc.Provider != string(models.InboxProviderSMTPIMAP) || err != nil {
return out, nil
}
for _, box := range s.mailboxesFor(ctx, userID, acc.ID) {
out = append(out, models.SyncFolder{Name: box.Name, Folder: imap.CanonicalFolder(box)})
}
sort.SliceStable(out, func(i, j int) bool {
li, lj := strings.EqualFold(out[i].Name, "INBOX"), strings.EqualFold(out[j].Name, "INBOX")
if li != lj {
return li
}
return strings.ToLower(out[i].Name) < strings.ToLower(out[j].Name)
})
return out, nil
}
// UpdateSyncSettings stores a normalized skip list for an IMAP mailbox. The
// purge of already-stored mail is exact-name: the worker retires each
// subfolder it stops following by name on its next pass, and the re-ship
// makes that pass apply the new list within a minute rather than at the
// reconciler's next republish.
func (s *emailService) UpdateSyncSettings(ctx context.Context, orgID, emailID string, body *models.UpdateSyncSettings) ([]string, *errx.Error) {
acc, xerr := s.Get(ctx, orgID, emailID)
if xerr != nil {
return nil, xerr
}
if body == nil {
return nil, errx.ErrInvalid
}
if acc.Provider != string(models.InboxProviderSMTPIMAP) {
return nil, errx.NewWithIdentifier(errx.BadRequest, "invalid_sync_folder", "folders can only be skipped on an IMAP mailbox")
}
folders, xerr := imap.NormalizeSkipFolders(body.SkipFolders)
if xerr != nil {
return nil, xerr
}
if xerr := s.emailRepository.SetSyncSkipFolders(ctx, orgID, emailID, folders); xerr != nil {
return nil, xerr
}
if s.unibox != nil && len(folders) > 0 {
if n, err := s.unibox.DeleteByFolderPaths(ctx, acc.ID, folders); err != nil {
log.Warn().Err(err).Str("email_id", acc.ID.String()).Msg("sync skip folders: purge of stored mail failed; the worker retires the folders on its next pass")
} else if n > 0 {
log.Info().Str("email_id", acc.ID.String()).Int64("messages", n).Msg("sync skip folders: stored mail from skipped folders dropped")
}
}
s.loadAccountBestEffort(ctx, acc.ID)
return folders, nil
}
@@ -59,6 +59,11 @@ func (s *stubRemovalRepo) Update(ctx context.Context, orgID, emailAccountID stri
return &models.Email{ID: id, Status: status}, nil
}
// GetSyncSkipFolders answers the loader's skip-list read with an empty list.
func (s *stubRemovalRepo) GetSyncSkipFolders(context.Context, uuid.UUID) ([]string, *errx.Error) {
return nil, nil
}
func (s *stubRemovalRepo) GetByID(ctx context.Context, emailAccountID uuid.UUID) (*models.Email, *errx.Error) {
if s.getErr != nil {
return nil, s.getErr
+142
View File
@@ -46,6 +46,22 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError {
// that reached it would re-file known mail as archive under a second UID.
folders = slices.DeleteFunc(folders, func(b models.Mailbox) bool { return imapVirtualFolder(&b) })
// The folders the owner excluded leave the listing here, before renames
// are followed and before the delete sweep: one already synced is retired
// like a folder the server dropped, and one never seen is never
// baselined. They are kept aside so mail that moves into one of them can
// be recognised as filed rather than lost.
var skipped []models.Mailbox
if skip := w.skipFolders(); len(skip) > 0 {
folders = slices.DeleteFunc(folders, func(b models.Mailbox) bool {
if !imap.SkipsFolder(b, skip) {
return false
}
skipped = append(skipped, b)
return true
})
}
// Before anything is matched by name, follow the folders whose name
// changed. A rename read as a delete plus a first sighting would orphan
// every message filed under the old name and re-import the folder's
@@ -93,6 +109,9 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError {
}
changed := imapFolderChanged(befBox, box, condStore)
// Decided against the cursor the previous pass left, before the
// block below moves it.
movedOut := len(skipped) > 0 && imapMovedOut(befBox, box)
fullyProcessed := true
var touched map[string]struct{}
if changed && !stats.aborted {
@@ -133,7 +152,12 @@ func (w *WMail) Sync(ctx context.Context) *errx.MailError {
if err := w.imapReconcileDrafts(ctx, box, touched, stats); err != nil {
return err
}
} else if movedOut && !stats.aborted {
if err := w.imapReconcileSkipped(ctx, box, skipped, stats); err != nil {
return err
}
}
befBox.Messages = box.Messages
// Without CONDSTORE a message marked read elsewhere moves no cursor,
// so read state is mirrored by a periodic scan instead. It runs after
@@ -166,6 +190,9 @@ outer:
EmailID: w.ID,
Mailbox: box.Name,
UIDValidity: box.UIDValidity,
// A folder that is still on the server but now excluded takes
// the mail already stored from it along.
Skipped: slices.ContainsFunc(skipped, func(s models.Mailbox) bool { return s.Name == box.Name }),
}); err != nil {
return nil
}
@@ -199,6 +226,121 @@ outer:
return nil
}
// skipFolders is the owner's exclusion list as the policy in force carries
// it; a republished ADD_EMAIL changes it between passes.
func (w *WMail) skipFolders() []string {
if w.gov == nil {
return nil
}
return w.gov.Policy().SkipFolders
}
// imapMovedOut reports whether messages left the folder since the last
// pass: the count is below the previous count plus the arrivals the UIDNEXT
// advance accounts for. An expunge moves neither cursor on every server, so
// the count is the one signal that always carries it. Only a count taken by
// this worker session counts: a folder seeded from the control plane has
// none, and the first pass baselines it.
func imapMovedOut(before, now *models.Mailbox) bool {
if before.Messages == 0 || now.UIDNext < before.UIDNext {
return false
}
arrivals := now.UIDNext - before.UIDNext
return now.Messages < before.Messages+arrivals
}
// imapReconcileSkipped retires the platform's rows for mail that left this
// folder for one the owner excluded from sync. Nothing else that leaves a
// folder is touched: a message can go somewhere the sync does not follow
// (Gmail's All Mail) and still be wanted, so a row goes only when its
// Message-ID is found in a skipped folder. Rows checked once and found
// nowhere are remembered for the session, so a folder the owner emptied by
// hand does not cost a search per row on every later pass.
func (w *WMail) imapReconcileSkipped(ctx context.Context, box *models.Mailbox, skipped []models.Mailbox, stats *tickStats) *errx.MailError {
if w.SyncContext == nil || len(skipped) == 0 {
return nil
}
stored, err := w.SyncContext.ListFolderMessages(ctx, w.UserID, w.ID, box.Name, box.UIDValidity)
if err != nil {
return w.controlPlaneError(err, stats)
}
if len(stored) == 0 {
return nil
}
client := w.SmtpImapData.ImapClient
_, gen, serr := client.SelectForSyncGen(box.Name)
if serr != nil {
return serr
}
// The same guard as the drafts reconciliation: UIDs only mean anything
// inside one generation.
if gen != box.UIDValidity {
return nil
}
present, aerr := client.SearchAll()
if aerr != nil {
return aerr
}
live := make(map[uint32]struct{}, len(present))
for _, uid := range present {
live[uint32(uid)] = struct{}{}
}
if w.skipChecked == nil || len(w.skipChecked) > imapSkipCheckedMax {
w.skipChecked = make(map[string]struct{})
}
for _, m := range stored {
if _, ok := live[m.UID]; ok {
continue
}
if ctx.Err() != nil {
return nil
}
key := fmt.Sprintf("%s\x00%d\x00%d", box.Name, box.UIDValidity, m.UID)
if _, done := w.skipChecked[key]; done {
continue
}
folder := w.imapFindInSkipped(ctx, skipped, m.MessageID)
if folder == "" {
w.skipChecked[key] = struct{}{}
continue
}
if err := w.onEvent(models.JobEventTypeRemoveEmail, &models.JobEventRemoveEmail{
UserID: w.UserID,
EmailID: w.ID,
ID: m.ID,
SkippedFolder: folder,
}); err != nil {
return w.controlPlaneError(err, stats)
}
w.skipChecked[key] = struct{}{}
}
return nil
}
// imapSkipCheckedMax bounds the per-session memory of rows already looked
// for in the skipped folders; past it the memory starts over.
const imapSkipCheckedMax = 20_000
// imapFindInSkipped names the skipped folder holding the message, or "".
// A key the sync made up for a message without a Message-ID was never on
// the wire, so there is nothing to search for.
func (w *WMail) imapFindInSkipped(ctx context.Context, skipped []models.Mailbox, messageID string) string {
if messageID == "" || strings.HasPrefix(messageID, "no-msgid/") {
return ""
}
for i := range skipped {
uid, err := w.SmtpImapData.ImapClient.FindUIDByMessageID(ctx, skipped[i].Name, messageID)
if err != nil {
log.Debug().Err(err).Str("email_id", w.ID.String()).Str("folder", skipped[i].Name).Msg("sync: search in skipped folder failed")
continue
}
if uid != 0 {
return skipped[i].Name
}
}
return ""
}
// imapFolderChanged reports whether a folder has anything new since the
// cursor we hold for it. With CONDSTORE the mod-sequence answers for new mail
// AND flag changes; without it only arrivals are visible here, and flag
@@ -0,0 +1,199 @@
package wmail
import (
"context"
"testing"
goimap "github.com/emersion/go-imap/v2"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/repository"
)
func (c *fakeImapConn) FindUIDByMessageID(_ context.Context, folder, messageID string) (uint32, error) {
c.finds++
return c.inSkipped[folder][messageID], nil
}
// skipBudget is fixedBudget with an owner's skip list in the policy.
type skipBudget struct {
*fixedBudget
skip []string
}
func (b *skipBudget) Policy() models.SyncPolicy {
return normalizePolicy(models.SyncPolicy{SkipFolders: b.skip})
}
func mailboxEvents(events []captured, kind models.JobEventType) []captured {
var out []captured
for _, e := range events {
if e.eventType == kind {
out = append(out, e)
}
}
return out
}
// A folder on the skip list is never baselined and never fetched, and its
// subfolders go with it; a folder that merely shares the prefix does not.
func TestImapSyncNeverBaselinesSkippedFolders(t *testing.T) {
conn := &fakeImapConn{folders: []models.Mailbox{
{Name: "INBOX", UIDValidity: 7, HighestModSeq: 100, Delim: "/"},
{Name: "Warmer", UIDValidity: 9, HighestModSeq: 100, Delim: "/"},
{Name: "Warmer/Replies", UIDValidity: 10, HighestModSeq: 100, Delim: "/"},
{Name: "Warmer2", UIDValidity: 11, HighestModSeq: 100, Delim: "/"},
}}
budget := &skipBudget{fixedBudget: &fixedBudget{allow: 10}, skip: []string{"warmer"}}
w, events := newIMAPTestMail(conn, budget, &models.Mailbox{Name: "INBOX", UIDValidity: 7, HighestModSeq: 100})
if err := w.Sync(t.Context()); err != nil {
t.Fatalf("Sync: %v", err)
}
baselined := map[string]bool{}
for _, e := range mailboxEvents(*events, models.JobEventTypeMailboxUpdate) {
baselined[e.body.(*models.JobEventMailboxUpdate).Data.Name] = true
}
if baselined["Warmer"] || baselined["Warmer/Replies"] {
t.Fatalf("a skipped folder was baselined: %v", baselined)
}
if !baselined["Warmer2"] {
t.Fatal("Warmer2 shares a prefix but is a different folder; it must be synced")
}
if len(mailboxEvents(*events, models.JobEventTypeMailboxDelete)) != 0 {
t.Fatal("nothing was tracked for the skipped folders, so nothing should be retired")
}
if len(w.SmtpImapData.Mailboxes) != 2 {
t.Fatalf("tracked %d folders, want INBOX and Warmer2", len(w.SmtpImapData.Mailboxes))
}
}
// A folder synced before the owner excluded it is retired like one the
// server dropped, with the marker that takes its stored mail along.
func TestImapSyncRetiresNewlySkippedFolder(t *testing.T) {
conn := &fakeImapConn{folders: []models.Mailbox{
{Name: "INBOX", UIDValidity: 7, HighestModSeq: 100, Delim: "/"},
{Name: "Warmer", UIDValidity: 9, HighestModSeq: 100, Delim: "/"},
}}
budget := &skipBudget{fixedBudget: &fixedBudget{allow: 10}, skip: []string{"Warmer"}}
w, events := newIMAPTestMail(conn, budget, &models.Mailbox{Name: "INBOX", UIDValidity: 7, HighestModSeq: 100})
w.SmtpImapData.Mailboxes = append(w.SmtpImapData.Mailboxes, &models.Mailbox{Name: "Warmer", UIDValidity: 9, HighestModSeq: 90})
w.tracker.setFolder("Warmer", models.SyncFolderCursor{UID: 40})
if err := w.Sync(t.Context()); err != nil {
t.Fatalf("Sync: %v", err)
}
deletes := mailboxEvents(*events, models.JobEventTypeMailboxDelete)
if len(deletes) != 1 {
t.Fatalf("got %d MAILBOX_DELETE events, want 1", len(deletes))
}
del := deletes[0].body.(*models.JobEventMailboxDelete)
if del.Mailbox != "Warmer" || !del.Skipped {
t.Fatalf("retired %+v, want Warmer with the skipped marker", del)
}
if len(w.SmtpImapData.Mailboxes) != 1 || w.SmtpImapData.Mailboxes[0].Name != "INBOX" {
t.Fatalf("tracked %v, want just INBOX", w.SmtpImapData.Mailboxes)
}
if cur := w.tracker.folder("Warmer"); cur.UID != 0 {
t.Fatalf("the backfill floor for Warmer survived: %+v", cur)
}
if conn.fetches != 0 {
t.Fatalf("fetched %d batches from a skipped folder, want none", conn.fetches)
}
}
// Mail that left a synced folder is removed only when it turns up in a
// skipped folder; mail that left for anywhere else is kept, and is not
// searched for again on the next pass.
func TestImapSyncRemovesMailMovedIntoSkippedFolder(t *testing.T) {
conn := &fakeImapConn{
folders: []models.Mailbox{
// Three messages were here last pass; one arrived and two left.
{Name: "INBOX", UIDValidity: 7, HighestModSeq: 100, UIDNext: 11, Messages: 2, Delim: "/"},
{Name: "Warmer", UIDValidity: 9, HighestModSeq: 100, Delim: "/"},
},
all: []goimap.UID{8, 10},
inSkipped: map[string]map[string]uint32{"Warmer": {"<9@fake.test>": 3}},
}
budget := &skipBudget{fixedBudget: &fixedBudget{allow: 10}, skip: []string{"Warmer"}}
w, events := newIMAPTestMail(conn, budget, &models.Mailbox{Name: "INBOX", UIDValidity: 7, HighestModSeq: 100, UIDNext: 10, Messages: 3})
w.EmailMessageMapRepository = knownMessageMap{id: uuid.New().String()}
warmup, deleted, kept := uuid.New(), uuid.New(), uuid.New()
w.SyncContext = &fakeSyncContext{stored: map[string][]repository.StoredFolderMessage{
"INBOX": {
{UID: 7, MessageID: "<7@fake.test>", ID: deleted},
{UID: 8, MessageID: "<8@fake.test>", ID: kept},
{UID: 9, MessageID: "<9@fake.test>", ID: warmup},
},
}}
if err := w.Sync(t.Context()); err != nil {
t.Fatalf("Sync: %v", err)
}
removed := removeIDs(*events)
if len(removed) != 1 || removed[0] != warmup {
t.Fatalf("removed %v, want exactly the message found in Warmer (%s)", removed, warmup)
}
for _, e := range mailboxEvents(*events, models.JobEventTypeRemoveEmail) {
if got := e.body.(*models.JobEventRemoveEmail).SkippedFolder; got != "Warmer" {
t.Fatalf("removal named folder %q, want Warmer", got)
}
}
if conn.finds != 2 {
t.Fatalf("searched the skipped folder %d times, want once per vanished row (2)", conn.finds)
}
// Next pass, nothing else changed: the row that was deleted for good is
// remembered and not searched for again.
conn.folders[0].Messages = 2
if err := w.Sync(t.Context()); err != nil {
t.Fatalf("second Sync: %v", err)
}
if conn.finds != 2 {
t.Fatalf("a settled row was searched for again (finds = %d)", conn.finds)
}
if len(removeIDs(*events)) != 1 {
t.Fatal("the second pass removed something")
}
}
// Without a skip list the count check is off entirely: a folder the owner
// emptied costs no lookups and loses no rows.
func TestImapSyncLeavesVanishedMailAloneWithoutSkipList(t *testing.T) {
conn := &fakeImapConn{
folders: []models.Mailbox{{Name: "INBOX", UIDValidity: 7, HighestModSeq: 100, UIDNext: 10, Messages: 1, Delim: "/"}},
all: []goimap.UID{8},
}
w, events := newIMAPTestMail(conn, &fixedBudget{allow: 10}, &models.Mailbox{Name: "INBOX", UIDValidity: 7, HighestModSeq: 100, UIDNext: 10, Messages: 3})
ctx := &fakeSyncContext{stored: map[string][]repository.StoredFolderMessage{
"INBOX": {{UID: 7, MessageID: "<7@fake.test>", ID: uuid.New()}},
}}
w.SyncContext = ctx
if err := w.Sync(t.Context()); err != nil {
t.Fatalf("Sync: %v", err)
}
if ctx.calls != 0 || conn.finds != 0 || len(removeIDs(*events)) != 0 {
t.Fatalf("folder rows were reconciled without a skip list: lookups=%d finds=%d removed=%d", ctx.calls, conn.finds, len(removeIDs(*events)))
}
}
func TestImapMovedOut(t *testing.T) {
cases := []struct {
name string
before, now models.Mailbox
want bool
}{
{"nothing changed", models.Mailbox{Messages: 3, UIDNext: 10}, models.Mailbox{Messages: 3, UIDNext: 10}, false},
{"one arrived", models.Mailbox{Messages: 3, UIDNext: 10}, models.Mailbox{Messages: 4, UIDNext: 11}, false},
{"one left", models.Mailbox{Messages: 3, UIDNext: 10}, models.Mailbox{Messages: 2, UIDNext: 10}, true},
{"one arrived and one left", models.Mailbox{Messages: 3, UIDNext: 10}, models.Mailbox{Messages: 3, UIDNext: 11}, true},
{"no baseline from this session", models.Mailbox{Messages: 0, UIDNext: 10}, models.Mailbox{Messages: 2, UIDNext: 10}, false},
{"cursor went backwards", models.Mailbox{Messages: 3, UIDNext: 10}, models.Mailbox{Messages: 1, UIDNext: 5}, false},
}
for _, tc := range cases {
if got := imapMovedOut(&tc.before, &tc.now); got != tc.want {
t.Errorf("%s: imapMovedOut = %v, want %v", tc.name, got, tc.want)
}
}
}
@@ -36,6 +36,10 @@ type fakeImapConn struct {
// selectGen, when non-zero, is the UIDVALIDITY SELECT reports, which the
// reconciliation compares against the one the listing gave it.
selectGen uint32
// inSkipped is what FindUIDByMessageID answers per folder name, and
// finds counts how often it was asked.
inSkipped map[string]map[string]uint32
finds int
}
func (c *fakeImapConn) Folders() ([]models.Mailbox, *errx.MailError) { return c.folders, nil }
+4
View File
@@ -98,6 +98,10 @@ type WMail struct {
// flagScan is the previous flag snapshot per folder name, used only on
// IMAP servers without CONDSTORE, which cannot say what changed.
flagScan map[string]*folderFlagScan
// skipChecked remembers, for this session, the stored rows that left a
// synced folder and were looked for in the skipped folders without being
// found, so they are not searched for again on every pass.
skipChecked map[string]struct{}
// transportFailures counts consecutive passes that could not reach the
// mail server, which paces the retry and keeps one outage to one warning.
transportFailures int
+86
View File
@@ -2,8 +2,10 @@ package imap
import (
"errors"
"fmt"
"sort"
"strings"
"unicode/utf8"
"github.com/emersion/go-imap/v2"
"github.com/rs/zerolog/log"
@@ -41,6 +43,7 @@ func (c *Client) foldersCapped(limit int) ([]models.Mailbox, *errx.MailError) {
status := &imap.StatusOptions{
UIDValidity: true,
UIDNext: true,
NumMessages: true,
// Asking a server without CONDSTORE for HIGHESTMODSEQ is a BAD.
HighestModSeq: caps.Has(imap.CapCondStore),
}
@@ -101,6 +104,9 @@ func (c *Client) foldersCapped(limit int) ([]models.Mailbox, *errx.MailError) {
box.UIDValidity = st.UIDValidity
box.UIDNext = uint32(st.UIDNext)
box.HighestModSeq = st.HighestModSeq
if st.NumMessages != nil {
box.Messages = *st.NumMessages
}
resp = append(resp, box)
}
@@ -262,6 +268,86 @@ func BackfillEligible(box models.Mailbox) bool {
return true
}
// SkipsFolder reports whether box is one the mailbox owner asked the sync to
// leave alone. A name in skip matches the folder listed under it and every
// folder below it, compared the way servers compare names: case does not
// count. INBOX and the special folders (by attribute or by name) never match,
// whatever the list says, because a sync without them is a broken mailbox
// rather than a quieter one.
func SkipsFolder(box models.Mailbox, skip []string) bool {
if len(skip) == 0 || !skippableFolder(box) {
return false
}
for _, s := range skip {
s = strings.TrimSpace(s)
if s == "" || strings.EqualFold(s, "INBOX") {
continue
}
if strings.EqualFold(box.Name, s) {
return true
}
if box.Delim != "" && hasPrefixFold(box.Name, s+box.Delim) {
return true
}
}
return false
}
// skippableFolder is false for INBOX and for any folder the mailbox needs
// whole: a special-use attribute or a recognised special name.
func skippableFolder(box models.Mailbox) bool {
if strings.EqualFold(strings.TrimSpace(box.Name), "INBOX") {
return false
}
for _, a := range box.Attrs {
switch strings.ToLower(a) {
case "\\inbox", "\\sent", "\\drafts", "\\junk", "\\trash", "\\archive", "\\all", "\\flagged", "\\important":
return false
}
}
return CanonicalFolder(box) == models.FolderInbox
}
// NormalizeSkipFolders is the write-side check on a skip list: trimmed,
// deduplicated without regard to case, bounded in count and length, no
// control characters, and none of the names the sync must keep. The result
// is what gets stored; the error names the first entry refused.
func NormalizeSkipFolders(names []string) ([]string, *errx.Error) {
out := make([]string, 0, len(names))
seen := make(map[string]struct{}, len(names))
for _, raw := range names {
name := strings.TrimSpace(raw)
if name == "" {
continue
}
if utf8.RuneCountInString(name) > config.SyncSkipFolderNameMax {
return nil, skipFolderError(name, "is longer than the folder name limit")
}
for _, r := range name {
if r < 0x20 || r == 0x7f {
return nil, skipFolderError(name, "contains a control character")
}
}
if !skippableFolder(models.Mailbox{Name: name}) {
return nil, skipFolderError(name, "is a folder the sync always follows")
}
key := strings.ToLower(name)
if _, dup := seen[key]; dup {
continue
}
seen[key] = struct{}{}
out = append(out, name)
}
if len(out) > config.SyncSkipFoldersMax {
return nil, errx.NewWithIdentifier(errx.BadRequest, "invalid_sync_folder", fmt.Sprintf("at most %d folders can be skipped", config.SyncSkipFoldersMax))
}
return out, nil
}
func skipFolderError(name, why string) *errx.Error {
return errx.NewWithIdentifier(errx.BadRequest, "invalid_sync_folder", fmt.Sprintf("folder %q %s", name, why))
}
// CanonicalFolder maps an IMAP folder to the canonical unibox folder.
// Special-use attributes are authoritative, with a name fallback for servers
// that do not advertise them; unrecognized user folders file as inbox so
@@ -0,0 +1,80 @@
package imap
import (
"strings"
"testing"
"github.com/warmbly/warmbly/internal/config"
"github.com/warmbly/warmbly/internal/models"
)
func TestSkipsFolder(t *testing.T) {
skip := []string{"Warmer", "Clients/Acme", " inbox "}
cases := []struct {
name string
box models.Mailbox
want bool
}{
{"exact", models.Mailbox{Name: "Warmer", Delim: "/"}, true},
{"case does not count", models.Mailbox{Name: "WARMER", Delim: "/"}, true},
{"subfolder", models.Mailbox{Name: "Warmer/Replies", Delim: "/"}, true},
{"subfolder under a dot server", models.Mailbox{Name: "Warmer.Replies", Delim: "."}, true},
{"nested entry", models.Mailbox{Name: "Clients/Acme/2026", Delim: "/"}, true},
{"prefix without a delimiter is a different folder", models.Mailbox{Name: "Warmer2", Delim: "/"}, false},
{"no delimiter reported means exact only", models.Mailbox{Name: "Warmer/Replies", Delim: ""}, false},
{"another folder", models.Mailbox{Name: "Receipts", Delim: "/"}, false},
{"INBOX is never skipped, even when listed", models.Mailbox{Name: "INBOX", Delim: "/"}, false},
{"an INBOX entry does not take the inbox's subfolders", models.Mailbox{Name: "INBOX/Work", Delim: "/"}, false},
{"special-use attribute wins over the name", models.Mailbox{Name: "Warmer", Delim: "/", Attrs: []string{"\\Sent"}}, false},
{"special name wins", models.Mailbox{Name: "Sent", Delim: "/"}, false},
}
for _, tc := range cases {
if got := SkipsFolder(tc.box, skip); got != tc.want {
t.Errorf("%s: SkipsFolder(%q) = %v, want %v", tc.name, tc.box.Name, got, tc.want)
}
}
if SkipsFolder(models.Mailbox{Name: "Warmer"}, nil) {
t.Error("an empty list skips nothing")
}
}
func TestNormalizeSkipFolders(t *testing.T) {
got, xerr := NormalizeSkipFolders([]string{" Warmer ", "", "warmer", "Clients/Acme", "INBOX/Newsletters"})
if xerr != nil {
t.Fatalf("unexpected refusal: %v", xerr)
}
want := []string{"Warmer", "Clients/Acme", "INBOX/Newsletters"}
if strings.Join(got, "|") != strings.Join(want, "|") {
t.Fatalf("normalized %v, want %v", got, want)
}
refused := [][]string{
{"INBOX"},
{"inbox"},
{"Sent"},
{"INBOX.Drafts"},
{"Junk"},
{"[Gmail]/Trash"},
{"Archive"},
{"bad\x01name"},
{strings.Repeat("x", config.SyncSkipFolderNameMax+1)},
}
for _, in := range refused {
if _, xerr := NormalizeSkipFolders(in); xerr == nil {
t.Errorf("NormalizeSkipFolders(%q) accepted, want a refusal", in)
} else if xerr.Identifier != "invalid_sync_folder" {
t.Errorf("NormalizeSkipFolders(%q) refused as %q, want invalid_sync_folder", in, xerr.Identifier)
}
}
many := make([]string, config.SyncSkipFoldersMax+1)
for i := range many {
many[i] = "Folder" + strings.Repeat("x", i)
}
if _, xerr := NormalizeSkipFolders(many); xerr == nil {
t.Error("a list over the cap was accepted")
}
if got, _ := NormalizeSkipFolders(nil); got == nil || len(got) != 0 {
t.Errorf("nil in, want an empty list out, got %#v", got)
}
}
+2
View File
@@ -95,6 +95,8 @@ const (
SyncBackfillPerMinute = 240 // backfill pacing per mailbox
SyncFloodPerHour = 5_000 // new live messages observed in one hour that mark a mailbox as flooding
SyncThrottleEscalationDays = 3 // throttled UTC days out of the last 7 that deactivate a mailbox
SyncSkipFoldersMax = 50 // folders one mailbox may exclude from sync
SyncSkipFolderNameMax = 255 // characters in one excluded folder name
// Forms. Funnel events feed analytics ranges up to 90 days, so the default
// window keeps double coverage. Operator-editable under Instance settings.
@@ -0,0 +1,2 @@
ALTER TABLE public.email_accounts
DROP COLUMN IF EXISTS sync_skip_folders;
@@ -0,0 +1,14 @@
-- A mailbox owner can name folders the sync leaves alone.
--
-- The IMAP sync follows every folder the server lists, and a folder it does
-- not recognise files as inbox so the mail in it stays visible. A folder a
-- third-party tool fills with its own machine traffic then lands in the
-- unified inbox, spends the mailbox's sync budget, and is classified like a
-- reply. The names here are matched against the server's listing and the
-- folders they name (and their subfolders) are never opened.
--
-- email_accounts.sync_skip_folders: folder names as the server lists them.
-- Empty means everything is synced. The special folders (inbox, sent,
-- drafts, spam, trash, archive) cannot be named here.
ALTER TABLE public.email_accounts
ADD COLUMN sync_skip_folders text[] NOT NULL DEFAULT '{}';
+4
View File
@@ -20,6 +20,10 @@ type JobEventRemoveEmail struct {
UserID uuid.UUID `json:"user_id" avro:"user_id"`
EmailID uuid.UUID `json:"email_id" avro:"email_id"`
ID uuid.UUID `json:"id" avro:"id"`
// SkippedFolder is set when the message was found in a folder the owner
// excluded from sync: it still exists in the mailbox, so the removal is
// filing, not deletion.
SkippedFolder string `json:"skipped_folder,omitempty" avro:"skipped_folder"`
}
type JobEventFlags struct {
+3
View File
@@ -18,6 +18,9 @@ type JobEventMailboxDelete struct {
Mailbox string `json:"mailbox,omitempty" avro:"mailbox"`
// UIDValidity is that legacy fallback and nothing else.
UIDValidity uint32 `json:"uid_validity" avro:"uid_validity"`
// Skipped means the folder is still on the server but the owner excluded
// it from sync, so the mail already stored from it is retired as well.
Skipped bool `json:"skipped,omitempty" avro:"skipped"`
}
// JobEventMailboxRename is a folder that kept its UIDVALIDITY under a new
+5
View File
@@ -22,6 +22,11 @@ type Mailbox struct {
// ("/" on Gmail, "." on many Dovecots). Empty when the server reported
// none, where the leaf is guessed instead.
Delim string `json:"delim,omitempty" avro:"delim"`
// Messages is the folder's message count at the last listing. Held by the
// worker between passes and not persisted: a drop that the arrivals do
// not explain is what makes a pass look for mail moved into a folder the
// sync does not follow.
Messages uint32 `json:"messages,omitempty" avro:"messages"`
UpdatedAt time.Time `json:"updated_at" avro:"updated_at"`
}
+19
View File
@@ -21,6 +21,25 @@ type SyncPolicy struct {
// OrgDailyMessages caps new plus backfilled messages stored across the
// whole organization per UTC day.
OrgDailyMessages int `json:"org_daily_messages" avro:"org_daily_messages"`
// SkipFolders names the folders the sync leaves alone, as the server
// lists them; each also covers its subfolders. Per mailbox, unlike the
// budgets above, and only meaningful on IMAP.
SkipFolders []string `json:"skip_folders,omitempty" avro:"skip_folders"`
}
// SyncFolder is one folder the sync has seen on the server, as GET
// /emails/:id/sync reports it: the name to use in skip_folders and the
// canonical folder it files under, so a client can tell which ones are the
// special folders that cannot be skipped.
type SyncFolder struct {
Name string `json:"name"`
Folder string `json:"folder"`
}
// UpdateSyncSettings is the body of PUT /emails/:id/sync. The list is the
// desired state, so a retry converges.
type UpdateSyncSettings struct {
SkipFolders []string `json:"skip_folders"`
}
// SyncBackfillStatus is where the initial import stands.
+47
View File
@@ -115,6 +115,13 @@ type EmailRepository interface {
SetWarmupLifecycle(ctx context.Context, orgID, emailAccountID, action string) (*models.Email, *errx.Error)
UpdateTrackingDomain(ctx context.Context, orgID, emailAccountID, domain string, verified bool, verifiedAt *time.Time) *errx.Error
UpdateTrackDirectMail(ctx context.Context, orgID, emailAccountID string, enabled bool) *errx.Error
// GetSyncSkipFolders is the folders this mailbox's sync leaves alone.
// Read without a tenant predicate by the loader, which ships it to the
// worker inside the mailbox's sync policy.
GetSyncSkipFolders(ctx context.Context, emailAccountID uuid.UUID) ([]string, *errx.Error)
// SetSyncSkipFolders replaces that list, scoped by organization like every
// other mailbox setting a workspace admin may change.
SetSyncSkipFolders(ctx context.Context, orgID, emailAccountID string, folders []string) *errx.Error
// ListOrganizationIDs names every workspace with a mailbox, for sweeps that
// run per workspace rather than per event.
ListOrganizationIDs(ctx context.Context) ([]uuid.UUID, error)
@@ -2121,6 +2128,46 @@ func (r *emailRepository) UpdateTrackDirectMail(ctx context.Context, orgID, emai
return nil
}
// GetSyncSkipFolders reads the mailbox's skip list; an empty list for a
// mailbox that never set one.
func (r *emailRepository) GetSyncSkipFolders(ctx context.Context, emailAccountID uuid.UUID) ([]string, *errx.Error) {
query := `SELECT sync_skip_folders FROM email_accounts WHERE id = $1`
var folders []string
if err := r.DB.QueryRow(ctx, query, emailAccountID).Scan(&folders); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, errx.ErrNotFound
}
db.CaptureError(err, query, []any{emailAccountID}, "query")
return nil, errx.InternalError()
}
return folders, nil
}
// SetSyncSkipFolders stores the skip list as given; the caller has already
// normalized it. A nil slice would bind as SQL NULL against a NOT NULL
// column, so an empty list is written as an empty array.
func (r *emailRepository) SetSyncSkipFolders(ctx context.Context, orgID, emailAccountID string, folders []string) *errx.Error {
if folders == nil {
folders = []string{}
}
query := `
UPDATE email_accounts
SET sync_skip_folders = $1, updated_at = NOW()
WHERE organization_id = $2 AND id = $3
`
params := []any{folders, orgID, emailAccountID}
cmd, err := r.DB.Exec(ctx, query, params...)
if err != nil {
db.CaptureError(err, query, params, "exec")
return errx.InternalError()
}
if cmd.RowsAffected() == 0 {
return errx.ErrNotFound
}
return nil
}
// ListOrganizationIDs returns every workspace that has at least one mailbox.
func (r *emailRepository) ListOrganizationIDs(ctx context.Context) ([]uuid.UUID, error) {
rows, err := r.DB.Query(ctx, `SELECT DISTINCT organization_id FROM email_accounts WHERE organization_id IS NOT NULL`)
+17
View File
@@ -69,6 +69,10 @@ type UniboxRepository interface {
// are left out: there is nothing to relay through.
SeenRelayTargets(ctx context.Context, orgID uuid.UUID, ids []uuid.UUID) ([]models.SeenRelayTarget, error)
Delete(ctx context.Context, userID, id uuid.UUID) error
// DeleteByFolderPaths drops every stored message one mailbox synced from
// the named source folders, for folders the owner has excluded from sync.
// Exact names only; the worker names each subfolder it retires itself.
DeleteByFolderPaths(ctx context.Context, emailID uuid.UUID, folderPaths []string) (int64, error)
ListWarmupReviewCandidates(ctx context.Context, afterID uuid.UUID, limit int) ([]models.JobEventNewEmail, error)
// ListUnprocessedCampaignReplies pages inbound messages that reply
// processing never claimed and that look like campaign replies: they
@@ -918,6 +922,19 @@ func (r *uniboxRepository) Delete(ctx context.Context, userID, id uuid.UUID) err
return tx.Commit(ctx)
}
// DeleteByFolderPaths removes the mirror rows for whole source folders. The
// mail stays where it is at the provider; only the platform's copy goes.
func (r *uniboxRepository) DeleteByFolderPaths(ctx context.Context, emailID uuid.UUID, folderPaths []string) (int64, error) {
if len(folderPaths) == 0 {
return 0, nil
}
tag, err := r.db.Exec(ctx, `DELETE FROM unibox_emails WHERE email_id = $1 AND folder_path = ANY($2)`, emailID, folderPaths)
if err != nil {
return 0, err
}
return tag.RowsAffected(), nil
}
// queryPreviewList executes a query returning preview rows with limit+1 pagination.
func (r *uniboxRepository) queryPreviewList(ctx context.Context, query string, args []any, limit int) (*models.MailSearchResult, error) {
rows, err := r.db.Query(ctx, query, args...)
+1 -1
View File
@@ -38,7 +38,7 @@ Run `warmblyctl <family> --help` for subcommands and `warmblyctl <family>
| `me` | Identity and granted scopes |
| `campaign` | list, get, create, update, delete, steps, senders, preflight, start, stop, test-email, logs, plan, pause-lead / resume-lead |
| `contact` | list (search), get, lookup, create, update, delete, notes, timeline, import, export |
| `mailbox` | list, get, update, delete, auth-check, sync, identity, refresh-identity, behavior, verify, send, warmup-start/pause/resume/stop/status |
| `mailbox` | list, get, update, delete, auth-check, sync, skip-folders, identity, refresh-identity, behavior, verify, send, warmup-start/pause/resume/stop/status |
| `inbox` | list, count, thread, seen, reply, compose, agent drafts, scheduled sends |
| `analytics` | dashboard, deliverability, warmup, accounts, campaigns, usage, audit-logs |
| `settings` | outreach and suppression settings |
+1 -1
View File
@@ -74,7 +74,7 @@ gives the arguments and flags. Ids are positional, not flags.
| `status` | one call for "what is happening": mailboxes needing attention, what is sending, what is unread |
| `campaign` | list, view, create, edit, delete, steps, senders, segments, preflight, test, start, stop, logs, plan, pause-lead / resume-lead |
| `contact` | list, view, create, edit, delete, lookup, timeline, emails, notes, import, export, verify |
| `mailbox` | list, view, edit, check, sync, identity, refresh-identity, behavior, warmup, hold, release, send |
| `mailbox` | list, view, edit, check, sync, skip-folders, identity, refresh-identity, behavior, warmup, hold, release, send |
| `inbox` | list, view, thread, read, reply, compose, drafts, scheduled, snooze |
| `suppression` | the list of addresses and domains that get no campaign mail |
| `segment`, `template`, `automation`, `form` | audiences, reply templates, automations, lead capture |
@@ -642,7 +642,7 @@ function OverviewTab({ status, loading, mailbox }: { status?: import("@/lib/api/
</div>
{/* Sync: import progress and fair-use status */}
<SyncStatusCard mailboxId={mailbox.id} />
<SyncStatusCard mailboxId={mailbox.id} provider={mailbox.provider} />
{/* Key stats */}
<div className="grid grid-cols-2 divide-x divide-y divide-slate-200/60">
@@ -1,7 +1,14 @@
import { useState } from "react";
import { motion } from "framer-motion";
import { CheckCircle2Icon, DownloadIcon, HourglassIcon, RefreshCwIcon } from "lucide-react";
import { CheckCircle2Icon, DownloadIcon, HourglassIcon, PlusIcon, RefreshCwIcon } from "lucide-react";
import toast from "react-hot-toast";
import { CheckSquare } from "@/components/ui/check-square";
import { TextInput } from "@/components/ui/field";
import type { AppError } from "@/lib/api/client/normalizeError";
import useSync from "@/lib/api/hooks/app/emails/useSync";
import type { SyncThrottleReason } from "@/lib/api/models/app/emails/SyncState";
import useUpdateSyncSkipFolders from "@/lib/api/hooks/app/emails/useUpdateSyncSkipFolders";
import type { SyncFolder, SyncThrottleReason } from "@/lib/api/models/app/emails/SyncState";
import buildError from "@/lib/helper/buildError";
import { cn } from "@/lib/utils";
// Sync card in the mailbox drawer: what the initial import has done, whether
@@ -35,7 +42,98 @@ function until(iso: string): string {
: d.toLocaleString([], { weekday: "short", hour: "2-digit", minute: "2-digit" });
}
export default function SyncStatusCard({ mailboxId }: { mailboxId: string }) {
// The folders a client may offer to skip: everything the worker listed
// except INBOX and the special folders, which the sync always follows.
function skippable(folders: SyncFolder[]): string[] {
return folders.filter((f) => f.folder === "inbox" && f.name.toUpperCase() !== "INBOX").map((f) => f.name);
}
// Folders the owner excluded from sync, with the ones the server lists as
// the choices. A skipped folder leaves the listing once the worker stops
// following it, so the rows are the union of both, and a name the listing
// does not have yet (a folder not yet seen, or one on a mailbox that has not
// synced) can be typed in.
function SkipFoldersSection({ mailboxId, listed, skipped }: { mailboxId: string; listed: string[]; skipped: string[] }) {
const mutation = useUpdateSyncSkipFolders(mailboxId);
const [draft, setDraft] = useState("");
const isSkipped = (name: string) => skipped.some((s) => s.toLowerCase() === name.toLowerCase());
const rows = [...listed, ...skipped.filter((s) => !listed.some((l) => l.toLowerCase() === s.toLowerCase()))];
const save = async (next: string[]) => {
try {
await mutation.mutateAsync(next);
} catch (e) {
toast.error(buildError(e as AppError));
}
};
const toggle = (name: string) =>
save(isSkipped(name) ? skipped.filter((s) => s.toLowerCase() !== name.toLowerCase()) : [...skipped, name]);
const add = () => {
const name = draft.trim();
if (!name) return;
setDraft("");
if (isSkipped(name)) return;
void save([...skipped, name]);
};
return (
<div className="mt-4">
<div className="text-[10px] uppercase tracking-[0.14em] text-slate-400 font-medium">Folders not synced</div>
<p className="mt-1 text-[11.5px] leading-relaxed text-slate-500">
Mail in a folder ticked here never reaches Warmbly, and what was already imported from it is removed. Use it
for a folder another tool fills, such as a second warmup service. Inbox, sent, drafts, spam, trash and archive
always sync.
</p>
{rows.length > 0 && (
<ul className="mt-2 -mx-2.5">
{rows.map((name) => (
<li key={name}>
<button
type="button"
disabled={mutation.isPending}
onClick={() => void toggle(name)}
className="w-full px-2.5 h-7 flex items-center gap-2 text-[12px] text-slate-700 hover:bg-slate-100 transition-colors disabled:opacity-60 rounded-md"
>
<CheckSquare checked={isSkipped(name)} />
<span className="truncate">{name}</span>
{!listed.some((l) => l.toLowerCase() === name.toLowerCase()) && (
<span className="ml-auto text-[10.5px] text-slate-400 shrink-0">skipped</span>
)}
</button>
</li>
))}
</ul>
)}
<div className="mt-2 flex items-center gap-1.5">
<TextInput
value={draft}
onChange={setDraft}
placeholder="Folder name as your mail server lists it"
disabled={mutation.isPending}
maxLength={255}
onKeyDown={(e) => {
if (e.key === "Enter") {
e.preventDefault();
add();
}
}}
className="flex-1"
/>
<button
type="button"
onClick={add}
disabled={mutation.isPending || !draft.trim()}
className="h-7 px-2.5 inline-flex items-center gap-1 rounded-md border border-slate-200 text-[12px] text-slate-700 hover:bg-slate-50 disabled:opacity-50 shrink-0"
>
<PlusIcon className="w-3 h-3" /> Skip
</button>
</div>
</div>
);
}
export default function SyncStatusCard({ mailboxId, provider }: { mailboxId: string; provider?: string }) {
const sync = useSync(mailboxId);
const state = sync.data?.state ?? null;
const policy = sync.data?.policy;
@@ -129,6 +227,14 @@ export default function SyncStatusCard({ mailboxId }: { mailboxId: string }) {
synced. Renaming one of them on your mail server clears this.
</p>
)}
{provider === "smtp_imap" && (
<SkipFoldersSection
mailboxId={mailboxId}
listed={skippable(sync.data.folders ?? [])}
skipped={sync.data.skip_folders ?? []}
/>
)}
</div>
);
}
@@ -0,0 +1,12 @@
import Request from "../../Request";
// Replaces the folders a mailbox's sync leaves alone. The list is the
// desired state, so a retry converges.
export default async function updateSync(id: string, skipFolders: string[]): Promise<{ skip_folders: string[] }> {
return await Request<{ skip_folders: string[] }>({
method: "PUT",
url: `/emails/${id}/sync`,
data: { skip_folders: skipFolders },
authorization: true,
});
}
@@ -0,0 +1,19 @@
import { useMutation, useQueryClient } from "@tanstack/react-query";
import updateSync from "@/lib/api/client/app/emails/updateSync";
import type EmailSync from "@/lib/api/models/app/emails/SyncState";
export default function useUpdateSyncSkipFolders(id: string) {
const queryClient = useQueryClient();
return useMutation({
mutationFn: (skipFolders: string[]) => updateSync(id, skipFolders),
onSuccess: (data) => {
queryClient.setQueryData<EmailSync>(["emails", id, "sync"], (prev) =>
prev ? { ...prev, skip_folders: data.skip_folders, policy: { ...prev.policy, skip_folders: data.skip_folders } } : prev,
);
// Mail already stored from a newly skipped folder is dropped on
// the server, so every inbox list is stale.
queryClient.invalidateQueries({ queryKey: ["unibox"] });
},
});
}
@@ -32,9 +32,21 @@ export interface SyncPolicy {
backfill_messages: number;
daily_messages: number;
org_daily_messages: number;
/** Folders the sync leaves alone, as the server lists them. IMAP only. */
skip_folders?: string[];
}
/** One folder the sync has seen on the server, with the canonical folder it files under. */
export interface SyncFolder {
name: string;
folder: "inbox" | "sent" | "drafts" | "spam" | "trash" | "archive" | string;
}
export default interface EmailSync {
state: SyncState | null;
policy: SyncPolicy;
/** The stored skip list; PUT /emails/:id/sync replaces it. */
skip_folders: string[];
/** What the worker last listed, INBOX first. Empty for Gmail and Outlook. */
folders: SyncFolder[];
}