Merge pull request #699 from warmbly/feature/contact-email-provider

feat: detect each contact's inbox provider from its domain's MX and SPF, show, sort and filter it across contact lists and segments, and use it for campaign ESP matching
This commit is contained in:
Matthew Meszaros
2026-09-26 13:52:02 +00:00
committed by GitHub
41 changed files with 1121 additions and 86 deletions
+1
View File
@@ -1282,6 +1282,7 @@ func main() {
formService.SetDomains(organizationRepoForHandler)
go jobs.NewFormEventsRetentionJob(formEventRepository).WireRetention(instanceSettings).Start(ctx, 12*time.Hour)
go jobs.NewFormsDomainSweep(organizationRepoForHandler).Start(ctx, time.Hour)
go jobs.NewContactMailHostSweep(contactRepoForHandler, cache, streamingPublisher).Start(ctx)
// A visibly bad import is filed on the workspace's posture. On its own
// it can only reach `watch`, which changes nothing.
if aware, ok := contactService.(contact.OrgRiskAware); ok && orgRiskService != nil {
+1
View File
@@ -92,6 +92,7 @@ Fields are named by their JSON key, with nested fields as a dotted path (`inner.
| `invalid_lead_status` | `POST /contacts/search` or `POST /contacts/export` was given a `lead_status` that is not one of the documented values |
| `invalid_sort_by` | `POST /contacts/search`, `POST /contacts/export` or a bulk action's `all` selection was given a `sort_by` of the form `custom:<key>` whose key could never be a custom-field name (letters, numbers, underscores, spaces or dashes) |
| `invalid_column`, `duplicate_column`, `too_many_columns`, `invalid_sort` | `PUT /me/views/:view` was given a column id that view cannot render, the same column twice, more than 64 columns, or a sort that names neither a sortable contact column nor a well-formed `custom:<key>` |
| `invalid_mail_host` | `POST /contacts/search`, `POST /contacts/export` or a bulk action's `all` selection was given a `mail_hosts` entry that is not a documented provider value |
| `invalid_engagement` | `POST /contacts/search` or `POST /contacts/export` was given an `engagement` that is not one of the documented values |
| `lead_filter_requires_campaign` | `lead_status` or `engagement` was set without exactly one `campaign_ids` entry; both filters describe a contact inside one campaign |
| `unknown_verification_status` | A contact's `verification_status` is not a value any known verification service writes |
+15 -4
View File
@@ -33,6 +33,7 @@ Every field is optional; an empty body matches all contacts in the organization.
| `category_ids` | string[] | No | Contact must have ALL of these categories. |
| `segment_ids` | string[] | No | Contact must be a member of ALL of these segments (conditions plus manual overrides). An id that is not a valid UUID is rejected with `400`; an unknown segment matches nothing. |
| `verification_status` | string | No | Filter by verification verdict: `valid`, `risky`, `invalid`, or `unknown`. |
| `mail_hosts` | string[] | No | Contacts whose inbox is hosted by any of these `mail_host` values (see [Email provider](#email-provider)). `""` matches contacts with no known provider: not checked yet, or a domain with no mail server. An unknown value is rejected with `invalid_mail_host`. |
| `min_campaigns` | integer | No | Minimum number of associated campaigns. |
| `max_campaigns` | integer | No | Maximum number of associated campaigns. |
| `subscribed` | boolean | No | Filter by subscription status. |
@@ -40,7 +41,7 @@ Every field is optional; an empty body matches all contacts in the organization.
| `created_before` | string (RFC 3339) | No | Created on or before this time. |
| `updated_after` | string (RFC 3339) | No | Updated on or after this time. |
| `updated_before` | string (RFC 3339) | No | Updated on or before this time. |
| `sort_by` | string | No | Sort column: `created_at` (the default), `updated_at`, `first_name`, `last_name`, `email`, `company`, `phone`, or `campaign_count`. `custom:<key>` sorts on a custom field, as text, with contacts that lack the field (or have it blank) grouped last when ascending and first when descending. A custom-field name that could never exist is rejected with `invalid_sort_by`; any other unknown value falls back to `created_at`. |
| `sort_by` | string | No | Sort column: `created_at` (the default), `updated_at`, `first_name`, `last_name`, `email`, `company`, `phone`, `campaign_count`, or `mail_host`. `mail_host` sorts by the provider value, with contacts that have no known provider grouped last when ascending and first when descending. `custom:<key>` sorts on a custom field, as text, with contacts that lack the field (or have it blank) grouped last when ascending and first when descending. A custom-field name that could never exist is rejected with `invalid_sort_by`; any other unknown value falls back to `created_at`. |
| `reverse` | boolean | No | Ascending when true. The default is descending. |
```json
@@ -79,6 +80,7 @@ Returns a `data` array of contacts plus a `pagination` envelope. Paginate by pas
"verification_provider": "builtin",
"verification_checked_at": "2026-06-10T11:58:00Z",
"is_catch_all": false,
"mail_host": "gmail",
"esp_provider": "gmail",
"updated_at": "2026-06-10T12:00:00Z",
"created_at": "2026-05-01T09:30:00Z"
@@ -94,9 +96,15 @@ Returns a `data` array of contacts plus a `pagination` envelope. Paginate by pas
Every contact carries its address verification: `verification_status` (`valid`, `risky`, `invalid`, or `unknown`), `verification_sub_status` (`catch_all`, `disposable`, `role`, `spamtrap`, `mailbox_full`, `no_mx`, `syntax`, `undisclosed`, or empty), `verification_source` (`probe` for the built-in check, `provider` for a connected verification service, `imported` for a verdict that came with the contact, `manual` for one a member set, empty when never checked), `verification_provider` (who produced it), `verification_reason`, `verification_checked_at`, `verification_requested_at` (set while a re-check a member asked for is waiting to run; the verdict stands until it lands), and `verification_confidence` (0 to 100, scored from the check plus what real mail to the address showed; see [what real mail teaches the check](/guides/deliverability/#what-real-mail-teaches-the-check)). Campaigns never send to `invalid`, and send to `risky` only when their `risky_emails` setting is on.
### Email provider
Every contact carries `mail_host`, who hosts the inbox the address belongs to, and `esp_provider`, that host's family. Warmbly reads them from the domain's DNS in the background, usually within a minute of the contact being added or its address changing: the MX records first, then the SPF record when a filtering gateway such as Proofpoint or Mimecast sits in front of the real host. Consumer domains like `gmail.com` are known without a lookup. Each domain is looked up once however many contacts share it, and nothing about the address is sent to a third party.
`mail_host` is one of `google_workspace`, `gmail`, `microsoft365`, `outlook`, `zoho`, `yahoo`, `aol`, `icloud`, `fastmail`, `godaddy`, `namecheap`, `ionos`, `hostinger`, `ovh`, `migadu`, `purelymail`, `rackspace`, `yandex`, `gmx`, `proton`, or `other` (the domain receives mail, on a host Warmbly does not name), and is empty until the check has run or when the domain has no mail server. `esp_provider` is `gmail` for either Google product, `outlook` for either Microsoft one, `other` for the rest, and empty with `mail_host`. Campaign [ESP matching](/guides/campaigns/) pairs senders and recipients by `esp_provider`. A domain whose provider could not be read is checked again a week later; one whose lookup failed is retried within the hour.
When the search filters by exactly one campaign, each contact additionally carries a `campaign_lead` object with its processing state inside that campaign (`status`, `sent`, `opened`, `machine_opened`, `clicked`, `replied`, `bounced`, `current_step`, `sender`, `last_activity_at`, `hold` when held, and `failure_reason` when failed). `sender` is the mailbox address the lead's whole sequence sends from, fixed when its first email went out and absent until then. `opened` counts steps opened by a person; steps fetched automatically by a mail client (Apple Mail Privacy Protection and similar) are in `machine_opened` instead, matching the machine opens the analytics summary reports. The `status` derivation, highest priority first, is `unsubscribed` (not subscribed), then `bounced`, `replied`, `failed` (a step could not be sent after every retry; `failure_reason` carries the sending worker's reason), `completed` (every email step sent, no reply), `paused` (the lead's flow is held, by an out-of-office auto-reply or by hand; the `hold` object carries `since`, `until`, `reason` and `source`), `active` (some steps sent, more to send), `undeliverable` (pre-send verification refused the address, so the campaign skips the lead and never sends to it), and `pending` (queued, nothing sent). A step counts as sent only once the sending worker has delivered it to the mailbox provider; a send the worker could not complete is retried on the campaign's next pass and never shows as sent. The `lead_status` filter narrows to one of these buckets.
When the search filters by exactly one campaign, the first page (no `cursor`) also includes a `lead_counts` object: per-status lead totals for that campaign, independent of the `lead_status` and `engagement` filters so every scope's total is available at once. The status buckets include `paused`. Alongside them it carries engagement totals that match the `engagement` filter: `contacted` (leads sent at least one step), `opened` (a human open on any step), `clicked`, and `replied_any` (a reply on any step, whatever the derived status).
When the search filters by exactly one campaign, the first page (no `cursor`) also includes a `lead_counts` object: per-status lead totals for that campaign, independent of the `lead_status` and `engagement` filters so every scope's total is available at once. The status buckets include `paused`. Alongside them it carries engagement totals that match the `engagement` filter: `contacted` (leads sent at least one step), `opened` (a human open on any step), `clicked`, and `replied_any` (a reply on any step, whatever the derived status). `providers` splits the campaign's leads by [email provider](#email-provider) family, the grouping ESP matching uses: `gmail`, `outlook`, `other` (including checked domains with no known provider, which ESP matching treats the same way), and `undetected` for leads whose provider has not been read yet.
```json
{
@@ -192,6 +200,7 @@ Returns the created contacts as a bare JSON array (same contact shape as search)
"campaigns": [],
"categories": [{ "id": "6f1c...", "title": "VIP", "color": "#0ea5e9" }],
"verification_status": "unknown",
"mail_host": "",
"esp_provider": "",
"updated_at": "2026-06-11T10:00:00Z",
"created_at": "2026-06-11T10:00:00Z"
@@ -532,6 +541,7 @@ Auth: **Scope** `READ_CONTACTS` · **Org permission** `view_contacts`
"campaigns": [],
"categories": [],
"verification_status": "valid",
"mail_host": "gmail",
"esp_provider": "gmail",
"updated_at": "2026-06-10T12:00:00Z",
"created_at": "2026-05-01T09:30:00Z"
@@ -566,6 +576,7 @@ Auth: **Scope** `READ_CONTACTS` · **Org permission** `view_contacts`
"campaigns": [{ "id": "c1...", "name": "Q3 Outbound" }],
"categories": [{ "id": "6f1c...", "title": "VIP", "color": "#0ea5e9" }],
"verification_status": "valid",
"mail_host": "gmail",
"esp_provider": "gmail",
"updated_at": "2026-06-10T12:00:00Z",
"created_at": "2026-05-01T09:30:00Z",
@@ -603,7 +614,7 @@ When the contact is suppressed, `suppression` is an object: `{ "id", "kind", "va
Partially updates a single contact. Only the fields present are changed. Campaign and category lists can be set wholesale or adjusted with diff-style add/remove.
`email` replaces the contact's address. It is stored lowercased, a display name (`Dana Reyes <dana@acme.com>`) is reduced to the address inside it, and anything that is not an address answers `400`. The address has to be free: one another contact already holds answers `409` with `code` `contact_email_taken` rather than merging the two. A changed address drops the contact's verification verdict back to `unknown`, clears the delivery evidence behind it and forgets the cached `esp_provider`, because all three belonged to the old mailbox; the next verification pass checks the new address and the recipient provider is derived again from the new domain. Steps already sent went to the old address and keep their history.
`email` replaces the contact's address. It is stored lowercased, a display name (`Dana Reyes <dana@acme.com>`) is reduced to the address inside it, and anything that is not an address answers `400`. The address has to be free: one another contact already holds answers `409` with `code` `contact_email_taken` rather than merging the two. A changed address drops the contact's verification verdict back to `unknown`, clears the delivery evidence behind it and forgets its `mail_host` and `esp_provider`, because all of them belonged to the old mailbox; the next verification pass checks the new address and the provider is read again from the new domain. Steps already sent went to the old address and keep their history.
Auth: **Scope** `WRITE_CONTACTS` · **Org permission** `manage_contacts`
@@ -1115,7 +1126,7 @@ Each condition names a `field`, an `operator`, and either a `value` (scalar oper
| Kind | Fields | Operators | Value |
| --- | --- | --- | --- |
| text | `first_name`, `last_name`, `email`, `email_domain`, `phone`, `company`, `custom.*` | `equals`, `not_equals`, `contains`, `not_contains`, `starts_with`, `ends_with`, `is_empty`, `is_not_empty` | `value` string; comparisons ignore case |
| enum | `source`, `verification_status`, `esp_provider` | `in`, `not_in` | `values`, drawn from the field's `options` |
| enum | `source`, `verification_status`, `mail_host`, `esp_provider` | `in`, `not_in` | `values`, drawn from the field's `options`; `option_labels` names them for display |
| bool | `subscribed`, `suppressed`, `is_catch_all` | `is_true`, `is_false` | none |
| date | `created_at`, `updated_at`, `last_sent_at`, `last_opened_at`, `last_clicked_at`, `last_replied_at` | `within_days`, `not_within_days` (`value` is a day count, 1 to 3650); `before`, `after` (`value` is `YYYY-MM-DD` or RFC 3339); `is_empty`, `is_not_empty` | see operators |
| number | `campaign_count`, `emails_sent`, `emails_opened`, `emails_clicked`, `emails_replied`, `emails_bounced` | `equals`, `not_equals`, `gt`, `gte`, `lt`, `lte` | `value`, a whole number |
+1 -1
View File
@@ -61,7 +61,7 @@ Whichever mode you pick, it decides which mailbox **starts** a lead. Every mailb
The Leads tab shows each lead's mailbox in its **Sender** column, and a contact's Activity tab shows it beside their progress. A lead has none until its first email goes out.
**ESP matching** optionally aligns sender and recipient providers (Gmail to Gmail). **Prefer same** falls back to any mailbox when no same-provider one is free; **Strict same** makes the contact wait instead. A contact left waiting this way waits alone: the campaign moves on to the next lead in the queue rather than stopping, and comes back to them when a matching mailbox is free. An unknown recipient provider never blocks a send. Like rotation, it applies to a lead's first email: a follow-up goes out from the mailbox the contact already knows.
**ESP matching** optionally aligns sender and recipient providers (Google to Google, Microsoft to Microsoft). The recipient's provider is read from their domain's mail records, so a lead on a company domain hosted by Google Workspace or Microsoft 365 is matched too, not only `gmail.com` and `outlook.com` addresses; the contacts list shows it in the **Email provider** column. A mailbox connected with an app password to Google Workspace or Microsoft 365 counts as that provider. Under **Settings** > **ESP matching**, the coverage panel shows how many of the campaign's leads are on Google, Microsoft and other providers, and which of your mailboxes serve each. **Prefer same** falls back to any mailbox when no same-provider one is free; **Strict same** makes the contact wait instead. A contact left waiting this way waits alone: the campaign moves on to the next lead in the queue rather than stopping, and comes back to them when a matching mailbox is free. An unknown recipient provider never blocks a send. Like rotation, it applies to a lead's first email: a follow-up goes out from the mailbox the contact already knows.
### When a lead's mailbox is busy, or gone
+8 -2
View File
@@ -89,7 +89,7 @@ See [Personalization & expressions](/guides/expressions/) for the full templatin
## Filtering the list
A filter bar sits above the contact list. **Category**, **Segment**, **Status** and **Campaign** are always there; open one, tick values, and the list updates immediately with the matching count next to the bar. **Add filter** adds a custom-field condition (field, contains/is/starts with/ends with, value), a date-added or last-updated range, a number-of-campaigns range, the address verification verdict and, on a campaign's Leads tab, lead status and engagement. Each active filter is a pill you can reopen to change or remove with its cross; **Clear** drops them all, and **Save as segment** turns the current set into a [segment](/guides/segments/). Free-text search, sort and the column chooser stay in the toolbar.
A filter bar sits above the contact list. **Category**, **Segment**, **Status** and **Campaign** are always there; open one, tick values, and the list updates immediately with the matching count next to the bar. **Add filter** adds a custom-field condition (field, contains/is/starts with/ends with, value), a date-added or last-updated range, a number-of-campaigns range, the address verification verdict, the [email provider](#email-provider) and, on a campaign's Leads tab, lead status and engagement. Each active filter is a pill you can reopen to change or remove with its cross; **Clear** drops them all, and **Save as segment** turns the current set into a [segment](/guides/segments/). Free-text search, sort and the column chooser stay in the toolbar.
Free-text search matches first name, last name, email, company and phone. Every word you type has to match one of those, so `Test Demo` finds the contact whose first name is Test and last name is Demo, and `Demo Acme` finds everyone named Demo at Acme. Words can be in any order, and only the first six count.
@@ -101,10 +101,16 @@ The layout is yours: it is saved to your account for this workspace, so it follo
## Sorting
Click a column header to sort by it, and click it again to flip the direction. Text columns (name, company, phone, custom fields) start ascending; dates and counts start with the newest or largest first. The **Sort** menu in the toolbar reaches the same orderings, including columns that are hidden or that a phone screen has no room for, and every custom field in the workspace.
Click a column header to sort by it, and click it again to flip the direction. Text columns (name, company, email provider, phone, custom fields) start ascending; dates and counts start with the newest or largest first. The **Sort** menu in the toolbar reaches the same orderings, including columns that are hidden or that a phone screen has no room for, and every custom field in the workspace.
A sort on a custom field orders the values as text. Contacts that do not have the field, or have it blank, are grouped at the end when ascending and at the start when descending. The sort is saved with your layout, so the list opens the way you left it.
## Email provider
The **Email provider** column shows who hosts each contact's inbox: Google Workspace, Gmail, Microsoft 365, Outlook.com, Yahoo Mail, iCloud Mail, Zoho Mail and the other hosts Warmbly recognises, with the provider's logo. It works for company domains too: `dana@acme.com` reads as Google Workspace when Acme's mail is hosted by Google. Warmbly reads it from the domain's mail records in the background, usually within a minute of a contact being added, and fills the list in live. A dash means the check has not reached the contact yet, or the domain has no mail server at all.
It is part of the standard layout on the contacts page (if you have already arranged your own columns, add it from **Columns**) and one tick away in the column chooser on a campaign's Leads tab. From there you can sort by it, filter with **Add filter** > **Email provider** (including **Unknown or not checked yet**), select every match and act on them in bulk (add them to a campaign, tag them, export them), or save the filter as a segment. Campaign [ESP matching](/guides/campaigns/) uses the same data to pair each lead with a same-provider mailbox.
## Selecting rows
Tick a row's checkbox to select it, or the one in the table header to select every row loaded so far. Contact lists load in pages as you scroll, so the header checkbox on its own only ever covers what is on screen.
+3 -1
View File
@@ -17,7 +17,7 @@ Give the segment a name and a color, then add conditions. Each condition is a fi
|-------|--------|-----------|
| Contact | First name, last name, email, email domain, phone, and every custom field | is, is not, contains, does not contain, starts with, ends with, is empty, is not empty |
| Contact | Subscribed, on the suppression list, catch-all domain | is yes, is no |
| Contact | Source, verification status, email provider | is any of, is none of |
| Contact | Source, verification status, email provider, email provider family | is any of, is none of |
| Contact | Created, updated | in the last N days, not in the last N days, after, before |
| Contact | Category | has any of, has none of, has none, has any |
| Company | Company name | the text operators above |
@@ -27,6 +27,8 @@ Give the segment a name and a color, then add conditions. Each condition is a fi
| Email engagement | Last email sent, last open, last click, last reply | in the last N days, not in the last N days, after, before, never, ever |
| Segments | In segment | is in any of, is in none of |
**Email provider** is who hosts the contact's inbox (Google Workspace, Microsoft 365, Yahoo Mail, and so on), read from the domain's mail records; **email provider family** groups those into Google, Microsoft and other, the grouping [ESP matching](/guides/campaigns/) uses. See [email provider](/guides/contacts-crm/#email-provider).
Engagement counts add up every campaign the contact has been in. Opens count human opens only; automated fetches are ignored, the same way the campaign analytics report them. Text comparisons ignore case.
A segment can be built on other segments (**In segment**), up to five levels deep. A segment cannot reference itself or form a loop, and a segment that others depend on cannot be deleted until those references are removed.
+14 -2
View File
@@ -29293,6 +29293,7 @@
"verification_status",
"verification_reason",
"is_catch_all",
"mail_host",
"esp_provider",
"updated_at",
"created_at"
@@ -29386,9 +29387,13 @@
"format": "date-time",
"description": "Set while a re-check a member asked for is waiting to run; the verdict stands until it lands."
},
"mail_host": {
"type": "string",
"description": "Who hosts the contact's inbox, read from the domain's MX (and SPF behind a filtering gateway) by a background check: google_workspace, gmail, microsoft365, outlook, zoho, yahoo, aol, icloud, fastmail, godaddy, namecheap, ionos, hostinger, ovh, migadu, purelymail, rackspace, yandex, gmx, proton, or other. Empty until checked, or when the domain has no mail server."
},
"esp_provider": {
"type": "string",
"description": "Recipient ESP derived from the domain: '' | gmail | outlook | other."
"description": "mail_host's family, the one campaign ESP matching uses: gmail (either Google product), outlook (either Microsoft one), other, or empty with mail_host."
},
"esp_resolved_at": {
"type": [
@@ -29603,6 +29608,13 @@
"type": "integer",
"description": "Maximum number of associated campaigns."
},
"mail_hosts": {
"type": "array",
"items": {
"type": "string"
},
"description": "Contacts whose mail_host is any of these; \"\" matches contacts with no known provider (not checked yet, or a domain with no mail server). An unknown value is rejected with 400 invalid_mail_host."
},
"subscribed": {
"type": "boolean"
},
@@ -29624,7 +29636,7 @@
},
"sort_by": {
"type": "string",
"description": "Sort column: created_at (the default), updated_at, first_name, last_name, email, company, phone, or campaign_count. custom:<key> sorts on a custom field as text, with contacts lacking the field grouped last when ascending and first when descending; a custom-field name that could never exist is rejected with 400 invalid_sort_by. Any other unknown value falls back to created_at."
"description": "Sort column: created_at (the default), updated_at, first_name, last_name, email, company, phone, campaign_count, or mail_host (contacts with no known provider grouped last when ascending and first when descending). custom:<key> sorts on a custom field as text, with contacts lacking the field grouped last when ascending and first when descending; a custom-field name that could never exist is rejected with 400 invalid_sort_by. Any other unknown value falls back to created_at."
},
"reverse": {
"type": "boolean",
+3
View File
@@ -76,6 +76,9 @@ func (s *contactService) Export(
if err := validateLeadFilters(*searchFilters); err != nil {
return "", "", 0, err
}
if err := validateMailHosts(*searchFilters); err != nil {
return "", "", 0, err
}
}
rows, xerr := s.contactRepository.ExportAll(ctx, orgID, searchFilters, contactIDs, models.MaxContactExportRows)
+16
View File
@@ -3,6 +3,7 @@ package contact
import (
"context"
"fmt"
"strconv"
"strings"
"time"
@@ -10,6 +11,7 @@ import (
"github.com/warmbly/warmbly/internal/config"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/pkg/mailhost"
"github.com/warmbly/warmbly/internal/utils"
"github.com/warmbly/warmbly/internal/utils/paging"
"github.com/warmbly/warmbly/internal/utils/validate"
@@ -118,6 +120,9 @@ func (s *contactService) Search(ctx context.Context, orgID, cursor, category, li
if err := validateSort(filters); err != nil {
return nil, err
}
if err := validateMailHosts(filters); err != nil {
return nil, err
}
return s.contactRepository.Search(ctx, orgID, categoryId, cursorPos, filters, limitN)
}
@@ -136,6 +141,17 @@ func validateSort(filters models.SearchContacts) *errx.Error {
return nil
}
// validateMailHosts refuses a provider filter naming no known host; "" is
// allowed and matches contacts the provider sweep has not reached yet.
func validateMailHosts(filters models.SearchContacts) *errx.Error {
for _, h := range filters.MailHosts {
if !mailhost.Valid(h) {
return errx.NewWithIdentifier(errx.BadRequest, "invalid_mail_host", "invalid mail_hosts: unknown provider "+strconv.Quote(h))
}
}
return nil
}
// validateLeadFilters gates the single-campaign Leads-view filters: an unknown
// value or the wrong campaign cardinality is a client contract error (400 with
// a stable code), not a silently-ignored no-op.
+3
View File
@@ -30,6 +30,9 @@ func (s *contactService) ResolveSelection(ctx context.Context, orgID uuid.UUID,
if xerr := validateSort(*sel.Filters); xerr != nil {
return nil, xerr
}
if xerr := validateMailHosts(*sel.Filters); xerr != nil {
return nil, xerr
}
ids, xerr := s.contactRepository.SearchIDs(ctx, orgID.String(), *sel.Filters, models.MaxContactBulkSelection)
if xerr != nil {
+18
View File
@@ -340,6 +340,24 @@ const (
UniboxLimitMax = 100
UniboxLimitDefault = 50
// ContactMailHostBatchSize is how many contacts one provider sweep pass
// reads; lookups are per distinct domain, so a pass costs far fewer.
ContactMailHostBatchSize = 2000
// ContactMailHostIntervalSeconds is how often the provider sweep passes. A
// full batch runs again immediately, so an import drains without waiting.
ContactMailHostIntervalSeconds = 60
// ContactMailHostConcurrency bounds parallel domain lookups in one pass.
ContactMailHostConcurrency = 16
// ContactMailHostRecheckDays is how long a domain with no known host waits
// before it is looked up again.
ContactMailHostRecheckDays = 7
// ContactMailHostRetryMinutes is how soon a lookup that failed transiently
// is tried again.
ContactMailHostRetryMinutes = 60
// ContactMailHostCacheHours is how long a domain's host is cached across
// sweeps and backend replicas.
ContactMailHostCacheHours = 24
// VerificationRecheckDays is how long a verification verdict is trusted
// before the address is checked again. Mailboxes get created and closed;
// a verdict from last quarter is a guess.
@@ -0,0 +1,3 @@
ALTER TABLE contacts
DROP CONSTRAINT IF EXISTS contacts_mail_host_check,
DROP COLUMN IF EXISTS mail_host;
@@ -0,0 +1,6 @@
-- Who hosts a contact's inbox (google_workspace, microsoft365, yahoo, ...), read
-- from the domain's MX by a backend sweep. esp_provider stays the coarse family
-- ESP matching reads. NOT VALID: every existing row holds the default.
ALTER TABLE contacts
ADD COLUMN mail_host text NOT NULL DEFAULT '',
ADD CONSTRAINT contacts_mail_host_check CHECK (mail_host ~ '^[a-z0-9_]{0,32}$') NOT VALID;
@@ -0,0 +1 @@
DROP INDEX CONCURRENTLY IF EXISTS idx_contacts_mail_host_pending;
@@ -0,0 +1,5 @@
-- Alone in its file for CONCURRENTLY. The provider sweep reads contacts whose
-- host is not known yet, oldest check first.
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_contacts_mail_host_pending
ON contacts (esp_resolved_at NULLS FIRST)
WHERE mail_host = '';
+24
View File
@@ -164,6 +164,8 @@ type ContactEvent struct {
// BulkOperationEvent for bulk operation signals
type BulkOperationEvent struct {
BaseEvent
// OrgID routes the event to every member of the organization.
OrgID string `json:"org_id,omitempty"`
OperationID string `json:"operation_id"`
OperationType string `json:"operation_type"`
EntityType string `json:"entity_type"`
@@ -416,6 +418,28 @@ func (p *StreamingPublisher) PublishContactsReload(ctx context.Context, userID,
}
}
// PublishOrgContactsReload tells every member of an organization to reload
// its contact lists, for a change no single member made.
func (p *StreamingPublisher) PublishOrgContactsReload(ctx context.Context, orgID, operationID string) {
if p.client == nil {
return
}
event := &BulkOperationEvent{
BaseEvent: BaseEvent{
EventType: EventContactsReload,
Timestamp: time.Now(),
},
OrgID: orgID,
OperationID: operationID,
EntityType: "contacts",
}
attrs := map[string]string{
"org_id": orgID,
"event_type": string(EventContactsReload),
}
_ = p.client.Publish(ctx, TopicBulkOps, event, attrs)
}
// PublishBulkProgress sends bulk operation progress update
func (p *StreamingPublisher) PublishBulkProgress(ctx context.Context, event *BulkOperationEvent) {
if p.client == nil {
+182
View File
@@ -0,0 +1,182 @@
package jobs
import (
"context"
"sync"
"time"
"github.com/google/uuid"
"github.com/redis/go-redis/v9"
"github.com/rs/zerolog/log"
"github.com/warmbly/warmbly/internal/config"
"github.com/warmbly/warmbly/internal/infrastructure/cache"
"github.com/warmbly/warmbly/internal/jobrun"
"github.com/warmbly/warmbly/internal/pkg/mailhost"
"github.com/warmbly/warmbly/internal/repository"
)
const (
// contactMailHostMaxPasses bounds one run; the next tick picks up the rest.
contactMailHostMaxPasses = 50
contactMailHostLease = "lock:contactmailhost"
contactMailHostLeaseTTL = 10 * time.Minute
// contactMailHostRunBudget ends a run while the lease still has room for
// the lookups already in flight, so two replicas never sweep at once.
contactMailHostRunBudget = contactMailHostLeaseTTL - 2*time.Minute
)
// releaseContactMailHostLease deletes the lease only while this run still holds it.
var releaseContactMailHostLease = redis.NewScript(`if redis.call("GET", KEYS[1]) == ARGV[1] then return redis.call("DEL", KEYS[1]) end return 0`)
// ContactMailHostStore is the slice of the contact repository the sweep uses.
type ContactMailHostStore interface {
ListMailHostPending(ctx context.Context, limit int) ([]repository.ContactMailHostPending, error)
SetContactMailHosts(ctx context.Context, results []repository.ContactMailHostResult) ([]uuid.UUID, error)
}
// ContactsReloader tells an organization's members their contact lists changed.
type ContactsReloader interface {
PublishOrgContactsReload(ctx context.Context, orgID, operationID string)
}
// ContactMailHostSweep works out who hosts each contact's inbox from the
// domain's DNS, one lookup per distinct domain, and stores it on the contact
// with the family ESP matching reads. Control plane only: it dials DNS for
// user-supplied domains.
type ContactMailHostSweep struct {
contacts ContactMailHostStore
cache *cache.Cache
resolver mailhost.RecipientResolver
reloader ContactsReloader
}
func NewContactMailHostSweep(contacts ContactMailHostStore, c *cache.Cache, reloader ContactsReloader) *ContactMailHostSweep {
return &ContactMailHostSweep{contacts: contacts, cache: c, reloader: reloader}
}
type cachedMailHost struct {
Host string `json:"host"`
}
func (j *ContactMailHostSweep) Run(ctx context.Context) error {
// One replica sweeps at a time; a Redis outage fails open.
if j.cache != nil {
token := uuid.NewString()
if got, err := j.cache.SetNX(ctx, contactMailHostLease, token, contactMailHostLeaseTTL).Result(); err == nil {
if !got {
return nil
}
defer releaseContactMailHostLease.Run(context.WithoutCancel(ctx), j.cache, []string{contactMailHostLease}, token)
}
}
deadline := time.Now().Add(contactMailHostRunBudget)
for range contactMailHostMaxPasses {
pending, err := j.contacts.ListMailHostPending(ctx, config.ContactMailHostBatchSize)
if err != nil {
return err
}
if len(pending) == 0 {
return nil
}
hosts, skipped := j.resolveDomains(ctx, pending, deadline)
results := make([]repository.ContactMailHostResult, 0, len(pending))
for _, p := range pending {
d := mailhost.NormalizeDomain(p.Email)
if skipped[d] {
continue
}
r, ok := hosts[d]
res := repository.ContactMailHostResult{ID: p.ID, Email: p.Email, Transient: !ok}
if ok {
res.MailHost, res.ESP = string(r), mailhost.ESPFamily(r)
}
results = append(results, res)
}
orgs, err := j.contacts.SetContactMailHosts(ctx, results)
if err != nil {
return err
}
if j.reloader != nil {
for _, org := range orgs {
j.reloader.PublishOrgContactsReload(ctx, org.String(), "contacts:mail_host")
}
}
if len(pending) < config.ContactMailHostBatchSize || len(skipped) > 0 || ctx.Err() != nil {
return ctx.Err()
}
}
return nil
}
// resolveDomains answers each distinct domain once. A domain missing from the
// result failed transiently; one in skipped was not asked before the deadline.
func (j *ContactMailHostSweep) resolveDomains(ctx context.Context, pending []repository.ContactMailHostPending, deadline time.Time) (map[string]mailhost.Host, map[string]bool) {
out := map[string]mailhost.Host{}
skipped := map[string]bool{}
var todo []string
for _, p := range pending {
d := mailhost.NormalizeDomain(p.Email)
if _, seen := out[d]; seen {
continue
}
out[d] = mailhost.Unknown
if d == "" {
continue
}
if h := mailhost.KnownDomain(d); h != mailhost.Unknown {
out[d] = h
continue
}
var hit cachedMailHost
if j.cache != nil && j.cache.GetJSON(ctx, contactMailHostKey(d), &hit) == nil {
out[d] = mailhost.Host(hit.Host)
continue
}
todo = append(todo, d)
}
var mu sync.Mutex
var wg sync.WaitGroup
sem := make(chan struct{}, config.ContactMailHostConcurrency)
for _, d := range todo {
sem <- struct{}{}
if time.Now().After(deadline) {
<-sem
skipped[d] = true
continue
}
wg.Add(1)
go func(d string) {
defer wg.Done()
defer func() { <-sem }()
h, ok := mailhost.Recipient(ctx, j.resolver, d)
if ok && j.cache != nil {
ttl := time.Duration(config.ContactMailHostCacheHours) * time.Hour
if err := j.cache.SetJSON(ctx, contactMailHostKey(d), cachedMailHost{Host: string(h)}, ttl); err != nil {
log.Debug().Err(err).Msg("contact mail host: cache write failed")
}
}
mu.Lock()
defer mu.Unlock()
if ok {
out[d] = h
} else {
delete(out, d)
}
}(d)
}
wg.Wait()
for d := range skipped {
delete(out, d)
}
return out, skipped
}
func contactMailHostKey(domain string) string { return "contactmailhost:" + domain }
// Start runs the sweep on boot and then on the interval until ctx ends.
func (j *ContactMailHostSweep) Start(ctx context.Context) {
interval := time.Duration(config.ContactMailHostIntervalSeconds) * time.Second
jobrun.Loop(ctx, "contact_mail_host", interval, true, j.Run)
}
+118
View File
@@ -0,0 +1,118 @@
package jobs
import (
"context"
"net"
"sync"
"testing"
"time"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/repository"
)
type mailHostDNS struct {
mx map[string][]string
mu sync.Mutex
calls map[string]int
}
func (d *mailHostDNS) LookupMX(_ context.Context, name string) ([]*net.MX, error) {
d.mu.Lock()
d.calls[name]++
d.mu.Unlock()
if name == "flaky.example" {
return nil, &net.DNSError{Err: "server misbehaving", Name: name, IsTemporary: true}
}
hosts, ok := d.mx[name]
if !ok {
return nil, &net.DNSError{Err: "no such host", Name: name, IsNotFound: true}
}
out := make([]*net.MX, len(hosts))
for i, h := range hosts {
out[i] = &net.MX{Host: h + ".", Pref: 10}
}
return out, nil
}
func (d *mailHostDNS) LookupTXT(context.Context, string) ([]string, error) {
return nil, &net.DNSError{Err: "no such host", IsNotFound: true}
}
type mailHostStore struct {
pending []repository.ContactMailHostPending
got []repository.ContactMailHostResult
}
func (s *mailHostStore) ListMailHostPending(context.Context, int) ([]repository.ContactMailHostPending, error) {
p := s.pending
s.pending = nil
return p, nil
}
func (s *mailHostStore) SetContactMailHosts(_ context.Context, r []repository.ContactMailHostResult) ([]uuid.UUID, error) {
s.got = append(s.got, r...)
return nil, nil
}
func TestContactMailHostSweep(t *testing.T) {
store := &mailHostStore{pending: []repository.ContactMailHostPending{
{ID: uuid.New(), Email: "dana@gmail.com"},
{ID: uuid.New(), Email: "a@acme.example"},
{ID: uuid.New(), Email: "b@acme.example"},
{ID: uuid.New(), Email: "c@flaky.example"},
{ID: uuid.New(), Email: "d@nomail.example"},
}}
dns := &mailHostDNS{mx: map[string][]string{"acme.example": {"aspmx.l.google.com"}}, calls: map[string]int{}}
j := NewContactMailHostSweep(store, nil, nil)
j.resolver = dns
if err := j.Run(context.Background()); err != nil {
t.Fatal(err)
}
want := []struct {
host, esp string
transient bool
}{
{"gmail", "gmail", false},
{"google_workspace", "gmail", false},
{"google_workspace", "gmail", false},
{"", "", true},
{"", "", false},
}
if len(store.got) != len(want) {
t.Fatalf("got %d results, want %d", len(store.got), len(want))
}
for i, w := range want {
g := store.got[i]
if g.MailHost != w.host || g.ESP != w.esp || g.Transient != w.transient {
t.Errorf("%s: got host=%q esp=%q transient=%v, want %q %q %v", g.Email, g.MailHost, g.ESP, g.Transient, w.host, w.esp, w.transient)
}
}
if dns.calls["acme.example"] != 1 {
t.Errorf("acme.example looked up %d times, want once per pass", dns.calls["acme.example"])
}
if dns.calls["gmail.com"] != 0 {
t.Errorf("a consumer domain dialled DNS")
}
}
func TestContactMailHostSweepSkipsPastDeadline(t *testing.T) {
dns := &mailHostDNS{mx: map[string][]string{"acme.example": {"aspmx.l.google.com"}}, calls: map[string]int{}}
j := NewContactMailHostSweep(&mailHostStore{}, nil, nil)
j.resolver = dns
pending := []repository.ContactMailHostPending{
{ID: uuid.New(), Email: "a@acme.example"},
{ID: uuid.New(), Email: "dana@gmail.com"},
}
hosts, skipped := j.resolveDomains(context.Background(), pending, time.Now().Add(-time.Second))
if !skipped["acme.example"] || dns.calls["acme.example"] != 0 {
t.Fatalf("a lookup past the deadline ran: skipped=%v calls=%v", skipped, dns.calls)
}
if _, ok := hosts["acme.example"]; ok {
t.Fatal("a skipped domain has an answer")
}
if hosts["gmail.com"] != "gmail" {
t.Fatalf("a known domain needs no lookup and is never skipped: %v", hosts)
}
}
+18 -3
View File
@@ -54,9 +54,11 @@ type Contact struct {
// to run; the verdict above stands until it lands.
VerificationRequestedAt *time.Time `json:"verification_requested_at,omitempty"`
// Recipient ESP/provider, derived in the control plane from the recipient
// domain (never an MX dial on the send hot path). '' | 'gmail' | 'outlook'
// | 'other'. Used by the campaign ESP-matching feature.
// MailHost is who hosts the contact's inbox (a mailhost.Host value), read
// from the domain's MX by the backend sweep; '' until it has run.
MailHost string `json:"mail_host"`
// ESPProvider is MailHost's family for campaign ESP matching: '' | 'gmail'
// | 'outlook' | 'other'. Resolved with it, never on the send hot path.
ESPProvider string `json:"esp_provider"`
ESPResolvedAt *time.Time `json:"esp_resolved_at,omitempty"`
@@ -235,6 +237,18 @@ type CampaignLeadCounts struct {
Opened int `json:"opened"`
Clicked int `json:"clicked"`
RepliedAny int `json:"replied_any"`
// Providers splits the leads by their inbox's family, as ESP matching sees it.
Providers CampaignLeadProviderCounts `json:"providers"`
}
// CampaignLeadProviderCounts are a campaign's leads by esp_provider family.
// Other includes checked domains with no known host, which match like other;
// Undetected counts the leads the provider check has not reached yet.
type CampaignLeadProviderCounts struct {
Google int `json:"gmail"`
Microsoft int `json:"outlook"`
Other int `json:"other"`
Undetected int `json:"undetected"`
}
// ContactsCounts are org-wide contact facet totals for the browse sidebar.
@@ -847,6 +861,7 @@ type SearchContacts struct {
MaxCampaigns *int `json:"max_campaigns"` // Maximum number of associated campaigns
Subscribed *bool `json:"subscribed"` // Filter by subscription status
VerificationStatus string `json:"verification_status"` // Filter by verification verdict: valid | risky | invalid | unknown
MailHosts []string `json:"mail_hosts"` // Contacts whose inbox host is any of these; "" matches not detected yet
CreatedAfter *time.Time `json:"created_after"` // Contacts created after this date
CreatedBefore *time.Time `json:"created_before"` // Contacts created before this date
UpdatedAfter *time.Time `json:"updated_after"` // Contacts updated after this date
+23 -1
View File
@@ -9,6 +9,7 @@ import (
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/pkg/mailhost"
"github.com/warmbly/warmbly/internal/utils"
)
@@ -109,6 +110,26 @@ type SegmentFieldSpec struct {
Kind SegmentFieldKind `json:"kind"`
// Options lists the accepted values of an enum field.
Options []string `json:"options,omitempty"`
// OptionLabels names an enum's values for display where the value alone
// is not readable; a value missing from it shows as itself.
OptionLabels map[string]string `json:"option_labels,omitempty"`
}
func mailHostOptions() []string {
hosts := mailhost.Hosts()
out := make([]string, len(hosts))
for i, h := range hosts {
out[i] = string(h)
}
return out
}
func mailHostLabels() map[string]string {
out := map[string]string{}
for _, h := range mailhost.Hosts() {
out[string(h)] = h.Label()
}
return out
}
// SegmentFieldCatalog is every non-custom field a condition may name.
@@ -123,7 +144,8 @@ var SegmentFieldCatalog = []SegmentFieldSpec{
{Field: "source", Label: "Source", Group: "Contact", Kind: SegmentFieldEnum, Options: []string{"unknown", "manual", "campaign", "import", "sheet_sync", "api", "ai_assistant", "form"}},
{Field: "verification_status", Label: "Verification status", Group: "Contact", Kind: SegmentFieldEnum, Options: []string{"valid", "risky", "invalid", "unknown"}},
{Field: "is_catch_all", Label: "Catch-all domain", Group: "Contact", Kind: SegmentFieldBool},
{Field: "esp_provider", Label: "Email provider", Group: "Contact", Kind: SegmentFieldEnum, Options: []string{"gmail", "outlook", "other"}},
{Field: "mail_host", Label: "Email provider", Group: "Contact", Kind: SegmentFieldEnum, Options: mailHostOptions(), OptionLabels: mailHostLabels()},
{Field: "esp_provider", Label: "Email provider family", Group: "Contact", Kind: SegmentFieldEnum, Options: []string{"gmail", "outlook", "other"}, OptionLabels: map[string]string{"gmail": "Google", "outlook": "Microsoft", "other": "Other"}},
{Field: "created_at", Label: "Created", Group: "Contact", Kind: SegmentFieldDate},
{Field: "updated_at", Label: "Updated", Group: "Contact", Kind: SegmentFieldDate},
{Field: "category", Label: "Category", Group: "Contact", Kind: SegmentFieldCategory},
+3 -3
View File
@@ -22,15 +22,15 @@ var KnownViews = map[string]bool{
// step with the dashboard's column registry (web/src/components/app/contacts/
// columns.tsx). A saved layout may name only these and custom fields.
var ViewBuiltinColumns = map[string][]string{
ViewContacts: {"name", "company", "phone", "status", "campaigns", "created_at", "updated_at"},
ViewCampaignLeads: {"name", "company", "phone", "progress", "opened", "clicked", "replied", "current_step", "sender", "last_activity", "created_at", "updated_at"},
ViewContacts: {"name", "company", "phone", "mail_host", "status", "campaigns", "created_at", "updated_at"},
ViewCampaignLeads: {"name", "company", "phone", "mail_host", "progress", "opened", "clicked", "replied", "current_step", "sender", "last_activity", "created_at", "updated_at"},
}
// ContactBuiltinSorts are the sort_by values the contacts search knows besides
// "custom:<key>", the same set contactSorts holds in the repository.
var ContactBuiltinSorts = map[string]bool{
"created_at": true, "updated_at": true, "first_name": true, "last_name": true,
"email": true, "company": true, "phone": true, "campaign_count": true,
"email": true, "company": true, "phone": true, "campaign_count": true, "mail_host": true,
}
// ViewPreferencesMaxColumns bounds one layout; no list here has anywhere near
+117
View File
@@ -0,0 +1,117 @@
package mailhost
import (
"context"
"net"
"strings"
"time"
)
const txtTimeout = 3 * time.Second
// RecipientResolver is the DNS surface Recipient needs; *net.Resolver satisfies it.
type RecipientResolver interface {
LookupMX(ctx context.Context, name string) ([]*net.MX, error)
LookupTXT(ctx context.Context, name string) ([]string, error)
}
// Recipient works out who hosts a recipient domain's inboxes from DNS alone.
// It fetches nothing over HTTPS and asks no third party, because the domain is
// a lead's. ok is false when a lookup failed transiently and the answer should
// be asked again later; a domain that has no mail comes back Unknown and ok.
func Recipient(ctx context.Context, r RecipientResolver, domain string) (Host, bool) {
norm := NormalizeDomain(domain)
if norm == "" {
return Unknown, true
}
if h := recipientKnown(norm); h != Unknown {
return h, true
}
if r == nil {
r = net.DefaultResolver
}
d := &Detector{resolver: mxOnly{r}}
mx, err := d.lookupMX(ctx, norm)
if err != nil {
return Unknown, notFound(err)
}
if len(mx) == 0 {
return Unknown, true
}
for _, host := range mx {
if h, _ := classify(host); h != Unknown {
return Refine(h, norm), true
}
}
// A filtering gateway or a relay fronts the MX; SPF usually still names the mailbox host.
if h := fromSPF(ctx, r, norm); h != Unknown {
return h, true
}
return Other, true
}
// KnownDomain names the host of a consumer mail domain, or of an address's
// domain, from the built-in list with no DNS; Unknown for anything else.
func KnownDomain(s string) Host {
return recipientKnown(NormalizeDomain(s))
}
// recipientKnown is knownDomain plus Microsoft 365 tenant domains, which are
// Microsoft's mail but not a consumer service, so SharedProvider leaves them out.
func recipientKnown(domain string) Host {
if h, ok := knownDomain(domain); ok {
return h
}
if strings.HasSuffix(domain, ".onmicrosoft.com") {
return Microsoft365
}
return Unknown
}
// ESPFamily is h's family as campaign ESP matching and contacts.esp_provider
// name it: "gmail", "outlook", "other", or "" when h is Unknown.
func ESPFamily(h Host) string {
switch {
case h == Unknown:
return ""
case h.Google():
return "gmail"
case h.Microsoft():
return "outlook"
}
return "other"
}
// fromSPF reads the provider out of the include: mechanisms of domain's SPF record.
func fromSPF(ctx context.Context, r RecipientResolver, domain string) Host {
ctx, cancel := context.WithTimeout(ctx, txtTimeout)
defer cancel()
txts, err := r.LookupTXT(ctx, domain)
if err != nil {
return Unknown
}
for _, t := range txts {
fields := strings.Fields(strings.ToLower(strings.TrimSpace(t)))
if len(fields) == 0 || fields[0] != "v=spf1" {
continue
}
for _, f := range fields[1:] {
f = strings.TrimLeft(f, "+~?")
inc, ok := strings.CutPrefix(f, "include:")
if !ok {
continue
}
if h, _ := classify(inc); h.Google() || h.Microsoft() || h == Zoho {
return Refine(h, domain)
}
}
}
return Unknown
}
// mxOnly adapts a RecipientResolver to lookupMX, which never asks for SRV.
type mxOnly struct{ RecipientResolver }
func (mxOnly) LookupSRV(context.Context, string, string, string) (string, []*net.SRV, error) {
return "", nil, &net.DNSError{Err: "not supported", IsNotFound: true}
}
+88
View File
@@ -0,0 +1,88 @@
package mailhost
import (
"context"
"errors"
"net"
"testing"
)
type recipientFake struct {
*fakeResolver
txt map[string][]string
}
func (f recipientFake) LookupTXT(_ context.Context, name string) ([]string, error) {
if t, ok := f.txt[name]; ok {
return t, nil
}
return nil, &net.DNSError{Err: "no such host", Name: name, IsNotFound: true}
}
func TestRecipient(t *testing.T) {
r := recipientFake{
fakeResolver: &fakeResolver{
mx: map[string][]string{
"acme.example": {"aspmx.l.google.com", "alt1.aspmx.l.google.com"},
"contoso.example": {"contoso-example.mail.protection.outlook.com"},
"guarded.example": {"mx0a-001.pphosted.com"},
"guarded2.example": {"eu-smtp-inbound-1.mimecast.com"},
"selfhost.example": {"mail.selfhost.example"},
"zoho.example": {"mx.zoho.eu"},
},
mxErr: map[string]error{
"flaky.example": &net.DNSError{Err: "server misbehaving", Name: "flaky.example", IsTemporary: true},
},
},
txt: map[string][]string{
"guarded.example": {"google-site-verification=abc", "v=spf1 include:pphosted.com include:_spf.google.com ~all"},
"guarded2.example": {"v=spf1 include:sendgrid.net include:spf.protection.outlook.com -all"},
"selfhost.example": {"v=spf1 mx include:sendgrid.net -all"},
},
}
cases := []struct {
domain string
want Host
ok bool
}{
{"gmail.com", Gmail, true},
{"hotmail.co.uk", Outlook, true},
{"icloud.com", ICloud, true},
{"contoso.onmicrosoft.com", Microsoft365, true},
{"yahoo.com", Yahoo, true},
{"acme.example", GoogleWorkspace, true},
{"contoso.example", Microsoft365, true},
{"guarded.example", GoogleWorkspace, true},
{"guarded2.example", Microsoft365, true},
{"selfhost.example", Other, true},
{"zoho.example", Zoho, true},
{"nomail.example", Unknown, true},
{"flaky.example", Unknown, false},
{"not a domain", Unknown, true},
}
for _, c := range cases {
got, ok := Recipient(context.Background(), r, c.domain)
if got != c.want || ok != c.ok {
t.Errorf("Recipient(%q) = %q, %v; want %q, %v", c.domain, got, ok, c.want, c.ok)
}
}
}
func TestRecipientKnownDomainSkipsDNS(t *testing.T) {
f := &fakeResolver{mxErr: map[string]error{"gmail.com": errors.New("must not be asked")}}
if got, ok := Recipient(context.Background(), recipientFake{fakeResolver: f}, "Dana@GMail.com"); got != Gmail || !ok {
t.Fatalf("got %q, %v", got, ok)
}
if f.mxCalls != 0 {
t.Fatalf("known domain dialled DNS %d times", f.mxCalls)
}
}
func TestKnownDomainTenant(t *testing.T) {
if got := KnownDomain("ops@contoso.onmicrosoft.com"); got != Microsoft365 {
t.Fatalf("tenant domain = %q", got)
}
if SharedProvider("contoso.onmicrosoft.com") {
t.Fatal("a tenant domain is not a consumer provider")
}
}
@@ -0,0 +1,158 @@
package repository
import (
"context"
"testing"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/models"
)
// Run against a migrated scratch database:
//
// WARMBLY_TEST_DB=postgres://warmbly:warmbly@localhost:15432/<db>?sslmode=disable \
// go test ./internal/repository/ -run LiveContactMailHost -v
func TestLiveContactMailHostSweepWritesAndSearchReads(t *testing.T) {
handle, pool := liveContactDB(t)
requireSchemaVersion(t, pool, 219)
f := newSharedOrgFixture(t, pool)
repo := NewContactRepostory(handle)
ctx := context.Background()
add := func(email string) uuid.UUID {
t.Helper()
id := uuid.New()
if _, err := pool.Exec(ctx, `
INSERT INTO contacts (id, user_id, organization_id, email, first_name, last_name, company, phone, custom_fields, updated_at, created_at)
VALUES ($1, $2, $3, $4, 'Live', 'Host', '', '', '{}'::jsonb, NOW(), NOW())`,
id, f.owner, f.org, email); err != nil {
t.Fatalf("insert %s: %v", email, err)
}
return id
}
workspace := add("ana@acme-i695.example")
flaky := add("bo@flaky-i695.example")
moved := add("cy@moved-i695.example")
pending, err := repo.ListMailHostPending(ctx, 100000)
if err != nil {
t.Fatalf("pending: %v", err)
}
seen := map[uuid.UUID]bool{}
for _, p := range pending {
seen[p.ID] = true
}
for _, id := range []uuid.UUID{f.contact, workspace, flaky, moved} {
if !seen[id] {
t.Fatalf("contact %s is not pending", id)
}
}
// The address changed after it was read, so its answer must not land.
if _, err := pool.Exec(ctx, `UPDATE contacts SET email = 'cy@elsewhere-i695.example' WHERE id = $1`, moved); err != nil {
t.Fatal(err)
}
orgs, err := repo.SetContactMailHosts(ctx, []ContactMailHostResult{
{ID: workspace, Email: "ana@acme-i695.example", MailHost: "google_workspace", ESP: "gmail"},
{ID: flaky, Email: "bo@flaky-i695.example", Transient: true},
{ID: moved, Email: "cy@moved-i695.example", MailHost: "yahoo", ESP: "other"},
})
if err != nil {
t.Fatalf("set: %v", err)
}
if len(orgs) != 1 || orgs[0] != f.org {
t.Fatalf("changed orgs = %v, want [%s]", orgs, f.org)
}
type row struct {
host, esp string
resolved bool
retryInHour bool
}
read := func(id uuid.UUID) row {
t.Helper()
var r row
if err := pool.QueryRow(ctx, `
SELECT mail_host, esp_provider, esp_resolved_at IS NOT NULL,
COALESCE(esp_resolved_at < NOW() - make_interval(days => 7) + make_interval(mins => 61), false)
FROM contacts WHERE id = $1`, id).Scan(&r.host, &r.esp, &r.resolved, &r.retryInHour); err != nil {
t.Fatal(err)
}
return r
}
if r := read(workspace); r.host != "google_workspace" || r.esp != "gmail" || !r.resolved {
t.Fatalf("workspace contact = %+v", r)
}
if r := read(flaky); r.host != "" || !r.resolved || !r.retryInHour {
t.Fatalf("transient contact = %+v, want unresolved host retried within the hour", r)
}
if r := read(moved); r.host != "" || r.resolved {
t.Fatalf("moved contact = %+v, want untouched", r)
}
pending, err = repo.ListMailHostPending(ctx, 100000)
if err != nil {
t.Fatal(err)
}
for _, p := range pending {
if p.ID == workspace || p.ID == flaky {
t.Fatalf("contact %s is pending again straight after its check", p.ID)
}
}
res, xerr := repo.Search(ctx, f.org.String(), nil, nil, models.SearchContacts{MailHosts: []string{"google_workspace"}}, 25)
if xerr != nil {
t.Fatalf("filter search: %v", xerr)
}
if len(res.Data) != 1 || res.Data[0].ID != workspace || res.Data[0].MailHost != "google_workspace" || res.Data[0].ESPProvider != "gmail" {
t.Fatalf("filter by provider returned %+v", res.Data)
}
res, xerr = repo.Search(ctx, f.org.String(), nil, nil, models.SearchContacts{MailHosts: []string{""}}, 25)
if xerr != nil {
t.Fatalf("undetected search: %v", xerr)
}
if len(res.Data) != 3 {
t.Fatalf("not-detected filter returned %d contacts, want 3", len(res.Data))
}
if _, err := pool.Exec(ctx, `INSERT INTO campaign_leads (campaign_id, contact_id) SELECT $1, id FROM contacts WHERE organization_id = $2 ON CONFLICT DO NOTHING`, f.campaign, f.org); err != nil {
t.Fatalf("enroll: %v", err)
}
counts, xerr := repo.CampaignLeadCounts(ctx, f.org.String(), f.campaign.String())
if xerr != nil {
t.Fatalf("lead counts: %v", xerr)
}
// The failed lookup was checked, so it counts with other; two were never reached.
if p := counts.Providers; p.Google != 1 || p.Microsoft != 0 || p.Other != 1 || p.Undetected != 2 {
t.Fatalf("lead counts by provider = %+v, want 1 Google, 1 other and 2 undetected", p)
}
for _, reverse := range []bool{true, false} {
res, xerr = repo.Search(ctx, f.org.String(), nil, nil, models.SearchContacts{SortBy: "mail_host", Reverse: reverse}, 25)
if xerr != nil {
t.Fatalf("sort (reverse=%v): %v", reverse, xerr)
}
if len(res.Data) != 4 {
t.Fatalf("sort (reverse=%v) returned %d contacts, want 4", reverse, len(res.Data))
}
// Undetected contacts sit after every known host ascending, before it descending.
want := 0
if !reverse {
want = 3
}
if res.Data[want].ID != workspace {
t.Fatalf("sort (reverse=%v): detected contact at %d, want %d", reverse, indexOf(res.Data, workspace), want)
}
}
}
func indexOf(cs []models.Contact, id uuid.UUID) int {
for i, c := range cs {
if c.ID == id {
return i
}
}
return -1
}
+115 -22
View File
@@ -60,10 +60,12 @@ type ContactRepository interface {
UndeliverableLeadIDs(ctx context.Context, orgID, campaignID uuid.UUID) ([]uuid.UUID, *errx.Error)
// VerificationCounts is the org's contacts by verdict.
VerificationCounts(ctx context.Context, orgID uuid.UUID) (models.ContactVerificationCounts, *errx.Error)
// SetContactESP caches the recipient ESP/provider resolved from the contact's
// domain (control-plane only, no MX dial). Best-effort: a failure should not
// block sending.
SetContactESP(ctx context.Context, contactID uuid.UUID, provider string) error
// ListMailHostPending returns contacts whose inbox host the provider sweep
// has not detected, never-checked first. Instance-wide: control plane only.
ListMailHostPending(ctx context.Context, limit int) ([]ContactMailHostPending, error)
// SetContactMailHosts stores the sweep's results and returns the
// organizations whose contacts changed.
SetContactMailHosts(ctx context.Context, results []ContactMailHostResult) ([]uuid.UUID, error)
GetByEmailsAndUser(ctx context.Context, userID uuid.UUID, emails []string) (map[string]models.Contact, *errx.Error)
// ResolveCategoryNames maps category titles (as typed in an imported file)
// to the workspace's category IDs, creating the ones that don't exist yet.
@@ -593,7 +595,7 @@ func (r *contactRepository) GetByID(ctx context.Context, contactID uuid.UUID) (*
c.custom_fields, c.subscribed, c.updated_at, c.created_at,
c.verification_status, c.verification_reason, c.is_catch_all, c.verification_checked_at,
c.verification_source, c.verification_provider, c.verification_sub_status, c.verification_confidence,
c.verification_requested_at, c.esp_provider, c.esp_resolved_at
c.verification_requested_at, c.mail_host, c.esp_provider, c.esp_resolved_at
FROM contacts c
WHERE c.id = $1
`
@@ -605,7 +607,7 @@ func (r *contactRepository) GetByID(ctx context.Context, contactID uuid.UUID) (*
&contact.UpdatedAt, &contact.CreatedAt,
&contact.VerificationStatus, &contact.VerificationReason, &contact.IsCatchAll, &contact.VerificationCheckedAt,
&contact.VerificationSource, &contact.VerificationProvider, &contact.VerificationSubStatus, &contact.VerificationConfidence,
&contact.VerificationRequestedAt, &contact.ESPProvider, &contact.ESPResolvedAt,
&contact.VerificationRequestedAt, &contact.MailHost, &contact.ESPProvider, &contact.ESPResolvedAt,
)
if err != nil {
if err == pgx.ErrNoRows {
@@ -620,17 +622,97 @@ func (r *contactRepository) GetByID(ctx context.Context, contactID uuid.UUID) (*
return &contact, nil
}
// SetContactESP caches the recipient ESP/provider on the contact row. It is a
// single keyed UPDATE and intentionally tolerant: callers treat any error as a
// best-effort cache miss and fall back to deriving the provider on the fly.
func (r *contactRepository) SetContactESP(ctx context.Context, contactID uuid.UUID, provider string) error {
// ContactMailHostPending is one contact the provider sweep has to look at.
type ContactMailHostPending struct {
ID uuid.UUID
Email string
}
// ContactMailHostResult is the sweep's answer for one contact. Email is the
// address it was resolved for, so an address changed meanwhile is left alone.
type ContactMailHostResult struct {
ID uuid.UUID
Email string
MailHost string
ESP string
// Transient marks a lookup that failed and should be retried soon.
Transient bool
}
func (r *contactRepository) ListMailHostPending(ctx context.Context, limit int) ([]ContactMailHostPending, error) {
query := `
UPDATE contacts
SET esp_provider = $2, esp_resolved_at = NOW()
WHERE id = $1
SELECT id, email
FROM contacts
WHERE mail_host = ''
AND (esp_resolved_at IS NULL OR esp_resolved_at < NOW() - make_interval(days => $2))
ORDER BY esp_resolved_at NULLS FIRST
LIMIT $1
`
_, err := r.DB.Exec(ctx, query, contactID, provider)
return err
rows, err := r.DB.Query(ctx, query, limit, config.ContactMailHostRecheckDays)
if err != nil {
db.CaptureError(err, query, []any{limit}, "query")
return nil, err
}
defer rows.Close()
var out []ContactMailHostPending
for rows.Next() {
var p ContactMailHostPending
if err := rows.Scan(&p.ID, &p.Email); err != nil {
return nil, err
}
out = append(out, p)
}
return out, rows.Err()
}
func (r *contactRepository) SetContactMailHosts(ctx context.Context, results []ContactMailHostResult) ([]uuid.UUID, error) {
if len(results) == 0 {
return nil, nil
}
ids := make([]uuid.UUID, len(results))
emails := make([]string, len(results))
hosts := make([]string, len(results))
esps := make([]string, len(results))
transient := make([]bool, len(results))
for i, res := range results {
ids[i], emails[i], hosts[i], esps[i], transient[i] = res.ID, res.Email, res.MailHost, res.ESP, res.Transient
}
// A transient failure is stamped as checked long enough ago that the
// pending read offers it again after the retry delay, not the recheck window.
query := `
WITH u AS (
SELECT * FROM unnest($1::uuid[], $2::text[], $3::text[], $4::text[], $5::bool[])
AS t(id, email, mail_host, esp, transient)
), changed AS (
UPDATE contacts c
SET mail_host = u.mail_host,
esp_provider = u.esp,
esp_resolved_at = CASE WHEN u.transient
THEN NOW() - make_interval(days => $6) + make_interval(mins => $7)
ELSE NOW() END
FROM u
WHERE c.id = u.id AND c.email = u.email AND c.mail_host = ''
RETURNING c.organization_id, (u.mail_host <> '') AS found
)
SELECT DISTINCT organization_id FROM changed
WHERE found AND organization_id IS NOT NULL
`
args := []any{ids, emails, hosts, esps, transient, config.ContactMailHostRecheckDays, config.ContactMailHostRetryMinutes}
rows, err := r.DB.Query(ctx, query, args...)
if err != nil {
db.CaptureError(err, query, nil, "query")
return nil, err
}
defer rows.Close()
var orgs []uuid.UUID
for rows.Next() {
var id uuid.UUID
if err := rows.Scan(&id); err != nil {
return nil, err
}
orgs = append(orgs, id)
}
return orgs, rows.Err()
}
// UpdateContactVerification stores the outcome of a verification pass on the
@@ -1080,6 +1162,8 @@ var contactSorts = map[string]contactSort{
"created_at": {expr: "c.created_at", kind: sortTimestamp},
"updated_at": {expr: "c.updated_at", kind: sortTimestamp},
"campaign_count": {expr: "COALESCE(cl.campaign_count,0)", kind: sortNumber},
// Contacts the provider sweep has not reached sort after every known host.
"mail_host": {expr: "NULLIF(c.mail_host, '')", kind: sortText, nullable: true},
}
// resolveContactSort turns a request's sort_by into the column the list orders
@@ -1214,6 +1298,11 @@ func (r *contactRepository) buildContactFilter(ctx context.Context, orgID string
args = append(args, filters.VerificationStatus)
argIndex++
}
if len(filters.MailHosts) > 0 {
whereClauses = append(whereClauses, fmt.Sprintf("c.mail_host = ANY($%d)", argIndex))
args = append(args, filters.MailHosts)
argIndex++
}
// -----------------------------
// Date filters
@@ -1554,7 +1643,7 @@ func (r *contactRepository) Search(
c.custom_fields, c.subscribed, c.updated_at, c.created_at,
c.verification_status, c.verification_reason, c.is_catch_all, c.verification_checked_at,
c.verification_source, c.verification_provider, c.verification_sub_status, c.verification_confidence,
c.verification_requested_at,
c.verification_requested_at, c.mail_host, c.esp_provider,
COALESCE(
(
SELECT json_agg(json_build_object('id', cam.id, 'name', cam.name))
@@ -1640,7 +1729,7 @@ func (r *contactRepository) Search(
&c.UpdatedAt, &c.CreatedAt,
&c.VerificationStatus, &c.VerificationReason, &c.IsCatchAll, &c.VerificationCheckedAt,
&c.VerificationSource, &c.VerificationProvider, &c.VerificationSubStatus, &c.VerificationConfidence,
&c.VerificationRequestedAt,
&c.VerificationRequestedAt, &c.MailHost, &c.ESPProvider,
&campaignsJSON, &categoriesJSON, &leadProgressJSON,
&sortValue,
); err != nil {
@@ -2093,7 +2182,11 @@ func (r *contactRepository) CampaignLeadCounts(ctx context.Context, orgID, campa
COUNT(*) FILTER (WHERE COALESCE(pr.has_sent, false)) AS contacted,
COUNT(*) FILTER (WHERE COALESCE(pr.has_opened, false)) AS opened,
COUNT(*) FILTER (WHERE COALESCE(pr.has_clicked, false)) AS clicked,
COUNT(*) FILTER (WHERE COALESCE(pr.has_replied, false)) AS replied_any
COUNT(*) FILTER (WHERE COALESCE(pr.has_replied, false)) AS replied_any,
COUNT(*) FILTER (WHERE c.esp_provider = 'gmail') AS provider_gmail,
COUNT(*) FILTER (WHERE c.esp_provider = 'outlook') AS provider_outlook,
COUNT(*) FILTER (WHERE c.esp_provider = 'other' OR (c.esp_provider = '' AND c.esp_resolved_at IS NOT NULL)) AS provider_other,
COUNT(*) FILTER (WHERE c.esp_provider = '' AND c.esp_resolved_at IS NULL) AS provider_undetected
FROM campaign_leads cl
JOIN contacts c ON c.id = cl.contact_id AND c.organization_id = $2
CROSS JOIN (SELECT COUNT(*) AS total_steps FROM sequences st WHERE st.campaign_id = $1 AND st.kind = 'email') ts
@@ -2115,6 +2208,7 @@ func (r *contactRepository) CampaignLeadCounts(ctx context.Context, orgID, campa
if err := r.DB.QueryRow(ctx, query, campaignID, orgID, config.CampaignSendMaxAttempts).Scan(
&out.Total, &out.Unsubscribed, &out.Bounced, &out.Replied, &out.Failed, &out.Completed, &out.Paused, &out.Processing, &out.Undeliverable, &out.Queued,
&out.Contacted, &out.Opened, &out.Clicked, &out.RepliedAny,
&out.Providers.Google, &out.Providers.Microsoft, &out.Providers.Other, &out.Providers.Undetected,
); err != nil {
if err == pgx.ErrNoRows {
return out, nil
@@ -2261,10 +2355,9 @@ func (r *contactRepository) Update(ctx context.Context, userID, contactID string
// the next pass and hand the new address a verdict it never
// earned.
"verification_evidence_reset_at = NOW()",
// esp_provider is derived from the address domain and cached
// forever: the scheduler only fills it when it is empty, so a
// gmail-to-outlook correction would keep routing ESP-matched
// sends by the old provider.
// The inbox host belongs to the old domain; clearing it puts
// the contact back in front of the provider sweep.
"mail_host = ''",
"esp_provider = ''",
"esp_resolved_at = NULL",
)
+1
View File
@@ -128,6 +128,7 @@ var segmentEnumColumns = map[string]string{
"source": "c.source",
"verification_status": "c.verification_status",
"esp_provider": "c.esp_provider",
"mail_host": "c.mail_host",
}
var segmentDateExprs = map[string]string{
+5 -13
View File
@@ -263,9 +263,9 @@ func (s *schedulerService) placeCampaignSend(ctx context.Context, campaign *mode
}
}
// STEP 3.5: Resolve the recipient ESP/provider for ESP matching. Cheap:
// prefer the cached contact.esp_provider, else derive from the domain
// string. NEVER dial MX on the hot path. Empty => unknown => wildcard.
// STEP 3.5: Resolve the recipient ESP/provider for ESP matching from what
// the provider sweep stored, else the domain string. NEVER dial MX on the
// hot path. Empty => unknown => wildcard.
// STEP 3.4: Recipient-timezone policy. Disabled, absent or unreadable all
// leave the send on the sending mailbox's clock.
sendPref := s.sendTimePreference(ctx, campaign.OrganizationID)
@@ -280,15 +280,7 @@ func (s *schedulerService) placeCampaignSend(ctx context.Context, campaign *mode
recipientProvider := ""
if campaign.ESPMatchMode != "off" && recipientContact != nil {
if recipientContact.ESPProvider != "" {
recipientProvider = recipientContact.ESPProvider
} else {
recipientProvider = providerForEmailDomain(recipientContact.Email)
// Opportunistically cache the derived provider (best-effort).
if recipientProvider != "" && !preview {
_ = s.contactRepo.SetContactESP(ctx, recipientContact.ID, recipientProvider)
}
}
recipientProvider = recipientESP(recipientContact)
}
// STEP 4: Calculate base time from sequence wait_after
@@ -506,7 +498,7 @@ func (s *schedulerService) placeCampaignSend(ctx context.Context, campaign *mode
RemainingToday: remaining,
WarmupAgeDays: warmupAgeDays,
Weight: computeWeight(remaining, warmupAgeDays),
ProviderMatch: providerMatches(acct.Provider),
ProviderMatch: providerMatches(senderESP(acct)),
Behavior: pass.behaviors[acct.ID],
OpenAt: openAt,
OpenLoc: openLoc,
+22 -17
View File
@@ -4,11 +4,11 @@ import (
"math"
"math/rand"
"sort"
"strings"
"time"
"github.com/warmbly/warmbly/internal/app/behavior"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/pkg/mailhost"
"github.com/warmbly/warmbly/internal/repository"
)
@@ -409,24 +409,29 @@ func campaignRampCeiling(enabled bool, start, increment, ceiling, level int) int
return v
}
// providerForEmailDomain maps a recipient email address to a coarse ESP bucket
// for provider matching. Pure string work — never dials MX. Unknown/other
// domains return "" so matching never blocks the first contact.
func providerForEmailDomain(email string) string {
at := strings.LastIndex(email, "@")
if at < 0 || at == len(email)-1 {
return ""
// senderESP is a mailbox's family for ESP matching: its email_provider, or
// for an smtp_imap mailbox on Google or Microsoft (an app-password import)
// that host's family.
func senderESP(acct models.Email) string {
if acct.Provider == "smtp_imap" {
if esp := mailhost.ESPFamily(mailhost.Host(acct.MailHost)); esp == "gmail" || esp == "outlook" {
return esp
}
}
domain := strings.ToLower(strings.TrimSpace(email[at+1:]))
switch domain {
case "gmail.com", "googlemail.com":
return "gmail"
case "outlook.com", "hotmail.com", "live.com", "msn.com", "office365.com", "microsoft.com":
return "outlook"
return acct.Provider
}
// recipientESP is the provider family ESP matching compares against a
// mailbox's email_provider: "gmail", "outlook", or "" for anything else,
// which matches every mailbox. A contact the provider sweep has not reached
// yet is read from its domain alone.
func recipientESP(c *models.Contact) string {
esp := c.ESPProvider
if c.MailHost == "" && esp == "" {
esp = mailhost.ESPFamily(mailhost.KnownDomain(c.Email))
}
// Subdomain / suffix heuristics for hosted Google/Microsoft mail.
if strings.HasSuffix(domain, ".onmicrosoft.com") {
return "outlook"
if esp == "gmail" || esp == "outlook" {
return esp
}
return ""
}
@@ -227,6 +227,7 @@ export function EspMatchingSection({
mode={newCampaign.esp_match_mode}
emailTags={newCampaign.email_tags}
explicitAccounts={explicitAccounts}
campaignId={newCampaign.id}
/>
</div>
);
@@ -4,10 +4,12 @@
// match and an SMTP wildcard is obvious. No connector lines, no new API.
//
// Grounded in the scheduler (internal/scheduler/campaign_scheduler.go):
// • recipient provider is derived from the domain → "gmail" | "outlook" |
// unknown (everything else).
// • a mailbox matches a recipient when: mailbox is smtp_imap (WILDCARD —
// serves any provider, even in strict), OR the recipient is unknown
// • recipient provider is the contact's esp_provider, read from its domain's
// MX → "gmail" (Google) | "outlook" (Microsoft) | anything else (wildcard).
// • a mailbox's provider is its connection, or for an SMTP/IMAP mailbox on
// Google Workspace or Microsoft 365 that host's family.
// • a mailbox matches a recipient when: mailbox is other SMTP/IMAP (a
// wildcard outside strict), OR the recipient is not Google/Microsoft
// (wildcard), OR provider equality (gmail↔gmail, outlook↔outlook).
// • strict + no matching mailbox → the recipient is DEFERRED (held to the next
// slot), never sent cross-provider and never skipped.
@@ -18,6 +20,8 @@
import React from "react";
import useEmails from "@/lib/api/hooks/app/emails/useEmails";
import useSearchContacts from "@/lib/api/hooks/app/contacts/useSearchContacts";
import { mailHostLogo } from "@/lib/mailHost";
import ProviderLogo from "./ProviderLogo";
type Mode = "off" | "prefer" | "strict";
@@ -25,21 +29,39 @@ type Status = "ok" | "warn" | "blocked" | "any";
const LABEL: Record<string, string> = {
gmail: "Google",
outlook: "Outlook",
outlook: "Microsoft",
smtp_imap: "Other / SMTP",
other: "Other domains",
};
// The family the scheduler matches a mailbox by (senderESP in the scheduler).
function mailboxFamily(e: { provider: string; mail_host?: string }): "gmail" | "outlook" | "smtp_imap" {
if (e.provider === "gmail" || e.provider === "outlook") return e.provider;
const host = mailHostLogo(e.mail_host);
return host === "google" ? "gmail" : host === "microsoft" ? "outlook" : "smtp_imap";
}
export default function EspCoveragePanel({
mode,
emailTags,
explicitAccounts,
campaignId,
}: {
mode: Mode;
emailTags: string[];
explicitAccounts: string[];
// The campaign whose leads are counted by provider; none on a draft.
campaignId?: string;
}) {
const { emails, isLoading } = useEmails({ query: "", tag: "", limit: 200 });
const leadSearch = useSearchContacts({
options: { query: "", custom_field_filters: [], campaign_ids: campaignId ? [campaignId] : [], sort_by: "created_at", reverse: false },
limit: 1,
enabled: !!campaignId,
});
const leads = leadSearch.data?.pages[0]?.lead_counts?.providers;
const leadCount = (n: number | undefined) =>
leads && n !== undefined ? <span className="ml-1.5 text-[11px] font-normal text-slate-400 tabular-nums">{n.toLocaleString()} {n === 1 ? "lead" : "leads"}</span> : null;
const pool = React.useMemo(() => {
const explicit = new Set(explicitAccounts);
@@ -58,8 +80,9 @@ export default function EspCoveragePanel({
let outlook = 0;
let smtp = 0;
for (const e of pool) {
if (e.provider === "gmail") gmail++;
else if (e.provider === "outlook") outlook++;
const family = mailboxFamily(e);
if (family === "gmail") gmail++;
else if (family === "outlook") outlook++;
else smtp++;
}
return { gmail, outlook, smtp, total: pool.length };
@@ -121,7 +144,7 @@ export default function EspCoveragePanel({
{(["gmail", "outlook"] as const).map((r) => {
const sameCount = r === "gmail" ? counts.gmail : counts.outlook;
const hasSame = sameCount > 0;
// SMTP/IMAP is a wildcard for Gmail/Outlook recipients only OUTSIDE
// SMTP/IMAP is a wildcard for Google/Microsoft recipients only OUTSIDE
// strict — under strict, "same provider" excludes unknown-ESP SMTP.
const wildServes = counts.smtp > 0 && mode !== "strict";
// prefer falls back cross-provider when no same + no wildcard.
@@ -146,6 +169,7 @@ export default function EspCoveragePanel({
<div className="min-w-0 flex-1">
<div className="text-[12.5px] font-medium text-slate-800 leading-tight">
{LABEL[r]} recipients
{leadCount(leads?.[r])}
</div>
<div className="mt-1.5 flex items-center gap-1.5 flex-wrap">
<span className="text-[10px] uppercase tracking-[0.1em] text-slate-400">via</span>
@@ -184,7 +208,10 @@ export default function EspCoveragePanel({
<div className="flex items-center gap-3 rounded-lg border border-slate-200 bg-white px-3 py-2.5">
<ProviderLogo provider="other" className="size-8" />
<div className="min-w-0 flex-1">
<div className="text-[12.5px] font-medium text-slate-800 leading-tight">Other domains</div>
<div className="text-[12.5px] font-medium text-slate-800 leading-tight">
Other providers
{leadCount(leads?.other)}
</div>
<div className="mt-1.5 flex items-center gap-1.5 flex-wrap">
<span className="text-[10px] uppercase tracking-[0.1em] text-slate-400">via</span>
<AnyMailbox />
@@ -197,16 +224,22 @@ export default function EspCoveragePanel({
{mode === "strict" && onlyWildcard && (
<p className="rounded-md bg-amber-50 border border-amber-200 px-2.5 py-2 text-[11px] text-amber-700 leading-relaxed">
All your mailboxes are SMTP/IMAP, which can send to any provider — so Strict can't narrow by
provider here and behaves like Off. Connect a Google or Outlook mailbox to truly restrict
provider here and behaves like Off. Connect a Google or Microsoft mailbox to truly restrict
same-provider sending.
</p>
)}
{leads && leads.undetected > 0 && (
<p className="text-[11px] text-slate-500 leading-relaxed">
{leads.undetected.toLocaleString()} {leads.undetected === 1 ? "lead's provider is" : "leads' providers are"} still being read from their domain's mail records. Until then only their address is used, so a gmail.com or outlook.com lead still matches.
</p>
)}
<p className="text-[11px] text-slate-400 leading-relaxed">
{mode === "off"
? "Provider matching is off — any recipient can be sent from any mailbox in the pool."
: mode === "strict"
? "Strict sends Google and Outlook recipients only from a same-provider mailbox (the sky match) — a recipient with no same-provider mailbox is held (deferred) until one frees up, never sent cross-provider. SMTP/IMAP mailboxes only carry non-Google/Outlook (“other”) domains under strict."
? "Strict sends Google and Microsoft recipients only from a same-provider mailbox (the sky match). A recipient with no same-provider mailbox is held until one frees up, never sent cross-provider. Other SMTP/IMAP mailboxes only carry recipients on other providers under strict."
: "Prefer uses a same-provider mailbox when one has capacity (the sky match), otherwise it falls back to another provider (the amber chip) — it never holds a recipient. SMTP/IMAP mailboxes can carry any provider."}
</p>
</div>
+30 -2
View File
@@ -14,7 +14,9 @@ import {
PhoneIcon,
type LucideIcon,
} from "lucide-react";
import ProviderLogo from "@/components/app/emails/ProviderLogo";
import clippedTitle from "@/lib/helper/clippedTitle";
import { mailHostLabel } from "@/lib/mailHost";
import type { ContactCampaignProgress, VerificationSource, VerificationStatus } from "@/lib/api/models/app/contacts/Contact";
import type { SearchContactsSortBy } from "@/lib/api/models/app/contacts/search-contacts.types";
import type { ViewName } from "@/lib/api/models/app/views/ViewPreferences";
@@ -43,6 +45,7 @@ export interface ContactRow {
verification_checked_at?: string | null;
verification_confidence?: number;
verification_requested_at?: string | null;
mail_host?: string;
created_at: Date;
updated_at?: Date;
}
@@ -185,6 +188,29 @@ function companyColumn(view: ViewName): ContactColumn {
};
}
// Who hosts the contact's inbox. Empty until the backend's DNS check reaches
// the contact, a minute or so after it is added.
function mailHostColumn(view: ViewName): ContactColumn {
return {
id: "mail_host",
label: "Email provider",
width: "w-40",
hideBelow: view === "campaign_leads" ? "2xl" : "xl",
sortKey: "mail_host",
sortAsc: true,
cellClassName: "text-[12px] text-slate-600",
cell: ({ c }) =>
c.mail_host ? (
<div className="flex items-center gap-1.5 min-w-0">
<ProviderLogo id={c.mail_host} size="xs" framed={false} />
<span className="truncate" {...clippedTitle}>{mailHostLabel(c.mail_host)}</span>
</div>
) : (
<Dash />
),
};
}
const phoneColumn: ContactColumn = {
id: "phone",
label: "Phone",
@@ -369,6 +395,7 @@ export function builtinColumns(view: ViewName): ContactColumn[] {
return [
nameColumn,
companyColumn(view),
mailHostColumn(view),
phoneColumn,
progressColumn,
engagement(
@@ -390,12 +417,12 @@ export function builtinColumns(view: ViewName): ContactColumn[] {
updatedColumn,
];
}
return [nameColumn, companyColumn(view), phoneColumn, statusColumn, campaignsColumn, addedColumn(view), updatedColumn];
return [nameColumn, companyColumn(view), mailHostColumn(view), phoneColumn, statusColumn, campaignsColumn, addedColumn(view), updatedColumn];
}
// The layout a member sees before choosing anything.
export const DEFAULT_COLUMNS: Record<ViewName, string[]> = {
contacts: ["name", "company", "phone", "status", "campaigns", "created_at"],
contacts: ["name", "company", "mail_host", "phone", "status", "campaigns", "created_at"],
campaign_leads: ["name", "company", "progress", "opened", "clicked", "replied", "current_step", "sender", "last_activity"],
};
@@ -448,6 +475,7 @@ export function sortOptions(view: ViewName): SortOption[] {
{ key: "email", label: "Email", asc: true },
{ key: "company", label: "Company", asc: true },
{ key: "phone", label: "Phone", asc: true },
{ key: "mail_host", label: "Email provider", asc: true },
];
if (view === "contacts") base.push({ key: "campaign_count", label: "Campaigns", asc: false });
return base;
@@ -6,6 +6,7 @@ import React from "react";
import { AnimatePresence, motion } from "framer-motion";
import { CheckIcon, ChevronDownIcon, LayersIcon, Loader2Icon, PlusIcon, XIcon } from "lucide-react";
import ProviderLogo from "@/components/app/emails/ProviderLogo";
import { DatePicker } from "@/components/ui/DatePicker";
import { NumberInput, TextInput } from "@/components/ui/field";
import {
@@ -27,6 +28,8 @@ import type SearchContacts from "@/lib/api/models/app/contacts/SearchContacts";
import type SearchContactsFilter from "@/lib/api/models/app/contacts/SearchContactsFilter";
import type { SearchContactsFilterType } from "@/lib/api/models/app/contacts/search-contacts.types";
import type { LeadEngagement, LeadStatus, VerificationStatus } from "@/lib/api/models/app/contacts/Contact";
import type { MailHost } from "@/lib/api/models/app/emails/MailboxImport";
import { MAIL_HOST_LABELS } from "@/lib/mailHost";
import { cn } from "@/lib/utils";
import { countActiveFilters, isCompleteCustomFilter } from "./helpers";
@@ -66,8 +69,18 @@ const VERIFICATION: { id: VerificationStatus; label: string }[] = [
{ id: "unknown", label: "Unverified" },
];
// Every inbox host the backend detects, then the contacts it has not reached yet.
const PROVIDERS: Option[] = [
...(Object.entries(MAIL_HOST_LABELS) as [MailHost, string][]).map(([id, label]) => ({
id,
label,
icon: <ProviderLogo id={id} size="xs" framed={false} />,
})),
{ id: "", label: "Unknown or not checked yet" },
];
// Optional pills, shown once added from the menu or when their value is set.
type ExtraKey = "created" | "updated" | "campaign_count" | "lead_status" | "engagement" | "verification";
type ExtraKey = "created" | "updated" | "campaign_count" | "lead_status" | "engagement" | "verification" | "provider";
function toIso(d?: Date): string {
return d ? new Date(d).toISOString().slice(0, 10) : "";
@@ -121,6 +134,8 @@ export default function FilterBar({
return !!filters.engagement;
case "verification":
return !!filters.verification_status;
case "provider":
return !!filters.mail_hosts?.length;
}
};
@@ -144,6 +159,8 @@ export default function FilterBar({
return { ...s, engagement: undefined };
case "verification":
return { ...s, verification_status: undefined };
case "provider":
return { ...s, mail_hosts: undefined };
}
});
}
@@ -195,6 +212,7 @@ export default function FilterBar({
{ key: "updated", label: "Last updated", hidden: shown("updated") },
{ key: "campaign_count", label: "Number of campaigns", hidden: shown("campaign_count") },
{ key: "verification", label: "Address verification", hidden: shown("verification") },
{ key: "provider", label: "Email provider", hidden: shown("provider") },
{ key: "lead_status", label: "Lead status", hidden: !campaignCtx || shown("lead_status") },
{ key: "engagement", label: "Engagement", hidden: !campaignCtx || shown("engagement") },
];
@@ -304,6 +322,25 @@ export default function FilterBar({
onRemove={() => removeExtra("verification")}
/>
)}
{shown("provider") && (
<Pill
id="provider"
label="Provider"
summary={summarize(filters.mail_hosts ?? [], PROVIDERS)}
active={!!filters.mail_hosts?.length}
openKey={openKey}
setOpenKey={setOpenKey}
onRemove={() => removeExtra("provider")}
>
<CheckList
value={filters.mail_hosts ?? []}
onChange={(v) => setFilters((s) => ({ ...s, mail_hosts: v.length ? (v as MailHost[]) : undefined }))}
options={PROVIDERS}
empty="No providers."
hint="Contacts at any selected provider."
/>
</Pill>
)}
{shown("lead_status") && (
<ChoicePill<LeadStatus | undefined>
id="lead_status"
@@ -485,6 +522,7 @@ interface Option {
id: string;
label: string;
color?: string;
icon?: React.ReactNode;
}
function summarize(ids: string[], options: Option[]): string {
@@ -547,6 +585,7 @@ function CheckList({
{checked && <CheckIcon className="w-2 h-2 text-white" />}
</span>
{o.color && <span className="size-2.5 rounded-full shrink-0" style={{ backgroundColor: o.color }} />}
{o.icon}
<span className="truncate">{o.label}</span>
</button>
);
@@ -16,6 +16,7 @@ export function countActiveFilters(f: SearchContacts, campaignContext: boolean):
if (!campaignContext && f.campaign_ids.length > 0) n++;
if (f.subscribed !== undefined) n++;
if (f.verification_status) n++;
if (f.mail_hosts?.length) n++;
if (f.min_campaigns !== undefined || f.max_campaigns !== undefined) n++;
if (f.created_after || f.created_before) n++;
if (f.updated_after || f.updated_before) n++;
@@ -37,6 +38,7 @@ export function hasNarrowingFilters(f: SearchContacts, base?: SearchContacts): b
(f.category_ids?.length ?? 0) > 0 ||
f.subscribed !== undefined ||
!!f.verification_status ||
(f.mail_hosts?.length ?? 0) > 0 ||
!!f.lead_status ||
!!f.engagement ||
f.min_campaigns !== undefined ||
@@ -488,7 +488,7 @@ function ValueInput({
/>
);
case "enum":
return <EnumMultiPicker value={values} onChange={setValues} options={spec.options ?? []} />;
return <EnumMultiPicker value={values} onChange={setValues} options={spec.options ?? []} labels={spec.option_labels} />;
case "category":
return <CategoryPicker value={values} onChange={setValues} placeholder="Pick categories…" allowCreate={false} />;
case "campaign":
@@ -210,11 +210,13 @@ export function EnumMultiPicker({
value,
onChange,
options,
labels,
}: {
value: string[];
onChange: (next: string[]) => void;
options: string[];
labels?: Record<string, string>;
}) {
const opts = React.useMemo<PickOption[]>(() => options.map((o) => ({ id: o, label: ENUM_LABELS[o] ?? o })), [options]);
return <MultiPicker value={value} onChange={onChange} options={opts} placeholder="Pick values…" searchable={false} />;
const opts = React.useMemo<PickOption[]>(() => options.map((o) => ({ id: o, label: labels?.[o] ?? ENUM_LABELS[o] ?? o })), [options, labels]);
return <MultiPicker value={value} onChange={onChange} options={opts} placeholder="Pick values…" searchable={options.length > 8} />;
}
@@ -43,6 +43,10 @@ export function filtersToSegment(f: SearchContacts, campaignID?: string): Segmen
conditions.push({ field: "subscribed", operator: f.subscribed ? "is_true" : "is_false" });
}
if (f.verification_status) conditions.push({ field: "verification_status", operator: "in", values: [f.verification_status] });
// A segment enum has no empty value, so "unknown" is reported rather than lost.
const hosts = (f.mail_hosts ?? []).filter(Boolean);
if (hosts.length > 0) conditions.push({ field: "mail_host", operator: "in", values: hosts });
if (f.mail_hosts?.includes("")) dropped.push("unknown email provider");
if (f.min_campaigns !== undefined) conditions.push({ field: "campaign_count", operator: "gte", value: String(f.min_campaigns) });
if (f.max_campaigns !== undefined) conditions.push({ field: "campaign_count", operator: "lte", value: String(f.max_campaigns) });
if (f.created_after) conditions.push({ field: "created_at", operator: "after", value: isoDate(f.created_after) });
@@ -1,5 +1,6 @@
import type MiniCampaign from "../campaigns/MiniCampaign";
import type MiniCategory from "./MiniCategory";
import type { MailHost } from "../emails/MailboxImport";
// LeadStatus mirrors models.ContactCampaignProgress.Status, a contact's
// processing state inside a single campaign. "completed" = every step sent, no
@@ -122,6 +123,11 @@ export default interface Contact {
verification_requested_at?: string | null;
is_catch_all?: boolean;
// Who hosts the contact's inbox, read from its domain's MX; "" until
// detected. esp_provider is its family, the one ESP matching uses.
mail_host?: MailHost;
esp_provider?: "" | "gmail" | "outlook" | "other";
// Present only in the campaign Leads view (single-campaign search). Drives
// the per-lead processing-state column.
campaign_lead?: ContactCampaignProgress | null;
@@ -1,6 +1,7 @@
import type { SearchContactsSortBy } from "./search-contacts.types";
import type SearchContactsFilter from "./SearchContactsFilter";
import type { LeadEngagement, LeadStatus, VerificationStatus } from "./Contact";
import type { MailHost } from "../emails/MailboxImport";
export default interface SearchContacts {
query: string;
@@ -17,6 +18,8 @@ export default interface SearchContacts {
max_campaigns?: number;
subscribed?: boolean;
verification_status?: VerificationStatus;
// Contacts whose inbox is hosted by any of these; "" matches no known provider.
mail_hosts?: MailHost[];
created_after?: Date;
created_before?: Date;
updated_after?: Date;
@@ -53,6 +53,17 @@ export interface CampaignLeadCounts {
opened: number;
clicked: number;
replied_any: number;
// Leads by their inbox's provider family, the grouping ESP matching uses.
providers?: CampaignLeadProviderCounts;
}
export interface CampaignLeadProviderCounts {
gmail: number;
outlook: number;
// Includes checked domains with no known provider; they match like other.
other: number;
// Leads whose provider the background check has not read yet.
undetected: number;
}
export default interface SearchContactsResult {
@@ -10,6 +10,7 @@ export type SearchContactsSortBy =
| 'company'
| 'phone'
| 'campaign_count'
| 'mail_host'
| `custom:${string}`;
export type SearchContactsFilterType =
@@ -25,6 +25,8 @@ export interface SegmentFieldSpec {
group: string;
kind: SegmentFieldKind;
options?: string[];
// Display names for an enum's values, where the value alone is not readable.
option_labels?: Record<string, string>;
}
export default interface Segment {