diff --git a/docs/content/docs/api/reference/contacts.mdx b/docs/content/docs/api/reference/contacts.mdx index 346c6d1c2..493c14a41 100644 --- a/docs/content/docs/api/reference/contacts.mdx +++ b/docs/content/docs/api/reference/contacts.mdx @@ -33,7 +33,7 @@ Every field is optional; an empty body matches all contacts in the organization. | `category_ids` | string[] | No | Contact must have ALL of these categories. | | `segment_ids` | string[] | No | Contact must be a member of ALL of these segments (conditions plus manual overrides). An id that is not a valid UUID is rejected with `400`; an unknown segment matches nothing. | | `verification_status` | string | No | Filter by verification verdict: `valid`, `risky`, `invalid`, or `unknown`. | -| `mail_hosts` | string[] | No | Contacts whose inbox is hosted by any of these `mail_host` values (see [Email provider](#email-provider)). `""` matches contacts whose provider is not detected yet. An unknown value is rejected with `invalid_mail_host`. | +| `mail_hosts` | string[] | No | Contacts whose inbox is hosted by any of these `mail_host` values (see [Email provider](#email-provider)). `""` matches contacts with no known provider: not checked yet, or a domain with no mail server. An unknown value is rejected with `invalid_mail_host`. | | `min_campaigns` | integer | No | Minimum number of associated campaigns. | | `max_campaigns` | integer | No | Maximum number of associated campaigns. | | `subscribed` | boolean | No | Filter by subscription status. | @@ -41,7 +41,7 @@ Every field is optional; an empty body matches all contacts in the organization. | `created_before` | string (RFC 3339) | No | Created on or before this time. | | `updated_after` | string (RFC 3339) | No | Updated on or after this time. | | `updated_before` | string (RFC 3339) | No | Updated on or before this time. | -| `sort_by` | string | No | Sort column: `created_at` (the default), `updated_at`, `first_name`, `last_name`, `email`, `company`, `phone`, `campaign_count`, or `mail_host`. `mail_host` sorts by the provider value, with contacts whose provider is not detected yet grouped last when ascending and first when descending. `custom:` sorts on a custom field, as text, with contacts that lack the field (or have it blank) grouped last when ascending and first when descending. A custom-field name that could never exist is rejected with `invalid_sort_by`; any other unknown value falls back to `created_at`. | +| `sort_by` | string | No | Sort column: `created_at` (the default), `updated_at`, `first_name`, `last_name`, `email`, `company`, `phone`, `campaign_count`, or `mail_host`. `mail_host` sorts by the provider value, with contacts that have no known provider grouped last when ascending and first when descending. `custom:` sorts on a custom field, as text, with contacts that lack the field (or have it blank) grouped last when ascending and first when descending. A custom-field name that could never exist is rejected with `invalid_sort_by`; any other unknown value falls back to `created_at`. | | `reverse` | boolean | No | Ascending when true. The default is descending. | ```json @@ -104,7 +104,7 @@ Every contact carries `mail_host`, who hosts the inbox the address belongs to, a When the search filters by exactly one campaign, each contact additionally carries a `campaign_lead` object with its processing state inside that campaign (`status`, `sent`, `opened`, `machine_opened`, `clicked`, `replied`, `bounced`, `current_step`, `sender`, `last_activity_at`, `hold` when held, and `failure_reason` when failed). `sender` is the mailbox address the lead's whole sequence sends from, fixed when its first email went out and absent until then. `opened` counts steps opened by a person; steps fetched automatically by a mail client (Apple Mail Privacy Protection and similar) are in `machine_opened` instead, matching the machine opens the analytics summary reports. The `status` derivation, highest priority first, is `unsubscribed` (not subscribed), then `bounced`, `replied`, `failed` (a step could not be sent after every retry; `failure_reason` carries the sending worker's reason), `completed` (every email step sent, no reply), `paused` (the lead's flow is held, by an out-of-office auto-reply or by hand; the `hold` object carries `since`, `until`, `reason` and `source`), `active` (some steps sent, more to send), `undeliverable` (pre-send verification refused the address, so the campaign skips the lead and never sends to it), and `pending` (queued, nothing sent). A step counts as sent only once the sending worker has delivered it to the mailbox provider; a send the worker could not complete is retried on the campaign's next pass and never shows as sent. The `lead_status` filter narrows to one of these buckets. -When the search filters by exactly one campaign, the first page (no `cursor`) also includes a `lead_counts` object: per-status lead totals for that campaign, independent of the `lead_status` and `engagement` filters so every scope's total is available at once. The status buckets include `paused`. Alongside them it carries engagement totals that match the `engagement` filter: `contacted` (leads sent at least one step), `opened` (a human open on any step), `clicked`, and `replied_any` (a reply on any step, whatever the derived status). `providers` splits the campaign's leads by [email provider](#email-provider) family, the grouping ESP matching uses: `gmail`, `outlook`, `other`, and `undetected` for leads whose provider has not been read yet. +When the search filters by exactly one campaign, the first page (no `cursor`) also includes a `lead_counts` object: per-status lead totals for that campaign, independent of the `lead_status` and `engagement` filters so every scope's total is available at once. The status buckets include `paused`. Alongside them it carries engagement totals that match the `engagement` filter: `contacted` (leads sent at least one step), `opened` (a human open on any step), `clicked`, and `replied_any` (a reply on any step, whatever the derived status). `providers` splits the campaign's leads by [email provider](#email-provider) family, the grouping ESP matching uses: `gmail`, `outlook`, `other` (including checked domains with no known provider, which ESP matching treats the same way), and `undetected` for leads whose provider has not been read yet. ```json { diff --git a/docs/content/docs/guides/contacts-crm.mdx b/docs/content/docs/guides/contacts-crm.mdx index 721e3bf50..101286f7b 100644 --- a/docs/content/docs/guides/contacts-crm.mdx +++ b/docs/content/docs/guides/contacts-crm.mdx @@ -109,7 +109,7 @@ A sort on a custom field orders the values as text. Contacts that do not have th The **Email provider** column shows who hosts each contact's inbox: Google Workspace, Gmail, Microsoft 365, Outlook.com, Yahoo Mail, iCloud Mail, Zoho Mail and the other hosts Warmbly recognises, with the provider's logo. It works for company domains too: `dana@acme.com` reads as Google Workspace when Acme's mail is hosted by Google. Warmbly reads it from the domain's mail records in the background, usually within a minute of a contact being added, and fills the list in live. A dash means the check has not reached the contact yet, or the domain has no mail server at all. -It is part of the standard layout on the contacts page (if you have already arranged your own columns, add it from **Columns**) and one tick away in the column chooser on a campaign's Leads tab. From there you can sort by it, filter with **Add filter** > **Email provider** (including **Not detected yet**), select every match and act on them in bulk (add them to a campaign, tag them, export them), or save the filter as a segment. Campaign [ESP matching](/guides/campaigns/) uses the same data to pair each lead with a same-provider mailbox. +It is part of the standard layout on the contacts page (if you have already arranged your own columns, add it from **Columns**) and one tick away in the column chooser on a campaign's Leads tab. From there you can sort by it, filter with **Add filter** > **Email provider** (including **Unknown or not checked yet**), select every match and act on them in bulk (add them to a campaign, tag them, export them), or save the filter as a segment. Campaign [ESP matching](/guides/campaigns/) uses the same data to pair each lead with a same-provider mailbox. ## Selecting rows diff --git a/docs/public/openapi.json b/docs/public/openapi.json index 404c5afbf..a52430c98 100644 --- a/docs/public/openapi.json +++ b/docs/public/openapi.json @@ -29613,7 +29613,7 @@ "items": { "type": "string" }, - "description": "Contacts whose mail_host is any of these; \"\" matches contacts whose provider is not detected yet. An unknown value is rejected with 400 invalid_mail_host." + "description": "Contacts whose mail_host is any of these; \"\" matches contacts with no known provider (not checked yet, or a domain with no mail server). An unknown value is rejected with 400 invalid_mail_host." }, "subscribed": { "type": "boolean" @@ -29636,7 +29636,7 @@ }, "sort_by": { "type": "string", - "description": "Sort column: created_at (the default), updated_at, first_name, last_name, email, company, phone, campaign_count, or mail_host (contacts whose provider is not detected yet grouped last when ascending and first when descending). custom: sorts on a custom field as text, with contacts lacking the field grouped last when ascending and first when descending; a custom-field name that could never exist is rejected with 400 invalid_sort_by. Any other unknown value falls back to created_at." + "description": "Sort column: created_at (the default), updated_at, first_name, last_name, email, company, phone, campaign_count, or mail_host (contacts with no known provider grouped last when ascending and first when descending). custom: sorts on a custom field as text, with contacts lacking the field grouped last when ascending and first when descending; a custom-field name that could never exist is rejected with 400 invalid_sort_by. Any other unknown value falls back to created_at." }, "reverse": { "type": "boolean", diff --git a/internal/jobs/contact_mail_host.go b/internal/jobs/contact_mail_host.go index 8d9875d14..e67294306 100644 --- a/internal/jobs/contact_mail_host.go +++ b/internal/jobs/contact_mail_host.go @@ -6,6 +6,7 @@ import ( "time" "github.com/google/uuid" + "github.com/redis/go-redis/v9" "github.com/rs/zerolog/log" "github.com/warmbly/warmbly/internal/config" @@ -20,8 +21,14 @@ const ( contactMailHostMaxPasses = 50 contactMailHostLease = "lock:contactmailhost" contactMailHostLeaseTTL = 10 * time.Minute + // contactMailHostRunBudget ends a run while the lease still has room for + // the lookups already in flight, so two replicas never sweep at once. + contactMailHostRunBudget = contactMailHostLeaseTTL - 2*time.Minute ) +// releaseContactMailHostLease deletes the lease only while this run still holds it. +var releaseContactMailHostLease = redis.NewScript(`if redis.call("GET", KEYS[1]) == ARGV[1] then return redis.call("DEL", KEYS[1]) end return 0`) + // ContactMailHostStore is the slice of the contact repository the sweep uses. type ContactMailHostStore interface { ListMailHostPending(ctx context.Context, limit int) ([]repository.ContactMailHostPending, error) @@ -55,13 +62,15 @@ type cachedMailHost struct { func (j *ContactMailHostSweep) Run(ctx context.Context) error { // One replica sweeps at a time; a Redis outage fails open. if j.cache != nil { - if got, err := j.cache.SetNX(ctx, contactMailHostLease, "1", contactMailHostLeaseTTL).Result(); err == nil { + token := uuid.NewString() + if got, err := j.cache.SetNX(ctx, contactMailHostLease, token, contactMailHostLeaseTTL).Result(); err == nil { if !got { return nil } - defer j.cache.Del(context.WithoutCancel(ctx), contactMailHostLease) + defer releaseContactMailHostLease.Run(context.WithoutCancel(ctx), j.cache, []string{contactMailHostLease}, token) } } + deadline := time.Now().Add(contactMailHostRunBudget) for range contactMailHostMaxPasses { pending, err := j.contacts.ListMailHostPending(ctx, config.ContactMailHostBatchSize) if err != nil { @@ -70,10 +79,14 @@ func (j *ContactMailHostSweep) Run(ctx context.Context) error { if len(pending) == 0 { return nil } - hosts := j.resolveDomains(ctx, pending) + hosts, skipped := j.resolveDomains(ctx, pending, deadline) results := make([]repository.ContactMailHostResult, 0, len(pending)) for _, p := range pending { - r, ok := hosts[mailhost.NormalizeDomain(p.Email)] + d := mailhost.NormalizeDomain(p.Email) + if skipped[d] { + continue + } + r, ok := hosts[d] res := repository.ContactMailHostResult{ID: p.ID, Email: p.Email, Transient: !ok} if ok { res.MailHost, res.ESP = string(r), mailhost.ESPFamily(r) @@ -89,7 +102,7 @@ func (j *ContactMailHostSweep) Run(ctx context.Context) error { j.reloader.PublishOrgContactsReload(ctx, org.String(), "contacts:mail_host") } } - if len(pending) < config.ContactMailHostBatchSize || ctx.Err() != nil { + if len(pending) < config.ContactMailHostBatchSize || len(skipped) > 0 || ctx.Err() != nil { return ctx.Err() } } @@ -97,9 +110,10 @@ func (j *ContactMailHostSweep) Run(ctx context.Context) error { } // resolveDomains answers each distinct domain once. A domain missing from the -// result failed transiently. -func (j *ContactMailHostSweep) resolveDomains(ctx context.Context, pending []repository.ContactMailHostPending) map[string]mailhost.Host { +// result failed transiently; one in skipped was not asked before the deadline. +func (j *ContactMailHostSweep) resolveDomains(ctx context.Context, pending []repository.ContactMailHostPending, deadline time.Time) (map[string]mailhost.Host, map[string]bool) { out := map[string]mailhost.Host{} + skipped := map[string]bool{} var todo []string for _, p := range pending { d := mailhost.NormalizeDomain(p.Email) @@ -110,6 +124,10 @@ func (j *ContactMailHostSweep) resolveDomains(ctx context.Context, pending []rep if d == "" { continue } + if h := mailhost.KnownDomain(d); h != mailhost.Unknown { + out[d] = h + continue + } var hit cachedMailHost if j.cache != nil && j.cache.GetJSON(ctx, contactMailHostKey(d), &hit) == nil { out[d] = mailhost.Host(hit.Host) @@ -122,8 +140,13 @@ func (j *ContactMailHostSweep) resolveDomains(ctx context.Context, pending []rep var wg sync.WaitGroup sem := make(chan struct{}, config.ContactMailHostConcurrency) for _, d := range todo { - wg.Add(1) sem <- struct{}{} + if time.Now().After(deadline) { + <-sem + skipped[d] = true + continue + } + wg.Add(1) go func(d string) { defer wg.Done() defer func() { <-sem }() @@ -144,7 +167,10 @@ func (j *ContactMailHostSweep) resolveDomains(ctx context.Context, pending []rep }(d) } wg.Wait() - return out + for d := range skipped { + delete(out, d) + } + return out, skipped } func contactMailHostKey(domain string) string { return "contactmailhost:" + domain } diff --git a/internal/jobs/contact_mail_host_test.go b/internal/jobs/contact_mail_host_test.go index fb89d5183..40b7c07ea 100644 --- a/internal/jobs/contact_mail_host_test.go +++ b/internal/jobs/contact_mail_host_test.go @@ -5,6 +5,7 @@ import ( "net" "sync" "testing" + "time" "github.com/google/uuid" @@ -95,3 +96,23 @@ func TestContactMailHostSweep(t *testing.T) { t.Errorf("a consumer domain dialled DNS") } } + +func TestContactMailHostSweepSkipsPastDeadline(t *testing.T) { + dns := &mailHostDNS{mx: map[string][]string{"acme.example": {"aspmx.l.google.com"}}, calls: map[string]int{}} + j := NewContactMailHostSweep(&mailHostStore{}, nil, nil) + j.resolver = dns + pending := []repository.ContactMailHostPending{ + {ID: uuid.New(), Email: "a@acme.example"}, + {ID: uuid.New(), Email: "dana@gmail.com"}, + } + hosts, skipped := j.resolveDomains(context.Background(), pending, time.Now().Add(-time.Second)) + if !skipped["acme.example"] || dns.calls["acme.example"] != 0 { + t.Fatalf("a lookup past the deadline ran: skipped=%v calls=%v", skipped, dns.calls) + } + if _, ok := hosts["acme.example"]; ok { + t.Fatal("a skipped domain has an answer") + } + if hosts["gmail.com"] != "gmail" { + t.Fatalf("a known domain needs no lookup and is never skipped: %v", hosts) + } +} diff --git a/internal/models/contact.go b/internal/models/contact.go index 92893fc07..0fb7d54f4 100644 --- a/internal/models/contact.go +++ b/internal/models/contact.go @@ -241,7 +241,8 @@ type CampaignLeadCounts struct { Providers CampaignLeadProviderCounts `json:"providers"` } -// CampaignLeadProviderCounts are a campaign's leads by esp_provider family; +// CampaignLeadProviderCounts are a campaign's leads by esp_provider family. +// Other includes checked domains with no known host, which match like other; // Undetected counts the leads the provider check has not reached yet. type CampaignLeadProviderCounts struct { Google int `json:"gmail"` diff --git a/internal/pkg/mailhost/recipient.go b/internal/pkg/mailhost/recipient.go index 14ca80a94..87a7b1256 100644 --- a/internal/pkg/mailhost/recipient.go +++ b/internal/pkg/mailhost/recipient.go @@ -24,7 +24,7 @@ func Recipient(ctx context.Context, r RecipientResolver, domain string) (Host, b if norm == "" { return Unknown, true } - if h, _, ok := known(norm); ok { + if h := recipientKnown(norm); h != Unknown { return h, true } if r == nil { @@ -53,8 +53,19 @@ func Recipient(ctx context.Context, r RecipientResolver, domain string) (Host, b // KnownDomain names the host of a consumer mail domain, or of an address's // domain, from the built-in list with no DNS; Unknown for anything else. func KnownDomain(s string) Host { - h, _ := knownDomain(NormalizeDomain(s)) - return h + return recipientKnown(NormalizeDomain(s)) +} + +// recipientKnown is knownDomain plus Microsoft 365 tenant domains, which are +// Microsoft's mail but not a consumer service, so SharedProvider leaves them out. +func recipientKnown(domain string) Host { + if h, ok := knownDomain(domain); ok { + return h + } + if strings.HasSuffix(domain, ".onmicrosoft.com") { + return Microsoft365 + } + return Unknown } // ESPFamily is h's family as campaign ESP matching and contacts.esp_provider diff --git a/internal/pkg/mailhost/recipient_test.go b/internal/pkg/mailhost/recipient_test.go index c8544c215..b9f28225d 100644 --- a/internal/pkg/mailhost/recipient_test.go +++ b/internal/pkg/mailhost/recipient_test.go @@ -48,6 +48,7 @@ func TestRecipient(t *testing.T) { {"gmail.com", Gmail, true}, {"hotmail.co.uk", Outlook, true}, {"icloud.com", ICloud, true}, + {"contoso.onmicrosoft.com", Microsoft365, true}, {"yahoo.com", Yahoo, true}, {"acme.example", GoogleWorkspace, true}, {"contoso.example", Microsoft365, true}, @@ -76,3 +77,12 @@ func TestRecipientKnownDomainSkipsDNS(t *testing.T) { t.Fatalf("known domain dialled DNS %d times", f.mxCalls) } } + +func TestKnownDomainTenant(t *testing.T) { + if got := KnownDomain("ops@contoso.onmicrosoft.com"); got != Microsoft365 { + t.Fatalf("tenant domain = %q", got) + } + if SharedProvider("contoso.onmicrosoft.com") { + t.Fatal("a tenant domain is not a consumer provider") + } +} diff --git a/internal/repository/contact_mail_host_live_test.go b/internal/repository/contact_mail_host_live_test.go index fbadc8a05..84ec9fa13 100644 --- a/internal/repository/contact_mail_host_live_test.go +++ b/internal/repository/contact_mail_host_live_test.go @@ -124,8 +124,9 @@ func TestLiveContactMailHostSweepWritesAndSearchReads(t *testing.T) { if xerr != nil { t.Fatalf("lead counts: %v", xerr) } - if p := counts.Providers; p.Google != 1 || p.Microsoft != 0 || p.Other != 0 || p.Undetected != 3 { - t.Fatalf("lead counts by provider = %+v, want 1 Google and 3 undetected", p) + // The failed lookup was checked, so it counts with other; two were never reached. + if p := counts.Providers; p.Google != 1 || p.Microsoft != 0 || p.Other != 1 || p.Undetected != 2 { + t.Fatalf("lead counts by provider = %+v, want 1 Google, 1 other and 2 undetected", p) } for _, reverse := range []bool{true, false} { diff --git a/internal/repository/pg_contact.go b/internal/repository/pg_contact.go index bdfc7efb0..715591843 100644 --- a/internal/repository/pg_contact.go +++ b/internal/repository/pg_contact.go @@ -2185,8 +2185,8 @@ func (r *contactRepository) CampaignLeadCounts(ctx context.Context, orgID, campa COUNT(*) FILTER (WHERE COALESCE(pr.has_replied, false)) AS replied_any, COUNT(*) FILTER (WHERE c.esp_provider = 'gmail') AS provider_gmail, COUNT(*) FILTER (WHERE c.esp_provider = 'outlook') AS provider_outlook, - COUNT(*) FILTER (WHERE c.esp_provider = 'other') AS provider_other, - COUNT(*) FILTER (WHERE c.esp_provider = '') AS provider_undetected + COUNT(*) FILTER (WHERE c.esp_provider = 'other' OR (c.esp_provider = '' AND c.esp_resolved_at IS NOT NULL)) AS provider_other, + COUNT(*) FILTER (WHERE c.esp_provider = '' AND c.esp_resolved_at IS NULL) AS provider_undetected FROM campaign_leads cl JOIN contacts c ON c.id = cl.contact_id AND c.organization_id = $2 CROSS JOIN (SELECT COUNT(*) AS total_steps FROM sequences st WHERE st.campaign_id = $1 AND st.kind = 'email') ts diff --git a/web/src/components/app/contacts/filters/FilterBar.tsx b/web/src/components/app/contacts/filters/FilterBar.tsx index 4842aef1d..427d326a5 100644 --- a/web/src/components/app/contacts/filters/FilterBar.tsx +++ b/web/src/components/app/contacts/filters/FilterBar.tsx @@ -76,7 +76,7 @@ const PROVIDERS: Option[] = [ label, icon: , })), - { id: "", label: "Not detected yet" }, + { id: "", label: "Unknown or not checked yet" }, ]; // Optional pills, shown once added from the menu or when their value is set. diff --git a/web/src/components/app/segments/filtersToSegment.ts b/web/src/components/app/segments/filtersToSegment.ts index d4c278714..fbc387df0 100644 --- a/web/src/components/app/segments/filtersToSegment.ts +++ b/web/src/components/app/segments/filtersToSegment.ts @@ -43,9 +43,10 @@ export function filtersToSegment(f: SearchContacts, campaignID?: string): Segmen conditions.push({ field: "subscribed", operator: f.subscribed ? "is_true" : "is_false" }); } if (f.verification_status) conditions.push({ field: "verification_status", operator: "in", values: [f.verification_status] }); - // A segment enum has no empty value, so "not detected yet" does not carry over. + // A segment enum has no empty value, so "unknown" is reported rather than lost. const hosts = (f.mail_hosts ?? []).filter(Boolean); if (hosts.length > 0) conditions.push({ field: "mail_host", operator: "in", values: hosts }); + if (f.mail_hosts?.includes("")) dropped.push("unknown email provider"); if (f.min_campaigns !== undefined) conditions.push({ field: "campaign_count", operator: "gte", value: String(f.min_campaigns) }); if (f.max_campaigns !== undefined) conditions.push({ field: "campaign_count", operator: "lte", value: String(f.max_campaigns) }); if (f.created_after) conditions.push({ field: "created_at", operator: "after", value: isoDate(f.created_after) }); diff --git a/web/src/lib/api/models/app/contacts/SearchContacts.ts b/web/src/lib/api/models/app/contacts/SearchContacts.ts index d8674638b..2f7e47321 100644 --- a/web/src/lib/api/models/app/contacts/SearchContacts.ts +++ b/web/src/lib/api/models/app/contacts/SearchContacts.ts @@ -18,7 +18,7 @@ export default interface SearchContacts { max_campaigns?: number; subscribed?: boolean; verification_status?: VerificationStatus; - // Contacts whose inbox is hosted by any of these; "" matches not detected yet. + // Contacts whose inbox is hosted by any of these; "" matches no known provider. mail_hosts?: MailHost[]; created_after?: Date; created_before?: Date; diff --git a/web/src/lib/api/models/app/contacts/SearchContactsResult.ts b/web/src/lib/api/models/app/contacts/SearchContactsResult.ts index d5eb93ce3..b33c99097 100644 --- a/web/src/lib/api/models/app/contacts/SearchContactsResult.ts +++ b/web/src/lib/api/models/app/contacts/SearchContactsResult.ts @@ -60,6 +60,7 @@ export interface CampaignLeadCounts { export interface CampaignLeadProviderCounts { gmail: number; outlook: number; + // Includes checked domains with no known provider; they match like other. other: number; // Leads whose provider the background check has not read yet. undetected: number;