Merge pull request #269 from warmbly/feature/effortless-issue-264-proposal

Address verification overhaul: MillionVerifier plugin, imported results, campaigns never stop on verification (#264)
This commit is contained in:
Matthew Meszaros
2026-08-30 00:29:06 -07:00
committed by GitHub
93 changed files with 4214 additions and 184 deletions
@@ -56,6 +56,7 @@ const STATUS_TONE: Record<string, string> = {
paused_trial_expired: "border-orange-300 text-orange-700 bg-orange-50",
paused_no_accounts: "border-orange-300 text-orange-700 bg-orange-50",
paused_guardrail: "border-rose-300 text-rose-700 bg-rose-50",
paused_undeliverable: "border-amber-300 text-amber-700 bg-amber-50",
};
const STATUS_OPTIONS = [
@@ -67,6 +68,7 @@ const STATUS_OPTIONS = [
{ value: "paused_trial_expired", label: "Paused — trial expired" },
{ value: "paused_no_accounts", label: "Paused — no accounts" },
{ value: "paused_guardrail", label: "Paused — guardrail" },
{ value: "paused_undeliverable", label: "Paused — undeliverable leads" },
];
const pct = (n: number, d: number) => (d ? ((n / d) * 100).toFixed(1) : "—");
+1 -1
View File
@@ -522,7 +522,7 @@ export interface AdminCampaignSearch {
q?: string;
user_id?: string;
org_id?: string;
status?: string; // "" | draft | active | paused | completed | paused_trial_expired | paused_no_accounts | paused_guardrail
status?: string; // "" | draft | active | paused | completed | paused_trial_expired | paused_no_accounts | paused_guardrail | paused_undeliverable
// Boolean flags
open_tracking?: boolean;
link_tracking?: boolean;
+36 -13
View File
@@ -1269,6 +1269,10 @@ func main() {
if aware, ok := campaignService.(campaign.AudienceAware); ok {
aware.WireAudience(campaignAudienceRepository)
}
// A start with nothing left to send says whether verification is why.
if aware, ok := campaignService.(campaign.ProgressAware); ok {
aware.WireProgress(campaignProgressRepository)
}
// Delete drops attachment objects and duplicate copies them, so the
// campaign service needs the store the attachment handler writes to.
if aware, ok := campaignService.(campaign.AttachmentAware); ok {
@@ -1632,23 +1636,42 @@ func main() {
warmupBatchPoller := jobs.NewWarmupBatchPoller(warmupContentService, 5*time.Minute)
go warmupBatchPoller.Start(ctx)
// Pre-send email verification: verify a capped batch of not-yet-checked
// contacts each tick so hard-bouncing addresses are dropped before any
// worker sends. CONTROL-PLANE ONLY — the SMTP RCPT probe dials remote MX
// on :25 from this backend host (a non-sending IP), never a worker.
// The HELO name must be a real, public FQDN: servers reject a bare or
// reserved greeting, and Postfix reports that rejection on RCPT, where
// the prober used to read it as a dead mailbox (issue #200). APP_URL's
// host is the instance's own public name, so it is the right fallback;
// when neither is usable the verifier declines to probe instead of
// inventing verdicts.
// Pre-send verification runs here, never on a worker (a sending IP).
// The HELO host must be a public FQDN or the probe declines to run.
emailVerifier := emailverify.New(emailverify.Config{
HeloHost: emailVerifyHeloHost(), // e.g. verify.warmbly.com
MailFrom: os.Getenv("EMAIL_VERIFY_MAIL_FROM"), // e.g. verify@warmbly.com
})
emailVerifyService = emailverifyapp.NewService(contactRepostory, emailVerifier)
emailVerificationJob := jobs.NewEmailVerificationJob(emailVerifyService, 100)
emailVerificationScheduler := jobs.NewEmailVerificationScheduler(emailVerificationJob, 15*time.Minute)
// Org key first, operator key second, built-in check last.
emailVerifyService = emailverifyapp.NewService(contactRepostory, emailverifyapp.Options{
Builtin: emailVerifier,
BuiltinReady: emailVerifier.ProbeReady(),
Providers: integrationServiceForHandler,
PlatformMillionVerifierKey: os.Getenv("EMAIL_VERIFY_MILLIONVERIFIER_API_KEY"),
})
verificationEvidence := emailverifyapp.NewEvidence(repository.NewVerificationEvidenceRepository(primaryDB))
emailVerifyService.SetEvidence(verificationEvidence)
if aware, ok := contactService.(contact.VerificationAware); ok {
aware.WireVerification(emailVerifyService)
}
if advancedService != nil {
if aware, ok := advancedService.(advanced.EvidenceAware); ok {
aware.WireEvidence(verificationEvidence)
}
}
go jobs.NewDeliveryEvidenceJob(verificationEvidence, 15*time.Minute, 2000).Start(ctx)
emailVerifyService.SetVerdictHook(func(ctx context.Context, orgID uuid.UUID) {
if campaignService != nil {
campaignService.ResumeVerificationPaused(ctx, orgID)
}
// Verdicts land outside any request; feed the audit spine by hand.
if streamingPublisher != nil {
streamingPublisher.PublishAuditCreated(ctx, orgID, uuid.Nil, "verify", string(models.AuditEntityContact), nil)
streamingPublisher.PublishAuditCreated(ctx, orgID, uuid.Nil, "verify", string(models.AuditEntityCampaign), nil)
}
})
emailVerificationJob := jobs.NewEmailVerificationJob(emailVerifyService, config.VerificationBatchSize)
emailVerificationScheduler := jobs.NewEmailVerificationScheduler(emailVerificationJob, time.Duration(config.VerificationIntervalSeconds)*time.Second)
go emailVerificationScheduler.Start(ctx)
// Seed inbox-placement testing: send a tokenized copy of a template
+9
View File
@@ -3,6 +3,7 @@ package main
import (
"context"
"github.com/warmbly/warmbly/internal/app/cloudlink"
emailverifyapp "github.com/warmbly/warmbly/internal/app/emailverify"
"log"
"os"
"os/signal"
@@ -292,6 +293,12 @@ func main() {
warmupService,
)
advancedService.WireDispatcher(webhookService)
// Replies, bounces, opens and clicks teach verification what real mail
// showed about each address.
verificationEvidence := emailverifyapp.NewEvidence(repository.NewVerificationEvidenceRepository(primaryDB))
if aware, ok := advancedService.(advanced.EvidenceAware); ok {
aware.WireEvidence(verificationEvidence)
}
// Reply/open/click instant action chains run in THIS process (inbox ingest +
// tracking consumer), so a "run_automation" node on an instant branch must be
// able to launch the flow here too. Without this it would be stamped sent and
@@ -392,6 +399,7 @@ func main() {
CampaignProgressRepo: campaignProgressRepo,
CampaignLogRepo: repository.NewCampaignLogRepository(primaryDB),
ContactRepo: contactRepo,
Evidence: verificationEvidence,
}
jobsService.InitEvents()
@@ -470,6 +478,7 @@ func main() {
streamingPublisher,
repository.NewTrackingDedupeRepository(primaryDB.Pool),
advancedService,
verificationEvidence,
); terr != nil {
log.Println("tracking consumer unavailable; opens/clicks not consumed:", terr)
} else {
+2
View File
@@ -76,6 +76,8 @@ All paths below are relative to the versioned base URL `https://api.warmbly.com/
| POST | `/contacts` | `WRITE_CONTACTS` |
| DELETE | `/contacts` | `BULK_CONTACTS` |
| PATCH | `/contacts` | `BULK_CONTACTS` |
| GET | `/contacts/verification` | `READ_CONTACTS` |
| POST | `/contacts/verification` | `BULK_CONTACTS` |
| GET | `/contacts/:id` | `READ_CONTACTS` |
| PATCH | `/contacts/:id` | `WRITE_CONTACTS` |
| DELETE | `/contacts/:id` | `WRITE_CONTACTS` |
+6
View File
@@ -78,6 +78,12 @@ Returned when the request cannot be processed due to invalid syntax.
| `invalid_lead_status` | `POST /contacts/search` or `POST /contacts/export` was given a `lead_status` that is not one of the documented values |
| `invalid_engagement` | `POST /contacts/search` or `POST /contacts/export` was given an `engagement` that is not one of the documented values |
| `lead_filter_requires_campaign` | `lead_status` or `engagement` was set without exactly one `campaign_ids` entry; both filters describe a contact inside one campaign |
| `unknown_verification_status` | A contact's `verification_status` is not a value any known verification service writes |
| `unknown_verification_provider` | A contact's `verification_provider` names a vocabulary the platform cannot read |
| `invalid_action` | `POST /contacts/verification` was given an `action` other than `verify`, `mark_deliverable` or `mark_undeliverable` |
| `no_contacts` | `POST /contacts/verification` selected no contacts: neither `contacts` nor a `campaign_id` with refused leads |
| `list_bounce_risk` | `POST /campaigns/:id/start` refused the launch on the list's projected bounce rate. Clean or verify the list, or repeat the request with `acknowledge_list_risk: true` |
| `leads_undeliverable` | `POST /campaigns/:id/start` found nothing to send because address verification refused every remaining lead; the campaign is parked at `paused_undeliverable` until they are re-verified or marked deliverable |
| `no_organization` | The request needs a workspace and the caller has none selected. Every entitlement, limit and suppression rule is scoped to a workspace, so a write that would run unscoped is refused rather than run without those checks. API keys always carry their workspace; a dashboard session picks one at sign-in, so this normally means the session predates the workspace being chosen. Select a workspace and retry |
### 401 Unauthorized
@@ -658,6 +658,9 @@ Start (activate) the campaign so it begins sending real mail. Works from `draft`
| Parameter | In | Type | Description |
| --- | --- | --- | --- |
| `id` | path | uuid | Campaign id. |
| `acknowledge_list_risk` | body | boolean | Optional. Launch even though the list's projected bounce rate would be refused (`list_bounce_risk`), for a list verified elsewhere. |
A start refused because every remaining lead was refused by address verification answers `leads_undeliverable` and parks the campaign at `paused_undeliverable`; re-verify the leads or mark them deliverable with [`POST /contacts/verification`](/api/reference/contacts/#verify-or-override-contacts), which resumes it.
### Response
+54 -3
View File
@@ -31,6 +31,7 @@ Every field is optional; an empty body matches all contacts in the organization.
| `lead_status` | string | No | Filter to one derived lead status: `pending`, `active`, `completed`, `replied`, `bounced`, `failed`, `undeliverable`, or `unsubscribed`. Requires exactly one `campaign_ids` entry, otherwise the request is rejected with `lead_filter_requires_campaign`; an unknown value is rejected with `invalid_lead_status`. |
| `engagement` | string | No | Filter by engagement inside that campaign: `opened`, `not_opened`, `clicked`, `not_clicked`, `replied`, `not_replied`, or `bounced`. `opened` means a human open (machine opens never count); the `not_*` values match only leads sent at least one step. Combines with `lead_status` as AND. Requires exactly one `campaign_ids` entry (`lead_filter_requires_campaign`); an unknown value is rejected with `invalid_engagement`. |
| `category_ids` | string[] | No | Contact must have ALL of these categories. |
| `verification_status` | string | No | Filter by verification verdict: `valid`, `risky`, `invalid`, or `unknown`. |
| `min_campaigns` | integer | No | Minimum number of associated campaigns. |
| `max_campaigns` | integer | No | Maximum number of associated campaigns. |
| `subscribed` | boolean | No | Filter by subscription status. |
@@ -71,7 +72,11 @@ Returns a `data` array of contacts plus a `pagination` envelope.
"campaigns": [{ "id": "c1...", "name": "Q3 Outbound" }],
"categories": [{ "id": "6f1c...", "title": "VIP", "color": "#0ea5e9" }],
"verification_status": "valid",
"verification_reason": "",
"verification_reason": "recipient accepted",
"verification_sub_status": "",
"verification_source": "probe",
"verification_provider": "builtin",
"verification_checked_at": "2026-06-10T11:58:00Z",
"is_catch_all": false,
"esp_provider": "gmail",
"updated_at": "2026-06-10T12:00:00Z",
@@ -86,6 +91,8 @@ Returns a `data` array of contacts plus a `pagination` envelope.
}
```
Every contact carries its address verification: `verification_status` (`valid`, `risky`, `invalid`, or `unknown`), `verification_sub_status` (`catch_all`, `disposable`, `role`, `spamtrap`, `mailbox_full`, `no_mx`, `syntax`, `undisclosed`, or empty), `verification_source` (`probe` for the built-in check, `provider` for a connected verification service, `imported` for a verdict that came with the contact, `manual` for one a member set, empty when never checked), `verification_provider` (who produced it), `verification_reason`, `verification_checked_at`, and `verification_confidence` (0 to 100, scored from the check plus what real mail to the address showed; see [what real mail teaches the check](/guides/deliverability/#what-real-mail-teaches-the-check)). Campaigns never send to `invalid`, and send to `risky` only when their `risky_emails` setting is on.
When the search filters by exactly one campaign, each contact additionally carries a `campaign_lead` object with its processing state inside that campaign (`status`, `sent`, `opened`, `machine_opened`, `clicked`, `replied`, `bounced`, `current_step`, `last_activity_at`, and `failure_reason` when failed). `opened` counts steps opened by a person; steps fetched automatically by a mail client (Apple Mail Privacy Protection and similar) are in `machine_opened` instead, matching the machine opens the analytics summary reports. The `status` derivation, highest priority first, is `unsubscribed` (not subscribed), then `bounced`, `replied`, `failed` (a step could not be sent after every retry; `failure_reason` carries the sending worker's reason), `completed` (every email step sent, no reply), `active` (some steps sent, more to send), `undeliverable` (pre-send verification refused the address, so the campaign skips the lead and never sends to it), and `pending` (queued, nothing sent). A step counts as sent only once the sending worker has delivered it to the mailbox provider; a send the worker could not complete is retried on the campaign's next pass and never shows as sent. The `lead_status` filter narrows to one of these buckets.
When the search filters by exactly one campaign, the first page (no `cursor`) also includes a `lead_counts` object: per-status lead totals for that campaign, independent of the `lead_status` and `engagement` filters so every scope's total is available at once. Alongside the status buckets it carries engagement totals that match the `engagement` filter: `contacted` (leads sent at least one step), `opened` (a human open on any step), `clicked`, and `replied_any` (a reply on any step, whatever the derived status).
@@ -145,6 +152,8 @@ A JSON array of contact objects (at least one, up to the per-request maximum; an
| `categories` | string[] | No | Category IDs to assign. |
| `custom_fields` | object | No | String key/value custom fields. Keys may use letters, numbers, underscores, spaces, and dashes. |
| `subscribed` | boolean | No | Marketing-consent flag. Omit it to let a new contact default to subscribed and an existing one keep whatever it already had. |
| `verification_status` | string | No | A verdict you already hold for the address, in Warmbly's vocabulary (`valid`, `risky`, `invalid`, `unknown`) or any known service's (`ok`, `catch-all`, `do_not_mail`, `deliverable`, `ok_for_all`, ...). Stored as an imported verdict that the background check leaves alone. A value no known service writes is rejected with `unknown_verification_status`. |
| `verification_provider` | string | No | The vocabulary `verification_status` is written in: `zerobounce`, `millionverifier`, `neverbounce`, `bouncer`, `kickbox`, `emailable`, `debounce`, `clearout`, `emaillistverify`, or `warmbly`. Omit it to have the value recognised by itself. An unknown name is rejected with `unknown_verification_provider`. |
An address you already have is matched (lowercased) and enriched rather than duplicated: fields you send replace what is stored, fields you omit or send empty are left alone, and `custom_fields` is merged key by key. Use `PATCH /contacts/:id` to clear a value.
@@ -351,7 +360,7 @@ Send `multipart/form-data` with a `file` field and an `options` field containing
| Field | Type | Required | Description |
| --- | --- | --- | --- |
| `mapping` | array | Yes | Column mappings: `{ "index", "target", "custom_key" }`. `target` is `ignore`, `email`, `first_name`, `last_name`, `company`, `phone`, `subscribed`, `categories`, or `custom` with the name in `custom_key`. `custom:<key>` is still accepted as the older spelling. Exactly one column must map to `email`. |
| `mapping` | array | Yes | Column mappings: `{ "index", "target", "custom_key", "verification_provider" }`. `target` is `ignore`, `email`, `first_name`, `last_name`, `company`, `phone`, `subscribed`, `categories`, `verification_status`, or `custom` with the name in `custom_key`. `custom:<key>` is still accepted as the older spelling. Exactly one column must map to `email`. A `verification_status` column is read in the vocabulary named by `verification_provider` (see [Create contacts](#create-contacts)), or recognised value by value when it is omitted; a cell nobody recognises leaves that contact unverified rather than failing the row. The preview suggests this target itself when a column's header or values look like another service's results. |
| `dedup` | string | Yes | `skip`, `update`, or `create_duplicate` for rows whose email matches an existing contact. |
| `has_header` | boolean | Yes | Whether the first row is a header. |
| `category_ids` | string[] | No | Categories to assign to imported contacts. |
@@ -402,6 +411,48 @@ Imports are capped at 50,000 rows. `errors` carries at most the first 1,000 entr
A custom-field name may use letters, numbers, underscores, spaces, and dashes (`Company Mobile`, `first-name`, `plan_tier`). Anything else is a `400` on the whole request, raised before any row is written, along with a mapping that names no `email` column or a `custom` column with no `custom_key`. Per-row `errors` are reserved for problems with the data itself.
## Verification overview
`GET /contacts/verification`
Reports which verifier checks this workspace's addresses and the contacts by verdict. **Scope** `READ_CONTACTS` · **Org permission** `view_contacts`.
### Response
```json
{
"provider": "millionverifier",
"connection_id": "9a1b...",
"credits": 48210,
"builtin_ready": true,
"counts": { "valid": 11240, "risky": 380, "invalid": 512, "unknown": 1890, "pending": 120 }
}
```
`provider` is `builtin` or `millionverifier`. `credits` is the connected service's remaining balance; `provider_error` is set instead when the service is connected but unusable (a rejected key, no credits), in which case the built-in check is in use. `builtin_ready` says whether the built-in mailbox probe can run on this instance. `pending` is the share of `unknown` nobody has checked yet.
## Verify or override contacts
`POST /contacts/verification`
Queues a fresh check of the listed contacts, or records a manual verdict on them. **Scope** `BULK_CONTACTS` · **Org permission** `manage_contacts`.
### Request body
| Field | Type | Required | Description |
| --- | --- | --- | --- |
| `action` | string | Yes | `verify` queues a re-check (each contact updates as its verdict lands); `mark_deliverable` records `valid`; `mark_undeliverable` records `invalid`. Manual verdicts are never re-checked automatically. Anything else is rejected with `invalid_action`. |
| `contacts` | string[] | No | Contact ids, up to the bulk maximum per request. |
| `campaign_id` | string | No | Instead of, or as well as, `contacts`: every lead of this campaign that verification refused. |
At least one contact must be selected (`no_contacts`). Marking leads deliverable resumes any campaign of the workspace that was paused for verification.
### Response
```json
{ "affected": 512, "action": "verify", "queued": true }
```
## Look up a contact by email
`GET /contacts/lookup`
@@ -439,7 +490,7 @@ Auth: **Scope** `READ_CONTACTS` · **Org permission** `view_contacts`
`GET /contacts/:id`
Returns the hydrated contact 360 payload: the contact plus an engagement summary and, when present, suppression state. Engagement and suppression counts are org-scoped; they are returned only when an organization is selected.
Returns the hydrated contact 360 payload: the contact plus an engagement summary, when present suppression state, and a `verification` object explaining the verdict: `status`, `confidence`, `reasons` (sentences, strongest first), `decisive` (true when real mail rather than a check decided the status), and `evidence`, the observations it was scored from, newest first, each `{ "kind", "detail", "observed_at" }` with `kind` one of `delivered`, `opened`, `clicked`, `replied`, `auto_replied`, `bounced_recipient`, `bounced_other`. Engagement and suppression counts are org-scoped; they are returned only when an organization is selected.
Auth: **Scope** `READ_CONTACTS` · **Org permission** `view_contacts`
@@ -217,11 +217,14 @@ Before a campaign sends to an address, the backend can check it: syntax, then `M
|---|---|---|---|
| `EMAIL_VERIFY_HELO_HOST` | The hostname the probe announces in `EHLO`/`HELO`. Must be a public, fully-qualified name that belongs to this instance | the host of `APP_URL` | yes |
| `EMAIL_VERIFY_MAIL_FROM` | The envelope sender the probe uses in `MAIL FROM` | `verify@` plus the `HELO` host | yes |
| `EMAIL_VERIFY_MILLIONVERIFIER_API_KEY` | An instance-wide [MillionVerifier](https://www.millionverifier.com/) key. Every workspace that has not connected its own key is checked through this one, spending its credits, instead of the built-in probe | unset | yes |
<Callout type="warn" title="An unqualified HELO name gets the whole session rejected">
Mail servers refuse a greeting that is not a real hostname (`localhost`, a bare name, or anything under a reserved suffix such as `.local`, `.internal` or `.lan`). Postfix in particular applies that rejection at `RCPT` time rather than at `HELO`, so it arrives looking exactly like `504 5.5.2 <localhost>: Helo command rejected: need fully-qualified hostname` on the recipient's address. Warmbly reads a reply like that as a rejected probe, not as a dead mailbox, so it never marks the contact invalid. If neither `EMAIL_VERIFY_HELO_HOST` nor a usable `APP_URL` host is set, the probe is skipped entirely and every address stays `unknown`, which still sends.
</Callout>
Verdicts are re-checked after 90 days (30 for an inconclusive one), in passes of 200 contacts a minute that repeat while a backlog remains. A workspace that connected MillionVerifier under Integrations is checked through its own credits whether or not the instance-wide key is set.
`EMAIL_VERIFY_HELO_HOST` is not `SMTP_EHLO_NAME`. `SMTP_EHLO_NAME` is the greeting your platform mail relay sees in [platform mail](#platform-mail); this one is the greeting recipients' servers see from the verifier.
## Encryption
+3 -1
View File
@@ -118,7 +118,7 @@ Optional **campaign dates** bound when it may send. Leave both blank to run open
The play and pause buttons work from the list row or the detail view. Starting moves the campaign to **active** and begins scheduling inside your windows and limits; pausing stops new scheduling immediately and resumes from where it left off.
A campaign can pause itself: **paused, no accounts** when it loses every sender, **paused, trial expired** when a trial ends, **auto-paused** when a guardrail trips, and plain **paused** if it ever loses its workspace, because unsubscribes, bounces and complaints are checked per workspace and cannot be honoured without one. It moves to **finished** once every contact completes the sequence or its end date passes. Configured to do so, it also stops following up with a contact the moment they reply.
A campaign can pause itself: **paused, no accounts** when it loses every sender, **paused, trial expired** when a trial ends, **auto-paused** when a guardrail trips, **needs verification** when address verification has refused every remaining lead (the campaign offers to re-verify them or send anyway; see [address verification](/guides/deliverability/#address-verification)), and plain **paused** if it ever loses its workspace, because unsubscribes, bounces and complaints are checked per workspace and cannot be honoured without one. It moves to **finished** once every contact completes the sequence or its end date passes. Configured to do so, it also stops following up with a contact the moment they reply.
A finished campaign can be started again: after extending or clearing its end date, or adding new leads, pressing play resumes it. If there is genuinely nothing left to send it finishes again immediately with a message saying so.
@@ -202,6 +202,8 @@ What it counts:
- **Deliverable recipients only.** Suppressed and unsubscribed leads are skipped at send time, so counting them would make a bad list look fine.
- **Lists of at least 50.** Below that a share means nothing, and no launch is refused on it.
A refused launch shows why and offers **Launch anyway**, for a list you verified with another service. Through the API the same override is `acknowledge_list_risk` on the start request.
Above `4%` projected bounce the launch is refused with the number and what to do about it. Between `2%` and `4%` it launches with a warning. The same projection appears in the launch dialog before you click, so a refusal is never a surprise.
## Safety posture
+1 -1
View File
@@ -43,7 +43,7 @@ The result step also reports what the addresses themselves look like: how many w
- **The import still happens.** These are your records, so nothing is refused here. A list bad enough to matter is stopped when you try to launch a campaign with it, which is where the damage would actually occur. See [the launch check](/guides/campaigns/).
- **Lists under 20 rows are not judged**, since a share of a handful of rows means nothing.
This is not address verification. It reads the addresses; verification asks the receiving server whether they exist, and runs separately in the background.
This is not address verification. It reads the addresses; verification asks the receiving server whether they exist, and runs separately in the background, within a minute of the import. A file that already carries verification results from another service (a `status` or `ZeroBounce Status` column, say) is recognised during mapping and offered as **Verification status**; those verdicts are kept and the background check skips them. See [address verification](/guides/deliverability/#address-verification).
## Custom fields
+29 -1
View File
@@ -98,7 +98,7 @@ If a campaign has already been paused because every one of its mailboxes was gat
**One-click unsubscribe** (RFC 8058) puts a native Unsubscribe control in the recipient's mail client. It drives complaint rate down, because someone who can opt out cleanly rarely hits "mark as spam" instead, and Google's bulk-sender guidance effectively requires it at volume. Unsubscribes are suppressed automatically.
**Pre-send verification** checks an address before sending and skips undeliverable ones, keeping bad addresses from becoming the hard bounces that drive the `5%` and `10%` thresholds above. A skipped lead shows as **Undeliverable** in the campaign's Leads view rather than sitting at Queued, and the campaign's activity log says how many it skipped when it finishes, so a campaign is never quietly held up by addresses it will not send to. Verification only records an address as invalid when the recipient's server rejects the address itself. A server that rejects the check for its own reasons (a policy block, a rate limit, or a greeting it does not accept) leaves the address unverified, and unverified addresses are still sent to. Self-hosted instances should set `EMAIL_VERIFY_HELO_HOST`, described in the [configuration reference](/development/configuration/#pre-send-verification).
**Pre-send verification** checks every address before a campaign sends to it and skips the undeliverable ones, keeping bad addresses from becoming the hard bounces that drive the `5%` and `10%` thresholds above. There is nothing to run: new contacts are checked in the background within a minute of being added, each contact carries its verdict (a small mark next to the email), and Settings > Sending shows the whole workspace at a glance. See [address verification](#address-verification) below for how verdicts are produced and what to do when one is wrong.
**Suppression** is the safety net: a suppressed recipient is never emailed again by any campaign. Warmbly suppresses automatically on a bounce, a spam complaint, or an unsubscribe, because once someone has bounced or complained the safest response is to stop, not to keep collecting negative signals.
@@ -114,6 +114,34 @@ Complaint reports are read from IMAP and Gmail mailboxes. Microsoft Graph return
**Domain authentication refusals** are separated from ordinary failures. When a receiving server refuses a message because your sending domain failed its checks (Microsoft's `5.7.515`, Gmail's `5.7.26`), that is not a bad recipient and not an outage: every send from that domain fails the same way until its DNS is fixed. Warmbly says so in the campaign's activity feed rather than blaming the address, does not suppress the recipient, and brings that domain's authentication check forward so the [domain authentication](/guides/deliverability/#domain-authentication) state reflects it.
## Address verification
Every address gets one of four verdicts. **Deliverable** and **unverified** are always sent to. **Undeliverable** is never sent to. **Risky** (a catch-all domain, a shared inbox such as `info@`) is sent to only when the campaign's **send to risky addresses** setting allows it.
Verdicts come from one of three places, and the source is shown on the contact:
- **The built-in check**, included on every plan: syntax, the domain's mail server, known disposable domains, and a mailbox probe against the recipient's server. It only records an address as undeliverable when that server rejects the address itself; a server that rejects the check for its own reasons (a policy block, a rate limit, a greeting it does not accept) leaves the address unverified. Microsoft 365 and Yahoo answer every probe the same way, so their addresses stay unverified rather than being called deliverable. The check also watches itself: when it starts rejecting an unusual share of addresses, its rejections are filed as unverified for an hour instead of being trusted, because a real list is never mostly dead. Self-hosted instances should set `EMAIL_VERIFY_HELO_HOST`, described in the [configuration reference](/development/configuration/#pre-send-verification).
- **MillionVerifier**, pay as you go: connect it once under Integrations with your own API key, and every check from then on uses your MillionVerifier credits (one per address) instead of the built-in probe. Its verdicts cover the catch-all and Microsoft cases the probe cannot. When the balance runs out or the key stops working, the connection is marked and the built-in check takes over until it is fixed. Nothing else changes.
- **Results you already have.** A list verified with another service (ZeroBounce, MillionVerifier, NeverBounce, Bouncer, Kickbox, Emailable, DeBounce, Clearout, EmailListVerify, or a Warmbly export) can bring its status column along. The import recognises the column from its header or its values, maps it to **Verification status**, and reads each service's own words (`catch-all`, `do_not_mail`, `ok_for_all`, and so on). Those verdicts are kept as they are; the built-in check leaves them alone. The same field is accepted when [creating contacts through the API](/api/reference/contacts/#create-contacts).
### What real mail teaches the check
A check sees an address once. The platform sees what happens after every send, and that evidence outranks any check:
- **Proof the mailbox is live**: a reply (an automatic one too, since a machine answered from it), a click, an open by a person (mail-client prefetches do not count), and a delivery the recipient's server kept without bouncing for three days. A reply, a click, a human open, or two clean deliveries make an address deliverable whatever a probe said about it.
- **Proof it is not**: a bounce whose text names the recipient (`5.1.1`, "user unknown"). A full mailbox, a policy block, or a reputation rejection says nothing about the address and is recorded as such.
- **Nothing at all**: a contact who never opens or replies. Silence is not evidence; there is no input for it and nothing in the ledger can say "did not open". A lead that ignored one campaign is exactly as deliverable as it was.
Every contact carries a **confidence** (0 to 100) next to its verdict, and the contact drawer's Deliverability card lists the reasons in plain words with the observations behind them. Newer evidence outranks older: a bounce after a reply makes the address undeliverable, a reply after a bounce makes it deliverable again, and old observations fade over months rather than switching off.
Verdicts age: an address is checked again after 90 days (30 for an inconclusive verdict), because mailboxes get created and closed, but one that real mail reached in the last six months is left alone. A verdict a teammate set by hand never expires and is never outvoted by evidence.
### When verification is wrong
Select the contacts and choose **Verify** in the selection bar: **Re-verify** queues a fresh check, and each row's mark updates as its verdict lands; **Mark deliverable** overrides the verdict for a list you know is good. Both are also available on a campaign's Leads tab.
A campaign never quietly finishes because of verification. When every remaining lead has been refused, the campaign shows **needs verification** and pauses, with two buttons on it: **Re-verify leads** checks them again and resumes sending automatically as soon as any pass, and **Send anyway** marks them deliverable and resumes at once. The [launch check](/guides/campaigns/#adding-leads) likewise offers **Launch anyway** when a list is refused for its projected bounce rate, for a list verified elsewhere.
## Workspace sending posture
Individual protections each watch one thing: a rate limit watches one user, warmup health watches one mailbox, verification watches one address. A workspace that is slightly wrong on several of those at once sits under every individual threshold and is never noticed. The workspace posture is the view that sees all of them together.
@@ -19,6 +19,7 @@ The Integrations page is a searchable directory with your existing connections a
| Slack | Notifications | one-click OAuth | Ping a channel on reply, bounce, or deliverability dips |
| Discord | Notifications | webhook URL | Ping a server channel on reply, bounce, or warmup health |
| Calendly, Cal.com | Meetings | minted inbound URL | Track booked, rescheduled, and canceled calls |
| MillionVerifier | Verification | API key | Check every contact's address through your pay-as-you-go credits instead of the built-in probe |
Three connect styles appear on the catalog cards: `one-click` OAuth, `api_key` for providers without an OAuth app, and `webhook` for a minted inbound URL or a pasted channel URL.
@@ -30,6 +31,10 @@ You must start the authorize flow; pasting credentials for an OAuth provider is
Zapier, Make, and n8n connect in one click with nothing stored on the integration. Warmbly sends events to the webhook URL you configure there; when they call back into Warmbly they use a scoped API key you create.
</Callout>
<Callout type="info" title="MillionVerifier is checked before it is saved">
The key is tested against your MillionVerifier account when you connect, so a mistyped key is refused rather than silently leaving contacts on the built-in check. From then on every new contact, and every re-verify, spends one credit there. When the balance runs out or the key is revoked, the connection is marked and the built-in check covers until it is fixed. See [address verification](/guides/deliverability/#address-verification).
</Callout>
**Credentials** (OAuth tokens, pasted keys, webhook URLs) are sealed with envelope encryption before touching the database. Only non-secret display details like an account name or Salesforce instance host are stored in the clear, so the dashboard can label the connection.
## CRM field mappings
@@ -23,6 +23,7 @@ The data is split into groups. Every export includes **Workspace**; the rest are
| Inbox | Unified inbox threads, message bodies, and mailbox sync state |
| Send history | Queued and completed send tasks with their payloads |
| Delivery events | Bounces, complaints, opens, clicks, placement tests, and website page views with the browser records that tie them to contacts |
| Verification evidence | What real mail showed about each contact's address (deliveries, opens, replies, bounces), so verdicts and confidence survive the move |
| Logs | Audit log, campaign logs, and notifications |
| Billing history | Subscription, credit ledger, and referral records |
+8 -1
View File
@@ -272,7 +272,14 @@ func (h *Handler) StartCampaign(c *gin.Context) {
id := c.Param("id")
if xerr := h.CampaignService.StartCampaign(c.Request.Context(), *orgID, id); xerr != nil {
// Optional body: {"acknowledge_list_risk": true} launches past the
// bounce-risk gate once the member has read the projection.
var opts models.StartCampaignOptions
if c.Request.ContentLength != 0 {
_ = c.ShouldBindJSON(&opts)
}
if xerr := h.CampaignService.StartCampaign(c.Request.Context(), *orgID, id, opts); xerr != nil {
errx.JSON(c, xerr)
return
}
+4
View File
@@ -51,6 +51,10 @@ func (h *Handler) AddContacts(c *gin.Context) {
// Audit log - bulk import
h.auditOrg(c, models.AuditActionImport, models.AuditEntityContact, nil, nil, map[string]string{"count": fmt.Sprintf("%d", len(data))})
// New addresses are checked right away rather than on the next tick.
if h.EmailVerifyService != nil {
h.EmailVerifyService.Kick()
}
c.JSON(http.StatusOK, resp)
}
+4
View File
@@ -163,5 +163,9 @@ func (h *Handler) ImportCommitContacts(c *gin.Context) {
"failed": fmt.Sprintf("%d", result.Failed),
})
// New addresses are checked right away rather than on the next tick.
if h.EmailVerifyService != nil {
h.EmailVerifyService.Kick()
}
c.JSON(http.StatusOK, result)
}
+66 -12
View File
@@ -8,6 +8,7 @@ import (
"github.com/warmbly/warmbly/internal/api/middleware"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/models"
)
// verifyEmailRequest is the optional JSON body for VerifyEmail. The address may
@@ -16,18 +17,12 @@ type verifyEmailRequest struct {
Email string `json:"email"`
}
// VerifyEmail verifies a single email address on demand (syntax -> MX -> SMTP
// RCPT probe -> catch-all detection) and returns the emailverify.Result. This
// is pre-send verification: it lets the user/admin confirm an address is
// deliverable *before* a worker ever sends to it, instead of learning from a
// hard bounce after the fact.
// VerifyEmail verifies a single email address on demand through whichever
// verifier the workspace uses (its connected provider, else the built-in
// check) and returns the emailverify.Result. Nothing is stored.
//
// Control-plane only: the SMTP RCPT probe behind this runs from the backend (a
// non-sending IP). Probing must never run from worker (sending) IPs — see
// internal/pkg/emailverify.
//
// Route registration is intentionally NOT done here; the parent workstream
// wires it in internal/api/routes.go behind the appropriate permission gates.
// Control-plane only: the SMTP RCPT probe behind the built-in check runs from
// the backend (a non-sending IP), never a worker. See internal/pkg/emailverify.
func (h *Handler) VerifyEmail(c *gin.Context) {
if _, err := middleware.GetUserUUID(c); err != nil {
errx.JSON(c, errx.ErrUnauthorized)
@@ -37,6 +32,10 @@ func (h *Handler) VerifyEmail(c *gin.Context) {
errx.JSON(c, errx.InternalError())
return
}
orgID, ok := requireOrgID(c)
if !ok {
return
}
var req verifyEmailRequest
// Body is optional; ignore a bind error and fall back to the query param.
@@ -50,6 +49,61 @@ func (h *Handler) VerifyEmail(c *gin.Context) {
return
}
res := h.EmailVerifyService.VerifyAddress(c.Request.Context(), email)
res := h.EmailVerifyService.VerifyAddress(c.Request.Context(), orgID, email)
c.JSON(http.StatusOK, res)
}
// GetContactVerification reports which verifier the workspace uses, its
// remaining credits, and the contacts by verdict.
// GET /contacts/verification
func (h *Handler) GetContactVerification(c *gin.Context) {
if h.EmailVerifyService == nil {
errx.JSON(c, errx.InternalError())
return
}
orgID, ok := requireOrgID(c)
if !ok {
return
}
out, xerr := h.EmailVerifyService.Overview(c.Request.Context(), orgID)
if xerr != nil {
errx.JSON(c, xerr)
return
}
c.JSON(http.StatusOK, out)
}
// RequestContactVerification queues a re-check of the listed contacts, or
// records a manual verdict on them.
// POST /contacts/verification
func (h *Handler) RequestContactVerification(c *gin.Context) {
if h.EmailVerifyService == nil {
errx.JSON(c, errx.InternalError())
return
}
orgID, ok := requireOrgID(c)
if !ok {
return
}
var req models.ContactVerificationRequest
if err := c.ShouldBindJSON(&req); err != nil {
errx.JSON(c, errx.ErrInvalid)
return
}
if len(req.Contacts) > maxBulkOperationSize {
errx.JSON(c, errx.NewWithIdentifier(errx.BadRequest, "too_many_contacts",
"too many contacts, maximum is "+itoa(maxBulkOperationSize)+" per request"))
return
}
resp, xerr := h.EmailVerifyService.Request(c.Request.Context(), orgID, req)
if xerr != nil {
errx.JSON(c, xerr)
return
}
// The audit spine refreshes every teammate's contact lists; verdicts
// then land live as the background pass records them.
h.auditOrg(c, models.AuditActionUpdate, models.AuditEntityContact, nil, nil, map[string]string{
"verification": req.Action, "count": itoa(resp.Affected),
})
c.JSON(http.StatusOK, resp)
}
+4
View File
@@ -550,6 +550,10 @@ func Run(
contacts.POST("", m.RequireAccess(models.PermManageContacts, models.APIPermWriteContacts), h.AddContacts)
contacts.DELETE("", m.RequireAccess(models.PermManageContacts, models.APIPermBulkContacts), h.DeleteContactBulk)
contacts.PATCH("", m.RequireAccess(models.PermManageContacts, models.APIPermBulkContacts), h.UpdateContactBulk)
// Address verification: who checks this workspace's contacts,
// and the member actions (re-verify, mark deliverable).
contacts.GET("/verification", m.RequireAccess(models.PermViewContacts, models.APIPermReadContacts), h.GetContactVerification)
contacts.POST("/verification", m.RequireAccess(models.PermManageContacts, models.APIPermBulkContacts), h.RequestContactVerification)
// Import + export power-tools. Read-only export gates on
// ReadContacts; the import endpoints write and so use the
// stricter Write/Bulk scopes that the rest of the contact
+36 -3
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/warmbly/warmbly/internal/pkg/emailverify"
"hash/fnv"
"math/rand"
"net/mail"
@@ -168,9 +169,11 @@ type service struct {
audienceRepo repository.CampaignAudienceRepository
// attachmentRepo lets preflight weigh attachments as the send path does.
// Optional/nil-safe: without it the content check scores none.
attachmentRepo repository.AttachmentRepository
notifier Notifier
realtime ReplyRealtimePublisher
attachmentRepo repository.AttachmentRepository
notifier Notifier
realtime ReplyRealtimePublisher
// evidence teaches verification what replies and bounces showed.
evidence EvidenceRecorder
automationRunner AutomationRunner
inboxAgent InboxAgent
}
@@ -919,6 +922,15 @@ func (s *service) ProcessIncomingReply(ctx context.Context, emailAccountID uuid.
// stop_on_reply and silently halt the sequence, and (b) match the plain
// "replied" branch. Both stop_on_reply and the "replied" condition key off
// replied_at IS NOT NULL, so gating the stamp here fixes both at once.
// Any reply, human or automatic, proves the mailbox is live; only a
// human one counts as engagement.
if s.evidence != nil {
kind := "replied"
if replyclassify.IsAutomated(replyResult.Class) {
kind = "auto_replied"
}
s.evidence.RecordEvidence(ctx, ctID, kind, msg.ID.String(), "")
}
if !replyclassify.IsAutomated(replyResult.Class) {
_ = s.campaignProgressRepo.RecordEmailReplied(ctx, cID, ctID, sID)
_ = s.repo.MarkVariantEvent(ctx, cID, ctID, string(models.DeliverabilityEventReply))
@@ -1154,6 +1166,15 @@ func (s *service) IngestDeliverabilityEvent(ctx context.Context, organizationID
switch eventType {
case models.DeliverabilityEventBounce:
_ = s.campaignProgressRepo.RecordEmailBounced(ctx, *req.CampaignID, *req.ContactID, *campaignTask.SequenceID)
// Only a bounce that names the recipient is evidence against
// the address; a full mailbox or a policy block is not.
if s.evidence != nil {
kind := "bounced_other"
if emailverify.NamesRecipient(req.Reason) {
kind = "bounced_recipient"
}
s.evidence.RecordEvidence(ctx, *req.ContactID, kind, req.IdempotencyKey, req.Reason)
}
case models.DeliverabilityEventComplaint:
_ = s.campaignProgressRepo.RecordEmailComplained(ctx, *req.CampaignID, *req.ContactID, *campaignTask.SequenceID)
}
@@ -1899,6 +1920,18 @@ func (s *service) listQualityCheck(ctx context.Context, orgID, campaignID uuid.U
}
// WireAudience attaches the launch-time list measurement.
// EvidenceRecorder mirrors emailverify.EvidenceRecorder without importing it.
type EvidenceRecorder interface {
RecordEvidence(ctx context.Context, contactID uuid.UUID, kind, ref, detail string)
}
// EvidenceAware lets main hand the service the verification evidence ledger.
type EvidenceAware interface {
WireEvidence(e EvidenceRecorder)
}
func (s *service) WireEvidence(e EvidenceRecorder) { s.evidence = e }
func (s *service) WireAudience(r repository.CampaignAudienceRepository) {
s.audienceRepo = r
}
+1 -1
View File
@@ -536,7 +536,7 @@ func (d Deps) setCampaignStatus(ctx context.Context, inv Invocation, args json.R
}
switch in.Action {
case "start":
if xerr := d.Campaigns.StartCampaign(ctx, inv.OrgID, in.CampaignID); xerr != nil {
if xerr := d.Campaigns.StartCampaign(ctx, inv.OrgID, in.CampaignID, models.StartCampaignOptions{}); xerr != nil {
return "", fromErrx(xerr)
}
d.logAudit(ctx, inv, models.AuditActionStart, models.AuditEntityCampaign, &cid, nil)
+31 -6
View File
@@ -364,7 +364,7 @@ func truncateUTF8(s string, n int) string {
return s[:n]
}
func (s *campaignService) StartCampaign(ctx context.Context, orgID uuid.UUID, campaignID string) *errx.Error {
func (s *campaignService) StartCampaign(ctx context.Context, orgID uuid.UUID, campaignID string, opts models.StartCampaignOptions) *errx.Error {
cID, parseErr := uuid.Parse(campaignID)
if parseErr != nil {
return errx.ErrUuid
@@ -392,14 +392,14 @@ func (s *campaignService) StartCampaign(ctx context.Context, orgID uuid.UUID, ca
// one just re-completes in enqueueCampaignWakeup with a clear message.
startable := map[string]bool{
"draft": true, "paused": true, "paused_no_accounts": true,
"paused_guardrail": true, "completed": true,
"paused_guardrail": true, "paused_undeliverable": true, "completed": true,
}
if !startable[campaign.Status] {
return errx.New(errx.BadRequest, "campaign must be in draft, paused, or completed status to start")
}
// Check cooldown
if campaign.LastStatusChangeAt != nil {
// Check cooldown. An automatic resume is not a member flapping a button.
if campaign.LastStatusChangeAt != nil && !opts.Automatic {
elapsed := time.Since(*campaign.LastStatusChangeAt)
if elapsed.Seconds() < campaignCooldownSeconds {
return errx.New(errx.BadRequest, "please wait before changing campaign status")
@@ -451,7 +451,10 @@ func (s *campaignService) StartCampaign(ctx context.Context, orgID uuid.UUID, ca
// Refuse a launch whose list is known to be largely undeliverable. Only
// KNOWN-invalid addresses count: a list nobody has verified is not evidence
// of a bad list, and blocking on that would refuse nearly every launch.
if s.audienceRepo != nil {
// The owner can acknowledge the risk and launch anyway (the list may have
// been verified elsewhere), and a campaign parked by verification has
// already been told, so its resume is not gated again.
if s.audienceRepo != nil && !opts.AcknowledgeListRisk && campaign.Status != "paused_undeliverable" {
audience, aerr := s.audienceRepo.GetCampaignAudience(ctx, orgID, cID)
switch {
case aerr != nil:
@@ -462,7 +465,7 @@ func (s *campaignService) StartCampaign(ctx context.Context, orgID uuid.UUID, ca
Msg("could not measure the campaign's list; launching without the check")
default:
if verdict := listgate.Project(audience); verdict.Block {
return errx.New(errx.BadRequest, verdict.Summary+" "+verdict.Remediation)
return errx.NewWithIdentifier(errx.BadRequest, "list_bounce_risk", verdict.Summary+" "+verdict.Remediation)
}
}
}
@@ -602,6 +605,13 @@ func (s *campaignService) enqueueCampaignWakeup(ctx context.Context, campaignID
_ = s.campaignRepository.UpdateStatusWithLock(ctx, campaignID, "paused_no_accounts")
return errx.New(errx.BadRequest, "no active email accounts found for campaign's email tags")
case errors.Is(err, scheduler.ErrCampaignCompleted):
if s.campaignProgressRepo != nil {
if n, cerr := s.campaignProgressRepo.CountUndeliverableLeads(ctx, campaignID); cerr == nil && n > 0 {
_ = s.campaignRepository.UpdateStatusWithLock(ctx, campaignID, "paused_undeliverable")
return errx.NewWithIdentifier(errx.BadRequest, "leads_undeliverable",
fmt.Sprintf("%d remaining lead(s) were refused by address verification; re-verify them or mark them deliverable to continue", n))
}
}
_ = s.campaignRepository.UpdateStatusWithLock(ctx, campaignID, "completed")
return errx.New(errx.BadRequest, "campaign has no remaining contacts to send")
case errors.Is(err, scheduler.ErrCampaignEnded):
@@ -650,6 +660,21 @@ func (s *campaignService) enqueueCampaignWakeup(ctx context.Context, campaignID
return nil
}
// ResumeVerificationPaused restarts every campaign of the org parked because
// verification refused its leads. Best effort: a campaign that still has
// nothing to send parks itself again inside StartCampaign.
func (s *campaignService) ResumeVerificationPaused(ctx context.Context, orgID uuid.UUID) {
ids, err := s.campaignRepository.ListIDsByStatus(ctx, orgID, "paused_undeliverable")
if err != nil || len(ids) == 0 {
return
}
for _, id := range ids {
if xerr := s.StartCampaign(ctx, orgID, id.String(), models.StartCampaignOptions{Automatic: true}); xerr != nil {
log.Info().Str("campaign_id", id.String()).Str("reason", xerr.Message).Msg("campaign stays parked after verification")
}
}
}
func (s *campaignService) StopCampaign(ctx context.Context, orgID uuid.UUID, campaignID string) *errx.Error {
cID, parseErr := uuid.Parse(campaignID)
if parseErr != nil {
+17 -1
View File
@@ -33,7 +33,11 @@ type CampaignService interface {
Duplicate(ctx context.Context, orgID, userID uuid.UUID, campaignID, name string) (*models.Campaign, *errx.Error)
// Start/Stop
StartCampaign(ctx context.Context, orgID uuid.UUID, campaignID string) *errx.Error
StartCampaign(ctx context.Context, orgID uuid.UUID, campaignID string, opts models.StartCampaignOptions) *errx.Error
// ResumeVerificationPaused restarts the org's campaigns parked at
// paused_undeliverable. Called after verification verdicts change; a
// campaign that still has nothing to send parks itself again.
ResumeVerificationPaused(ctx context.Context, orgID uuid.UUID)
StopCampaign(ctx context.Context, orgID uuid.UUID, campaignID string) *errx.Error
// Logs
@@ -71,6 +75,9 @@ type campaignService struct {
// audienceRepo measures the campaign's list at launch. Optional/nil-safe:
// without it no launch is ever refused on list quality.
audienceRepo repository.CampaignAudienceRepository
// campaignProgressRepo counts the leads verification refused, so a start
// with nothing left to send can say why. Optional/nil-safe.
campaignProgressRepo repository.CampaignProgressRepository
}
// WireAudience attaches the launch-time list gate.
@@ -90,6 +97,15 @@ type AttachmentAware interface {
WireAttachments(repo repository.AttachmentRepository, store storage.Store)
}
// ProgressAware lets main hand the campaign service the progress repository.
type ProgressAware interface {
WireProgress(r repository.CampaignProgressRepository)
}
func (s *campaignService) WireProgress(r repository.CampaignProgressRepository) {
s.campaignProgressRepo = r
}
func (s *campaignService) WireAttachments(repo repository.AttachmentRepository, store storage.Store) {
s.attachmentRepo = repo
s.storage = store
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"github.com/warmbly/warmbly/internal/pkg/emailverify"
"strings"
"time"
@@ -196,6 +197,13 @@ func (s *JobsService) failCampaignSend(ctx context.Context, task *repository.Tas
return nil
}
// A server that rejected the RECIPIENT at send time is a bounce in all but
// delivery route; a rejection of the sender, the session or the content
// says nothing about the address.
if s.Evidence != nil && ct.ContactID != nil && ct.SequenceID != nil && emailverify.NamesRecipient(reason) {
s.Evidence.RecordEvidence(ctx, *ct.ContactID, "bounced_recipient", "send:"+ct.SequenceID.String(), reason)
}
attempts, exhausted, rolledBack := 0, false, false
if ct.ContactID != nil && ct.SequenceID != nil && s.CampaignProgressRepo != nil {
attempts, exhausted, rolledBack, err = s.CampaignProgressRepo.RecordSendFailure(ctx, campaignID, *ct.ContactID, *ct.SequenceID, reason)
+11
View File
@@ -27,6 +27,7 @@ type TrackingConsumer struct {
campaignProgressRepo repository.CampaignProgressRepository
campaignRepo repository.CampaignRepository
contactRepo repository.ContactRepository
evidence advanced.EvidenceRecorder
streamingPublisher *pubsub.StreamingPublisher
dedupeRepo repository.TrackingDedupeRepository
// advancedService fires INSTANT open/click action chains the moment a
@@ -50,6 +51,7 @@ func NewTrackingConsumer(
streamingPublisher *pubsub.StreamingPublisher,
dedupeRepo repository.TrackingDedupeRepository,
advancedService advanced.Service,
evidence advanced.EvidenceRecorder,
) (*TrackingConsumer, error) {
return &TrackingConsumer{
bus: bus,
@@ -61,6 +63,7 @@ func NewTrackingConsumer(
streamingPublisher: streamingPublisher,
dedupeRepo: dedupeRepo,
advancedService: advancedService,
evidence: evidence,
topic: topic,
group: group,
}, nil
@@ -161,6 +164,11 @@ func (tc *TrackingConsumer) HandleTrackingEvent(ctx context.Context, event *even
machineOpen)
if !machineOpen {
instantKind = "open"
// A human open proves the mailbox is live; a prefetch proves
// only that a proxy fetched an image.
if tc.evidence != nil {
tc.evidence.RecordEvidence(ctx, *campaignTask.ContactID, "opened", campaignTask.SequenceID.String(), "")
}
}
case events.EventTypeEmailClicked:
err = tc.campaignProgressRepo.RecordEmailClicked(ctx,
@@ -168,6 +176,9 @@ func (tc *TrackingConsumer) HandleTrackingEvent(ctx context.Context, event *even
*campaignTask.ContactID,
*campaignTask.SequenceID)
instantKind = "click"
if tc.evidence != nil {
tc.evidence.RecordEvidence(ctx, *campaignTask.ContactID, "clicked", campaignTask.SequenceID.String(), "")
}
default:
// Unknown event type, skip
return nil
+2
View File
@@ -83,6 +83,8 @@ type JobsService struct {
CampaignProgressRepo repository.CampaignProgressRepository
CampaignLogRepo repository.CampaignLogRepository
ContactRepo repository.ContactRepository
// Evidence teaches verification what send results showed.
Evidence advanced.EvidenceRecorder
eventHandlers map[models.JobEventType]func(ctx context.Context, body any) error
}
+2
View File
@@ -134,6 +134,8 @@ func constraintCopy(c scheduler.ContactSendConstraint, st *models.ContactCampaig
return "Campaign has finished"
case "paused_guardrail":
return "Campaign was auto-paused by a guardrail"
case "paused_undeliverable":
return "Campaign is paused: verification refused its remaining leads"
}
return "Campaign is paused"
case scheduler.ConstraintNoMailbox:
+8 -1
View File
@@ -168,7 +168,14 @@ func (s *contactService) Delete(ctx context.Context, userID string, orgID uuid.U
}
func (s *contactService) GetDetail(ctx context.Context, userID uuid.UUID, orgID *uuid.UUID, contactID uuid.UUID) (*models.ContactDetail, *errx.Error) {
return s.contactRepository.GetDetail(ctx, userID, orgID, contactID)
detail, xerr := s.contactRepository.GetDetail(ctx, userID, orgID, contactID)
if xerr != nil || detail == nil {
return detail, xerr
}
if s.explainer != nil {
detail.Verification = s.explainer.Explain(ctx, contactID)
}
return detail, nil
}
func (s *contactService) GetByEmail(ctx context.Context, orgID *uuid.UUID, email string) (*models.Contact, *errx.Error) {
+46 -2
View File
@@ -16,6 +16,7 @@ import (
"github.com/warmbly/warmbly/internal/email"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/pkg/emailverify"
"github.com/warmbly/warmbly/internal/pkg/listquality"
"github.com/warmbly/warmbly/internal/utils"
"github.com/xuri/excelize/v2"
@@ -60,7 +61,7 @@ func (s *contactService) ImportPreview(ctx context.Context, r io.Reader, filenam
Columns: headers,
HasHeader: hasHeader,
SampleRows: sample,
SuggestedMapping: suggestMapping(headers),
SuggestedMapping: suggestMapping(headers, sample),
}, nil
}
@@ -105,6 +106,15 @@ func resolveMapping(mapping []models.ContactImportColumnMapping) ([]importColumn
models.ContactImportTargetPhone,
models.ContactImportTargetSubscribed,
models.ContactImportTargetCategories:
case models.ContactImportTargetVerificationStatus:
if p := strings.TrimSpace(m.VerificationProvider); p != "" {
k, ok := emailverify.KnownVocabulary(p)
if !ok {
return nil, errx.New(errx.BadRequest,
"unknown verification provider "+strconv.Quote(p)+" for column "+strconv.Itoa(m.Index+1))
}
key = k
}
case models.ContactImportTargetCustom:
default:
// An unrecognised target with a custom key is how older clients
@@ -703,10 +713,38 @@ func padRow(row []string, n int) []string {
// suggestMapping uses fuzzy header matches to pick a target for each
// column. Anything we don't recognise becomes ignore — better than
// inventing a custom-field key the user didn't ask for.
func suggestMapping(headers []string) []models.ContactImportColumnMapping {
func suggestMapping(headers []string, sample [][]string) []models.ContactImportColumnMapping {
out := make([]models.ContactImportColumnMapping, len(headers))
for i, h := range headers {
out[i] = guessTarget(i, h)
if out[i].Target != models.ContactImportTargetIgnore {
continue
}
// A verdict column from another verification service: the header
// says so, or every sample value is a word one of them writes.
provider, headerSays := emailverify.IsStatusHeader(h)
values := make([]string, 0, len(sample))
for _, row := range sample {
if i < len(row) {
values = append(values, row[i])
}
}
detected, valuesSay := emailverify.DetectVocabulary(values)
if !headerSays && !valuesSay {
continue
}
// A generic header ("status") only counts when the values agree,
// otherwise a CRM's deal-stage column would be read as a verdict.
if headerSays && provider == "" && !valuesSay {
continue
}
if provider == "" {
provider = detected
}
if provider == emailverify.ProviderBuiltin {
provider = ""
}
out[i] = models.ContactImportColumnMapping{Index: i, Target: models.ContactImportTargetVerificationStatus, VerificationProvider: provider}
}
return out
}
@@ -786,6 +824,12 @@ func buildAddContact(
categories = appendUnique(categories, name)
}
}
case models.ContactImportTargetVerificationStatus:
// Lenient on purpose: a stray "n/a" must not drop the lead.
if _, ok := emailverify.NormalizeExternal(col.customKey, val); ok {
ac.VerificationStatus = val
ac.VerificationProvider = col.customKey
}
case models.ContactImportTargetCustom:
ac.CustomFields[col.customKey] = val
}
+16
View File
@@ -101,8 +101,24 @@ type contactService struct {
// orgRisk files import-quality findings on the workspace's posture.
// Optional/nil-safe: without it a bad import is reported but not fused.
orgRisk orgrisk.Service
// explainer builds the verification "why" for the contact drawer.
explainer VerificationExplainer
}
// VerificationAware is implemented by the contact service so main can hand
// it the explainer.
type VerificationAware interface {
WireVerification(e VerificationExplainer)
}
// VerificationExplainer mirrors emailverify.Service.Explain.
type VerificationExplainer interface {
Explain(ctx context.Context, contactID uuid.UUID) *models.ContactVerificationDetail
}
// WireVerification attaches the verification explainer.
func (s *contactService) WireVerification(e VerificationExplainer) { s.explainer = e }
// WireOrgRisk attaches the organization risk posture.
func (s *contactService) WireOrgRisk(r orgrisk.Service) { s.orgRisk = r }
+73
View File
@@ -0,0 +1,73 @@
package emailverify
import (
"sync"
"time"
)
// breaker is the in-house probe's self-check. Two incidents (#200, #264) had
// the probe reject good addresses wholesale because of its own environment
// (a rejected HELO, a blocklisted IP). A real list is never mostly dead, so
// when the share of "invalid" over the last window crosses the threshold the
// probe's rejections are filed as unknown, which still sends, until it has
// cooled down.
type breaker struct {
mu sync.Mutex
window int
threshold float64
cooldown time.Duration
ring []bool
pos int
filled int
invalid int
trippedAt time.Time
}
func newBreaker(window int, thresholdPct float64, cooldown time.Duration) *breaker {
if window < 10 {
window = 10
}
return &breaker{window: window, threshold: thresholdPct, cooldown: cooldown, ring: make([]bool, window)}
}
// observe records one probe verdict and reports whether the breaker is open.
func (b *breaker) observe(invalid bool) bool {
b.mu.Lock()
defer b.mu.Unlock()
if !b.trippedAt.IsZero() {
if time.Since(b.trippedAt) < b.cooldown {
return true
}
// Cooled down: start measuring afresh.
b.trippedAt = time.Time{}
b.ring = make([]bool, b.window)
b.pos, b.filled, b.invalid = 0, 0, 0
}
if b.filled == b.window {
if b.ring[b.pos] {
b.invalid--
}
} else {
b.filled++
}
b.ring[b.pos] = invalid
if invalid {
b.invalid++
}
b.pos = (b.pos + 1) % b.window
if b.filled < b.window/2 {
return false
}
if float64(b.invalid)/float64(b.filled)*100 >= b.threshold {
b.trippedAt = time.Now()
return true
}
return false
}
// open reports whether the breaker is currently tripped.
func (b *breaker) open() bool {
b.mu.Lock()
defer b.mu.Unlock()
return !b.trippedAt.IsZero() && time.Since(b.trippedAt) < b.cooldown
}
+37
View File
@@ -0,0 +1,37 @@
package emailverify
import (
"testing"
"time"
)
func TestBreakerTripsOnAnInvalidFlood(t *testing.T) {
b := newBreaker(20, 40, time.Hour)
for i := 0; i < 9; i++ {
if b.observe(false) {
t.Fatal("closed breaker tripped on valid verdicts")
}
}
// 9 of 19 invalid = 47%, above the threshold once half the window is filled.
tripped := false
for i := 0; i < 10; i++ {
if b.observe(true) {
tripped = true
}
}
if !tripped || !b.open() {
t.Fatal("breaker did not trip")
}
if !b.observe(false) {
t.Fatal("open breaker must stay open during cooldown")
}
}
func TestBreakerNeedsHalfAWindow(t *testing.T) {
b := newBreaker(20, 40, time.Hour)
for i := 0; i < 5; i++ {
if b.observe(true) {
t.Fatal("five verdicts are not evidence")
}
}
}
+156
View File
@@ -0,0 +1,156 @@
package emailverify
import (
"context"
"time"
"github.com/google/uuid"
"github.com/rs/zerolog/log"
"github.com/warmbly/warmbly/internal/config"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/pkg/emailverify"
"github.com/warmbly/warmbly/internal/repository"
)
// EvidenceRecorder is what the send, tracking, reply and bounce paths call
// to teach verification what real mail showed. Every call is idempotent on
// (contact, kind, ref) and best effort: a failure is logged, never returned
// into the caller's path.
type EvidenceRecorder interface {
RecordEvidence(ctx context.Context, contactID uuid.UUID, kind, ref, detail string)
}
// Evidence scores contacts from the ledger. It is separate from Service so
// the consumer, which never verifies, can record and rescore without the
// verifier and its providers.
type Evidence struct {
repo repository.VerificationEvidenceRepository
// onChange runs after a rescore changed the contact's status, e.g. to
// resume a campaign parked on the old verdict.
onChange func(ctx context.Context, contactID uuid.UUID)
}
func NewEvidence(repo repository.VerificationEvidenceRepository) *Evidence {
return &Evidence{repo: repo}
}
func (e *Evidence) SetOnChange(fn func(ctx context.Context, contactID uuid.UUID)) { e.onChange = fn }
func (e *Evidence) RecordEvidence(ctx context.Context, contactID uuid.UUID, kind, ref, detail string) {
if e == nil || e.repo == nil || contactID == uuid.Nil {
return
}
inserted, err := e.repo.Record(ctx, contactID, kind, ref, detail, time.Now().UTC())
if err != nil {
log.Warn().Err(err).Str("contact_id", contactID.String()).Str("kind", kind).Msg("verification evidence not recorded")
return
}
if inserted {
e.Rescore(ctx, contactID)
}
}
// Rescore recomputes the contact's status and confidence from its verdict
// and ledger, and persists the result.
func (e *Evidence) Rescore(ctx context.Context, contactID uuid.UUID) {
if e == nil || e.repo == nil {
return
}
verdict, err := e.repo.Verdict(ctx, contactID)
if err != nil {
return
}
rows, err := e.repo.ListForContact(ctx, contactID)
if err != nil {
return
}
scored := emailverify.Score(verdict, toEvidence(rows), time.Now().UTC())
reason := ""
if len(scored.Reasons) > 0 {
reason = scored.Reasons[0]
}
if err := e.repo.SetScore(ctx, contactID, string(scored.Status), scored.Confidence, reason, scored.LastPositiveAt, scored.Decisive); err != nil {
return
}
if scored.Decisive && scored.Status != verdict.Status && e.onChange != nil {
e.onChange(ctx, contactID)
}
}
// Explain builds the "why" a member reads on the contact drawer.
func (e *Evidence) Explain(ctx context.Context, contactID uuid.UUID) *models.ContactVerificationDetail {
if e == nil || e.repo == nil {
return nil
}
verdict, err := e.repo.Verdict(ctx, contactID)
if err != nil {
return nil
}
rows, err := e.repo.ListForContact(ctx, contactID)
if err != nil {
return nil
}
scored := emailverify.Score(verdict, toEvidence(rows), time.Now().UTC())
return &models.ContactVerificationDetail{
Status: string(scored.Status),
Confidence: scored.Confidence,
Reasons: scored.Reasons,
Decisive: scored.Decisive,
Evidence: rows,
}
}
// CreditCleanDeliveries turns sends that never bounced into evidence, then
// rescores the contacts credited. Returns how many contacts changed.
func (e *Evidence) CreditCleanDeliveries(ctx context.Context, limit int) (int, error) {
if e == nil || e.repo == nil {
return 0, nil
}
ids, err := e.repo.CreditCleanDeliveries(ctx, time.Duration(config.VerificationDeliveryWindowHours)*time.Hour, limit)
if err != nil {
return 0, err
}
for _, id := range ids {
if ctx.Err() != nil {
break
}
e.Rescore(ctx, id)
}
return len(ids), nil
}
// Apply folds the ledger into a fresh check result before it is stored, so a
// probe can never demote an address real mail has proven.
func (e *Evidence) Apply(ctx context.Context, contactID uuid.UUID, res emailverify.Result) emailverify.Result {
source := models.VerificationSourceProbe
if res.Provider != "" && res.Provider != emailverify.ProviderBuiltin {
source = models.VerificationSourceProvider
}
verdict := emailverify.Verdict{Status: res.Status, Source: source, CheckedAt: res.CheckedAt}
var rows []models.ContactVerificationEvidence
if e != nil && e.repo != nil {
rows, _ = e.repo.ListForContact(ctx, contactID)
}
scored := emailverify.Score(verdict, toEvidence(rows), time.Now().UTC())
res.Confidence = scored.Confidence
if scored.Decisive && scored.Status != res.Status {
res.Status = scored.Status
if len(scored.Reasons) > 0 {
res.Reason = scored.Reasons[0] + " (check said: " + res.Reason + ")"
}
if scored.Status == emailverify.StatusValid {
res.SubStatus = emailverify.SubStatusNone
res.IsCatchAll = false
}
}
return res
}
func toEvidence(rows []models.ContactVerificationEvidence) []emailverify.Evidence {
out := make([]emailverify.Evidence, 0, len(rows))
for _, r := range rows {
out = append(out, emailverify.Evidence{Kind: r.Kind, Detail: r.Detail, ObservedAt: r.ObservedAt})
}
return out
}
+400 -40
View File
@@ -1,82 +1,442 @@
// Package emailverify (app layer) orchestrates pre-send email verification:
// it loads contacts, runs them through a pkg/emailverify.Verifier, and persists
// the result back onto the contact. This is the control-plane home for
// verification — the SMTP RCPT probe inside the Verifier dials remote MX hosts
// on :25 and must never run from a worker (a sending IP). See
// internal/pkg/emailverify for the probing details and the in-repo-vs-paid
// backend split.
// it loads contacts due for a check, picks the verifier for their organization
// (the paid provider connected on the workspace, else the in-house probe),
// persists each verdict, and offers the member-facing actions (re-verify,
// mark deliverable). The SMTP probe inside the in-house verifier dials remote
// MX hosts on :25 and must never run from a worker (a sending IP).
package emailverify
import (
"context"
"errors"
"strings"
"sync"
"time"
"github.com/google/uuid"
"github.com/rs/zerolog/log"
"github.com/warmbly/warmbly/internal/config"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/pkg/emailverify"
"github.com/warmbly/warmbly/internal/repository"
)
// Provider is a paid verification backend bound to one organization.
type Provider struct {
Name string
// ConnectionID is the integration connection carrying the key; nil for
// the instance-wide key an operator configured.
ConnectionID *uuid.UUID
Client *emailverify.MillionVerifier
}
// ProviderSource resolves the paid provider an organization connected.
// Implemented by the integration service.
type ProviderSource interface {
// VerificationProviderFor returns nil when the org has no usable provider.
VerificationProviderFor(ctx context.Context, orgID uuid.UUID) (*Provider, error)
// ReportVerificationProviderError flips the connection's health when the
// provider rejected the key or ran out of credits.
ReportVerificationProviderError(ctx context.Context, connectionID uuid.UUID, err error)
}
// Service verifies contact email addresses before they are ever sent to.
type Service interface {
// VerifyContact verifies a single stored contact by id and persists the
// result. Returns the Result so callers (admin/on-demand) can surface it.
VerifyContact(ctx context.Context, contactID uuid.UUID) (emailverify.Result, *errx.Error)
// VerifyAddress verifies an arbitrary address for an organization without
// touching the DB, through whichever verifier the org uses.
VerifyAddress(ctx context.Context, orgID uuid.UUID, email string) emailverify.Result
// VerifyAddress verifies an arbitrary address without touching the DB. Used
// by the on-demand handler for addresses that aren't stored contacts yet.
VerifyAddress(ctx context.Context, email string) emailverify.Result
// VerifyPending verifies up to `limit` not-yet-checked contacts, persisting
// each result. Returns the number processed. Driven by the ticker scheduler.
// VerifyPending verifies up to `limit` contacts due for a check. Returns
// the number processed. Driven by the scheduler.
VerifyPending(ctx context.Context, limit int) (int, *errx.Error)
// Request applies a member action: queue a re-check, or record a manual
// verdict.
Request(ctx context.Context, orgID uuid.UUID, req models.ContactVerificationRequest) (*models.ContactVerificationResponse, *errx.Error)
// Overview reports which verifier the org uses, its credits, and the
// org's contacts by verdict.
Overview(ctx context.Context, orgID uuid.UUID) (*models.VerificationOverview, *errx.Error)
// Kick wakes the scheduler for an immediate pass (after a re-verify
// request or an import), instead of waiting for the next interval.
Kick()
// Wake is the channel the scheduler selects on.
Wake() <-chan struct{}
// SetVerdictHook registers a callback run once per organization after a
// pass that changed its contacts, e.g. to resume a campaign that was
// parked waiting for verification.
SetVerdictHook(fn func(ctx context.Context, orgID uuid.UUID))
// SetEvidence attaches the evidence ledger, so a check never overrides
// what real mail has shown.
SetEvidence(e *Evidence)
// Explain builds the contact's verification detail for the drawer.
Explain(ctx context.Context, contactID uuid.UUID) *models.ContactVerificationDetail
}
type service struct {
repo repository.ContactRepository
verifier emailverify.Verifier
repo repository.ContactRepository
builtin emailverify.Verifier
providers ProviderSource
// platform is the instance-wide paid client an operator configured for
// workspaces that bring no key of their own; nil when unset.
platform *emailverify.MillionVerifier
builtinReady bool
wake chan struct{}
hook func(ctx context.Context, orgID uuid.UUID)
evidence *Evidence
// breakers are per organization: one tenant's list must not decide
// whether another tenant's rejections are trusted.
breakersMu sync.Mutex
breakers map[uuid.UUID]*breaker
creditsMu sync.Mutex
credits map[string]creditsEntry
}
// NewService wires the verification service. verifier is the pluggable backend:
// the in-house emailverify.SMTPVerifier in dev/self-host, or a paid provider
// (ZeroBounce/NeverBounce/Bouncer) implementing the same interface in prod.
func NewService(repo repository.ContactRepository, verifier emailverify.Verifier) Service {
return &service{repo: repo, verifier: verifier}
type creditsEntry struct {
n int
err error
at time.Time
}
func (s *service) VerifyAddress(ctx context.Context, email string) emailverify.Result {
return s.verifier.Verify(ctx, email)
// Options configures the service.
type Options struct {
// Builtin is the in-house verifier. Required.
Builtin emailverify.Verifier
// BuiltinReady reports whether Builtin can run its SMTP probe (a public
// HELO host is configured). Shown to members so a self-hosted instance
// without one knows why every verdict is "unknown".
BuiltinReady bool
// Providers resolves per-org paid providers. Optional.
Providers ProviderSource
// PlatformMillionVerifierKey is the operator's own key, used for every
// workspace without a key of its own. Optional.
PlatformMillionVerifierKey string
}
func (s *service) VerifyContact(ctx context.Context, contactID uuid.UUID) (emailverify.Result, *errx.Error) {
contact, xerr := s.repo.GetByID(ctx, contactID)
if xerr != nil {
return emailverify.Result{}, xerr
// NewService wires the verification service.
func NewService(repo repository.ContactRepository, opts Options) Service {
s := &service{
repo: repo,
builtin: opts.Builtin,
builtinReady: opts.BuiltinReady,
providers: opts.Providers,
wake: make(chan struct{}, 1),
breakers: map[uuid.UUID]*breaker{},
credits: map[string]creditsEntry{},
}
res := s.verifier.Verify(ctx, contact.Email)
if xerr := s.repo.UpdateContactVerification(ctx, contactID, res); xerr != nil {
return res, xerr
if k := strings.TrimSpace(opts.PlatformMillionVerifierKey); k != "" {
s.platform = emailverify.NewMillionVerifier(k, "")
}
return res, nil
return s
}
func (s *service) Kick() {
select {
case s.wake <- struct{}{}:
default:
}
}
func (s *service) Wake() <-chan struct{} { return s.wake }
func (s *service) SetVerdictHook(fn func(ctx context.Context, orgID uuid.UUID)) { s.hook = fn }
func (s *service) SetEvidence(e *Evidence) { s.evidence = e }
func (s *service) Explain(ctx context.Context, contactID uuid.UUID) *models.ContactVerificationDetail {
return s.evidence.Explain(ctx, contactID)
}
func (s *service) breakerFor(orgID uuid.UUID) *breaker {
s.breakersMu.Lock()
defer s.breakersMu.Unlock()
b, ok := s.breakers[orgID]
if !ok {
b = newBreaker(config.VerificationBreakerWindow, config.VerificationBreakerInvalidPct, time.Duration(config.VerificationBreakerCooldownMinutes)*time.Minute)
s.breakers[orgID] = b
}
return b
}
// providerFor resolves the org's paid provider: its own connection first,
// then the operator's instance-wide key.
func (s *service) providerFor(ctx context.Context, orgID uuid.UUID) *Provider {
if s.providers != nil {
p, err := s.providers.VerificationProviderFor(ctx, orgID)
if err != nil {
log.Warn().Err(err).Str("organization_id", orgID.String()).Msg("verification: could not resolve provider; using built-in")
} else if p != nil {
return p
}
}
if s.platform != nil {
return &Provider{Name: emailverify.ProviderMillionVerifier, Client: s.platform}
}
return nil
}
func (s *service) VerifyAddress(ctx context.Context, orgID uuid.UUID, email string) emailverify.Result {
if p := s.providerFor(ctx, orgID); p != nil {
res, err := p.Client.Check(ctx, email)
if err == nil {
return res
}
s.noteProviderError(ctx, p, err)
}
return s.verifyBuiltin(ctx, orgID, email)
}
// verifyBuiltin runs the in-house verifier under the org's self-check breaker.
func (s *service) verifyBuiltin(ctx context.Context, orgID uuid.UUID, email string) emailverify.Result {
res := s.builtin.Verify(ctx, email)
// Syntax, no-MX and disposable verdicts do not come from a probe, so they
// neither feed nor fall under the breaker.
probeVerdict := res.SubStatus == emailverify.SubStatusNone || res.SubStatus == emailverify.SubStatusCatchAll || res.SubStatus == emailverify.SubStatusRole
if !probeVerdict {
return res
}
if s.breakerFor(orgID).observe(res.Status == emailverify.StatusInvalid) && res.Status == emailverify.StatusInvalid {
res.Status = emailverify.StatusUnknown
res.Reason = "probe rejection not trusted: the in-house check is rejecting an unusual share of addresses and is cooling down (" + res.Reason + ")"
}
return res
}
func (s *service) noteProviderError(ctx context.Context, p *Provider, err error) {
if p == nil || err == nil {
return
}
if errors.Is(err, emailverify.ErrMillionVerifierKey) || errors.Is(err, emailverify.ErrMillionVerifierCredits) {
if p.ConnectionID != nil && s.providers != nil {
s.providers.ReportVerificationProviderError(ctx, *p.ConnectionID, err)
}
s.creditsMu.Lock()
s.credits[cacheKey(p)] = creditsEntry{err: err, at: time.Now()}
s.creditsMu.Unlock()
}
}
func cacheKey(p *Provider) string {
if p.ConnectionID != nil {
return p.ConnectionID.String()
}
return "platform"
}
// providerUsable checks (cached for a minute) that the provider's key works
// and has credits, so a pass never burns a whole batch on a dead key.
func (s *service) providerUsable(ctx context.Context, p *Provider) (int, error) {
key := cacheKey(p)
s.creditsMu.Lock()
e, ok := s.credits[key]
s.creditsMu.Unlock()
if ok && time.Since(e.at) < time.Minute {
return e.n, e.err
}
n, err := p.Client.Credits(ctx)
if err == nil && n <= 0 {
err = emailverify.ErrMillionVerifierCredits
}
s.creditsMu.Lock()
s.credits[key] = creditsEntry{n: n, err: err, at: time.Now()}
s.creditsMu.Unlock()
if err != nil {
s.noteProviderError(ctx, p, err)
}
return n, err
}
func (s *service) VerifyPending(ctx context.Context, limit int) (int, *errx.Error) {
contacts, xerr := s.repo.ListUnverifiedContacts(ctx, limit)
cands, xerr := s.repo.ListVerificationCandidates(ctx, limit)
if xerr != nil {
return 0, xerr
}
if len(cands) == 0 {
return 0, nil
}
// Group by organization: the verifier is chosen once per org.
byOrg := map[uuid.UUID][]repository.VerificationCandidate{}
var order []uuid.UUID
for _, c := range cands {
if _, seen := byOrg[c.OrganizationID]; !seen {
order = append(order, c.OrganizationID)
}
byOrg[c.OrganizationID] = append(byOrg[c.OrganizationID], c)
}
processed := 0
for i := range contacts {
// Honour cancellation between addresses; each probe can take seconds.
for _, orgID := range order {
if err := ctx.Err(); err != nil {
break
}
res := s.verifier.Verify(ctx, contacts[i].Email)
if xerr := s.repo.UpdateContactVerification(ctx, contacts[i].ID, res); xerr != nil {
// Skip this one; a transient DB error shouldn't abort the whole tick.
continue
n := s.verifyOrgBatch(ctx, orgID, byOrg[orgID])
processed += n
if n > 0 && s.hook != nil {
s.hook(ctx, orgID)
}
processed++
}
return processed, nil
}
// verifyOrgBatch checks one org's candidates with its verifier, in parallel
// up to the verifier's concurrency, and persists every verdict.
func (s *service) verifyOrgBatch(ctx context.Context, orgID uuid.UUID, cands []repository.VerificationCandidate) int {
verify := func(ctx context.Context, email string) emailverify.Result { return s.verifyBuiltin(ctx, orgID, email) }
workers := config.VerificationProbeConcurrency
if p := s.providerFor(ctx, orgID); p != nil {
if _, err := s.providerUsable(ctx, p); err != nil {
log.Warn().Err(err).Str("organization_id", orgID.String()).Msg("verification: paid provider unusable; using built-in check")
} else {
workers = config.VerificationProviderConcurrency
verify = func(ctx context.Context, email string) emailverify.Result {
res, err := p.Client.Check(ctx, email)
if err != nil {
s.noteProviderError(ctx, p, err)
// Fall back for this address so the pass still makes progress.
return s.verifyBuiltin(ctx, orgID, email)
}
return res
}
}
}
var (
wg sync.WaitGroup
mu sync.Mutex
processed int
sem = make(chan struct{}, workers)
)
for _, c := range cands {
if err := ctx.Err(); err != nil {
break
}
sem <- struct{}{}
wg.Add(1)
go func(c repository.VerificationCandidate) {
defer wg.Done()
defer func() { <-sem }()
// What real mail showed outranks what the check says.
res := s.evidence.Apply(ctx, c.ID, verify(ctx, c.Email))
if xerr := s.repo.UpdateContactVerification(ctx, c.ID, res); xerr != nil {
// Skip this one; a transient DB error shouldn't abort the whole pass.
return
}
mu.Lock()
processed++
mu.Unlock()
}(c)
}
wg.Wait()
return processed
}
func (s *service) Request(ctx context.Context, orgID uuid.UUID, req models.ContactVerificationRequest) (*models.ContactVerificationResponse, *errx.Error) {
var ids []uuid.UUID
for _, raw := range req.Contacts {
id, err := uuid.Parse(strings.TrimSpace(raw))
if err != nil {
return nil, errx.ErrUuid
}
ids = append(ids, id)
}
if req.CampaignID != "" {
cid, err := uuid.Parse(strings.TrimSpace(req.CampaignID))
if err != nil {
return nil, errx.ErrUuid
}
more, xerr := s.repo.UndeliverableLeadIDs(ctx, orgID, cid)
if xerr != nil {
return nil, xerr
}
ids = append(ids, more...)
}
if len(ids) == 0 {
return nil, errx.NewWithIdentifier(errx.BadRequest, "no_contacts", "no contacts selected")
}
resp := &models.ContactVerificationResponse{Action: req.Action}
switch req.Action {
case models.ContactVerificationActionVerify:
n, xerr := s.repo.ResetContactsVerification(ctx, orgID, ids)
if xerr != nil {
return nil, xerr
}
resp.Affected, resp.Queued = n, true
s.Kick()
case models.ContactVerificationActionMarkDeliverable:
n, xerr := s.repo.SetContactsVerification(ctx, orgID, ids, models.ContactVerificationWrite{
Status: string(emailverify.StatusValid),
Reason: "marked deliverable by a member",
Provider: "manual",
Source: models.VerificationSourceManual,
})
if xerr != nil {
return nil, xerr
}
resp.Affected = n
if s.hook != nil {
s.hook(ctx, orgID)
}
case models.ContactVerificationActionMarkUndeliverable:
n, xerr := s.repo.SetContactsVerification(ctx, orgID, ids, models.ContactVerificationWrite{
Status: string(emailverify.StatusInvalid),
Reason: "marked undeliverable by a member",
Provider: "manual",
Source: models.VerificationSourceManual,
})
if xerr != nil {
return nil, xerr
}
resp.Affected = n
default:
return nil, errx.NewWithIdentifier(errx.BadRequest, "invalid_action", "action must be verify, mark_deliverable or mark_undeliverable")
}
return resp, nil
}
func (s *service) Overview(ctx context.Context, orgID uuid.UUID) (*models.VerificationOverview, *errx.Error) {
counts, xerr := s.repo.VerificationCounts(ctx, orgID)
if xerr != nil {
return nil, xerr
}
out := &models.VerificationOverview{
Provider: emailverify.ProviderBuiltin,
BuiltinReady: s.builtinReady,
Counts: counts,
}
if p := s.providerFor(ctx, orgID); p != nil {
if p.ConnectionID != nil {
id := p.ConnectionID.String()
out.ConnectionID = &id
}
n, err := s.providerUsable(ctx, p)
if err != nil {
out.ProviderError = providerErrorText(err)
} else {
out.Provider = p.Name
out.Credits = &n
}
}
return out, nil
}
func providerErrorText(err error) string {
switch {
case errors.Is(err, emailverify.ErrMillionVerifierKey):
return "MillionVerifier rejected the API key. Reconnect it with a current key."
case errors.Is(err, emailverify.ErrMillionVerifierCredits):
return "The MillionVerifier account has no credits left. Top it up to keep using it; the built-in check is used meanwhile."
default:
return "MillionVerifier could not be reached; the built-in check is used meanwhile."
}
}
+14
View File
@@ -191,5 +191,19 @@ func Catalog() []models.IntegrationCatalogEntry {
// Lead Sync feature under Contacts), but it is no longer surfaced as an
// integration tile and has no event-driven append-row automation. See
// internal/app/leadsync.
// Verification --------------------------------------------------------
{
Provider: models.IntegrationMillionVerifier,
Name: "MillionVerifier",
Tagline: "Pay-as-you-go address verification for every contact you import.",
Category: models.IntegrationCategoryVerification,
AuthMethod: string(models.IntegrationAuthAPIKey),
DocsURL: "https://www.millionverifier.com/",
Highlights: []string{
"Every new contact is checked automatically; nothing to run",
"One credit per address, bought from MillionVerifier as you go",
"Replaces the built-in check while credits last, falls back when they run out",
},
},
}
}
+96
View File
@@ -7,6 +7,8 @@ import (
"encoding/json"
"errors"
"fmt"
emailverifyapp "github.com/warmbly/warmbly/internal/app/emailverify"
"github.com/warmbly/warmbly/internal/pkg/emailverify"
"strings"
"time"
@@ -161,6 +163,11 @@ type Service interface {
// configured default channel. No-op (nil) when no Slack is connected.
NotifySlack(ctx context.Context, orgID uuid.UUID, title, body string) error
// VerificationProviderFor and ReportVerificationProviderError implement
// emailverify.ProviderSource: the org's paid verification backend, if any.
VerificationProviderFor(ctx context.Context, orgID uuid.UUID) (*emailverifyapp.Provider, error)
ReportVerificationProviderError(ctx context.Context, connectionID uuid.UUID, err error)
// Repo exposes the underlying repository for the inbound webhook handlers.
Repo() repository.IntegrationRepository
}
@@ -264,6 +271,16 @@ func (s *service) Connect(ctx context.Context, orgID, userID uuid.UUID, provider
displayFields := buildDisplayFields(provider, config)
// A verification key is checked before it is stored: a mistyped key would
// otherwise quietly leave every contact on the built-in check.
if provider == models.IntegrationMillionVerifier {
credits, err := checkMillionVerifierKey(ctx, config)
if err != nil {
return nil, err
}
displayFields["credits"] = credits
}
var inboundSecret string
var err error
if provider == models.IntegrationCalendly || provider == models.IntegrationCalCom {
@@ -1327,10 +1344,89 @@ func buildDisplayFields(provider models.IntegrationProvider, config map[string]a
pick("server")
case models.IntegrationZapier, models.IntegrationMake, models.IntegrationN8N:
// Outbound-via-Warmbly-API providers: minimal display fields.
case models.IntegrationMillionVerifier:
// Credits are filled in at connect time from the provider.
}
return df
}
// checkMillionVerifierKey validates a pasted key against the provider and
// returns the account's credit balance.
func checkMillionVerifierKey(ctx context.Context, config map[string]any) (int, error) {
key, _ := config["api_key"].(string)
key = strings.TrimSpace(key)
if key == "" {
return 0, errors.New("paste your MillionVerifier API key")
}
credits, err := emailverify.NewMillionVerifier(key, "").Credits(ctx)
switch {
case errors.Is(err, emailverify.ErrMillionVerifierKey):
return 0, errors.New("MillionVerifier rejected this API key")
case errors.Is(err, emailverify.ErrMillionVerifierCredits):
// A valid key with an empty balance still connects; the built-in
// check covers until it is topped up.
return 0, nil
case err != nil:
return 0, fmt.Errorf("could not reach MillionVerifier: %w", err)
}
return credits, nil
}
// VerificationProviderFor returns the org's connected MillionVerifier client,
// or nil when none is connected. A disconnected or reauth-required connection
// does not count.
func (s *service) VerificationProviderFor(ctx context.Context, orgID uuid.UUID) (*emailverifyapp.Provider, error) {
conns, err := s.repo.ListConnections(ctx, orgID)
if err != nil {
return nil, err
}
for _, c := range conns {
if c.Provider != models.IntegrationMillionVerifier {
continue
}
if c.Status != models.IntegrationStatusConnected && c.Status != models.IntegrationStatusDegraded {
continue
}
sec, err := s.repo.GetConnectionSecrets(ctx, c.ID)
if err != nil {
return nil, err
}
cfg, err := s.openConfig(ctx, sec)
if err != nil {
return nil, err
}
key, _ := cfg["api_key"].(string)
if strings.TrimSpace(key) == "" {
continue
}
id := c.ID
return &emailverifyapp.Provider{
Name: emailverify.ProviderMillionVerifier,
ConnectionID: &id,
Client: emailverify.NewMillionVerifier(key, ""),
}, nil
}
return nil, nil
}
// ReportVerificationProviderError records why the provider stopped being
// usable on the connection card.
func (s *service) ReportVerificationProviderError(ctx context.Context, connectionID uuid.UUID, err error) {
if err == nil {
return
}
status, health := models.IntegrationStatusDegraded, models.IntegrationHealthDegraded
detail := err.Error()
switch {
case errors.Is(err, emailverify.ErrMillionVerifierKey):
status, health = models.IntegrationStatusReauthRequired, models.IntegrationHealthDown
detail = "MillionVerifier rejected the API key; reconnect with a current key"
case errors.Is(err, emailverify.ErrMillionVerifierCredits):
detail = "MillionVerifier account is out of credits; the built-in check is used until it is topped up"
}
_ = s.repo.SetConnectionStatus(ctx, connectionID, status, health, detail)
}
// slackChannelFor resolves the channel to post org notifications to. The
// OAuth connect flow doesn't capture a default channel, so we look (in order)
// at the connection's own config, then reuse whatever channel the org already
+6
View File
@@ -253,6 +253,12 @@ var Tables = []Table{
Name: "contact_categories", Group: models.OrgDataGroupContacts,
Scope: `contact_id IN ` + orgContacts,
},
{
// What real mail showed about each address; the verdict on the
// contact row is scored from it, so it travels with the contacts.
Name: "contact_verification_evidence", Group: models.OrgDataGroupContacts,
Scope: `contact_id IN ` + orgContacts,
},
{
Name: "contact_notes", Group: models.OrgDataGroupContacts,
Scope: scopeOrg,
+32
View File
@@ -158,6 +158,38 @@ const (
UniboxLimitMax = 100
UniboxLimitDefault = 50
// VerificationRecheckDays is how long a verification verdict is trusted
// before the address is checked again. Mailboxes get created and closed;
// a verdict from last quarter is a guess.
VerificationRecheckDays = 90
// VerificationUnknownRecheckDays is the shorter shelf life of an
// inconclusive verdict (greylisted, timeout, undisclosing provider).
VerificationUnknownRecheckDays = 30
// VerificationEvidenceFreshDays is how long real mail to an address (a
// delivery, an open, a reply) excuses it from being re-checked.
VerificationEvidenceFreshDays = 180
// VerificationDeliveryWindowHours is how long after a send with no
// bounce the delivery counts as evidence the mailbox exists.
VerificationDeliveryWindowHours = 72
// VerificationBatchSize is how many contacts one scheduler pass checks.
VerificationBatchSize = 200
// VerificationIntervalSeconds is how often the scheduler passes. A pass
// that finds a full batch runs again immediately, so a large import drains
// at the verifier's speed rather than one batch per interval.
VerificationIntervalSeconds = 60
// VerificationProbeConcurrency bounds parallel in-house SMTP probes.
VerificationProbeConcurrency = 4
// VerificationProviderConcurrency bounds parallel paid-provider lookups.
VerificationProviderConcurrency = 8
// VerificationBreakerWindow and VerificationBreakerInvalidPct are the
// in-house probe's self-check: when this share of the last window of
// probe verdicts is "invalid", the probe itself is suspect (issue #200,
// #264) and its invalid verdicts are filed as unknown for
// VerificationBreakerCooldownMinutes.
VerificationBreakerWindow = 200
VerificationBreakerInvalidPct = 40.0
VerificationBreakerCooldownMinutes = 60
// WarmupVerifyHeader is the custom header carrying the warmup
// verification token on outbound warmup mail. The name is intentionally
// generic (not "X-Warmbly-*") so anti-spam vendors cannot trivially
@@ -0,0 +1,14 @@
-- Postgres cannot drop a single enum value, so 'paused_undeliverable' stays on
-- campaign_status. Campaigns parked in it return to a plain pause first.
UPDATE campaigns SET status = 'paused' WHERE status = 'paused_undeliverable';
DROP INDEX IF EXISTS public.idx_contacts_verification_pending;
CREATE INDEX IF NOT EXISTS idx_contacts_verification_pending
ON public.contacts (verification_checked_at)
WHERE verification_status = 'unknown' AND verification_checked_at IS NULL;
ALTER TABLE public.contacts DROP CONSTRAINT IF EXISTS contacts_verification_source_check;
ALTER TABLE public.contacts
DROP COLUMN IF EXISTS verification_source,
DROP COLUMN IF EXISTS verification_provider,
DROP COLUMN IF EXISTS verification_sub_status;
@@ -0,0 +1,37 @@
-- Verification provenance: who produced a contact's verdict and how specific
-- it is, so an external or manual verdict is never overwritten by the in-house
-- probe, and a campaign that runs out of deliverable leads pauses for the
-- owner instead of quietly finishing.
--
-- verification_source: '' (never checked) | 'probe' (in-house SMTP probe) |
-- 'provider' (a paid backend on the org's integration) | 'imported' (a
-- status column brought in with the list or through the API) | 'manual'
-- (a member marked it deliverable). Only 'manual' is never re-checked.
-- verification_provider: which backend or vocabulary produced the verdict.
-- verification_sub_status: catch_all | disposable | role | spamtrap |
-- mailbox_full | no_mx | syntax | undisclosed | ''.
ALTER TABLE public.contacts
ADD COLUMN IF NOT EXISTS verification_source text NOT NULL DEFAULT '',
ADD COLUMN IF NOT EXISTS verification_provider text NOT NULL DEFAULT '',
ADD COLUMN IF NOT EXISTS verification_sub_status text NOT NULL DEFAULT '';
ALTER TABLE public.contacts
ADD CONSTRAINT contacts_verification_source_check
CHECK (verification_source IN ('', 'probe', 'provider', 'imported', 'manual'));
-- Every verdict recorded so far came from the probe.
UPDATE public.contacts
SET verification_source = 'probe', verification_provider = 'builtin'
WHERE verification_checked_at IS NOT NULL AND verification_source = '';
-- The scheduler now re-checks aged verdicts too, so the pending index covers
-- the check timestamp itself (NULLs first) instead of only never-checked rows.
DROP INDEX IF EXISTS public.idx_contacts_verification_pending;
CREATE INDEX IF NOT EXISTS idx_contacts_verification_pending
ON public.contacts (verification_checked_at NULLS FIRST, created_at)
WHERE verification_source <> 'manual';
-- A campaign whose remaining leads were all refused by verification parks
-- here, resumable once the owner re-verifies or marks them deliverable.
ALTER TYPE public.campaign_status ADD VALUE IF NOT EXISTS 'paused_undeliverable';
@@ -0,0 +1,5 @@
DROP INDEX IF EXISTS public.idx_campaign_progress_delivery_evidence;
ALTER TABLE public.contacts
DROP COLUMN IF EXISTS verification_confidence,
DROP COLUMN IF EXISTS verification_evidence_at;
DROP TABLE IF EXISTS public.contact_verification_evidence;
@@ -0,0 +1,35 @@
-- Evidence ledger for address verification. A probe sees an address once;
-- the platform sees what happens after every send. Each row is one observed
-- fact about a contact's mailbox (a clean delivery, a human open, a reply, a
-- bounce naming the recipient) and the contact's verdict is scored from the
-- ledger plus the last probe or provider verdict. Silence (no open, no
-- reply) is never recorded: it is not evidence of anything.
CREATE TABLE IF NOT EXISTS public.contact_verification_evidence (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
contact_id uuid NOT NULL REFERENCES public.contacts(id) ON DELETE CASCADE,
kind text NOT NULL CHECK (kind IN (
'delivered', 'opened', 'clicked', 'replied', 'auto_replied',
'bounced_recipient', 'bounced_other')),
-- ref makes an observation idempotent: the campaign step it came from,
-- or the message id of the bounce or reply.
ref text NOT NULL DEFAULT '',
detail text NOT NULL DEFAULT '',
observed_at timestamptz NOT NULL DEFAULT NOW(),
created_at timestamptz NOT NULL DEFAULT NOW(),
UNIQUE (contact_id, kind, ref)
);
CREATE INDEX IF NOT EXISTS idx_contact_verification_evidence_contact
ON public.contact_verification_evidence (contact_id, observed_at DESC);
ALTER TABLE public.contacts
ADD COLUMN IF NOT EXISTS verification_confidence smallint NOT NULL DEFAULT 0,
-- The most recent positive observation (a delivery, an open, a reply),
-- which is what excuses an address from being re-checked.
ADD COLUMN IF NOT EXISTS verification_evidence_at timestamptz;
-- The delivery-evidence job scans sent steps that never bounced.
CREATE INDEX IF NOT EXISTS idx_campaign_progress_delivery_evidence
ON public.campaign_contact_progress (sent_at)
WHERE sent_at IS NOT NULL AND bounced_at IS NULL;
+27 -16
View File
@@ -9,12 +9,12 @@ import (
emailverifyapp "github.com/warmbly/warmbly/internal/app/emailverify"
)
// EmailVerificationJob verifies a capped batch of not-yet-checked contacts each
// run, so the platform can drop hard-bouncing addresses before any worker sends
// to them. It is a thin wrapper around emailverify.Service.VerifyPending; the
// per-tick cap bounds how much outbound SMTP-probe work one pass does.
// EmailVerificationJob verifies a batch of contacts due for a check each run,
// so the platform can drop hard-bouncing addresses before any worker sends
// to them. A run that fills its batch is repeated until the backlog is below
// one batch, so a large import drains at the verifier's speed.
//
// Control-plane only: the underlying verifier dials remote MX hosts on :25 and
// Control-plane only: the in-house verifier dials remote MX hosts on :25 and
// must run from the backend/consumer, never a worker (sending) IP.
type EmailVerificationJob struct {
svc emailverifyapp.Service
@@ -22,7 +22,7 @@ type EmailVerificationJob struct {
}
// NewEmailVerificationJob creates the job. batchSize caps how many contacts are
// verified per tick (defaults to 100 when non-positive).
// verified per pass (defaults to 100 when non-positive).
func NewEmailVerificationJob(svc emailverifyapp.Service, batchSize int) *EmailVerificationJob {
if batchSize <= 0 {
batchSize = 100
@@ -30,20 +30,26 @@ func NewEmailVerificationJob(svc emailverifyapp.Service, batchSize int) *EmailVe
return &EmailVerificationJob{svc: svc, batchSize: batchSize}
}
// Run performs one capped verification pass. Safe to call frequently it
// no-ops when there are no unverified contacts.
// Run drains the backlog in batches. Safe to call frequently; it no-ops when
// nothing is due.
func (j *EmailVerificationJob) Run(ctx context.Context) error {
if j.svc == nil {
return nil
}
if _, err := j.svc.VerifyPending(ctx, j.batchSize); err != nil {
sentry.CaptureException(err)
return err
for {
n, err := j.svc.VerifyPending(ctx, j.batchSize)
if err != nil {
sentry.CaptureException(err)
return err
}
if n < j.batchSize || ctx.Err() != nil {
return nil
}
}
return nil
}
// EmailVerificationScheduler runs the job on a fixed interval.
// EmailVerificationScheduler runs the job on a fixed interval and whenever
// the service is kicked (a re-verify request, an import).
type EmailVerificationScheduler struct {
job *EmailVerificationJob
interval time.Duration
@@ -64,17 +70,22 @@ func (s *EmailVerificationScheduler) Start(ctx context.Context) {
ticker := time.NewTicker(s.interval)
defer ticker.Stop()
var wake <-chan struct{}
if s.job != nil && s.job.svc != nil {
wake = s.job.svc.Wake()
}
for {
select {
case <-ticker.C:
if err := s.job.Run(ctx); err != nil {
sentry.CaptureException(err)
}
case <-wake:
case <-s.stopCh:
return
case <-ctx.Done():
return
}
if err := s.job.Run(ctx); err != nil {
sentry.CaptureException(err)
}
}
}
+50
View File
@@ -0,0 +1,50 @@
package jobs
import (
"context"
"time"
"github.com/getsentry/sentry-go"
emailverifyapp "github.com/warmbly/warmbly/internal/app/emailverify"
)
// DeliveryEvidenceJob turns campaign sends that never bounced into
// verification evidence: a delivery the recipient's server kept is the
// strongest proof the mailbox exists, and it costs nothing to observe.
type DeliveryEvidenceJob struct {
evidence *emailverifyapp.Evidence
interval time.Duration
batch int
}
func NewDeliveryEvidenceJob(evidence *emailverifyapp.Evidence, interval time.Duration, batch int) *DeliveryEvidenceJob {
if batch <= 0 {
batch = 2000
}
return &DeliveryEvidenceJob{evidence: evidence, interval: interval, batch: batch}
}
// Start runs the job on its interval until ctx ends. A full batch repeats
// at once so a backlog drains.
func (j *DeliveryEvidenceJob) Start(ctx context.Context) {
ticker := time.NewTicker(j.interval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
case <-ctx.Done():
return
}
for {
n, err := j.evidence.CreditCleanDeliveries(ctx, j.batch)
if err != nil {
sentry.CaptureException(err)
break
}
if n < j.batch || ctx.Err() != nil {
break
}
}
}
}
+1 -1
View File
@@ -309,7 +309,7 @@ type AdminCampaignSearch struct {
Query string `form:"q"`
UserID *uuid.UUID `form:"user_id"`
OrgID *uuid.UUID `form:"org_id"`
Status string `form:"status"` // draft, active, paused, completed, paused_trial_expired, paused_no_accounts
Status string `form:"status"` // draft, active, paused, completed, paused_trial_expired, paused_no_accounts, paused_guardrail, paused_undeliverable
// Boolean flags
OpenTracking bool `form:"open_tracking"`
+11
View File
@@ -358,3 +358,14 @@ type CreateSequenceInput struct {
BodyCode *bool `json:"body_code,omitempty"`
WaitAfter *int `json:"wait_after,omitempty"`
}
// StartCampaignOptions qualifies a start request.
type StartCampaignOptions struct {
// AcknowledgeListRisk launches past the bounce-risk gate: the member has
// read the projection and takes the risk (the list may have been verified
// elsewhere).
AcknowledgeListRisk bool `json:"acknowledge_list_risk"`
// Automatic marks a start the platform initiated (a resume after
// verification), which skips the member-facing cooldown.
Automatic bool `json:"-"`
}
+125 -6
View File
@@ -38,6 +38,17 @@ type Contact struct {
VerificationReason string `json:"verification_reason"`
IsCatchAll bool `json:"is_catch_all"`
VerificationCheckedAt *time.Time `json:"verification_checked_at,omitempty"`
// VerificationSource says who produced the verdict (see the
// VerificationSource* constants); VerificationProvider names the backend or
// the external vocabulary; VerificationSubStatus refines the status
// (catch_all, disposable, role, ...).
VerificationSource string `json:"verification_source"`
VerificationProvider string `json:"verification_provider"`
VerificationSubStatus string `json:"verification_sub_status"`
// VerificationConfidence is how sure the platform is of the status, 0 to
// 100, scored from the last check plus what real mail to the address
// showed (deliveries, opens, replies, bounces).
VerificationConfidence int `json:"verification_confidence"`
// Recipient ESP/provider, derived in the control plane from the recipient
// domain (never an MX dial on the send hot path). '' | 'gmail' | 'outlook'
@@ -178,12 +189,103 @@ type CampaignLeadCounts struct {
// ContactsCounts are org-wide contact facet totals for the browse sidebar.
type ContactsCounts struct {
Total int `json:"total"`
Subscribed int `json:"subscribed"`
Unsubscribed int `json:"unsubscribed"`
InCampaign int `json:"in_campaign"`
NotContacted int `json:"not_contacted"`
Categories []ContactCategoryCount `json:"categories"`
Total int `json:"total"`
Subscribed int `json:"subscribed"`
Unsubscribed int `json:"unsubscribed"`
InCampaign int `json:"in_campaign"`
NotContacted int `json:"not_contacted"`
Categories []ContactCategoryCount `json:"categories"`
Verification ContactVerificationCounts `json:"verification"`
}
// ContactVerificationCounts is the org's contacts by verification status.
// Pending is the subset of Unknown nobody has checked yet.
type ContactVerificationCounts struct {
Valid int `json:"valid"`
Risky int `json:"risky"`
Invalid int `json:"invalid"`
Unknown int `json:"unknown"`
Pending int `json:"pending"`
}
// ContactVerificationDetail is the "why" behind a contact's verdict.
type ContactVerificationDetail struct {
Status string `json:"status"`
Confidence int `json:"confidence"`
Reasons []string `json:"reasons"`
// Decisive is true when real mail, not a check, decided the status.
Decisive bool `json:"decisive"`
Evidence []ContactVerificationEvidence `json:"evidence"`
}
// ContactVerificationEvidence is one observed fact about the mailbox.
type ContactVerificationEvidence struct {
Kind string `json:"kind"`
Detail string `json:"detail,omitempty"`
ObservedAt time.Time `json:"observed_at"`
}
// Verification provenance values for contacts.verification_source.
const (
VerificationSourceNone = ""
VerificationSourceProbe = "probe"
VerificationSourceProvider = "provider"
VerificationSourceImported = "imported"
VerificationSourceManual = "manual"
)
// ContactVerificationWrite is a verdict to store on a contact, already
// normalised into Warmbly's vocabulary.
type ContactVerificationWrite struct {
Status string
SubStatus string
Reason string
Provider string
Source string
}
// Actions for POST /contacts/verification.
const (
ContactVerificationActionVerify = "verify"
ContactVerificationActionMarkDeliverable = "mark_deliverable"
ContactVerificationActionMarkUndeliverable = "mark_undeliverable"
)
// ContactVerificationRequest is the body of POST /contacts/verification.
type ContactVerificationRequest struct {
Contacts []string `json:"contacts"`
// CampaignID selects every lead of one campaign that verification refused
// (the "re-verify skipped leads" action), instead of listing ids.
CampaignID string `json:"campaign_id,omitempty"`
Action string `json:"action"`
}
// ContactVerificationResponse reports how many contacts the action touched.
type ContactVerificationResponse struct {
Affected int `json:"affected"`
Action string `json:"action"`
// Queued is true for the verify action: the check runs in the background
// and each contact updates live as its verdict lands.
Queued bool `json:"queued"`
}
// VerificationOverview is what Settings shows about address verification.
type VerificationOverview struct {
// Provider is who checks this workspace's addresses: "builtin" or
// "millionverifier".
Provider string `json:"provider"`
// ConnectionID is the integration connection behind a paid provider.
ConnectionID *string `json:"connection_id,omitempty"`
// Credits is the paid provider's remaining balance when it could be read.
Credits *int `json:"credits,omitempty"`
// ProviderError is set when the paid provider is connected but unusable
// (bad key, no credits), in which case the built-in check is in use.
ProviderError string `json:"provider_error,omitempty"`
// BuiltinReady says whether the in-house probe can reach mail servers from
// this instance (a HELO host is configured). Off, it still checks syntax,
// MX, and disposable domains.
BuiltinReady bool `json:"builtin_ready"`
Counts ContactVerificationCounts `json:"counts"`
}
// ContactCategoryCount is the number of org contacts carrying one category.
@@ -226,6 +328,9 @@ type ContactDetail struct {
Contact
Engagement ContactEngagement `json:"engagement"`
Suppression *ContactSuppression `json:"suppression,omitempty"`
// Verification explains the verdict: the reasons behind it and the
// observations it was scored from.
Verification *ContactVerificationDetail `json:"verification,omitempty"`
// First-touch attribution. Source never changes after creation.
Source ContactSource `json:"source"`
@@ -383,6 +488,19 @@ type AddContact struct {
CustomFields map[string]string `json:"custom_fields"`
// VerificationStatus is a verdict the caller already holds for this
// address, in Warmbly's vocabulary or any provider's the platform knows
// (ZeroBounce, MillionVerifier, NeverBounce, ...). VerificationProvider
// optionally names that vocabulary; without it the value is recognised by
// itself. An unrecognised value is a 400. Stored as an imported verdict,
// which the background check leaves alone until it ages out.
VerificationStatus string `json:"verification_status,omitempty"`
VerificationProvider string `json:"verification_provider,omitempty"`
// Verification is the normalised verdict derived from VerificationStatus.
// Filled by the repository, never read from the request.
Verification *ContactVerificationWrite `json:"-"`
// Subscribed is the marketing-consent flag to store. nil means "don't
// decide": a new contact defaults to subscribed, an existing one keeps
// whatever it already had. Set explicitly by the importer when the file
@@ -454,6 +572,7 @@ type SearchContacts struct {
MinCampaigns *int `json:"min_campaigns"` // Minimum number of associated campaigns
MaxCampaigns *int `json:"max_campaigns"` // Maximum number of associated campaigns
Subscribed *bool `json:"subscribed"` // Filter by subscription status
VerificationStatus string `json:"verification_status"` // Filter by verification verdict: valid | risky | invalid | unknown
CreatedAfter *time.Time `json:"created_after"` // Contacts created after this date
CreatedBefore *time.Time `json:"created_before"` // Contacts created before this date
UpdatedAfter *time.Time `json:"updated_after"` // Contacts updated after this date
+10
View File
@@ -42,6 +42,11 @@ const (
ContactImportTargetPhone ContactImportColumnTarget = "phone"
ContactImportTargetSubscribed ContactImportColumnTarget = "subscribed"
ContactImportTargetCategories ContactImportColumnTarget = "categories"
// ContactImportTargetVerificationStatus reads a verdict column written by
// Warmbly or another verification service (ZeroBounce, MillionVerifier,
// NeverBounce, ...). Values are recognised by vocabulary; a value nobody
// knows leaves the contact unverified rather than failing the row.
ContactImportTargetVerificationStatus ContactImportColumnTarget = "verification_status"
// ContactImportTargetCustom routes the column into Contact.CustomFields
// under ContactImportColumnMapping.CustomKey. "custom:<key>" is accepted
// as an equivalent legacy spelling.
@@ -56,6 +61,11 @@ type ContactImportColumnMapping struct {
Index int `json:"index"`
Target ContactImportColumnTarget `json:"target"`
// VerificationProvider names the vocabulary of a verification_status
// column when the header or its values made it clear (e.g. "zerobounce").
// Optional; without it each value is recognised by itself.
VerificationProvider string `json:"verification_provider,omitempty"`
// CustomKey is only used when Target == "custom:<anything>". It
// is split out so the client can render a nicer label without
// having to parse the target string.
+5
View File
@@ -35,6 +35,9 @@ const (
// Data
IntegrationGoogleSheets IntegrationProvider = "google_sheets"
// Verification
IntegrationMillionVerifier IntegrationProvider = "millionverifier"
)
// AllIntegrationProviders lists every provider the dashboard exposes. The
@@ -52,6 +55,7 @@ var AllIntegrationProviders = []IntegrationProvider{
IntegrationCalendly,
IntegrationCalCom,
IntegrationGoogleSheets,
IntegrationMillionVerifier,
}
func IsValidIntegrationProvider(s string) bool {
@@ -118,6 +122,7 @@ const (
IntegrationCategoryNotifications IntegrationCategory = "notifications"
IntegrationCategoryMeetings IntegrationCategory = "meetings"
IntegrationCategoryData IntegrationCategory = "data"
IntegrationCategoryVerification IntegrationCategory = "verification"
)
// IntegrationCatalogEntry is the static metadata for one provider that the
+128 -7
View File
@@ -43,6 +43,8 @@ import (
"strconv"
"strings"
"time"
"github.com/warmbly/warmbly/internal/pkg/signuprisk"
)
// Status is the verification outcome for a single address. It is a small closed
@@ -65,15 +67,45 @@ const (
StatusUnknown Status = "unknown"
)
// SubStatus refines a Status with the reason class a paid provider or the
// in-house checks can name. Empty when nothing more specific is known.
type SubStatus string
const (
SubStatusNone SubStatus = ""
SubStatusCatchAll SubStatus = "catch_all"
SubStatusDisposable SubStatus = "disposable"
SubStatusRole SubStatus = "role"
SubStatusSpamTrap SubStatus = "spamtrap"
SubStatusMailboxFull SubStatus = "mailbox_full"
SubStatusNoMX SubStatus = "no_mx"
SubStatusSyntax SubStatus = "syntax"
// SubStatusUndisclosed marks a provider (Microsoft, Yahoo) that answers
// every RCPT the same way, so an SMTP probe cannot judge the mailbox.
SubStatusUndisclosed SubStatus = "undisclosed"
)
// Provider names for Result.Provider and contacts.verification_provider.
const (
ProviderBuiltin = "builtin"
ProviderMillionVerifier = "millionverifier"
)
// Result is the outcome of verifying one address. It round-trips into the
// contacts table (verification_status / verification_reason / is_catch_all /
// verification_checked_at).
// verification_checked_at / verification_sub_status / verification_provider).
type Result struct {
Email string `json:"email"`
Status Status `json:"status"`
SubStatus SubStatus `json:"sub_status,omitempty"`
Reason string `json:"reason"`
IsCatchAll bool `json:"is_catch_all"`
HasMX bool `json:"has_mx"`
// Provider is who produced the verdict: ProviderBuiltin, a paid backend,
// or the vocabulary an imported result was recognised as.
Provider string `json:"provider,omitempty"`
// Confidence is the scored certainty of Status once evidence is applied.
Confidence int `json:"confidence,omitempty"`
CheckedAt time.Time `json:"checked_at"`
}
@@ -125,6 +157,9 @@ func (c Config) withDefaults() Config {
return c
}
// ProbeReady reports whether the configured HELO host lets the SMTP probe run.
func (v *SMTPVerifier) ProbeReady() bool { return isFQDN(v.cfg.HeloHost) }
// SMTPVerifier is the in-house Verifier: syntax -> MX -> SMTP RCPT probe ->
// catch-all detection. It opens exactly one connection to the lowest-preference
// MX and probes both the real address and a random localpart on the same
@@ -135,6 +170,9 @@ type SMTPVerifier struct {
// smtpPort is always "25" in production (MX hosts listen nowhere else);
// it exists so tests can point probe() at a local server.
smtpPort string
// domains remembers per-domain facts (no MX, catch-all, undisclosing
// provider) so a 50k list at 2k domains costs 2k probes, not 50k.
domains *domainCache
}
// New constructs the in-house SMTP verifier. The resolver mirrors dnsauth's
@@ -144,6 +182,7 @@ func New(cfg Config) *SMTPVerifier {
cfg: cfg.withDefaults(),
resolver: &net.Resolver{},
smtpPort: "25",
domains: newDomainCache(domainCacheTTL),
}
}
@@ -152,12 +191,13 @@ func New(cfg Config) *SMTPVerifier {
// code path.
func (v *SMTPVerifier) Verify(ctx context.Context, email string) Result {
now := time.Now().UTC()
res := Result{Email: email, CheckedAt: now, Status: StatusUnknown}
res := Result{Email: email, CheckedAt: now, Status: StatusUnknown, Provider: ProviderBuiltin}
// 1. Syntax (RFC 5322-ish via net/mail). A parse failure is a hard invalid.
addr, err := mail.ParseAddress(email)
if err != nil {
res.Status = StatusInvalid
res.SubStatus = SubStatusSyntax
res.Reason = "invalid syntax"
return res
}
@@ -166,14 +206,49 @@ func (v *SMTPVerifier) Verify(ctx context.Context, email string) Result {
at := strings.LastIndex(normalized, "@")
if at <= 0 || at == len(normalized)-1 {
res.Status = StatusInvalid
res.SubStatus = SubStatusSyntax
res.Reason = "invalid syntax"
return res
}
localpart := normalized[:at]
domain := normalized[at+1:]
// 2. MX lookup. No MX (and no usable fallback) is a hard invalid: nowhere to
// deliver. A lookup *error* (timeout/SERVFAIL) is unknown, not invalid.
// Address-level facts that need no network. A disposable domain is never
// worth a send; a role address is deliverable but flagged, so the campaign's
// risky toggle decides.
if signuprisk.IsDisposable(normalized) {
res.Status = StatusInvalid
res.SubStatus = SubStatusDisposable
res.Reason = "disposable email domain"
return res
}
role := isRoleLocalpart(localpart)
// 2. Domain facts, cached: no MX and catch-all are properties of the
// domain, and providers that never disclose mailboxes are known by MX.
if cached, ok := v.domains.get(domain); ok {
switch cached.kind {
case domainNoMX:
res.Status = StatusInvalid
res.SubStatus = SubStatusNoMX
res.Reason = "no MX records"
return res
case domainCatchAll:
res.HasMX = true
res.IsCatchAll = true
res.Status = StatusRisky
res.SubStatus = SubStatusCatchAll
res.Reason = "catch-all domain; acceptance is not conclusive"
return res
case domainUndisclosed:
res.HasMX = true
res.Status = StatusUnknown
res.SubStatus = SubStatusUndisclosed
res.Reason = cached.reason
return res
}
}
hosts, mxErr := v.lookupMXHosts(ctx, domain)
if mxErr != nil {
res.Status = StatusUnknown
@@ -181,12 +256,26 @@ func (v *SMTPVerifier) Verify(ctx context.Context, email string) Result {
return res
}
if len(hosts) == 0 {
v.domains.put(domain, domainFact{kind: domainNoMX})
res.Status = StatusInvalid
res.SubStatus = SubStatusNoMX
res.Reason = "no MX records"
return res
}
res.HasMX = true
// Providers that accept every RCPT and bounce later (Microsoft 365,
// Yahoo) make a probe meaningless: say so instead of spending a
// connection and calling the result "valid".
if fp := fingerprintMX(hosts); fp.undisclosed {
reason := fp.name + " does not disclose mailboxes to an SMTP probe"
v.domains.put(domain, domainFact{kind: domainUndisclosed, reason: reason})
res.Status = StatusUnknown
res.SubStatus = SubStatusUndisclosed
res.Reason = reason
return res
}
// 3. SMTP RCPT probe against the lowest-preference (highest priority) MX.
// Refused outright without a real HELO identity: a probe that announces a
// bare or reserved name gets the SESSION rejected, and that rejection
@@ -196,17 +285,25 @@ func (v *SMTPVerifier) Verify(ctx context.Context, email string) Result {
res.Reason = "smtp probe skipped: set EMAIL_VERIFY_HELO_HOST to a public fully-qualified hostname for this instance"
return res
}
probe := v.probe(ctx, hosts[0], localpart, domain)
probe := v.probeHosts(ctx, hosts, localpart, domain)
switch probe.outcome {
case probeAccepted:
// 4. Catch-all check already folded into probe(): if the random control
// localpart was also accepted, the 250 on the real address is meaningless.
if probe.catchAll {
v.domains.put(domain, domainFact{kind: domainCatchAll})
res.IsCatchAll = true
res.Status = StatusRisky
res.SubStatus = SubStatusCatchAll
res.Reason = "catch-all domain; acceptance is not conclusive"
return res
}
if role {
res.Status = StatusRisky
res.SubStatus = SubStatusRole
res.Reason = "role address (shared inbox); recipient accepted"
return res
}
res.Status = StatusValid
res.Reason = "recipient accepted"
return res
@@ -221,6 +318,27 @@ func (v *SMTPVerifier) Verify(ctx context.Context, email string) Result {
}
}
// probeHosts tries the MX hosts in preference order until one answers the
// session. A dial or handshake failure on the primary is common (it is the
// busiest host) and says nothing about the mailbox; only a host that talked
// to us gets to decide.
func (v *SMTPVerifier) probeHosts(ctx context.Context, hosts []string, localpart, domain string) probeResult {
var last probeResult
for i, host := range hosts {
if i >= maxMXAttempts {
break
}
if err := ctx.Err(); err != nil {
return probeResult{outcome: probeUnknown, reason: "cancelled: " + err.Error()}
}
last = v.probe(ctx, host, localpart, domain)
if last.outcome != probeUnknown || !last.sessionFailed {
return last
}
}
return last
}
// lookupMXHosts returns MX hosts ordered by ascending preference (most-preferred
// first). When a domain publishes no MX, RFC 5321 permits implicit-MX fallback
// to the A/AAAA record of the domain itself; we honour that so apex-only mail
@@ -276,6 +394,9 @@ type probeResult struct {
outcome probeOutcome
catchAll bool
reason string
// sessionFailed is set when the host never judged the address (dial or
// handshake failure), so the next MX is worth trying.
sessionFailed bool
}
// probe opens one SMTP session to host:25, greets it, sets the envelope sender,
@@ -293,7 +414,7 @@ func (v *SMTPVerifier) probe(ctx context.Context, host, localpart, domain string
if err != nil {
// Most commonly: outbound :25 blocked by the cloud provider, or the MX is
// firewalled/tarpitting. Either way we cannot conclude invalid.
return probeResult{outcome: probeUnknown, reason: "smtp dial failed (port 25 may be blocked): " + err.Error()}
return probeResult{outcome: probeUnknown, sessionFailed: true, reason: "smtp dial failed (port 25 may be blocked): " + err.Error()}
}
// Bound the whole session.
deadline := time.Now().Add(v.cfg.CommandTimeout * 4)
@@ -305,7 +426,7 @@ func (v *SMTPVerifier) probe(ctx context.Context, host, localpart, domain string
client, err := smtp.NewClient(conn, host)
if err != nil {
_ = conn.Close()
return probeResult{outcome: probeUnknown, reason: "smtp handshake failed: " + err.Error()}
return probeResult{outcome: probeUnknown, sessionFailed: true, reason: "smtp handshake failed: " + err.Error()}
}
defer func() { _ = client.Close() }()
+133
View File
@@ -0,0 +1,133 @@
package emailverify
import (
"strings"
"sync"
"time"
)
const (
// domainCacheTTL bounds how long a domain fact (no MX, catch-all,
// undisclosing provider) is trusted before it is re-learned.
domainCacheTTL = 24 * time.Hour
// domainCacheMax caps the in-process cache; the oldest entries are evicted.
domainCacheMax = 50_000
// maxMXAttempts bounds how many MX hosts one address may cost.
maxMXAttempts = 3
)
type domainKind int
const (
domainNoMX domainKind = iota + 1
domainCatchAll
domainUndisclosed
)
type domainFact struct {
kind domainKind
reason string
at time.Time
}
// domainCache is a small TTL map keyed by domain. In-process only: it is a
// cost optimisation, and a cold cache after a restart is merely slower.
type domainCache struct {
mu sync.Mutex
ttl time.Duration
data map[string]domainFact
}
func newDomainCache(ttl time.Duration) *domainCache {
return &domainCache{ttl: ttl, data: make(map[string]domainFact)}
}
func (c *domainCache) get(domain string) (domainFact, bool) {
if c == nil {
return domainFact{}, false
}
c.mu.Lock()
defer c.mu.Unlock()
f, ok := c.data[domain]
if !ok {
return domainFact{}, false
}
if time.Since(f.at) > c.ttl {
delete(c.data, domain)
return domainFact{}, false
}
return f, true
}
func (c *domainCache) put(domain string, f domainFact) {
if c == nil {
return
}
c.mu.Lock()
defer c.mu.Unlock()
if len(c.data) >= domainCacheMax {
// Evict the stalest quarter rather than scanning on every insert.
cutoff := time.Now().Add(-c.ttl / 4)
for k, v := range c.data {
if v.at.Before(cutoff) {
delete(c.data, k)
}
}
if len(c.data) >= domainCacheMax {
for k := range c.data {
delete(c.data, k)
if len(c.data) < domainCacheMax*3/4 {
break
}
}
}
}
f.at = time.Now()
c.data[domain] = f
}
// mxFingerprint is what the MX hostnames reveal about the receiving provider.
type mxFingerprint struct {
name string
// undisclosed providers accept every RCPT and reject at delivery time, so
// an accepted probe proves nothing and a rejected one never happens.
undisclosed bool
}
// undisclosingMX lists MX suffixes of providers whose RCPT answer is not a
// verdict. Microsoft 365 answers 250 for any address on a tenant and bounces
// later; Yahoo/AOL tarpit and accept. Google, by contrast, answers truthfully
// (550 5.1.1) and is left to the probe.
var undisclosingMX = []struct{ suffix, name string }{
{".mail.protection.outlook.com", "Microsoft 365"},
{".olc.protection.outlook.com", "Outlook.com"},
{".mail.eo.outlook.com", "Microsoft 365"},
{".yahoodns.net", "Yahoo"},
{".mx.aol.com", "AOL"},
}
func fingerprintMX(hosts []string) mxFingerprint {
for _, h := range hosts {
lh := strings.ToLower(strings.TrimSuffix(h, "."))
for _, u := range undisclosingMX {
if strings.HasSuffix(lh, u.suffix) {
return mxFingerprint{name: u.name, undisclosed: true}
}
}
}
return mxFingerprint{}
}
// roleLocalparts is the same shared-inbox vocabulary the import quality check
// and the launch gate use, so one list is never described three ways.
var roleLocalparts = map[string]bool{
"info": true, "sales": true, "support": true, "contact": true,
"admin": true, "hello": true, "help": true, "office": true, "team": true,
"billing": true, "careers": true, "jobs": true, "marketing": true,
"noreply": true, "no-reply": true, "webmaster": true,
"enquiries": true, "enquiry": true, "postmaster": true, "abuse": true,
}
func isRoleLocalpart(local string) bool {
return roleLocalparts[strings.ToLower(local)]
}
+179
View File
@@ -0,0 +1,179 @@
package emailverify
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
)
// MillionVerifier is the paid Verifier backed by MillionVerifier's single
// address API. Pay-as-you-go credits, one credit per lookup; the customer's
// own key is stored on their integration connection.
type MillionVerifier struct {
apiKey string
baseURL string
client *http.Client
}
const (
millionVerifierAPI = "https://api.millionverifier.com"
millionVerifierTimeout = 20 * time.Second
// millionVerifierProbeTimeout is the per-address server-side budget, in
// seconds, sent as the API's timeout parameter.
millionVerifierProbeTimeout = 15
)
var (
// ErrMillionVerifierKey is returned when the API rejects the key.
ErrMillionVerifierKey = errors.New("millionverifier rejected the API key")
// ErrMillionVerifierCredits is returned when the account is out of credits.
ErrMillionVerifierCredits = errors.New("millionverifier account has no credits left")
)
// NewMillionVerifier constructs the client. baseURL is overridable for tests.
func NewMillionVerifier(apiKey string, baseURL string) *MillionVerifier {
if baseURL == "" {
baseURL = millionVerifierAPI
}
return &MillionVerifier{
apiKey: strings.TrimSpace(apiKey),
baseURL: strings.TrimRight(baseURL, "/"),
client: &http.Client{Timeout: millionVerifierTimeout},
}
}
type mvSingleResponse struct {
Email string `json:"email"`
Quality string `json:"quality"`
Result string `json:"result"`
ResultCode int `json:"resultcode"`
SubResult string `json:"subresult"`
Free bool `json:"free"`
Role bool `json:"role"`
Error string `json:"error"`
Credits int `json:"credits"`
}
type mvCreditsResponse struct {
Credits int `json:"credits"`
Error string `json:"error"`
}
// Verify implements Verifier. An API failure is never an "invalid" verdict:
// it degrades to unknown with the reason, and the typed error is reported
// through Check so the connection can be marked degraded.
func (m *MillionVerifier) Verify(ctx context.Context, email string) Result {
res, _ := m.Check(ctx, email)
return res
}
// Check is Verify with the transport/account error surfaced.
func (m *MillionVerifier) Check(ctx context.Context, email string) (Result, error) {
now := time.Now().UTC()
res := Result{Email: strings.ToLower(strings.TrimSpace(email)), CheckedAt: now, Status: StatusUnknown, Provider: ProviderMillionVerifier}
q := url.Values{}
q.Set("api", m.apiKey)
q.Set("email", res.Email)
q.Set("timeout", fmt.Sprintf("%d", millionVerifierProbeTimeout))
req, err := http.NewRequestWithContext(ctx, http.MethodGet, m.baseURL+"/api/v3/?"+q.Encode(), nil)
if err != nil {
res.Reason = "millionverifier request failed: " + err.Error()
return res, err
}
resp, err := m.client.Do(req)
if err != nil {
res.Reason = "millionverifier unreachable: " + err.Error()
return res, err
}
defer resp.Body.Close()
body, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {
res.Reason = "millionverifier rejected the API key"
return res, ErrMillionVerifierKey
}
if resp.StatusCode != http.StatusOK {
res.Reason = fmt.Sprintf("millionverifier answered HTTP %d", resp.StatusCode)
return res, fmt.Errorf("millionverifier: http %d", resp.StatusCode)
}
var out mvSingleResponse
if err := json.Unmarshal(body, &out); err != nil {
res.Reason = "millionverifier answered with an unreadable body"
return res, err
}
if out.Error != "" {
return res, m.accountError(&res, out.Error)
}
v, ok := NormalizeExternal(ProviderMillionVerifier, out.Result)
if !ok {
res.Reason = "millionverifier result not recognised: " + out.Result
return res, nil
}
res.Status, res.SubStatus = v.Status, v.SubStatus
res.IsCatchAll = v.SubStatus == SubStatusCatchAll
res.HasMX = out.Result != "invalid" || out.SubResult != ""
if out.Role && res.Status == StatusValid {
res.Status = StatusRisky
res.SubStatus = SubStatusRole
}
reason := out.Result
if out.SubResult != "" {
reason += " (" + out.SubResult + ")"
}
res.Reason = "millionverifier: " + reason
return res, nil
}
// Credits returns the account's remaining credits, and validates the key.
func (m *MillionVerifier) Credits(ctx context.Context) (int, error) {
q := url.Values{}
q.Set("api", m.apiKey)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, m.baseURL+"/api/v3/credits?"+q.Encode(), nil)
if err != nil {
return 0, err
}
resp, err := m.client.Do(req)
if err != nil {
return 0, err
}
defer resp.Body.Close()
body, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {
return 0, ErrMillionVerifierKey
}
if resp.StatusCode != http.StatusOK {
return 0, fmt.Errorf("millionverifier: http %d", resp.StatusCode)
}
var out mvCreditsResponse
if err := json.Unmarshal(body, &out); err != nil {
return 0, err
}
if out.Error != "" {
return 0, m.accountError(nil, out.Error)
}
return out.Credits, nil
}
func (m *MillionVerifier) accountError(res *Result, msg string) error {
lower := strings.ToLower(msg)
var err error
switch {
case strings.Contains(lower, "api key") || strings.Contains(lower, "apikey") || strings.Contains(lower, "unauthori"):
err = ErrMillionVerifierKey
case strings.Contains(lower, "credit"):
err = ErrMillionVerifierCredits
default:
err = fmt.Errorf("millionverifier: %s", msg)
}
if res != nil {
res.Reason = "millionverifier: " + msg
}
return err
}
@@ -0,0 +1,60 @@
package emailverify
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"testing"
)
func TestMillionVerifierMapsResults(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Query().Get("api") != "key" {
http.Error(w, `{"error":"api key not found"}`, http.StatusOK)
return
}
switch r.URL.Path {
case "/api/v3/credits":
_, _ = w.Write([]byte(`{"credits": 42}`))
case "/api/v3/":
switch r.URL.Query().Get("email") {
case "good@x.com":
_, _ = w.Write([]byte(`{"email":"good@x.com","quality":"good","result":"ok","resultcode":1,"subresult":"","free":false,"role":false,"error":"","credits":41}`))
case "info@x.com":
_, _ = w.Write([]byte(`{"email":"info@x.com","quality":"good","result":"ok","resultcode":1,"subresult":"","free":false,"role":true,"error":"","credits":40}`))
case "gone@x.com":
_, _ = w.Write([]byte(`{"email":"gone@x.com","quality":"bad","result":"invalid","resultcode":6,"subresult":"user_unknown","free":false,"role":false,"error":"","credits":39}`))
default:
_, _ = w.Write([]byte(`{"error":"insufficient credits"}`))
}
}
}))
defer srv.Close()
mv := NewMillionVerifier("key", srv.URL)
if n, err := mv.Credits(context.Background()); err != nil || n != 42 {
t.Fatalf("credits = %d, %v", n, err)
}
res, err := mv.Check(context.Background(), "good@x.com")
if err != nil || res.Status != StatusValid || res.Provider != ProviderMillionVerifier {
t.Fatalf("good = %+v, %v", res, err)
}
res, _ = mv.Check(context.Background(), "info@x.com")
if res.Status != StatusRisky || res.SubStatus != SubStatusRole {
t.Fatalf("role = %+v", res)
}
res, _ = mv.Check(context.Background(), "gone@x.com")
if res.Status != StatusInvalid || res.Reason != "millionverifier: invalid (user_unknown)" {
t.Fatalf("gone = %+v", res)
}
res, err = mv.Check(context.Background(), "broke@x.com")
if !errors.Is(err, ErrMillionVerifierCredits) || res.Status != StatusUnknown {
t.Fatalf("no credits = %+v, %v", res, err)
}
bad := NewMillionVerifier("wrong", srv.URL)
if _, err := bad.Credits(context.Background()); !errors.Is(err, ErrMillionVerifierKey) {
t.Fatalf("bad key = %v", err)
}
}
+287
View File
@@ -0,0 +1,287 @@
package emailverify
import (
"fmt"
"math"
"sort"
"strings"
"time"
)
// Evidence kinds. Positive kinds prove the mailbox is live; the recipient
// bounce proves it is not. Nothing here describes interest: a contact who
// never opens or replies has told us nothing about their address.
const (
EvidenceDelivered = "delivered"
EvidenceOpened = "opened"
EvidenceClicked = "clicked"
EvidenceReplied = "replied"
EvidenceAutoReplied = "auto_replied"
EvidenceBouncedRecipient = "bounced_recipient"
EvidenceBouncedOther = "bounced_other"
)
// Evidence is one observed fact about a mailbox.
type Evidence struct {
Kind string
Detail string
ObservedAt time.Time
}
// Verdict is the last probe, provider, import or manual verdict on the
// contact row.
type Verdict struct {
Status Status
// Source is the contacts.verification_source value.
Source string
CheckedAt time.Time
}
// Scored is the derived state of an address.
type Scored struct {
Status Status
// Confidence is how sure the platform is of Status, 0 to 100.
Confidence int
// Reasons are the sentences a member reads, strongest first.
Reasons []string
// Decisive reports whether real mail (not a check) decided the status.
Decisive bool
// LastPositiveAt is the most recent positive observation.
LastPositiveAt time.Time
}
// Evidence weights. Weights decay with a half-life so last week's reply
// outranks last year's, but nothing ever reaches zero on its own.
var evidenceWeights = map[string]struct {
weight float64
halfLife time.Duration
positive bool
// cap bounds how many observations of this kind count.
cap int
}{
EvidenceReplied: {45, 365 * 24 * time.Hour, true, 3},
EvidenceAutoReplied: {30, 180 * 24 * time.Hour, true, 2},
EvidenceClicked: {35, 270 * 24 * time.Hour, true, 3},
EvidenceOpened: {25, 180 * 24 * time.Hour, true, 3},
EvidenceDelivered: {14, 180 * 24 * time.Hour, true, 4},
EvidenceBouncedRecipient: {70, 365 * 24 * time.Hour, false, 2},
}
// verdictBase is how much a check is trusted on its own, by source.
func verdictBase(v Verdict) (int, string) {
switch v.Source {
case "manual":
return 95, "marked by a teammate"
case "provider":
switch v.Status {
case StatusValid, StatusInvalid:
return 88, "checked by the verification service"
case StatusRisky:
return 55, "flagged by the verification service"
}
return 30, "the verification service could not decide"
case "imported":
switch v.Status {
case StatusValid, StatusInvalid:
return 80, "verified before it was imported"
case StatusRisky:
return 50, "flagged before it was imported"
}
return 25, "imported without a decisive result"
case "probe":
switch v.Status {
case StatusValid:
return 75, "the mail server accepted the address"
case StatusInvalid:
return 80, "the mail server rejected the address"
case StatusRisky:
return 45, "the domain accepts any address, so the check proves nothing"
}
return 20, "the check was inconclusive"
}
return 0, "never checked"
}
// PositiveDecisiveScore is the evidence score above which real mail decides
// the address is deliverable regardless of what a check said.
const PositiveDecisiveScore = 20.0
// Score derives an address's status and confidence from its last verdict and
// the evidence ledger. Rules, in order:
//
// 1. A manual verdict wins outright.
// 2. A bounce naming the recipient, newer than every positive observation,
// makes the address undeliverable.
// 3. Enough positive evidence (a reply, a click, a human open, or repeated
// clean deliveries) makes it deliverable, whatever a probe said.
// 4. Otherwise the verdict stands, with the evidence nudging confidence.
//
// Absence of engagement never appears here: nothing in the ledger says "did
// not open", and Score has no input for it.
func Score(v Verdict, evidence []Evidence, now time.Time) Scored {
base, baseReason := verdictBase(v)
out := Scored{Status: v.Status, Confidence: base}
if out.Status == "" {
out.Status = StatusUnknown
}
sort.Slice(evidence, func(i, j int) bool { return evidence[i].ObservedAt.After(evidence[j].ObservedAt) })
counts := map[string]int{}
var positive, negative float64
var lastPositive, lastNegative time.Time
var reasons []string
for _, e := range evidence {
w, ok := evidenceWeights[e.Kind]
if !ok {
continue
}
if counts[e.Kind] >= w.cap {
continue
}
counts[e.Kind]++
age := now.Sub(e.ObservedAt)
if age < 0 {
age = 0
}
value := w.weight * math.Pow(0.5, age.Hours()/w.halfLife.Hours())
if w.positive {
positive += value
if e.ObservedAt.After(lastPositive) {
lastPositive = e.ObservedAt
}
} else {
negative += value
if e.ObservedAt.After(lastNegative) {
lastNegative = e.ObservedAt
}
}
if counts[e.Kind] == 1 {
reasons = append(reasons, evidenceReason(e, counts, evidence, now))
}
}
out.LastPositiveAt = lastPositive
switch {
case v.Source == "manual":
out.Reasons = append([]string{baseReason}, reasons...)
out.Confidence = clamp(base + int(positive/4))
return out
case !lastNegative.IsZero() && lastNegative.After(lastPositive):
out.Status = StatusInvalid
out.Decisive = true
out.Confidence = clamp(60 + int(negative/2))
out.Reasons = prepend(reasons, EvidenceBouncedRecipient)
return out
case positive >= PositiveDecisiveScore:
out.Status = StatusValid
out.Decisive = true
out.Confidence = clamp(70 + int(positive/2))
out.Reasons = reasons
if v.Status == StatusRisky || v.Status == StatusInvalid {
out.Reasons = append(out.Reasons, "real mail outranks the earlier check ("+baseReason+")")
}
return out
}
// The verdict stands; a little evidence either way moves confidence.
switch out.Status {
case StatusValid:
out.Confidence = clamp(base + int(positive/2) - int(negative))
case StatusInvalid:
out.Confidence = clamp(base + int(negative/2) - int(positive))
default:
out.Confidence = clamp(base + int(positive/2))
}
out.Reasons = append([]string{baseReason}, reasons...)
return out
}
func prepend(reasons []string, kind string) []string {
out := make([]string, 0, len(reasons))
for _, r := range reasons {
if strings.HasPrefix(r, "bounced") {
out = append([]string{r}, out...)
continue
}
out = append(out, r)
}
return out
}
func evidenceReason(e Evidence, counts map[string]int, all []Evidence, now time.Time) string {
n := 0
for _, x := range all {
if x.Kind == e.Kind {
n++
}
}
when := humanAge(now.Sub(e.ObservedAt))
switch e.Kind {
case EvidenceReplied:
return "replied " + when
case EvidenceAutoReplied:
return "sent an automatic reply " + when + " (the mailbox is live)"
case EvidenceClicked:
return "clicked a link " + when
case EvidenceOpened:
return "opened an email " + when
case EvidenceDelivered:
if n > 1 {
return fmt.Sprintf("delivered %d times without a bounce, last %s", n, when)
}
return "delivered without a bounce " + when
case EvidenceBouncedRecipient:
d := strings.TrimSpace(e.Detail)
if d != "" {
return "bounced " + when + ": " + d
}
return "bounced " + when + ": the server said the mailbox does not exist"
}
return e.Kind + " " + when
}
func humanAge(d time.Duration) string {
days := int(d.Hours() / 24)
switch {
case days <= 0:
return "today"
case days == 1:
return "yesterday"
case days < 30:
return fmt.Sprintf("%d days ago", days)
case days < 365:
return fmt.Sprintf("%d months ago", days/30)
}
return fmt.Sprintf("%d years ago", days/365)
}
func clamp(n int) int {
if n < 0 {
return 0
}
if n > 100 {
return 100
}
return n
}
// NamesRecipient reports whether a bounce or rejection text says the
// RECIPIENT does not exist, as opposed to a full mailbox, a policy block, a
// reputation rejection or a greeting the server disliked. Only the former is
// evidence against the address.
func NamesRecipient(text string) bool {
lower := strings.ToLower(text)
if containsAny(lower, recipientMarkers) {
return true
}
if containsAny(lower, probeMarkers) {
return false
}
// Enhanced status anywhere in the text: 5.1.1 / 5.1.2 / 5.1.3 / 5.1.6 /
// 5.1.10 address errors and 5.2.1 disabled mailbox.
for _, code := range []string{"5.1.1", "5.1.2", "5.1.3", "5.1.6", "5.1.10", "5.2.1"} {
if strings.Contains(lower, code+" ") || strings.HasSuffix(lower, code) || strings.Contains(lower, code+":") {
return true
}
}
return false
}
+90
View File
@@ -0,0 +1,90 @@
package emailverify
import (
"testing"
"time"
)
func TestScoreSilenceIsNotEvidence(t *testing.T) {
now := time.Now()
probeValid := Verdict{Status: StatusValid, Source: "probe", CheckedAt: now.Add(-24 * time.Hour)}
// A contact sent to five times with nothing recorded: nothing changes.
got := Score(probeValid, nil, now)
if got.Status != StatusValid || got.Confidence != 75 {
t.Fatalf("silence changed the verdict: %+v", got)
}
}
func TestScoreRealMailBeatsTheProbe(t *testing.T) {
now := time.Now()
probeInvalid := Verdict{Status: StatusInvalid, Source: "probe", CheckedAt: now.Add(-time.Hour)}
got := Score(probeInvalid, []Evidence{{Kind: EvidenceReplied, ObservedAt: now.Add(-3 * 24 * time.Hour)}}, now)
if got.Status != StatusValid || !got.Decisive {
t.Fatalf("a reply did not override the probe: %+v", got)
}
// Two clean deliveries are enough; one is not.
one := Score(probeInvalid, []Evidence{{Kind: EvidenceDelivered, ObservedAt: now}}, now)
if one.Status != StatusInvalid {
t.Fatalf("one delivery overrode the probe: %+v", one)
}
two := Score(probeInvalid, []Evidence{{Kind: EvidenceDelivered, ObservedAt: now, Detail: "a"}, {Kind: EvidenceDelivered, ObservedAt: now.Add(-time.Hour), Detail: "b"}}, now)
if two.Status != StatusValid {
t.Fatalf("two deliveries did not override the probe: %+v", two)
}
}
func TestScoreNewerBounceWinsOverOlderEngagement(t *testing.T) {
now := time.Now()
v := Verdict{Status: StatusValid, Source: "provider"}
got := Score(v, []Evidence{
{Kind: EvidenceReplied, ObservedAt: now.Add(-60 * 24 * time.Hour)},
{Kind: EvidenceBouncedRecipient, ObservedAt: now.Add(-24 * time.Hour), Detail: "550 5.1.1 user unknown"},
}, now)
if got.Status != StatusInvalid || !got.Decisive {
t.Fatalf("a newer bounce did not win: %+v", got)
}
// And a reply after the bounce (the mailbox came back) wins again.
back := Score(v, []Evidence{
{Kind: EvidenceBouncedRecipient, ObservedAt: now.Add(-60 * 24 * time.Hour)},
{Kind: EvidenceReplied, ObservedAt: now.Add(-24 * time.Hour)},
}, now)
if back.Status != StatusValid {
t.Fatalf("a reply after a bounce did not win: %+v", back)
}
}
func TestScoreManualWinsOutright(t *testing.T) {
now := time.Now()
got := Score(Verdict{Status: StatusValid, Source: "manual"}, []Evidence{{Kind: EvidenceBouncedRecipient, ObservedAt: now}}, now)
if got.Status != StatusValid {
t.Fatalf("manual lost to evidence: %+v", got)
}
}
func TestScoreEvidenceDecays(t *testing.T) {
now := time.Now()
v := Verdict{Status: StatusUnknown, Source: "probe"}
fresh := Score(v, []Evidence{{Kind: EvidenceOpened, ObservedAt: now}}, now)
old := Score(v, []Evidence{{Kind: EvidenceOpened, ObservedAt: now.Add(-3 * 365 * 24 * time.Hour)}}, now)
if fresh.Status != StatusValid {
t.Fatalf("a fresh human open should decide: %+v", fresh)
}
if old.Status != StatusUnknown || old.Confidence >= fresh.Confidence {
t.Fatalf("a three-year-old open should only nudge: %+v", old)
}
}
func TestNamesRecipient(t *testing.T) {
yes := []string{"550 5.1.1 The email account that you tried to reach does not exist", "User unknown", "smtp; 550 5.1.1 <a@b>: Recipient address rejected: User unknown in virtual mailbox table"}
no := []string{"552 5.2.2 Mailbox full", "550 5.7.1 Service unavailable, Client host blocked using Spamhaus", "451 4.7.1 Greylisted, try again later", "554 5.7.1 Relay access denied"}
for _, s := range yes {
if !NamesRecipient(s) {
t.Fatalf("should name recipient: %q", s)
}
}
for _, s := range no {
if NamesRecipient(s) {
t.Fatalf("must not name recipient: %q", s)
}
}
}
+233
View File
@@ -0,0 +1,233 @@
package emailverify
import "strings"
// External verification vocabularies. A customer who verified a list with
// another service brings a status column along; these tables read the words
// each service writes and fold them into Warmbly's four statuses, so nobody
// has to pick a provider or translate a column by hand.
// ExternalVerdict is one recognised external status.
type ExternalVerdict struct {
Status Status
SubStatus SubStatus
}
// vocabularies maps a provider name to the statuses it emits. Keys are
// lower-cased with spaces, dashes and underscores removed (see vocabKey).
var vocabularies = map[string]map[string]ExternalVerdict{
ProviderMillionVerifier: {
"ok": {Status: StatusValid},
"good": {Status: StatusValid},
"catchall": {Status: StatusRisky, SubStatus: SubStatusCatchAll},
"unknown": {Status: StatusUnknown},
"error": {Status: StatusUnknown},
"disposable": {Status: StatusInvalid, SubStatus: SubStatusDisposable},
"invalid": {Status: StatusInvalid},
"bad": {Status: StatusInvalid},
"risky": {Status: StatusRisky},
},
"zerobounce": {
"valid": {Status: StatusValid},
"invalid": {Status: StatusInvalid},
"catchall": {Status: StatusRisky, SubStatus: SubStatusCatchAll},
"spamtrap": {Status: StatusInvalid, SubStatus: SubStatusSpamTrap},
"abuse": {Status: StatusInvalid, SubStatus: SubStatusSpamTrap},
"donotmail": {Status: StatusInvalid},
"unknown": {Status: StatusUnknown},
"disposable": {Status: StatusInvalid, SubStatus: SubStatusDisposable},
"toxic": {Status: StatusInvalid, SubStatus: SubStatusSpamTrap},
"rolebased": {Status: StatusRisky, SubStatus: SubStatusRole},
"mailboxfull": {Status: StatusRisky, SubStatus: SubStatusMailboxFull},
},
"neverbounce": {
"valid": {Status: StatusValid},
"invalid": {Status: StatusInvalid},
"disposable": {Status: StatusInvalid, SubStatus: SubStatusDisposable},
"catchall": {Status: StatusRisky, SubStatus: SubStatusCatchAll},
"unknown": {Status: StatusUnknown},
},
"bouncer": {
"deliverable": {Status: StatusValid},
"undeliverable": {Status: StatusInvalid},
"risky": {Status: StatusRisky},
"unknown": {Status: StatusUnknown},
"acceptall": {Status: StatusRisky, SubStatus: SubStatusCatchAll},
"disposable": {Status: StatusInvalid, SubStatus: SubStatusDisposable},
},
"kickbox": {
"deliverable": {Status: StatusValid},
"undeliverable": {Status: StatusInvalid},
"risky": {Status: StatusRisky},
"unknown": {Status: StatusUnknown},
"acceptall": {Status: StatusRisky, SubStatus: SubStatusCatchAll},
},
"emailable": {
"deliverable": {Status: StatusValid},
"undeliverable": {Status: StatusInvalid},
"risky": {Status: StatusRisky},
"unknown": {Status: StatusUnknown},
"acceptall": {Status: StatusRisky, SubStatus: SubStatusCatchAll},
},
"debounce": {
"safetosend": {Status: StatusValid},
"valid": {Status: StatusValid},
"deliverable": {Status: StatusValid},
"invalid": {Status: StatusInvalid},
"disposable": {Status: StatusInvalid, SubStatus: SubStatusDisposable},
"spamtrap": {Status: StatusInvalid, SubStatus: SubStatusSpamTrap},
"acceptall": {Status: StatusRisky, SubStatus: SubStatusCatchAll},
"catchall": {Status: StatusRisky, SubStatus: SubStatusCatchAll},
"role": {Status: StatusRisky, SubStatus: SubStatusRole},
"unknown": {Status: StatusUnknown},
"risky": {Status: StatusRisky},
"undeliverable": {Status: StatusInvalid},
},
"clearout": {
"valid": {Status: StatusValid},
"invalid": {Status: StatusInvalid},
"catchall": {Status: StatusRisky, SubStatus: SubStatusCatchAll},
"unknown": {Status: StatusUnknown},
},
"emaillistverify": {
"ok": {Status: StatusValid},
"valid": {Status: StatusValid},
"okforall": {Status: StatusRisky, SubStatus: SubStatusCatchAll},
"acceptall": {Status: StatusRisky, SubStatus: SubStatusCatchAll},
"catchall": {Status: StatusRisky, SubStatus: SubStatusCatchAll},
"invalid": {Status: StatusInvalid},
"invalidmx": {Status: StatusInvalid, SubStatus: SubStatusNoMX},
"invalidsyntax": {Status: StatusInvalid, SubStatus: SubStatusSyntax},
"emaildisabled": {Status: StatusInvalid},
"deadserver": {Status: StatusInvalid, SubStatus: SubStatusNoMX},
"disposable": {Status: StatusInvalid, SubStatus: SubStatusDisposable},
"spamtrap": {Status: StatusInvalid, SubStatus: SubStatusSpamTrap},
"unknown": {Status: StatusUnknown},
"role": {Status: StatusRisky, SubStatus: SubStatusRole},
"smtpprotocol": {Status: StatusUnknown},
"antispamsystem": {Status: StatusUnknown},
"unknownerror": {Status: StatusUnknown},
"attemptrejected": {Status: StatusUnknown},
"relaydenied": {Status: StatusUnknown},
"mailboxfull": {Status: StatusRisky, SubStatus: SubStatusMailboxFull},
"greylisted": {Status: StatusUnknown},
"noresponse": {Status: StatusUnknown},
"ipblocked": {Status: StatusUnknown},
"servicenotavailable": {Status: StatusUnknown},
},
// Warmbly's own statuses, so a Warmbly export re-imports losslessly.
ProviderBuiltin: {
"valid": {Status: StatusValid},
"risky": {Status: StatusRisky},
"invalid": {Status: StatusInvalid},
"unknown": {Status: StatusUnknown},
},
}
// vocabProviderOrder decides which provider wins when several recognise every
// value. Specific vocabularies first; the generic one last.
var vocabProviderOrder = []string{
"zerobounce", ProviderMillionVerifier, "neverbounce", "emaillistverify",
"debounce", "bouncer", "kickbox", "emailable", "clearout", ProviderBuiltin,
}
// vocabProviderAliases maps how a provider is written in a column header to
// its vocabulary key.
var vocabProviderAliases = map[string]string{
"millionverifier": ProviderMillionVerifier, "mv": ProviderMillionVerifier,
"zerobounce": "zerobounce", "zb": "zerobounce",
"neverbounce": "neverbounce", "nb": "neverbounce",
"bouncer": "bouncer", "usebouncer": "bouncer",
"kickbox": "kickbox",
"emailable": "emailable",
"debounce": "debounce",
"clearout": "clearout",
"emaillistverify": "emaillistverify", "elv": "emaillistverify",
"warmbly": ProviderBuiltin, "builtin": ProviderBuiltin,
}
func vocabKey(raw string) string {
k := strings.ToLower(strings.TrimSpace(raw))
return strings.NewReplacer(" ", "", "_", "", "-", "", ".", "").Replace(k)
}
// KnownVocabulary reports whether name (as written by a customer or an API
// caller) is a vocabulary this package can read, and returns its key.
func KnownVocabulary(name string) (string, bool) {
k, ok := vocabProviderAliases[vocabKey(name)]
return k, ok
}
// NormalizeExternal reads one external status. provider may be empty, in
// which case every vocabulary is consulted and the first that recognises the
// value wins. Returns false for a value nobody recognises.
func NormalizeExternal(provider, raw string) (ExternalVerdict, bool) {
key := vocabKey(raw)
if key == "" {
return ExternalVerdict{}, false
}
if provider != "" {
p, ok := KnownVocabulary(provider)
if !ok {
return ExternalVerdict{}, false
}
v, ok := vocabularies[p][key]
return v, ok
}
for _, p := range vocabProviderOrder {
if v, ok := vocabularies[p][key]; ok {
return v, ok
}
}
return ExternalVerdict{}, false
}
// DetectVocabulary looks at a column's values and names the provider whose
// vocabulary they all belong to. Blank cells are ignored; a column with no
// non-blank cell, or with one value nobody knows, is not a status column.
func DetectVocabulary(values []string) (string, bool) {
seen := 0
for _, p := range vocabProviderOrder {
all := true
seen = 0
for _, raw := range values {
key := vocabKey(raw)
if key == "" {
continue
}
seen++
if _, ok := vocabularies[p][key]; !ok {
all = false
break
}
}
if all && seen > 0 {
return p, true
}
}
return "", false
}
// IsStatusHeader reports whether a column header reads like a verification
// status column, and names the provider if the header says so.
func IsStatusHeader(header string) (provider string, ok bool) {
k := vocabKey(header)
if k == "" {
return "", false
}
for alias, p := range vocabProviderAliases {
if strings.HasPrefix(k, alias) {
rest := strings.TrimPrefix(k, alias)
if rest == "" || strings.Contains(rest, "status") || strings.Contains(rest, "result") || strings.Contains(rest, "verif") || strings.Contains(rest, "quality") {
return p, true
}
}
}
switch k {
case "verificationstatus", "verification", "verified", "emailstatus", "emailverification",
"verifystatus", "verificationresult", "verifyresult", "validationstatus", "validation",
"emailvalidation", "deliverability", "deliverabilitystatus", "result", "status", "quality":
return "", true
}
return "", false
}
+84
View File
@@ -0,0 +1,84 @@
package emailverify
import "testing"
func TestNormalizeExternalReadsEveryKnownVocabulary(t *testing.T) {
cases := []struct {
provider, raw string
want Status
sub SubStatus
}{
{"zerobounce", "valid", StatusValid, SubStatusNone},
{"zerobounce", "catch-all", StatusRisky, SubStatusCatchAll},
{"zerobounce", "spamtrap", StatusInvalid, SubStatusSpamTrap},
{"millionverifier", "ok", StatusValid, SubStatusNone},
{"millionverifier", "catch_all", StatusRisky, SubStatusCatchAll},
{"millionverifier", "disposable", StatusInvalid, SubStatusDisposable},
{"neverbounce", "catchall", StatusRisky, SubStatusCatchAll},
{"bouncer", "deliverable", StatusValid, SubStatusNone},
{"kickbox", "accept-all", StatusRisky, SubStatusCatchAll},
{"emaillistverify", "ok_for_all", StatusRisky, SubStatusCatchAll},
{"", "Safe to Send", StatusValid, SubStatusNone},
{"", "INVALID", StatusInvalid, SubStatusNone},
{"", "unknown", StatusUnknown, SubStatusNone},
}
for _, c := range cases {
got, ok := NormalizeExternal(c.provider, c.raw)
if !ok {
t.Fatalf("%s/%q not recognised", c.provider, c.raw)
}
if got.Status != c.want || got.SubStatus != c.sub {
t.Fatalf("%s/%q = %v/%v, want %v/%v", c.provider, c.raw, got.Status, got.SubStatus, c.want, c.sub)
}
}
if _, ok := NormalizeExternal("", "qualified lead"); ok {
t.Fatal("a CRM stage must not read as a verdict")
}
if _, ok := NormalizeExternal("nosuchprovider", "valid"); ok {
t.Fatal("an unknown provider must not be read")
}
}
func TestDetectVocabularyNamesTheProviderOrRefuses(t *testing.T) {
if p, ok := DetectVocabulary([]string{"valid", "catch-all", "", "do_not_mail"}); !ok || p != "zerobounce" {
t.Fatalf("zerobounce column = %q/%v", p, ok)
}
if p, ok := DetectVocabulary([]string{"ok", "catch_all", "unknown"}); !ok || p != ProviderMillionVerifier {
t.Fatalf("millionverifier column = %q/%v", p, ok)
}
if _, ok := DetectVocabulary([]string{"valid", "won", "lost"}); ok {
t.Fatal("mixed column must not be a verdict column")
}
if _, ok := DetectVocabulary([]string{"", ""}); ok {
t.Fatal("blank column must not be a verdict column")
}
}
func TestIsStatusHeader(t *testing.T) {
if p, ok := IsStatusHeader("ZeroBounce Status"); !ok || p != "zerobounce" {
t.Fatalf("header = %q/%v", p, ok)
}
if p, ok := IsStatusHeader("verification_status"); !ok || p != "" {
t.Fatalf("generic header = %q/%v", p, ok)
}
if _, ok := IsStatusHeader("Deal stage"); ok {
t.Fatal("deal stage is not a status header")
}
}
func TestFingerprintMXNamesUndisclosingProviders(t *testing.T) {
if fp := fingerprintMX([]string{"acme-com.mail.protection.outlook.com"}); !fp.undisclosed || fp.name != "Microsoft 365" {
t.Fatalf("m365 = %+v", fp)
}
if fp := fingerprintMX([]string{"aspmx.l.google.com"}); fp.undisclosed {
t.Fatalf("google must be probed: %+v", fp)
}
}
func TestDomainCacheExpires(t *testing.T) {
c := newDomainCache(0)
c.put("a.com", domainFact{kind: domainCatchAll})
if _, ok := c.get("a.com"); ok {
t.Fatal("zero ttl entry must be expired")
}
}
+1 -1
View File
@@ -1606,7 +1606,7 @@ func (r *adminRepository) StopCampaign(ctx context.Context, campaignID uuid.UUID
UPDATE campaigns
SET status = 'paused', last_status_change_at = NOW(), updated_at = NOW()
WHERE id = $1
AND status IN ('active', 'paused', 'paused_no_accounts', 'paused_trial_expired', 'paused_guardrail')
AND status IN ('active', 'paused', 'paused_no_accounts', 'paused_trial_expired', 'paused_guardrail', 'paused_undeliverable')
`, campaignID)
if err != nil {
return false, err
+29 -5
View File
@@ -68,6 +68,8 @@ type CampaignRepository interface {
// reply routing a contact onto a live branch). The reconciler re-checks each.
ListStaleParkedCampaigns(ctx context.Context, staleAfter time.Duration, limit int) ([]ParkedCampaignTask, error)
CountActiveForOrganization(ctx context.Context, orgID uuid.UUID) (int, error)
// ListIDsByStatus lists the org's campaigns parked at one status.
ListIDsByStatus(ctx context.Context, orgID uuid.UUID, status string) ([]uuid.UUID, error)
AccountHasActiveCampaign(ctx context.Context, accountID uuid.UUID) (bool, error)
// CountActiveCampaignsForAccount returns how many active campaigns send
// from the given mailbox (matched through the campaign's email tags OR an
@@ -833,12 +835,13 @@ func (r *campaignRepository) Update(ctx context.Context, userID, campaignID stri
}
if data.Status != nil {
// Valid statuses: draft, active, paused, completed, paused_trial_expired,
// paused_no_accounts, paused_guardrail
// paused_no_accounts, paused_guardrail, paused_undeliverable
status := *data.Status
validStatuses := map[string]bool{
"draft": true, "active": true, "paused": true,
"completed": true, "paused_trial_expired": true,
"paused_no_accounts": true, "paused_guardrail": true,
"paused_undeliverable": true,
}
if !validStatuses[status] {
return nil, errx.ErrInvalid
@@ -1331,10 +1334,13 @@ func (r *campaignRepository) GetSequencesRoutingByCampaignID(ctx context.Context
// validCampaignTransitions defines which status transitions are allowed.
// Key is the current status, values are the statuses it can transition to.
var validCampaignTransitions = map[string]map[string]bool{
"draft": {"active": true},
"active": {"paused": true, "completed": true, "paused_no_accounts": true, "paused_trial_expired": true, "paused_guardrail": true},
"paused": {"active": true, "draft": true},
"paused_no_accounts": {"active": true, "paused": true},
"draft": {"active": true},
"active": {"paused": true, "completed": true, "paused_no_accounts": true, "paused_trial_expired": true, "paused_guardrail": true, "paused_undeliverable": true},
"paused": {"active": true, "draft": true},
"paused_no_accounts": {"active": true, "paused": true},
// Parked because verification refused every remaining lead; resumes once
// they are re-verified or marked deliverable.
"paused_undeliverable": {"active": true, "paused": true},
"paused_trial_expired": {"active": true, "paused": true},
// An auto-pause is resumable, but only deliberately: the owner has to
// restart the campaign (or park it) after looking at why it tripped.
@@ -1563,6 +1569,24 @@ func (r *campaignRepository) CountActiveForOrganization(ctx context.Context, org
return count, err
}
// ListIDsByStatus lists the org's campaigns parked at one status.
func (r *campaignRepository) ListIDsByStatus(ctx context.Context, orgID uuid.UUID, status string) ([]uuid.UUID, error) {
rows, err := r.DB.Query(ctx, `SELECT id FROM campaigns WHERE organization_id = $1 AND status = $2`, orgID, status)
if err != nil {
return nil, err
}
defer rows.Close()
var out []uuid.UUID
for rows.Next() {
var id uuid.UUID
if err := rows.Scan(&id); err != nil {
return nil, err
}
out = append(out, id)
}
return out, rows.Err()
}
// AccountHasActiveCampaign reports whether the mailbox backs at least one active
// campaign, counting BOTH tag-based campaigns AND explicit-sender campaigns
// (campaign_senders). The explicit lane matters for the warmup health-check
+72 -10
View File
@@ -1368,18 +1368,80 @@ func branchHasPositiveReplyCondition(b *models.Branch) bool {
return false
}
// CountUndeliverableLeads counts the leads FindNextRoutedPair excludes, using
// the same predicate it filters on.
// CountUndeliverableLeads counts the leads that verification alone keeps out
// of routing: the campaign's flow would still send them a step if their
// verdict were ignored. It runs the same router the send path runs, so a lead
// whose flow has ended (a STOP branch, no outgoing connection, every step
// attempted, replied) is not counted, and neither is one another gate
// excludes (bounced, failed, suppressed).
func (r *campaignProgressRepository) CountUndeliverableLeads(ctx context.Context, campaignID uuid.UUID) (int, error) {
var n int
err := r.db.QueryRow(ctx, `
SELECT COUNT(*)
FROM campaign_leads cl
JOIN contacts c ON c.id = cl.contact_id
WHERE cl.campaign_id = $1
AND `+undeliverableClause("$1"), campaignID).Scan(&n)
router, err := r.loadRouter(ctx, campaignID)
if err != nil {
return 0, err
}
return n, nil
if router == nil {
return 0, nil
}
query := `
SELECT cl.contact_id,
lp.sequence_id, lp.sent_at, lp.opened_at, lp.clicked_at, lp.replied_at, COALESCE(lp.reply_class, ''), COALESCE(lp.ai_label, ''),
COALESCE(ss.ids, '{}') AS sent_ids,
EXISTS (
SELECT 1 FROM campaign_contact_progress rp
WHERE rp.campaign_id = $1 AND rp.contact_id = cl.contact_id AND rp.replied_at IS NOT NULL
) AS has_replied
FROM campaign_leads cl
JOIN contacts c ON c.id = cl.contact_id
LEFT JOIN LATERAL (
SELECT sequence_id, sent_at, opened_at, clicked_at, replied_at, reply_class, ai_label
FROM campaign_contact_progress p
WHERE p.campaign_id = $1 AND p.contact_id = cl.contact_id AND p.sent_at IS NOT NULL
ORDER BY p.sent_at DESC LIMIT 1
) lp ON true
LEFT JOIN LATERAL (
SELECT array_agg(sequence_id) AS ids
FROM campaign_contact_progress p2
WHERE p2.campaign_id = $1 AND p2.contact_id = cl.contact_id
AND (p2.sent_at IS NOT NULL OR p2.dispatched_at IS NOT NULL)
) ss ON true
WHERE cl.campaign_id = $1
AND NOT EXISTS (
SELECT 1 FROM campaign_contact_progress b
WHERE b.contact_id = cl.contact_id AND b.bounced_at IS NOT NULL
)
AND NOT EXISTS (
SELECT 1 FROM campaign_contact_progress f
WHERE f.campaign_id = $1 AND f.contact_id = cl.contact_id
AND f.sent_at IS NULL AND f.failed_at IS NOT NULL
AND f.send_attempts >= $2
)
AND NOT EXISTS (
SELECT 1 FROM suppressed_recipients sr
JOIN campaigns camp ON camp.organization_id = sr.organization_id
WHERE camp.id = $1
AND LOWER(sr.email) = LOWER(c.email)
AND (sr.expires_at IS NULL OR sr.expires_at > NOW())
)
AND ` + undeliverableClause("$1") + `
`
rows, err := r.db.Query(ctx, query, campaignID, config.CampaignSendMaxAttempts)
if err != nil {
return 0, err
}
defer rows.Close()
n := 0
for rows.Next() {
var in routeInput
var contactID uuid.UUID
if serr := rows.Scan(&contactID, &in.lastSeq, &in.sentAt, &in.openedAt, &in.clickedAt, &in.repliedAt, &in.replyClass, &in.aiLabel, &in.sentIDs, &in.hasReplied); serr != nil {
return 0, serr
}
res := router.route(campaignID, contactID, in)
// A step to send (now or later) or a condition still deciding: the
// flow is not over for this lead.
if res.Target != nil || res.WaitUntil != nil {
n++
}
}
return n, rows.Err()
}
+246 -31
View File
@@ -41,7 +41,20 @@ type ContactRepository interface {
// have never been conclusively checked (status 'unknown', never verified) so
// the batch scheduler can work them off a cap per tick.
UpdateContactVerification(ctx context.Context, contactID uuid.UUID, res emailverify.Result) *errx.Error
ListUnverifiedContacts(ctx context.Context, limit int) ([]models.Contact, *errx.Error)
// ListVerificationCandidates returns contacts due for a check: never
// checked, or checked long enough ago that the verdict has aged out.
// Manual verdicts are never candidates. Oldest first.
ListVerificationCandidates(ctx context.Context, limit int) ([]VerificationCandidate, *errx.Error)
// SetContactsVerification stores one verdict on many of the org's contacts
// (a manual "mark deliverable"). Returns how many rows changed.
SetContactsVerification(ctx context.Context, orgID uuid.UUID, ids []uuid.UUID, w models.ContactVerificationWrite) (int, *errx.Error)
// ResetContactsVerification clears the verdict so the scheduler checks the
// contacts again on its next pass. Returns how many rows changed.
ResetContactsVerification(ctx context.Context, orgID uuid.UUID, ids []uuid.UUID) (int, *errx.Error)
// UndeliverableLeadIDs lists the campaign's leads verification refused.
UndeliverableLeadIDs(ctx context.Context, orgID, campaignID uuid.UUID) ([]uuid.UUID, *errx.Error)
// VerificationCounts is the org's contacts by verdict.
VerificationCounts(ctx context.Context, orgID uuid.UUID) (models.ContactVerificationCounts, *errx.Error)
// SetContactESP caches the recipient ESP/provider resolved from the contact's
// domain (control-plane only, no MX dial). Best-effort: a failure should not
// block sending.
@@ -233,6 +246,13 @@ func (r *contactRepository) Add(ctx context.Context, userID string, orgID uuid.U
}
lead.SourceDetail = strings.TrimSpace(lead.SourceDetail)
// A verdict the caller brought along, in any vocabulary we can read.
if v, xerr := verificationFromRequest(lead.VerificationStatus, lead.VerificationProvider); xerr != nil {
return nil, xerr
} else if v != nil {
lead.Verification = v
}
normalized = append(normalized, lead)
campaignIDs = append(campaignIDs, cids)
categoryIDs = append(categoryIDs, cats)
@@ -248,6 +268,10 @@ func (r *contactRepository) Add(ctx context.Context, userID string, orgID uuid.U
// Upsert contacts in a single batch round-trip.
insertBatch := pgx.Batch{}
for _, lead := range normalized {
var vStatus, vSub, vReason, vProvider string
if lead.Verification != nil {
vStatus, vSub, vReason, vProvider = lead.Verification.Status, lead.Verification.SubStatus, lead.Verification.Reason, lead.Verification.Provider
}
insertBatch.Queue(
// $9 and $10 are the same value: a parameter used both as an
// INSERT value and inside the DO UPDATE set gives Postgres two
@@ -255,10 +279,16 @@ func (r *contactRepository) Add(ctx context.Context, userID string, orgID uuid.U
// explicit casts. NULL means "leave the flag alone".
`INSERT INTO contacts (
id, user_id, organization_id, first_name, last_name, email, company, phone, custom_fields, subscribed,
source, source_detail, first_seen_at
source, source_detail, first_seen_at,
verification_status, verification_sub_status, verification_reason, verification_provider,
verification_source, is_catch_all, verification_checked_at
) VALUES (
gen_random_uuid(), $1, $2, $3, $4, LOWER($5), $6, $7, $8, COALESCE($9::boolean, TRUE),
$11, $12, NOW()
$11, $12, NOW(),
COALESCE(NULLIF($13::text, ''), 'unknown'), $14, $15, $16,
CASE WHEN $13::text <> '' THEN 'imported' ELSE '' END,
($14::text = 'catch_all'),
CASE WHEN $13::text <> '' THEN NOW() ELSE NULL END
)
ON CONFLICT (user_id, (LOWER(email))) DO UPDATE SET
organization_id = COALESCE(contacts.organization_id, EXCLUDED.organization_id),
@@ -270,12 +300,22 @@ func (r *contactRepository) Add(ctx context.Context, userID string, orgID uuid.U
phone = COALESCE(NULLIF(EXCLUDED.phone, ''), contacts.phone),
custom_fields = contacts.custom_fields || EXCLUDED.custom_fields,
subscribed = COALESCE($10::boolean, contacts.subscribed),
-- A verdict in the file replaces whatever was recorded; a file
-- without one leaves the existing verdict alone.
verification_status = CASE WHEN $13::text <> '' THEN $13 ELSE contacts.verification_status END,
verification_sub_status = CASE WHEN $13::text <> '' THEN $14 ELSE contacts.verification_sub_status END,
verification_reason = CASE WHEN $13::text <> '' THEN $15 ELSE contacts.verification_reason END,
verification_provider = CASE WHEN $13::text <> '' THEN $16 ELSE contacts.verification_provider END,
verification_source = CASE WHEN $13::text <> '' THEN 'imported' ELSE contacts.verification_source END,
is_catch_all = CASE WHEN $13::text <> '' THEN ($14::text = 'catch_all') ELSE contacts.is_catch_all END,
verification_checked_at = CASE WHEN $13::text <> '' THEN NOW() ELSE contacts.verification_checked_at END,
updated_at = NOW()
-- xmax = 0 only on a fresh row: the source is first-touch, so an
-- upsert that hit an existing contact is not a creation.
RETURNING id, first_name, last_name, email, company, phone, custom_fields, subscribed, updated_at, created_at, (xmax = 0)`,
userID, orgID, lead.FirstName, lead.LastName, lead.Email, lead.Company, lead.Phone, lead.CustomFields,
lead.Subscribed, lead.Subscribed, string(lead.Source), lead.SourceDetail,
vStatus, vSub, vReason, vProvider,
)
}
@@ -469,6 +509,7 @@ func (r *contactRepository) GetByID(ctx context.Context, contactID uuid.UUID) (*
c.id, c.first_name, c.last_name, c.email, c.company, c.phone,
c.custom_fields, c.subscribed, c.updated_at, c.created_at,
c.verification_status, c.verification_reason, c.is_catch_all, c.verification_checked_at,
c.verification_source, c.verification_provider, c.verification_sub_status, c.verification_confidence,
c.esp_provider, c.esp_resolved_at
FROM contacts c
WHERE c.id = $1
@@ -480,6 +521,7 @@ func (r *contactRepository) GetByID(ctx context.Context, contactID uuid.UUID) (*
&contact.Company, &contact.Phone, &contact.CustomFields, &contact.Subscribed,
&contact.UpdatedAt, &contact.CreatedAt,
&contact.VerificationStatus, &contact.VerificationReason, &contact.IsCatchAll, &contact.VerificationCheckedAt,
&contact.VerificationSource, &contact.VerificationProvider, &contact.VerificationSubStatus, &contact.VerificationConfidence,
&contact.ESPProvider, &contact.ESPResolvedAt,
)
if err != nil {
@@ -520,6 +562,14 @@ func (r *contactRepository) UpdateContactVerification(ctx context.Context, conta
if checkedAt.IsZero() {
checkedAt = time.Now().UTC()
}
provider := res.Provider
if provider == "" {
provider = emailverify.ProviderBuiltin
}
source := models.VerificationSourceProvider
if provider == emailverify.ProviderBuiltin {
source = models.VerificationSourceProbe
}
query := `
UPDATE contacts
@@ -527,10 +577,14 @@ func (r *contactRepository) UpdateContactVerification(ctx context.Context, conta
verification_reason = $3,
is_catch_all = $4,
verification_checked_at = $5,
verification_source = $6,
verification_provider = $7,
verification_sub_status = $8,
verification_confidence = $9,
updated_at = NOW()
WHERE id = $1
`
params := []any{contactID, status, res.Reason, res.IsCatchAll, checkedAt}
params := []any{contactID, status, res.Reason, res.IsCatchAll, checkedAt, source, provider, string(res.SubStatus), res.Confidence}
cmd, err := r.DB.Exec(ctx, query, params...)
if err != nil {
db.CaptureError(err, query, params, "exec")
@@ -542,55 +596,172 @@ func (r *contactRepository) UpdateContactVerification(ctx context.Context, conta
return nil
}
// ListUnverifiedContacts returns up to `limit` contacts that have never been
// conclusively verified (status 'unknown' and no recorded check). Oldest
// contacts first so a backlog drains in creation order. The pre-send gate only
// drops 'invalid', so 'risky'/'valid'/already-checked rows are intentionally
// excluded here — they don't need re-verification on every tick.
func (r *contactRepository) ListUnverifiedContacts(ctx context.Context, limit int) ([]models.Contact, *errx.Error) {
// VerificationCandidate is one contact due for a verification check.
type VerificationCandidate struct {
ID uuid.UUID
OrganizationID uuid.UUID
Email string
// Requested is true when a member asked for this check (the verdict was
// reset), so it is worth spending a paid credit on even when the
// organization has none to spare.
Requested bool
}
// ListVerificationCandidates returns up to `limit` contacts due for a check.
// Never-checked contacts come first (a reset counts as never checked), then
// verdicts older than their shelf life: an unknown verdict is retried after
// config.VerificationUnknownRecheckDays, everything else after
// config.VerificationRecheckDays. Manual verdicts are never re-checked.
func (r *contactRepository) ListVerificationCandidates(ctx context.Context, limit int) ([]VerificationCandidate, *errx.Error) {
if limit <= 0 {
limit = 100
}
query := `
SELECT
c.id, c.first_name, c.last_name, c.email, c.company, c.phone,
c.custom_fields, c.subscribed, c.updated_at, c.created_at,
c.verification_status, c.verification_reason, c.is_catch_all, c.verification_checked_at
SELECT c.id, c.organization_id, c.email, c.verification_checked_at IS NULL
FROM contacts c
WHERE c.verification_status = 'unknown' AND c.verification_checked_at IS NULL
ORDER BY c.created_at ASC
WHERE c.organization_id IS NOT NULL
AND c.verification_source <> 'manual'
-- Real mail seen recently excuses the address from a check.
AND (c.verification_evidence_at IS NULL OR c.verification_evidence_at < NOW() - make_interval(days => $4))
AND (
c.verification_checked_at IS NULL
OR (c.verification_status = 'unknown' AND c.verification_checked_at < NOW() - make_interval(days => $2))
OR c.verification_checked_at < NOW() - make_interval(days => $3)
)
ORDER BY c.verification_checked_at ASC NULLS FIRST, c.created_at ASC
LIMIT $1
`
rows, err := r.DB.Query(ctx, query, limit)
params := []any{limit, config.VerificationUnknownRecheckDays, config.VerificationRecheckDays, config.VerificationEvidenceFreshDays}
rows, err := r.DB.Query(ctx, query, params...)
if err != nil {
db.CaptureError(err, query, []any{limit}, "query")
db.CaptureError(err, query, params, "query")
return nil, errx.InternalError()
}
defer rows.Close()
out := make([]models.Contact, 0, limit)
out := make([]VerificationCandidate, 0, limit)
for rows.Next() {
var c models.Contact
if err := rows.Scan(
&c.ID, &c.FirstName, &c.LastName, &c.Email,
&c.Company, &c.Phone, &c.CustomFields, &c.Subscribed,
&c.UpdatedAt, &c.CreatedAt,
&c.VerificationStatus, &c.VerificationReason, &c.IsCatchAll, &c.VerificationCheckedAt,
); err != nil {
db.CaptureError(err, "", nil, "ListUnverifiedContacts scan")
var c VerificationCandidate
if err := rows.Scan(&c.ID, &c.OrganizationID, &c.Email, &c.Requested); err != nil {
db.CaptureError(err, "", nil, "ListVerificationCandidates scan")
return nil, errx.InternalError()
}
c.Campaigns = []models.MiniCampaign{}
c.Categories = []models.MiniCategory{}
out = append(out, c)
}
if err := rows.Err(); err != nil {
db.CaptureError(err, "", nil, "ListUnverifiedContacts rows")
db.CaptureError(err, "", nil, "ListVerificationCandidates rows")
return nil, errx.InternalError()
}
return out, nil
}
// SetContactsVerification writes one verdict onto the org's listed contacts.
func (r *contactRepository) SetContactsVerification(ctx context.Context, orgID uuid.UUID, ids []uuid.UUID, w models.ContactVerificationWrite) (int, *errx.Error) {
if len(ids) == 0 {
return 0, nil
}
query := `
UPDATE contacts
SET verification_status = $3,
verification_sub_status = $4,
verification_reason = $5,
verification_provider = $6,
verification_source = $7,
is_catch_all = ($4 = 'catch_all'),
verification_checked_at = NOW(),
updated_at = NOW()
WHERE organization_id = $1 AND id = ANY($2)
`
params := []any{orgID, ids, w.Status, w.SubStatus, w.Reason, w.Provider, w.Source}
cmd, err := r.DB.Exec(ctx, query, params...)
if err != nil {
db.CaptureError(err, query, params, "exec")
return 0, errx.InternalError()
}
return int(cmd.RowsAffected()), nil
}
// ResetContactsVerification returns the org's listed contacts to "never
// checked" so the next scheduler pass picks them up first.
func (r *contactRepository) ResetContactsVerification(ctx context.Context, orgID uuid.UUID, ids []uuid.UUID) (int, *errx.Error) {
if len(ids) == 0 {
return 0, nil
}
query := `
UPDATE contacts
SET verification_status = 'unknown',
verification_sub_status = '',
verification_reason = 'verification requested',
verification_provider = '',
verification_source = '',
is_catch_all = false,
verification_checked_at = NULL,
updated_at = NOW()
WHERE organization_id = $1 AND id = ANY($2)
`
params := []any{orgID, ids}
cmd, err := r.DB.Exec(ctx, query, params...)
if err != nil {
db.CaptureError(err, query, params, "exec")
return 0, errx.InternalError()
}
return int(cmd.RowsAffected()), nil
}
// UndeliverableLeadIDs lists the campaign's leads the routing predicate skips
// for verification reasons (invalid, or risky with the risky toggle off).
func (r *contactRepository) UndeliverableLeadIDs(ctx context.Context, orgID, campaignID uuid.UUID) ([]uuid.UUID, *errx.Error) {
query := `
SELECT c.id
FROM campaign_leads cl
JOIN contacts c ON c.id = cl.contact_id
JOIN campaigns cp ON cp.id = cl.campaign_id
WHERE cl.campaign_id = $1 AND cp.organization_id = $2
AND (c.verification_status = 'invalid' OR (c.verification_status = 'risky' AND NOT cp.risky_emails))
`
params := []any{campaignID, orgID}
rows, err := r.DB.Query(ctx, query, params...)
if err != nil {
db.CaptureError(err, query, params, "query")
return nil, errx.InternalError()
}
defer rows.Close()
var out []uuid.UUID
for rows.Next() {
var id uuid.UUID
if err := rows.Scan(&id); err != nil {
db.CaptureError(err, "", nil, "UndeliverableLeadIDs scan")
return nil, errx.InternalError()
}
out = append(out, id)
}
if err := rows.Err(); err != nil {
db.CaptureError(err, "", nil, "UndeliverableLeadIDs rows")
return nil, errx.InternalError()
}
return out, nil
}
// VerificationCounts is the org's contacts by verdict.
func (r *contactRepository) VerificationCounts(ctx context.Context, orgID uuid.UUID) (models.ContactVerificationCounts, *errx.Error) {
var c models.ContactVerificationCounts
query := `
SELECT
COUNT(*) FILTER (WHERE verification_status = 'valid'),
COUNT(*) FILTER (WHERE verification_status = 'risky'),
COUNT(*) FILTER (WHERE verification_status = 'invalid'),
COUNT(*) FILTER (WHERE verification_status NOT IN ('valid','risky','invalid')),
COUNT(*) FILTER (WHERE verification_checked_at IS NULL)
FROM contacts
WHERE organization_id = $1
`
if err := r.DB.QueryRow(ctx, query, orgID).Scan(&c.Valid, &c.Risky, &c.Invalid, &c.Unknown, &c.Pending); err != nil {
db.CaptureError(err, query, []any{orgID}, "queryrow")
return c, errx.InternalError()
}
return c, nil
}
func (r *contactRepository) GetByEmailAndOrganization(ctx context.Context, organizationID uuid.UUID, email string) (*models.Contact, *errx.Error) {
query := `
SELECT
@@ -752,6 +923,11 @@ func (r *contactRepository) Search(
args = append(args, *filters.Subscribed)
argIndex++
}
if filters.VerificationStatus != "" {
whereClauses = append(whereClauses, fmt.Sprintf("c.verification_status = $%d", argIndex))
args = append(args, filters.VerificationStatus)
argIndex++
}
// -----------------------------
// Date filters
@@ -1003,6 +1179,8 @@ func (r *contactRepository) Search(
SELECT
c.id, c.first_name, c.last_name, c.email, c.company, c.phone,
c.custom_fields, c.subscribed, c.updated_at, c.created_at,
c.verification_status, c.verification_reason, c.is_catch_all, c.verification_checked_at,
c.verification_source, c.verification_provider, c.verification_sub_status, c.verification_confidence,
COALESCE(cl.campaign_count,0) AS campaign_count,
COALESCE(
(
@@ -1082,7 +1260,10 @@ func (r *contactRepository) Search(
if err := rows.Scan(
&c.ID, &c.FirstName, &c.LastName, &c.Email,
&c.Company, &c.Phone, &c.CustomFields, &c.Subscribed,
&c.UpdatedAt, &c.CreatedAt, &campaignCount, &campaignsJSON, &categoriesJSON, &leadProgressJSON,
&c.UpdatedAt, &c.CreatedAt,
&c.VerificationStatus, &c.VerificationReason, &c.IsCatchAll, &c.VerificationCheckedAt,
&c.VerificationSource, &c.VerificationProvider, &c.VerificationSubStatus, &c.VerificationConfidence,
&campaignCount, &campaignsJSON, &categoriesJSON, &leadProgressJSON,
); err != nil {
db.CaptureError(err, "", nil, "scan")
return nil, errx.InternalError()
@@ -3074,3 +3255,37 @@ func (r *contactRepository) ListTimeline(ctx context.Context, userID uuid.UUID,
HasMore: hasMore,
}, nil
}
// verificationFromRequest normalises a verdict a caller supplied with a
// contact. Empty means none; an unrecognised value is the caller's error.
func verificationFromRequest(status, provider string) (*models.ContactVerificationWrite, *errx.Error) {
status = strings.TrimSpace(status)
if status == "" {
return nil, nil
}
provider = strings.TrimSpace(provider)
if provider != "" {
if _, ok := emailverify.KnownVocabulary(provider); !ok {
return nil, errx.NewWithIdentifier(errx.BadRequest, "unknown_verification_provider",
"unknown verification_provider "+strconv.Quote(provider))
}
}
v, ok := emailverify.NormalizeExternal(provider, status)
if !ok {
return nil, errx.NewWithIdentifier(errx.BadRequest, "unknown_verification_status",
"verification_status "+strconv.Quote(status)+" is not a value any known verification service writes")
}
name := provider
if name == "" {
name = "imported"
} else if k, ok := emailverify.KnownVocabulary(provider); ok {
name = k
}
return &models.ContactVerificationWrite{
Status: string(v.Status),
SubStatus: string(v.SubStatus),
Reason: "imported verdict: " + strings.ToLower(status),
Provider: name,
Source: models.VerificationSourceImported,
}, nil
}
@@ -0,0 +1,175 @@
package repository
import (
"context"
"errors"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/infrastructure/db"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/pkg/emailverify"
)
// VerificationEvidenceRepository owns the evidence ledger behind a contact's
// verification verdict and the derived score on the contact row.
type VerificationEvidenceRepository interface {
// Record stores one observation. Idempotent on (contact, kind, ref);
// returns whether a new row was written.
Record(ctx context.Context, contactID uuid.UUID, kind, ref, detail string, observedAt time.Time) (bool, error)
// ListForContact returns the contact's evidence, newest first.
ListForContact(ctx context.Context, contactID uuid.UUID) ([]models.ContactVerificationEvidence, error)
// Verdict reads the contact's current check verdict for scoring.
Verdict(ctx context.Context, contactID uuid.UUID) (emailverify.Verdict, error)
// SetScore writes the derived status and confidence.
SetScore(ctx context.Context, contactID uuid.UUID, status string, confidence int, reason string, lastPositive time.Time, decisive bool) error
// CreditCleanDeliveries records a delivered observation for every campaign
// step sent at least `window` ago that never bounced and is not yet in
// the ledger. Returns the contacts credited.
CreditCleanDeliveries(ctx context.Context, window time.Duration, limit int) ([]uuid.UUID, error)
}
type verificationEvidenceRepository struct {
DB *db.DB
}
func NewVerificationEvidenceRepository(database *db.DB) VerificationEvidenceRepository {
return &verificationEvidenceRepository{DB: database}
}
func (r *verificationEvidenceRepository) Record(ctx context.Context, contactID uuid.UUID, kind, ref, detail string, observedAt time.Time) (bool, error) {
if observedAt.IsZero() {
observedAt = time.Now().UTC()
}
query := `
INSERT INTO contact_verification_evidence (contact_id, kind, ref, detail, observed_at)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (contact_id, kind, ref) DO NOTHING
`
params := []any{contactID, kind, ref, detail, observedAt}
cmd, err := r.DB.Exec(ctx, query, params...)
if err != nil {
db.CaptureError(err, query, params, "exec")
return false, err
}
return cmd.RowsAffected() > 0, nil
}
func (r *verificationEvidenceRepository) ListForContact(ctx context.Context, contactID uuid.UUID) ([]models.ContactVerificationEvidence, error) {
query := `
SELECT kind, detail, observed_at
FROM contact_verification_evidence
WHERE contact_id = $1
ORDER BY observed_at DESC
LIMIT 50
`
rows, err := r.DB.Query(ctx, query, contactID)
if err != nil {
db.CaptureError(err, query, []any{contactID}, "query")
return nil, err
}
defer rows.Close()
out := []models.ContactVerificationEvidence{}
for rows.Next() {
var e models.ContactVerificationEvidence
if err := rows.Scan(&e.Kind, &e.Detail, &e.ObservedAt); err != nil {
return nil, err
}
out = append(out, e)
}
return out, rows.Err()
}
func (r *verificationEvidenceRepository) Verdict(ctx context.Context, contactID uuid.UUID) (emailverify.Verdict, error) {
var v emailverify.Verdict
var status, source string
var checked *time.Time
query := `SELECT verification_status, verification_source, verification_checked_at FROM contacts WHERE id = $1`
if err := r.DB.QueryRow(ctx, query, contactID).Scan(&status, &source, &checked); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return v, errx.ErrNotFound
}
db.CaptureError(err, query, []any{contactID}, "queryrow")
return v, err
}
v.Status, v.Source = emailverify.Status(status), source
if checked != nil {
v.CheckedAt = *checked
}
return v, nil
}
func (r *verificationEvidenceRepository) SetScore(ctx context.Context, contactID uuid.UUID, status string, confidence int, reason string, lastPositive time.Time, decisive bool) error {
var lp *time.Time
if !lastPositive.IsZero() {
lp = &lastPositive
}
// A decisive score from real mail replaces the status and says so in the
// reason; otherwise only the confidence moves and the check's own reason
// stays.
query := `
UPDATE contacts
SET verification_confidence = $2,
verification_evidence_at = COALESCE($3, verification_evidence_at),
verification_status = CASE WHEN $5 THEN $4 ELSE verification_status END,
verification_reason = CASE WHEN $5 THEN $6 ELSE verification_reason END,
updated_at = NOW()
WHERE id = $1
`
params := []any{contactID, confidence, lp, status, decisive, reason}
if _, err := r.DB.Exec(ctx, query, params...); err != nil {
db.CaptureError(err, query, params, "exec")
return err
}
return nil
}
func (r *verificationEvidenceRepository) CreditCleanDeliveries(ctx context.Context, window time.Duration, limit int) ([]uuid.UUID, error) {
if limit <= 0 {
limit = 1000
}
// One evidence row per sent step; the ref is the step so a re-run of the
// job is a no-op, and a step that bounces later is excluded here and
// recorded as a bounce by the deliverability path instead.
query := `
WITH due AS (
SELECT p.contact_id, p.campaign_id, p.sequence_id, p.sent_at
FROM campaign_contact_progress p
WHERE p.sent_at IS NOT NULL AND p.bounced_at IS NULL
AND p.sent_at < NOW() - make_interval(secs => $1)
AND NOT EXISTS (
SELECT 1 FROM contact_verification_evidence e
WHERE e.contact_id = p.contact_id AND e.kind = 'delivered'
AND e.ref = p.campaign_id::text || ':' || p.sequence_id::text
)
ORDER BY p.sent_at
LIMIT $2
),
ins AS (
INSERT INTO contact_verification_evidence (contact_id, kind, ref, detail, observed_at)
SELECT contact_id, 'delivered', campaign_id::text || ':' || sequence_id::text, '', sent_at FROM due
ON CONFLICT (contact_id, kind, ref) DO NOTHING
RETURNING contact_id
)
SELECT DISTINCT contact_id FROM ins
`
params := []any{window.Seconds(), limit}
rows, err := r.DB.Query(ctx, query, params...)
if err != nil {
db.CaptureError(err, query, params, "query")
return nil, err
}
defer rows.Close()
var out []uuid.UUID
for rows.Next() {
var id uuid.UUID
if err := rows.Scan(&id); err != nil {
return nil, err
}
out = append(out, id)
}
return out, rows.Err()
}
@@ -0,0 +1,87 @@
package repository
import (
"context"
"testing"
"time"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/pkg/emailverify"
)
// A clean delivery is credited once per step, a bounce naming the recipient
// is recorded, and the score lands on the contact row.
func TestLiveVerificationEvidenceLedger(t *testing.T) {
handle, pool := liveContactDB(t)
f := newSharedOrgFixture(t, pool)
ctx := context.Background()
repo := NewVerificationEvidenceRepository(handle)
contact := addLead(t, f, "live"+uuid.New().String()[:6]+"@test.local", "invalid", true)
seq := uuid.New()
if _, err := pool.Exec(ctx, `INSERT INTO sequences (id, campaign_id, organization_id, name, subject, body_plain, body_html, position, kind) VALUES ($1, $2, $3, 'Step 1', 's', 'b', 'b', 1, 'email')`, seq, f.campaign, f.org); err != nil {
t.Fatalf("sequence: %v", err)
}
if _, err := pool.Exec(ctx, `INSERT INTO campaign_contact_progress (campaign_id, contact_id, sequence_id, sent_at) VALUES ($1, $2, $3, NOW() - interval '4 days')`, f.campaign, contact, seq); err != nil {
t.Fatalf("progress: %v", err)
}
// The shared dev database may hold a backlog of older sends, so drain
// until our step is reached.
found := false
for i := 0; i < 50 && !found; i++ {
ids, err := repo.CreditCleanDeliveries(ctx, 72*time.Hour, 500)
if err != nil {
t.Fatalf("credit: %v", err)
}
if len(ids) == 0 {
break
}
for _, id := range ids {
if id == contact {
found = true
}
}
}
if !found {
t.Fatal("delivery not credited")
}
again, err := repo.CreditCleanDeliveries(ctx, 72*time.Hour, 100)
if err != nil {
t.Fatalf("credit again: %v", err)
}
for _, id := range again {
if id == contact {
t.Fatal("delivery credited twice")
}
}
inserted, err := repo.Record(ctx, contact, emailverify.EvidenceReplied, "msg-1", "", time.Now())
if err != nil || !inserted {
t.Fatalf("record: %v %v", inserted, err)
}
if dup, _ := repo.Record(ctx, contact, emailverify.EvidenceReplied, "msg-1", "", time.Now()); dup {
t.Fatal("same reply recorded twice")
}
rows, err := repo.ListForContact(ctx, contact)
if err != nil || len(rows) != 2 {
t.Fatalf("list: %d %v", len(rows), err)
}
verdict, err := repo.Verdict(ctx, contact)
if err != nil || verdict.Status != emailverify.StatusInvalid {
t.Fatalf("verdict: %+v %v", verdict, err)
}
if err := repo.SetScore(ctx, contact, "valid", 91, "replied today", time.Now(), true); err != nil {
t.Fatalf("set score: %v", err)
}
var status string
var confidence int
if err := pool.QueryRow(ctx, `SELECT verification_status, verification_confidence FROM contacts WHERE id = $1`, contact).Scan(&status, &confidence); err != nil {
t.Fatal(err)
}
if status != "valid" || confidence != 91 {
t.Fatalf("score not applied: %s %d", status, confidence)
}
}
+8 -1
View File
@@ -82,7 +82,14 @@ func (s *tasksService) ReconcileCampaignSchedules(ctx context.Context, limit int
// record which of the several possible causes it was.
s.autoPauseCampaign(ctx, id, uuid.Nil, autoPauseReason(cerr))
case errors.Is(cerr, scheduler.ErrCampaignCompleted), errors.Is(cerr, scheduler.ErrCampaignEnded):
// Nothing left to send (or past its end date): close it out.
// Nothing left to send (or past its end date): close it out, unless
// what is left was refused by verification, which parks it instead.
if errors.Is(cerr, scheduler.ErrCampaignCompleted) {
if n, uerr := s.campaignProgressRepo.CountUndeliverableLeads(ctx, id); uerr == nil && n > 0 {
s.pauseUndeliverable(ctx, id, uuid.Nil, n)
continue
}
}
s.campaignRepo.UpdateStatus(ctx, id, "completed")
default:
// Transient error (DB blip): leave it; the next pass retries.
+40 -2
View File
@@ -226,9 +226,16 @@ func (s *tasksService) HandleCampaignTask(task *proto.ProcessTask) *errx.Error {
if errors.Is(err, scheduler.ErrCampaignEnded) {
reason = "Campaign ended: reached its end date"
}
// Leads verification refused are never routed, so say so here
// rather than letting "all emails sent" cover for them.
// Leads verification refused are never routed. A campaign that ran
// out of leads only because of them is not finished: park it for the
// owner to re-verify or override (issue #264), instead of letting
// "all emails sent" cover for a verifier that may have been wrong.
if n, cerr := s.campaignProgressRepo.CountUndeliverableLeads(ctx, campaign.ID); cerr == nil && n > 0 {
if errors.Is(err, scheduler.ErrCampaignCompleted) {
s.pauseUndeliverable(ctx, campaign.ID, taskID, n)
executionStatus = "completed"
return nil
}
reason = fmt.Sprintf("%s (%d lead(s) skipped: address verification refused them)", reason, n)
}
s.campaignRepo.UpdateStatus(ctx, campaign.ID, "completed")
@@ -854,6 +861,37 @@ func autoPauseReason(err error) string {
}
}
// UndeliverablePauseReason is the activity-log line for a campaign parked
// because verification refused every remaining lead.
func UndeliverablePauseReason(n int) string {
return fmt.Sprintf("Campaign paused: %d remaining lead(s) were refused by address verification. Re-verify them or mark them deliverable to continue", n)
}
// pauseUndeliverable parks a campaign whose only remaining leads were refused
// by verification. Resumable: re-verifying or marking them deliverable
// restarts it.
func (s *tasksService) pauseUndeliverable(ctx context.Context, campaignID, taskID uuid.UUID, n int) {
s.campaignRepo.UpdateStatusWithLock(ctx, campaignID, "paused_undeliverable")
if taskID != uuid.Nil {
s.taskRepo.UpdateTaskStatus(ctx, taskID, "completed")
}
if s.campaignLogRepo != nil {
s.campaignLogRepo.CreateLog(ctx, &repository.CampaignLogEntry{
CampaignID: campaignID,
EventType: "auto_paused",
Message: UndeliverablePauseReason(n),
Metadata: map[string]interface{}{"undeliverable": n},
})
}
if s.streamingPublisher != nil {
s.streamingPublisher.PublishCampaignEvent(ctx, &pubsub.CampaignEvent{
BaseEvent: pubsub.BaseEvent{EventType: pubsub.EventCampaignCompleted},
CampaignID: campaignID.String(),
Status: "paused_undeliverable",
})
}
}
// autoPauseCampaign parks a campaign that has nothing it can send from. The
// reason is carried through to the activity log because "paused_no_accounts"
// covers several very different fixes (connect a mailbox, widen a sending
+6 -2
View File
@@ -20,6 +20,7 @@ import { CampaignContext } from "@/hooks/context/campaign";
import { useConfirm } from "@/hooks/context/confirm";
import LaunchCampaignDialog from "@/components/app/campaigns/LaunchCampaignDialog";
import CampaignActionsMenu from "@/components/app/campaigns/CampaignActionsMenu";
import UndeliverableBanner from "@/components/app/campaigns/UndeliverableBanner";
import { canStartCampaign } from "@/components/app/campaigns/useCampaignActions";
import toast from "react-hot-toast";
import { CAMPAIGN_DELETED_EVENT, type CampaignDeletedDetail } from "@/lib/realtime/campaignDeleted";
@@ -37,6 +38,7 @@ const TABS = [
const STATUS_PILL: Record<string, string> = {
active: "bg-emerald-50 text-emerald-700 border-emerald-200",
paused: "bg-amber-50 text-amber-700 border-amber-200",
paused_undeliverable: "bg-amber-50 text-amber-700 border-amber-200",
draft: "bg-slate-100 text-slate-600 border-slate-200",
completed: "bg-slate-100 text-slate-600 border-slate-200",
};
@@ -138,7 +140,7 @@ export default function CampaignLayout() {
<span
className={`shrink-0 inline-flex items-center h-5 px-2 rounded-md border text-[10px] uppercase tracking-[0.12em] font-medium ${pill}`}
>
{status}
{status === "paused_undeliverable" ? "needs verification" : status}
</span>
<ResourceViewers resource={`campaign:${campaign.id}`} className="shrink-0" />
</div>
@@ -173,6 +175,8 @@ export default function CampaignLayout() {
</div>
</div>
<UndeliverableBanner campaignId={campaign.id} status={status} />
<div className="shrink-0 px-3 flex items-center gap-1 border-b border-slate-200 overflow-x-auto no-scrollbar">
{TABS.map(({ label, path, Icon }) => {
const fullPath = `/app/campaigns/${id}${path}`;
@@ -208,7 +212,7 @@ export default function CampaignLayout() {
<LaunchCampaignDialog
campaign={launchOpen ? campaign : null}
onClose={() => setLaunchOpen(false)}
onConfirm={(cid) => startCampaign.mutateAsync(cid)}
onConfirm={(cid, options) => startCampaign.mutateAsync({ id: cid, options })}
/>
</CampaignContext.Provider>
);
+6 -1
View File
@@ -79,6 +79,7 @@ const STATUS_LABEL: Record<string, string> = {
paused_no_accounts: "no accounts",
paused_trial_expired: "trial expired",
paused_guardrail: "auto-paused",
paused_undeliverable: "needs verification",
completed: "finished",
draft: "draft",
};
@@ -93,6 +94,7 @@ const STATUS_TONE: Record<string, string> = {
paused_no_accounts: "text-amber-600",
paused_trial_expired: "text-amber-600",
paused_guardrail: "text-rose-600",
paused_undeliverable: "text-amber-600",
draft: "text-slate-500",
};
@@ -116,6 +118,9 @@ function CampaignStatusMark({ status }: { status: string }) {
} else if (status === "paused_guardrail") {
Icon = AlertTriangleIcon;
title = "Paused automatically — a deliverability guardrail was breached";
} else if (status === "paused_undeliverable") {
Icon = AlertTriangleIcon;
title = "Paused — address verification refused the remaining leads";
} else if (status === "paused_no_accounts" || status === "paused_trial_expired") {
Icon = AlertTriangleIcon;
title = status === "paused_no_accounts" ? "Paused — no sending accounts" : "Paused — trial expired";
@@ -603,7 +608,7 @@ export default function CampaignsPage() {
<LaunchCampaignDialog
campaign={launchTarget}
onClose={() => setLaunchTarget(null)}
onConfirm={(id) => startCampaign.mutateAsync(id)}
onConfirm={(id, options) => startCampaign.mutateAsync({ id, options })}
/>
</Page>
);
@@ -56,6 +56,15 @@ interface FieldDef {
// Credential fields for non-OAuth providers only. OAuth providers never paste.
const FIELDS_BY_PROVIDER: Record<string, FieldDef[]> = {
millionverifier: [
{
key: "api_key",
label: "MillionVerifier API key",
type: "password",
required: true,
helper: "API → API key in your MillionVerifier account. The key is checked before it is saved; one credit is spent per address verified.",
},
],
close: [
{ key: "workspace", label: "Organization", placeholder: "Acme" },
{
@@ -8,6 +8,10 @@ import { cn } from "@/lib/utils";
import { RAW_BRAND_LOGOS } from "./brandLogos";
const BRAND_ICON: Record<string, { hex: string; path: string }> = {
millionverifier: {
hex: "#1FB25A",
path: "M12 2C6.477 2 2 6.477 2 12s4.477 10 10 10 10-4.477 10-10S17.523 2 12 2zm-1.2 14.4-4.2-4.2 1.7-1.7 2.5 2.5 6.1-6.1 1.7 1.7-7.8 7.8z",
},
discord: {
hex: "#5865F2",
path: "M20.317 4.3698a19.7913 19.7913 0 00-4.8851-1.5152.0741.0741 0 00-.0785.0371c-.211.3753-.4447.8648-.6083 1.2495-1.8447-.2762-3.68-.2762-5.4868 0-.1636-.3933-.4058-.8742-.6177-1.2495a.077.077 0 00-.0785-.037 19.7363 19.7363 0 00-4.8852 1.515.0699.0699 0 00-.0321.0277C.5334 9.0458-.319 13.5799.0992 18.0578a.0824.0824 0 00.0312.0561c2.0528 1.5076 4.0413 2.4228 5.9929 3.0294a.0777.0777 0 00.0842-.0276c.4616-.6304.8731-1.2952 1.226-1.9942a.076.076 0 00-.0416-.1057c-.6528-.2476-1.2743-.5495-1.8722-.8923a.077.077 0 01-.0076-.1277c.1258-.0943.2517-.1923.3718-.2914a.0743.0743 0 01.0776-.0105c3.9278 1.7933 8.18 1.7933 12.0614 0a.0739.0739 0 01.0785.0095c.1202.099.246.1981.3728.2924a.077.077 0 01-.0066.1276 12.2986 12.2986 0 01-1.873.8914.0766.0766 0 00-.0407.1067c.3604.698.7719 1.3628 1.225 1.9932a.076.076 0 00.0842.0286c1.961-.6067 3.9495-1.5219 6.0023-3.0294a.077.077 0 00.0313-.0552c.5004-5.177-.8382-9.6739-3.5485-13.6604a.061.061 0 00-.0312-.0286zM8.02 15.3312c-1.1825 0-2.1569-1.0857-2.1569-2.419 0-1.3332.9555-2.4189 2.157-2.4189 1.2108 0 2.1757 1.0952 2.1568 2.419 0 1.3332-.9555 2.4189-2.1569 2.4189zm7.9748 0c-1.1825 0-2.1569-1.0857-2.1569-2.419 0-1.3332.9554-2.4189 2.1569-2.4189 1.2108 0 2.1757 1.0952 2.1568 2.419 0 1.3332-.946 2.4189-2.1568 2.4189Z",
@@ -18,6 +18,7 @@ import {
useOutreachSettings,
useUpdateOutreachSettings,
} from "@/lib/api/hooks/app/outreach/useOutreachSettings";
import VerificationSettings from "@/components/app/contacts/VerificationSettings";
import {
DEFAULT_PREFERRED_HOURS,
describeHours,
@@ -107,6 +108,7 @@ function SendingSettings() {
description="When campaign mail goes out, measured in your recipient's day."
actions={<SaveStatus status={autosave.status} onRetry={autosave.retry} />}
>
<VerificationSettings />
<Section
eyebrow="Send-time optimization"
description="Hold each campaign email until it lands inside the hours you pick, in the recipient's own timezone. It only ever delays a send, never brings one forward, and it still obeys the campaign schedule and each mailbox's working hours."
@@ -88,10 +88,13 @@ export default function LaunchCampaignDialog({
}: {
campaign: Campaign | null;
onClose: () => void;
onConfirm: (id: string) => Promise<unknown>;
onConfirm: (id: string, options?: { acknowledge_list_risk?: boolean }) => Promise<unknown>;
}) {
const [phase, setPhase] = React.useState<Phase>("idle");
const [error, setError] = React.useState<string | null>(null);
// The backend refused the launch on projected bounce rate. The member can
// take the risk explicitly (a list verified elsewhere), once they read why.
const [riskBlocked, setRiskBlocked] = React.useState(false);
const timer = React.useRef<number | null>(null);
// The list passes a campaign whose `sequences` is null (the list endpoint
@@ -111,6 +114,7 @@ export default function LaunchCampaignDialog({
if (campaign) {
setPhase("idle");
setError(null);
setRiskBlocked(false);
}
}, [campaign]);
@@ -130,16 +134,18 @@ export default function LaunchCampaignDialog({
return () => document.removeEventListener("keydown", onKey);
}, [campaign, phase, onClose]);
async function launch() {
async function launch(acknowledge = false) {
if (!campaign || phase === "launching") return;
setError(null);
setPhase("launching");
try {
await onConfirm(campaign.id);
await onConfirm(campaign.id, acknowledge ? { acknowledge_list_risk: true } : undefined);
setPhase("done");
timer.current = window.setTimeout(onClose, 1200);
} catch (e) {
setError(buildError(e as unknown as AppError));
const err = e as unknown as AppError;
setError(buildError(err));
setRiskBlocked(err?.code === "list_bounce_risk");
setPhase("idle");
}
}
@@ -322,9 +328,21 @@ export default function LaunchCampaignDialog({
>
Cancel
</button>
{riskBlocked && (
<motion.button
initial={{ opacity: 0, x: 8 }}
animate={{ opacity: 1, x: 0 }}
type="button"
onClick={() => launch(true)}
disabled={phase === "launching"}
className="h-8 px-3 rounded-md border border-amber-300 bg-amber-50 hover:bg-amber-100 text-amber-900 text-[12.5px] font-medium transition-colors disabled:opacity-50"
>
Launch anyway
</motion.button>
)}
<button
type="button"
onClick={launch}
onClick={() => launch()}
disabled={phase === "launching"}
className="h-8 px-3.5 rounded-md bg-sky-600 hover:bg-sky-700 text-white text-[12.5px] font-medium inline-flex items-center gap-2 transition-colors disabled:opacity-80"
>
@@ -0,0 +1,120 @@
// Shown on a campaign parked at paused_undeliverable: verification refused
// every remaining lead. Two ways out, both one click: re-check the leads, or
// trust a list verified elsewhere and send anyway.
import React from "react";
import { AnimatePresence, motion } from "framer-motion";
import { Link } from "react-router-dom";
import { Loader2Icon, RefreshCcwIcon, SendIcon, ShieldAlertIcon } from "lucide-react";
import toast from "react-hot-toast";
import { useConfirm } from "@/hooks/context/confirm";
import PermissionButton from "@/components/ui/PermissionButton";
import { useRequestContactVerification } from "@/lib/api/hooks/app/contacts/useContactVerification";
import useStartCampaign from "@/lib/api/hooks/app/campaigns/useStartCampaign";
import type { AppError } from "@/lib/api/client/normalizeError";
import buildError from "@/lib/helper/buildError";
export default function UndeliverableBanner({
campaignId,
status,
}: {
campaignId: string;
status: string;
}) {
const confirm = useConfirm();
const request = useRequestContactVerification();
const start = useStartCampaign();
const [action, setAction] = React.useState<"verify" | "send" | null>(null);
const reverify = async () => {
setAction("verify");
try {
const res = await request.mutateAsync({ campaign_id: campaignId, action: "verify" });
toast.success(`Re-checking ${res.affected} ${res.affected === 1 ? "lead" : "leads"}. Sending resumes as soon as any pass.`);
} catch (e) {
toast.error(buildError(e as AppError));
} finally {
setAction(null);
}
};
const sendAnyway = () => {
confirm?.show(
"Send to the refused leads anyway? They are marked deliverable and the campaign resumes. Use this only for a list you verified elsewhere: sending to dead addresses costs you bounces.",
async () => {
setAction("send");
try {
await request.mutateAsync({ campaign_id: campaignId, action: "mark_deliverable" });
await start.mutateAsync({ id: campaignId, options: { acknowledge_list_risk: true } });
toast.success("Campaign resumed");
} catch (e) {
toast.error(buildError(e as AppError));
} finally {
setAction(null);
}
},
);
};
return (
<AnimatePresence initial={false}>
{status === "paused_undeliverable" && (
<motion.div
key="undeliverable"
initial={{ opacity: 0, y: -8, height: 0 }}
animate={{ opacity: 1, y: 0, height: "auto" }}
exit={{ opacity: 0, y: -8, height: 0 }}
transition={{ type: "spring", duration: 0.4, bounce: 0.2 }}
className="overflow-hidden"
>
<div className="mx-5 mb-3 rounded-md border border-amber-200 bg-amber-50/70 px-3.5 py-3 flex flex-col md:flex-row md:items-center gap-3">
<motion.span
initial={{ rotate: -12, scale: 0.8 }}
animate={{ rotate: 0, scale: 1 }}
transition={{ type: "spring", duration: 0.5, bounce: 0.5 }}
className="shrink-0 w-7 h-7 rounded-md bg-amber-100 text-amber-700 inline-flex items-center justify-center"
>
<ShieldAlertIcon className="w-4 h-4" />
</motion.span>
<div className="min-w-0 flex-1">
<p className="text-[12.5px] font-medium text-amber-900">
Sending paused: address verification refused the remaining leads
</p>
<p className="text-[11.5px] text-amber-800/80 leading-snug mt-0.5">
Re-check them now, or send anyway if you verified this list with another service.
Verification settings live under{" "}
<Link to="/app/settings/sending" className="underline underline-offset-2 hover:text-amber-900">
Settings
</Link>
.
</p>
</div>
<div className="shrink-0 flex items-center gap-1.5">
<PermissionButton
permission="MANAGE_CONTACTS"
type="button"
onClick={reverify}
disabled={action !== null}
className="h-7 px-2.5 rounded-md border border-amber-300 bg-white hover:bg-amber-100 text-amber-900 text-[12px] font-medium inline-flex items-center gap-1.5 transition-colors disabled:opacity-60"
>
{action === "verify" ? <Loader2Icon className="w-3.5 h-3.5 animate-spin" /> : <RefreshCcwIcon className="w-3.5 h-3.5" />}
Re-verify leads
</PermissionButton>
<PermissionButton
permission="SEND_CAMPAIGNS"
type="button"
onClick={sendAnyway}
disabled={action !== null}
className="h-7 px-2.5 rounded-md bg-amber-600 hover:bg-amber-700 text-white text-[12px] font-medium inline-flex items-center gap-1.5 transition-colors disabled:opacity-60"
>
{action === "send" ? <Loader2Icon className="w-3.5 h-3.5 animate-spin" /> : <SendIcon className="w-3.5 h-3.5" />}
Send anyway
</PermissionButton>
</div>
</div>
</motion.div>
)}
</AnimatePresence>
);
}
@@ -19,7 +19,7 @@ export interface CampaignLike {
}
// Statuses from which a start is accepted by the backend.
const STARTABLE = new Set(["draft", "paused", "paused_no_accounts", "paused_guardrail", "completed"]);
const STARTABLE = new Set(["draft", "paused", "paused_no_accounts", "paused_guardrail", "paused_undeliverable", "completed"]);
export function canStartCampaign(status: string | null | undefined): boolean {
return STARTABLE.has(status ?? "draft");
@@ -13,7 +13,7 @@ import type SearchContacts from "@/lib/api/models/app/contacts/SearchContacts";
import type SearchContactsFilter from "@/lib/api/models/app/contacts/SearchContactsFilter";
import type { SearchContactsFilterType, SearchContactsSortBy } from "@/lib/api/models/app/contacts/search-contacts.types";
import type MiniCampaign from "@/lib/api/models/app/campaigns/MiniCampaign";
import type { LeadEngagement, LeadStatus } from "@/lib/api/models/app/contacts/Contact";
import type { LeadEngagement, LeadStatus, VerificationStatus } from "@/lib/api/models/app/contacts/Contact";
import React from "react";
import { AnimatePresence, motion } from "framer-motion";
@@ -292,6 +292,18 @@ export default function ContactFilters({
</>
)}
<Section label="Address verification" />
<div className="px-4 py-3">
<ChoiceRow
value={draft.verification_status}
onChange={(v) => setDraft((s) => ({ ...s, verification_status: v }))}
options={VERIFICATION_OPTIONS}
/>
<p className="text-[10.5px] text-slate-400 mt-1.5 leading-tight">
Undeliverable addresses are never sent to. Risky ones (catch-all domains, shared inboxes) send only when the campaign allows it.
</p>
</div>
<Section label="Subscription" />
<div className="px-4 py-3">
<Toggle3
@@ -459,6 +471,14 @@ const LEAD_STATUS_OPTIONS: { id: LeadStatus | undefined; label: string }[] = [
{ id: "unsubscribed", label: "Unsubscribed" },
];
const VERIFICATION_OPTIONS: { id: VerificationStatus | undefined; label: string }[] = [
{ id: undefined, label: "Any" },
{ id: "valid", label: "Deliverable" },
{ id: "risky", label: "Risky" },
{ id: "invalid", label: "Undeliverable" },
{ id: "unknown", label: "Unverified" },
];
const ENGAGEMENT_OPTIONS: { id: LeadEngagement | undefined; label: string }[] = [
{ id: undefined, label: "Any" },
{ id: "opened", label: "Opened" },
@@ -629,6 +649,7 @@ function countActiveFilters(f: SearchContacts, hasCampaignContext: boolean): num
if (f.query) n++;
n += f.filters.length;
if (f.subscribed !== undefined) n++;
if (f.verification_status) n++;
if (f.lead_status) n++;
if (f.engagement) n++;
if (f.min_campaigns !== undefined) n++;
@@ -29,6 +29,7 @@ import {
PlusIcon,
RefreshCcwIcon,
Settings2Icon,
ShieldCheckIcon,
SheetIcon,
SparklesIcon,
TrashIcon,
@@ -41,6 +42,8 @@ import { useConfirm } from "@/hooks/context/confirm";
import useSearchContacts from "@/lib/api/hooks/app/contacts/useSearchContacts";
import type SearchContacts from "@/lib/api/models/app/contacts/SearchContacts";
import useDeleteContacts from "@/lib/api/hooks/app/contacts/useDeleteContacts";
import { useRequestContactVerification } from "@/lib/api/hooks/app/contacts/useContactVerification";
import VerificationBadge from "./VerificationBadge";
import { useBatchResearch } from "@/lib/api/hooks/app/contacts/useContactResearch";
import useIntegrationConnections from "@/lib/api/hooks/app/integrations/useIntegrationConnections";
import { usePushContacts } from "@/lib/api/hooks/app/integrations/usePushContacts";
@@ -56,7 +59,7 @@ import ContactFilters from "./ContactFilters";
import ContactEdit from "./ContactEdit";
import type { ContactSlideTab } from "./contact-edit/tabs";
import type MiniCampaign from "@/lib/api/models/app/campaigns/MiniCampaign";
import type { ContactCampaignProgress, LeadEngagement, LeadStatus } from "@/lib/api/models/app/contacts/Contact";
import type { ContactCampaignProgress, LeadEngagement, LeadStatus, VerificationSource, VerificationStatus } from "@/lib/api/models/app/contacts/Contact";
import type { CampaignLeadCounts } from "@/lib/api/models/app/contacts/SearchContactsResult";
import ContactsEditBulk from "./ContactsEditBulk";
import { NewContactDialog } from "./NewContactDialog";
@@ -247,6 +250,36 @@ export default function ContactsTable({
);
}
// Bulk verification actions. A re-check is queued and each row's mark
// updates live as its verdict lands; marking deliverable is immediate.
const verification = useRequestContactVerification();
function bulkVerify() {
if (selected.length === 0) return;
const ids = selected;
confirm?.show(
`Re-verify ${ids.length} ${ids.length === 1 ? "address" : "addresses"}? Verdicts land in the background${
ids.length > 50 ? " over the next few minutes" : ""
}.`,
async () => {
const res = await verification.mutateAsync({ contacts: ids, action: "verify" });
toast.success(`Re-checking ${res.affected} ${res.affected === 1 ? "address" : "addresses"}`);
setSelected([]);
},
);
}
function bulkMarkDeliverable() {
if (selected.length === 0) return;
const ids = selected;
confirm?.show(
`Mark ${ids.length} ${ids.length === 1 ? "address" : "addresses"} deliverable? Campaigns will send to them even if verification refused them. Use this for a list you verified elsewhere.`,
async () => {
const res = await verification.mutateAsync({ contacts: ids, action: "mark_deliverable" });
toast.success(`${res.affected} marked deliverable`);
setSelected([]);
},
);
}
const embedded = !!current_campaign;
// Leads-view scope chips write straight into the search request, so the
// rows, the total and pagination all come from the server for that scope.
@@ -402,6 +435,9 @@ export default function ContactsTable({
onBulkEdit={() => setBulkEdit(true)}
onResearch={bulkResearch}
researching={batchResearch.isPending}
onVerify={bulkVerify}
onMarkDeliverable={bulkMarkDeliverable}
verifying={verification.isPending}
onDelete={() =>
confirm?.show(
`Are you sure you want to delete ${selected.length} contacts?`,
@@ -613,6 +649,9 @@ export default function ContactsTable({
onBulkEdit={() => setBulkEdit(true)}
onResearch={bulkResearch}
researching={batchResearch.isPending}
onVerify={bulkVerify}
onMarkDeliverable={bulkMarkDeliverable}
verifying={verification.isPending}
onDelete={() =>
confirm?.show(
`Are you sure you want to delete ${selected.length} contacts?`,
@@ -689,6 +728,12 @@ function ContactsTableBody({
campaigns: { id: string }[];
categories?: { id: string; title: string; color: string }[];
campaign_lead?: ContactCampaignProgress | null;
verification_status?: VerificationStatus;
verification_sub_status?: string;
verification_source?: VerificationSource;
verification_provider?: string;
verification_checked_at?: string | null;
verification_confidence?: number;
created_at: Date;
}[];
selected: string[];
@@ -862,6 +907,7 @@ function ContactsTableBody({
<div className="text-[10.5px] text-slate-400 truncate font-mono leading-tight flex items-center gap-1">
<MailIcon className="w-2.5 h-2.5 shrink-0" />
<span className="truncate">{c.email}</span>
<VerificationBadge contact={c} />
</div>
</div>
</div>
@@ -1319,6 +1365,9 @@ function SelectionBar({
onBulkEdit,
onResearch,
researching,
onVerify,
onMarkDeliverable,
verifying,
onDelete,
onClear,
}: {
@@ -1330,6 +1379,9 @@ function SelectionBar({
onBulkEdit: () => void;
onResearch: () => void;
researching: boolean;
onVerify: () => void;
onMarkDeliverable: () => void;
verifying: boolean;
onDelete: () => void;
onClear: () => void;
}) {
@@ -1391,6 +1443,23 @@ function SelectionBar({
{researching ? <Loader2Icon className="w-3 h-3 animate-spin" /> : <SparklesIcon className="w-3 h-3" />}
<span className="hidden sm:inline">Research</span>
</button>
<PopoverMenu side="top" align="center">
<PopoverMenuTrigger asChild>
<button
type="button"
disabled={verifying}
className="h-7 px-2.5 rounded text-[12px] text-slate-700 hover:text-emerald-700 hover:bg-emerald-50 font-medium inline-flex items-center gap-1.5 transition-colors disabled:opacity-60"
>
{verifying ? <Loader2Icon className="w-3 h-3 animate-spin" /> : <ShieldCheckIcon className="w-3 h-3" />}
<span className="hidden sm:inline">Verify</span>
</button>
</PopoverMenuTrigger>
<PopoverMenuContent>
<PopoverMenuLabel>Address verification</PopoverMenuLabel>
<PopoverMenuItem onSelect={onVerify}>Re-verify {count}</PopoverMenuItem>
<PopoverMenuItem onSelect={onMarkDeliverable}>Mark deliverable</PopoverMenuItem>
</PopoverMenuContent>
</PopoverMenu>
<button
type="button"
onClick={onDelete}
@@ -28,6 +28,7 @@ import {
DownloadIcon,
FileSpreadsheetIcon,
Loader2Icon,
ShieldCheckIcon,
UploadCloudIcon,
XIcon,
} from "lucide-react";
@@ -57,6 +58,7 @@ import {
CUSTOM_KEY_RULES,
DEDUP_OPTIONS,
STANDARD_TARGETS,
VERIFICATION_VOCABULARY_LABELS,
describeError,
isCustomTarget,
isValidCustomKey,
@@ -545,6 +547,22 @@ export function MapStep({
header={col}
onChange={(next) => updateMapping(idx, next)}
/>
<AnimatePresence initial={false}>
{m.target === "verification_status" && (
<motion.p
key="vocab"
initial={{ opacity: 0, y: -4 }}
animate={{ opacity: 1, y: 0 }}
exit={{ opacity: 0, y: -4 }}
className="mt-1 text-[10.5px] text-emerald-700 inline-flex items-center gap-1"
>
<ShieldCheckIcon className="w-3 h-3" />
{m.verification_provider && VERIFICATION_VOCABULARY_LABELS[m.verification_provider]
? `${VERIFICATION_VOCABULARY_LABELS[m.verification_provider]} results recognised`
: "Verification results recognised; these leads skip the built-in check"}
</motion.p>
)}
</AnimatePresence>
</td>
</tr>
);
@@ -0,0 +1,76 @@
// The pre-send verdict on a contact's address, as a small animated mark next
// to the email. Verdicts land in the background, so the mark springs in when
// a row's status changes rather than just appearing.
import { AnimatePresence, motion } from "framer-motion";
import { AlertTriangleIcon, CircleDashedIcon, ShieldCheckIcon, ShieldXIcon } from "lucide-react";
import type Contact from "@/lib/api/models/app/contacts/Contact";
import { cn } from "@/lib/utils";
const META = {
valid: { label: "Deliverable", tone: "text-emerald-600", Icon: ShieldCheckIcon },
risky: { label: "Risky", tone: "text-amber-600", Icon: AlertTriangleIcon },
invalid: { label: "Undeliverable", tone: "text-rose-600", Icon: ShieldXIcon },
unknown: { label: "Not verified yet", tone: "text-slate-300", Icon: CircleDashedIcon },
} as const;
const SUB_LABEL: Record<string, string> = {
catch_all: "catch-all domain",
disposable: "disposable domain",
role: "shared inbox",
spamtrap: "spam trap",
mailbox_full: "mailbox full",
no_mx: "no mail server",
syntax: "malformed",
undisclosed: "provider does not disclose mailboxes",
};
const SOURCE_LABEL: Record<string, string> = {
probe: "checked by Warmbly",
provider: "checked by MillionVerifier",
imported: "imported with the list",
manual: "marked by a teammate",
};
export function verificationTitle(c: Pick<Contact, "verification_status" | "verification_sub_status" | "verification_source" | "verification_provider" | "verification_confidence">): string {
const status = c.verification_status ?? "unknown";
const meta = META[status] ?? META.unknown;
const parts: string[] = [c.verification_confidence ? `${meta.label} (${c.verification_confidence}% sure)` : meta.label];
if (c.verification_sub_status && SUB_LABEL[c.verification_sub_status]) parts.push(SUB_LABEL[c.verification_sub_status]);
if (c.verification_source && SOURCE_LABEL[c.verification_source]) {
const src = c.verification_source === "imported" && c.verification_provider && c.verification_provider !== "imported"
? `imported from ${c.verification_provider}`
: SOURCE_LABEL[c.verification_source];
parts.push(src);
}
return parts.join(" · ");
}
export default function VerificationBadge({
contact,
className,
}: {
contact: Pick<Contact, "verification_status" | "verification_sub_status" | "verification_source" | "verification_provider" | "verification_checked_at" | "verification_confidence">;
className?: string;
}) {
const status = contact.verification_status ?? "unknown";
const meta = META[status] ?? META.unknown;
const Icon = meta.Icon;
const pending = status === "unknown" && !contact.verification_checked_at;
return (
<AnimatePresence mode="popLayout" initial={false}>
<motion.span
key={status}
initial={{ scale: 0.4, opacity: 0 }}
animate={{ scale: 1, opacity: 1 }}
exit={{ scale: 0.4, opacity: 0 }}
transition={{ type: "spring", duration: 0.35, bounce: 0.45 }}
title={pending ? "Verification queued" : verificationTitle(contact)}
aria-label={verificationTitle(contact)}
className={cn("inline-flex shrink-0", meta.tone, pending && "animate-[spin_3s_linear_infinite]", className)}
>
<Icon className="w-2.5 h-2.5" />
</motion.span>
</AnimatePresence>
);
}
@@ -0,0 +1,130 @@
// Address verification at a glance: who checks this workspace's contacts,
// what it has found so far, and the one-click path to pay-as-you-go
// MillionVerifier. Verdicts change in the background, so the bar and the
// numbers animate as they land.
import React from "react";
import { Link } from "react-router-dom";
import { motion } from "framer-motion";
import { AlertTriangleIcon, ArrowRightIcon, CoinsIcon, ShieldCheckIcon } from "lucide-react";
import { Section } from "@/app/app/settings/_components/SectionShell";
import AnimatedNumber from "@/components/ui/AnimatedNumber";
import { useContactVerification } from "@/lib/api/hooks/app/contacts/useContactVerification";
import { cn } from "@/lib/utils";
const SEGMENTS = [
{ key: "valid", label: "Deliverable", color: "bg-emerald-500", text: "text-emerald-700" },
{ key: "risky", label: "Risky", color: "bg-amber-400", text: "text-amber-700" },
{ key: "invalid", label: "Undeliverable", color: "bg-rose-500", text: "text-rose-700" },
{ key: "unknown", label: "Unverified", color: "bg-slate-200", text: "text-slate-500" },
] as const;
export default function VerificationSettings() {
const { data, isLoading } = useContactVerification();
const counts = data?.counts;
const total = counts ? counts.valid + counts.risky + counts.invalid + counts.unknown : 0;
const paid = data?.provider === "millionverifier";
return (
<Section
eyebrow="Address verification"
description="Every contact you add is checked before any campaign sends to it, so bad addresses never become bounces. Nothing to run: verdicts land in the background and appear on each contact."
>
{isLoading || !data ? (
<div className="h-16 rounded-md bg-slate-100 animate-pulse" />
) : (
<div className="space-y-4">
<div className="flex flex-col md:flex-row md:items-center gap-3">
<div className="flex items-center gap-2.5 min-w-0">
<motion.span
initial={{ scale: 0.6, opacity: 0 }}
animate={{ scale: 1, opacity: 1 }}
transition={{ type: "spring", duration: 0.5, bounce: 0.4 }}
className={cn(
"w-8 h-8 rounded-md inline-flex items-center justify-center shrink-0",
paid ? "bg-emerald-50 text-emerald-600" : "bg-sky-50 text-sky-600",
)}
>
<ShieldCheckIcon className="w-4 h-4" />
</motion.span>
<div className="min-w-0">
<p className="text-[12.5px] font-medium text-slate-900">
{paid ? "MillionVerifier" : "Built-in check"}
<span className="ml-1.5 text-[10px] uppercase tracking-[0.14em] text-slate-400 font-medium">
{paid ? "pay as you go" : "included"}
</span>
</p>
<p className="text-[11.5px] text-slate-500 leading-snug">
{paid
? "One credit per address, from your own MillionVerifier balance."
: data.builtin_ready
? "Syntax, mail server, disposable domains and a mailbox probe. Catch-all domains and Microsoft 365 stay unverified."
: "Syntax, mail server and disposable-domain checks. The mailbox probe is off on this instance, so most addresses stay unverified."}
</p>
</div>
</div>
<div className="md:ml-auto shrink-0 flex items-center gap-2">
{paid && data.credits !== undefined && (
<span className="inline-flex items-center gap-1.5 h-7 px-2.5 rounded-md bg-slate-50 border border-slate-200 text-[12px] text-slate-700">
<CoinsIcon className="w-3.5 h-3.5 text-amber-500" />
<AnimatedNumber value={data.credits} className="font-medium tabular-nums" /> credits
</span>
)}
<Link
to="/app/integrations"
className="inline-flex items-center gap-1.5 h-7 px-2.5 rounded-md bg-sky-600 hover:bg-sky-700 text-white text-[12px] font-medium transition-colors"
>
{paid ? "Manage connection" : "Connect MillionVerifier"}
<ArrowRightIcon className="w-3.5 h-3.5" />
</Link>
</div>
</div>
{data.provider_error && (
<div className="flex items-start gap-2 rounded-md border border-amber-200 bg-amber-50 px-3 py-2">
<AlertTriangleIcon className="w-3.5 h-3.5 text-amber-600 shrink-0 mt-0.5" />
<p className="text-[12px] text-amber-800 leading-snug">{data.provider_error}</p>
</div>
)}
<div>
<div className="flex h-2 w-full overflow-hidden rounded-full bg-slate-100">
{SEGMENTS.map((s) => {
const n = counts?.[s.key] ?? 0;
return (
<motion.div
key={s.key}
className={s.color}
initial={{ width: 0 }}
animate={{ width: total ? `${(n / total) * 100}%` : "0%" }}
transition={{ type: "spring", duration: 0.8, bounce: 0.1 }}
/>
);
})}
</div>
<div className="mt-2.5 flex flex-wrap gap-x-5 gap-y-1.5">
{SEGMENTS.map((s) => (
<div key={s.key} className="inline-flex items-center gap-1.5 text-[11.5px]">
<span className={cn("w-2 h-2 rounded-full", s.color)} />
<span className="text-slate-500">{s.label}</span>
<AnimatedNumber value={counts?.[s.key] ?? 0} className={cn("font-medium tabular-nums", s.text)} />
</div>
))}
{(counts?.pending ?? 0) > 0 && (
<motion.span
initial={{ opacity: 0 }}
animate={{ opacity: 1 }}
className="inline-flex items-center gap-1.5 text-[11.5px] text-slate-400"
>
<span className="w-2 h-2 rounded-full bg-sky-400 animate-pulse" />
<AnimatedNumber value={counts?.pending ?? 0} className="tabular-nums" /> in the queue
</motion.span>
)}
</div>
</div>
</div>
)}
</Section>
);
}
@@ -579,6 +579,8 @@ function campaignStatusLabel(status: string): string {
return "paused";
case "paused_guardrail":
return "auto-paused";
case "paused_undeliverable":
return "paused, needs verification";
case "paused_no_accounts":
return "paused, no accounts";
case "paused_trial_expired":
@@ -20,6 +20,7 @@ import type ContactDetail from "@/lib/api/models/app/contacts/ContactDetail";
import type Contact from "@/lib/api/models/app/contacts/Contact";
import { fmtAbsolute, fmtRelative } from "./format";
import { sourceLabel } from "./ActivityTab";
import VerificationCard from "./VerificationCard";
export default function OverviewTab({
contact,
@@ -51,6 +52,10 @@ export default function OverviewTab({
</div>
)}
<Section title="Deliverability">
<VerificationCard detail={detail?.verification} loading={detailLoading} />
</Section>
<Section title="Engagement">
<div className="grid grid-cols-2 md:grid-cols-3 gap-1.5">
<StatTile
@@ -0,0 +1,136 @@
// The "why" behind a contact's deliverability verdict: an animated
// confidence ring, the reasons in plain words, and the observations they
// were scored from. Absence of engagement is never listed, because it is not
// evidence of anything.
import { AnimatePresence, motion } from "framer-motion";
import {
AlertTriangleIcon,
CircleDashedIcon,
MailCheckIcon,
MailOpenIcon,
MailWarningIcon,
MousePointerClickIcon,
ReplyIcon,
ShieldCheckIcon,
ShieldXIcon,
} from "lucide-react";
import type { ContactVerificationDetail, VerificationEvidenceKind } from "@/lib/api/models/app/contacts/ContactDetail";
import { fmtRelative } from "./format";
import { cn } from "@/lib/utils";
const STATUS = {
valid: { label: "Deliverable", ring: "stroke-emerald-500", text: "text-emerald-700", Icon: ShieldCheckIcon },
risky: { label: "Risky", ring: "stroke-amber-500", text: "text-amber-700", Icon: AlertTriangleIcon },
invalid: { label: "Undeliverable", ring: "stroke-rose-500", text: "text-rose-700", Icon: ShieldXIcon },
unknown: { label: "Not verified", ring: "stroke-slate-300", text: "text-slate-500", Icon: CircleDashedIcon },
} as const;
const EVIDENCE: Record<VerificationEvidenceKind, { label: string; Icon: typeof MailCheckIcon; tone: string }> = {
delivered: { label: "Delivered, no bounce", Icon: MailCheckIcon, tone: "text-emerald-600" },
opened: { label: "Opened by a person", Icon: MailOpenIcon, tone: "text-emerald-600" },
clicked: { label: "Clicked a link", Icon: MousePointerClickIcon, tone: "text-emerald-600" },
replied: { label: "Replied", Icon: ReplyIcon, tone: "text-emerald-600" },
auto_replied: { label: "Automatic reply (mailbox is live)", Icon: ReplyIcon, tone: "text-emerald-600" },
bounced_recipient: { label: "Bounced: mailbox does not exist", Icon: MailWarningIcon, tone: "text-rose-600" },
bounced_other: { label: "Bounced for another reason", Icon: MailWarningIcon, tone: "text-slate-500" },
};
export default function VerificationCard({
detail,
loading,
}: {
detail?: ContactVerificationDetail | null;
loading: boolean;
}) {
if (loading && !detail) {
return <div className="h-20 rounded-md border border-slate-200 bg-slate-50 animate-pulse" />;
}
if (!detail) return null;
const meta = STATUS[detail.status] ?? STATUS.unknown;
const Icon = meta.Icon;
const r = 16;
const c = 2 * Math.PI * r;
const pct = Math.max(0, Math.min(100, detail.confidence));
return (
<div className="rounded-md border border-slate-200 bg-white overflow-hidden">
<div className="px-3 py-2.5 flex items-center gap-3">
<div className="relative w-11 h-11 shrink-0">
<svg viewBox="0 0 40 40" className="w-11 h-11 -rotate-90">
<circle cx="20" cy="20" r={r} className="stroke-slate-100" strokeWidth="4" fill="none" />
<motion.circle
cx="20"
cy="20"
r={r}
className={meta.ring}
strokeWidth="4"
strokeLinecap="round"
fill="none"
strokeDasharray={c}
initial={{ strokeDashoffset: c }}
animate={{ strokeDashoffset: c - (c * pct) / 100 }}
transition={{ type: "spring", duration: 1, bounce: 0.15 }}
/>
</svg>
<motion.span
key={detail.status}
initial={{ scale: 0.5, opacity: 0 }}
animate={{ scale: 1, opacity: 1 }}
transition={{ type: "spring", duration: 0.4, bounce: 0.5 }}
className={cn("absolute inset-0 flex items-center justify-center", meta.text)}
>
<Icon className="w-4 h-4" />
</motion.span>
</div>
<div className="min-w-0 flex-1">
<div className="flex items-baseline gap-1.5">
<span className={cn("text-[13px] font-semibold", meta.text)}>{meta.label}</span>
<span className="text-[11px] text-slate-400 tabular-nums">{pct}% sure</span>
{detail.decisive && (
<span className="ml-auto text-[10px] uppercase tracking-[0.12em] text-slate-400 font-medium">
from real mail
</span>
)}
</div>
<ul className="mt-0.5 space-y-0.5">
<AnimatePresence initial={false}>
{detail.reasons.slice(0, 3).map((reason, i) => (
<motion.li
key={reason}
initial={{ opacity: 0, x: -6 }}
animate={{ opacity: 1, x: 0 }}
transition={{ delay: i * 0.06 }}
className="text-[11.5px] text-slate-600 leading-snug"
>
{reason.charAt(0).toUpperCase() + reason.slice(1)}
</motion.li>
))}
</AnimatePresence>
</ul>
</div>
</div>
{detail.evidence.length > 0 && (
<div className="border-t border-slate-100 divide-y divide-slate-100">
{detail.evidence.slice(0, 6).map((e, i) => {
const m = EVIDENCE[e.kind] ?? EVIDENCE.bounced_other;
const EIcon = m.Icon;
return (
<motion.div
key={`${e.kind}-${e.observed_at}-${i}`}
initial={{ opacity: 0 }}
animate={{ opacity: 1 }}
transition={{ delay: 0.15 + i * 0.05 }}
className="px-3 py-1.5 flex items-center gap-2 text-[11.5px]"
>
<EIcon className={cn("w-3 h-3 shrink-0", m.tone)} />
<span className="text-slate-700 truncate">{m.label}</span>
<span className="ml-auto text-slate-400 shrink-0">{fmtRelative(e.observed_at)}</span>
</motion.div>
);
})}
</div>
)}
</div>
);
}
@@ -20,8 +20,24 @@ export const STANDARD_TARGETS: { id: string; label: string }[] = [
{ id: "phone", label: "Phone" },
{ id: "subscribed", label: "Subscribed" },
{ id: "categories", label: "Categories" },
{ id: "verification_status", label: "Verification status" },
];
// Vocabularies the verification_status target can read, for the mapping
// row's "recognised as" badge. Mirrors emailverify.KnownVocabulary.
export const VERIFICATION_VOCABULARY_LABELS: Record<string, string> = {
zerobounce: "ZeroBounce",
millionverifier: "MillionVerifier",
neverbounce: "NeverBounce",
bouncer: "Bouncer",
kickbox: "Kickbox",
emailable: "Emailable",
debounce: "DeBounce",
clearout: "Clearout",
emaillistverify: "EmailListVerify",
builtin: "Warmbly",
};
export const DEDUP_OPTIONS: { id: ImportDedupStrategy; label: string; hint: string }[] = [
{ id: "skip", label: "Skip existing", hint: "If a contact with this email exists, leave it alone." },
{ id: "update", label: "Update existing", hint: "Merge new values onto the existing contact." },
@@ -1,9 +1,15 @@
import Request from "../../Request";
export default async function startCampaign(id: string): Promise<void> {
export interface StartCampaignOptions {
// Launch past the bounce-risk gate after reading the projection.
acknowledge_list_risk?: boolean;
}
export default async function startCampaign(id: string, options?: StartCampaignOptions): Promise<void> {
return await Request<void>({
method: "POST",
url: `/campaigns/${id}/start`,
data: options ?? {},
authorization: true,
})
}
@@ -15,6 +15,7 @@ export type ImportColumnTarget =
| "phone"
| "subscribed"
| "categories"
| "verification_status"
| string; // "custom:<key>"
export type ImportDedupStrategy = "skip" | "update" | "create_duplicate";
@@ -23,6 +24,10 @@ export interface ImportColumnMapping {
index: number;
target: ImportColumnTarget;
custom_key?: string;
// For a verification_status column: the vocabulary the header or its
// values were recognised as (e.g. "zerobounce"). Empty means each value
// is recognised by itself.
verification_provider?: string;
}
export interface ImportPreview {
@@ -0,0 +1,49 @@
// Address verification: who checks this workspace's contacts, and the member
// actions on a selection (re-verify, mark deliverable / undeliverable).
import Request from "../../Request";
import type { ContactVerificationCounts } from "@/lib/api/models/app/contacts/SearchContactsResult";
export interface VerificationOverview {
// "builtin" (the in-house check) or "millionverifier".
provider: "builtin" | "millionverifier" | string;
connection_id?: string;
credits?: number;
// Set when a provider is connected but unusable (bad key, no credits).
provider_error?: string;
// Whether the built-in check can reach mail servers from this instance.
builtin_ready: boolean;
counts: ContactVerificationCounts;
}
export type VerificationAction = "verify" | "mark_deliverable" | "mark_undeliverable";
export interface VerificationRequest {
contacts?: string[];
// Every lead of this campaign that verification refused.
campaign_id?: string;
action: VerificationAction;
}
export interface VerificationResponse {
affected: number;
action: VerificationAction;
queued: boolean;
}
export async function getContactVerification(): Promise<VerificationOverview> {
return await Request<VerificationOverview>({
method: "GET",
url: "/contacts/verification",
authorization: true,
});
}
export async function requestContactVerification(req: VerificationRequest): Promise<VerificationResponse> {
return await Request<VerificationResponse>({
method: "POST",
url: "/contacts/verification",
data: req,
authorization: true,
});
}
@@ -1,11 +1,12 @@
import { useMutation, useQueryClient } from "@tanstack/react-query";
import startCampaign from "@/lib/api/client/app/campaigns/startCampaign";
import startCampaign, { type StartCampaignOptions } from "@/lib/api/client/app/campaigns/startCampaign";
export default function useStartCampaign() {
const queryClient = useQueryClient();
return useMutation({
mutationFn: (id: string) => startCampaign(id),
mutationFn: (arg: string | { id: string; options?: StartCampaignOptions }) =>
typeof arg === "string" ? startCampaign(arg) : startCampaign(arg.id, arg.options),
onSuccess: () => {
queryClient.invalidateQueries({
queryKey: ["campaigns"],
@@ -0,0 +1,26 @@
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import {
getContactVerification,
requestContactVerification,
type VerificationRequest,
} from "@/lib/api/client/app/contacts/verification";
export function useContactVerification(enabled = true) {
return useQuery({
queryKey: ["contacts", "verification"],
queryFn: getContactVerification,
staleTime: 60 * 1000,
enabled,
});
}
export function useRequestContactVerification() {
const queryClient = useQueryClient();
return useMutation({
mutationFn: (req: VerificationRequest) => requestContactVerification(req),
onSuccess: () => {
queryClient.invalidateQueries({ queryKey: ["contacts"] });
queryClient.invalidateQueries({ queryKey: ["campaigns"] });
},
});
}
@@ -7,6 +7,10 @@ export default interface AddContact {
campaigns: string[];
categories?: string[];
// A verdict you already hold for the address, in Warmbly's vocabulary or
// any known provider's (ZeroBounce, MillionVerifier, NeverBounce, ...).
verification_status?: string;
verification_provider?: string;
custom_fields: Record<string, string>;
// First-touch source hint. The dashboard may say "manual" or "campaign"
@@ -49,6 +49,15 @@ export interface ContactCampaignProgress {
failure_reason?: string;
}
// VerificationStatus mirrors emailverify.Status: the pre-send verdict on the
// address. "invalid" is never sent to; "risky" (catch-all, role) only when the
// campaign allows it; "unknown" and "valid" always send.
export type VerificationStatus = "valid" | "risky" | "invalid" | "unknown";
// VerificationSource says who produced the verdict: the in-house probe, a
// connected provider, a status column imported with the list, or a member.
export type VerificationSource = "" | "probe" | "provider" | "imported" | "manual";
export default interface Contact {
id: string;
@@ -64,6 +73,17 @@ export default interface Contact {
campaigns: MiniCampaign[];
categories: MiniCategory[];
verification_status?: VerificationStatus;
verification_reason?: string;
verification_sub_status?: string;
verification_source?: VerificationSource;
verification_provider?: string;
verification_checked_at?: string | null;
// How sure the platform is of the status, 0 to 100, scored from the last
// check plus what real mail to the address showed.
verification_confidence?: number;
is_catch_all?: boolean;
// Present only in the campaign Leads view (single-campaign search). Drives
// the per-lead processing-state column.
campaign_lead?: ContactCampaignProgress | null;
@@ -33,9 +33,36 @@ export type ContactSource =
| "api"
| "ai_assistant";
// One observed fact about the mailbox. Silence is never recorded: a contact
// who does not open or reply has said nothing about their address.
export type VerificationEvidenceKind =
| "delivered"
| "opened"
| "clicked"
| "replied"
| "auto_replied"
| "bounced_recipient"
| "bounced_other";
export interface ContactVerificationEvidence {
kind: VerificationEvidenceKind;
detail?: string;
observed_at: string;
}
export interface ContactVerificationDetail {
status: "valid" | "risky" | "invalid" | "unknown";
confidence: number;
reasons: string[];
// True when real mail, not a check, decided the status.
decisive: boolean;
evidence: ContactVerificationEvidence[];
}
export default interface ContactDetail extends Contact {
engagement: ContactEngagement;
suppression?: ContactSuppression | null;
verification?: ContactVerificationDetail | null;
// First-touch attribution; never changes after creation.
source: ContactSource;
@@ -1,6 +1,6 @@
import type { SearchContactsSortBy } from "./search-contacts.types";
import type SearchContactsFilter from "./SearchContactsFilter";
import type { LeadEngagement, LeadStatus } from "./Contact";
import type { LeadEngagement, LeadStatus, VerificationStatus } from "./Contact";
export default interface SearchContacts {
query: string;
@@ -14,6 +14,7 @@ export default interface SearchContacts {
min_campaigns?: number;
max_campaigns?: number;
subscribed?: boolean;
verification_status?: VerificationStatus;
created_after?: Date;
created_before?: Date;
updated_after?: Date;
@@ -16,6 +16,17 @@ export interface ContactsCounts {
in_campaign: number;
not_contacted: number;
categories: ContactCategoryCount[];
verification?: ContactVerificationCounts;
}
// Org contacts by verification verdict. pending is the subset of unknown
// nobody has checked yet.
export interface ContactVerificationCounts {
valid: number;
risky: number;
invalid: number;
unknown: number;
pending: number;
}
// Per-status lead totals for one campaign's Leads view. Returned on the first
@@ -13,7 +13,9 @@ export type IntegrationProvider =
| "slack"
| "discord"
| "calendly"
| "cal_com";
| "cal_com"
| "google_sheets"
| "millionverifier";
export type IntegrationAuthMethod = "oauth" | "api_key" | "webhook";
@@ -32,7 +34,8 @@ export type IntegrationCategory =
| "automation"
| "notifications"
| "meetings"
| "data";
| "data"
| "verification";
export interface IntegrationCatalogEntry {
provider: IntegrationProvider;
@@ -281,6 +284,7 @@ export const CATEGORY_LABELS: Record<IntegrationCategory, string> = {
notifications: "Notifications",
meetings: "Meetings",
data: "Data",
verification: "Verification",
};
export const CATEGORY_ORDER: IntegrationCategory[] = [
@@ -289,6 +293,7 @@ export const CATEGORY_ORDER: IntegrationCategory[] = [
"automation",
"meetings",
"data",
"verification",
];
// Reply-intent classifier buckets, used to filter reply automations
@@ -356,6 +361,8 @@ export const PROVIDER_LABELS: Record<IntegrationProvider, string> = {
discord: "Discord",
calendly: "Calendly",
cal_com: "Cal.com",
google_sheets: "Google Sheets",
millionverifier: "MillionVerifier",
};
// A connection is bookable when it's a connected scheduling provider with a