From a02ff7c936824e5baa96d1818b8bc823a666080a Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Sat, 29 Aug 2026 23:11:22 -0700 Subject: [PATCH 1/3] feat: address verification overhaul for #264: MillionVerifier as a pay-as-you-go integration plugin with org-sealed key and automatic fallback to the built-in check, built-in prober gains domain cache, Microsoft/Yahoo fingerprinting, MX fallback, disposable/role sub-statuses and a self-check breaker, imports and POST /contacts accept verification results in any known provider vocabulary with auto-detected columns, verdict provenance and expiry columns (migration 000110), campaigns park at paused_undeliverable with re-verify/send-anyway instead of finishing, POST/GET /contacts/verification bulk actions and overview, launch gate override via acknowledge_list_risk, animated verification marks, banner and settings card in the dashboard, and docs --- admin/src/app/dashboard/CampaignsPage.tsx | 2 + admin/src/lib/api/models/admin.ts | 2 +- cmd/backend/main.go | 30 +- docs/content/docs/api/endpoints.mdx | 2 + docs/content/docs/api/error-codes.mdx | 6 + docs/content/docs/api/reference/campaigns.mdx | 3 + docs/content/docs/api/reference/contacts.mdx | 55 ++- .../docs/development/configuration.mdx | 3 + docs/content/docs/guides/campaigns.mdx | 4 +- docs/content/docs/guides/contacts-crm.mdx | 2 +- docs/content/docs/guides/deliverability.mdx | 20 +- docs/content/docs/guides/integrations.mdx | 5 + internal/api/handler/campaign.go | 9 +- internal/api/handler/contact.go | 4 + internal/api/handler/contact_io.go | 4 + internal/api/handler/email_verification.go | 78 +++- internal/api/routes.go | 4 + internal/app/aitools/tools_campaigns.go | 2 +- internal/app/campaign/handlers.go | 37 +- internal/app/campaign/service.go | 18 +- internal/app/contact/campaign_state.go | 2 + internal/app/contact/import.go | 48 +- internal/app/emailverify/breaker.go | 73 ++++ internal/app/emailverify/breaker_test.go | 37 ++ internal/app/emailverify/service.go | 411 ++++++++++++++++-- internal/app/integration/catalog.go | 14 + internal/app/integration/service.go | 96 ++++ internal/config/constants.go | 26 ++ .../000110_verification_provenance.down.sql | 14 + .../000110_verification_provenance.up.sql | 37 ++ internal/jobs/email_verification.go | 43 +- internal/models/admin.go | 2 +- internal/models/campaign.go | 11 + internal/models/contact.go | 107 ++++- internal/models/contact_import.go | 10 + internal/models/integration.go | 5 + internal/pkg/emailverify/emailverify.go | 135 +++++- internal/pkg/emailverify/fingerprint.go | 133 ++++++ internal/pkg/emailverify/millionverifier.go | 179 ++++++++ .../pkg/emailverify/millionverifier_test.go | 60 +++ internal/pkg/emailverify/vocab.go | 233 ++++++++++ internal/pkg/emailverify/vocab_test.go | 84 ++++ internal/repository/pg_admin.go | 2 +- internal/repository/pg_campaign.go | 34 +- internal/repository/pg_contact.go | 274 ++++++++++-- internal/tasks/campaign_reconciler.go | 9 +- internal/tasks/campaign_task.go | 42 +- web/src/app/app/campaigns/[id]/layout.tsx | 8 +- web/src/app/app/campaigns/page.tsx | 7 +- .../_components/ConnectDrawer.tsx | 9 + .../_components/ProviderGlyph.tsx | 4 + web/src/app/app/settings/sending/page.tsx | 2 + .../app/campaigns/LaunchCampaignDialog.tsx | 28 +- .../app/campaigns/UndeliverableBanner.tsx | 120 +++++ .../app/campaigns/useCampaignActions.ts | 2 +- .../app/contacts/ContactFilters.tsx | 23 +- .../components/app/contacts/ContactsTable.tsx | 70 ++- .../components/app/contacts/ImportWizard.tsx | 18 + .../app/contacts/VerificationBadge.tsx | 76 ++++ .../app/contacts/VerificationSettings.tsx | 130 ++++++ .../app/contacts/contact-edit/ActivityTab.tsx | 2 + .../components/app/contacts/importShared.ts | 16 + .../api/client/app/campaigns/startCampaign.ts | 8 +- .../api/client/app/contacts/importContacts.ts | 5 + .../api/client/app/contacts/verification.ts | 49 +++ .../hooks/app/campaigns/useStartCampaign.ts | 5 +- .../app/contacts/useContactVerification.ts | 26 ++ .../lib/api/models/app/contacts/AddContact.ts | 4 + .../lib/api/models/app/contacts/Contact.ts | 17 + .../api/models/app/contacts/SearchContacts.ts | 3 +- .../app/contacts/SearchContactsResult.ts | 11 + .../models/app/integrations/Integration.ts | 11 +- 72 files changed, 2905 insertions(+), 160 deletions(-) create mode 100644 internal/app/emailverify/breaker.go create mode 100644 internal/app/emailverify/breaker_test.go create mode 100644 internal/infrastructure/db/migrations/000110_verification_provenance.down.sql create mode 100644 internal/infrastructure/db/migrations/000110_verification_provenance.up.sql create mode 100644 internal/pkg/emailverify/fingerprint.go create mode 100644 internal/pkg/emailverify/millionverifier.go create mode 100644 internal/pkg/emailverify/millionverifier_test.go create mode 100644 internal/pkg/emailverify/vocab.go create mode 100644 internal/pkg/emailverify/vocab_test.go create mode 100644 web/src/components/app/campaigns/UndeliverableBanner.tsx create mode 100644 web/src/components/app/contacts/VerificationBadge.tsx create mode 100644 web/src/components/app/contacts/VerificationSettings.tsx create mode 100644 web/src/lib/api/client/app/contacts/verification.ts create mode 100644 web/src/lib/api/hooks/app/contacts/useContactVerification.ts diff --git a/admin/src/app/dashboard/CampaignsPage.tsx b/admin/src/app/dashboard/CampaignsPage.tsx index ebd32bde..cd2bacd0 100644 --- a/admin/src/app/dashboard/CampaignsPage.tsx +++ b/admin/src/app/dashboard/CampaignsPage.tsx @@ -56,6 +56,7 @@ const STATUS_TONE: Record = { paused_trial_expired: "border-orange-300 text-orange-700 bg-orange-50", paused_no_accounts: "border-orange-300 text-orange-700 bg-orange-50", paused_guardrail: "border-rose-300 text-rose-700 bg-rose-50", + paused_undeliverable: "border-amber-300 text-amber-700 bg-amber-50", }; const STATUS_OPTIONS = [ @@ -67,6 +68,7 @@ const STATUS_OPTIONS = [ { value: "paused_trial_expired", label: "Paused — trial expired" }, { value: "paused_no_accounts", label: "Paused — no accounts" }, { value: "paused_guardrail", label: "Paused — guardrail" }, + { value: "paused_undeliverable", label: "Paused — undeliverable leads" }, ]; const pct = (n: number, d: number) => (d ? ((n / d) * 100).toFixed(1) : "—"); diff --git a/admin/src/lib/api/models/admin.ts b/admin/src/lib/api/models/admin.ts index a820aa38..43d52943 100644 --- a/admin/src/lib/api/models/admin.ts +++ b/admin/src/lib/api/models/admin.ts @@ -522,7 +522,7 @@ export interface AdminCampaignSearch { q?: string; user_id?: string; org_id?: string; - status?: string; // "" | draft | active | paused | completed | paused_trial_expired | paused_no_accounts | paused_guardrail + status?: string; // "" | draft | active | paused | completed | paused_trial_expired | paused_no_accounts | paused_guardrail | paused_undeliverable // Boolean flags open_tracking?: boolean; link_tracking?: boolean; diff --git a/cmd/backend/main.go b/cmd/backend/main.go index d8276c1a..41652300 100644 --- a/cmd/backend/main.go +++ b/cmd/backend/main.go @@ -1269,6 +1269,10 @@ func main() { if aware, ok := campaignService.(campaign.AudienceAware); ok { aware.WireAudience(campaignAudienceRepository) } + // A start with nothing left to send says whether verification is why. + if aware, ok := campaignService.(campaign.ProgressAware); ok { + aware.WireProgress(campaignProgressRepository) + } // Delete drops attachment objects and duplicate copies them, so the // campaign service needs the store the attachment handler writes to. if aware, ok := campaignService.(campaign.AttachmentAware); ok { @@ -1646,9 +1650,29 @@ func main() { HeloHost: emailVerifyHeloHost(), // e.g. verify.warmbly.com MailFrom: os.Getenv("EMAIL_VERIFY_MAIL_FROM"), // e.g. verify@warmbly.com }) - emailVerifyService = emailverifyapp.NewService(contactRepostory, emailVerifier) - emailVerificationJob := jobs.NewEmailVerificationJob(emailVerifyService, 100) - emailVerificationScheduler := jobs.NewEmailVerificationScheduler(emailVerificationJob, 15*time.Minute) + // A workspace that connected MillionVerifier is checked through its own + // credits; EMAIL_VERIFY_MILLIONVERIFIER_API_KEY is the operator's key + // for every workspace without one. Verdict changes resume campaigns + // parked because verification refused their leads. + emailVerifyService = emailverifyapp.NewService(contactRepostory, emailverifyapp.Options{ + Builtin: emailVerifier, + BuiltinReady: emailVerifier.ProbeReady(), + Providers: integrationServiceForHandler, + PlatformMillionVerifierKey: os.Getenv("EMAIL_VERIFY_MILLIONVERIFIER_API_KEY"), + }) + emailVerifyService.SetVerdictHook(func(ctx context.Context, orgID uuid.UUID) { + if campaignService != nil { + campaignService.ResumeVerificationPaused(ctx, orgID) + } + // Verdicts land outside any request, so the audit spine is fed by + // hand: every member's contact and campaign views refresh live. + if streamingPublisher != nil { + streamingPublisher.PublishAuditCreated(ctx, orgID, uuid.Nil, "verify", string(models.AuditEntityContact), nil) + streamingPublisher.PublishAuditCreated(ctx, orgID, uuid.Nil, "verify", string(models.AuditEntityCampaign), nil) + } + }) + emailVerificationJob := jobs.NewEmailVerificationJob(emailVerifyService, config.VerificationBatchSize) + emailVerificationScheduler := jobs.NewEmailVerificationScheduler(emailVerificationJob, time.Duration(config.VerificationIntervalSeconds)*time.Second) go emailVerificationScheduler.Start(ctx) // Seed inbox-placement testing: send a tokenized copy of a template diff --git a/docs/content/docs/api/endpoints.mdx b/docs/content/docs/api/endpoints.mdx index 7acc0d80..cc8cbfe2 100644 --- a/docs/content/docs/api/endpoints.mdx +++ b/docs/content/docs/api/endpoints.mdx @@ -76,6 +76,8 @@ All paths below are relative to the versioned base URL `https://api.warmbly.com/ | POST | `/contacts` | `WRITE_CONTACTS` | | DELETE | `/contacts` | `BULK_CONTACTS` | | PATCH | `/contacts` | `BULK_CONTACTS` | +| GET | `/contacts/verification` | `READ_CONTACTS` | +| POST | `/contacts/verification` | `BULK_CONTACTS` | | GET | `/contacts/:id` | `READ_CONTACTS` | | PATCH | `/contacts/:id` | `WRITE_CONTACTS` | | DELETE | `/contacts/:id` | `WRITE_CONTACTS` | diff --git a/docs/content/docs/api/error-codes.mdx b/docs/content/docs/api/error-codes.mdx index b2d13a15..24b7539e 100644 --- a/docs/content/docs/api/error-codes.mdx +++ b/docs/content/docs/api/error-codes.mdx @@ -78,6 +78,12 @@ Returned when the request cannot be processed due to invalid syntax. | `invalid_lead_status` | `POST /contacts/search` or `POST /contacts/export` was given a `lead_status` that is not one of the documented values | | `invalid_engagement` | `POST /contacts/search` or `POST /contacts/export` was given an `engagement` that is not one of the documented values | | `lead_filter_requires_campaign` | `lead_status` or `engagement` was set without exactly one `campaign_ids` entry; both filters describe a contact inside one campaign | +| `unknown_verification_status` | A contact's `verification_status` is not a value any known verification service writes | +| `unknown_verification_provider` | A contact's `verification_provider` names a vocabulary the platform cannot read | +| `invalid_action` | `POST /contacts/verification` was given an `action` other than `verify`, `mark_deliverable` or `mark_undeliverable` | +| `no_contacts` | `POST /contacts/verification` selected no contacts: neither `contacts` nor a `campaign_id` with refused leads | +| `list_bounce_risk` | `POST /campaigns/:id/start` refused the launch on the list's projected bounce rate. Clean or verify the list, or repeat the request with `acknowledge_list_risk: true` | +| `leads_undeliverable` | `POST /campaigns/:id/start` found nothing to send because address verification refused every remaining lead; the campaign is parked at `paused_undeliverable` until they are re-verified or marked deliverable | | `no_organization` | The request needs a workspace and the caller has none selected. Every entitlement, limit and suppression rule is scoped to a workspace, so a write that would run unscoped is refused rather than run without those checks. API keys always carry their workspace; a dashboard session picks one at sign-in, so this normally means the session predates the workspace being chosen. Select a workspace and retry | ### 401 Unauthorized diff --git a/docs/content/docs/api/reference/campaigns.mdx b/docs/content/docs/api/reference/campaigns.mdx index 10249756..a40dcdc0 100644 --- a/docs/content/docs/api/reference/campaigns.mdx +++ b/docs/content/docs/api/reference/campaigns.mdx @@ -658,6 +658,9 @@ Start (activate) the campaign so it begins sending real mail. Works from `draft` | Parameter | In | Type | Description | | --- | --- | --- | --- | | `id` | path | uuid | Campaign id. | +| `acknowledge_list_risk` | body | boolean | Optional. Launch even though the list's projected bounce rate would be refused (`list_bounce_risk`), for a list verified elsewhere. | + +A start refused because every remaining lead was refused by address verification answers `leads_undeliverable` and parks the campaign at `paused_undeliverable`; re-verify the leads or mark them deliverable with [`POST /contacts/verification`](/api/reference/contacts/#verify-or-override-contacts), which resumes it. ### Response diff --git a/docs/content/docs/api/reference/contacts.mdx b/docs/content/docs/api/reference/contacts.mdx index b9e9f61e..3fe156d2 100644 --- a/docs/content/docs/api/reference/contacts.mdx +++ b/docs/content/docs/api/reference/contacts.mdx @@ -31,6 +31,7 @@ Every field is optional; an empty body matches all contacts in the organization. | `lead_status` | string | No | Filter to one derived lead status: `pending`, `active`, `completed`, `replied`, `bounced`, `failed`, `undeliverable`, or `unsubscribed`. Requires exactly one `campaign_ids` entry, otherwise the request is rejected with `lead_filter_requires_campaign`; an unknown value is rejected with `invalid_lead_status`. | | `engagement` | string | No | Filter by engagement inside that campaign: `opened`, `not_opened`, `clicked`, `not_clicked`, `replied`, `not_replied`, or `bounced`. `opened` means a human open (machine opens never count); the `not_*` values match only leads sent at least one step. Combines with `lead_status` as AND. Requires exactly one `campaign_ids` entry (`lead_filter_requires_campaign`); an unknown value is rejected with `invalid_engagement`. | | `category_ids` | string[] | No | Contact must have ALL of these categories. | +| `verification_status` | string | No | Filter by verification verdict: `valid`, `risky`, `invalid`, or `unknown`. | | `min_campaigns` | integer | No | Minimum number of associated campaigns. | | `max_campaigns` | integer | No | Maximum number of associated campaigns. | | `subscribed` | boolean | No | Filter by subscription status. | @@ -71,7 +72,11 @@ Returns a `data` array of contacts plus a `pagination` envelope. "campaigns": [{ "id": "c1...", "name": "Q3 Outbound" }], "categories": [{ "id": "6f1c...", "title": "VIP", "color": "#0ea5e9" }], "verification_status": "valid", - "verification_reason": "", + "verification_reason": "recipient accepted", + "verification_sub_status": "", + "verification_source": "probe", + "verification_provider": "builtin", + "verification_checked_at": "2026-06-10T11:58:00Z", "is_catch_all": false, "esp_provider": "gmail", "updated_at": "2026-06-10T12:00:00Z", @@ -86,6 +91,8 @@ Returns a `data` array of contacts plus a `pagination` envelope. } ``` +Every contact carries its address verification: `verification_status` (`valid`, `risky`, `invalid`, or `unknown`), `verification_sub_status` (`catch_all`, `disposable`, `role`, `spamtrap`, `mailbox_full`, `no_mx`, `syntax`, `undisclosed`, or empty), `verification_source` (`probe` for the built-in check, `provider` for a connected verification service, `imported` for a verdict that came with the contact, `manual` for one a member set, empty when never checked), `verification_provider` (who produced it), `verification_reason`, and `verification_checked_at`. Campaigns never send to `invalid`, and send to `risky` only when their `risky_emails` setting is on. + When the search filters by exactly one campaign, each contact additionally carries a `campaign_lead` object with its processing state inside that campaign (`status`, `sent`, `opened`, `machine_opened`, `clicked`, `replied`, `bounced`, `current_step`, `last_activity_at`, and `failure_reason` when failed). `opened` counts steps opened by a person; steps fetched automatically by a mail client (Apple Mail Privacy Protection and similar) are in `machine_opened` instead, matching the machine opens the analytics summary reports. The `status` derivation, highest priority first, is `unsubscribed` (not subscribed), then `bounced`, `replied`, `failed` (a step could not be sent after every retry; `failure_reason` carries the sending worker's reason), `completed` (every email step sent, no reply), `active` (some steps sent, more to send), `undeliverable` (pre-send verification refused the address, so the campaign skips the lead and never sends to it), and `pending` (queued, nothing sent). A step counts as sent only once the sending worker has delivered it to the mailbox provider; a send the worker could not complete is retried on the campaign's next pass and never shows as sent. The `lead_status` filter narrows to one of these buckets. When the search filters by exactly one campaign, the first page (no `cursor`) also includes a `lead_counts` object: per-status lead totals for that campaign, independent of the `lead_status` and `engagement` filters so every scope's total is available at once. Alongside the status buckets it carries engagement totals that match the `engagement` filter: `contacted` (leads sent at least one step), `opened` (a human open on any step), `clicked`, and `replied_any` (a reply on any step, whatever the derived status). @@ -145,6 +152,8 @@ A JSON array of contact objects (at least one, up to the per-request maximum; an | `categories` | string[] | No | Category IDs to assign. | | `custom_fields` | object | No | String key/value custom fields. Keys may use letters, numbers, underscores, spaces, and dashes. | | `subscribed` | boolean | No | Marketing-consent flag. Omit it to let a new contact default to subscribed and an existing one keep whatever it already had. | +| `verification_status` | string | No | A verdict you already hold for the address, in Warmbly's vocabulary (`valid`, `risky`, `invalid`, `unknown`) or any known service's (`ok`, `catch-all`, `do_not_mail`, `deliverable`, `ok_for_all`, ...). Stored as an imported verdict that the background check leaves alone. A value no known service writes is rejected with `unknown_verification_status`. | +| `verification_provider` | string | No | The vocabulary `verification_status` is written in: `zerobounce`, `millionverifier`, `neverbounce`, `bouncer`, `kickbox`, `emailable`, `debounce`, `clearout`, `emaillistverify`, or `warmbly`. Omit it to have the value recognised by itself. An unknown name is rejected with `unknown_verification_provider`. | An address you already have is matched (lowercased) and enriched rather than duplicated: fields you send replace what is stored, fields you omit or send empty are left alone, and `custom_fields` is merged key by key. Use `PATCH /contacts/:id` to clear a value. @@ -351,7 +360,7 @@ Send `multipart/form-data` with a `file` field and an `options` field containing | Field | Type | Required | Description | | --- | --- | --- | --- | -| `mapping` | array | Yes | Column mappings: `{ "index", "target", "custom_key" }`. `target` is `ignore`, `email`, `first_name`, `last_name`, `company`, `phone`, `subscribed`, `categories`, or `custom` with the name in `custom_key`. `custom:` is still accepted as the older spelling. Exactly one column must map to `email`. | +| `mapping` | array | Yes | Column mappings: `{ "index", "target", "custom_key", "verification_provider" }`. `target` is `ignore`, `email`, `first_name`, `last_name`, `company`, `phone`, `subscribed`, `categories`, `verification_status`, or `custom` with the name in `custom_key`. `custom:` is still accepted as the older spelling. Exactly one column must map to `email`. A `verification_status` column is read in the vocabulary named by `verification_provider` (see [Create contacts](#create-contacts)), or recognised value by value when it is omitted; a cell nobody recognises leaves that contact unverified rather than failing the row. The preview suggests this target itself when a column's header or values look like another service's results. | | `dedup` | string | Yes | `skip`, `update`, or `create_duplicate` for rows whose email matches an existing contact. | | `has_header` | boolean | Yes | Whether the first row is a header. | | `category_ids` | string[] | No | Categories to assign to imported contacts. | @@ -402,6 +411,48 @@ Imports are capped at 50,000 rows. `errors` carries at most the first 1,000 entr A custom-field name may use letters, numbers, underscores, spaces, and dashes (`Company Mobile`, `first-name`, `plan_tier`). Anything else is a `400` on the whole request, raised before any row is written, along with a mapping that names no `email` column or a `custom` column with no `custom_key`. Per-row `errors` are reserved for problems with the data itself. +## Verification overview + +`GET /contacts/verification` + +Reports which verifier checks this workspace's addresses and the contacts by verdict. **Scope** `READ_CONTACTS` · **Org permission** `view_contacts`. + +### Response + +```json +{ + "provider": "millionverifier", + "connection_id": "9a1b...", + "credits": 48210, + "builtin_ready": true, + "counts": { "valid": 11240, "risky": 380, "invalid": 512, "unknown": 1890, "pending": 120 } +} +``` + +`provider` is `builtin` or `millionverifier`. `credits` is the connected service's remaining balance; `provider_error` is set instead when the service is connected but unusable (a rejected key, no credits), in which case the built-in check is in use. `builtin_ready` says whether the built-in mailbox probe can run on this instance. `pending` is the share of `unknown` nobody has checked yet. + +## Verify or override contacts + +`POST /contacts/verification` + +Queues a fresh check of the listed contacts, or records a manual verdict on them. **Scope** `BULK_CONTACTS` · **Org permission** `manage_contacts`. + +### Request body + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `action` | string | Yes | `verify` queues a re-check (each contact updates as its verdict lands); `mark_deliverable` records `valid`; `mark_undeliverable` records `invalid`. Manual verdicts are never re-checked automatically. Anything else is rejected with `invalid_action`. | +| `contacts` | string[] | No | Contact ids, up to the bulk maximum per request. | +| `campaign_id` | string | No | Instead of, or as well as, `contacts`: every lead of this campaign that verification refused. | + +At least one contact must be selected (`no_contacts`). Marking leads deliverable resumes any campaign of the workspace that was paused for verification. + +### Response + +```json +{ "affected": 512, "action": "verify", "queued": true } +``` + ## Look up a contact by email `GET /contacts/lookup` diff --git a/docs/content/docs/development/configuration.mdx b/docs/content/docs/development/configuration.mdx index 5a2d38a4..6d11b5bc 100644 --- a/docs/content/docs/development/configuration.mdx +++ b/docs/content/docs/development/configuration.mdx @@ -217,11 +217,14 @@ Before a campaign sends to an address, the backend can check it: syntax, then `M |---|---|---|---| | `EMAIL_VERIFY_HELO_HOST` | The hostname the probe announces in `EHLO`/`HELO`. Must be a public, fully-qualified name that belongs to this instance | the host of `APP_URL` | yes | | `EMAIL_VERIFY_MAIL_FROM` | The envelope sender the probe uses in `MAIL FROM` | `verify@` plus the `HELO` host | yes | +| `EMAIL_VERIFY_MILLIONVERIFIER_API_KEY` | An instance-wide [MillionVerifier](https://www.millionverifier.com/) key. Every workspace that has not connected its own key is checked through this one, spending its credits, instead of the built-in probe | unset | yes | Mail servers refuse a greeting that is not a real hostname (`localhost`, a bare name, or anything under a reserved suffix such as `.local`, `.internal` or `.lan`). Postfix in particular applies that rejection at `RCPT` time rather than at `HELO`, so it arrives looking exactly like `504 5.5.2 : Helo command rejected: need fully-qualified hostname` on the recipient's address. Warmbly reads a reply like that as a rejected probe, not as a dead mailbox, so it never marks the contact invalid. If neither `EMAIL_VERIFY_HELO_HOST` nor a usable `APP_URL` host is set, the probe is skipped entirely and every address stays `unknown`, which still sends. +Verdicts are re-checked after 90 days (30 for an inconclusive one), in passes of 200 contacts a minute that repeat while a backlog remains. A workspace that connected MillionVerifier under Integrations is checked through its own credits whether or not the instance-wide key is set. + `EMAIL_VERIFY_HELO_HOST` is not `SMTP_EHLO_NAME`. `SMTP_EHLO_NAME` is the greeting your platform mail relay sees in [platform mail](#platform-mail); this one is the greeting recipients' servers see from the verifier. ## Encryption diff --git a/docs/content/docs/guides/campaigns.mdx b/docs/content/docs/guides/campaigns.mdx index d86203fe..3bf04f83 100644 --- a/docs/content/docs/guides/campaigns.mdx +++ b/docs/content/docs/guides/campaigns.mdx @@ -118,7 +118,7 @@ Optional **campaign dates** bound when it may send. Leave both blank to run open The play and pause buttons work from the list row or the detail view. Starting moves the campaign to **active** and begins scheduling inside your windows and limits; pausing stops new scheduling immediately and resumes from where it left off. -A campaign can pause itself: **paused, no accounts** when it loses every sender, **paused, trial expired** when a trial ends, **auto-paused** when a guardrail trips, and plain **paused** if it ever loses its workspace, because unsubscribes, bounces and complaints are checked per workspace and cannot be honoured without one. It moves to **finished** once every contact completes the sequence or its end date passes. Configured to do so, it also stops following up with a contact the moment they reply. +A campaign can pause itself: **paused, no accounts** when it loses every sender, **paused, trial expired** when a trial ends, **auto-paused** when a guardrail trips, **needs verification** when address verification has refused every remaining lead (the campaign offers to re-verify them or send anyway; see [address verification](/guides/deliverability/#address-verification)), and plain **paused** if it ever loses its workspace, because unsubscribes, bounces and complaints are checked per workspace and cannot be honoured without one. It moves to **finished** once every contact completes the sequence or its end date passes. Configured to do so, it also stops following up with a contact the moment they reply. A finished campaign can be started again: after extending or clearing its end date, or adding new leads, pressing play resumes it. If there is genuinely nothing left to send it finishes again immediately with a message saying so. @@ -202,6 +202,8 @@ What it counts: - **Deliverable recipients only.** Suppressed and unsubscribed leads are skipped at send time, so counting them would make a bad list look fine. - **Lists of at least 50.** Below that a share means nothing, and no launch is refused on it. +A refused launch shows why and offers **Launch anyway**, for a list you verified with another service. Through the API the same override is `acknowledge_list_risk` on the start request. + Above `4%` projected bounce the launch is refused with the number and what to do about it. Between `2%` and `4%` it launches with a warning. The same projection appears in the launch dialog before you click, so a refusal is never a surprise. ## Safety posture diff --git a/docs/content/docs/guides/contacts-crm.mdx b/docs/content/docs/guides/contacts-crm.mdx index 5e603687..bd82a564 100644 --- a/docs/content/docs/guides/contacts-crm.mdx +++ b/docs/content/docs/guides/contacts-crm.mdx @@ -43,7 +43,7 @@ The result step also reports what the addresses themselves look like: how many w - **The import still happens.** These are your records, so nothing is refused here. A list bad enough to matter is stopped when you try to launch a campaign with it, which is where the damage would actually occur. See [the launch check](/guides/campaigns/). - **Lists under 20 rows are not judged**, since a share of a handful of rows means nothing. -This is not address verification. It reads the addresses; verification asks the receiving server whether they exist, and runs separately in the background. +This is not address verification. It reads the addresses; verification asks the receiving server whether they exist, and runs separately in the background, within a minute of the import. A file that already carries verification results from another service (a `status` or `ZeroBounce Status` column, say) is recognised during mapping and offered as **Verification status**; those verdicts are kept and the background check skips them. See [address verification](/guides/deliverability/#address-verification). ## Custom fields diff --git a/docs/content/docs/guides/deliverability.mdx b/docs/content/docs/guides/deliverability.mdx index 7cd977a0..f3b56ecc 100644 --- a/docs/content/docs/guides/deliverability.mdx +++ b/docs/content/docs/guides/deliverability.mdx @@ -98,7 +98,7 @@ If a campaign has already been paused because every one of its mailboxes was gat **One-click unsubscribe** (RFC 8058) puts a native Unsubscribe control in the recipient's mail client. It drives complaint rate down, because someone who can opt out cleanly rarely hits "mark as spam" instead, and Google's bulk-sender guidance effectively requires it at volume. Unsubscribes are suppressed automatically. -**Pre-send verification** checks an address before sending and skips undeliverable ones, keeping bad addresses from becoming the hard bounces that drive the `5%` and `10%` thresholds above. A skipped lead shows as **Undeliverable** in the campaign's Leads view rather than sitting at Queued, and the campaign's activity log says how many it skipped when it finishes, so a campaign is never quietly held up by addresses it will not send to. Verification only records an address as invalid when the recipient's server rejects the address itself. A server that rejects the check for its own reasons (a policy block, a rate limit, or a greeting it does not accept) leaves the address unverified, and unverified addresses are still sent to. Self-hosted instances should set `EMAIL_VERIFY_HELO_HOST`, described in the [configuration reference](/development/configuration/#pre-send-verification). +**Pre-send verification** checks every address before a campaign sends to it and skips the undeliverable ones, keeping bad addresses from becoming the hard bounces that drive the `5%` and `10%` thresholds above. There is nothing to run: new contacts are checked in the background within a minute of being added, each contact carries its verdict (a small mark next to the email), and Settings > Sending shows the whole workspace at a glance. See [address verification](#address-verification) below for how verdicts are produced and what to do when one is wrong. **Suppression** is the safety net: a suppressed recipient is never emailed again by any campaign. Warmbly suppresses automatically on a bounce, a spam complaint, or an unsubscribe, because once someone has bounced or complained the safest response is to stop, not to keep collecting negative signals. @@ -114,6 +114,24 @@ Complaint reports are read from IMAP and Gmail mailboxes. Microsoft Graph return **Domain authentication refusals** are separated from ordinary failures. When a receiving server refuses a message because your sending domain failed its checks (Microsoft's `5.7.515`, Gmail's `5.7.26`), that is not a bad recipient and not an outage: every send from that domain fails the same way until its DNS is fixed. Warmbly says so in the campaign's activity feed rather than blaming the address, does not suppress the recipient, and brings that domain's authentication check forward so the [domain authentication](/guides/deliverability/#domain-authentication) state reflects it. +## Address verification + +Every address gets one of four verdicts. **Deliverable** and **unverified** are always sent to. **Undeliverable** is never sent to. **Risky** (a catch-all domain, a shared inbox such as `info@`) is sent to only when the campaign's **send to risky addresses** setting allows it. + +Verdicts come from one of three places, and the source is shown on the contact: + +- **The built-in check**, included on every plan: syntax, the domain's mail server, known disposable domains, and a mailbox probe against the recipient's server. It only records an address as undeliverable when that server rejects the address itself; a server that rejects the check for its own reasons (a policy block, a rate limit, a greeting it does not accept) leaves the address unverified. Microsoft 365 and Yahoo answer every probe the same way, so their addresses stay unverified rather than being called deliverable. The check also watches itself: when it starts rejecting an unusual share of addresses, its rejections are filed as unverified for an hour instead of being trusted, because a real list is never mostly dead. Self-hosted instances should set `EMAIL_VERIFY_HELO_HOST`, described in the [configuration reference](/development/configuration/#pre-send-verification). +- **MillionVerifier**, pay as you go: connect it once under Integrations with your own API key, and every check from then on uses your MillionVerifier credits (one per address) instead of the built-in probe. Its verdicts cover the catch-all and Microsoft cases the probe cannot. When the balance runs out or the key stops working, the connection is marked and the built-in check takes over until it is fixed. Nothing else changes. +- **Results you already have.** A list verified with another service (ZeroBounce, MillionVerifier, NeverBounce, Bouncer, Kickbox, Emailable, DeBounce, Clearout, EmailListVerify, or a Warmbly export) can bring its status column along. The import recognises the column from its header or its values, maps it to **Verification status**, and reads each service's own words (`catch-all`, `do_not_mail`, `ok_for_all`, and so on). Those verdicts are kept as they are; the built-in check leaves them alone. The same field is accepted when [creating contacts through the API](/api/reference/contacts/#create-contacts). + +Verdicts age: an address is checked again after 90 days (30 for an inconclusive verdict), because mailboxes get created and closed. A verdict a teammate set by hand never expires. + +### When verification is wrong + +Select the contacts and choose **Verify** in the selection bar: **Re-verify** queues a fresh check, and each row's mark updates as its verdict lands; **Mark deliverable** overrides the verdict for a list you know is good. Both are also available on a campaign's Leads tab. + +A campaign never quietly finishes because of verification. When every remaining lead has been refused, the campaign shows **needs verification** and pauses, with two buttons on it: **Re-verify leads** checks them again and resumes sending automatically as soon as any pass, and **Send anyway** marks them deliverable and resumes at once. The [launch check](/guides/campaigns/#adding-leads) likewise offers **Launch anyway** when a list is refused for its projected bounce rate, for a list verified elsewhere. + ## Workspace sending posture Individual protections each watch one thing: a rate limit watches one user, warmup health watches one mailbox, verification watches one address. A workspace that is slightly wrong on several of those at once sits under every individual threshold and is never noticed. The workspace posture is the view that sees all of them together. diff --git a/docs/content/docs/guides/integrations.mdx b/docs/content/docs/guides/integrations.mdx index f737c55e..b7835eed 100644 --- a/docs/content/docs/guides/integrations.mdx +++ b/docs/content/docs/guides/integrations.mdx @@ -19,6 +19,7 @@ The Integrations page is a searchable directory with your existing connections a | Slack | Notifications | one-click OAuth | Ping a channel on reply, bounce, or deliverability dips | | Discord | Notifications | webhook URL | Ping a server channel on reply, bounce, or warmup health | | Calendly, Cal.com | Meetings | minted inbound URL | Track booked, rescheduled, and canceled calls | +| MillionVerifier | Verification | API key | Check every contact's address through your pay-as-you-go credits instead of the built-in probe | Three connect styles appear on the catalog cards: `one-click` OAuth, `api_key` for providers without an OAuth app, and `webhook` for a minted inbound URL or a pasted channel URL. @@ -30,6 +31,10 @@ You must start the authorize flow; pasting credentials for an OAuth provider is Zapier, Make, and n8n connect in one click with nothing stored on the integration. Warmbly sends events to the webhook URL you configure there; when they call back into Warmbly they use a scoped API key you create. + +The key is tested against your MillionVerifier account when you connect, so a mistyped key is refused rather than silently leaving contacts on the built-in check. From then on every new contact, and every re-verify, spends one credit there. When the balance runs out or the key is revoked, the connection is marked and the built-in check covers until it is fixed. See [address verification](/guides/deliverability/#address-verification). + + **Credentials** (OAuth tokens, pasted keys, webhook URLs) are sealed with envelope encryption before touching the database. Only non-secret display details like an account name or Salesforce instance host are stored in the clear, so the dashboard can label the connection. ## CRM field mappings diff --git a/internal/api/handler/campaign.go b/internal/api/handler/campaign.go index 029eec49..b5a2974f 100644 --- a/internal/api/handler/campaign.go +++ b/internal/api/handler/campaign.go @@ -272,7 +272,14 @@ func (h *Handler) StartCampaign(c *gin.Context) { id := c.Param("id") - if xerr := h.CampaignService.StartCampaign(c.Request.Context(), *orgID, id); xerr != nil { + // Optional body: {"acknowledge_list_risk": true} launches past the + // bounce-risk gate once the member has read the projection. + var opts models.StartCampaignOptions + if c.Request.ContentLength != 0 { + _ = c.ShouldBindJSON(&opts) + } + + if xerr := h.CampaignService.StartCampaign(c.Request.Context(), *orgID, id, opts); xerr != nil { errx.JSON(c, xerr) return } diff --git a/internal/api/handler/contact.go b/internal/api/handler/contact.go index 0533f070..e168f39d 100644 --- a/internal/api/handler/contact.go +++ b/internal/api/handler/contact.go @@ -51,6 +51,10 @@ func (h *Handler) AddContacts(c *gin.Context) { // Audit log - bulk import h.auditOrg(c, models.AuditActionImport, models.AuditEntityContact, nil, nil, map[string]string{"count": fmt.Sprintf("%d", len(data))}) + // New addresses are checked right away rather than on the next tick. + if h.EmailVerifyService != nil { + h.EmailVerifyService.Kick() + } c.JSON(http.StatusOK, resp) } diff --git a/internal/api/handler/contact_io.go b/internal/api/handler/contact_io.go index ff614f64..0d783715 100644 --- a/internal/api/handler/contact_io.go +++ b/internal/api/handler/contact_io.go @@ -163,5 +163,9 @@ func (h *Handler) ImportCommitContacts(c *gin.Context) { "failed": fmt.Sprintf("%d", result.Failed), }) + // New addresses are checked right away rather than on the next tick. + if h.EmailVerifyService != nil { + h.EmailVerifyService.Kick() + } c.JSON(http.StatusOK, result) } diff --git a/internal/api/handler/email_verification.go b/internal/api/handler/email_verification.go index 5949005e..b4ef5880 100644 --- a/internal/api/handler/email_verification.go +++ b/internal/api/handler/email_verification.go @@ -8,6 +8,7 @@ import ( "github.com/warmbly/warmbly/internal/api/middleware" "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" ) // verifyEmailRequest is the optional JSON body for VerifyEmail. The address may @@ -16,18 +17,12 @@ type verifyEmailRequest struct { Email string `json:"email"` } -// VerifyEmail verifies a single email address on demand (syntax -> MX -> SMTP -// RCPT probe -> catch-all detection) and returns the emailverify.Result. This -// is pre-send verification: it lets the user/admin confirm an address is -// deliverable *before* a worker ever sends to it, instead of learning from a -// hard bounce after the fact. +// VerifyEmail verifies a single email address on demand through whichever +// verifier the workspace uses (its connected provider, else the built-in +// check) and returns the emailverify.Result. Nothing is stored. // -// Control-plane only: the SMTP RCPT probe behind this runs from the backend (a -// non-sending IP). Probing must never run from worker (sending) IPs — see -// internal/pkg/emailverify. -// -// Route registration is intentionally NOT done here; the parent workstream -// wires it in internal/api/routes.go behind the appropriate permission gates. +// Control-plane only: the SMTP RCPT probe behind the built-in check runs from +// the backend (a non-sending IP), never a worker. See internal/pkg/emailverify. func (h *Handler) VerifyEmail(c *gin.Context) { if _, err := middleware.GetUserUUID(c); err != nil { errx.JSON(c, errx.ErrUnauthorized) @@ -37,6 +32,10 @@ func (h *Handler) VerifyEmail(c *gin.Context) { errx.JSON(c, errx.InternalError()) return } + orgID, ok := requireOrgID(c) + if !ok { + return + } var req verifyEmailRequest // Body is optional; ignore a bind error and fall back to the query param. @@ -50,6 +49,61 @@ func (h *Handler) VerifyEmail(c *gin.Context) { return } - res := h.EmailVerifyService.VerifyAddress(c.Request.Context(), email) + res := h.EmailVerifyService.VerifyAddress(c.Request.Context(), orgID, email) c.JSON(http.StatusOK, res) } + +// GetContactVerification reports which verifier the workspace uses, its +// remaining credits, and the contacts by verdict. +// GET /contacts/verification +func (h *Handler) GetContactVerification(c *gin.Context) { + if h.EmailVerifyService == nil { + errx.JSON(c, errx.InternalError()) + return + } + orgID, ok := requireOrgID(c) + if !ok { + return + } + out, xerr := h.EmailVerifyService.Overview(c.Request.Context(), orgID) + if xerr != nil { + errx.JSON(c, xerr) + return + } + c.JSON(http.StatusOK, out) +} + +// RequestContactVerification queues a re-check of the listed contacts, or +// records a manual verdict on them. +// POST /contacts/verification +func (h *Handler) RequestContactVerification(c *gin.Context) { + if h.EmailVerifyService == nil { + errx.JSON(c, errx.InternalError()) + return + } + orgID, ok := requireOrgID(c) + if !ok { + return + } + var req models.ContactVerificationRequest + if err := c.ShouldBindJSON(&req); err != nil { + errx.JSON(c, errx.ErrInvalid) + return + } + if len(req.Contacts) > maxBulkOperationSize { + errx.JSON(c, errx.NewWithIdentifier(errx.BadRequest, "too_many_contacts", + "too many contacts, maximum is "+itoa(maxBulkOperationSize)+" per request")) + return + } + resp, xerr := h.EmailVerifyService.Request(c.Request.Context(), orgID, req) + if xerr != nil { + errx.JSON(c, xerr) + return + } + // The audit spine refreshes every teammate's contact lists; verdicts + // then land live as the background pass records them. + h.auditOrg(c, models.AuditActionUpdate, models.AuditEntityContact, nil, nil, map[string]string{ + "verification": req.Action, "count": itoa(resp.Affected), + }) + c.JSON(http.StatusOK, resp) +} diff --git a/internal/api/routes.go b/internal/api/routes.go index be842f0d..66bba3ca 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -550,6 +550,10 @@ func Run( contacts.POST("", m.RequireAccess(models.PermManageContacts, models.APIPermWriteContacts), h.AddContacts) contacts.DELETE("", m.RequireAccess(models.PermManageContacts, models.APIPermBulkContacts), h.DeleteContactBulk) contacts.PATCH("", m.RequireAccess(models.PermManageContacts, models.APIPermBulkContacts), h.UpdateContactBulk) + // Address verification: who checks this workspace's contacts, + // and the member actions (re-verify, mark deliverable). + contacts.GET("/verification", m.RequireAccess(models.PermViewContacts, models.APIPermReadContacts), h.GetContactVerification) + contacts.POST("/verification", m.RequireAccess(models.PermManageContacts, models.APIPermBulkContacts), h.RequestContactVerification) // Import + export power-tools. Read-only export gates on // ReadContacts; the import endpoints write and so use the // stricter Write/Bulk scopes that the rest of the contact diff --git a/internal/app/aitools/tools_campaigns.go b/internal/app/aitools/tools_campaigns.go index 62ba7b2a..e27f0f41 100644 --- a/internal/app/aitools/tools_campaigns.go +++ b/internal/app/aitools/tools_campaigns.go @@ -536,7 +536,7 @@ func (d Deps) setCampaignStatus(ctx context.Context, inv Invocation, args json.R } switch in.Action { case "start": - if xerr := d.Campaigns.StartCampaign(ctx, inv.OrgID, in.CampaignID); xerr != nil { + if xerr := d.Campaigns.StartCampaign(ctx, inv.OrgID, in.CampaignID, models.StartCampaignOptions{}); xerr != nil { return "", fromErrx(xerr) } d.logAudit(ctx, inv, models.AuditActionStart, models.AuditEntityCampaign, &cid, nil) diff --git a/internal/app/campaign/handlers.go b/internal/app/campaign/handlers.go index 5ae7da52..e738aff4 100644 --- a/internal/app/campaign/handlers.go +++ b/internal/app/campaign/handlers.go @@ -364,7 +364,7 @@ func truncateUTF8(s string, n int) string { return s[:n] } -func (s *campaignService) StartCampaign(ctx context.Context, orgID uuid.UUID, campaignID string) *errx.Error { +func (s *campaignService) StartCampaign(ctx context.Context, orgID uuid.UUID, campaignID string, opts models.StartCampaignOptions) *errx.Error { cID, parseErr := uuid.Parse(campaignID) if parseErr != nil { return errx.ErrUuid @@ -392,14 +392,14 @@ func (s *campaignService) StartCampaign(ctx context.Context, orgID uuid.UUID, ca // one just re-completes in enqueueCampaignWakeup with a clear message. startable := map[string]bool{ "draft": true, "paused": true, "paused_no_accounts": true, - "paused_guardrail": true, "completed": true, + "paused_guardrail": true, "paused_undeliverable": true, "completed": true, } if !startable[campaign.Status] { return errx.New(errx.BadRequest, "campaign must be in draft, paused, or completed status to start") } - // Check cooldown - if campaign.LastStatusChangeAt != nil { + // Check cooldown. An automatic resume is not a member flapping a button. + if campaign.LastStatusChangeAt != nil && !opts.Automatic { elapsed := time.Since(*campaign.LastStatusChangeAt) if elapsed.Seconds() < campaignCooldownSeconds { return errx.New(errx.BadRequest, "please wait before changing campaign status") @@ -451,7 +451,10 @@ func (s *campaignService) StartCampaign(ctx context.Context, orgID uuid.UUID, ca // Refuse a launch whose list is known to be largely undeliverable. Only // KNOWN-invalid addresses count: a list nobody has verified is not evidence // of a bad list, and blocking on that would refuse nearly every launch. - if s.audienceRepo != nil { + // The owner can acknowledge the risk and launch anyway (the list may have + // been verified elsewhere), and a campaign parked by verification has + // already been told, so its resume is not gated again. + if s.audienceRepo != nil && !opts.AcknowledgeListRisk && campaign.Status != "paused_undeliverable" { audience, aerr := s.audienceRepo.GetCampaignAudience(ctx, orgID, cID) switch { case aerr != nil: @@ -462,7 +465,7 @@ func (s *campaignService) StartCampaign(ctx context.Context, orgID uuid.UUID, ca Msg("could not measure the campaign's list; launching without the check") default: if verdict := listgate.Project(audience); verdict.Block { - return errx.New(errx.BadRequest, verdict.Summary+" "+verdict.Remediation) + return errx.NewWithIdentifier(errx.BadRequest, "list_bounce_risk", verdict.Summary+" "+verdict.Remediation) } } } @@ -602,6 +605,13 @@ func (s *campaignService) enqueueCampaignWakeup(ctx context.Context, campaignID _ = s.campaignRepository.UpdateStatusWithLock(ctx, campaignID, "paused_no_accounts") return errx.New(errx.BadRequest, "no active email accounts found for campaign's email tags") case errors.Is(err, scheduler.ErrCampaignCompleted): + if s.campaignProgressRepo != nil { + if n, cerr := s.campaignProgressRepo.CountUndeliverableLeads(ctx, campaignID); cerr == nil && n > 0 { + _ = s.campaignRepository.UpdateStatusWithLock(ctx, campaignID, "paused_undeliverable") + return errx.NewWithIdentifier(errx.BadRequest, "leads_undeliverable", + fmt.Sprintf("%d remaining lead(s) were refused by address verification; re-verify them or mark them deliverable to continue", n)) + } + } _ = s.campaignRepository.UpdateStatusWithLock(ctx, campaignID, "completed") return errx.New(errx.BadRequest, "campaign has no remaining contacts to send") case errors.Is(err, scheduler.ErrCampaignEnded): @@ -650,6 +660,21 @@ func (s *campaignService) enqueueCampaignWakeup(ctx context.Context, campaignID return nil } +// ResumeVerificationPaused restarts every campaign of the org parked because +// verification refused its leads. Best effort: a campaign that still has +// nothing to send parks itself again inside StartCampaign. +func (s *campaignService) ResumeVerificationPaused(ctx context.Context, orgID uuid.UUID) { + ids, err := s.campaignRepository.ListIDsByStatus(ctx, orgID, "paused_undeliverable") + if err != nil || len(ids) == 0 { + return + } + for _, id := range ids { + if xerr := s.StartCampaign(ctx, orgID, id.String(), models.StartCampaignOptions{Automatic: true}); xerr != nil { + log.Info().Str("campaign_id", id.String()).Str("reason", xerr.Message).Msg("campaign stays parked after verification") + } + } +} + func (s *campaignService) StopCampaign(ctx context.Context, orgID uuid.UUID, campaignID string) *errx.Error { cID, parseErr := uuid.Parse(campaignID) if parseErr != nil { diff --git a/internal/app/campaign/service.go b/internal/app/campaign/service.go index 86fdb0e2..1033844b 100644 --- a/internal/app/campaign/service.go +++ b/internal/app/campaign/service.go @@ -33,7 +33,11 @@ type CampaignService interface { Duplicate(ctx context.Context, orgID, userID uuid.UUID, campaignID, name string) (*models.Campaign, *errx.Error) // Start/Stop - StartCampaign(ctx context.Context, orgID uuid.UUID, campaignID string) *errx.Error + StartCampaign(ctx context.Context, orgID uuid.UUID, campaignID string, opts models.StartCampaignOptions) *errx.Error + // ResumeVerificationPaused restarts the org's campaigns parked at + // paused_undeliverable. Called after verification verdicts change; a + // campaign that still has nothing to send parks itself again. + ResumeVerificationPaused(ctx context.Context, orgID uuid.UUID) StopCampaign(ctx context.Context, orgID uuid.UUID, campaignID string) *errx.Error // Logs @@ -71,6 +75,9 @@ type campaignService struct { // audienceRepo measures the campaign's list at launch. Optional/nil-safe: // without it no launch is ever refused on list quality. audienceRepo repository.CampaignAudienceRepository + // campaignProgressRepo counts the leads verification refused, so a start + // with nothing left to send can say why. Optional/nil-safe. + campaignProgressRepo repository.CampaignProgressRepository } // WireAudience attaches the launch-time list gate. @@ -90,6 +97,15 @@ type AttachmentAware interface { WireAttachments(repo repository.AttachmentRepository, store storage.Store) } +// ProgressAware lets main hand the campaign service the progress repository. +type ProgressAware interface { + WireProgress(r repository.CampaignProgressRepository) +} + +func (s *campaignService) WireProgress(r repository.CampaignProgressRepository) { + s.campaignProgressRepo = r +} + func (s *campaignService) WireAttachments(repo repository.AttachmentRepository, store storage.Store) { s.attachmentRepo = repo s.storage = store diff --git a/internal/app/contact/campaign_state.go b/internal/app/contact/campaign_state.go index 54d535bf..456910ce 100644 --- a/internal/app/contact/campaign_state.go +++ b/internal/app/contact/campaign_state.go @@ -134,6 +134,8 @@ func constraintCopy(c scheduler.ContactSendConstraint, st *models.ContactCampaig return "Campaign has finished" case "paused_guardrail": return "Campaign was auto-paused by a guardrail" + case "paused_undeliverable": + return "Campaign is paused: verification refused its remaining leads" } return "Campaign is paused" case scheduler.ConstraintNoMailbox: diff --git a/internal/app/contact/import.go b/internal/app/contact/import.go index 4ddfcce7..72eca850 100644 --- a/internal/app/contact/import.go +++ b/internal/app/contact/import.go @@ -16,6 +16,7 @@ import ( "github.com/warmbly/warmbly/internal/email" "github.com/warmbly/warmbly/internal/errx" "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/pkg/emailverify" "github.com/warmbly/warmbly/internal/pkg/listquality" "github.com/warmbly/warmbly/internal/utils" "github.com/xuri/excelize/v2" @@ -60,7 +61,7 @@ func (s *contactService) ImportPreview(ctx context.Context, r io.Reader, filenam Columns: headers, HasHeader: hasHeader, SampleRows: sample, - SuggestedMapping: suggestMapping(headers), + SuggestedMapping: suggestMapping(headers, sample), }, nil } @@ -105,6 +106,15 @@ func resolveMapping(mapping []models.ContactImportColumnMapping) ([]importColumn models.ContactImportTargetPhone, models.ContactImportTargetSubscribed, models.ContactImportTargetCategories: + case models.ContactImportTargetVerificationStatus: + if p := strings.TrimSpace(m.VerificationProvider); p != "" { + k, ok := emailverify.KnownVocabulary(p) + if !ok { + return nil, errx.New(errx.BadRequest, + "unknown verification provider "+strconv.Quote(p)+" for column "+strconv.Itoa(m.Index+1)) + } + key = k + } case models.ContactImportTargetCustom: default: // An unrecognised target with a custom key is how older clients @@ -703,10 +713,38 @@ func padRow(row []string, n int) []string { // suggestMapping uses fuzzy header matches to pick a target for each // column. Anything we don't recognise becomes ignore — better than // inventing a custom-field key the user didn't ask for. -func suggestMapping(headers []string) []models.ContactImportColumnMapping { +func suggestMapping(headers []string, sample [][]string) []models.ContactImportColumnMapping { out := make([]models.ContactImportColumnMapping, len(headers)) for i, h := range headers { out[i] = guessTarget(i, h) + if out[i].Target != models.ContactImportTargetIgnore { + continue + } + // A verdict column from another verification service: the header + // says so, or every sample value is a word one of them writes. + provider, headerSays := emailverify.IsStatusHeader(h) + values := make([]string, 0, len(sample)) + for _, row := range sample { + if i < len(row) { + values = append(values, row[i]) + } + } + detected, valuesSay := emailverify.DetectVocabulary(values) + if !headerSays && !valuesSay { + continue + } + // A generic header ("status") only counts when the values agree, + // otherwise a CRM's deal-stage column would be read as a verdict. + if headerSays && provider == "" && !valuesSay { + continue + } + if provider == "" { + provider = detected + } + if provider == emailverify.ProviderBuiltin { + provider = "" + } + out[i] = models.ContactImportColumnMapping{Index: i, Target: models.ContactImportTargetVerificationStatus, VerificationProvider: provider} } return out } @@ -786,6 +824,12 @@ func buildAddContact( categories = appendUnique(categories, name) } } + case models.ContactImportTargetVerificationStatus: + // Lenient on purpose: a stray "n/a" must not drop the lead. + if _, ok := emailverify.NormalizeExternal(col.customKey, val); ok { + ac.VerificationStatus = val + ac.VerificationProvider = col.customKey + } case models.ContactImportTargetCustom: ac.CustomFields[col.customKey] = val } diff --git a/internal/app/emailverify/breaker.go b/internal/app/emailverify/breaker.go new file mode 100644 index 00000000..b818ad28 --- /dev/null +++ b/internal/app/emailverify/breaker.go @@ -0,0 +1,73 @@ +package emailverify + +import ( + "sync" + "time" +) + +// breaker is the in-house probe's self-check. Two incidents (#200, #264) had +// the probe reject good addresses wholesale because of its own environment +// (a rejected HELO, a blocklisted IP). A real list is never mostly dead, so +// when the share of "invalid" over the last window crosses the threshold the +// probe's rejections are filed as unknown, which still sends, until it has +// cooled down. +type breaker struct { + mu sync.Mutex + window int + threshold float64 + cooldown time.Duration + ring []bool + pos int + filled int + invalid int + trippedAt time.Time +} + +func newBreaker(window int, thresholdPct float64, cooldown time.Duration) *breaker { + if window < 10 { + window = 10 + } + return &breaker{window: window, threshold: thresholdPct, cooldown: cooldown, ring: make([]bool, window)} +} + +// observe records one probe verdict and reports whether the breaker is open. +func (b *breaker) observe(invalid bool) bool { + b.mu.Lock() + defer b.mu.Unlock() + if !b.trippedAt.IsZero() { + if time.Since(b.trippedAt) < b.cooldown { + return true + } + // Cooled down: start measuring afresh. + b.trippedAt = time.Time{} + b.ring = make([]bool, b.window) + b.pos, b.filled, b.invalid = 0, 0, 0 + } + if b.filled == b.window { + if b.ring[b.pos] { + b.invalid-- + } + } else { + b.filled++ + } + b.ring[b.pos] = invalid + if invalid { + b.invalid++ + } + b.pos = (b.pos + 1) % b.window + if b.filled < b.window/2 { + return false + } + if float64(b.invalid)/float64(b.filled)*100 >= b.threshold { + b.trippedAt = time.Now() + return true + } + return false +} + +// open reports whether the breaker is currently tripped. +func (b *breaker) open() bool { + b.mu.Lock() + defer b.mu.Unlock() + return !b.trippedAt.IsZero() && time.Since(b.trippedAt) < b.cooldown +} diff --git a/internal/app/emailverify/breaker_test.go b/internal/app/emailverify/breaker_test.go new file mode 100644 index 00000000..2e5e17f0 --- /dev/null +++ b/internal/app/emailverify/breaker_test.go @@ -0,0 +1,37 @@ +package emailverify + +import ( + "testing" + "time" +) + +func TestBreakerTripsOnAnInvalidFlood(t *testing.T) { + b := newBreaker(20, 40, time.Hour) + for i := 0; i < 9; i++ { + if b.observe(false) { + t.Fatal("closed breaker tripped on valid verdicts") + } + } + // 9 of 19 invalid = 47%, above the threshold once half the window is filled. + tripped := false + for i := 0; i < 10; i++ { + if b.observe(true) { + tripped = true + } + } + if !tripped || !b.open() { + t.Fatal("breaker did not trip") + } + if !b.observe(false) { + t.Fatal("open breaker must stay open during cooldown") + } +} + +func TestBreakerNeedsHalfAWindow(t *testing.T) { + b := newBreaker(20, 40, time.Hour) + for i := 0; i < 5; i++ { + if b.observe(true) { + t.Fatal("five verdicts are not evidence") + } + } +} diff --git a/internal/app/emailverify/service.go b/internal/app/emailverify/service.go index e9f97f30..4e684367 100644 --- a/internal/app/emailverify/service.go +++ b/internal/app/emailverify/service.go @@ -1,82 +1,413 @@ // Package emailverify (app layer) orchestrates pre-send email verification: -// it loads contacts, runs them through a pkg/emailverify.Verifier, and persists -// the result back onto the contact. This is the control-plane home for -// verification — the SMTP RCPT probe inside the Verifier dials remote MX hosts -// on :25 and must never run from a worker (a sending IP). See -// internal/pkg/emailverify for the probing details and the in-repo-vs-paid -// backend split. +// it loads contacts due for a check, picks the verifier for their organization +// (the paid provider connected on the workspace, else the in-house probe), +// persists each verdict, and offers the member-facing actions (re-verify, +// mark deliverable). The SMTP probe inside the in-house verifier dials remote +// MX hosts on :25 and must never run from a worker (a sending IP). package emailverify import ( "context" + "errors" + "strings" + "sync" + "time" "github.com/google/uuid" + "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/config" "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/pkg/emailverify" "github.com/warmbly/warmbly/internal/repository" ) +// Provider is a paid verification backend bound to one organization. +type Provider struct { + Name string + // ConnectionID is the integration connection carrying the key; nil for + // the instance-wide key an operator configured. + ConnectionID *uuid.UUID + Client *emailverify.MillionVerifier +} + +// ProviderSource resolves the paid provider an organization connected. +// Implemented by the integration service. +type ProviderSource interface { + // VerificationProviderFor returns nil when the org has no usable provider. + VerificationProviderFor(ctx context.Context, orgID uuid.UUID) (*Provider, error) + // ReportVerificationProviderError flips the connection's health when the + // provider rejected the key or ran out of credits. + ReportVerificationProviderError(ctx context.Context, connectionID uuid.UUID, err error) +} + // Service verifies contact email addresses before they are ever sent to. type Service interface { - // VerifyContact verifies a single stored contact by id and persists the - // result. Returns the Result so callers (admin/on-demand) can surface it. - VerifyContact(ctx context.Context, contactID uuid.UUID) (emailverify.Result, *errx.Error) + // VerifyAddress verifies an arbitrary address for an organization without + // touching the DB, through whichever verifier the org uses. + VerifyAddress(ctx context.Context, orgID uuid.UUID, email string) emailverify.Result - // VerifyAddress verifies an arbitrary address without touching the DB. Used - // by the on-demand handler for addresses that aren't stored contacts yet. - VerifyAddress(ctx context.Context, email string) emailverify.Result - - // VerifyPending verifies up to `limit` not-yet-checked contacts, persisting - // each result. Returns the number processed. Driven by the ticker scheduler. + // VerifyPending verifies up to `limit` contacts due for a check. Returns + // the number processed. Driven by the scheduler. VerifyPending(ctx context.Context, limit int) (int, *errx.Error) + + // Request applies a member action: queue a re-check, or record a manual + // verdict. + Request(ctx context.Context, orgID uuid.UUID, req models.ContactVerificationRequest) (*models.ContactVerificationResponse, *errx.Error) + + // Overview reports which verifier the org uses, its credits, and the + // org's contacts by verdict. + Overview(ctx context.Context, orgID uuid.UUID) (*models.VerificationOverview, *errx.Error) + + // Kick wakes the scheduler for an immediate pass (after a re-verify + // request or an import), instead of waiting for the next interval. + Kick() + // Wake is the channel the scheduler selects on. + Wake() <-chan struct{} + + // SetVerdictHook registers a callback run once per organization after a + // pass that changed its contacts, e.g. to resume a campaign that was + // parked waiting for verification. + SetVerdictHook(fn func(ctx context.Context, orgID uuid.UUID)) } type service struct { - repo repository.ContactRepository - verifier emailverify.Verifier + repo repository.ContactRepository + builtin emailverify.Verifier + providers ProviderSource + // platform is the instance-wide paid client an operator configured for + // workspaces that bring no key of their own; nil when unset. + platform *emailverify.MillionVerifier + builtinReady bool + + wake chan struct{} + hook func(ctx context.Context, orgID uuid.UUID) + + breaker *breaker + + creditsMu sync.Mutex + credits map[string]creditsEntry } -// NewService wires the verification service. verifier is the pluggable backend: -// the in-house emailverify.SMTPVerifier in dev/self-host, or a paid provider -// (ZeroBounce/NeverBounce/Bouncer) implementing the same interface in prod. -func NewService(repo repository.ContactRepository, verifier emailverify.Verifier) Service { - return &service{repo: repo, verifier: verifier} +type creditsEntry struct { + n int + err error + at time.Time } -func (s *service) VerifyAddress(ctx context.Context, email string) emailverify.Result { - return s.verifier.Verify(ctx, email) +// Options configures the service. +type Options struct { + // Builtin is the in-house verifier. Required. + Builtin emailverify.Verifier + // BuiltinReady reports whether Builtin can run its SMTP probe (a public + // HELO host is configured). Shown to members so a self-hosted instance + // without one knows why every verdict is "unknown". + BuiltinReady bool + // Providers resolves per-org paid providers. Optional. + Providers ProviderSource + // PlatformMillionVerifierKey is the operator's own key, used for every + // workspace without a key of its own. Optional. + PlatformMillionVerifierKey string } -func (s *service) VerifyContact(ctx context.Context, contactID uuid.UUID) (emailverify.Result, *errx.Error) { - contact, xerr := s.repo.GetByID(ctx, contactID) - if xerr != nil { - return emailverify.Result{}, xerr +// NewService wires the verification service. +func NewService(repo repository.ContactRepository, opts Options) Service { + s := &service{ + repo: repo, + builtin: opts.Builtin, + builtinReady: opts.BuiltinReady, + providers: opts.Providers, + wake: make(chan struct{}, 1), + breaker: newBreaker(config.VerificationBreakerWindow, config.VerificationBreakerInvalidPct, time.Duration(config.VerificationBreakerCooldownMinutes)*time.Minute), + credits: map[string]creditsEntry{}, } - res := s.verifier.Verify(ctx, contact.Email) - if xerr := s.repo.UpdateContactVerification(ctx, contactID, res); xerr != nil { - return res, xerr + if k := strings.TrimSpace(opts.PlatformMillionVerifierKey); k != "" { + s.platform = emailverify.NewMillionVerifier(k, "") } - return res, nil + return s +} + +func (s *service) Kick() { + select { + case s.wake <- struct{}{}: + default: + } +} + +func (s *service) Wake() <-chan struct{} { return s.wake } + +func (s *service) SetVerdictHook(fn func(ctx context.Context, orgID uuid.UUID)) { s.hook = fn } + +// providerFor resolves the org's paid provider: its own connection first, +// then the operator's instance-wide key. +func (s *service) providerFor(ctx context.Context, orgID uuid.UUID) *Provider { + if s.providers != nil { + p, err := s.providers.VerificationProviderFor(ctx, orgID) + if err != nil { + log.Warn().Err(err).Str("organization_id", orgID.String()).Msg("verification: could not resolve provider; using built-in") + } else if p != nil { + return p + } + } + if s.platform != nil { + return &Provider{Name: emailverify.ProviderMillionVerifier, Client: s.platform} + } + return nil +} + +func (s *service) VerifyAddress(ctx context.Context, orgID uuid.UUID, email string) emailverify.Result { + if p := s.providerFor(ctx, orgID); p != nil { + res, err := p.Client.Check(ctx, email) + if err == nil { + return res + } + s.noteProviderError(ctx, p, err) + } + return s.verifyBuiltin(ctx, email) +} + +// verifyBuiltin runs the in-house verifier under the self-check breaker. +func (s *service) verifyBuiltin(ctx context.Context, email string) emailverify.Result { + res := s.builtin.Verify(ctx, email) + // Syntax, no-MX and disposable verdicts do not come from a probe, so they + // neither feed nor fall under the breaker. + probeVerdict := res.SubStatus == emailverify.SubStatusNone || res.SubStatus == emailverify.SubStatusCatchAll || res.SubStatus == emailverify.SubStatusRole + if !probeVerdict { + return res + } + if s.breaker.observe(res.Status == emailverify.StatusInvalid) && res.Status == emailverify.StatusInvalid { + res.Status = emailverify.StatusUnknown + res.Reason = "probe rejection not trusted: the in-house check is rejecting an unusual share of addresses and is cooling down (" + res.Reason + ")" + } + return res +} + +func (s *service) noteProviderError(ctx context.Context, p *Provider, err error) { + if p == nil || err == nil { + return + } + if errors.Is(err, emailverify.ErrMillionVerifierKey) || errors.Is(err, emailverify.ErrMillionVerifierCredits) { + if p.ConnectionID != nil && s.providers != nil { + s.providers.ReportVerificationProviderError(ctx, *p.ConnectionID, err) + } + s.creditsMu.Lock() + s.credits[cacheKey(p)] = creditsEntry{err: err, at: time.Now()} + s.creditsMu.Unlock() + } +} + +func cacheKey(p *Provider) string { + if p.ConnectionID != nil { + return p.ConnectionID.String() + } + return "platform" +} + +// providerUsable checks (cached for a minute) that the provider's key works +// and has credits, so a pass never burns a whole batch on a dead key. +func (s *service) providerUsable(ctx context.Context, p *Provider) (int, error) { + key := cacheKey(p) + s.creditsMu.Lock() + e, ok := s.credits[key] + s.creditsMu.Unlock() + if ok && time.Since(e.at) < time.Minute { + return e.n, e.err + } + n, err := p.Client.Credits(ctx) + if err == nil && n <= 0 { + err = emailverify.ErrMillionVerifierCredits + } + s.creditsMu.Lock() + s.credits[key] = creditsEntry{n: n, err: err, at: time.Now()} + s.creditsMu.Unlock() + if err != nil { + s.noteProviderError(ctx, p, err) + } + return n, err } func (s *service) VerifyPending(ctx context.Context, limit int) (int, *errx.Error) { - contacts, xerr := s.repo.ListUnverifiedContacts(ctx, limit) + cands, xerr := s.repo.ListVerificationCandidates(ctx, limit) if xerr != nil { return 0, xerr } + if len(cands) == 0 { + return 0, nil + } + + // Group by organization: the verifier is chosen once per org. + byOrg := map[uuid.UUID][]repository.VerificationCandidate{} + var order []uuid.UUID + for _, c := range cands { + if _, seen := byOrg[c.OrganizationID]; !seen { + order = append(order, c.OrganizationID) + } + byOrg[c.OrganizationID] = append(byOrg[c.OrganizationID], c) + } + processed := 0 - for i := range contacts { - // Honour cancellation between addresses; each probe can take seconds. + for _, orgID := range order { if err := ctx.Err(); err != nil { break } - res := s.verifier.Verify(ctx, contacts[i].Email) - if xerr := s.repo.UpdateContactVerification(ctx, contacts[i].ID, res); xerr != nil { - // Skip this one; a transient DB error shouldn't abort the whole tick. - continue + n := s.verifyOrgBatch(ctx, orgID, byOrg[orgID]) + processed += n + if n > 0 && s.hook != nil { + s.hook(ctx, orgID) } - processed++ } return processed, nil } + +// verifyOrgBatch checks one org's candidates with its verifier, in parallel +// up to the verifier's concurrency, and persists every verdict. +func (s *service) verifyOrgBatch(ctx context.Context, orgID uuid.UUID, cands []repository.VerificationCandidate) int { + verify := s.verifyBuiltin + workers := config.VerificationProbeConcurrency + if p := s.providerFor(ctx, orgID); p != nil { + if _, err := s.providerUsable(ctx, p); err != nil { + log.Warn().Err(err).Str("organization_id", orgID.String()).Msg("verification: paid provider unusable; using built-in check") + } else { + workers = config.VerificationProviderConcurrency + verify = func(ctx context.Context, email string) emailverify.Result { + res, err := p.Client.Check(ctx, email) + if err != nil { + s.noteProviderError(ctx, p, err) + // Fall back for this address so the pass still makes progress. + return s.verifyBuiltin(ctx, email) + } + return res + } + } + } + + var ( + wg sync.WaitGroup + mu sync.Mutex + processed int + sem = make(chan struct{}, workers) + ) + for _, c := range cands { + if err := ctx.Err(); err != nil { + break + } + sem <- struct{}{} + wg.Add(1) + go func(c repository.VerificationCandidate) { + defer wg.Done() + defer func() { <-sem }() + res := verify(ctx, c.Email) + if xerr := s.repo.UpdateContactVerification(ctx, c.ID, res); xerr != nil { + // Skip this one; a transient DB error shouldn't abort the whole pass. + return + } + mu.Lock() + processed++ + mu.Unlock() + }(c) + } + wg.Wait() + return processed +} + +func (s *service) Request(ctx context.Context, orgID uuid.UUID, req models.ContactVerificationRequest) (*models.ContactVerificationResponse, *errx.Error) { + var ids []uuid.UUID + for _, raw := range req.Contacts { + id, err := uuid.Parse(strings.TrimSpace(raw)) + if err != nil { + return nil, errx.ErrUuid + } + ids = append(ids, id) + } + if req.CampaignID != "" { + cid, err := uuid.Parse(strings.TrimSpace(req.CampaignID)) + if err != nil { + return nil, errx.ErrUuid + } + more, xerr := s.repo.UndeliverableLeadIDs(ctx, orgID, cid) + if xerr != nil { + return nil, xerr + } + ids = append(ids, more...) + } + if len(ids) == 0 { + return nil, errx.NewWithIdentifier(errx.BadRequest, "no_contacts", "no contacts selected") + } + + resp := &models.ContactVerificationResponse{Action: req.Action} + switch req.Action { + case models.ContactVerificationActionVerify: + n, xerr := s.repo.ResetContactsVerification(ctx, orgID, ids) + if xerr != nil { + return nil, xerr + } + resp.Affected, resp.Queued = n, true + s.Kick() + case models.ContactVerificationActionMarkDeliverable: + n, xerr := s.repo.SetContactsVerification(ctx, orgID, ids, models.ContactVerificationWrite{ + Status: string(emailverify.StatusValid), + Reason: "marked deliverable by a member", + Provider: "manual", + Source: models.VerificationSourceManual, + }) + if xerr != nil { + return nil, xerr + } + resp.Affected = n + if s.hook != nil { + s.hook(ctx, orgID) + } + case models.ContactVerificationActionMarkUndeliverable: + n, xerr := s.repo.SetContactsVerification(ctx, orgID, ids, models.ContactVerificationWrite{ + Status: string(emailverify.StatusInvalid), + Reason: "marked undeliverable by a member", + Provider: "manual", + Source: models.VerificationSourceManual, + }) + if xerr != nil { + return nil, xerr + } + resp.Affected = n + default: + return nil, errx.NewWithIdentifier(errx.BadRequest, "invalid_action", "action must be verify, mark_deliverable or mark_undeliverable") + } + return resp, nil +} + +func (s *service) Overview(ctx context.Context, orgID uuid.UUID) (*models.VerificationOverview, *errx.Error) { + counts, xerr := s.repo.VerificationCounts(ctx, orgID) + if xerr != nil { + return nil, xerr + } + out := &models.VerificationOverview{ + Provider: emailverify.ProviderBuiltin, + BuiltinReady: s.builtinReady, + Counts: counts, + } + if p := s.providerFor(ctx, orgID); p != nil { + if p.ConnectionID != nil { + id := p.ConnectionID.String() + out.ConnectionID = &id + } + n, err := s.providerUsable(ctx, p) + if err != nil { + out.ProviderError = providerErrorText(err) + } else { + out.Provider = p.Name + out.Credits = &n + } + } + return out, nil +} + +func providerErrorText(err error) string { + switch { + case errors.Is(err, emailverify.ErrMillionVerifierKey): + return "MillionVerifier rejected the API key. Reconnect it with a current key." + case errors.Is(err, emailverify.ErrMillionVerifierCredits): + return "The MillionVerifier account has no credits left. Top it up to keep using it; the built-in check is used meanwhile." + default: + return "MillionVerifier could not be reached; the built-in check is used meanwhile." + } +} diff --git a/internal/app/integration/catalog.go b/internal/app/integration/catalog.go index 05a3bdf7..80403c16 100644 --- a/internal/app/integration/catalog.go +++ b/internal/app/integration/catalog.go @@ -191,5 +191,19 @@ func Catalog() []models.IntegrationCatalogEntry { // Lead Sync feature under Contacts), but it is no longer surfaced as an // integration tile and has no event-driven append-row automation. See // internal/app/leadsync. + // Verification -------------------------------------------------------- + { + Provider: models.IntegrationMillionVerifier, + Name: "MillionVerifier", + Tagline: "Pay-as-you-go address verification for every contact you import.", + Category: models.IntegrationCategoryVerification, + AuthMethod: string(models.IntegrationAuthAPIKey), + DocsURL: "https://www.millionverifier.com/", + Highlights: []string{ + "Every new contact is checked automatically; nothing to run", + "One credit per address, bought from MillionVerifier as you go", + "Replaces the built-in check while credits last, falls back when they run out", + }, + }, } } diff --git a/internal/app/integration/service.go b/internal/app/integration/service.go index 3a905fdf..2a51da4c 100644 --- a/internal/app/integration/service.go +++ b/internal/app/integration/service.go @@ -7,6 +7,8 @@ import ( "encoding/json" "errors" "fmt" + emailverifyapp "github.com/warmbly/warmbly/internal/app/emailverify" + "github.com/warmbly/warmbly/internal/pkg/emailverify" "strings" "time" @@ -161,6 +163,11 @@ type Service interface { // configured default channel. No-op (nil) when no Slack is connected. NotifySlack(ctx context.Context, orgID uuid.UUID, title, body string) error + // VerificationProviderFor and ReportVerificationProviderError implement + // emailverify.ProviderSource: the org's paid verification backend, if any. + VerificationProviderFor(ctx context.Context, orgID uuid.UUID) (*emailverifyapp.Provider, error) + ReportVerificationProviderError(ctx context.Context, connectionID uuid.UUID, err error) + // Repo exposes the underlying repository for the inbound webhook handlers. Repo() repository.IntegrationRepository } @@ -264,6 +271,16 @@ func (s *service) Connect(ctx context.Context, orgID, userID uuid.UUID, provider displayFields := buildDisplayFields(provider, config) + // A verification key is checked before it is stored: a mistyped key would + // otherwise quietly leave every contact on the built-in check. + if provider == models.IntegrationMillionVerifier { + credits, err := checkMillionVerifierKey(ctx, config) + if err != nil { + return nil, err + } + displayFields["credits"] = credits + } + var inboundSecret string var err error if provider == models.IntegrationCalendly || provider == models.IntegrationCalCom { @@ -1327,10 +1344,89 @@ func buildDisplayFields(provider models.IntegrationProvider, config map[string]a pick("server") case models.IntegrationZapier, models.IntegrationMake, models.IntegrationN8N: // Outbound-via-Warmbly-API providers: minimal display fields. + case models.IntegrationMillionVerifier: + // Credits are filled in at connect time from the provider. } return df } +// checkMillionVerifierKey validates a pasted key against the provider and +// returns the account's credit balance. +func checkMillionVerifierKey(ctx context.Context, config map[string]any) (int, error) { + key, _ := config["api_key"].(string) + key = strings.TrimSpace(key) + if key == "" { + return 0, errors.New("paste your MillionVerifier API key") + } + credits, err := emailverify.NewMillionVerifier(key, "").Credits(ctx) + switch { + case errors.Is(err, emailverify.ErrMillionVerifierKey): + return 0, errors.New("MillionVerifier rejected this API key") + case errors.Is(err, emailverify.ErrMillionVerifierCredits): + // A valid key with an empty balance still connects; the built-in + // check covers until it is topped up. + return 0, nil + case err != nil: + return 0, fmt.Errorf("could not reach MillionVerifier: %w", err) + } + return credits, nil +} + +// VerificationProviderFor returns the org's connected MillionVerifier client, +// or nil when none is connected. A disconnected or reauth-required connection +// does not count. +func (s *service) VerificationProviderFor(ctx context.Context, orgID uuid.UUID) (*emailverifyapp.Provider, error) { + conns, err := s.repo.ListConnections(ctx, orgID) + if err != nil { + return nil, err + } + for _, c := range conns { + if c.Provider != models.IntegrationMillionVerifier { + continue + } + if c.Status != models.IntegrationStatusConnected && c.Status != models.IntegrationStatusDegraded { + continue + } + sec, err := s.repo.GetConnectionSecrets(ctx, c.ID) + if err != nil { + return nil, err + } + cfg, err := s.openConfig(ctx, sec) + if err != nil { + return nil, err + } + key, _ := cfg["api_key"].(string) + if strings.TrimSpace(key) == "" { + continue + } + id := c.ID + return &emailverifyapp.Provider{ + Name: emailverify.ProviderMillionVerifier, + ConnectionID: &id, + Client: emailverify.NewMillionVerifier(key, ""), + }, nil + } + return nil, nil +} + +// ReportVerificationProviderError records why the provider stopped being +// usable on the connection card. +func (s *service) ReportVerificationProviderError(ctx context.Context, connectionID uuid.UUID, err error) { + if err == nil { + return + } + status, health := models.IntegrationStatusDegraded, models.IntegrationHealthDegraded + detail := err.Error() + switch { + case errors.Is(err, emailverify.ErrMillionVerifierKey): + status, health = models.IntegrationStatusReauthRequired, models.IntegrationHealthDown + detail = "MillionVerifier rejected the API key; reconnect with a current key" + case errors.Is(err, emailverify.ErrMillionVerifierCredits): + detail = "MillionVerifier account is out of credits; the built-in check is used until it is topped up" + } + _ = s.repo.SetConnectionStatus(ctx, connectionID, status, health, detail) +} + // slackChannelFor resolves the channel to post org notifications to. The // OAuth connect flow doesn't capture a default channel, so we look (in order) // at the connection's own config, then reuse whatever channel the org already diff --git a/internal/config/constants.go b/internal/config/constants.go index 75b4b291..ea6d5af7 100644 --- a/internal/config/constants.go +++ b/internal/config/constants.go @@ -158,6 +158,32 @@ const ( UniboxLimitMax = 100 UniboxLimitDefault = 50 + // VerificationRecheckDays is how long a verification verdict is trusted + // before the address is checked again. Mailboxes get created and closed; + // a verdict from last quarter is a guess. + VerificationRecheckDays = 90 + // VerificationUnknownRecheckDays is the shorter shelf life of an + // inconclusive verdict (greylisted, timeout, undisclosing provider). + VerificationUnknownRecheckDays = 30 + // VerificationBatchSize is how many contacts one scheduler pass checks. + VerificationBatchSize = 200 + // VerificationIntervalSeconds is how often the scheduler passes. A pass + // that finds a full batch runs again immediately, so a large import drains + // at the verifier's speed rather than one batch per interval. + VerificationIntervalSeconds = 60 + // VerificationProbeConcurrency bounds parallel in-house SMTP probes. + VerificationProbeConcurrency = 4 + // VerificationProviderConcurrency bounds parallel paid-provider lookups. + VerificationProviderConcurrency = 8 + // VerificationBreakerWindow and VerificationBreakerInvalidPct are the + // in-house probe's self-check: when this share of the last window of + // probe verdicts is "invalid", the probe itself is suspect (issue #200, + // #264) and its invalid verdicts are filed as unknown for + // VerificationBreakerCooldownMinutes. + VerificationBreakerWindow = 200 + VerificationBreakerInvalidPct = 40.0 + VerificationBreakerCooldownMinutes = 60 + // WarmupVerifyHeader is the custom header carrying the warmup // verification token on outbound warmup mail. The name is intentionally // generic (not "X-Warmbly-*") so anti-spam vendors cannot trivially diff --git a/internal/infrastructure/db/migrations/000110_verification_provenance.down.sql b/internal/infrastructure/db/migrations/000110_verification_provenance.down.sql new file mode 100644 index 00000000..08c9d4ce --- /dev/null +++ b/internal/infrastructure/db/migrations/000110_verification_provenance.down.sql @@ -0,0 +1,14 @@ +-- Postgres cannot drop a single enum value, so 'paused_undeliverable' stays on +-- campaign_status. Campaigns parked in it return to a plain pause first. +UPDATE campaigns SET status = 'paused' WHERE status = 'paused_undeliverable'; + +DROP INDEX IF EXISTS public.idx_contacts_verification_pending; +CREATE INDEX IF NOT EXISTS idx_contacts_verification_pending + ON public.contacts (verification_checked_at) + WHERE verification_status = 'unknown' AND verification_checked_at IS NULL; + +ALTER TABLE public.contacts DROP CONSTRAINT IF EXISTS contacts_verification_source_check; +ALTER TABLE public.contacts + DROP COLUMN IF EXISTS verification_source, + DROP COLUMN IF EXISTS verification_provider, + DROP COLUMN IF EXISTS verification_sub_status; diff --git a/internal/infrastructure/db/migrations/000110_verification_provenance.up.sql b/internal/infrastructure/db/migrations/000110_verification_provenance.up.sql new file mode 100644 index 00000000..93f2982a --- /dev/null +++ b/internal/infrastructure/db/migrations/000110_verification_provenance.up.sql @@ -0,0 +1,37 @@ +-- Verification provenance: who produced a contact's verdict and how specific +-- it is, so an external or manual verdict is never overwritten by the in-house +-- probe, and a campaign that runs out of deliverable leads pauses for the +-- owner instead of quietly finishing. +-- +-- verification_source: '' (never checked) | 'probe' (in-house SMTP probe) | +-- 'provider' (a paid backend on the org's integration) | 'imported' (a +-- status column brought in with the list or through the API) | 'manual' +-- (a member marked it deliverable). Only 'manual' is never re-checked. +-- verification_provider: which backend or vocabulary produced the verdict. +-- verification_sub_status: catch_all | disposable | role | spamtrap | +-- mailbox_full | no_mx | syntax | undisclosed | ''. + +ALTER TABLE public.contacts + ADD COLUMN IF NOT EXISTS verification_source text NOT NULL DEFAULT '', + ADD COLUMN IF NOT EXISTS verification_provider text NOT NULL DEFAULT '', + ADD COLUMN IF NOT EXISTS verification_sub_status text NOT NULL DEFAULT ''; + +ALTER TABLE public.contacts + ADD CONSTRAINT contacts_verification_source_check + CHECK (verification_source IN ('', 'probe', 'provider', 'imported', 'manual')); + +-- Every verdict recorded so far came from the probe. +UPDATE public.contacts +SET verification_source = 'probe', verification_provider = 'builtin' +WHERE verification_checked_at IS NOT NULL AND verification_source = ''; + +-- The scheduler now re-checks aged verdicts too, so the pending index covers +-- the check timestamp itself (NULLs first) instead of only never-checked rows. +DROP INDEX IF EXISTS public.idx_contacts_verification_pending; +CREATE INDEX IF NOT EXISTS idx_contacts_verification_pending + ON public.contacts (verification_checked_at NULLS FIRST, created_at) + WHERE verification_source <> 'manual'; + +-- A campaign whose remaining leads were all refused by verification parks +-- here, resumable once the owner re-verifies or marks them deliverable. +ALTER TYPE public.campaign_status ADD VALUE IF NOT EXISTS 'paused_undeliverable'; diff --git a/internal/jobs/email_verification.go b/internal/jobs/email_verification.go index 7e26e1e0..d3002d63 100644 --- a/internal/jobs/email_verification.go +++ b/internal/jobs/email_verification.go @@ -9,12 +9,12 @@ import ( emailverifyapp "github.com/warmbly/warmbly/internal/app/emailverify" ) -// EmailVerificationJob verifies a capped batch of not-yet-checked contacts each -// run, so the platform can drop hard-bouncing addresses before any worker sends -// to them. It is a thin wrapper around emailverify.Service.VerifyPending; the -// per-tick cap bounds how much outbound SMTP-probe work one pass does. +// EmailVerificationJob verifies a batch of contacts due for a check each run, +// so the platform can drop hard-bouncing addresses before any worker sends +// to them. A run that fills its batch is repeated until the backlog is below +// one batch, so a large import drains at the verifier's speed. // -// Control-plane only: the underlying verifier dials remote MX hosts on :25 and +// Control-plane only: the in-house verifier dials remote MX hosts on :25 and // must run from the backend/consumer, never a worker (sending) IP. type EmailVerificationJob struct { svc emailverifyapp.Service @@ -22,7 +22,7 @@ type EmailVerificationJob struct { } // NewEmailVerificationJob creates the job. batchSize caps how many contacts are -// verified per tick (defaults to 100 when non-positive). +// verified per pass (defaults to 100 when non-positive). func NewEmailVerificationJob(svc emailverifyapp.Service, batchSize int) *EmailVerificationJob { if batchSize <= 0 { batchSize = 100 @@ -30,20 +30,26 @@ func NewEmailVerificationJob(svc emailverifyapp.Service, batchSize int) *EmailVe return &EmailVerificationJob{svc: svc, batchSize: batchSize} } -// Run performs one capped verification pass. Safe to call frequently — it -// no-ops when there are no unverified contacts. +// Run drains the backlog in batches. Safe to call frequently; it no-ops when +// nothing is due. func (j *EmailVerificationJob) Run(ctx context.Context) error { if j.svc == nil { return nil } - if _, err := j.svc.VerifyPending(ctx, j.batchSize); err != nil { - sentry.CaptureException(err) - return err + for { + n, err := j.svc.VerifyPending(ctx, j.batchSize) + if err != nil { + sentry.CaptureException(err) + return err + } + if n < j.batchSize || ctx.Err() != nil { + return nil + } } - return nil } -// EmailVerificationScheduler runs the job on a fixed interval. +// EmailVerificationScheduler runs the job on a fixed interval and whenever +// the service is kicked (a re-verify request, an import). type EmailVerificationScheduler struct { job *EmailVerificationJob interval time.Duration @@ -64,17 +70,22 @@ func (s *EmailVerificationScheduler) Start(ctx context.Context) { ticker := time.NewTicker(s.interval) defer ticker.Stop() + var wake <-chan struct{} + if s.job != nil && s.job.svc != nil { + wake = s.job.svc.Wake() + } for { select { case <-ticker.C: - if err := s.job.Run(ctx); err != nil { - sentry.CaptureException(err) - } + case <-wake: case <-s.stopCh: return case <-ctx.Done(): return } + if err := s.job.Run(ctx); err != nil { + sentry.CaptureException(err) + } } } diff --git a/internal/models/admin.go b/internal/models/admin.go index 7d22dcc4..dcab573e 100644 --- a/internal/models/admin.go +++ b/internal/models/admin.go @@ -309,7 +309,7 @@ type AdminCampaignSearch struct { Query string `form:"q"` UserID *uuid.UUID `form:"user_id"` OrgID *uuid.UUID `form:"org_id"` - Status string `form:"status"` // draft, active, paused, completed, paused_trial_expired, paused_no_accounts + Status string `form:"status"` // draft, active, paused, completed, paused_trial_expired, paused_no_accounts, paused_guardrail, paused_undeliverable // Boolean flags OpenTracking bool `form:"open_tracking"` diff --git a/internal/models/campaign.go b/internal/models/campaign.go index 39152452..bf841a2a 100644 --- a/internal/models/campaign.go +++ b/internal/models/campaign.go @@ -358,3 +358,14 @@ type CreateSequenceInput struct { BodyCode *bool `json:"body_code,omitempty"` WaitAfter *int `json:"wait_after,omitempty"` } + +// StartCampaignOptions qualifies a start request. +type StartCampaignOptions struct { + // AcknowledgeListRisk launches past the bounce-risk gate: the member has + // read the projection and takes the risk (the list may have been verified + // elsewhere). + AcknowledgeListRisk bool `json:"acknowledge_list_risk"` + // Automatic marks a start the platform initiated (a resume after + // verification), which skips the member-facing cooldown. + Automatic bool `json:"-"` +} diff --git a/internal/models/contact.go b/internal/models/contact.go index 72556b61..6bfcb0e9 100644 --- a/internal/models/contact.go +++ b/internal/models/contact.go @@ -38,6 +38,13 @@ type Contact struct { VerificationReason string `json:"verification_reason"` IsCatchAll bool `json:"is_catch_all"` VerificationCheckedAt *time.Time `json:"verification_checked_at,omitempty"` + // VerificationSource says who produced the verdict (see the + // VerificationSource* constants); VerificationProvider names the backend or + // the external vocabulary; VerificationSubStatus refines the status + // (catch_all, disposable, role, ...). + VerificationSource string `json:"verification_source"` + VerificationProvider string `json:"verification_provider"` + VerificationSubStatus string `json:"verification_sub_status"` // Recipient ESP/provider, derived in the control plane from the recipient // domain (never an MX dial on the send hot path). '' | 'gmail' | 'outlook' @@ -178,12 +185,86 @@ type CampaignLeadCounts struct { // ContactsCounts are org-wide contact facet totals for the browse sidebar. type ContactsCounts struct { - Total int `json:"total"` - Subscribed int `json:"subscribed"` - Unsubscribed int `json:"unsubscribed"` - InCampaign int `json:"in_campaign"` - NotContacted int `json:"not_contacted"` - Categories []ContactCategoryCount `json:"categories"` + Total int `json:"total"` + Subscribed int `json:"subscribed"` + Unsubscribed int `json:"unsubscribed"` + InCampaign int `json:"in_campaign"` + NotContacted int `json:"not_contacted"` + Categories []ContactCategoryCount `json:"categories"` + Verification ContactVerificationCounts `json:"verification"` +} + +// ContactVerificationCounts is the org's contacts by verification status. +// Pending is the subset of Unknown nobody has checked yet. +type ContactVerificationCounts struct { + Valid int `json:"valid"` + Risky int `json:"risky"` + Invalid int `json:"invalid"` + Unknown int `json:"unknown"` + Pending int `json:"pending"` +} + +// Verification provenance values for contacts.verification_source. +const ( + VerificationSourceNone = "" + VerificationSourceProbe = "probe" + VerificationSourceProvider = "provider" + VerificationSourceImported = "imported" + VerificationSourceManual = "manual" +) + +// ContactVerificationWrite is a verdict to store on a contact, already +// normalised into Warmbly's vocabulary. +type ContactVerificationWrite struct { + Status string + SubStatus string + Reason string + Provider string + Source string +} + +// Actions for POST /contacts/verification. +const ( + ContactVerificationActionVerify = "verify" + ContactVerificationActionMarkDeliverable = "mark_deliverable" + ContactVerificationActionMarkUndeliverable = "mark_undeliverable" +) + +// ContactVerificationRequest is the body of POST /contacts/verification. +type ContactVerificationRequest struct { + Contacts []string `json:"contacts"` + // CampaignID selects every lead of one campaign that verification refused + // (the "re-verify skipped leads" action), instead of listing ids. + CampaignID string `json:"campaign_id,omitempty"` + Action string `json:"action"` +} + +// ContactVerificationResponse reports how many contacts the action touched. +type ContactVerificationResponse struct { + Affected int `json:"affected"` + Action string `json:"action"` + // Queued is true for the verify action: the check runs in the background + // and each contact updates live as its verdict lands. + Queued bool `json:"queued"` +} + +// VerificationOverview is what Settings shows about address verification. +type VerificationOverview struct { + // Provider is who checks this workspace's addresses: "builtin" or + // "millionverifier". + Provider string `json:"provider"` + // ConnectionID is the integration connection behind a paid provider. + ConnectionID *string `json:"connection_id,omitempty"` + // Credits is the paid provider's remaining balance when it could be read. + Credits *int `json:"credits,omitempty"` + // ProviderError is set when the paid provider is connected but unusable + // (bad key, no credits), in which case the built-in check is in use. + ProviderError string `json:"provider_error,omitempty"` + // BuiltinReady says whether the in-house probe can reach mail servers from + // this instance (a HELO host is configured). Off, it still checks syntax, + // MX, and disposable domains. + BuiltinReady bool `json:"builtin_ready"` + Counts ContactVerificationCounts `json:"counts"` } // ContactCategoryCount is the number of org contacts carrying one category. @@ -383,6 +464,19 @@ type AddContact struct { CustomFields map[string]string `json:"custom_fields"` + // VerificationStatus is a verdict the caller already holds for this + // address, in Warmbly's vocabulary or any provider's the platform knows + // (ZeroBounce, MillionVerifier, NeverBounce, ...). VerificationProvider + // optionally names that vocabulary; without it the value is recognised by + // itself. An unrecognised value is a 400. Stored as an imported verdict, + // which the background check leaves alone until it ages out. + VerificationStatus string `json:"verification_status,omitempty"` + VerificationProvider string `json:"verification_provider,omitempty"` + + // Verification is the normalised verdict derived from VerificationStatus. + // Filled by the repository, never read from the request. + Verification *ContactVerificationWrite `json:"-"` + // Subscribed is the marketing-consent flag to store. nil means "don't // decide": a new contact defaults to subscribed, an existing one keeps // whatever it already had. Set explicitly by the importer when the file @@ -454,6 +548,7 @@ type SearchContacts struct { MinCampaigns *int `json:"min_campaigns"` // Minimum number of associated campaigns MaxCampaigns *int `json:"max_campaigns"` // Maximum number of associated campaigns Subscribed *bool `json:"subscribed"` // Filter by subscription status + VerificationStatus string `json:"verification_status"` // Filter by verification verdict: valid | risky | invalid | unknown CreatedAfter *time.Time `json:"created_after"` // Contacts created after this date CreatedBefore *time.Time `json:"created_before"` // Contacts created before this date UpdatedAfter *time.Time `json:"updated_after"` // Contacts updated after this date diff --git a/internal/models/contact_import.go b/internal/models/contact_import.go index 85ba3101..92513f8a 100644 --- a/internal/models/contact_import.go +++ b/internal/models/contact_import.go @@ -42,6 +42,11 @@ const ( ContactImportTargetPhone ContactImportColumnTarget = "phone" ContactImportTargetSubscribed ContactImportColumnTarget = "subscribed" ContactImportTargetCategories ContactImportColumnTarget = "categories" + // ContactImportTargetVerificationStatus reads a verdict column written by + // Warmbly or another verification service (ZeroBounce, MillionVerifier, + // NeverBounce, ...). Values are recognised by vocabulary; a value nobody + // knows leaves the contact unverified rather than failing the row. + ContactImportTargetVerificationStatus ContactImportColumnTarget = "verification_status" // ContactImportTargetCustom routes the column into Contact.CustomFields // under ContactImportColumnMapping.CustomKey. "custom:" is accepted // as an equivalent legacy spelling. @@ -56,6 +61,11 @@ type ContactImportColumnMapping struct { Index int `json:"index"` Target ContactImportColumnTarget `json:"target"` + // VerificationProvider names the vocabulary of a verification_status + // column when the header or its values made it clear (e.g. "zerobounce"). + // Optional; without it each value is recognised by itself. + VerificationProvider string `json:"verification_provider,omitempty"` + // CustomKey is only used when Target == "custom:". It // is split out so the client can render a nicer label without // having to parse the target string. diff --git a/internal/models/integration.go b/internal/models/integration.go index 06eaaa64..0bd3f417 100644 --- a/internal/models/integration.go +++ b/internal/models/integration.go @@ -35,6 +35,9 @@ const ( // Data IntegrationGoogleSheets IntegrationProvider = "google_sheets" + + // Verification + IntegrationMillionVerifier IntegrationProvider = "millionverifier" ) // AllIntegrationProviders lists every provider the dashboard exposes. The @@ -52,6 +55,7 @@ var AllIntegrationProviders = []IntegrationProvider{ IntegrationCalendly, IntegrationCalCom, IntegrationGoogleSheets, + IntegrationMillionVerifier, } func IsValidIntegrationProvider(s string) bool { @@ -118,6 +122,7 @@ const ( IntegrationCategoryNotifications IntegrationCategory = "notifications" IntegrationCategoryMeetings IntegrationCategory = "meetings" IntegrationCategoryData IntegrationCategory = "data" + IntegrationCategoryVerification IntegrationCategory = "verification" ) // IntegrationCatalogEntry is the static metadata for one provider that the diff --git a/internal/pkg/emailverify/emailverify.go b/internal/pkg/emailverify/emailverify.go index e139eb54..51f60a4b 100644 --- a/internal/pkg/emailverify/emailverify.go +++ b/internal/pkg/emailverify/emailverify.go @@ -43,6 +43,8 @@ import ( "strconv" "strings" "time" + + "github.com/warmbly/warmbly/internal/pkg/signuprisk" ) // Status is the verification outcome for a single address. It is a small closed @@ -65,16 +67,44 @@ const ( StatusUnknown Status = "unknown" ) +// SubStatus refines a Status with the reason class a paid provider or the +// in-house checks can name. Empty when nothing more specific is known. +type SubStatus string + +const ( + SubStatusNone SubStatus = "" + SubStatusCatchAll SubStatus = "catch_all" + SubStatusDisposable SubStatus = "disposable" + SubStatusRole SubStatus = "role" + SubStatusSpamTrap SubStatus = "spamtrap" + SubStatusMailboxFull SubStatus = "mailbox_full" + SubStatusNoMX SubStatus = "no_mx" + SubStatusSyntax SubStatus = "syntax" + // SubStatusUndisclosed marks a provider (Microsoft, Yahoo) that answers + // every RCPT the same way, so an SMTP probe cannot judge the mailbox. + SubStatusUndisclosed SubStatus = "undisclosed" +) + +// Provider names for Result.Provider and contacts.verification_provider. +const ( + ProviderBuiltin = "builtin" + ProviderMillionVerifier = "millionverifier" +) + // Result is the outcome of verifying one address. It round-trips into the // contacts table (verification_status / verification_reason / is_catch_all / -// verification_checked_at). +// verification_checked_at / verification_sub_status / verification_provider). type Result struct { Email string `json:"email"` Status Status `json:"status"` + SubStatus SubStatus `json:"sub_status,omitempty"` Reason string `json:"reason"` IsCatchAll bool `json:"is_catch_all"` HasMX bool `json:"has_mx"` - CheckedAt time.Time `json:"checked_at"` + // Provider is who produced the verdict: ProviderBuiltin, a paid backend, + // or the vocabulary an imported result was recognised as. + Provider string `json:"provider,omitempty"` + CheckedAt time.Time `json:"checked_at"` } // Verifier is the single contract the rest of the platform depends on. The @@ -125,6 +155,9 @@ func (c Config) withDefaults() Config { return c } +// ProbeReady reports whether the configured HELO host lets the SMTP probe run. +func (v *SMTPVerifier) ProbeReady() bool { return isFQDN(v.cfg.HeloHost) } + // SMTPVerifier is the in-house Verifier: syntax -> MX -> SMTP RCPT probe -> // catch-all detection. It opens exactly one connection to the lowest-preference // MX and probes both the real address and a random localpart on the same @@ -135,6 +168,9 @@ type SMTPVerifier struct { // smtpPort is always "25" in production (MX hosts listen nowhere else); // it exists so tests can point probe() at a local server. smtpPort string + // domains remembers per-domain facts (no MX, catch-all, undisclosing + // provider) so a 50k list at 2k domains costs 2k probes, not 50k. + domains *domainCache } // New constructs the in-house SMTP verifier. The resolver mirrors dnsauth's @@ -144,6 +180,7 @@ func New(cfg Config) *SMTPVerifier { cfg: cfg.withDefaults(), resolver: &net.Resolver{}, smtpPort: "25", + domains: newDomainCache(domainCacheTTL), } } @@ -152,12 +189,13 @@ func New(cfg Config) *SMTPVerifier { // code path. func (v *SMTPVerifier) Verify(ctx context.Context, email string) Result { now := time.Now().UTC() - res := Result{Email: email, CheckedAt: now, Status: StatusUnknown} + res := Result{Email: email, CheckedAt: now, Status: StatusUnknown, Provider: ProviderBuiltin} // 1. Syntax (RFC 5322-ish via net/mail). A parse failure is a hard invalid. addr, err := mail.ParseAddress(email) if err != nil { res.Status = StatusInvalid + res.SubStatus = SubStatusSyntax res.Reason = "invalid syntax" return res } @@ -166,14 +204,49 @@ func (v *SMTPVerifier) Verify(ctx context.Context, email string) Result { at := strings.LastIndex(normalized, "@") if at <= 0 || at == len(normalized)-1 { res.Status = StatusInvalid + res.SubStatus = SubStatusSyntax res.Reason = "invalid syntax" return res } localpart := normalized[:at] domain := normalized[at+1:] - // 2. MX lookup. No MX (and no usable fallback) is a hard invalid: nowhere to - // deliver. A lookup *error* (timeout/SERVFAIL) is unknown, not invalid. + // Address-level facts that need no network. A disposable domain is never + // worth a send; a role address is deliverable but flagged, so the campaign's + // risky toggle decides. + if signuprisk.IsDisposable(normalized) { + res.Status = StatusInvalid + res.SubStatus = SubStatusDisposable + res.Reason = "disposable email domain" + return res + } + role := isRoleLocalpart(localpart) + + // 2. Domain facts, cached: no MX and catch-all are properties of the + // domain, and providers that never disclose mailboxes are known by MX. + if cached, ok := v.domains.get(domain); ok { + switch cached.kind { + case domainNoMX: + res.Status = StatusInvalid + res.SubStatus = SubStatusNoMX + res.Reason = "no MX records" + return res + case domainCatchAll: + res.HasMX = true + res.IsCatchAll = true + res.Status = StatusRisky + res.SubStatus = SubStatusCatchAll + res.Reason = "catch-all domain; acceptance is not conclusive" + return res + case domainUndisclosed: + res.HasMX = true + res.Status = StatusUnknown + res.SubStatus = SubStatusUndisclosed + res.Reason = cached.reason + return res + } + } + hosts, mxErr := v.lookupMXHosts(ctx, domain) if mxErr != nil { res.Status = StatusUnknown @@ -181,12 +254,26 @@ func (v *SMTPVerifier) Verify(ctx context.Context, email string) Result { return res } if len(hosts) == 0 { + v.domains.put(domain, domainFact{kind: domainNoMX}) res.Status = StatusInvalid + res.SubStatus = SubStatusNoMX res.Reason = "no MX records" return res } res.HasMX = true + // Providers that accept every RCPT and bounce later (Microsoft 365, + // Yahoo) make a probe meaningless: say so instead of spending a + // connection and calling the result "valid". + if fp := fingerprintMX(hosts); fp.undisclosed { + reason := fp.name + " does not disclose mailboxes to an SMTP probe" + v.domains.put(domain, domainFact{kind: domainUndisclosed, reason: reason}) + res.Status = StatusUnknown + res.SubStatus = SubStatusUndisclosed + res.Reason = reason + return res + } + // 3. SMTP RCPT probe against the lowest-preference (highest priority) MX. // Refused outright without a real HELO identity: a probe that announces a // bare or reserved name gets the SESSION rejected, and that rejection @@ -196,17 +283,25 @@ func (v *SMTPVerifier) Verify(ctx context.Context, email string) Result { res.Reason = "smtp probe skipped: set EMAIL_VERIFY_HELO_HOST to a public fully-qualified hostname for this instance" return res } - probe := v.probe(ctx, hosts[0], localpart, domain) + probe := v.probeHosts(ctx, hosts, localpart, domain) switch probe.outcome { case probeAccepted: // 4. Catch-all check already folded into probe(): if the random control // localpart was also accepted, the 250 on the real address is meaningless. if probe.catchAll { + v.domains.put(domain, domainFact{kind: domainCatchAll}) res.IsCatchAll = true res.Status = StatusRisky + res.SubStatus = SubStatusCatchAll res.Reason = "catch-all domain; acceptance is not conclusive" return res } + if role { + res.Status = StatusRisky + res.SubStatus = SubStatusRole + res.Reason = "role address (shared inbox); recipient accepted" + return res + } res.Status = StatusValid res.Reason = "recipient accepted" return res @@ -221,6 +316,27 @@ func (v *SMTPVerifier) Verify(ctx context.Context, email string) Result { } } +// probeHosts tries the MX hosts in preference order until one answers the +// session. A dial or handshake failure on the primary is common (it is the +// busiest host) and says nothing about the mailbox; only a host that talked +// to us gets to decide. +func (v *SMTPVerifier) probeHosts(ctx context.Context, hosts []string, localpart, domain string) probeResult { + var last probeResult + for i, host := range hosts { + if i >= maxMXAttempts { + break + } + if err := ctx.Err(); err != nil { + return probeResult{outcome: probeUnknown, reason: "cancelled: " + err.Error()} + } + last = v.probe(ctx, host, localpart, domain) + if last.outcome != probeUnknown || !last.sessionFailed { + return last + } + } + return last +} + // lookupMXHosts returns MX hosts ordered by ascending preference (most-preferred // first). When a domain publishes no MX, RFC 5321 permits implicit-MX fallback // to the A/AAAA record of the domain itself; we honour that so apex-only mail @@ -276,6 +392,9 @@ type probeResult struct { outcome probeOutcome catchAll bool reason string + // sessionFailed is set when the host never judged the address (dial or + // handshake failure), so the next MX is worth trying. + sessionFailed bool } // probe opens one SMTP session to host:25, greets it, sets the envelope sender, @@ -293,7 +412,7 @@ func (v *SMTPVerifier) probe(ctx context.Context, host, localpart, domain string if err != nil { // Most commonly: outbound :25 blocked by the cloud provider, or the MX is // firewalled/tarpitting. Either way we cannot conclude invalid. - return probeResult{outcome: probeUnknown, reason: "smtp dial failed (port 25 may be blocked): " + err.Error()} + return probeResult{outcome: probeUnknown, sessionFailed: true, reason: "smtp dial failed (port 25 may be blocked): " + err.Error()} } // Bound the whole session. deadline := time.Now().Add(v.cfg.CommandTimeout * 4) @@ -305,7 +424,7 @@ func (v *SMTPVerifier) probe(ctx context.Context, host, localpart, domain string client, err := smtp.NewClient(conn, host) if err != nil { _ = conn.Close() - return probeResult{outcome: probeUnknown, reason: "smtp handshake failed: " + err.Error()} + return probeResult{outcome: probeUnknown, sessionFailed: true, reason: "smtp handshake failed: " + err.Error()} } defer func() { _ = client.Close() }() diff --git a/internal/pkg/emailverify/fingerprint.go b/internal/pkg/emailverify/fingerprint.go new file mode 100644 index 00000000..b65580f4 --- /dev/null +++ b/internal/pkg/emailverify/fingerprint.go @@ -0,0 +1,133 @@ +package emailverify + +import ( + "strings" + "sync" + "time" +) + +const ( + // domainCacheTTL bounds how long a domain fact (no MX, catch-all, + // undisclosing provider) is trusted before it is re-learned. + domainCacheTTL = 24 * time.Hour + // domainCacheMax caps the in-process cache; the oldest entries are evicted. + domainCacheMax = 50_000 + // maxMXAttempts bounds how many MX hosts one address may cost. + maxMXAttempts = 3 +) + +type domainKind int + +const ( + domainNoMX domainKind = iota + 1 + domainCatchAll + domainUndisclosed +) + +type domainFact struct { + kind domainKind + reason string + at time.Time +} + +// domainCache is a small TTL map keyed by domain. In-process only: it is a +// cost optimisation, and a cold cache after a restart is merely slower. +type domainCache struct { + mu sync.Mutex + ttl time.Duration + data map[string]domainFact +} + +func newDomainCache(ttl time.Duration) *domainCache { + return &domainCache{ttl: ttl, data: make(map[string]domainFact)} +} + +func (c *domainCache) get(domain string) (domainFact, bool) { + if c == nil { + return domainFact{}, false + } + c.mu.Lock() + defer c.mu.Unlock() + f, ok := c.data[domain] + if !ok { + return domainFact{}, false + } + if time.Since(f.at) > c.ttl { + delete(c.data, domain) + return domainFact{}, false + } + return f, true +} + +func (c *domainCache) put(domain string, f domainFact) { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + if len(c.data) >= domainCacheMax { + // Evict the stalest quarter rather than scanning on every insert. + cutoff := time.Now().Add(-c.ttl / 4) + for k, v := range c.data { + if v.at.Before(cutoff) { + delete(c.data, k) + } + } + if len(c.data) >= domainCacheMax { + for k := range c.data { + delete(c.data, k) + if len(c.data) < domainCacheMax*3/4 { + break + } + } + } + } + f.at = time.Now() + c.data[domain] = f +} + +// mxFingerprint is what the MX hostnames reveal about the receiving provider. +type mxFingerprint struct { + name string + // undisclosed providers accept every RCPT and reject at delivery time, so + // an accepted probe proves nothing and a rejected one never happens. + undisclosed bool +} + +// undisclosingMX lists MX suffixes of providers whose RCPT answer is not a +// verdict. Microsoft 365 answers 250 for any address on a tenant and bounces +// later; Yahoo/AOL tarpit and accept. Google, by contrast, answers truthfully +// (550 5.1.1) and is left to the probe. +var undisclosingMX = []struct{ suffix, name string }{ + {".mail.protection.outlook.com", "Microsoft 365"}, + {".olc.protection.outlook.com", "Outlook.com"}, + {".mail.eo.outlook.com", "Microsoft 365"}, + {".yahoodns.net", "Yahoo"}, + {".mx.aol.com", "AOL"}, +} + +func fingerprintMX(hosts []string) mxFingerprint { + for _, h := range hosts { + lh := strings.ToLower(strings.TrimSuffix(h, ".")) + for _, u := range undisclosingMX { + if strings.HasSuffix(lh, u.suffix) { + return mxFingerprint{name: u.name, undisclosed: true} + } + } + } + return mxFingerprint{} +} + +// roleLocalparts is the same shared-inbox vocabulary the import quality check +// and the launch gate use, so one list is never described three ways. +var roleLocalparts = map[string]bool{ + "info": true, "sales": true, "support": true, "contact": true, + "admin": true, "hello": true, "help": true, "office": true, "team": true, + "billing": true, "careers": true, "jobs": true, "marketing": true, + "noreply": true, "no-reply": true, "webmaster": true, + "enquiries": true, "enquiry": true, "postmaster": true, "abuse": true, +} + +func isRoleLocalpart(local string) bool { + return roleLocalparts[strings.ToLower(local)] +} diff --git a/internal/pkg/emailverify/millionverifier.go b/internal/pkg/emailverify/millionverifier.go new file mode 100644 index 00000000..155df667 --- /dev/null +++ b/internal/pkg/emailverify/millionverifier.go @@ -0,0 +1,179 @@ +package emailverify + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +// MillionVerifier is the paid Verifier backed by MillionVerifier's single +// address API. Pay-as-you-go credits, one credit per lookup; the customer's +// own key is stored on their integration connection. +type MillionVerifier struct { + apiKey string + baseURL string + client *http.Client +} + +const ( + millionVerifierAPI = "https://api.millionverifier.com" + millionVerifierTimeout = 20 * time.Second + // millionVerifierProbeTimeout is the per-address server-side budget, in + // seconds, sent as the API's timeout parameter. + millionVerifierProbeTimeout = 15 +) + +var ( + // ErrMillionVerifierKey is returned when the API rejects the key. + ErrMillionVerifierKey = errors.New("millionverifier rejected the API key") + // ErrMillionVerifierCredits is returned when the account is out of credits. + ErrMillionVerifierCredits = errors.New("millionverifier account has no credits left") +) + +// NewMillionVerifier constructs the client. baseURL is overridable for tests. +func NewMillionVerifier(apiKey string, baseURL string) *MillionVerifier { + if baseURL == "" { + baseURL = millionVerifierAPI + } + return &MillionVerifier{ + apiKey: strings.TrimSpace(apiKey), + baseURL: strings.TrimRight(baseURL, "/"), + client: &http.Client{Timeout: millionVerifierTimeout}, + } +} + +type mvSingleResponse struct { + Email string `json:"email"` + Quality string `json:"quality"` + Result string `json:"result"` + ResultCode int `json:"resultcode"` + SubResult string `json:"subresult"` + Free bool `json:"free"` + Role bool `json:"role"` + Error string `json:"error"` + Credits int `json:"credits"` +} + +type mvCreditsResponse struct { + Credits int `json:"credits"` + Error string `json:"error"` +} + +// Verify implements Verifier. An API failure is never an "invalid" verdict: +// it degrades to unknown with the reason, and the typed error is reported +// through Check so the connection can be marked degraded. +func (m *MillionVerifier) Verify(ctx context.Context, email string) Result { + res, _ := m.Check(ctx, email) + return res +} + +// Check is Verify with the transport/account error surfaced. +func (m *MillionVerifier) Check(ctx context.Context, email string) (Result, error) { + now := time.Now().UTC() + res := Result{Email: strings.ToLower(strings.TrimSpace(email)), CheckedAt: now, Status: StatusUnknown, Provider: ProviderMillionVerifier} + + q := url.Values{} + q.Set("api", m.apiKey) + q.Set("email", res.Email) + q.Set("timeout", fmt.Sprintf("%d", millionVerifierProbeTimeout)) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, m.baseURL+"/api/v3/?"+q.Encode(), nil) + if err != nil { + res.Reason = "millionverifier request failed: " + err.Error() + return res, err + } + resp, err := m.client.Do(req) + if err != nil { + res.Reason = "millionverifier unreachable: " + err.Error() + return res, err + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10)) + if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden { + res.Reason = "millionverifier rejected the API key" + return res, ErrMillionVerifierKey + } + if resp.StatusCode != http.StatusOK { + res.Reason = fmt.Sprintf("millionverifier answered HTTP %d", resp.StatusCode) + return res, fmt.Errorf("millionverifier: http %d", resp.StatusCode) + } + var out mvSingleResponse + if err := json.Unmarshal(body, &out); err != nil { + res.Reason = "millionverifier answered with an unreadable body" + return res, err + } + if out.Error != "" { + return res, m.accountError(&res, out.Error) + } + + v, ok := NormalizeExternal(ProviderMillionVerifier, out.Result) + if !ok { + res.Reason = "millionverifier result not recognised: " + out.Result + return res, nil + } + res.Status, res.SubStatus = v.Status, v.SubStatus + res.IsCatchAll = v.SubStatus == SubStatusCatchAll + res.HasMX = out.Result != "invalid" || out.SubResult != "" + if out.Role && res.Status == StatusValid { + res.Status = StatusRisky + res.SubStatus = SubStatusRole + } + reason := out.Result + if out.SubResult != "" { + reason += " (" + out.SubResult + ")" + } + res.Reason = "millionverifier: " + reason + return res, nil +} + +// Credits returns the account's remaining credits, and validates the key. +func (m *MillionVerifier) Credits(ctx context.Context) (int, error) { + q := url.Values{} + q.Set("api", m.apiKey) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, m.baseURL+"/api/v3/credits?"+q.Encode(), nil) + if err != nil { + return 0, err + } + resp, err := m.client.Do(req) + if err != nil { + return 0, err + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10)) + if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden { + return 0, ErrMillionVerifierKey + } + if resp.StatusCode != http.StatusOK { + return 0, fmt.Errorf("millionverifier: http %d", resp.StatusCode) + } + var out mvCreditsResponse + if err := json.Unmarshal(body, &out); err != nil { + return 0, err + } + if out.Error != "" { + return 0, m.accountError(nil, out.Error) + } + return out.Credits, nil +} + +func (m *MillionVerifier) accountError(res *Result, msg string) error { + lower := strings.ToLower(msg) + var err error + switch { + case strings.Contains(lower, "api key") || strings.Contains(lower, "apikey") || strings.Contains(lower, "unauthori"): + err = ErrMillionVerifierKey + case strings.Contains(lower, "credit"): + err = ErrMillionVerifierCredits + default: + err = fmt.Errorf("millionverifier: %s", msg) + } + if res != nil { + res.Reason = "millionverifier: " + msg + } + return err +} diff --git a/internal/pkg/emailverify/millionverifier_test.go b/internal/pkg/emailverify/millionverifier_test.go new file mode 100644 index 00000000..f975cd29 --- /dev/null +++ b/internal/pkg/emailverify/millionverifier_test.go @@ -0,0 +1,60 @@ +package emailverify + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" +) + +func TestMillionVerifierMapsResults(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("api") != "key" { + http.Error(w, `{"error":"api key not found"}`, http.StatusOK) + return + } + switch r.URL.Path { + case "/api/v3/credits": + _, _ = w.Write([]byte(`{"credits": 42}`)) + case "/api/v3/": + switch r.URL.Query().Get("email") { + case "good@x.com": + _, _ = w.Write([]byte(`{"email":"good@x.com","quality":"good","result":"ok","resultcode":1,"subresult":"","free":false,"role":false,"error":"","credits":41}`)) + case "info@x.com": + _, _ = w.Write([]byte(`{"email":"info@x.com","quality":"good","result":"ok","resultcode":1,"subresult":"","free":false,"role":true,"error":"","credits":40}`)) + case "gone@x.com": + _, _ = w.Write([]byte(`{"email":"gone@x.com","quality":"bad","result":"invalid","resultcode":6,"subresult":"user_unknown","free":false,"role":false,"error":"","credits":39}`)) + default: + _, _ = w.Write([]byte(`{"error":"insufficient credits"}`)) + } + } + })) + defer srv.Close() + + mv := NewMillionVerifier("key", srv.URL) + if n, err := mv.Credits(context.Background()); err != nil || n != 42 { + t.Fatalf("credits = %d, %v", n, err) + } + res, err := mv.Check(context.Background(), "good@x.com") + if err != nil || res.Status != StatusValid || res.Provider != ProviderMillionVerifier { + t.Fatalf("good = %+v, %v", res, err) + } + res, _ = mv.Check(context.Background(), "info@x.com") + if res.Status != StatusRisky || res.SubStatus != SubStatusRole { + t.Fatalf("role = %+v", res) + } + res, _ = mv.Check(context.Background(), "gone@x.com") + if res.Status != StatusInvalid || res.Reason != "millionverifier: invalid (user_unknown)" { + t.Fatalf("gone = %+v", res) + } + res, err = mv.Check(context.Background(), "broke@x.com") + if !errors.Is(err, ErrMillionVerifierCredits) || res.Status != StatusUnknown { + t.Fatalf("no credits = %+v, %v", res, err) + } + + bad := NewMillionVerifier("wrong", srv.URL) + if _, err := bad.Credits(context.Background()); !errors.Is(err, ErrMillionVerifierKey) { + t.Fatalf("bad key = %v", err) + } +} diff --git a/internal/pkg/emailverify/vocab.go b/internal/pkg/emailverify/vocab.go new file mode 100644 index 00000000..5a6b24f9 --- /dev/null +++ b/internal/pkg/emailverify/vocab.go @@ -0,0 +1,233 @@ +package emailverify + +import "strings" + +// External verification vocabularies. A customer who verified a list with +// another service brings a status column along; these tables read the words +// each service writes and fold them into Warmbly's four statuses, so nobody +// has to pick a provider or translate a column by hand. + +// ExternalVerdict is one recognised external status. +type ExternalVerdict struct { + Status Status + SubStatus SubStatus +} + +// vocabularies maps a provider name to the statuses it emits. Keys are +// lower-cased with spaces, dashes and underscores removed (see vocabKey). +var vocabularies = map[string]map[string]ExternalVerdict{ + ProviderMillionVerifier: { + "ok": {Status: StatusValid}, + "good": {Status: StatusValid}, + "catchall": {Status: StatusRisky, SubStatus: SubStatusCatchAll}, + "unknown": {Status: StatusUnknown}, + "error": {Status: StatusUnknown}, + "disposable": {Status: StatusInvalid, SubStatus: SubStatusDisposable}, + "invalid": {Status: StatusInvalid}, + "bad": {Status: StatusInvalid}, + "risky": {Status: StatusRisky}, + }, + "zerobounce": { + "valid": {Status: StatusValid}, + "invalid": {Status: StatusInvalid}, + "catchall": {Status: StatusRisky, SubStatus: SubStatusCatchAll}, + "spamtrap": {Status: StatusInvalid, SubStatus: SubStatusSpamTrap}, + "abuse": {Status: StatusInvalid, SubStatus: SubStatusSpamTrap}, + "donotmail": {Status: StatusInvalid}, + "unknown": {Status: StatusUnknown}, + "disposable": {Status: StatusInvalid, SubStatus: SubStatusDisposable}, + "toxic": {Status: StatusInvalid, SubStatus: SubStatusSpamTrap}, + "rolebased": {Status: StatusRisky, SubStatus: SubStatusRole}, + "mailboxfull": {Status: StatusRisky, SubStatus: SubStatusMailboxFull}, + }, + "neverbounce": { + "valid": {Status: StatusValid}, + "invalid": {Status: StatusInvalid}, + "disposable": {Status: StatusInvalid, SubStatus: SubStatusDisposable}, + "catchall": {Status: StatusRisky, SubStatus: SubStatusCatchAll}, + "unknown": {Status: StatusUnknown}, + }, + "bouncer": { + "deliverable": {Status: StatusValid}, + "undeliverable": {Status: StatusInvalid}, + "risky": {Status: StatusRisky}, + "unknown": {Status: StatusUnknown}, + "acceptall": {Status: StatusRisky, SubStatus: SubStatusCatchAll}, + "disposable": {Status: StatusInvalid, SubStatus: SubStatusDisposable}, + }, + "kickbox": { + "deliverable": {Status: StatusValid}, + "undeliverable": {Status: StatusInvalid}, + "risky": {Status: StatusRisky}, + "unknown": {Status: StatusUnknown}, + "acceptall": {Status: StatusRisky, SubStatus: SubStatusCatchAll}, + }, + "emailable": { + "deliverable": {Status: StatusValid}, + "undeliverable": {Status: StatusInvalid}, + "risky": {Status: StatusRisky}, + "unknown": {Status: StatusUnknown}, + "acceptall": {Status: StatusRisky, SubStatus: SubStatusCatchAll}, + }, + "debounce": { + "safetosend": {Status: StatusValid}, + "valid": {Status: StatusValid}, + "deliverable": {Status: StatusValid}, + "invalid": {Status: StatusInvalid}, + "disposable": {Status: StatusInvalid, SubStatus: SubStatusDisposable}, + "spamtrap": {Status: StatusInvalid, SubStatus: SubStatusSpamTrap}, + "acceptall": {Status: StatusRisky, SubStatus: SubStatusCatchAll}, + "catchall": {Status: StatusRisky, SubStatus: SubStatusCatchAll}, + "role": {Status: StatusRisky, SubStatus: SubStatusRole}, + "unknown": {Status: StatusUnknown}, + "risky": {Status: StatusRisky}, + "undeliverable": {Status: StatusInvalid}, + }, + "clearout": { + "valid": {Status: StatusValid}, + "invalid": {Status: StatusInvalid}, + "catchall": {Status: StatusRisky, SubStatus: SubStatusCatchAll}, + "unknown": {Status: StatusUnknown}, + }, + "emaillistverify": { + "ok": {Status: StatusValid}, + "valid": {Status: StatusValid}, + "okforall": {Status: StatusRisky, SubStatus: SubStatusCatchAll}, + "acceptall": {Status: StatusRisky, SubStatus: SubStatusCatchAll}, + "catchall": {Status: StatusRisky, SubStatus: SubStatusCatchAll}, + "invalid": {Status: StatusInvalid}, + "invalidmx": {Status: StatusInvalid, SubStatus: SubStatusNoMX}, + "invalidsyntax": {Status: StatusInvalid, SubStatus: SubStatusSyntax}, + "emaildisabled": {Status: StatusInvalid}, + "deadserver": {Status: StatusInvalid, SubStatus: SubStatusNoMX}, + "disposable": {Status: StatusInvalid, SubStatus: SubStatusDisposable}, + "spamtrap": {Status: StatusInvalid, SubStatus: SubStatusSpamTrap}, + "unknown": {Status: StatusUnknown}, + "role": {Status: StatusRisky, SubStatus: SubStatusRole}, + "smtpprotocol": {Status: StatusUnknown}, + "antispamsystem": {Status: StatusUnknown}, + "unknownerror": {Status: StatusUnknown}, + "attemptrejected": {Status: StatusUnknown}, + "relaydenied": {Status: StatusUnknown}, + "mailboxfull": {Status: StatusRisky, SubStatus: SubStatusMailboxFull}, + "greylisted": {Status: StatusUnknown}, + "noresponse": {Status: StatusUnknown}, + "ipblocked": {Status: StatusUnknown}, + "servicenotavailable": {Status: StatusUnknown}, + }, + // Warmbly's own statuses, so a Warmbly export re-imports losslessly. + ProviderBuiltin: { + "valid": {Status: StatusValid}, + "risky": {Status: StatusRisky}, + "invalid": {Status: StatusInvalid}, + "unknown": {Status: StatusUnknown}, + }, +} + +// vocabProviderOrder decides which provider wins when several recognise every +// value. Specific vocabularies first; the generic one last. +var vocabProviderOrder = []string{ + "zerobounce", ProviderMillionVerifier, "neverbounce", "emaillistverify", + "debounce", "bouncer", "kickbox", "emailable", "clearout", ProviderBuiltin, +} + +// vocabProviderAliases maps how a provider is written in a column header to +// its vocabulary key. +var vocabProviderAliases = map[string]string{ + "millionverifier": ProviderMillionVerifier, "mv": ProviderMillionVerifier, + "zerobounce": "zerobounce", "zb": "zerobounce", + "neverbounce": "neverbounce", "nb": "neverbounce", + "bouncer": "bouncer", "usebouncer": "bouncer", + "kickbox": "kickbox", + "emailable": "emailable", + "debounce": "debounce", + "clearout": "clearout", + "emaillistverify": "emaillistverify", "elv": "emaillistverify", + "warmbly": ProviderBuiltin, "builtin": ProviderBuiltin, +} + +func vocabKey(raw string) string { + k := strings.ToLower(strings.TrimSpace(raw)) + return strings.NewReplacer(" ", "", "_", "", "-", "", ".", "").Replace(k) +} + +// KnownVocabulary reports whether name (as written by a customer or an API +// caller) is a vocabulary this package can read, and returns its key. +func KnownVocabulary(name string) (string, bool) { + k, ok := vocabProviderAliases[vocabKey(name)] + return k, ok +} + +// NormalizeExternal reads one external status. provider may be empty, in +// which case every vocabulary is consulted and the first that recognises the +// value wins. Returns false for a value nobody recognises. +func NormalizeExternal(provider, raw string) (ExternalVerdict, bool) { + key := vocabKey(raw) + if key == "" { + return ExternalVerdict{}, false + } + if provider != "" { + p, ok := KnownVocabulary(provider) + if !ok { + return ExternalVerdict{}, false + } + v, ok := vocabularies[p][key] + return v, ok + } + for _, p := range vocabProviderOrder { + if v, ok := vocabularies[p][key]; ok { + return v, ok + } + } + return ExternalVerdict{}, false +} + +// DetectVocabulary looks at a column's values and names the provider whose +// vocabulary they all belong to. Blank cells are ignored; a column with no +// non-blank cell, or with one value nobody knows, is not a status column. +func DetectVocabulary(values []string) (string, bool) { + seen := 0 + for _, p := range vocabProviderOrder { + all := true + seen = 0 + for _, raw := range values { + key := vocabKey(raw) + if key == "" { + continue + } + seen++ + if _, ok := vocabularies[p][key]; !ok { + all = false + break + } + } + if all && seen > 0 { + return p, true + } + } + return "", false +} + +// IsStatusHeader reports whether a column header reads like a verification +// status column, and names the provider if the header says so. +func IsStatusHeader(header string) (provider string, ok bool) { + k := vocabKey(header) + if k == "" { + return "", false + } + for alias, p := range vocabProviderAliases { + if strings.HasPrefix(k, alias) { + rest := strings.TrimPrefix(k, alias) + if rest == "" || strings.Contains(rest, "status") || strings.Contains(rest, "result") || strings.Contains(rest, "verif") || strings.Contains(rest, "quality") { + return p, true + } + } + } + switch k { + case "verificationstatus", "verification", "verified", "emailstatus", "emailverification", + "verifystatus", "verificationresult", "verifyresult", "validationstatus", "validation", + "emailvalidation", "deliverability", "deliverabilitystatus", "result", "status", "quality": + return "", true + } + return "", false +} diff --git a/internal/pkg/emailverify/vocab_test.go b/internal/pkg/emailverify/vocab_test.go new file mode 100644 index 00000000..f3c51812 --- /dev/null +++ b/internal/pkg/emailverify/vocab_test.go @@ -0,0 +1,84 @@ +package emailverify + +import "testing" + +func TestNormalizeExternalReadsEveryKnownVocabulary(t *testing.T) { + cases := []struct { + provider, raw string + want Status + sub SubStatus + }{ + {"zerobounce", "valid", StatusValid, SubStatusNone}, + {"zerobounce", "catch-all", StatusRisky, SubStatusCatchAll}, + {"zerobounce", "spamtrap", StatusInvalid, SubStatusSpamTrap}, + {"millionverifier", "ok", StatusValid, SubStatusNone}, + {"millionverifier", "catch_all", StatusRisky, SubStatusCatchAll}, + {"millionverifier", "disposable", StatusInvalid, SubStatusDisposable}, + {"neverbounce", "catchall", StatusRisky, SubStatusCatchAll}, + {"bouncer", "deliverable", StatusValid, SubStatusNone}, + {"kickbox", "accept-all", StatusRisky, SubStatusCatchAll}, + {"emaillistverify", "ok_for_all", StatusRisky, SubStatusCatchAll}, + {"", "Safe to Send", StatusValid, SubStatusNone}, + {"", "INVALID", StatusInvalid, SubStatusNone}, + {"", "unknown", StatusUnknown, SubStatusNone}, + } + for _, c := range cases { + got, ok := NormalizeExternal(c.provider, c.raw) + if !ok { + t.Fatalf("%s/%q not recognised", c.provider, c.raw) + } + if got.Status != c.want || got.SubStatus != c.sub { + t.Fatalf("%s/%q = %v/%v, want %v/%v", c.provider, c.raw, got.Status, got.SubStatus, c.want, c.sub) + } + } + if _, ok := NormalizeExternal("", "qualified lead"); ok { + t.Fatal("a CRM stage must not read as a verdict") + } + if _, ok := NormalizeExternal("nosuchprovider", "valid"); ok { + t.Fatal("an unknown provider must not be read") + } +} + +func TestDetectVocabularyNamesTheProviderOrRefuses(t *testing.T) { + if p, ok := DetectVocabulary([]string{"valid", "catch-all", "", "do_not_mail"}); !ok || p != "zerobounce" { + t.Fatalf("zerobounce column = %q/%v", p, ok) + } + if p, ok := DetectVocabulary([]string{"ok", "catch_all", "unknown"}); !ok || p != ProviderMillionVerifier { + t.Fatalf("millionverifier column = %q/%v", p, ok) + } + if _, ok := DetectVocabulary([]string{"valid", "won", "lost"}); ok { + t.Fatal("mixed column must not be a verdict column") + } + if _, ok := DetectVocabulary([]string{"", ""}); ok { + t.Fatal("blank column must not be a verdict column") + } +} + +func TestIsStatusHeader(t *testing.T) { + if p, ok := IsStatusHeader("ZeroBounce Status"); !ok || p != "zerobounce" { + t.Fatalf("header = %q/%v", p, ok) + } + if p, ok := IsStatusHeader("verification_status"); !ok || p != "" { + t.Fatalf("generic header = %q/%v", p, ok) + } + if _, ok := IsStatusHeader("Deal stage"); ok { + t.Fatal("deal stage is not a status header") + } +} + +func TestFingerprintMXNamesUndisclosingProviders(t *testing.T) { + if fp := fingerprintMX([]string{"acme-com.mail.protection.outlook.com"}); !fp.undisclosed || fp.name != "Microsoft 365" { + t.Fatalf("m365 = %+v", fp) + } + if fp := fingerprintMX([]string{"aspmx.l.google.com"}); fp.undisclosed { + t.Fatalf("google must be probed: %+v", fp) + } +} + +func TestDomainCacheExpires(t *testing.T) { + c := newDomainCache(0) + c.put("a.com", domainFact{kind: domainCatchAll}) + if _, ok := c.get("a.com"); ok { + t.Fatal("zero ttl entry must be expired") + } +} diff --git a/internal/repository/pg_admin.go b/internal/repository/pg_admin.go index d0a55f66..3eb25d09 100644 --- a/internal/repository/pg_admin.go +++ b/internal/repository/pg_admin.go @@ -1606,7 +1606,7 @@ func (r *adminRepository) StopCampaign(ctx context.Context, campaignID uuid.UUID UPDATE campaigns SET status = 'paused', last_status_change_at = NOW(), updated_at = NOW() WHERE id = $1 - AND status IN ('active', 'paused', 'paused_no_accounts', 'paused_trial_expired', 'paused_guardrail') + AND status IN ('active', 'paused', 'paused_no_accounts', 'paused_trial_expired', 'paused_guardrail', 'paused_undeliverable') `, campaignID) if err != nil { return false, err diff --git a/internal/repository/pg_campaign.go b/internal/repository/pg_campaign.go index 5cfe42f9..b9e318fe 100644 --- a/internal/repository/pg_campaign.go +++ b/internal/repository/pg_campaign.go @@ -68,6 +68,8 @@ type CampaignRepository interface { // reply routing a contact onto a live branch). The reconciler re-checks each. ListStaleParkedCampaigns(ctx context.Context, staleAfter time.Duration, limit int) ([]ParkedCampaignTask, error) CountActiveForOrganization(ctx context.Context, orgID uuid.UUID) (int, error) + // ListIDsByStatus lists the org's campaigns parked at one status. + ListIDsByStatus(ctx context.Context, orgID uuid.UUID, status string) ([]uuid.UUID, error) AccountHasActiveCampaign(ctx context.Context, accountID uuid.UUID) (bool, error) // CountActiveCampaignsForAccount returns how many active campaigns send // from the given mailbox (matched through the campaign's email tags OR an @@ -833,12 +835,13 @@ func (r *campaignRepository) Update(ctx context.Context, userID, campaignID stri } if data.Status != nil { // Valid statuses: draft, active, paused, completed, paused_trial_expired, - // paused_no_accounts, paused_guardrail + // paused_no_accounts, paused_guardrail, paused_undeliverable status := *data.Status validStatuses := map[string]bool{ "draft": true, "active": true, "paused": true, "completed": true, "paused_trial_expired": true, "paused_no_accounts": true, "paused_guardrail": true, + "paused_undeliverable": true, } if !validStatuses[status] { return nil, errx.ErrInvalid @@ -1331,10 +1334,13 @@ func (r *campaignRepository) GetSequencesRoutingByCampaignID(ctx context.Context // validCampaignTransitions defines which status transitions are allowed. // Key is the current status, values are the statuses it can transition to. var validCampaignTransitions = map[string]map[string]bool{ - "draft": {"active": true}, - "active": {"paused": true, "completed": true, "paused_no_accounts": true, "paused_trial_expired": true, "paused_guardrail": true}, - "paused": {"active": true, "draft": true}, - "paused_no_accounts": {"active": true, "paused": true}, + "draft": {"active": true}, + "active": {"paused": true, "completed": true, "paused_no_accounts": true, "paused_trial_expired": true, "paused_guardrail": true, "paused_undeliverable": true}, + "paused": {"active": true, "draft": true}, + "paused_no_accounts": {"active": true, "paused": true}, + // Parked because verification refused every remaining lead; resumes once + // they are re-verified or marked deliverable. + "paused_undeliverable": {"active": true, "paused": true}, "paused_trial_expired": {"active": true, "paused": true}, // An auto-pause is resumable, but only deliberately: the owner has to // restart the campaign (or park it) after looking at why it tripped. @@ -1563,6 +1569,24 @@ func (r *campaignRepository) CountActiveForOrganization(ctx context.Context, org return count, err } +// ListIDsByStatus lists the org's campaigns parked at one status. +func (r *campaignRepository) ListIDsByStatus(ctx context.Context, orgID uuid.UUID, status string) ([]uuid.UUID, error) { + rows, err := r.DB.Query(ctx, `SELECT id FROM campaigns WHERE organization_id = $1 AND status = $2`, orgID, status) + if err != nil { + return nil, err + } + defer rows.Close() + var out []uuid.UUID + for rows.Next() { + var id uuid.UUID + if err := rows.Scan(&id); err != nil { + return nil, err + } + out = append(out, id) + } + return out, rows.Err() +} + // AccountHasActiveCampaign reports whether the mailbox backs at least one active // campaign, counting BOTH tag-based campaigns AND explicit-sender campaigns // (campaign_senders). The explicit lane matters for the warmup health-check diff --git a/internal/repository/pg_contact.go b/internal/repository/pg_contact.go index 4003146d..61bf9f78 100644 --- a/internal/repository/pg_contact.go +++ b/internal/repository/pg_contact.go @@ -41,7 +41,20 @@ type ContactRepository interface { // have never been conclusively checked (status 'unknown', never verified) so // the batch scheduler can work them off a cap per tick. UpdateContactVerification(ctx context.Context, contactID uuid.UUID, res emailverify.Result) *errx.Error - ListUnverifiedContacts(ctx context.Context, limit int) ([]models.Contact, *errx.Error) + // ListVerificationCandidates returns contacts due for a check: never + // checked, or checked long enough ago that the verdict has aged out. + // Manual verdicts are never candidates. Oldest first. + ListVerificationCandidates(ctx context.Context, limit int) ([]VerificationCandidate, *errx.Error) + // SetContactsVerification stores one verdict on many of the org's contacts + // (a manual "mark deliverable"). Returns how many rows changed. + SetContactsVerification(ctx context.Context, orgID uuid.UUID, ids []uuid.UUID, w models.ContactVerificationWrite) (int, *errx.Error) + // ResetContactsVerification clears the verdict so the scheduler checks the + // contacts again on its next pass. Returns how many rows changed. + ResetContactsVerification(ctx context.Context, orgID uuid.UUID, ids []uuid.UUID) (int, *errx.Error) + // UndeliverableLeadIDs lists the campaign's leads verification refused. + UndeliverableLeadIDs(ctx context.Context, orgID, campaignID uuid.UUID) ([]uuid.UUID, *errx.Error) + // VerificationCounts is the org's contacts by verdict. + VerificationCounts(ctx context.Context, orgID uuid.UUID) (models.ContactVerificationCounts, *errx.Error) // SetContactESP caches the recipient ESP/provider resolved from the contact's // domain (control-plane only, no MX dial). Best-effort: a failure should not // block sending. @@ -233,6 +246,13 @@ func (r *contactRepository) Add(ctx context.Context, userID string, orgID uuid.U } lead.SourceDetail = strings.TrimSpace(lead.SourceDetail) + // A verdict the caller brought along, in any vocabulary we can read. + if v, xerr := verificationFromRequest(lead.VerificationStatus, lead.VerificationProvider); xerr != nil { + return nil, xerr + } else if v != nil { + lead.Verification = v + } + normalized = append(normalized, lead) campaignIDs = append(campaignIDs, cids) categoryIDs = append(categoryIDs, cats) @@ -248,6 +268,10 @@ func (r *contactRepository) Add(ctx context.Context, userID string, orgID uuid.U // Upsert contacts in a single batch round-trip. insertBatch := pgx.Batch{} for _, lead := range normalized { + var vStatus, vSub, vReason, vProvider string + if lead.Verification != nil { + vStatus, vSub, vReason, vProvider = lead.Verification.Status, lead.Verification.SubStatus, lead.Verification.Reason, lead.Verification.Provider + } insertBatch.Queue( // $9 and $10 are the same value: a parameter used both as an // INSERT value and inside the DO UPDATE set gives Postgres two @@ -255,10 +279,16 @@ func (r *contactRepository) Add(ctx context.Context, userID string, orgID uuid.U // explicit casts. NULL means "leave the flag alone". `INSERT INTO contacts ( id, user_id, organization_id, first_name, last_name, email, company, phone, custom_fields, subscribed, - source, source_detail, first_seen_at + source, source_detail, first_seen_at, + verification_status, verification_sub_status, verification_reason, verification_provider, + verification_source, is_catch_all, verification_checked_at ) VALUES ( gen_random_uuid(), $1, $2, $3, $4, LOWER($5), $6, $7, $8, COALESCE($9::boolean, TRUE), - $11, $12, NOW() + $11, $12, NOW(), + COALESCE(NULLIF($13::text, ''), 'unknown'), $14, $15, $16, + CASE WHEN $13::text <> '' THEN 'imported' ELSE '' END, + ($14::text = 'catch_all'), + CASE WHEN $13::text <> '' THEN NOW() ELSE NULL END ) ON CONFLICT (user_id, (LOWER(email))) DO UPDATE SET organization_id = COALESCE(contacts.organization_id, EXCLUDED.organization_id), @@ -270,12 +300,22 @@ func (r *contactRepository) Add(ctx context.Context, userID string, orgID uuid.U phone = COALESCE(NULLIF(EXCLUDED.phone, ''), contacts.phone), custom_fields = contacts.custom_fields || EXCLUDED.custom_fields, subscribed = COALESCE($10::boolean, contacts.subscribed), + -- A verdict in the file replaces whatever was recorded; a file + -- without one leaves the existing verdict alone. + verification_status = CASE WHEN $13::text <> '' THEN $13 ELSE contacts.verification_status END, + verification_sub_status = CASE WHEN $13::text <> '' THEN $14 ELSE contacts.verification_sub_status END, + verification_reason = CASE WHEN $13::text <> '' THEN $15 ELSE contacts.verification_reason END, + verification_provider = CASE WHEN $13::text <> '' THEN $16 ELSE contacts.verification_provider END, + verification_source = CASE WHEN $13::text <> '' THEN 'imported' ELSE contacts.verification_source END, + is_catch_all = CASE WHEN $13::text <> '' THEN ($14::text = 'catch_all') ELSE contacts.is_catch_all END, + verification_checked_at = CASE WHEN $13::text <> '' THEN NOW() ELSE contacts.verification_checked_at END, updated_at = NOW() -- xmax = 0 only on a fresh row: the source is first-touch, so an -- upsert that hit an existing contact is not a creation. RETURNING id, first_name, last_name, email, company, phone, custom_fields, subscribed, updated_at, created_at, (xmax = 0)`, userID, orgID, lead.FirstName, lead.LastName, lead.Email, lead.Company, lead.Phone, lead.CustomFields, lead.Subscribed, lead.Subscribed, string(lead.Source), lead.SourceDetail, + vStatus, vSub, vReason, vProvider, ) } @@ -469,6 +509,7 @@ func (r *contactRepository) GetByID(ctx context.Context, contactID uuid.UUID) (* c.id, c.first_name, c.last_name, c.email, c.company, c.phone, c.custom_fields, c.subscribed, c.updated_at, c.created_at, c.verification_status, c.verification_reason, c.is_catch_all, c.verification_checked_at, + c.verification_source, c.verification_provider, c.verification_sub_status, c.esp_provider, c.esp_resolved_at FROM contacts c WHERE c.id = $1 @@ -480,6 +521,7 @@ func (r *contactRepository) GetByID(ctx context.Context, contactID uuid.UUID) (* &contact.Company, &contact.Phone, &contact.CustomFields, &contact.Subscribed, &contact.UpdatedAt, &contact.CreatedAt, &contact.VerificationStatus, &contact.VerificationReason, &contact.IsCatchAll, &contact.VerificationCheckedAt, + &contact.VerificationSource, &contact.VerificationProvider, &contact.VerificationSubStatus, &contact.ESPProvider, &contact.ESPResolvedAt, ) if err != nil { @@ -520,6 +562,14 @@ func (r *contactRepository) UpdateContactVerification(ctx context.Context, conta if checkedAt.IsZero() { checkedAt = time.Now().UTC() } + provider := res.Provider + if provider == "" { + provider = emailverify.ProviderBuiltin + } + source := models.VerificationSourceProvider + if provider == emailverify.ProviderBuiltin { + source = models.VerificationSourceProbe + } query := ` UPDATE contacts @@ -527,10 +577,13 @@ func (r *contactRepository) UpdateContactVerification(ctx context.Context, conta verification_reason = $3, is_catch_all = $4, verification_checked_at = $5, + verification_source = $6, + verification_provider = $7, + verification_sub_status = $8, updated_at = NOW() WHERE id = $1 ` - params := []any{contactID, status, res.Reason, res.IsCatchAll, checkedAt} + params := []any{contactID, status, res.Reason, res.IsCatchAll, checkedAt, source, provider, string(res.SubStatus)} cmd, err := r.DB.Exec(ctx, query, params...) if err != nil { db.CaptureError(err, query, params, "exec") @@ -542,55 +595,170 @@ func (r *contactRepository) UpdateContactVerification(ctx context.Context, conta return nil } -// ListUnverifiedContacts returns up to `limit` contacts that have never been -// conclusively verified (status 'unknown' and no recorded check). Oldest -// contacts first so a backlog drains in creation order. The pre-send gate only -// drops 'invalid', so 'risky'/'valid'/already-checked rows are intentionally -// excluded here — they don't need re-verification on every tick. -func (r *contactRepository) ListUnverifiedContacts(ctx context.Context, limit int) ([]models.Contact, *errx.Error) { +// VerificationCandidate is one contact due for a verification check. +type VerificationCandidate struct { + ID uuid.UUID + OrganizationID uuid.UUID + Email string + // Requested is true when a member asked for this check (the verdict was + // reset), so it is worth spending a paid credit on even when the + // organization has none to spare. + Requested bool +} + +// ListVerificationCandidates returns up to `limit` contacts due for a check. +// Never-checked contacts come first (a reset counts as never checked), then +// verdicts older than their shelf life: an unknown verdict is retried after +// config.VerificationUnknownRecheckDays, everything else after +// config.VerificationRecheckDays. Manual verdicts are never re-checked. +func (r *contactRepository) ListVerificationCandidates(ctx context.Context, limit int) ([]VerificationCandidate, *errx.Error) { if limit <= 0 { limit = 100 } query := ` - SELECT - c.id, c.first_name, c.last_name, c.email, c.company, c.phone, - c.custom_fields, c.subscribed, c.updated_at, c.created_at, - c.verification_status, c.verification_reason, c.is_catch_all, c.verification_checked_at + SELECT c.id, c.organization_id, c.email, c.verification_checked_at IS NULL FROM contacts c - WHERE c.verification_status = 'unknown' AND c.verification_checked_at IS NULL - ORDER BY c.created_at ASC + WHERE c.organization_id IS NOT NULL + AND c.verification_source <> 'manual' + AND ( + c.verification_checked_at IS NULL + OR (c.verification_status = 'unknown' AND c.verification_checked_at < NOW() - make_interval(days => $2)) + OR c.verification_checked_at < NOW() - make_interval(days => $3) + ) + ORDER BY c.verification_checked_at ASC NULLS FIRST, c.created_at ASC LIMIT $1 ` - rows, err := r.DB.Query(ctx, query, limit) + params := []any{limit, config.VerificationUnknownRecheckDays, config.VerificationRecheckDays} + rows, err := r.DB.Query(ctx, query, params...) if err != nil { - db.CaptureError(err, query, []any{limit}, "query") + db.CaptureError(err, query, params, "query") return nil, errx.InternalError() } defer rows.Close() - out := make([]models.Contact, 0, limit) + out := make([]VerificationCandidate, 0, limit) for rows.Next() { - var c models.Contact - if err := rows.Scan( - &c.ID, &c.FirstName, &c.LastName, &c.Email, - &c.Company, &c.Phone, &c.CustomFields, &c.Subscribed, - &c.UpdatedAt, &c.CreatedAt, - &c.VerificationStatus, &c.VerificationReason, &c.IsCatchAll, &c.VerificationCheckedAt, - ); err != nil { - db.CaptureError(err, "", nil, "ListUnverifiedContacts scan") + var c VerificationCandidate + if err := rows.Scan(&c.ID, &c.OrganizationID, &c.Email, &c.Requested); err != nil { + db.CaptureError(err, "", nil, "ListVerificationCandidates scan") return nil, errx.InternalError() } - c.Campaigns = []models.MiniCampaign{} - c.Categories = []models.MiniCategory{} out = append(out, c) } if err := rows.Err(); err != nil { - db.CaptureError(err, "", nil, "ListUnverifiedContacts rows") + db.CaptureError(err, "", nil, "ListVerificationCandidates rows") return nil, errx.InternalError() } return out, nil } +// SetContactsVerification writes one verdict onto the org's listed contacts. +func (r *contactRepository) SetContactsVerification(ctx context.Context, orgID uuid.UUID, ids []uuid.UUID, w models.ContactVerificationWrite) (int, *errx.Error) { + if len(ids) == 0 { + return 0, nil + } + query := ` + UPDATE contacts + SET verification_status = $3, + verification_sub_status = $4, + verification_reason = $5, + verification_provider = $6, + verification_source = $7, + is_catch_all = ($4 = 'catch_all'), + verification_checked_at = NOW(), + updated_at = NOW() + WHERE organization_id = $1 AND id = ANY($2) + ` + params := []any{orgID, ids, w.Status, w.SubStatus, w.Reason, w.Provider, w.Source} + cmd, err := r.DB.Exec(ctx, query, params...) + if err != nil { + db.CaptureError(err, query, params, "exec") + return 0, errx.InternalError() + } + return int(cmd.RowsAffected()), nil +} + +// ResetContactsVerification returns the org's listed contacts to "never +// checked" so the next scheduler pass picks them up first. +func (r *contactRepository) ResetContactsVerification(ctx context.Context, orgID uuid.UUID, ids []uuid.UUID) (int, *errx.Error) { + if len(ids) == 0 { + return 0, nil + } + query := ` + UPDATE contacts + SET verification_status = 'unknown', + verification_sub_status = '', + verification_reason = 'verification requested', + verification_provider = '', + verification_source = '', + is_catch_all = false, + verification_checked_at = NULL, + updated_at = NOW() + WHERE organization_id = $1 AND id = ANY($2) + ` + params := []any{orgID, ids} + cmd, err := r.DB.Exec(ctx, query, params...) + if err != nil { + db.CaptureError(err, query, params, "exec") + return 0, errx.InternalError() + } + return int(cmd.RowsAffected()), nil +} + +// UndeliverableLeadIDs lists the campaign's leads the routing predicate skips +// for verification reasons (invalid, or risky with the risky toggle off). +func (r *contactRepository) UndeliverableLeadIDs(ctx context.Context, orgID, campaignID uuid.UUID) ([]uuid.UUID, *errx.Error) { + query := ` + SELECT c.id + FROM campaign_leads cl + JOIN contacts c ON c.id = cl.contact_id + JOIN campaigns cp ON cp.id = cl.campaign_id + WHERE cl.campaign_id = $1 AND cp.organization_id = $2 + AND (c.verification_status = 'invalid' OR (c.verification_status = 'risky' AND NOT cp.risky_emails)) + ` + params := []any{campaignID, orgID} + rows, err := r.DB.Query(ctx, query, params...) + if err != nil { + db.CaptureError(err, query, params, "query") + return nil, errx.InternalError() + } + defer rows.Close() + var out []uuid.UUID + for rows.Next() { + var id uuid.UUID + if err := rows.Scan(&id); err != nil { + db.CaptureError(err, "", nil, "UndeliverableLeadIDs scan") + return nil, errx.InternalError() + } + out = append(out, id) + } + if err := rows.Err(); err != nil { + db.CaptureError(err, "", nil, "UndeliverableLeadIDs rows") + return nil, errx.InternalError() + } + return out, nil +} + +// VerificationCounts is the org's contacts by verdict. +func (r *contactRepository) VerificationCounts(ctx context.Context, orgID uuid.UUID) (models.ContactVerificationCounts, *errx.Error) { + var c models.ContactVerificationCounts + query := ` + SELECT + COUNT(*) FILTER (WHERE verification_status = 'valid'), + COUNT(*) FILTER (WHERE verification_status = 'risky'), + COUNT(*) FILTER (WHERE verification_status = 'invalid'), + COUNT(*) FILTER (WHERE verification_status NOT IN ('valid','risky','invalid')), + COUNT(*) FILTER (WHERE verification_checked_at IS NULL) + FROM contacts + WHERE organization_id = $1 + ` + if err := r.DB.QueryRow(ctx, query, orgID).Scan(&c.Valid, &c.Risky, &c.Invalid, &c.Unknown, &c.Pending); err != nil { + db.CaptureError(err, query, []any{orgID}, "queryrow") + return c, errx.InternalError() + } + return c, nil +} + func (r *contactRepository) GetByEmailAndOrganization(ctx context.Context, organizationID uuid.UUID, email string) (*models.Contact, *errx.Error) { query := ` SELECT @@ -752,6 +920,11 @@ func (r *contactRepository) Search( args = append(args, *filters.Subscribed) argIndex++ } + if filters.VerificationStatus != "" { + whereClauses = append(whereClauses, fmt.Sprintf("c.verification_status = $%d", argIndex)) + args = append(args, filters.VerificationStatus) + argIndex++ + } // ----------------------------- // Date filters @@ -1003,6 +1176,8 @@ func (r *contactRepository) Search( SELECT c.id, c.first_name, c.last_name, c.email, c.company, c.phone, c.custom_fields, c.subscribed, c.updated_at, c.created_at, + c.verification_status, c.verification_reason, c.is_catch_all, c.verification_checked_at, + c.verification_source, c.verification_provider, c.verification_sub_status, COALESCE(cl.campaign_count,0) AS campaign_count, COALESCE( ( @@ -1082,7 +1257,10 @@ func (r *contactRepository) Search( if err := rows.Scan( &c.ID, &c.FirstName, &c.LastName, &c.Email, &c.Company, &c.Phone, &c.CustomFields, &c.Subscribed, - &c.UpdatedAt, &c.CreatedAt, &campaignCount, &campaignsJSON, &categoriesJSON, &leadProgressJSON, + &c.UpdatedAt, &c.CreatedAt, + &c.VerificationStatus, &c.VerificationReason, &c.IsCatchAll, &c.VerificationCheckedAt, + &c.VerificationSource, &c.VerificationProvider, &c.VerificationSubStatus, + &campaignCount, &campaignsJSON, &categoriesJSON, &leadProgressJSON, ); err != nil { db.CaptureError(err, "", nil, "scan") return nil, errx.InternalError() @@ -3074,3 +3252,37 @@ func (r *contactRepository) ListTimeline(ctx context.Context, userID uuid.UUID, HasMore: hasMore, }, nil } + +// verificationFromRequest normalises a verdict a caller supplied with a +// contact. Empty means none; an unrecognised value is the caller's error. +func verificationFromRequest(status, provider string) (*models.ContactVerificationWrite, *errx.Error) { + status = strings.TrimSpace(status) + if status == "" { + return nil, nil + } + provider = strings.TrimSpace(provider) + if provider != "" { + if _, ok := emailverify.KnownVocabulary(provider); !ok { + return nil, errx.NewWithIdentifier(errx.BadRequest, "unknown_verification_provider", + "unknown verification_provider "+strconv.Quote(provider)) + } + } + v, ok := emailverify.NormalizeExternal(provider, status) + if !ok { + return nil, errx.NewWithIdentifier(errx.BadRequest, "unknown_verification_status", + "verification_status "+strconv.Quote(status)+" is not a value any known verification service writes") + } + name := provider + if name == "" { + name = "imported" + } else if k, ok := emailverify.KnownVocabulary(provider); ok { + name = k + } + return &models.ContactVerificationWrite{ + Status: string(v.Status), + SubStatus: string(v.SubStatus), + Reason: "imported verdict: " + strings.ToLower(status), + Provider: name, + Source: models.VerificationSourceImported, + }, nil +} diff --git a/internal/tasks/campaign_reconciler.go b/internal/tasks/campaign_reconciler.go index dc3ec5c3..2ea610bb 100644 --- a/internal/tasks/campaign_reconciler.go +++ b/internal/tasks/campaign_reconciler.go @@ -82,7 +82,14 @@ func (s *tasksService) ReconcileCampaignSchedules(ctx context.Context, limit int // record which of the several possible causes it was. s.autoPauseCampaign(ctx, id, uuid.Nil, autoPauseReason(cerr)) case errors.Is(cerr, scheduler.ErrCampaignCompleted), errors.Is(cerr, scheduler.ErrCampaignEnded): - // Nothing left to send (or past its end date): close it out. + // Nothing left to send (or past its end date): close it out, unless + // what is left was refused by verification, which parks it instead. + if errors.Is(cerr, scheduler.ErrCampaignCompleted) { + if n, uerr := s.campaignProgressRepo.CountUndeliverableLeads(ctx, id); uerr == nil && n > 0 { + s.pauseUndeliverable(ctx, id, uuid.Nil, n) + continue + } + } s.campaignRepo.UpdateStatus(ctx, id, "completed") default: // Transient error (DB blip): leave it; the next pass retries. diff --git a/internal/tasks/campaign_task.go b/internal/tasks/campaign_task.go index a96e4240..ac1cbdf0 100644 --- a/internal/tasks/campaign_task.go +++ b/internal/tasks/campaign_task.go @@ -226,9 +226,16 @@ func (s *tasksService) HandleCampaignTask(task *proto.ProcessTask) *errx.Error { if errors.Is(err, scheduler.ErrCampaignEnded) { reason = "Campaign ended: reached its end date" } - // Leads verification refused are never routed, so say so here - // rather than letting "all emails sent" cover for them. + // Leads verification refused are never routed. A campaign that ran + // out of leads only because of them is not finished: park it for the + // owner to re-verify or override (issue #264), instead of letting + // "all emails sent" cover for a verifier that may have been wrong. if n, cerr := s.campaignProgressRepo.CountUndeliverableLeads(ctx, campaign.ID); cerr == nil && n > 0 { + if errors.Is(err, scheduler.ErrCampaignCompleted) { + s.pauseUndeliverable(ctx, campaign.ID, taskID, n) + executionStatus = "completed" + return nil + } reason = fmt.Sprintf("%s (%d lead(s) skipped: address verification refused them)", reason, n) } s.campaignRepo.UpdateStatus(ctx, campaign.ID, "completed") @@ -854,6 +861,37 @@ func autoPauseReason(err error) string { } } +// UndeliverablePauseReason is the activity-log line for a campaign parked +// because verification refused every remaining lead. +func UndeliverablePauseReason(n int) string { + return fmt.Sprintf("Campaign paused: %d remaining lead(s) were refused by address verification. Re-verify them or mark them deliverable to continue", n) +} + +// pauseUndeliverable parks a campaign whose only remaining leads were refused +// by verification. Resumable: re-verifying or marking them deliverable +// restarts it. +func (s *tasksService) pauseUndeliverable(ctx context.Context, campaignID, taskID uuid.UUID, n int) { + s.campaignRepo.UpdateStatusWithLock(ctx, campaignID, "paused_undeliverable") + if taskID != uuid.Nil { + s.taskRepo.UpdateTaskStatus(ctx, taskID, "completed") + } + if s.campaignLogRepo != nil { + s.campaignLogRepo.CreateLog(ctx, &repository.CampaignLogEntry{ + CampaignID: campaignID, + EventType: "auto_paused", + Message: UndeliverablePauseReason(n), + Metadata: map[string]interface{}{"undeliverable": n}, + }) + } + if s.streamingPublisher != nil { + s.streamingPublisher.PublishCampaignEvent(ctx, &pubsub.CampaignEvent{ + BaseEvent: pubsub.BaseEvent{EventType: pubsub.EventCampaignCompleted}, + CampaignID: campaignID.String(), + Status: "paused_undeliverable", + }) + } +} + // autoPauseCampaign parks a campaign that has nothing it can send from. The // reason is carried through to the activity log because "paused_no_accounts" // covers several very different fixes (connect a mailbox, widen a sending diff --git a/web/src/app/app/campaigns/[id]/layout.tsx b/web/src/app/app/campaigns/[id]/layout.tsx index 7cacf454..3dbd550d 100644 --- a/web/src/app/app/campaigns/[id]/layout.tsx +++ b/web/src/app/app/campaigns/[id]/layout.tsx @@ -20,6 +20,7 @@ import { CampaignContext } from "@/hooks/context/campaign"; import { useConfirm } from "@/hooks/context/confirm"; import LaunchCampaignDialog from "@/components/app/campaigns/LaunchCampaignDialog"; import CampaignActionsMenu from "@/components/app/campaigns/CampaignActionsMenu"; +import UndeliverableBanner from "@/components/app/campaigns/UndeliverableBanner"; import { canStartCampaign } from "@/components/app/campaigns/useCampaignActions"; import toast from "react-hot-toast"; import { CAMPAIGN_DELETED_EVENT, type CampaignDeletedDetail } from "@/lib/realtime/campaignDeleted"; @@ -37,6 +38,7 @@ const TABS = [ const STATUS_PILL: Record = { active: "bg-emerald-50 text-emerald-700 border-emerald-200", paused: "bg-amber-50 text-amber-700 border-amber-200", + paused_undeliverable: "bg-amber-50 text-amber-700 border-amber-200", draft: "bg-slate-100 text-slate-600 border-slate-200", completed: "bg-slate-100 text-slate-600 border-slate-200", }; @@ -138,7 +140,7 @@ export default function CampaignLayout() { - {status} + {status === "paused_undeliverable" ? "needs verification" : status} @@ -173,6 +175,8 @@ export default function CampaignLayout() { + +
{TABS.map(({ label, path, Icon }) => { const fullPath = `/app/campaigns/${id}${path}`; @@ -208,7 +212,7 @@ export default function CampaignLayout() { setLaunchOpen(false)} - onConfirm={(cid) => startCampaign.mutateAsync(cid)} + onConfirm={(cid, options) => startCampaign.mutateAsync({ id: cid, options })} /> ); diff --git a/web/src/app/app/campaigns/page.tsx b/web/src/app/app/campaigns/page.tsx index f83d46d1..d05e16a5 100644 --- a/web/src/app/app/campaigns/page.tsx +++ b/web/src/app/app/campaigns/page.tsx @@ -79,6 +79,7 @@ const STATUS_LABEL: Record = { paused_no_accounts: "no accounts", paused_trial_expired: "trial expired", paused_guardrail: "auto-paused", + paused_undeliverable: "needs verification", completed: "finished", draft: "draft", }; @@ -93,6 +94,7 @@ const STATUS_TONE: Record = { paused_no_accounts: "text-amber-600", paused_trial_expired: "text-amber-600", paused_guardrail: "text-rose-600", + paused_undeliverable: "text-amber-600", draft: "text-slate-500", }; @@ -116,6 +118,9 @@ function CampaignStatusMark({ status }: { status: string }) { } else if (status === "paused_guardrail") { Icon = AlertTriangleIcon; title = "Paused automatically — a deliverability guardrail was breached"; + } else if (status === "paused_undeliverable") { + Icon = AlertTriangleIcon; + title = "Paused — address verification refused the remaining leads"; } else if (status === "paused_no_accounts" || status === "paused_trial_expired") { Icon = AlertTriangleIcon; title = status === "paused_no_accounts" ? "Paused — no sending accounts" : "Paused — trial expired"; @@ -603,7 +608,7 @@ export default function CampaignsPage() { setLaunchTarget(null)} - onConfirm={(id) => startCampaign.mutateAsync(id)} + onConfirm={(id, options) => startCampaign.mutateAsync({ id, options })} /> ); diff --git a/web/src/app/app/integrations/_components/ConnectDrawer.tsx b/web/src/app/app/integrations/_components/ConnectDrawer.tsx index c875c053..01f7b3c7 100644 --- a/web/src/app/app/integrations/_components/ConnectDrawer.tsx +++ b/web/src/app/app/integrations/_components/ConnectDrawer.tsx @@ -56,6 +56,15 @@ interface FieldDef { // Credential fields for non-OAuth providers only. OAuth providers never paste. const FIELDS_BY_PROVIDER: Record = { + millionverifier: [ + { + key: "api_key", + label: "MillionVerifier API key", + type: "password", + required: true, + helper: "API → API key in your MillionVerifier account. The key is checked before it is saved; one credit is spent per address verified.", + }, + ], close: [ { key: "workspace", label: "Organization", placeholder: "Acme" }, { diff --git a/web/src/app/app/integrations/_components/ProviderGlyph.tsx b/web/src/app/app/integrations/_components/ProviderGlyph.tsx index c1b2139b..ebba2994 100644 --- a/web/src/app/app/integrations/_components/ProviderGlyph.tsx +++ b/web/src/app/app/integrations/_components/ProviderGlyph.tsx @@ -8,6 +8,10 @@ import { cn } from "@/lib/utils"; import { RAW_BRAND_LOGOS } from "./brandLogos"; const BRAND_ICON: Record = { + millionverifier: { + hex: "#1FB25A", + path: "M12 2C6.477 2 2 6.477 2 12s4.477 10 10 10 10-4.477 10-10S17.523 2 12 2zm-1.2 14.4-4.2-4.2 1.7-1.7 2.5 2.5 6.1-6.1 1.7 1.7-7.8 7.8z", + }, discord: { hex: "#5865F2", path: "M20.317 4.3698a19.7913 19.7913 0 00-4.8851-1.5152.0741.0741 0 00-.0785.0371c-.211.3753-.4447.8648-.6083 1.2495-1.8447-.2762-3.68-.2762-5.4868 0-.1636-.3933-.4058-.8742-.6177-1.2495a.077.077 0 00-.0785-.037 19.7363 19.7363 0 00-4.8852 1.515.0699.0699 0 00-.0321.0277C.5334 9.0458-.319 13.5799.0992 18.0578a.0824.0824 0 00.0312.0561c2.0528 1.5076 4.0413 2.4228 5.9929 3.0294a.0777.0777 0 00.0842-.0276c.4616-.6304.8731-1.2952 1.226-1.9942a.076.076 0 00-.0416-.1057c-.6528-.2476-1.2743-.5495-1.8722-.8923a.077.077 0 01-.0076-.1277c.1258-.0943.2517-.1923.3718-.2914a.0743.0743 0 01.0776-.0105c3.9278 1.7933 8.18 1.7933 12.0614 0a.0739.0739 0 01.0785.0095c.1202.099.246.1981.3728.2924a.077.077 0 01-.0066.1276 12.2986 12.2986 0 01-1.873.8914.0766.0766 0 00-.0407.1067c.3604.698.7719 1.3628 1.225 1.9932a.076.076 0 00.0842.0286c1.961-.6067 3.9495-1.5219 6.0023-3.0294a.077.077 0 00.0313-.0552c.5004-5.177-.8382-9.6739-3.5485-13.6604a.061.061 0 00-.0312-.0286zM8.02 15.3312c-1.1825 0-2.1569-1.0857-2.1569-2.419 0-1.3332.9555-2.4189 2.157-2.4189 1.2108 0 2.1757 1.0952 2.1568 2.419 0 1.3332-.9555 2.4189-2.1569 2.4189zm7.9748 0c-1.1825 0-2.1569-1.0857-2.1569-2.419 0-1.3332.9554-2.4189 2.1569-2.4189 1.2108 0 2.1757 1.0952 2.1568 2.419 0 1.3332-.946 2.4189-2.1568 2.4189Z", diff --git a/web/src/app/app/settings/sending/page.tsx b/web/src/app/app/settings/sending/page.tsx index e51e1f37..f11bf828 100644 --- a/web/src/app/app/settings/sending/page.tsx +++ b/web/src/app/app/settings/sending/page.tsx @@ -18,6 +18,7 @@ import { useOutreachSettings, useUpdateOutreachSettings, } from "@/lib/api/hooks/app/outreach/useOutreachSettings"; +import VerificationSettings from "@/components/app/contacts/VerificationSettings"; import { DEFAULT_PREFERRED_HOURS, describeHours, @@ -107,6 +108,7 @@ function SendingSettings() { description="When campaign mail goes out, measured in your recipient's day." actions={} > +
void; - onConfirm: (id: string) => Promise; + onConfirm: (id: string, options?: { acknowledge_list_risk?: boolean }) => Promise; }) { const [phase, setPhase] = React.useState("idle"); const [error, setError] = React.useState(null); + // The backend refused the launch on projected bounce rate. The member can + // take the risk explicitly (a list verified elsewhere), once they read why. + const [riskBlocked, setRiskBlocked] = React.useState(false); const timer = React.useRef(null); // The list passes a campaign whose `sequences` is null (the list endpoint @@ -111,6 +114,7 @@ export default function LaunchCampaignDialog({ if (campaign) { setPhase("idle"); setError(null); + setRiskBlocked(false); } }, [campaign]); @@ -130,16 +134,18 @@ export default function LaunchCampaignDialog({ return () => document.removeEventListener("keydown", onKey); }, [campaign, phase, onClose]); - async function launch() { + async function launch(acknowledge = false) { if (!campaign || phase === "launching") return; setError(null); setPhase("launching"); try { - await onConfirm(campaign.id); + await onConfirm(campaign.id, acknowledge ? { acknowledge_list_risk: true } : undefined); setPhase("done"); timer.current = window.setTimeout(onClose, 1200); } catch (e) { - setError(buildError(e as unknown as AppError)); + const err = e as unknown as AppError; + setError(buildError(err)); + setRiskBlocked(err?.code === "list_bounce_risk"); setPhase("idle"); } } @@ -322,9 +328,21 @@ export default function LaunchCampaignDialog({ > Cancel + {riskBlocked && ( + launch(true)} + disabled={phase === "launching"} + className="h-8 px-3 rounded-md border border-amber-300 bg-amber-50 hover:bg-amber-100 text-amber-900 text-[12.5px] font-medium transition-colors disabled:opacity-50" + > + Launch anyway + + )}
@@ -1319,6 +1364,9 @@ function SelectionBar({ onBulkEdit, onResearch, researching, + onVerify, + onMarkDeliverable, + verifying, onDelete, onClear, }: { @@ -1330,6 +1378,9 @@ function SelectionBar({ onBulkEdit: () => void; onResearch: () => void; researching: boolean; + onVerify: () => void; + onMarkDeliverable: () => void; + verifying: boolean; onDelete: () => void; onClear: () => void; }) { @@ -1391,6 +1442,23 @@ function SelectionBar({ {researching ? : } Research + + + + + + Address verification + Re-verify {count} + Mark deliverable + +