diff --git a/docs/content/docs/api/reference/contacts.mdx b/docs/content/docs/api/reference/contacts.mdx index 01314566..6e1ecf9a 100644 --- a/docs/content/docs/api/reference/contacts.mdx +++ b/docs/content/docs/api/reference/contacts.mdx @@ -155,7 +155,7 @@ A JSON array of contact objects (at least one, up to the per-request maximum; an | `custom_fields` | object | No | String key/value custom fields. Keys may use letters, numbers, underscores, spaces, and dashes. | | `subscribed` | boolean | No | Marketing-consent flag. Omit it to let a new contact default to subscribed and an existing one keep whatever it already had. | | `verification_status` | string | No | A verdict you already hold for the address, in Warmbly's vocabulary (`valid`, `risky`, `invalid`, `unknown`) or any known service's (`ok`, `catch-all`, `do_not_mail`, `deliverable`, `ok_for_all`, ...). Stored as an imported verdict that the background check leaves alone. A value no known service writes is rejected with `unknown_verification_status`. | -| `verification_provider` | string | No | The vocabulary `verification_status` is written in: `zerobounce`, `millionverifier`, `neverbounce`, `bouncer`, `kickbox`, `emailable`, `debounce`, `clearout`, `emaillistverify`, or `warmbly`. Omit it to have the value recognised by itself. An unknown name is rejected with `unknown_verification_provider`. | +| `verification_provider` | string | No | The vocabulary `verification_status` is written in: `zerobounce`, `millionverifier`, `cleanmylist`, `neverbounce`, `bouncer`, `kickbox`, `emailable`, `debounce`, `clearout`, `emaillistverify`, or `warmbly`. Omit it to have the value recognised by itself. An unknown name is rejected with `unknown_verification_provider`. | An address you already have is matched (lowercased) and enriched rather than duplicated: fields you send replace what is stored, fields you omit or send empty are left alone, and `custom_fields` is merged key by key. Use `PATCH /contacts/:id` to clear a value. @@ -467,7 +467,7 @@ Reports which verifier checks this workspace's addresses and the contacts by ver } ``` -`provider` is `builtin` or `millionverifier`. `credits` is the connected service's remaining balance; `provider_error` is set instead when the service is connected but unusable (a rejected key, no credits), in which case the built-in check is in use. `builtin_ready` says whether the built-in mailbox probe can run on this instance. `pending` is the share of `unknown` nobody has checked yet. +`provider` is `builtin`, `millionverifier`, or `cleanmylist`. `credits` is the connected service's remaining balance when available (omitted for CleanMyList); `provider_error` is set instead when the service is connected but unusable (a rejected key, no credits), in which case the built-in check is in use. `builtin_ready` says whether the built-in mailbox probe can run on this instance. `pending` is the share of `unknown` nobody has checked yet. ## Verify or override contacts diff --git a/docs/content/docs/guides/deliverability.mdx b/docs/content/docs/guides/deliverability.mdx index efd23972..3d5f3976 100644 --- a/docs/content/docs/guides/deliverability.mdx +++ b/docs/content/docs/guides/deliverability.mdx @@ -118,11 +118,12 @@ Complaint reports are read from IMAP and Gmail mailboxes. Microsoft Graph return Every address gets one of four verdicts. **Deliverable** and **unverified** are always sent to. **Undeliverable** is never sent to. **Risky** (a catch-all domain, a shared inbox such as `info@`) is sent to only when the campaign's **send to risky addresses** setting allows it. -Verdicts come from one of three places, and the source is shown on the contact: +Verdicts come from the following sources, and the source is shown on the contact: - **The built-in check**, included on every plan: syntax, the domain's mail server, known disposable domains, and a mailbox probe against the recipient's server. It only records an address as undeliverable when that server rejects the address itself; a server that rejects the check for its own reasons (a policy block, a rate limit, a greeting it does not accept) leaves the address unverified. Microsoft 365 and Yahoo answer every probe the same way, so their addresses stay unverified rather than being called deliverable. The check also watches itself: when it starts rejecting an unusual share of addresses, its rejections are filed as unverified for an hour instead of being trusted, because a real list is never mostly dead. Self-hosted instances should set `EMAIL_VERIFY_HELO_HOST`, described in the [configuration reference](/development/configuration/#pre-send-verification). - **MillionVerifier**, pay as you go: connect it once under Integrations with your own API key, and every check from then on uses your MillionVerifier credits (one per address) instead of the built-in probe. Its verdicts cover the catch-all and Microsoft cases the probe cannot. When the balance runs out or the key stops working, the connection is marked and the built-in check takes over until it is fixed. Nothing else changes. -- **Results you already have.** A list verified with another service (ZeroBounce, MillionVerifier, NeverBounce, Bouncer, Kickbox, Emailable, DeBounce, Clearout, EmailListVerify, or a Warmbly export) can bring its status column along. The import recognises the column from its header or its values, maps it to **Verification status**, and reads each service's own words (`catch-all`, `do_not_mail`, `ok_for_all`, and so on). Those verdicts are kept as they are; the built-in check leaves them alone. The same field is accepted when [creating contacts through the API](/api/reference/contacts/#create-contacts). +- **CleanMyList**: connect a workspace API key under Integrations to verify new contacts and re-check existing ones through your CleanMyList account. Checks use plan allowance first, then credits. The built-in check takes over when verification fails. The balance stays in CleanMyList because its API does not expose it. See [setup instructions](/guides/integrations/#connect-cleanmylist). +- **Results you already have.** A list verified with another service (ZeroBounce, MillionVerifier, CleanMyList, NeverBounce, Bouncer, Kickbox, Emailable, DeBounce, Clearout, EmailListVerify, or a Warmbly export) can bring its status column along. The import recognises the column from its header or its values, maps it to **Verification status**, and reads each service's own words (`catch-all`, `do_not_mail`, `ok_for_all`, and so on). Those verdicts are kept as they are; the built-in check leaves them alone. The same field is accepted when [creating contacts through the API](/api/reference/contacts/#create-contacts). ### What real mail teaches the check diff --git a/docs/content/docs/guides/integrations.mdx b/docs/content/docs/guides/integrations.mdx index e61039fa..c0902633 100644 --- a/docs/content/docs/guides/integrations.mdx +++ b/docs/content/docs/guides/integrations.mdx @@ -20,6 +20,7 @@ The Integrations page is a searchable directory with your existing connections a | Discord | Notifications | webhook URL | Ping a server channel on reply, bounce, or warmup health | | Calendly, Cal.com | Meetings | minted inbound URL | Track booked, rescheduled, and canceled calls | | MillionVerifier | Verification | API key | Check every contact's address through your pay-as-you-go credits instead of the built-in probe | +| CleanMyList | Verification | API key | Verify contacts using your CleanMyList plan allowance and credits | Three connect styles appear on the catalog cards: `one-click` OAuth, `api_key` for providers without an OAuth app, and `webhook` for a minted inbound URL or a pasted channel URL. @@ -37,6 +38,16 @@ The key is tested against your MillionVerifier account when you connect, so a mi **Credentials** (OAuth tokens, pasted keys, webhook URLs) are sealed with envelope encryption before touching the database. Only non-secret display details like an account name or Salesforce instance host are stored in the clear, so the dashboard can label the connection. +### Connect CleanMyList + +1. Verify your account email in [CleanMyList](https://www.cleanmylist.io/), then create a workspace key under **App → API keys**. +2. In Warmbly, open **Integrations → CleanMyList → Connect** and paste the key. Warmbly validates it with a free, read-only account request and stores it encrypted. +3. New contacts and re-verification requests use CleanMyList. Each address uses one unit of plan allowance first, then credits. Its deliverable, risky, undeliverable, and unknown verdicts map to Warmbly's corresponding statuses. + +CleanMyList does not expose a balance through its API, so Warmbly omits the credit counter. View your allowance and credits in CleanMyList. If verification fails, including an exhausted account, Warmbly uses the built-in check. Fix the account or reconnect a revoked key to restore verification. + +If both verification services are connected, the most recently created connected or degraded connection takes precedence. Disconnect it to use the other service. See the [CleanMyList API documentation](https://www.cleanmylist.io/developers) and [address verification](/guides/deliverability/#address-verification). + ## CRM field mappings A mapping decides how a Warmbly contact projects onto provider fields. Every connection ships a sensible default: diff --git a/internal/app/emailverify/provider_test.go b/internal/app/emailverify/provider_test.go new file mode 100644 index 00000000..1496719e --- /dev/null +++ b/internal/app/emailverify/provider_test.go @@ -0,0 +1,80 @@ +package emailverify + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" + verify "github.com/warmbly/warmbly/internal/pkg/emailverify" + "github.com/warmbly/warmbly/internal/repository" +) + +type providerSource struct { + ProviderSource + provider *Provider + reported error +} + +func (s *providerSource) VerificationProviderFor(context.Context, uuid.UUID) (*Provider, error) { + return s.provider, nil +} +func (s *providerSource) ReportVerificationProviderError(_ context.Context, _ uuid.UUID, err error) { + s.reported = err +} +func (s *providerSource) ClearVerificationProviderError(context.Context, uuid.UUID) { s.reported = nil } + +type contactCountsRepo struct{ repository.ContactRepository } + +func (contactCountsRepo) VerificationCounts(context.Context, uuid.UUID) (models.ContactVerificationCounts, *errx.Error) { + return models.ContactVerificationCounts{}, nil +} + +type builtinVerifier struct{} + +func (builtinVerifier) Verify(_ context.Context, email string) verify.Result { + return verify.Result{Email: email, Status: verify.StatusUnknown, Provider: verify.ProviderBuiltin} +} + +func TestCleanMyListProviderWithoutBalance(t *testing.T) { + status := http.StatusOK + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/jobs" { + _, _ = w.Write([]byte(`{"jobs":[]}`)) + return + } + w.WriteHeader(status) + _, _ = w.Write([]byte(`{"verdict":"risky","reason_code":"catch_all"}`)) + })) + defer srv.Close() + id := uuid.New() + source := &providerSource{provider: &Provider{Name: "cleanmylist", ConnectionID: &id, Client: verify.NewCleanMyList("key", srv.URL)}} + s := NewService(contactCountsRepo{}, Options{Builtin: builtinVerifier{}, Providers: source}) + org := uuid.New() + overview, err := s.Overview(context.Background(), org) + if err != nil || overview.Provider != "cleanmylist" || overview.Credits != nil || overview.ProviderError != "" { + t.Fatalf("overview = %+v, %v", overview, err) + } + if res := s.VerifyAddress(context.Background(), org, "test@example.com"); res.Provider != "cleanmylist" || res.Status != verify.StatusRisky { + t.Fatalf("result = %+v", res) + } + status = http.StatusPaymentRequired + if res := s.VerifyAddress(context.Background(), org, "test@example.com"); res.Provider != verify.ProviderBuiltin || source.reported == nil { + t.Fatalf("fallback = %+v, report = %v", res, source.reported) + } + overview, err = s.Overview(context.Background(), org) + if err != nil || overview.Provider != verify.ProviderBuiltin || overview.ProviderError == "" { + t.Fatalf("empty-account overview = %+v, %v", overview, err) + } + status = http.StatusOK + if res := s.VerifyAddress(context.Background(), org, "test@example.com"); res.Provider != "cleanmylist" || source.reported != nil { + t.Fatalf("recovered account = %+v, report = %v", res, source.reported) + } + overview, err = s.Overview(context.Background(), org) + if err != nil || overview.Provider != "cleanmylist" || overview.ProviderError != "" { + t.Fatalf("recovered overview = %+v, %v", overview, err) + } +} diff --git a/internal/app/emailverify/service.go b/internal/app/emailverify/service.go index 3cff9d7c..25db5f0f 100644 --- a/internal/app/emailverify/service.go +++ b/internal/app/emailverify/service.go @@ -29,7 +29,7 @@ type Provider struct { // ConnectionID is the integration connection carrying the key; nil for // the instance-wide key an operator configured. ConnectionID *uuid.UUID - Client *emailverify.MillionVerifier + Client emailverify.ProviderClient } // ProviderSource resolves the paid provider an organization connected. @@ -104,7 +104,7 @@ type service struct { } type creditsEntry struct { - n int + n *int err error at time.Time } @@ -190,6 +190,7 @@ func (s *service) VerifyAddress(ctx context.Context, orgID uuid.UUID, email stri if p := s.providerFor(ctx, orgID); p != nil { res, err := p.Client.Check(ctx, email) if err == nil { + s.noteProviderOK(ctx, p) return res } s.noteProviderError(ctx, p, err) @@ -217,7 +218,7 @@ func (s *service) noteProviderError(ctx context.Context, p *Provider, err error) if p == nil || err == nil { return } - if errors.Is(err, emailverify.ErrMillionVerifierKey) || errors.Is(err, emailverify.ErrMillionVerifierCredits) { + if errors.Is(err, emailverify.ErrProviderKey) || errors.Is(err, emailverify.ErrProviderCredits) { if p.ConnectionID != nil && s.providers != nil { s.providers.ReportVerificationProviderError(ctx, *p.ConnectionID, err) } @@ -234,9 +235,8 @@ func cacheKey(p *Provider) string { return "platform" } -// providerUsable checks (cached for a minute) that the provider's key works -// and has credits, so a pass never burns a whole batch on a dead key. -func (s *service) providerUsable(ctx context.Context, p *Provider) (int, error) { +// providerUsable caches account health and any available balance for a minute. +func (s *service) providerUsable(ctx context.Context, p *Provider) (*int, error) { key := cacheKey(p) s.creditsMu.Lock() e, ok := s.credits[key] @@ -244,16 +244,16 @@ func (s *service) providerUsable(ctx context.Context, p *Provider) (int, error) if ok && time.Since(e.at) < time.Minute { return e.n, e.err } - n, err := p.Client.Credits(ctx) - if err == nil && n <= 0 { - err = emailverify.ErrMillionVerifierCredits + n, err := p.Client.Account(ctx) + if err == nil && n != nil && *n <= 0 { + err = emailverify.ErrProviderCredits } s.creditsMu.Lock() s.credits[key] = creditsEntry{n: n, err: err, at: time.Now()} s.creditsMu.Unlock() if err != nil { s.noteProviderError(ctx, p, err) - } else { + } else if n != nil { s.noteProviderOK(ctx, p) } return n, err @@ -263,6 +263,13 @@ func (s *service) providerUsable(ctx context.Context, p *Provider) (int, error) // a balance again. The write is guarded in the repository, so a pass that finds // the connection already healthy costs nothing. func (s *service) noteProviderOK(ctx context.Context, p *Provider) { + if p != nil { + s.creditsMu.Lock() + if s.credits[cacheKey(p)].err != nil { + delete(s.credits, cacheKey(p)) + } + s.creditsMu.Unlock() + } if p == nil || p.ConnectionID == nil || s.providers == nil { return } @@ -319,6 +326,7 @@ func (s *service) verifyOrgBatch(ctx context.Context, orgID uuid.UUID, cands []r // Fall back for this address so the pass still makes progress. return s.verifyBuiltin(ctx, orgID, email) } + s.noteProviderOK(ctx, p) return res } } @@ -438,7 +446,7 @@ func (s *service) Overview(ctx context.Context, orgID uuid.UUID) (*models.Verifi out.ProviderError = providerErrorText(err) } else { out.Provider = p.Name - out.Credits = &n + out.Credits = n } } return out, nil @@ -446,11 +454,11 @@ func (s *service) Overview(ctx context.Context, orgID uuid.UUID) (*models.Verifi func providerErrorText(err error) string { switch { - case errors.Is(err, emailverify.ErrMillionVerifierKey): - return "MillionVerifier rejected the API key. Reconnect it with a current key." - case errors.Is(err, emailverify.ErrMillionVerifierCredits): - return "The MillionVerifier account has no credits left. Top it up to keep using it; the built-in check is used meanwhile." + case errors.Is(err, emailverify.ErrProviderKey): + return "The verification service rejected the API key or account. Check the account and reconnect with a current key." + case errors.Is(err, emailverify.ErrProviderCredits): + return "The verification account has no allowance or credits left. Top it up to keep using it; the built-in check is used meanwhile." default: - return "MillionVerifier could not be reached; the built-in check is used meanwhile." + return "The verification service could not be reached; the built-in check is used meanwhile." } } diff --git a/internal/app/integration/catalog.go b/internal/app/integration/catalog.go index 80403c16..8bde5e36 100644 --- a/internal/app/integration/catalog.go +++ b/internal/app/integration/catalog.go @@ -205,5 +205,18 @@ func Catalog() []models.IntegrationCatalogEntry { "Replaces the built-in check while credits last, falls back when they run out", }, }, + { + Provider: models.IntegrationCleanMyList, + Name: "CleanMyList", + Tagline: "Verify contact addresses with your CleanMyList account.", + Category: models.IntegrationCategoryVerification, + AuthMethod: string(models.IntegrationAuthAPIKey), + DocsURL: "https://www.cleanmylist.io/developers", + Highlights: []string{ + "Checks new contacts and re-verifies existing addresses", + "Uses your CleanMyList plan allowance first, then credits", + "Falls back to the built-in check when the service is unavailable", + }, + }, } } diff --git a/internal/app/integration/service.go b/internal/app/integration/service.go index a935a044..beb2e1d9 100644 --- a/internal/app/integration/service.go +++ b/internal/app/integration/service.go @@ -9,6 +9,7 @@ import ( "fmt" emailverifyapp "github.com/warmbly/warmbly/internal/app/emailverify" "github.com/warmbly/warmbly/internal/pkg/emailverify" + "slices" "strings" "time" @@ -274,12 +275,14 @@ func (s *service) Connect(ctx context.Context, orgID, userID uuid.UUID, provider // A verification key is checked before it is stored: a mistyped key would // otherwise quietly leave every contact on the built-in check. - if provider == models.IntegrationMillionVerifier { - credits, err := checkMillionVerifierKey(ctx, config) + if slices.Contains(models.VerificationProviders, provider) { + credits, err := checkVerificationKey(ctx, provider, config) if err != nil { return nil, err } - displayFields["credits"] = credits + if credits != nil { + displayFields["credits"] = *credits + } } var inboundSecret string @@ -1351,35 +1354,42 @@ func buildDisplayFields(provider models.IntegrationProvider, config map[string]a pick("server") case models.IntegrationZapier, models.IntegrationMake, models.IntegrationN8N: // Outbound-via-Warmbly-API providers: minimal display fields. - case models.IntegrationMillionVerifier: + case models.IntegrationMillionVerifier, models.IntegrationCleanMyList: // Credits are filled in at connect time from the provider. } return df } -// checkMillionVerifierKey validates a pasted key against the provider and -// returns the account's credit balance. -func checkMillionVerifierKey(ctx context.Context, config map[string]any) (int, error) { - key, _ := config["api_key"].(string) - key = strings.TrimSpace(key) - if key == "" { - return 0, errors.New("paste your MillionVerifier API key") +func verificationClient(provider models.IntegrationProvider, key string) emailverify.ProviderClient { + switch provider { + case models.IntegrationMillionVerifier: + return emailverify.NewMillionVerifier(key, "") + case models.IntegrationCleanMyList: + return emailverify.NewCleanMyList(key, "") + default: + return nil } - credits, err := emailverify.NewMillionVerifier(key, "").Credits(ctx) - switch { - case errors.Is(err, emailverify.ErrMillionVerifierKey): - return 0, errors.New("MillionVerifier rejected this API key") - case errors.Is(err, emailverify.ErrMillionVerifierCredits): - // A valid key with an empty balance still connects; the built-in - // check covers until it is topped up. - return 0, nil - case err != nil: - return 0, fmt.Errorf("could not reach MillionVerifier: %w", err) - } - return credits, nil } -// VerificationProviderFor returns the org's connected MillionVerifier client, +// checkVerificationKey validates a pasted key without spending credits. +func checkVerificationKey(ctx context.Context, provider models.IntegrationProvider, config map[string]any) (*int, error) { + key, _ := config["api_key"].(string) + if strings.TrimSpace(key) == "" { + return nil, errors.New("paste your verification service API key") + } + balance, err := verificationClient(provider, key).Account(ctx) + if errors.Is(err, emailverify.ErrProviderCredits) { + // A valid empty account can connect; the built-in check covers it. + zero := 0 + return &zero, nil + } + if err != nil { + return nil, fmt.Errorf("%s: %w", provider, err) + } + return balance, nil +} + +// VerificationProviderFor returns the org's newest connected verification client, // or nil when none is connected. A disconnected or reauth-required connection // does not count. func (s *service) VerificationProviderFor(ctx context.Context, orgID uuid.UUID) (*emailverifyapp.Provider, error) { @@ -1388,7 +1398,7 @@ func (s *service) VerificationProviderFor(ctx context.Context, orgID uuid.UUID) return nil, err } for _, c := range conns { - if c.Provider != models.IntegrationMillionVerifier { + if !slices.Contains(models.VerificationProviders, c.Provider) { continue } if c.Status != models.IntegrationStatusConnected && c.Status != models.IntegrationStatusDegraded { @@ -1408,9 +1418,9 @@ func (s *service) VerificationProviderFor(ctx context.Context, orgID uuid.UUID) } id := c.ID return &emailverifyapp.Provider{ - Name: emailverify.ProviderMillionVerifier, + Name: string(c.Provider), ConnectionID: &id, - Client: emailverify.NewMillionVerifier(key, ""), + Client: verificationClient(c.Provider, key), }, nil } return nil, nil @@ -1425,11 +1435,11 @@ func (s *service) ReportVerificationProviderError(ctx context.Context, connectio status, health := models.IntegrationStatusDegraded, models.IntegrationHealthDegraded detail := err.Error() switch { - case errors.Is(err, emailverify.ErrMillionVerifierKey): + case errors.Is(err, emailverify.ErrProviderKey): status, health = models.IntegrationStatusReauthRequired, models.IntegrationHealthDown - detail = "MillionVerifier rejected the API key; reconnect with a current key" - case errors.Is(err, emailverify.ErrMillionVerifierCredits): - detail = "MillionVerifier account is out of credits; the built-in check is used until it is topped up" + detail = err.Error() + "; check the account and reconnect with a current key" + case errors.Is(err, emailverify.ErrProviderCredits): + detail = "The verification account has no allowance or credits left; the built-in check is used until it is topped up" } _ = s.repo.SetConnectionStatus(ctx, connectionID, status, health, detail) } diff --git a/internal/models/contact.go b/internal/models/contact.go index 5e6d4167..bb128945 100644 --- a/internal/models/contact.go +++ b/internal/models/contact.go @@ -303,7 +303,7 @@ type ContactVerificationResponse struct { // VerificationOverview is what Settings shows about address verification. type VerificationOverview struct { // Provider is who checks this workspace's addresses: "builtin" or - // "millionverifier". + // "millionverifier" or "cleanmylist". Provider string `json:"provider"` // ConnectionID is the integration connection behind a paid provider. ConnectionID *string `json:"connection_id,omitempty"` diff --git a/internal/models/integration.go b/internal/models/integration.go index 5446defb..96ff7985 100644 --- a/internal/models/integration.go +++ b/internal/models/integration.go @@ -38,11 +38,12 @@ const ( // Verification IntegrationMillionVerifier IntegrationProvider = "millionverifier" + IntegrationCleanMyList IntegrationProvider = "cleanmylist" ) // VerificationProviders are the providers that verify contact addresses. A // connection to one of these makes a built-in verdict worth re-checking. -var VerificationProviders = []IntegrationProvider{IntegrationMillionVerifier} +var VerificationProviders = []IntegrationProvider{IntegrationMillionVerifier, IntegrationCleanMyList} // AllIntegrationProviders lists every provider the dashboard exposes. The // order here is the catalog order users see. @@ -60,6 +61,7 @@ var AllIntegrationProviders = []IntegrationProvider{ IntegrationCalCom, IntegrationGoogleSheets, IntegrationMillionVerifier, + IntegrationCleanMyList, } func IsValidIntegrationProvider(s string) bool { diff --git a/internal/pkg/emailverify/cleanmylist.go b/internal/pkg/emailverify/cleanmylist.go new file mode 100644 index 00000000..402e47b5 --- /dev/null +++ b/internal/pkg/emailverify/cleanmylist.go @@ -0,0 +1,119 @@ +package emailverify + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// CleanMyList verifies addresses using the workspace's API key. +type CleanMyList struct { + apiKey string + baseURL string + client *http.Client +} + +func NewCleanMyList(apiKey, baseURL string) *CleanMyList { + if baseURL == "" { + baseURL = "https://www.cleanmylist.io" + } + return &CleanMyList{apiKey: strings.TrimSpace(apiKey), baseURL: strings.TrimRight(baseURL, "/"), client: &http.Client{Timeout: 60 * time.Second}} +} + +func (c *CleanMyList) Check(ctx context.Context, email string) (Result, error) { + res := Result{Email: strings.ToLower(strings.TrimSpace(email)), CheckedAt: time.Now().UTC(), Status: StatusUnknown, Provider: ProviderCleanMyList} + body, _ := json.Marshal(map[string]any{"email": res.Email, "smtp_probe": true}) + var out struct { + Verdict string `json:"verdict"` + Score int `json:"score"` + Reason string `json:"reason"` + ReasonCode string `json:"reason_code"` + Checks []struct { + Name string `json:"name"` + Status string `json:"status"` + ReasonCode string `json:"reason_code"` + } `json:"checks"` + } + if err := c.request(ctx, http.MethodPost, "/v1/verify", body, &out); err != nil { + res.Reason = err.Error() + return res, err + } + v, ok := NormalizeExternal(ProviderCleanMyList, out.Verdict) + if !ok { + return res, fmt.Errorf("cleanmylist: unrecognised verdict %q", out.Verdict) + } + res.Status = v.Status + switch out.ReasonCode { + case "catch_all": + res.SubStatus = SubStatusCatchAll + case "role_account": + res.SubStatus = SubStatusRole + case "disposable": + res.SubStatus = SubStatusDisposable + case "invalid_syntax": + res.SubStatus = SubStatusSyntax + case "no_mx", "null_mx": + res.SubStatus = SubStatusNoMX + } + res.IsCatchAll = res.SubStatus == SubStatusCatchAll + res.Confidence = out.Score + res.Reason = "cleanmylist: " + out.Reason + if out.ReasonCode != "" { + res.Reason += " (" + out.ReasonCode + ")" + } + for _, check := range out.Checks { + if check.Name == "catch_all" && check.ReasonCode == "catch_all" { + res.IsCatchAll = true + } + if check.Name == "mx_records" && check.Status == "pass" { + res.HasMX = true + } + } + return res, nil +} + +// Account validates the key without spending allowance; no balance API exists. +func (c *CleanMyList) Account(ctx context.Context) (*int, error) { + var out struct { + Jobs json.RawMessage `json:"jobs"` + } + err := c.request(ctx, http.MethodGet, "/v1/jobs", nil, &out) + if err == nil && (len(out.Jobs) == 0 || out.Jobs[0] != '[') { + err = fmt.Errorf("cleanmylist: unreadable account response") + } + return nil, err +} + +func (c *CleanMyList) request(ctx context.Context, method, path string, body []byte, out any) error { + req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+c.apiKey) + req.Header.Set("Content-Type", "application/json") + resp, err := c.client.Do(req) + if err != nil { + return fmt.Errorf("cleanmylist: %w", err) + } + defer resp.Body.Close() + switch resp.StatusCode { + case http.StatusUnauthorized: + return ErrProviderKey + case http.StatusForbidden: + return fmt.Errorf("verify your CleanMyList account email before connecting: %w", ErrProviderKey) + case http.StatusPaymentRequired: + return ErrProviderCredits + case http.StatusOK: + if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(out); err != nil { + return fmt.Errorf("cleanmylist: unreadable response: %w", err) + } + return nil + default: + return fmt.Errorf("cleanmylist: HTTP %d", resp.StatusCode) + } +} diff --git a/internal/pkg/emailverify/cleanmylist_test.go b/internal/pkg/emailverify/cleanmylist_test.go new file mode 100644 index 00000000..abe1b0e3 --- /dev/null +++ b/internal/pkg/emailverify/cleanmylist_test.go @@ -0,0 +1,122 @@ +package emailverify + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "testing" +) + +func TestCleanMyListChecksAddress(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/v1/verify" || r.Header.Get("Authorization") != "Bearer key" || r.Header.Get("Content-Type") != "application/json" { + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + } + var body struct { + Email string `json:"email"` + SMTPProbe bool `json:"smtp_probe"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.Email != "good@example.com" || !body.SMTPProbe { + t.Errorf("request body = %+v, %v", body, err) + } + _, _ = w.Write([]byte(`{"verdict":"deliverable","score":95,"reason":"All checks passed","reason_code":"ok","checks":[{"name":"mx_records","status":"pass"}]}`)) + })) + defer srv.Close() + + res, err := NewCleanMyList(" key ", srv.URL).Check(context.Background(), " Good@Example.com ") + if err != nil || res.Email != "good@example.com" || res.Status != StatusValid || res.Provider != "cleanmylist" || res.Confidence != 95 || !res.HasMX || res.CheckedAt.IsZero() { + t.Fatalf("check = %+v, %v", res, err) + } +} + +func TestCleanMyListValidatesAccountWithoutSpendingCredits(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/v1/jobs" || r.Header.Get("Authorization") != "Bearer key" { + t.Errorf("unexpected account request: %s %s", r.Method, r.URL.Path) + } + _, _ = w.Write([]byte(`{"jobs":[]}`)) + })) + defer srv.Close() + if balance, err := NewCleanMyList("key", srv.URL).Account(context.Background()); err != nil || balance != nil { + t.Fatalf("account = %v, %v; balance must remain unknown", balance, err) + } +} + +func TestCleanMyListFailuresNeverRejectAnAddress(t *testing.T) { + for _, tc := range []struct { + name string + status int + body string + want error + }{ + {"invalid key", 401, `{"error":{"code":"unauthorized"}}`, ErrProviderKey}, + {"unverified account", 403, `{"error":{"code":"email_not_verified"}}`, ErrProviderKey}, + {"no allowance", 402, `{"error":{"code":"out_of_credits"}}`, ErrProviderCredits}, + {"rate limited", 429, `{"error":{"code":"rate_limited"}}`, nil}, + {"server failure", 503, `unavailable`, nil}, + {"malformed response", 200, `Bad gateway`, nil}, + {"missing verdict", 200, `{}`, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tc.status) + _, _ = w.Write([]byte(tc.body)) + })) + defer srv.Close() + client := NewCleanMyList("key", srv.URL) + res, err := client.Check(context.Background(), "test@example.com") + if err == nil || res.Status != StatusUnknown || (tc.want != nil && !errors.Is(err, tc.want)) { + t.Fatalf("check = %+v, %v", res, err) + } + if tc.status != 200 { + if _, err := client.Account(context.Background()); err == nil || (tc.want != nil && !errors.Is(err, tc.want)) { + t.Fatalf("account error = %v", err) + } + } + }) + } +} + +func TestCleanMyListMapsVerdicts(t *testing.T) { + for _, tc := range []struct { + verdict, reason string + status Status + sub SubStatus + }{ + {"risky", "catch_all", StatusRisky, SubStatusCatchAll}, + {"risky", "role_account", StatusRisky, SubStatusRole}, + {"undeliverable", "disposable", StatusInvalid, SubStatusDisposable}, + {"undeliverable", "invalid_syntax", StatusInvalid, SubStatusSyntax}, + {"undeliverable", "null_mx", StatusInvalid, SubStatusNoMX}, + {"undeliverable", "mailbox_not_found", StatusInvalid, SubStatusNone}, + {"unknown", "verification_error", StatusUnknown, SubStatusNone}, + } { + t.Run(tc.reason, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintf(w, `{"verdict":%q,"reason_code":%q,"reason":"Explanation"}`, tc.verdict, tc.reason) + })) + defer srv.Close() + res, err := NewCleanMyList("key", srv.URL).Check(context.Background(), "test@example.com") + if err != nil || res.Status != tc.status || res.SubStatus != tc.sub || res.IsCatchAll != (tc.sub == SubStatusCatchAll) || res.Reason != "cleanmylist: Explanation ("+tc.reason+")" { + t.Fatalf("check = %+v, %v", res, err) + } + if v, ok := NormalizeExternal("CleanMyList", tc.verdict); !ok || v.Status != tc.status { + t.Fatalf("imported verdict = %+v, %v", v, ok) + } + }) + } +} + +func TestCleanMyListPreservesCatchAllAlongsideRoleReason(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"verdict":"risky","reason_code":"role_account","checks":[{"name":"catch_all","status":"warn","reason_code":"catch_all"}]}`)) + })) + defer srv.Close() + res, err := NewCleanMyList("key", srv.URL).Check(context.Background(), "info@example.com") + if err != nil || res.SubStatus != SubStatusRole || !res.IsCatchAll { + t.Fatalf("role on a catch-all domain = %+v, %v", res, err) + } +} diff --git a/internal/pkg/emailverify/emailverify.go b/internal/pkg/emailverify/emailverify.go index 803e667f..9cfae7bc 100644 --- a/internal/pkg/emailverify/emailverify.go +++ b/internal/pkg/emailverify/emailverify.go @@ -89,6 +89,7 @@ const ( const ( ProviderBuiltin = "builtin" ProviderMillionVerifier = "millionverifier" + ProviderCleanMyList = "cleanmylist" ) // Result is the outcome of verifying one address. It round-trips into the diff --git a/internal/pkg/emailverify/millionverifier.go b/internal/pkg/emailverify/millionverifier.go index 155df667..719dea28 100644 --- a/internal/pkg/emailverify/millionverifier.go +++ b/internal/pkg/emailverify/millionverifier.go @@ -3,7 +3,6 @@ package emailverify import ( "context" "encoding/json" - "errors" "fmt" "io" "net/http" @@ -31,9 +30,9 @@ const ( var ( // ErrMillionVerifierKey is returned when the API rejects the key. - ErrMillionVerifierKey = errors.New("millionverifier rejected the API key") + ErrMillionVerifierKey = ErrProviderKey // ErrMillionVerifierCredits is returned when the account is out of credits. - ErrMillionVerifierCredits = errors.New("millionverifier account has no credits left") + ErrMillionVerifierCredits = ErrProviderCredits ) // NewMillionVerifier constructs the client. baseURL is overridable for tests. @@ -131,6 +130,12 @@ func (m *MillionVerifier) Check(ctx context.Context, email string) (Result, erro return res, nil } +// Account validates the key and returns the remaining credits. +func (m *MillionVerifier) Account(ctx context.Context) (*int, error) { + n, err := m.Credits(ctx) + return &n, err +} + // Credits returns the account's remaining credits, and validates the key. func (m *MillionVerifier) Credits(ctx context.Context) (int, error) { q := url.Values{} diff --git a/internal/pkg/emailverify/provider.go b/internal/pkg/emailverify/provider.go new file mode 100644 index 00000000..afffa899 --- /dev/null +++ b/internal/pkg/emailverify/provider.go @@ -0,0 +1,18 @@ +package emailverify + +import ( + "context" + "errors" +) + +var ( + ErrProviderKey = errors.New("verification provider rejected the API key") + ErrProviderCredits = errors.New("verification account has no allowance or credits left") +) + +// ProviderClient exposes paid verification and a non-billable account check. +type ProviderClient interface { + Check(context.Context, string) (Result, error) + // Account returns nil for the balance when the provider does not expose it. + Account(context.Context) (*int, error) +} diff --git a/internal/pkg/emailverify/vocab.go b/internal/pkg/emailverify/vocab.go index 5a6b24f9..e257648d 100644 --- a/internal/pkg/emailverify/vocab.go +++ b/internal/pkg/emailverify/vocab.go @@ -16,6 +16,12 @@ type ExternalVerdict struct { // vocabularies maps a provider name to the statuses it emits. Keys are // lower-cased with spaces, dashes and underscores removed (see vocabKey). var vocabularies = map[string]map[string]ExternalVerdict{ + ProviderCleanMyList: { + "deliverable": {Status: StatusValid}, + "undeliverable": {Status: StatusInvalid}, + "risky": {Status: StatusRisky}, + "unknown": {Status: StatusUnknown}, + }, ProviderMillionVerifier: { "ok": {Status: StatusValid}, "good": {Status: StatusValid}, @@ -128,12 +134,13 @@ var vocabularies = map[string]map[string]ExternalVerdict{ // value. Specific vocabularies first; the generic one last. var vocabProviderOrder = []string{ "zerobounce", ProviderMillionVerifier, "neverbounce", "emaillistverify", - "debounce", "bouncer", "kickbox", "emailable", "clearout", ProviderBuiltin, + "debounce", "bouncer", "kickbox", "emailable", "clearout", ProviderCleanMyList, ProviderBuiltin, } // vocabProviderAliases maps how a provider is written in a column header to // its vocabulary key. var vocabProviderAliases = map[string]string{ + "cleanmylist": ProviderCleanMyList, "millionverifier": ProviderMillionVerifier, "mv": ProviderMillionVerifier, "zerobounce": "zerobounce", "zb": "zerobounce", "neverbounce": "neverbounce", "nb": "neverbounce", diff --git a/web/src/app/app/integrations/_components/ConnectDrawer.tsx b/web/src/app/app/integrations/_components/ConnectDrawer.tsx index 01f7b3c7..31075ebc 100644 --- a/web/src/app/app/integrations/_components/ConnectDrawer.tsx +++ b/web/src/app/app/integrations/_components/ConnectDrawer.tsx @@ -56,6 +56,15 @@ interface FieldDef { // Credential fields for non-OAuth providers only. OAuth providers never paste. const FIELDS_BY_PROVIDER: Record = { + cleanmylist: [ + { + key: "api_key", + label: "CleanMyList API key", + type: "password", + required: true, + helper: "App → API keys in CleanMyList. Verify your account email first. Connecting is free; checks use your plan allowance, then credits.", + }, + ], millionverifier: [ { key: "api_key", diff --git a/web/src/app/app/integrations/_components/brandLogos.ts b/web/src/app/app/integrations/_components/brandLogos.ts index 0b095b07..c91e911d 100644 --- a/web/src/app/app/integrations/_components/brandLogos.ts +++ b/web/src/app/app/integrations/_components/brandLogos.ts @@ -59,6 +59,10 @@ const PIPEDRIVE: RawBrandLogo = { }; export const RAW_BRAND_LOGOS: Record = { + cleanmylist: { + viewBox: "0 0 64 64", + inner: '', + }, slack: SLACK, close: CLOSE, pipedrive: PIPEDRIVE, diff --git a/web/src/components/app/contacts/VerificationBadge.test.ts b/web/src/components/app/contacts/VerificationBadge.test.ts new file mode 100644 index 00000000..4506fda1 --- /dev/null +++ b/web/src/components/app/contacts/VerificationBadge.test.ts @@ -0,0 +1,12 @@ +import { describe, expect, it } from "vitest"; +import { verificationTitle } from "./VerificationBadge"; + +describe("verification provider attribution", () => { + it.each(["CleanMyList", "MillionVerifier"])("names %s on provider verdicts", (name) => { + expect(verificationTitle({ + verification_status: "valid", + verification_source: "provider", + verification_provider: name.toLowerCase(), + })).toBe(`Deliverable · checked by ${name}`); + }); +}); diff --git a/web/src/components/app/contacts/VerificationBadge.tsx b/web/src/components/app/contacts/VerificationBadge.tsx index c153868c..84717fa3 100644 --- a/web/src/components/app/contacts/VerificationBadge.tsx +++ b/web/src/components/app/contacts/VerificationBadge.tsx @@ -6,6 +6,7 @@ import { AnimatePresence, motion } from "framer-motion"; import { AlertTriangleIcon, CircleDashedIcon, ShieldCheckIcon, ShieldXIcon } from "lucide-react"; import type Contact from "@/lib/api/models/app/contacts/Contact"; import { cn } from "@/lib/utils"; +import { PROVIDER_LABELS, type IntegrationProvider } from "@/lib/api/models/app/integrations/Integration"; const META = { valid: { label: "Deliverable", tone: "text-emerald-600", Icon: ShieldCheckIcon }, @@ -27,7 +28,7 @@ const SUB_LABEL: Record = { const SOURCE_LABEL: Record = { probe: "checked by Warmbly", - provider: "checked by MillionVerifier", + provider: "checked by a verification service", imported: "imported with the list", manual: "marked by a teammate", }; @@ -38,7 +39,10 @@ export function verificationTitle(c: Pick

- {paid ? "MillionVerifier" : "Built-in check"} + {paid ? providerName : "Built-in check"} - {paid ? "pay as you go" : "included"} + {paid ? "connected service" : "included"}

{paid - ? "One credit per address, from your own MillionVerifier balance." + ? data.provider === "cleanmylist" + ? "Uses your CleanMyList plan allowance, then credits. View your balance in CleanMyList." + : `One credit per address, from your own ${providerName} balance.` : data.builtin_ready ? "Syntax, mail server, disposable domains and a mailbox probe. Catch-all domains and Microsoft 365 stay unverified." : "Syntax, mail server and disposable-domain checks. The mailbox probe is off on this instance, so most addresses stay unverified."} @@ -65,7 +64,7 @@ export default function VerificationSettings() {

- {paid && data.credits !== undefined && ( + {paid && data.credits != null && ( credits @@ -75,7 +74,7 @@ export default function VerificationSettings() { to="/app/integrations" className="inline-flex items-center gap-1.5 h-7 px-2.5 rounded-md bg-sky-600 hover:bg-sky-700 text-white text-[12px] font-medium transition-colors" > - {paid ? "Manage connection" : "Connect MillionVerifier"} + {paid ? "Manage connection" : "Connect verification service"}
diff --git a/web/src/components/app/contacts/importShared.ts b/web/src/components/app/contacts/importShared.ts index bc2d6544..0a2189b5 100644 --- a/web/src/components/app/contacts/importShared.ts +++ b/web/src/components/app/contacts/importShared.ts @@ -28,6 +28,7 @@ export const STANDARD_TARGETS: { id: string; label: string }[] = [ export const VERIFICATION_VOCABULARY_LABELS: Record = { zerobounce: "ZeroBounce", millionverifier: "MillionVerifier", + cleanmylist: "CleanMyList", neverbounce: "NeverBounce", bouncer: "Bouncer", kickbox: "Kickbox", diff --git a/web/src/lib/api/client/app/contacts/verification.ts b/web/src/lib/api/client/app/contacts/verification.ts index 0b15b381..eb41b2fe 100644 --- a/web/src/lib/api/client/app/contacts/verification.ts +++ b/web/src/lib/api/client/app/contacts/verification.ts @@ -6,8 +6,8 @@ import type ContactSelection from "@/lib/api/models/app/contacts/ContactSelectio import type { ContactVerificationCounts } from "@/lib/api/models/app/contacts/SearchContactsResult"; export interface VerificationOverview { - // "builtin" (the in-house check) or "millionverifier". - provider: "builtin" | "millionverifier" | string; + // "builtin" (the in-house check) or the connected verification provider. + provider: "builtin" | "millionverifier" | "cleanmylist" | string; connection_id?: string; credits?: number; // Set when a provider is connected but unusable (bad key, no credits). diff --git a/web/src/lib/api/models/app/integrations/Integration.ts b/web/src/lib/api/models/app/integrations/Integration.ts index eee367fc..ca99b94e 100644 --- a/web/src/lib/api/models/app/integrations/Integration.ts +++ b/web/src/lib/api/models/app/integrations/Integration.ts @@ -15,7 +15,8 @@ export type IntegrationProvider = | "calendly" | "cal_com" | "google_sheets" - | "millionverifier"; + | "millionverifier" + | "cleanmylist"; export type IntegrationAuthMethod = "oauth" | "api_key" | "webhook"; @@ -365,6 +366,7 @@ export const PROVIDER_LABELS: Record = { cal_com: "Cal.com", google_sheets: "Google Sheets", millionverifier: "MillionVerifier", + cleanmylist: "CleanMyList", }; // A connection is bookable when it's a connected scheduling provider with a