mirror of
https://github.com/warmbly/warmbly.git
synced 2026-11-17 08:18:51 +00:00
feat: decide bounce suppression from the DSN status code and server reply through a shared dsn.Blame rule so mailbox-full, policy, reputation and system bounces keep the address and lead while address-invalid ones suppress, carry the per-recipient status and diagnostic in inbound bounce reasons, reuse the rule in emailverify, and add an optional TYPESAFE_ENDPOINT
This commit is contained in:
+1
-1
@@ -711,7 +711,7 @@ func main() {
|
||||
inboxTagRepository = repository.NewInboxTagRepository(primaryDB.Pool)
|
||||
if key := config.TypeSafeAPIKey(); key != "" {
|
||||
// One TypeSafe client for every typed judgment in this process.
|
||||
typeSafeClient = typesafe.NewClient(key, typesafe.WithBillingRedis(cache.Client))
|
||||
typeSafeClient = typesafe.NewClient(key, typesafe.WithBillingRedis(cache.Client), typesafe.WithEndpoint(config.TypeSafeEndpoint()))
|
||||
}
|
||||
unsubscribeLinkRepository = repository.NewUnsubscribeLinkRepository(primaryDB.Pool)
|
||||
customDomainRepository = repository.NewCustomDomainRepository(primaryDB.Pool)
|
||||
|
||||
@@ -305,7 +305,7 @@ func main() {
|
||||
// no key is configured, and every feature that reads it stays off.
|
||||
var typeSafeClient *typesafe.Client
|
||||
if key := config.TypeSafeAPIKey(); key != "" {
|
||||
typeSafeClient = typesafe.NewClient(key, typesafe.WithBillingRedis(redisCache.Client))
|
||||
typeSafeClient = typesafe.NewClient(key, typesafe.WithBillingRedis(redisCache.Client), typesafe.WithEndpoint(config.TypeSafeEndpoint()))
|
||||
}
|
||||
if aiProviderC != nil {
|
||||
replyclassify.SetModelClassifier(func(ctx context.Context, system, user string) (string, error) {
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"github.com/warmbly/warmbly/internal/app/inboxtag"
|
||||
"github.com/warmbly/warmbly/internal/config"
|
||||
"github.com/warmbly/warmbly/internal/pkg/typesafe"
|
||||
"github.com/warmbly/warmbly/internal/repository"
|
||||
)
|
||||
|
||||
@@ -92,7 +93,7 @@ func runInboxTagBackfill(ctx context.Context, args []string) error {
|
||||
// The asker is nil-safe on a dry run, but constructing it is what proves a
|
||||
// key is present, so it is built either way.
|
||||
svc := inboxtag.NewService(
|
||||
inboxtag.NewClient(config.TypeSafeAPIKey()),
|
||||
inboxtag.NewClient(config.TypeSafeAPIKey(), typesafe.WithEndpoint(config.TypeSafeEndpoint())),
|
||||
repo,
|
||||
categories,
|
||||
categories,
|
||||
|
||||
@@ -387,4 +387,5 @@ CHECK_ORIGIN=false
|
||||
# Phase 1 writes labels and a relevance score and takes no action: no snooze,
|
||||
# no lead hold, no task, no suppression. Docs: /guides/inbox-tagging/
|
||||
# TYPESAFE_API_KEY=
|
||||
# TYPESAFE_ENDPOINT=https://api.typesafe.ai/v1/systemone
|
||||
# INBOX_TAGGING_ENABLED=false
|
||||
|
||||
@@ -381,6 +381,7 @@ services:
|
||||
# review page from it, the consumer does the classifying. Both halves are
|
||||
# required, so an instance that sets neither never reaches the network.
|
||||
TYPESAFE_API_KEY: ${TYPESAFE_API_KEY:-}
|
||||
TYPESAFE_ENDPOINT: ${TYPESAFE_ENDPOINT:-}
|
||||
INBOX_TAGGING_ENABLED: ${INBOX_TAGGING_ENABLED:-false}
|
||||
# Gmail mailbox OAuth (leave unset to connect only SMTP/IMAP + Outlook
|
||||
# mailboxes). See the self-hosting docs for the Google Cloud setup.
|
||||
@@ -506,6 +507,7 @@ services:
|
||||
# key set only here still works; the backend needs it to report whether
|
||||
# the feature is on. Off unless both are set.
|
||||
TYPESAFE_API_KEY: ${TYPESAFE_API_KEY:-}
|
||||
TYPESAFE_ENDPOINT: ${TYPESAFE_ENDPOINT:-}
|
||||
INBOX_TAGGING_ENABLED: ${INBOX_TAGGING_ENABLED:-false}
|
||||
volumes:
|
||||
- ${WARMBLY_BLOBS:-blobs}:/data/blobs
|
||||
|
||||
@@ -778,6 +778,7 @@ Salesforce reads `SALESFORCE_OAUTH_CLIENT_ID` and `SALESFORCE_OAUTH_CLIENT_SECRE
|
||||
| `MAILHOST_ISPDB` | `true` lets mailbox server detection ask Mozilla's Thunderbird ISPDB (`autoconfig.thunderbird.net`) about a domain whose MX records and own autoconfig file name no known provider. Only the domain name is sent. See [data control](/development/data-control/#outbound-calls) | `false` on a self-host, `true` on Warmbly Cloud |
|
||||
| `MAILVENDOR_SANDBOX_URL` | Development only. Points every inbox vendor client at the [sandbox's mock vendor API](/development/sandbox/#mock-inbox-vendors) (`<url>/inboxkit`, `<url>/mailforge`) instead of the real vendors. Ignored unless `APP_ENV` is `dev` | unset |
|
||||
| `TYPESAFE_API_KEY` | TypeSafe API key. Turns on the typed judgments: reply classification, the copy judgment in the Advisor and the editor, warmup content review (otherwise performed by OpenAI when its generator is configured), bounce classification, import column mapping and per-form triage. Inbox tagging needs the switch below as well. See [data control](/development/data-control/#typesafe-judgments) for what each one sends | unset |
|
||||
| `TYPESAFE_ENDPOINT` | URL the TypeSafe judgments are posted to, for a regional or proxied TypeSafe endpoint. Read by the backend, the consumer and `warmblyctl inbox-tag`; has no effect without `TYPESAFE_API_KEY` | `https://api.typesafe.ai/v1/systemone` |
|
||||
| `INBOX_TAGGING_ENABLED` | `true` enables automatic inbox tagging when `TYPESAFE_API_KEY` is also set. Message subject, current body, and the previous outbound body are sent to TypeSafe | `false` |
|
||||
|
||||
<Callout type="warn" title="An unset provider still uses a key">
|
||||
|
||||
@@ -74,7 +74,7 @@ Mail over a daily budget is deferred, never dropped: the provider cursor is held
|
||||
|
||||
### TypeSafe judgments
|
||||
|
||||
`TYPESAFE_API_KEY` turns on a family of typed judgments, each of which sends a bounded piece of text to TypeSafe and stores a verdict. Without the key none of these TypeSafe calls runs. A separately configured OpenAI warmup generator still performs generation and review as described below. What each one sends:
|
||||
`TYPESAFE_API_KEY` turns on a family of typed judgments, each of which sends a bounded piece of text to TypeSafe (`https://api.typesafe.ai/v1/systemone`, or `TYPESAFE_ENDPOINT` when set) and stores a verdict. Without the key none of these TypeSafe calls runs. A separately configured OpenAI warmup generator still performs generation and review as described below. What each one sends:
|
||||
|
||||
| Judgment | On by | What is sent | Where the verdict lives |
|
||||
|---|---|---|---|
|
||||
@@ -82,7 +82,7 @@ Mail over a daily budget is deferred, never dropped: the provider cursor is held
|
||||
| Reply classification | The key alone | The subject and body of a reply to a campaign, only when no tagging verdict exists for it and the offline layers could not decide | `campaign_contact_progress.reply_class` and `reply_intent` |
|
||||
| Copy judgment | The key alone | The subject and body of a campaign step, at Advisor evaluation and on the editor's re-check | `copy_judgments`, keyed by a hash of the copy and never exported |
|
||||
| Warmup content lint | The key alone | A generated warmup thread before it enters the bank | Review state is stored with the generated thread; rejected threads stay archived and are counted on the generation job |
|
||||
| Bounce classification | The key alone | The bounce reason text, only when it does not name the recipient | `deliverability_events.metadata` |
|
||||
| Bounce classification | The key alone | The bounce reason text (the status code and the receiving server's reply), only when its status code and wording do not already say whether the address is at fault | `deliverability_events.metadata` |
|
||||
| Import column mapping | The key alone | For an import or Google Sheets preview with columns no header matched: every column header, the kind of value each column holds (email addresses, dates, short text and so on, worked out on the instance), and the workspace's custom field names. Never a cell value, and nothing at all unless the first row is clearly headers: none of its cells may be an address, a date or a number, and the Email column must have a header of its own. A contact's row fails that, since its Email cell is the address or blank | Nothing is stored; the answer only fills in the suggested mapping |
|
||||
| Form triage | A per-form switch | The submitted answers, labelled by field; never the visitor's address, agent or URL | `form_submissions.triage`, which travels with the Contacts group |
|
||||
| Mailbox import columns | The key alone | For a column of an import file that no header name or value pattern placed: the column's header text and a description of its values (such as "16 lowercase letters" or "hostnames"), never a value | Nowhere; the answer only sets that column's suggested field, which can be changed before importing |
|
||||
|
||||
@@ -186,11 +186,21 @@ If a campaign has already been paused because every one of its mailboxes was gat
|
||||
|
||||
**Pre-send verification** checks every address before a campaign sends to it and skips the undeliverable ones, keeping bad addresses from becoming the hard bounces that drive the `5%` and `10%` thresholds above. There is nothing to run: new contacts are checked in the background within a minute of being added, each contact carries its verdict (a small mark next to the email), and Settings > Sending shows the whole workspace at a glance. See [address verification](#address-verification) below for how verdicts are produced and what to do when one is wrong.
|
||||
|
||||
**Suppression** is the safety net: a suppressed recipient is never emailed again by any campaign. Warmbly suppresses automatically on a bounce, a spam complaint, or an unsubscribe, because once someone has bounced or complained the safest response is to stop, not to keep collecting negative signals. When the operator has configured TypeSafe (`TYPESAFE_API_KEY`), a bounce whose reason does not name the recipient is classified first, and the bounce reason text is sent to TypeSafe for this. A reputation or policy block, a full mailbox, or a temporary failure at 80% confidence or more still records the bounce, and still counts toward the campaign breaker and mailbox health, but does not suppress the address and does not drop the lead from its campaign, because the address is not what failed. The list, with anything you add by hand (addresses or whole domains), is under **Contacts > Suppression list**.
|
||||
**Suppression** is the safety net: a suppressed recipient is never emailed again by any campaign. Warmbly suppresses automatically on a bounce, a spam complaint, or an unsubscribe, because once someone has bounced or complained the safest response is to stop, not to keep collecting negative signals. A bounce suppresses only when it could be about the address. Warmbly reads the bounce's status code and the receiving server's reply, whether the refusal came back as a failure notice or during the send itself:
|
||||
|
||||
| The bounce says | Examples | Address suppressed, lead dropped from campaigns |
|
||||
|---|---|---|
|
||||
| The address does not exist or is disabled | `5.1.1` user unknown, `5.1.2` no such domain, `5.2.1` mailbox disabled, "no such user", "does not exist", Microsoft's `5.4.1` "Recipient address rejected: Access denied" | Yes |
|
||||
| The mailbox is full | `5.2.2`, "mailbox full", "over quota" | No |
|
||||
| The sender was refused for policy, reputation or authentication | `5.7.x` without wording about the recipient, "blocked using Spamhaus", "client host rejected", `5.1.7`/`5.1.8` sender address | No |
|
||||
| The receiving system had a problem | `5.3.x`, `5.4.x` other than `5.4.1` and `5.4.4`, any `4.x.x` | No |
|
||||
| Nothing telling | No status code and no recognisable wording | Yes, unless TypeSafe classifies it (below) |
|
||||
|
||||
A bounce that is not about the address still records the bounce, and still counts toward the campaign breaker and mailbox health, but does not suppress the address and does not drop the lead from its campaign: the step is offered again later. When the operator has configured TypeSafe (`TYPESAFE_API_KEY`), a bounce in the last row is classified, and its reason text is sent to TypeSafe for this. A reputation or policy block, a full mailbox, or a temporary failure at 80% confidence or more is then treated like the rows above it. The list, with anything you add by hand (addresses or whole domains), is under **Contacts > Suppression list**.
|
||||
|
||||
The list is held per workspace, and the check runs against the sending campaign's workspace before every send. If a campaign somehow has no workspace, that check cannot be applied, so the campaign is paused with the reason in its activity feed instead of sending unchecked.
|
||||
|
||||
Bounces are detected from the delivery-status report the recipient's server sends back. That includes the internationalized form (RFC 6533, `message/global-delivery-status`) that servers such as Postfix write for mail with non-ASCII addresses or headers. Only permanent failures (`5.x.x`, such as a nonexistent address) record a bounce and suppress; temporary ones (`4.x.x`, such as a full mailbox or greylisting) never do. This works the same for Gmail and Microsoft Graph mailboxes, where the send reports success and the bounce arrives later. Some failure notices carry no status code at all, such as Gmail's notice when a Google Group refuses a post. Those are read from the failed-recipient header the notice carries, and count as permanent unless the notice says delivery is still being retried.
|
||||
Bounces are detected from the delivery-status report the recipient's server sends back. That includes the internationalized form (RFC 6533, `message/global-delivery-status`) that servers such as Postfix write for mail with non-ASCII addresses or headers. Only permanent failures (`5.x.x`) record a bounce, and only those the table above says are about the address suppress; temporary ones (`4.x.x`, such as greylisting) never do. This works the same for Gmail and Microsoft Graph mailboxes, where the send reports success and the bounce arrives later. Some failure notices carry no status code at all, such as Gmail's notice when a Google Group refuses a post. Those are read from the failed-recipient header the notice carries, and count as permanent unless the notice says delivery is still being retried.
|
||||
|
||||
**Spam complaints** arrive the same way. When a recipient presses "mark as spam", their provider mails an abuse report back to the sending mailbox. Warmbly reads those reports, resolves them to the campaign send they name, and records a complaint: the recipient is suppressed and the number reaches your complaint rate. Providers that redact who complained still identify the message, so the right lead is suppressed anyway. Only genuine abuse reports count. A "not spam" report, where someone rescued your email from junk, is not recorded as a complaint. Who gets suppressed is always taken from the send the report names, never from an address written in the report, so a forged report cannot reach a contact that send never went to. This works for IMAP, Gmail and Microsoft Graph mailboxes. Graph syncs only a report's rendered notice, so for a message that looks like a report, the worker reads its raw message from Graph to find the parts that name the send. Temporary Graph failures leave the message pending for a later sync rather than losing the complaint. A missing or oversized raw message does not prevent other mail from syncing. The raw fetch is limited to 1 MiB and 15 seconds; parsing is limited to 16 nesting levels, 64 parts and 1 MiB of decoded report content. Malformed or over-limit reports are not counted as complaints.
|
||||
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
package advanced
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
"github.com/warmbly/warmbly/internal/pkg/typesafe"
|
||||
"github.com/warmbly/warmbly/internal/repository"
|
||||
)
|
||||
|
||||
type suppressionRepo struct {
|
||||
repository.AdvancedOutreachRepository
|
||||
events []*models.DeliverabilityEvent
|
||||
suppressed []string
|
||||
}
|
||||
|
||||
func (r *suppressionRepo) CreateDeliverabilityEvent(_ context.Context, e *models.DeliverabilityEvent) error {
|
||||
r.events = append(r.events, e)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *suppressionRepo) GetOutreachSettings(context.Context, uuid.UUID) (*models.AdvancedOutreachSettings, error) {
|
||||
s := &models.AdvancedOutreachSettings{}
|
||||
s.BouncePipeline.AutoSuppressOnBounce = true
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (r *suppressionRepo) UpsertSuppressedRecipient(_ context.Context, e *models.SuppressedRecipient) error {
|
||||
r.suppressed = append(r.suppressed, e.Email)
|
||||
return nil
|
||||
}
|
||||
|
||||
type countingJudge struct {
|
||||
calls int
|
||||
answer string
|
||||
}
|
||||
|
||||
func (j *countingJudge) Ask(context.Context, any, map[string]typesafe.Question) (*typesafe.Response, error) {
|
||||
j.calls++
|
||||
return &typesafe.Response{Answers: map[string]typesafe.Answer{"cause": {Choice: j.answer, Confidence: 0.95}}}, nil
|
||||
}
|
||||
|
||||
// A bounce suppresses the address only when it could be about the address;
|
||||
// the judge is asked only about a reason the status and wording cannot place.
|
||||
func TestBounceSuppressesOnlyWhenTheAddressFailed(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, reason string
|
||||
judge string
|
||||
suppress bool
|
||||
judgeCalls int
|
||||
}{
|
||||
{"user unknown", "550 5.1.1 <[EMAIL]>: Recipient address rejected: User unknown", "reputation_block", true, 0},
|
||||
{"exchange missing mailbox", "550 5.4.1 Recipient address rejected: Access denied. AS(201806281)", "reputation_block", true, 0},
|
||||
{"gmail policy block", "550-5.7.1 Our system has detected an unusual rate of unsolicited mail originating from your IP address.", "recipient_invalid", false, 0},
|
||||
{"spamhaus", "550 5.7.1 Service unavailable, Client host [[IP]] blocked using Spamhaus", "recipient_invalid", false, 0},
|
||||
{"mailbox full", "552 5.2.2 Mailbox full", "recipient_invalid", false, 0},
|
||||
{"synchronous refusal carrying only the status", "The mail server rejected the recipient: The SMTP server refused this recipient (SMTP rcpt to status 550); enhanced status 5.7.1", "recipient_invalid", false, 0},
|
||||
{"nothing telling, judge vouches", "Undelivered Mail Returned to Sender", "reputation_block", false, 1},
|
||||
{"nothing telling, judge does not", "Undelivered Mail Returned to Sender", "recipient_invalid", true, 1},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
repo := &suppressionRepo{}
|
||||
judge := &countingJudge{answer: tc.judge}
|
||||
s := &service{repo: repo, bounceJudge: judge}
|
||||
if xerr := s.IngestDeliverabilityEvent(context.Background(), uuid.New(), &models.IngestDeliverabilityEventRequest{
|
||||
EventType: models.DeliverabilityEventBounce, RecipientEmail: "[EMAIL]", Reason: tc.reason,
|
||||
}); xerr != nil {
|
||||
t.Fatal(xerr)
|
||||
}
|
||||
if len(repo.events) != 1 {
|
||||
t.Fatalf("the bounce event must always be recorded, got %d", len(repo.events))
|
||||
}
|
||||
if got := len(repo.suppressed) > 0; got != tc.suppress {
|
||||
t.Fatalf("suppressed = %v, want %v", got, tc.suppress)
|
||||
}
|
||||
if judge.calls != tc.judgeCalls {
|
||||
t.Fatalf("judge asked %d times, want %d", judge.calls, tc.judgeCalls)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// No judge configured: a reason nothing places keeps the conservative default.
|
||||
repo := &suppressionRepo{}
|
||||
s := &service{repo: repo}
|
||||
if xerr := s.IngestDeliverabilityEvent(context.Background(), uuid.New(), &models.IngestDeliverabilityEventRequest{
|
||||
EventType: models.DeliverabilityEventBounce, RecipientEmail: "[EMAIL]", Reason: "554 Transaction failed",
|
||||
}); xerr != nil {
|
||||
t.Fatal(xerr)
|
||||
}
|
||||
if len(repo.suppressed) != 1 {
|
||||
t.Fatal("an unplaced bounce with no judge must still suppress")
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/warmbly/warmbly/internal/pkg/dsn"
|
||||
"github.com/warmbly/warmbly/internal/pkg/emailverify"
|
||||
"github.com/warmbly/warmbly/internal/pkg/typesafe"
|
||||
"github.com/warmbly/warmbly/internal/utils/validate"
|
||||
@@ -1949,30 +1950,40 @@ func (s *service) IngestDeliverabilityEvent(ctx context.Context, organizationID
|
||||
provider = "manual"
|
||||
}
|
||||
|
||||
// A bounce reason that does not name the recipient is classified so a
|
||||
// reputation or policy block is not held against the address. The verdict
|
||||
// goes in metadata only: the reason text is the contract downstream
|
||||
// classifiers read. One call per ambiguous bounce, no cache.
|
||||
var verdict *bounceclass.Verdict
|
||||
if eventType == models.DeliverabilityEventBounce && s.bounceJudge != nil &&
|
||||
strings.TrimSpace(req.Reason) != "" && !emailverify.NamesRecipient(req.Reason) {
|
||||
// Bounded: a bounce storm must not queue behind a rate-limited judge.
|
||||
jctx, cancel := context.WithTimeout(typesafe.WithUsage(ctx, "bounce_classification", organizationID.String()), bounceJudgeTimeout)
|
||||
v, vErr := bounceclass.Classify(jctx, s.bounceJudge, req.Reason)
|
||||
cancel()
|
||||
switch {
|
||||
case vErr != nil:
|
||||
log.Debug().Err(vErr).Str("recipient", req.RecipientEmail).Msg("bounce reason not classified")
|
||||
case v != nil:
|
||||
verdict = v
|
||||
// A bounce's status code and wording decide first whether the address is
|
||||
// what failed; the judge only classifies a reason they cannot place. The
|
||||
// cause goes in metadata only: the reason text is the contract downstream
|
||||
// classifiers read. One judge call per ambiguous bounce, no cache.
|
||||
var cause string
|
||||
addressFine := false
|
||||
if eventType == models.DeliverabilityEventBounce {
|
||||
rule := dsn.Blame(req.Reason)
|
||||
cause, addressFine = string(rule), rule.AddressIsFine()
|
||||
source := "rule"
|
||||
if rule == dsn.CauseUnknown && s.bounceJudge != nil && strings.TrimSpace(req.Reason) != "" {
|
||||
// Bounded: a bounce storm must not queue behind a rate-limited judge.
|
||||
jctx, cancel := context.WithTimeout(typesafe.WithUsage(ctx, "bounce_classification", organizationID.String()), bounceJudgeTimeout)
|
||||
v, vErr := bounceclass.Classify(jctx, s.bounceJudge, req.Reason)
|
||||
cancel()
|
||||
switch {
|
||||
case vErr != nil:
|
||||
log.Debug().Err(vErr).Str("recipient", req.RecipientEmail).Msg("bounce reason not classified")
|
||||
case v != nil:
|
||||
cause, addressFine, source = v.Cause, v.AddressIsFine(), "judge"
|
||||
if req.Metadata == nil {
|
||||
req.Metadata = map[string]interface{}{}
|
||||
}
|
||||
req.Metadata["bounce_cause_confidence"] = v.Confidence
|
||||
}
|
||||
}
|
||||
if cause != "" {
|
||||
if req.Metadata == nil {
|
||||
req.Metadata = map[string]interface{}{}
|
||||
}
|
||||
req.Metadata["bounce_cause"] = v.Cause
|
||||
req.Metadata["bounce_cause_confidence"] = v.Confidence
|
||||
req.Metadata["bounce_cause"] = cause
|
||||
req.Metadata["bounce_cause_source"] = source
|
||||
}
|
||||
}
|
||||
addressFine := verdict.AddressIsFine()
|
||||
|
||||
if err := s.repo.CreateDeliverabilityEvent(ctx, &models.DeliverabilityEvent{
|
||||
OrganizationID: organizationID,
|
||||
@@ -2005,9 +2016,8 @@ func (s *service) IngestDeliverabilityEvent(ctx context.Context, organizationID
|
||||
log.Info().
|
||||
Str("organization_id", organizationID.String()).
|
||||
Str("recipient", req.RecipientEmail).
|
||||
Str("bounce_cause", verdict.Cause).
|
||||
Float64("confidence", verdict.Confidence).
|
||||
Msg("bounce classified as not about the address; recipient not suppressed and lead kept")
|
||||
Str("bounce_cause", cause).
|
||||
Msg("bounce not about the address; recipient not suppressed and lead kept")
|
||||
}
|
||||
|
||||
if shouldSuppress {
|
||||
@@ -2119,7 +2129,7 @@ func (s *service) IngestDeliverabilityEvent(ctx context.Context, organizationID
|
||||
title := "Bounce: " + req.RecipientEmail
|
||||
// A reputation block is about the mailbox, not the lead, so
|
||||
// the title says who refused rather than who bounced.
|
||||
if addressFine && verdict.Cause == bounceclass.CauseReputationBlock {
|
||||
if addressFine && cause == bounceclass.CauseReputationBlock {
|
||||
title = "Provider refused mail from " + displayname.DisplayableOr(camp.Name, "a campaign")
|
||||
}
|
||||
if eventType == models.DeliverabilityEventComplaint {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Package bounceclass names what a bounce is about when the reason text does
|
||||
// not say so, so a rejection aimed at the sending mailbox is not held against
|
||||
// the recipient. Only reasons that do not name the recipient reach it.
|
||||
// the recipient. Only reasons dsn.Blame cannot place reach it.
|
||||
package bounceclass
|
||||
|
||||
import (
|
||||
@@ -8,17 +8,18 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/warmbly/warmbly/internal/pkg/dsn"
|
||||
"github.com/warmbly/warmbly/internal/pkg/typesafe"
|
||||
)
|
||||
|
||||
// Cause is the mutually exclusive answer: what the receiving server refused.
|
||||
// These identifiers are stored in event metadata, so changing one changes
|
||||
// history.
|
||||
// history. They share dsn's names, so a rule and a verdict read alike.
|
||||
const (
|
||||
CauseRecipientInvalid = "recipient_invalid"
|
||||
CauseMailboxFull = "mailbox_full"
|
||||
CauseReputationBlock = "reputation_block"
|
||||
CauseTransient = "transient"
|
||||
CauseRecipientInvalid = string(dsn.CauseRecipientInvalid)
|
||||
CauseMailboxFull = string(dsn.CauseMailboxFull)
|
||||
CauseReputationBlock = string(dsn.CauseReputationBlock)
|
||||
CauseTransient = string(dsn.CauseTransient)
|
||||
CauseOther = "other"
|
||||
)
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/warmbly/warmbly/internal/errx"
|
||||
"github.com/warmbly/warmbly/internal/infrastructure/pubsub"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
"github.com/warmbly/warmbly/internal/pkg/dsn"
|
||||
"github.com/warmbly/warmbly/internal/pkg/emailverify"
|
||||
"github.com/warmbly/warmbly/internal/pkg/mailhdr"
|
||||
"github.com/warmbly/warmbly/internal/repository"
|
||||
@@ -336,11 +337,10 @@ func (s *JobsService) failCampaignSend(ctx context.Context, task *repository.Tas
|
||||
}
|
||||
}
|
||||
|
||||
// A recipient the mail server refused at RCPT is a hard bounce that simply
|
||||
// arrived synchronously. Retrying it from the same mailbox only spends
|
||||
// reputation, so it goes through the bounce pipeline (progress, optional
|
||||
// suppression, guardrails, warmup health, webhooks) and the lead is
|
||||
// dropped as bounced instead of being offered again.
|
||||
// A recipient the mail server refused at RCPT is a bounce that simply
|
||||
// arrived synchronously, so it goes through the bounce pipeline (progress,
|
||||
// optional suppression, guardrails, warmup health, webhooks). The lead is
|
||||
// dropped as bounced only when the refusal blames the address.
|
||||
if rolledBack && !copyRefused && code == string(errx.MailErrorCodeRecipientRejected) {
|
||||
recorded, err := s.recordSynchronousBounce(ctx, task, ct, campaign, recipient, reason)
|
||||
if err != nil {
|
||||
@@ -557,6 +557,12 @@ func sendFailureReason(result models.SendEmailResult) (reason, code string) {
|
||||
if reason == "" {
|
||||
reason = strings.TrimSpace(result.Error.UserMessage)
|
||||
}
|
||||
// The enhanced status is what decides whether a refusal blames the address.
|
||||
if f := result.Error.Failure; f != nil && len(f.EnhancedStatus) <= 11 && !strings.Contains(reason, f.EnhancedStatus) {
|
||||
if _, ok := dsn.StatusIn(f.EnhancedStatus); ok {
|
||||
reason = strings.TrimSpace(reason + "; enhanced status " + f.EnhancedStatus)
|
||||
}
|
||||
}
|
||||
}
|
||||
if reason == "" {
|
||||
reason = strings.TrimSpace(result.LegacyErrorMsg)
|
||||
|
||||
@@ -27,8 +27,8 @@ type Client struct {
|
||||
}
|
||||
|
||||
// NewClient builds a client for one API key.
|
||||
func NewClient(apiKey string) *Client {
|
||||
return &Client{inner: typesafe.NewClient(apiKey)}
|
||||
func NewClient(apiKey string, options ...typesafe.Option) *Client {
|
||||
return &Client{inner: typesafe.NewClient(apiKey, options...)}
|
||||
}
|
||||
|
||||
// NewAsker wraps an already-built client, so one process shares one HTTP
|
||||
|
||||
@@ -136,6 +136,12 @@ var table = []Entry{
|
||||
DocsAnchor: docsAI,
|
||||
Resolve: envValue("TYPESAFE_API_KEY"),
|
||||
},
|
||||
{
|
||||
Key: "TYPESAFE_ENDPOINT", Group: GroupDeployment, RuntimeChangeable: ChangeBootOnly,
|
||||
Effect: "URL the TypeSafe judgments are sent to.",
|
||||
DocsAnchor: docsAI,
|
||||
Resolve: envOr("TYPESAFE_ENDPOINT", "https://api.typesafe.ai/v1/systemone"),
|
||||
},
|
||||
{
|
||||
Key: "INBOX_TAGGING_ENABLED", Group: GroupDeployment, RuntimeChangeable: ChangeBootOnly,
|
||||
Effect: "Enables TypeSafe classification of inbound message content when its API key is configured.",
|
||||
|
||||
@@ -56,7 +56,7 @@ func (w *WMail) bounceReport(msg *models.EmailMessageData) *models.JobEventInbou
|
||||
EmailID: w.ID,
|
||||
OriginalMessageID: strings.Trim(originalID, "<>"),
|
||||
FailedRecipient: report.FailedRecipient,
|
||||
Reason: msg.Subject,
|
||||
Reason: report.Reason(msg.Subject),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -20,6 +20,11 @@ func TypeSafeAPIKey() string {
|
||||
return strings.TrimSpace(os.Getenv("TYPESAFE_API_KEY"))
|
||||
}
|
||||
|
||||
// TypeSafeEndpoint overrides the TypeSafe API URL; empty keeps the default.
|
||||
func TypeSafeEndpoint() string {
|
||||
return strings.TrimSpace(os.Getenv("TYPESAFE_ENDPOINT"))
|
||||
}
|
||||
|
||||
// InboxTaggingEnabled reports whether the feature should run. Both halves are
|
||||
// required, and the key check is here rather than at the call site so there is
|
||||
// one answer to "is this on" for the API, the consumer and the dashboard.
|
||||
|
||||
@@ -15,6 +15,7 @@ type JobEventInboundBounce struct {
|
||||
OriginalMessageID string `json:"original_message_id" avro:"original_message_id"`
|
||||
// FailedRecipient is the address that bounced, when the DSN exposed it.
|
||||
FailedRecipient string `json:"failed_recipient" avro:"failed_recipient"`
|
||||
// Reason is a short human string (the bounce subject) for the event record.
|
||||
// Reason is the DSN's status and diagnostic, or its subject when it has
|
||||
// neither (older workers always send the subject).
|
||||
Reason string `json:"reason" avro:"reason"`
|
||||
}
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
package dsn
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// Cause names what a refusal was about. The identifiers are stored in event
|
||||
// metadata, so changing one changes history.
|
||||
type Cause string
|
||||
|
||||
const (
|
||||
// CauseUnknown means the text does not say, so callers keep their default.
|
||||
CauseUnknown Cause = ""
|
||||
CauseRecipientInvalid Cause = "recipient_invalid"
|
||||
CauseMailboxFull Cause = "mailbox_full"
|
||||
CauseReputationBlock Cause = "reputation_block"
|
||||
CauseTransient Cause = "transient"
|
||||
)
|
||||
|
||||
// AddressIsFine reports a refusal that affirmatively names something other
|
||||
// than the recipient's address: a full mailbox, a policy or reputation block,
|
||||
// a system or network problem, or a temporary failure.
|
||||
func (c Cause) AddressIsFine() bool {
|
||||
switch c {
|
||||
case CauseMailboxFull, CauseReputationBlock, CauseTransient:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// mailboxFullMarkers say the mailbox exists and is over quota. They win over
|
||||
// recipientMarkers because Postfix quota policies answer "Recipient address
|
||||
// rejected: Mailbox full".
|
||||
var mailboxFullMarkers = []string{
|
||||
"mailbox full",
|
||||
"mailbox is full",
|
||||
"over quota",
|
||||
"quota exceeded",
|
||||
"exceeded storage",
|
||||
"insufficient storage",
|
||||
"out of storage",
|
||||
"quotaexceeded",
|
||||
}
|
||||
|
||||
// recipientMarkers name the RECIPIENT as the problem. They are decisive
|
||||
// regardless of the numeric code, because providers disagree about which code
|
||||
// carries them (Microsoft answers a missing mailbox with 5.4.1, some Exim
|
||||
// builds with 554).
|
||||
var recipientMarkers = []string{
|
||||
"user unknown",
|
||||
"unknown user",
|
||||
"no such user",
|
||||
"no such recipient",
|
||||
"no such mailbox",
|
||||
"user not found",
|
||||
"recipient not found",
|
||||
"recipient unknown",
|
||||
"unknown recipient",
|
||||
"recipient address rejected",
|
||||
"recipient rejected",
|
||||
"invalid recipient",
|
||||
"invalid mailbox",
|
||||
"mailbox unavailable",
|
||||
"mailbox not found",
|
||||
"mailbox does not exist",
|
||||
"mailbox is disabled",
|
||||
"mailbox disabled",
|
||||
"account has been disabled",
|
||||
"account is disabled",
|
||||
"account does not exist",
|
||||
"address does not exist",
|
||||
"does not exist",
|
||||
"doesn't exist",
|
||||
"unrouteable address",
|
||||
"unroutable address",
|
||||
"no mailbox here",
|
||||
"not a valid mailbox",
|
||||
"user doesn't have an account",
|
||||
}
|
||||
|
||||
// transientMarkers say the sender should try again later.
|
||||
var transientMarkers = []string{
|
||||
"greylist",
|
||||
"grey-list",
|
||||
"try again",
|
||||
"temporarily",
|
||||
"rate limit",
|
||||
"too many",
|
||||
}
|
||||
|
||||
// sessionMarkers name the SENDER as the problem: its greeting, envelope
|
||||
// sender, connecting IP, authentication or reputation, or a policy applied to
|
||||
// the session. A refusal carrying one says nothing about the mailbox.
|
||||
var sessionMarkers = []string{
|
||||
"helo",
|
||||
"ehlo",
|
||||
"fully-qualified",
|
||||
"fully qualified",
|
||||
"relay access denied",
|
||||
"relay denied",
|
||||
"relaying denied",
|
||||
"not permitted to relay",
|
||||
"unable to relay",
|
||||
"client host rejected",
|
||||
"client host blocked",
|
||||
"sender address rejected",
|
||||
"sender rejected",
|
||||
"sender verify failed",
|
||||
"sender not allowed",
|
||||
"authentication required",
|
||||
"auth required",
|
||||
"not authenticated",
|
||||
"not authorized",
|
||||
"blacklist",
|
||||
"blocklist",
|
||||
"blocked using",
|
||||
"listed in",
|
||||
"spamhaus",
|
||||
"reputation",
|
||||
"service unavailable",
|
||||
"connection refused",
|
||||
"policy reasons",
|
||||
"local policy",
|
||||
"security policy",
|
||||
}
|
||||
|
||||
// Status is an RFC 3463 enhanced status code (class.subject.detail).
|
||||
type Status struct {
|
||||
Class, Subject, Detail int
|
||||
}
|
||||
|
||||
// StatusIn returns the most specific enhanced status code in text: the first
|
||||
// that is not an undefined x.0.0, else the first. Tokens are whole, so an IP
|
||||
// address or a version string never reads as one.
|
||||
func StatusIn(text string) (Status, bool) {
|
||||
var first Status
|
||||
found := false
|
||||
for _, tok := range strings.FieldsFunc(text, func(r rune) bool { return !unicode.IsDigit(r) && r != '.' }) {
|
||||
s, ok := parseStatus(strings.Trim(tok, "."))
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if s.Subject != 0 || s.Detail != 0 {
|
||||
return s, true
|
||||
}
|
||||
if !found {
|
||||
first, found = s, true
|
||||
}
|
||||
}
|
||||
return first, found
|
||||
}
|
||||
|
||||
func parseStatus(tok string) (Status, bool) {
|
||||
parts := strings.Split(tok, ".")
|
||||
if len(parts) != 3 {
|
||||
return Status{}, false
|
||||
}
|
||||
var n [3]int
|
||||
for i, p := range parts {
|
||||
v, err := strconv.Atoi(p)
|
||||
if err != nil || len(p) > 3 {
|
||||
return Status{}, false
|
||||
}
|
||||
n[i] = v
|
||||
}
|
||||
if n[0] != 2 && n[0] != 4 && n[0] != 5 {
|
||||
return Status{}, false
|
||||
}
|
||||
return Status{Class: n[0], Subject: n[1], Detail: n[2]}, true
|
||||
}
|
||||
|
||||
// Blame decides from a refusal's text (its enhanced status code and wording)
|
||||
// what was refused. CauseUnknown means nothing in it is telling.
|
||||
func Blame(text string) Cause {
|
||||
lower := strings.ToLower(text)
|
||||
status, hasStatus := StatusIn(lower)
|
||||
|
||||
if hasStatus && status.Class == 4 {
|
||||
return CauseTransient
|
||||
}
|
||||
if hasStatus && status.Class == 5 && status.Subject == 2 && status.Detail == 2 || containsAny(lower, mailboxFullMarkers) {
|
||||
return CauseMailboxFull
|
||||
}
|
||||
if containsAny(lower, recipientMarkers) {
|
||||
return CauseRecipientInvalid
|
||||
}
|
||||
if containsAny(lower, transientMarkers) {
|
||||
return CauseTransient
|
||||
}
|
||||
if containsAny(lower, sessionMarkers) {
|
||||
return CauseReputationBlock
|
||||
}
|
||||
if !hasStatus || status.Class != 5 {
|
||||
return CauseUnknown
|
||||
}
|
||||
switch status.Subject {
|
||||
case 1:
|
||||
switch status.Detail {
|
||||
case 1, 2, 3, 6, 10: // bad mailbox, bad domain, bad syntax, moved, null MX
|
||||
return CauseRecipientInvalid
|
||||
case 7, 8: // the SENDER's address
|
||||
return CauseReputationBlock
|
||||
}
|
||||
case 2:
|
||||
switch status.Detail {
|
||||
case 1: // disabled
|
||||
return CauseRecipientInvalid
|
||||
case 3: // message too large
|
||||
return CauseReputationBlock
|
||||
}
|
||||
case 3: // receiving mail system
|
||||
return CauseTransient
|
||||
case 4:
|
||||
// Exchange answers a missing mailbox with 5.4.1 and a missing domain
|
||||
// with 5.4.4, so neither vouches for the address.
|
||||
if status.Detail != 1 && status.Detail != 4 {
|
||||
return CauseTransient
|
||||
}
|
||||
case 6, 7: // content, policy
|
||||
return CauseReputationBlock
|
||||
}
|
||||
return CauseUnknown
|
||||
}
|
||||
|
||||
func containsAny(haystack string, needles []string) bool {
|
||||
for _, n := range needles {
|
||||
if strings.Contains(haystack, n) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package dsn
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestBlame(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, text string
|
||||
want Cause
|
||||
}{
|
||||
// The address is what failed.
|
||||
{"gmail no such user", "550-5.1.1 The email account that you tried to reach does not exist. Please try double-checking the recipient's email address for typos or unnecessary spaces. https://support.google.com/mail/?p=NoSuchUser", CauseRecipientInvalid},
|
||||
{"postfix user unknown", "smtp; 550 5.1.1 <[EMAIL]>: Recipient address rejected: User unknown in virtual mailbox table", CauseRecipientInvalid},
|
||||
{"bare 5.1.1", "5.1.1", CauseRecipientInvalid},
|
||||
{"no such domain", "5.1.2 smtp; 550 5.1.2 Host unknown", CauseRecipientInvalid},
|
||||
{"exchange missing mailbox under 5.4.1", "550 5.4.1 Recipient address rejected: Access denied. AS(201806281) [DB5EUR03FT012.eop-EUR03.prod.protection.outlook.com]", CauseRecipientInvalid},
|
||||
{"exchange resolver", "550 5.1.10 RESOLVER.ADR.RecipientNotFound; Recipient not found by SMTP address lookup", CauseRecipientInvalid},
|
||||
{"gmail disabled", "550 5.2.1 The email account that you tried to reach is disabled.", CauseRecipientInvalid},
|
||||
{"exim unrouteable", "550 Unrouteable address", CauseRecipientInvalid},
|
||||
|
||||
// Something other than the address failed.
|
||||
{"gmail blocked", "550-5.7.1 [[IP]] Our system has detected an unusual rate of unsolicited mail originating from your IP address. To protect our users from spam, mail sent from your IP address has been blocked.", CauseReputationBlock},
|
||||
{"microsoft spamhaus", "550 5.7.1 Service unavailable, Client host [[IP]] blocked using Spamhaus. To request removal from this list see https://www.spamhaus.org/query/ip/[IP] (AS3130)", CauseReputationBlock},
|
||||
{"microsoft ip reputation", "550 5.7.1 Unfortunately, messages from [[IP]] weren't sent. Please contact your Internet service provider", CauseReputationBlock},
|
||||
{"gmail unauthenticated", "550-5.7.26 This mail has been blocked because the sender is unauthenticated.", CauseReputationBlock},
|
||||
{"sender address refused", "550 5.1.8 <bounce@localhost>: Sender address rejected", CauseReputationBlock},
|
||||
{"message too large", "552 5.2.3 Message size exceeds fixed maximum message size", CauseReputationBlock},
|
||||
{"mailbox full", "552 5.2.2 Mailbox full", CauseMailboxFull},
|
||||
{"gmail over quota", "552-5.2.2 The email account that you tried to reach is over quota.", CauseMailboxFull},
|
||||
{"quota policy names the recipient field", "552 <[EMAIL]>: Recipient address rejected: Mailbox full", CauseMailboxFull},
|
||||
{"system problem", "554 5.3.0 Other mail system problem", CauseTransient},
|
||||
{"routing loop", "554 5.4.6 Routing loop detected", CauseTransient},
|
||||
{"greylisted", "451 4.7.1 Greylisted, try again later", CauseTransient},
|
||||
{"deferred over quota", "452 4.2.2 The email account that you tried to reach is over quota", CauseTransient},
|
||||
|
||||
// Nothing telling: the caller keeps its conservative default.
|
||||
{"empty", "", CauseUnknown},
|
||||
{"subject only", "Undelivered Mail Returned to Sender", CauseUnknown},
|
||||
{"bare transaction failure", "554 Transaction failed", CauseUnknown},
|
||||
{"bare 5.4.1", "5.4.1", CauseUnknown},
|
||||
{"undefined status", "5.0.0", CauseUnknown},
|
||||
{"ip address is not a status", "Delivery to [IP] failed", CauseUnknown},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := Blame(tc.text); got != tc.want {
|
||||
t.Fatalf("Blame(%q) = %q, want %q", tc.text, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddressIsFine(t *testing.T) {
|
||||
for cause, want := range map[Cause]bool{
|
||||
CauseUnknown: false, CauseRecipientInvalid: false,
|
||||
CauseMailboxFull: true, CauseReputationBlock: true, CauseTransient: true,
|
||||
} {
|
||||
if cause.AddressIsFine() != want {
|
||||
t.Errorf("%q.AddressIsFine() = %v, want %v", cause, !want, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ package dsn
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// Report is the extracted result of parsing a bounce message.
|
||||
@@ -23,10 +24,23 @@ type Report struct {
|
||||
// OriginalMessageID is the Message-ID of the original outbound message the
|
||||
// bounce concerns, recovered from the returned headers section.
|
||||
OriginalMessageID string
|
||||
// Status is the first recipient's enhanced status code, e.g. "5.2.2".
|
||||
Status string
|
||||
// Action is the first recipient's Action field, lowercased.
|
||||
Action string
|
||||
// Diagnostic is the remote server's reply from Diagnostic-Code without its
|
||||
// type prefix, whitespace collapsed and bounded by ReasonLimit.
|
||||
Diagnostic string
|
||||
}
|
||||
|
||||
// ReasonLimit bounds Report.Reason in runes; a notice can paste a whole transcript.
|
||||
const ReasonLimit = 500
|
||||
|
||||
var (
|
||||
reStatus = regexp.MustCompile(`(?im)^\s*Status:\s*([245])\.\d{1,3}\.\d{1,3}`)
|
||||
reStatusCode = regexp.MustCompile(`(?im)^\s*Status:\s*([245]\.\d{1,3}\.\d{1,3})`)
|
||||
reDiagField = regexp.MustCompile(`(?im)^[ \t]*Diagnostic-Code:[^;\r\n]*;[ \t]*(.*)$`)
|
||||
reFieldName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9-]*:`)
|
||||
reAction = regexp.MustCompile(`(?im)^\s*Action:\s*(failed|delayed|delivered|relayed|expanded)`)
|
||||
reFinalRcpt = regexp.MustCompile(`(?im)^\s*(?:Final|Original)-Recipient:\s*[^;\r\n]*;\s*<?([^\s<>]+@[^\s<>]+?)>?\s*$`)
|
||||
reMessageID = regexp.MustCompile(`(?im)^\s*Message-ID:\s*<([^>\s]+)>`)
|
||||
@@ -104,6 +118,7 @@ func Parse(body string) Report {
|
||||
}
|
||||
if a := reAction.FindStringSubmatch(body); a != nil {
|
||||
r.IsBounce = true
|
||||
r.Action = strings.ToLower(a[1])
|
||||
if strings.EqualFold(a[1], "failed") && r.hasNo4xx(body) {
|
||||
// "failed" is permanent per RFC 3464, but a co-present 4.x.x status
|
||||
// (retry-then-fail) means transient — defer to the status code.
|
||||
@@ -119,10 +134,56 @@ func Parse(body string) Report {
|
||||
if ids := reMessageID.FindAllStringSubmatch(body, -1); len(ids) > 0 {
|
||||
r.OriginalMessageID = strings.TrimSpace(ids[len(ids)-1][1])
|
||||
}
|
||||
if m := reStatusCode.FindStringSubmatch(body); m != nil {
|
||||
r.Status = m[1]
|
||||
}
|
||||
r.Diagnostic = diagnosticText(body)
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
// diagnosticText reads the first Diagnostic-Code value with its folded
|
||||
// continuation lines.
|
||||
func diagnosticText(body string) string {
|
||||
loc := reDiagField.FindStringSubmatchIndex(body)
|
||||
if loc == nil {
|
||||
return ""
|
||||
}
|
||||
parts := []string{body[loc[2]:loc[3]]}
|
||||
for _, line := range strings.Split(body[loc[1]:], "\n")[1:] {
|
||||
line = strings.TrimRight(line, "\r")
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if trimmed == "" || (line[0] != ' ' && line[0] != '\t') || reFieldName.MatchString(trimmed) {
|
||||
break
|
||||
}
|
||||
parts = append(parts, trimmed)
|
||||
}
|
||||
return bound(strings.Join(parts, " "))
|
||||
}
|
||||
|
||||
// bound collapses whitespace, drops control characters and caps the length.
|
||||
func bound(s string) string {
|
||||
s = strings.Join(strings.FieldsFunc(s, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) }), " ")
|
||||
if r := []rune(s); len(r) > ReasonLimit {
|
||||
s = string(r[:ReasonLimit])
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Reason is the notice's own account of the failure, its status code and the
|
||||
// remote server's reply, or fallback when the notice carries neither.
|
||||
func (r Report) Reason(fallback string) string {
|
||||
switch {
|
||||
case r.Diagnostic != "" && (r.Status == "" || strings.Contains(r.Diagnostic, r.Status)):
|
||||
return r.Diagnostic
|
||||
case r.Diagnostic != "":
|
||||
return bound(r.Status + " " + r.Diagnostic)
|
||||
case r.Status != "":
|
||||
return r.Status
|
||||
}
|
||||
return bound(fallback)
|
||||
}
|
||||
|
||||
func (r Report) hasNo4xx(body string) bool {
|
||||
if m := reStatus.FindStringSubmatch(body); m != nil {
|
||||
return m[1] != "4"
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package dsn
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestReasonCarriesStatusAndDiagnostic(t *testing.T) {
|
||||
full := `Final-Recipient: rfc822; [EMAIL]
|
||||
Action: failed
|
||||
Status: 5.2.2
|
||||
Diagnostic-Code: smtp; 552-5.2.2 The email account that you tried to reach is
|
||||
over quota and inactive.
|
||||
Remote-MTA: dns; gmail-smtp-in.l.google.com
|
||||
`
|
||||
policy := "Final-Recipient: rfc822; [EMAIL]\nAction: failed\nStatus: 5.7.1\nDiagnostic-Code: smtp; 550 Service unavailable, Client host blocked using Spamhaus\n"
|
||||
for _, tc := range []struct {
|
||||
name, body, subject, want string
|
||||
}{
|
||||
{"folded diagnostic that already carries the status", full, "Delivery Status Notification (Failure)",
|
||||
"552-5.2.2 The email account that you tried to reach is over quota and inactive."},
|
||||
{"status joined to a diagnostic without one", policy, "Undeliverable",
|
||||
"5.7.1 550 Service unavailable, Client host blocked using Spamhaus"},
|
||||
{"status alone", "Action: failed\nStatus: 5.1.1\n", "Undeliverable", "5.1.1"},
|
||||
{"no status falls back to the subject", "Your message could not be delivered.", "Undeliverable: hello", "Undeliverable: hello"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := Parse(tc.body).Reason(tc.subject); got != tc.want {
|
||||
t.Fatalf("Reason = %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
r := Parse(full)
|
||||
if r.Status != "5.2.2" || r.Action != "failed" {
|
||||
t.Fatalf("status %q action %q", r.Status, r.Action)
|
||||
}
|
||||
if !Blame(r.Reason("")).AddressIsFine() {
|
||||
t.Fatal("a full mailbox must not blame the address")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReasonIsBounded(t *testing.T) {
|
||||
long := "Status: 5.1.1\nDiagnostic-Code: smtp; 550 5.1.1 " + strings.Repeat("x", 4*ReasonLimit) + "\x07\n"
|
||||
got := Parse(long).Reason("")
|
||||
if n := len([]rune(got)); n > ReasonLimit {
|
||||
t.Fatalf("reason is %d runes, limit %d", n, ReasonLimit)
|
||||
}
|
||||
if strings.ContainsRune(got, '\x07') {
|
||||
t.Fatal("control characters must not reach the reason")
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/warmbly/warmbly/internal/pkg/dsn"
|
||||
)
|
||||
|
||||
// A 5xx reply to RCPT TO does NOT on its own mean the recipient is unknown.
|
||||
@@ -15,168 +17,28 @@ import (
|
||||
// hostname" *on the RCPT command*. Reading that as "this address is invalid"
|
||||
// permanently drops a perfectly good contact, which is exactly what issue #200
|
||||
// hit. Only a reply that talks about the RECIPIENT may produce StatusInvalid;
|
||||
// everything else degrades to StatusUnknown, which still sends.
|
||||
|
||||
// recipientMarkers are phrases that name the RECIPIENT as the problem. They are
|
||||
// decisive regardless of the numeric code, because providers disagree about
|
||||
// which code carries them (Microsoft answers a missing mailbox with 5.4.1,
|
||||
// some Exim builds with 554).
|
||||
var recipientMarkers = []string{
|
||||
"user unknown",
|
||||
"unknown user",
|
||||
"no such user",
|
||||
"no such recipient",
|
||||
"no such mailbox",
|
||||
"user not found",
|
||||
"recipient not found",
|
||||
"recipient unknown",
|
||||
"unknown recipient",
|
||||
"recipient address rejected",
|
||||
"recipient rejected",
|
||||
"invalid recipient",
|
||||
"invalid mailbox",
|
||||
"mailbox unavailable",
|
||||
"mailbox not found",
|
||||
"mailbox does not exist",
|
||||
"mailbox is disabled",
|
||||
"mailbox disabled",
|
||||
"account has been disabled",
|
||||
"account is disabled",
|
||||
"account does not exist",
|
||||
"address does not exist",
|
||||
"does not exist",
|
||||
"doesn't exist",
|
||||
"unrouteable address",
|
||||
"unroutable address",
|
||||
"no mailbox here",
|
||||
"not a valid mailbox",
|
||||
"user doesn't have an account",
|
||||
}
|
||||
|
||||
// probeMarkers are phrases that name OUR probe as the problem: the greeting,
|
||||
// the envelope sender, the connecting IP, or a policy applied to the session.
|
||||
// A reply carrying one of these says nothing about whether the mailbox exists.
|
||||
var probeMarkers = []string{
|
||||
"helo",
|
||||
"ehlo",
|
||||
"fully-qualified",
|
||||
"fully qualified",
|
||||
"relay access denied",
|
||||
"relay denied",
|
||||
"relaying denied",
|
||||
"not permitted to relay",
|
||||
"unable to relay",
|
||||
"client host rejected",
|
||||
"client host blocked",
|
||||
"sender address rejected",
|
||||
"sender rejected",
|
||||
"sender verify failed",
|
||||
"sender not allowed",
|
||||
"authentication required",
|
||||
"auth required",
|
||||
"not authenticated",
|
||||
"not authorized",
|
||||
"blacklist",
|
||||
"blocklist",
|
||||
"blocked using",
|
||||
"listed in",
|
||||
"spamhaus",
|
||||
"reputation",
|
||||
"rate limit",
|
||||
"too many",
|
||||
"greylist",
|
||||
"grey-list",
|
||||
"try again",
|
||||
"temporarily",
|
||||
"service unavailable",
|
||||
"connection refused",
|
||||
"policy reasons",
|
||||
"local policy",
|
||||
"security policy",
|
||||
}
|
||||
|
||||
// enhancedStatus is an RFC 3463 status code (class.subject.detail) parsed off
|
||||
// the front of a reply text, e.g. "5.1.1" in "5.1.1 <a@b>: User unknown".
|
||||
type enhancedStatus struct {
|
||||
class, subject, detail int
|
||||
ok bool
|
||||
}
|
||||
|
||||
func parseEnhancedStatus(msg string) enhancedStatus {
|
||||
field := strings.TrimSpace(msg)
|
||||
if i := strings.IndexAny(field, " \t"); i >= 0 {
|
||||
field = field[:i]
|
||||
}
|
||||
parts := strings.Split(strings.TrimSuffix(field, ":"), ".")
|
||||
if len(parts) != 3 {
|
||||
return enhancedStatus{}
|
||||
}
|
||||
nums := make([]int, 3)
|
||||
for i, p := range parts {
|
||||
n, err := strconv.Atoi(p)
|
||||
if err != nil || n < 0 {
|
||||
return enhancedStatus{}
|
||||
}
|
||||
nums[i] = n
|
||||
}
|
||||
if nums[0] != 2 && nums[0] != 4 && nums[0] != 5 {
|
||||
return enhancedStatus{}
|
||||
}
|
||||
return enhancedStatus{class: nums[0], subject: nums[1], detail: nums[2], ok: true}
|
||||
}
|
||||
// everything else degrades to StatusUnknown, which still sends. The rule is
|
||||
// dsn.Blame, shared with the bounce pipeline.
|
||||
|
||||
// classifyPermanentRcpt decides whether a permanent (5xx) RCPT rejection is
|
||||
// about the recipient (a hard invalid we may record) or about the probe (which
|
||||
// must degrade to unknown so a good contact is never dropped).
|
||||
func classifyPermanentRcpt(code int, msg string) (probeOutcome, string) {
|
||||
lower := strings.ToLower(msg)
|
||||
prefix := "recipient rejected (" + strconv.Itoa(code) + "): "
|
||||
rejected := "recipient rejected (" + strconv.Itoa(code) + "): " + msg
|
||||
unjudged := "probe rejected, address not judged (" + strconv.Itoa(code) + "): " + msg
|
||||
|
||||
// 1. An explicit recipient phrase is decisive. Checked first because
|
||||
// Microsoft's "Recipient address rejected: Access denied" carries both a
|
||||
// recipient phrase and a policy-sounding one.
|
||||
if containsAny(lower, recipientMarkers) {
|
||||
return probeRejected, prefix + msg
|
||||
}
|
||||
// 2. An explicit probe/session phrase means the server never judged the
|
||||
// address at all.
|
||||
if containsAny(lower, probeMarkers) {
|
||||
return probeUnknown, "probe rejected, address not judged (" + strconv.Itoa(code) + "): " + msg
|
||||
}
|
||||
// 3. Otherwise trust the enhanced status code when the server sent one.
|
||||
if es := parseEnhancedStatus(msg); es.ok && es.class == 5 {
|
||||
switch es.subject {
|
||||
case 1: // Addressing. 5.1.7/5.1.8 describe the SENDER address.
|
||||
switch es.detail {
|
||||
case 1, 2, 3, 6, 10:
|
||||
return probeRejected, prefix + msg
|
||||
}
|
||||
case 2: // Mailbox. Only "disabled" means gone; full/too-large mailboxes exist.
|
||||
if es.detail == 1 {
|
||||
return probeRejected, prefix + msg
|
||||
}
|
||||
}
|
||||
// 5.3 system, 5.4 routing, 5.5 protocol, 5.6 content, 5.7 policy, and
|
||||
// every unlisted addressing/mailbox detail describe the session.
|
||||
return probeUnknown, "probe rejected, address not judged (" + strconv.Itoa(code) + "): " + msg
|
||||
}
|
||||
// 4. No enhanced status: only the codes that conventionally answer an
|
||||
// unknown mailbox at RCPT count. 500-504 reject the COMMAND, 552 is a
|
||||
// full mailbox, 554 is a catch-all "transaction failed".
|
||||
switch code {
|
||||
case 550, 551, 553:
|
||||
return probeRejected, prefix + msg
|
||||
}
|
||||
return probeUnknown, "probe rejected, address not judged (" + strconv.Itoa(code) + "): " + msg
|
||||
}
|
||||
|
||||
func containsAny(haystack string, needles []string) bool {
|
||||
for _, n := range needles {
|
||||
if strings.Contains(haystack, n) {
|
||||
return true
|
||||
switch dsn.Blame(msg) {
|
||||
case dsn.CauseRecipientInvalid:
|
||||
return probeRejected, rejected
|
||||
case dsn.CauseUnknown:
|
||||
// Without an enhanced status only the codes that conventionally answer
|
||||
// an unknown mailbox count: 500-504 reject the COMMAND, 552 is a full
|
||||
// mailbox, 554 a catch-all "transaction failed".
|
||||
if _, ok := dsn.StatusIn(msg); !ok && (code == 550 || code == 551 || code == 553) {
|
||||
return probeRejected, rejected
|
||||
}
|
||||
}
|
||||
return false
|
||||
return probeUnknown, unjudged
|
||||
}
|
||||
|
||||
// reservedHeloTLDs never resolve on the public internet, so announcing one is
|
||||
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/warmbly/warmbly/internal/pkg/dsn"
|
||||
)
|
||||
|
||||
// Evidence kinds. Positive kinds prove the mailbox is live; the recipient
|
||||
@@ -291,19 +293,5 @@ func clamp(n int) int {
|
||||
// reputation rejection or a greeting the server disliked. Only the former is
|
||||
// evidence against the address.
|
||||
func NamesRecipient(text string) bool {
|
||||
lower := strings.ToLower(text)
|
||||
if containsAny(lower, recipientMarkers) {
|
||||
return true
|
||||
}
|
||||
if containsAny(lower, probeMarkers) {
|
||||
return false
|
||||
}
|
||||
// Enhanced status anywhere in the text: 5.1.1 / 5.1.2 / 5.1.3 / 5.1.6 /
|
||||
// 5.1.10 address errors and 5.2.1 disabled mailbox.
|
||||
for _, code := range []string{"5.1.1", "5.1.2", "5.1.3", "5.1.6", "5.1.10", "5.2.1"} {
|
||||
if strings.Contains(lower, code+" ") || strings.HasSuffix(lower, code) || strings.Contains(lower, code+":") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
return dsn.Blame(text) == dsn.CauseRecipientInvalid
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
@@ -138,6 +139,15 @@ type Client struct {
|
||||
|
||||
type Option func(*Client)
|
||||
|
||||
// WithEndpoint replaces the default endpoint; an empty value keeps it.
|
||||
func WithEndpoint(endpoint string) Option {
|
||||
return func(c *Client) {
|
||||
if endpoint = strings.TrimSpace(endpoint); endpoint != "" {
|
||||
c.endpoint = endpoint
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func NewClient(apiKey string, options ...Option) *Client {
|
||||
c := &Client{
|
||||
endpoint: defaultEndpoint,
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package typesafe
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestWithEndpoint(t *testing.T) {
|
||||
if c := NewClient("k", WithEndpoint(" ")); c.endpoint != defaultEndpoint {
|
||||
t.Fatalf("an empty override must keep the default, got %q", c.endpoint)
|
||||
}
|
||||
hit := false
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
hit = true
|
||||
_, _ = w.Write([]byte(`{"model":"jev-1.13.0","answers":{"q":{"type":"noul","noul":0.9}}}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
if _, err := NewClient("k", WithEndpoint(srv.URL)).Ask(context.Background(), map[string]string{"a": "b"}, map[string]Question{"q": Noul("x")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !hit {
|
||||
t.Fatal("the request did not go to the configured endpoint")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user