diff --git a/admin/src/app/dashboard/configuration/SettingsTab.tsx b/admin/src/app/dashboard/configuration/SettingsTab.tsx index dd566905..9e7bd8b8 100644 --- a/admin/src/app/dashboard/configuration/SettingsTab.tsx +++ b/admin/src/app/dashboard/configuration/SettingsTab.tsx @@ -119,12 +119,37 @@ const RETENTION_PRESETS = [ }, ] as const; +// Engagement-classification windows, mirroring internal/config/constants.go. +// The clock starts when the send is handed to a worker, which is why these are +// larger than "how fast could a person read this": the window also has to +// cover provider queueing and transit before the recipient's gateway sees it. +const MACHINE_WINDOW_MIN_SECONDS = 1; +const MACHINE_WINDOW_MAX_SECONDS = 900; + +const TRACKING_FIELDS = [ + { + key: "machineWindowOpen", + setting: "machine_window_open_seconds", + label: "Automated open window (seconds)", + help: "An open arriving this soon after a send was dispatched is recorded as automated. Raise it when delivery-time scanners are being counted as opens, lower it when recipients who read immediately are being missed.", + }, + { + key: "machineWindowClick", + setting: "machine_window_click_seconds", + label: "Automated click window (seconds)", + help: "The same window for clicks, kept separate because the two mistakes cost different things: a misjudged open loses a metric, a misjudged click loses the automation behind an interested lead.", + }, +] as const; + +type TrackingFieldKey = (typeof TRACKING_FIELDS)[number]["key"]; + interface FormState { linksEnabled: boolean; ttlHours: string; allowInvitedSignup: boolean; sync: Record; retention: Record; + tracking: Record; enforceDomainAuth: boolean; authGraceHours: string; } @@ -145,6 +170,10 @@ function toForm(s: InstanceSettings): FormState { formDays: String(s.retention.form_event_days), auditDays: String(s.retention.audit_log_days), }, + tracking: { + machineWindowOpen: String(s.tracking.machine_window_open_seconds), + machineWindowClick: String(s.tracking.machine_window_click_seconds), + }, enforceDomainAuth: s.deliverability.enforce_domain_auth, authGraceHours: String(s.deliverability.auth_grace_hours), }; @@ -199,6 +228,10 @@ export function SettingsTab({ onDirtyChange, onSwitchTab }: SettingsTabProps) { RETENTION_FIELDS.some( (f) => form.retention[f.key] !== String(server.retention[f.setting]), ); + const trackingDirty = + !!server && + !!form && + TRACKING_FIELDS.some((f) => form.tracking[f.key] !== String(server.tracking[f.setting])); const dirty = !!server && !!form && @@ -208,6 +241,7 @@ export function SettingsTab({ onDirtyChange, onSwitchTab }: SettingsTabProps) { form.enforceDomainAuth !== server.deliverability.enforce_domain_auth || form.authGraceHours !== String(server.deliverability.auth_grace_hours) || retentionDirty || + trackingDirty || syncDirty); useEffect(() => { @@ -222,6 +256,16 @@ export function SettingsTab({ onDirtyChange, onSwitchTab }: SettingsTabProps) { syncFieldValid(form.retention[f.key], RETENTION_MIN_DAYS, RETENTION_MAX_DAYS), ); + const trackingValid = + form !== null && + TRACKING_FIELDS.every((f) => + syncFieldValid( + form.tracking[f.key], + MACHINE_WINDOW_MIN_SECONDS, + MACHINE_WINDOW_MAX_SECONDS, + ), + ); + const authGrace = form ? Number(form.authGraceHours) : NaN; const authGraceValid = form !== null && @@ -256,6 +300,12 @@ export function SettingsTab({ onDirtyChange, onSwitchTab }: SettingsTabProps) { ); return; } + if (!trackingValid) { + toast.error( + `Every automated-engagement window must be a whole number of seconds between ${MACHINE_WINDOW_MIN_SECONDS} and ${MACHINE_WINDOW_MAX_SECONDS}`, + ); + return; + } if (!authGraceValid) { toast.error( `The authentication grace period must be a whole number of hours between ${AUTH_GRACE_MIN_HOURS} and ${AUTH_GRACE_MAX_HOURS}`, @@ -276,6 +326,10 @@ export function SettingsTab({ onDirtyChange, onSwitchTab }: SettingsTabProps) { form_event_days: Number(form.retention.formDays), audit_log_days: Number(form.retention.auditDays), }, + tracking: { + machine_window_open_seconds: Number(form.tracking.machineWindowOpen), + machine_window_click_seconds: Number(form.tracking.machineWindowClick), + }, deliverability: { enforce_domain_auth: form.enforceDomainAuth, auth_grace_hours: authGrace, @@ -541,6 +595,69 @@ export function SettingsTab({ onDirtyChange, onSwitchTab }: SettingsTabProps) { + + + Automated engagement + + Security gateways fetch the tracking pixel and walk every link + when a message arrives, using an ordinary browser's user + agent. An open or click landing inside these windows is recorded + as automated: still kept as delivery evidence and still shown on + the timeline, but it does not count as engagement, fire a branch + or automation, or send a webhook. Nothing is discarded either way. + The clock starts when the send is handed to a worker, so the + window also covers the provider's queue and the transit to + the recipient. Known scanner networks are matched separately and + are not bounded by time. A change applies within a minute and + only to events recorded after it: opens and clicks already + stored keep the label they were given when they arrived. + + + + {TRACKING_FIELDS.map((f) => { + const valid = syncFieldValid( + form.tracking[f.key], + MACHINE_WINDOW_MIN_SECONDS, + MACHINE_WINDOW_MAX_SECONDS, + ); + return ( +
+ + + setForm({ + ...form, + tracking: { + ...form.tracking, + [f.key]: e.target.value, + }, + }) + } + aria-invalid={!valid} + className="mt-1" + /> +

+ {f.help} Between {MACHINE_WINDOW_MIN_SECONDS} and{" "} + {MACHINE_WINDOW_MAX_SECONDS.toLocaleString()} seconds. +

+ {!valid && ( +

+ Enter a whole number between{" "} + {MACHINE_WINDOW_MIN_SECONDS} and{" "} + {MACHINE_WINDOW_MAX_SECONDS.toLocaleString()}. +

+ )} +
+ ); + })} + + + Sending-domain authentication diff --git a/admin/src/lib/api/client/admin/instance.ts b/admin/src/lib/api/client/admin/instance.ts index 19430ecd..a82fd656 100644 --- a/admin/src/lib/api/client/admin/instance.ts +++ b/admin/src/lib/api/client/admin/instance.ts @@ -121,6 +121,15 @@ export interface InstanceSettings { form_event_days: number; audit_log_days: number; }; + // How soon after a send an open or click is recorded as automated. The + // clock starts at dispatch to the worker, so the window also covers the + // provider's queue and transit to the recipient, not just reading time. + // Nothing is discarded: an automated event stays as delivery evidence but + // does not count as engagement or fire anything. + tracking: { + machine_window_open_seconds: number; + machine_window_click_seconds: number; + }; // The sending-domain authentication gate. A mailbox whose domain has been // failing SPF or DMARC for longer than the grace window stops sending cold // mail and warmup mail until the records are fixed. diff --git a/cmd/consumer/main.go b/cmd/consumer/main.go index 4d4a2f0c..631d89d4 100644 --- a/cmd/consumer/main.go +++ b/cmd/consumer/main.go @@ -536,8 +536,13 @@ func main() { log.Println("tracking consumer unavailable; opens/clicks not consumed:", terr) } else { // The engagement prune reads its window from the instance settings on - // every pass, so shortening it in the admin panel needs no restart. - trackingConsumer.WireRetention(instancesettings.NewService(instancesettings.NewStore(primaryDB.Pool))) + // every pass, and the machine-window rule reads its windows per event, + // so editing either in the admin panel needs no restart. Both go + // through this process's own read cache, so an edit lands within its + // TTL rather than instantly. + trackingSettings := instancesettings.NewService(instancesettings.NewStore(primaryDB.Pool)) + trackingConsumer.WireRetention(trackingSettings) + trackingConsumer.WireTrackingPolicy(trackingSettings) defer trackingConsumer.Close() go func() { if err := trackingConsumer.Start(ctx); err != nil { diff --git a/docs/content/docs/api/reference/analytics.mdx b/docs/content/docs/api/reference/analytics.mdx index a09387ab..c499f12d 100644 --- a/docs/content/docs/api/reference/analytics.mdx +++ b/docs/content/docs/api/reference/analytics.mdx @@ -271,7 +271,7 @@ Returns a single campaign's performance summary plus per-sequence-step stats. Th `engagement` groups the campaign's human opens and clicks by country (ISO code), mail client or browser, and device type (`desktop`, `mobile`, `tablet`), counting distinct contacts per bucket; a click counts as an open. Each list holds the busiest eight; an empty `key` is unknown. Country needs the GeoLite2 database on the consumer, otherwise every country row is unknown. -`machine_opens` is the subset of `unique_opens` from automated fetchers (Apple MPP prefetch, UA-less clients, opens inside ten seconds of the send); human opens are `unique_opens` minus `machine_opens`. `machine_clicks` counts steps whose only clicks were automated (a security gateway walking the links); those are not part of `unique_clicks` or `total_clicks`, which only ever count a person's click. +`machine_opens` is the subset of `unique_opens` from automated fetchers (Apple MPP prefetch, UA-less clients, opens inside the instance's automated-open window, which starts when the step is dispatched to a worker); human opens are `unique_opens` minus `machine_opens`. `machine_clicks` counts steps whose only clicks were automated (a security gateway walking the links); those are not part of `unique_clicks` or `total_clicks`, which only ever count a person's click. ## Get campaign daily stats diff --git a/docs/content/docs/api/reference/contacts.mdx b/docs/content/docs/api/reference/contacts.mdx index 6e1ecf9a..48e0d031 100644 --- a/docs/content/docs/api/reference/contacts.mdx +++ b/docs/content/docs/api/reference/contacts.mdx @@ -782,7 +782,7 @@ Returns a `data` array and the standard `pagination` envelope. Paginate by passi `type` is one of `email_sent`, `email_opened`, `email_clicked`, `email_replied`, `email_bounced`, `reply_received`, `deliverability`, `suppressed`, `note`, `meeting_booked`, `meeting_rescheduled`, `meeting_canceled`, `contact_created`, `campaign_added`, `campaign_removed`, `category_added`, `category_removed`, `form_submitted`, or `page_hit`. Fields not relevant to an event type are omitted. -`email_opened` and `email_clicked` carry `machine`: `true` when an automated fetcher did it rather than the person (a mail privacy proxy, a fetch inside ten seconds of the send, several links followed within seconds). Per-link `email_clicked` events carry `machine_reason` (`prefetch`, `instant` or `burst`), and per-event `email_opened` rows carry it too (`prefetch` or `instant`); an open summarised from the lead alone carries only the flag. Both kinds carry an `origin` object when the event was logged: `client` when the user agent names a mail client or image proxy, `device_type`, `os`, `browser`, `browser_version`, and `country_code`, `region`, `city` when the consumer could resolve them. Opens appear once per event, so a contact who opened from two devices has two rows. Automated clicks are on the feed for the record but never count the step as clicked. An `email_clicked` event carries `link` with the link's `id`, `url`, `label` (its anchor text), the `utm_*` parameters the URL carried, and the `user_agent`; every link in an email is tracked on its own, so each link clicked is its own event. Clicks recorded before per-link attribution have no `link`. +`email_opened` and `email_clicked` carry `machine`: `true` when an automated fetcher did it rather than the person (a mail privacy proxy, a fetch inside the instance's automated-engagement window, which starts when the step is dispatched to a worker, several links followed within seconds). Per-link `email_clicked` events carry `machine_reason` (`prefetch`, `instant` or `burst`), and per-event `email_opened` rows carry it too (`prefetch` or `instant`); an open summarised from the lead alone carries only the flag. Both kinds carry an `origin` object when the event was logged: `client` when the user agent names a mail client or image proxy, `device_type`, `os`, `browser`, `browser_version`, and `country_code`, `region`, `city` when the consumer could resolve them. Opens appear once per event, so a contact who opened from two devices has two rows. Automated clicks are on the feed for the record but never count the step as clicked. An `email_clicked` event carries `link` with the link's `id`, `url`, `label` (its anchor text), the `utm_*` parameters the URL carried, and the `user_agent`; every link in an email is tracked on its own, so each link clicked is its own event. Clicks recorded before per-link attribution have no `link`. A `form_submitted` event carries `form_id` and `form_name`. A `page_hit` event is a page view on your own site from a browser tied to the contact through an email-link ticket (see [Website tracking](/guides/website-tracking/)); `subject` is the page title, or its path when the page has none, and `page_hit` carries the full view: `url`, `path`, `title`, `referrer`, `referrer_domain`, `landing` (the first view of a session), the `utm_*` parameters, `device_type`, `os`, `browser`, `browser_version`, `device_brand`, `language`, `timezone`, `screen_width`, `screen_height`, and `country_code`, `region`, `city` when known. diff --git a/docs/content/docs/development/configuration.mdx b/docs/content/docs/development/configuration.mdx index 648dc8be..70439be5 100644 --- a/docs/content/docs/development/configuration.mdx +++ b/docs/content/docs/development/configuration.mdx @@ -554,7 +554,7 @@ The Rust open and click service. It reads its own environment, so these have to | `TRACKING_TRUSTED_PROXIES` | CIDRs the tracking service accepts a forwarded client address from. Empty trusts nothing and uses the socket peer, which is correct for a directly exposed service; set it behind a reverse proxy or the per-source rate limits and the location stored with page views are caller-controlled. Same convention as the backend's `TRUSTED_PROXIES` | empty | | `TRACKING_IP_HASH_KEY` | Secret the source-address token in tracking events is keyed with. The token names one source for deduplication, rate limits and the click burst rule; keyed, it cannot be turned back into the address by enumeration | `INTERNAL_API_TOKEN` | | `TRACKING_CLIENT_IP_HEADER` | The one header a trusted proxy sets with the client address. No other header is read, so a caller cannot smuggle an address past a generic proxy in `CF-Connecting-IP`. For `x-forwarded-for` the proxy-appended last entry is used; set `cf-connecting-ip` behind Cloudflare | `x-forwarded-for` | -| `TRACKING_SCANNER_BUILTINS` | Whether the known-scanner catalogue shipped with Warmbly is loaded. `false` leaves only your own entries | `true` | +| `TRACKING_SCANNER_BUILTINS` | Whether the known-scanner catalogue shipped with Warmbly is loaded: Microsoft 365's Exchange Online Protection ranges and Barracuda's Email Gateway Defense blocks. `false` leaves only your own entries | `true` | | `TRACKING_SCANNER_NETWORKS` | Extra scanner sources, comma separated, each a CIDR or `asn:` with an optional label. For sources that never carry a person's own request, so their pixel fetches and their clicks are both treated as automated | empty | | `TRACKING_SCANNER_CLICK_NETWORKS` | The same, for sources that also proxy a mail client's own image fetches. Only their click tickets are treated as automated, because their pixel fetches are genuine opens | empty | | `TRACKING_SCANNER_ASN_HEADER` | Header a trusted proxy sets with the source ASN, which is what makes `asn:` entries match. Read only from a proxy in `TRACKING_TRUSTED_PROXIES`. On Cloudflare, a transform rule writing `ip.src.asnum`. Empty disables ASN matching | empty | @@ -569,6 +569,8 @@ The Rust open and click service. It reads its own environment, so these have to | `SENTRY_DSN` | The same through Sentry. An invalid value logs and disables it rather than stopping the service | unset | | `WARMBLY_RELEASE` | The build reported errors are tagged with | `dev` | +The shipped catalogue also carries Proofpoint, Mimecast and Cisco Secure Email by ASN, and the rest of Microsoft's and Google's networks, all commented out. The file (`tracking/scanner-networks.txt`) explains each one; the short version is that a whole network is only safe to treat as a scanner when no recipient can be behind it, and these fail that test in two different ways. Microsoft's and Google's clouds carry ordinary browsing, so a recipient on Windows 365 or a corporate NAT gateway sits inside them. Proofpoint Isolation and Mimecast Browser Isolation render a clicked page in the vendor's own cloud and stream it to the recipient, so when a policy sends a link to isolation the click on your ticket comes from the vendor's network with a person on the other end. Isolation is usually scoped to uncategorised URLs, which is what a new outreach domain looks like. Enable these when your recipients' scanner noise costs you more than the clicks and automations they will take with them, and prefer leaving the automated-engagement windows to catch the delivery-time half. + ## Realtime service The Elixir websocket service. Its runtime configuration is read only when the release boots in `prod`, which is how the shipped image runs. Like tracking, it reads its own environment and appears nowhere in the admin panel. @@ -615,6 +617,8 @@ These are the only settings a browser can change, and no environment variable ow | `retention.engagement_event_days` | integer, 1 to 3650 | `365` | How long the per-event open and click logs (client, device, approximate location) are kept. Campaign progress keeps its own summary that outlives them, so counts, filters and branching never change | | `retention.form_event_days` | integer, 1 to 3650 | `180` | How long form funnel events (views, starts, field-level drop-off) are kept. Funnel reports range up to 90 days, so anything shorter shortens the report too | | `retention.audit_log_days` | integer, 1 to 3650 | `90` | How long the audit trail is kept. It carries IP addresses, user agents and change payloads, so this is also how long that data is held | +| `tracking.machine_window_open_seconds` | integer, 1 to 900 | `60` | How soon after a send was dispatched an open is recorded as automated rather than a person's. The clock starts when the send is handed to a worker, so this window also covers the sending provider's queue and the transit to the recipient, not just reading time. Raise it when delivery-time scanners are being counted as opens; lower it when recipients who read immediately are being missed. A change applies within a minute and only to events recorded after it | +| `tracking.machine_window_click_seconds` | integer, 1 to 900 | `30` | The same window for click tickets. Kept separate because the two mistakes cost different things: a misjudged open loses a metric, a misjudged click loses the automation behind an interested lead | | `deliverability.enforce_domain_auth` | boolean | `true` | Whether a sending domain that fails SPF or DMARC stops cold campaign sending and warmup sending from every mailbox on it. Off keeps the check running and still shows the state and the Advisor card, it just never blocks | | `deliverability.auth_grace_hours` | integer, 1 to 720 | `72` | How long a domain must stay failing before the gate applies. The clock starts when the background check first sees the failure, so this is also how much warning the owner gets | diff --git a/docs/content/docs/development/events.mdx b/docs/content/docs/development/events.mdx index e1364756..770b9150 100644 --- a/docs/content/docs/development/events.mdx +++ b/docs/content/docs/development/events.mdx @@ -83,7 +83,7 @@ Produced by the Rust tracking service when a pixel loads or a tracked link is cl `event_type` is `EMAIL_OPENED` or `EMAIL_CLICKED`; `original_url` and `link_id` (the click ticket, which names the link's stored destination and anchor text) are set only for clicks; IPs are stored as hashes, never raw. The struct is `events.TrackingEvent` in `internal/events/schemas.go`, mirrored in `tracking/src/events.rs`. -The consumer classifies each event before it counts. An open is automated when the user agent is a mail privacy proxy or missing, or when it arrives within `TrackingMachineWindowSeconds` of the step's dispatch; it is recorded with `opened_machine` and upgraded by a later human open. A click is automated for a missing user agent, for arriving inside the same window, or when the same source clicked another link of the same email within `TrackingClickBurstSeconds`; every click is logged per link in `email_link_clicks` with its reason, and only a human click stamps `clicked_at`, fires instant actions, or emits a webhook. A burst is only recognisable from its second click, so a human click's stamp and log row are written at once but its effects (evidence, instant actions, webhook, realtime event) run after the burst window plus a second, on the classification the click has by then; a burst recognised meanwhile relabels the earlier click and withdraws the stamp when no human click remains. A consumer restart inside the window loses only those deferred effects, and routing still follows the clicked branch at the next step boundary. +The consumer classifies each event before it counts. An open is automated when the user agent is a mail privacy proxy or missing, or when it arrives within `tracking.machine_window_open_seconds` of the step's dispatch; it is recorded with `opened_machine` and upgraded by a later human open. A click is automated for a missing user agent, for arriving inside `tracking.machine_window_click_seconds` of the same dispatch (`30` by default, set independently of the open window and accepting the same 1 to 900 seconds; it ships shorter because a misjudged click costs an automation rather than a metric), or when the same source clicked another link of the same email within `TrackingClickBurstSeconds`; every click is logged per link in `email_link_clicks` with its reason, and only a human click stamps `clicked_at`, fires instant actions, or emits a webhook. A burst is only recognisable from its second click, so a human click's stamp and log row are written at once but its effects (evidence, instant actions, webhook, realtime event) run after the burst window plus a second, on the classification the click has by then; a burst recognised meanwhile relabels the earlier click and withdraws the stamp when no human click remains. A consumer restart inside the window loses only those deferred effects, and routing still follows the clicked branch at the next step boundary. Website page views do not ride this topic. The tracking service forwards each accepted view to the backend's internal API (`POST /api/v1/internal/page-hits`) instead, because the backend is where the user agent and IP are turned into device and location, and the IP must not sit in a durable stream on the way there. diff --git a/docs/content/docs/guides/analytics.mdx b/docs/content/docs/guides/analytics.mdx index d478d2a4..e65f7a8c 100644 --- a/docs/content/docs/guides/analytics.mdx +++ b/docs/content/docs/guides/analytics.mdx @@ -20,7 +20,7 @@ Six rules govern the counts: - **Tracking must be on.** Opens and clicks need open or link tracking enabled on the campaign, and a tracking host on the install. A [custom tracking domain](/guides/mailboxes/#custom-tracking-domain) per mailbox is optional; without one they go through the shared host and still count. With tracking off they stay at zero while sends and replies still count. - **Replies are human replies.** Out-of-office and autoresponders never count, never stamp the contact as replied, and never trip stop-on-reply. -- **Bots are filtered.** Crawlers, CLI agents, prefetches, chat link previews, and security gateways that announce themselves are served normally but never counted. The ones that do not announce themselves are caught by where they came from and by what they do. Requests from a known mail-filtering network are treated as automated whatever their user agent claims, and a click from one is still redirected, just without the ticket that would file the visit against the recipient. Microsoft 365's own filtering layer, where Safe Links and delivery-time link scanning run, is recognised out of the box. The rest are caught by behaviour: an open or click inside ten seconds of the send (nobody reads that fast; the clock starts when the send is handed to the worker, before the mail has even been delivered) and clicks on two or more links of one email from the same source within five seconds (a scanner walking the message). Otherwise one corporate scanner would "click" every link seconds after delivery. A self-hosted instance can name its own scanner networks; see [tracking service configuration](/development/configuration/). +- **Bots are filtered.** Crawlers, CLI agents, prefetches, chat link previews, and security gateways that announce themselves are served normally but never counted. The ones that do not announce themselves are caught by where they came from and by what they do. Requests from a known mail-filtering network are treated as automated whatever their user agent claims, and a click from one is still redirected, just without the ticket that would file the visit against the recipient. Microsoft 365's own filtering layer, where Safe Links and delivery-time link scanning run, is recognised out of the box, as is Barracuda's. The rest are caught by behaviour: an open within a minute of the send, or a click within thirty seconds, and clicks on two or more links of one email from the same source within five seconds (a scanner walking the message). The clock on the first two starts when the send is handed to a worker, before the mail has even been delivered, so those windows cover the sending provider's queue and the transit to the recipient as well as reading time. A self-hosted instance can change both under Instance settings when its recipients' gateways are slower or faster than that. Otherwise one corporate scanner would "click" every link seconds after delivery. A self-hosted instance can name its own scanner networks; see [tracking service configuration](/development/configuration/). - **Auto-opens are labeled, not hidden.** Privacy proxies like Apple Mail Privacy Protection, and instant opens, still count (they confirm delivery) but are tagged and shown separately (`12 auto`). A later real open upgrades them to human. **Auto-opens never trigger opened-based branches or automations.** - **A click is an open.** Opens need the mail client to load images, and many clients block them. A click by the person proves the email was read, so it counts as an open as well, and a lead can never read "clicked, not opened". - **Auto-clicks are kept but never counted.** A click classified as automated is logged on the contact's activity with the link it hit and the rule that caught it, and the campaign overview shows how many steps had only automated clicks (`3 auto`). It never makes the step clicked, never fires a clicked branch or automation, and never sends a webhook. A person clicking the same link later counts normally. A burst is only recognisable from its second click, so a click that looks human waits for the burst window plus one second (six seconds) before it fires anything (clicked branches, automations, webhooks, the live feed): if a second link follows inside that window, both clicks are relabeled, the stamp is withdrawn and nothing fires. diff --git a/docs/content/docs/guides/campaigns.mdx b/docs/content/docs/guides/campaigns.mdx index 13c0f59e..25cb89af 100644 --- a/docs/content/docs/guides/campaigns.mdx +++ b/docs/content/docs/guides/campaigns.mdx @@ -116,7 +116,7 @@ A lead is **Processing** only while steps remain, so a finished campaign reads a ### Who opened, clicked and replied -Next to each lead's status, the Leads list shows three engagement columns: **Opened**, **Clicked** and **Replied**, each with the number of emails in the sequence the person engaged with. A dash means the lead was emailed and has not engaged; the cell is blank for a lead not emailed yet. An open counts when a person opened the email, or clicked a link in it. Mail clients that fetch every image automatically (Apple Mail Privacy Protection, for example) show as **auto** instead, the same opens the campaign overview reports as automatic, so they never pass for engagement. Clicks are held to the same standard: a link followed from a known mail security network, within ten seconds of the send, or several links of one email followed within a few seconds of each other, is a security gateway scanning the message, not the recipient. Those clicks are kept on the contact's activity marked **auto**, counted apart on the campaign overview, and never make a lead **Clicked**, never fire a clicked branch or automation, and never send a webhook. See [Link tracking and UTM parameters](#link-tracking-and-utm-parameters). +Next to each lead's status, the Leads list shows three engagement columns: **Opened**, **Clicked** and **Replied**, each with the number of emails in the sequence the person engaged with. A dash means the lead was emailed and has not engaged; the cell is blank for a lead not emailed yet. An open counts when a person opened the email, or clicked a link in it. Mail clients that fetch every image automatically (Apple Mail Privacy Protection, for example) show as **auto** instead, the same opens the campaign overview reports as automatic, so they never pass for engagement. Clicks are held to the same standard: a link followed from a known mail security network, within the instance's automated-click window after the send (thirty seconds by default), or several links of one email followed within a few seconds of each other, is a security gateway scanning the message, not the recipient. Those clicks are kept on the contact's activity marked **auto**, counted apart on the campaign overview, and never make a lead **Clicked**, never fire a clicked branch or automation, and never send a webhook. See [Link tracking and UTM parameters](#link-tracking-and-utm-parameters). Opens depend on the recipient's mail client loading images. Outlook, many corporate setups and some privacy-minded clients block them, so a person can read an email and still show no open; the info icon on the column says as much. A click is stronger evidence, so a click by the person always counts as an open too. Open the contact's Activity tab to see each open and click with the mail client and location it came from. diff --git a/internal/api/handler/admin_instance.go b/internal/api/handler/admin_instance.go index c10ab46a..19005928 100644 --- a/internal/api/handler/admin_instance.go +++ b/internal/api/handler/admin_instance.go @@ -127,6 +127,8 @@ func instanceSettingsAuditDetails(doc instancesettings.Document) map[string]any "retention_engagement_event_days": doc.Retention.EngagementEventDays, "retention_form_event_days": doc.Retention.FormEventDays, "retention_audit_log_days": doc.Retention.AuditLogDays, + "tracking_machine_window_open": doc.Tracking.MachineWindowOpenSeconds, + "tracking_machine_window_click": doc.Tracking.MachineWindowClickSeconds, "deliverability_enforce_domain_auth": doc.Deliverability.EnforceDomainAuth, "deliverability_auth_grace_hours": doc.Deliverability.AuthGraceHours, "notification_channels": len(doc.Notifications.Channels), diff --git a/internal/app/consumer/event_tracking.go b/internal/app/consumer/event_tracking.go index 7c4dea2f..56116135 100644 --- a/internal/app/consumer/event_tracking.go +++ b/internal/app/consumer/event_tracking.go @@ -53,6 +53,7 @@ type TrackingConsumer struct { // retention is the operator-editable window the engagement prune obeys. // Injected post-construction; nil keeps the compiled default. retention RetentionSource + tracking TrackingPolicySource topic string group string } @@ -76,6 +77,30 @@ func (tc *TrackingConsumer) engagementRetentionDays(ctx context.Context) int { return tc.retention.RetentionWindows(ctx).EngagementEventDays } +// TrackingPolicySource is the operator-editable engagement-classification +// section, satisfied by instancesettings.Service. Read per event rather than +// held from boot, so an edit in the admin panel needs no restart. The +// consumer's own service caches for instancesettings.cacheTTL and the backend +// that wrote the row is a different process, so an edit lands within that TTL, +// not on the very next event. +type TrackingPolicySource interface { + TrackingPolicy(ctx context.Context) instancesettings.Tracking +} + +// WireTrackingPolicy attaches the instance settings the machine-window rule +// reads its windows from. +func (tc *TrackingConsumer) WireTrackingPolicy(src TrackingPolicySource) { tc.tracking = src } + +// machineWindows are the windows the next classification uses. An unwired +// source falls back to the compiled defaults, which is what the consumer runs +// on before the settings document has ever been written. +func (tc *TrackingConsumer) machineWindows(ctx context.Context) instancesettings.Tracking { + if tc.tracking == nil { + return instancesettings.DefaultTracking() + } + return tc.tracking.TrackingPolicy(ctx) +} + // NewTrackingConsumer wires the tracking consumer to the shared event bus. func NewTrackingConsumer( bus eventbus.EventBus, @@ -273,11 +298,12 @@ func (tc *TrackingConsumer) HandleTrackingEvent(ctx context.Context, event *even // labelled, and must never fire open-triggered automations. var machine bool var reason string + windows := tc.machineWindows(ctx) switch event.EventType { case events.EventTypeEmailOpened: - machine, reason = classifyOpen(event.UserAgent, event.Scanner, sentAt, at) + machine, reason = classifyOpen(event.UserAgent, event.Scanner, sentAt, at, windows.OpenWindow()) case events.EventTypeEmailClicked: - machine, reason = classifyClick(event.UserAgent, event.Scanner, sentAt, at) + machine, reason = classifyClick(event.UserAgent, event.Scanner, sentAt, at, windows.ClickWindow()) default: // Unknown event type, skip return nil diff --git a/internal/app/consumer/open_class.go b/internal/app/consumer/open_class.go index 8e30975f..ae4996a6 100644 --- a/internal/app/consumer/open_class.go +++ b/internal/app/consumer/open_class.go @@ -5,7 +5,6 @@ import ( "time" "github.com/mileusna/useragent" - "github.com/warmbly/warmbly/internal/config" "github.com/warmbly/warmbly/internal/repository" ) @@ -34,15 +33,27 @@ func isMachineOpen(userAgent *string) bool { } // isInstant reports whether an engagement arrived so soon after the step was -// dispatched that no person could have read the email yet. Security -// gateways (Safe Links, Proofpoint, Mimecast) open the pixel and walk every -// link at delivery time with an ordinary browser UA, which is exactly what -// the UA rules cannot see. An unknown dispatch time never counts as instant. -func isInstant(sentAt *time.Time, at time.Time) bool { +// dispatched that no person could have read the email yet. Security gateways +// (Safe Links, Proofpoint, Mimecast) open the pixel and walk every link at +// delivery time with an ordinary browser UA, which is exactly what the UA +// rules cannot see. +// +// The anchor is dispatch to the worker, so `window` has to cover the SMTP +// handshake, the sending provider's queue and transit to the recipient before +// the arrival scan it is aimed at. It is operator-editable for that reason: +// how long that takes is a property of the deployment, not of the code. +// +// An unknown dispatch time never counts as instant. Neither does an event +// stamped BEFORE the dispatch, which means the two clocks disagree rather than +// that someone read the mail early: the timing rule abstains there and the +// event is left to the user agent and source-network rules, which is the only +// honest answer when the one input this rule has is known to be wrong. +func isInstant(sentAt *time.Time, at time.Time, window time.Duration) bool { if sentAt == nil { return false } - return at.Sub(*sentAt) < time.Duration(config.TrackingMachineWindowSeconds)*time.Second + since := at.Sub(*sentAt) + return since >= 0 && since < window } // isScannerSource reports whether the tracking edge recognised the request's @@ -56,14 +67,14 @@ func isScannerSource(scanner *string) bool { // classifyClick applies the per-event click rules (the burst rule needs the // click log and lives in the consumer). It returns whether the click is // automated and the reason recorded with it; an empty reason is a person. -func classifyClick(userAgent, scanner *string, sentAt *time.Time, at time.Time) (bool, string) { +func classifyClick(userAgent, scanner *string, sentAt *time.Time, at time.Time, window time.Duration) (bool, string) { if isScannerSource(scanner) { return true, repository.LinkClickReasonScanner } if userAgent == nil || strings.TrimSpace(*userAgent) == "" { return true, repository.LinkClickReasonPrefetch } - if isInstant(sentAt, at) { + if isInstant(sentAt, at, window) { return true, repository.LinkClickReasonInstant } return false, "" @@ -86,14 +97,14 @@ func eventTime(stamp string) time.Time { // caught it: scanner for a fetch from a known mail-filtering network, // prefetch for a mail proxy or a fetch with no browser, instant for a fetch // inside the machine window after dispatch. An empty reason is a person. -func classifyOpen(userAgent, scanner *string, sentAt *time.Time, at time.Time) (bool, string) { +func classifyOpen(userAgent, scanner *string, sentAt *time.Time, at time.Time, window time.Duration) (bool, string) { if isScannerSource(scanner) { return true, repository.EmailOpenReasonScanner } if isMachineOpen(userAgent) { return true, repository.EmailOpenReasonPrefetch } - if isInstant(sentAt, at) { + if isInstant(sentAt, at, window) { return true, repository.EmailOpenReasonInstant } return false, "" diff --git a/internal/app/consumer/open_class_test.go b/internal/app/consumer/open_class_test.go index e43e27ed..d1a3187e 100644 --- a/internal/app/consumer/open_class_test.go +++ b/internal/app/consumer/open_class_test.go @@ -4,6 +4,7 @@ import ( "testing" "time" + "github.com/warmbly/warmbly/internal/app/instancesettings" "github.com/warmbly/warmbly/internal/repository" ) @@ -11,28 +12,77 @@ func strp(s string) *string { return &s } func TestIsInstantUsesTheDispatchClock(t *testing.T) { sent := time.Now() - if !isInstant(&sent, sent.Add(3*time.Second)) { + window := time.Minute + if !isInstant(&sent, sent.Add(3*time.Second), window) { t.Fatal("three seconds after dispatch is a machine") } - if isInstant(&sent, sent.Add(45*time.Second)) { - t.Fatal("forty-five seconds after dispatch can be a person") + if isInstant(&sent, sent.Add(90*time.Second), window) { + t.Fatal("past the window is a person") } - if isInstant(nil, sent) { + if isInstant(nil, sent, window) { t.Fatal("an unknown dispatch time must never count as instant") } + // The window is a half-open interval, so the boundary itself is already + // out. Without this the two windows would overlap by a second. + if isInstant(&sent, sent.Add(window), window) { + t.Fatal("the boundary is outside the window") + } + // A stamp before the dispatch means the two clocks disagree, not that + // someone read the mail early. The old form compared a raw difference, so + // every skewed event fell inside the window and was marked automated; + // the rule abstains instead and lets the user agent and network decide. + if isInstant(&sent, sent.Add(-time.Hour), window) { + t.Fatal("an event stamped before dispatch is not instant") + } +} + +// The window is a deployment property, not a constant: the clock starts when +// the send is handed to the worker, so it has to cover provider queueing and +// transit before the recipient's gateway has even seen the message. +func TestMachineWindowsAreOperatorEditable(t *testing.T) { + sent := time.Now() + chrome := strp("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36") + at := sent.Add(90 * time.Second) + + if m, _ := classifyOpen(chrome, nil, &sent, at, instancesettings.DefaultTracking().OpenWindow()); m { + t.Fatal("ninety seconds is past the shipped open window") + } + widened := instancesettings.Tracking{MachineWindowOpenSeconds: 120} + widened.Normalize() + if m, r := classifyOpen(chrome, nil, &sent, at, widened.OpenWindow()); !m || r != repository.EmailOpenReasonInstant { + t.Fatalf("a widened window catches it, got %v %q", m, r) + } +} + +// Opens and clicks are tuned separately because the two mistakes cost +// different things: a misjudged open loses a metric, a misjudged click loses +// the automation behind an interested lead. +func TestOpenAndClickWindowsAreIndependent(t *testing.T) { + sent := time.Now() + chrome := strp("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36") + windows := instancesettings.DefaultTracking() + at := sent.Add(45 * time.Second) + + if m, r := classifyOpen(chrome, nil, &sent, at, windows.OpenWindow()); !m || r != repository.EmailOpenReasonInstant { + t.Fatalf("forty-five seconds is inside the shipped open window, got %v %q", m, r) + } + if m, r := classifyClick(chrome, nil, &sent, at, windows.ClickWindow()); m || r != "" { + t.Fatalf("the same moment is outside the shipped click window, got %v %q", m, r) + } } func TestClassifyClick(t *testing.T) { sent := time.Now() + clickWindow := instancesettings.DefaultTracking().ClickWindow() chrome := strp("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36") - if m, r := classifyClick(nil, nil, &sent, sent.Add(time.Minute)); !m || r != repository.LinkClickReasonPrefetch { + if m, r := classifyClick(nil, nil, &sent, sent.Add(time.Minute), clickWindow); !m || r != repository.LinkClickReasonPrefetch { t.Fatalf("no user agent = prefetch, got %v %q", m, r) } - if m, r := classifyClick(chrome, nil, &sent, sent.Add(2*time.Second)); !m || r != repository.LinkClickReasonInstant { + if m, r := classifyClick(chrome, nil, &sent, sent.Add(2*time.Second), clickWindow); !m || r != repository.LinkClickReasonInstant { t.Fatalf("a browser UA two seconds after dispatch = instant, got %v %q", m, r) } - if m, r := classifyClick(chrome, nil, &sent, sent.Add(time.Minute)); m || r != "" { + if m, r := classifyClick(chrome, nil, &sent, sent.Add(time.Minute), clickWindow); m || r != "" { t.Fatalf("a browser a minute later is a person, got %v %q", m, r) } } @@ -44,21 +94,23 @@ func TestClassifyClick(t *testing.T) { func TestClassifyScannerSourceOutranksTheUserAgent(t *testing.T) { sent := time.Now() late := sent.Add(time.Hour) + windows := instancesettings.DefaultTracking() + openWindow, clickWindow := windows.OpenWindow(), windows.ClickWindow() chrome := strp("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36") net := strp("microsoft-365-protection") - if m, r := classifyClick(chrome, net, &sent, late); !m || r != repository.LinkClickReasonScanner { + if m, r := classifyClick(chrome, net, &sent, late, clickWindow); !m || r != repository.LinkClickReasonScanner { t.Fatalf("a click from a scanner network = scanner, got %v %q", m, r) } - if m, r := classifyOpen(chrome, net, &sent, late); !m || r != repository.EmailOpenReasonScanner { + if m, r := classifyOpen(chrome, net, &sent, late, openWindow); !m || r != repository.EmailOpenReasonScanner { t.Fatalf("an open from a scanner network = scanner, got %v %q", m, r) } // An empty label is the same as none: the edge recognised nothing, and a // blank string must not silently condemn every event that carries it. - if m, r := classifyOpen(chrome, strp(" "), &sent, late); m || r != "" { + if m, r := classifyOpen(chrome, strp(" "), &sent, late, openWindow); m || r != "" { t.Fatalf("a blank scanner label is not a verdict, got %v %q", m, r) } - if m, r := classifyClick(chrome, nil, &sent, late); m || r != "" { + if m, r := classifyClick(chrome, nil, &sent, late, clickWindow); m || r != "" { t.Fatalf("no scanner label leaves the click a person's, got %v %q", m, r) } } diff --git a/internal/app/consumer/tracking_window_live_test.go b/internal/app/consumer/tracking_window_live_test.go new file mode 100644 index 00000000..ccdde3a1 --- /dev/null +++ b/internal/app/consumer/tracking_window_live_test.go @@ -0,0 +1,175 @@ +package jobs + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/warmbly/warmbly/internal/app/instancesettings" + "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/repository" +) + +// Live checks that the operator-editable machine windows survive the trip +// through the settings document's jsonb column and reach the classifier. +// Skipped unless WARMBLY_TEST_DB is set: +// +// WARMBLY_TEST_DB=postgres://warmbly:warmbly@localhost:15432/warmbly_dev?sslmode=disable \ +// go test ./internal/app/consumer/ -run Live -v +// +// The unit tests cover the arithmetic. What they cannot cover is the storage: +// the whole section is one jsonb document, so a section that fails to marshal, +// or that an existing instance's document lacks entirely, is only visible +// against a real row. + +// settingsSandbox snapshots the singleton settings row and restores it +// verbatim afterwards. The document is instance-wide, not fixture-owned, so a +// cleanup that wrote Defaults() back would silently reset the settings of +// whatever database the test was pointed at. +// It returns the snapshot, because a test must assert against the values the +// database actually held rather than against the shipped defaults: this suite +// is meant to run against a real instance's database, where an operator may +// have set anything. +func settingsSandbox(t *testing.T, ctx context.Context, store instancesettings.Store) instancesettings.Document { + t.Helper() + before, err := store.Get(ctx) + if err != nil { + t.Fatalf("snapshot settings: %v", err) + } + t.Cleanup(func() { + if err := store.Put(ctx, before, nil); err != nil { + t.Errorf("restore settings: %v", err) + } + }) + return before +} + +// An instance that upgrades has a settings row written before this section +// existed. Its document has no "tracking" key at all, and the windows have to +// come back as the shipped defaults rather than as zero, which would read as +// "no window" and let every delivery-time scan count as a person. +func TestLiveTrackingWindowsDefaultOnADocumentWithoutTheSection(t *testing.T) { + handle := liveDB(t) + ctx := context.Background() + store := instancesettings.NewStore(handle.Pool) + _ = settingsSandbox(t, ctx, store) + + // A document exactly as an older version would have written it. + old := instancesettings.Defaults() + raw, err := json.Marshal(old) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var stripped map[string]any + if err := json.Unmarshal(raw, &stripped); err != nil { + t.Fatalf("unmarshal: %v", err) + } + delete(stripped, "tracking") + if _, ok := stripped["tracking"]; ok { + t.Fatal("the fixture still carries a tracking section") + } + pruned, err := json.Marshal(stripped) + if err != nil { + t.Fatalf("marshal pruned: %v", err) + } + var doc instancesettings.Document + if err := json.Unmarshal(pruned, &doc); err != nil { + t.Fatalf("unmarshal pruned: %v", err) + } + if err := store.Put(ctx, doc, nil); err != nil { + t.Fatalf("put: %v", err) + } + + got, err := store.Get(ctx) + if err != nil { + t.Fatalf("get: %v", err) + } + if got.Tracking.MachineWindowOpenSeconds != config.TrackingMachineWindowOpenSecondsDefault { + t.Errorf("open window = %d, want the shipped %d on a document with no tracking section", + got.Tracking.MachineWindowOpenSeconds, config.TrackingMachineWindowOpenSecondsDefault) + } + if got.Tracking.MachineWindowClickSeconds != config.TrackingMachineWindowClickSecondsDefault { + t.Errorf("click window = %d, want the shipped %d on a document with no tracking section", + got.Tracking.MachineWindowClickSeconds, config.TrackingMachineWindowClickSecondsDefault) + } +} + +// The operator's saved value has to reach the classifier, which is the whole +// point of the setting. This walks the real path: a patch through the service, +// the jsonb row, and the consumer reading it back to classify an event. +func TestLiveTrackingWindowReachesTheClassifier(t *testing.T) { + handle := liveDB(t) + ctx := context.Background() + store := instancesettings.NewStore(handle.Pool) + before := settingsSandbox(t, ctx, store) + + widened := 300 + patch := instancesettings.Patch{Tracking: &struct { + MachineWindowOpenSeconds *int `json:"machine_window_open_seconds"` + MachineWindowClickSeconds *int `json:"machine_window_click_seconds"` + }{MachineWindowOpenSeconds: &widened}} + + if _, err := instancesettings.NewService(store).Put(ctx, patch, nil); err != nil { + t.Fatalf("put: %v", err) + } + + // A reader that has never cached, as the consumer process is on the next + // poll after an edit. + tc := &TrackingConsumer{} + tc.WireTrackingPolicy(instancesettings.NewService(store)) + + windows := tc.machineWindows(ctx) + if got, want := windows.OpenWindow(), time.Duration(widened)*time.Second; got != want { + t.Fatalf("open window = %v, want %v", got, want) + } + // The click window was not part of the patch and must come back exactly as + // it was, rather than being cleared by a partial write. Compared against + // the snapshot, not against the shipped default, so the assertion holds on + // a database where an operator has already set one. + if got, want := windows.ClickWindow(), before.Tracking.ClickWindow(); got != want { + t.Fatalf("click window = %v, want it untouched at %v", got, want) + } + + sent := time.Now() + chrome := strp("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36") + at := sent.Add(4 * time.Minute) + + // Four minutes is well past the shipped 60s and inside the saved 300s, so + // this only passes if the stored value is what the classifier used. + if m, r := classifyOpen(chrome, nil, &sent, at, windows.OpenWindow()); !m || r != repository.EmailOpenReasonInstant { + t.Fatalf("an open inside the saved window is automated, got %v %q", m, r) + } + if m, _ := classifyOpen(chrome, nil, &sent, at, instancesettings.DefaultTracking().OpenWindow()); m { + t.Fatal("the same open is a person's under the shipped window; the test proves nothing otherwise") + } +} + +// An out-of-range value must be clamped on the way in, not stored and applied. +// Normalize runs on write and on read, so a hand-edited row is bounded too. +func TestLiveTrackingWindowClampsThroughStorage(t *testing.T) { + handle := liveDB(t) + ctx := context.Background() + store := instancesettings.NewStore(handle.Pool) + _ = settingsSandbox(t, ctx, store) + + doc := instancesettings.Defaults() + doc.Tracking.MachineWindowOpenSeconds = config.TrackingMachineWindowSecondsMax + 10_000 + doc.Tracking.MachineWindowClickSeconds = -5 + if err := store.Put(ctx, doc, nil); err != nil { + t.Fatalf("put: %v", err) + } + + got, err := store.Get(ctx) + if err != nil { + t.Fatalf("get: %v", err) + } + if got.Tracking.MachineWindowOpenSeconds != config.TrackingMachineWindowSecondsMax { + t.Errorf("open window = %d, want it clamped to %d", + got.Tracking.MachineWindowOpenSeconds, config.TrackingMachineWindowSecondsMax) + } + if got.Tracking.MachineWindowClickSeconds != config.TrackingMachineWindowClickSecondsDefault { + t.Errorf("click window = %d, want a negative to resolve to the default %d", + got.Tracking.MachineWindowClickSeconds, config.TrackingMachineWindowClickSecondsDefault) + } +} diff --git a/internal/app/instancesettings/document.go b/internal/app/instancesettings/document.go index 1933a779..9fd7bb58 100644 --- a/internal/app/instancesettings/document.go +++ b/internal/app/instancesettings/document.go @@ -69,6 +69,61 @@ type Retention struct { AuditLogDays int `json:"audit_log_days"` } +// Tracking holds the engagement-classification windows. Zero means "compiled +// default" on read, so a document written before the section existed still +// resolves; the accepted range is clamped in Normalize. +// +// Nothing here drops an event. A classified-automated open or click is still +// stored as delivery evidence and still shown on the timeline; it just does +// not stamp the step, fire a branch or automation, or send a webhook. +type Tracking struct { + // MachineWindowOpenSeconds is how soon after a step was dispatched an open + // is treated as automated. The clock starts at dispatch to the worker, so + // this window also covers provider queueing and transit to the recipient. + MachineWindowOpenSeconds int `json:"machine_window_open_seconds"` + // MachineWindowClickSeconds is the same window for click tickets, kept + // separately because a misjudged click costs an automation rather than a + // metric. + MachineWindowClickSeconds int `json:"machine_window_click_seconds"` +} + +// DefaultTracking is the compiled classification window for each event kind. +func DefaultTracking() Tracking { + return Tracking{ + MachineWindowOpenSeconds: config.TrackingMachineWindowOpenSecondsDefault, + MachineWindowClickSeconds: config.TrackingMachineWindowClickSecondsDefault, + } +} + +// Normalize clamps both windows into the accepted range. Zero and negative +// resolve to the compiled default rather than to "never automated", so a +// document written before this section existed keeps the shipped behaviour. +func (t *Tracking) Normalize() { + clamp := func(v, def int) int { + if v <= 0 { + return def + } + if v < config.TrackingMachineWindowSecondsMin { + return config.TrackingMachineWindowSecondsMin + } + if v > config.TrackingMachineWindowSecondsMax { + return config.TrackingMachineWindowSecondsMax + } + return v + } + t.MachineWindowOpenSeconds = clamp(t.MachineWindowOpenSeconds, config.TrackingMachineWindowOpenSecondsDefault) + t.MachineWindowClickSeconds = clamp(t.MachineWindowClickSeconds, config.TrackingMachineWindowClickSecondsDefault) +} + +// OpenWindow and ClickWindow are the normalized windows as durations. +func (t Tracking) OpenWindow() time.Duration { + return time.Duration(t.MachineWindowOpenSeconds) * time.Second +} + +func (t Tracking) ClickWindow() time.Duration { + return time.Duration(t.MachineWindowClickSeconds) * time.Second +} + // Bounds on the domain-authentication grace window. One hour is the shortest // window that still absorbs a resolver blip; 30 days is the longest a domain // should keep sending cold mail unauthenticated while being warned about it. @@ -98,6 +153,7 @@ type Document struct { Access Access `json:"access"` Sync Sync `json:"sync"` Retention Retention `json:"retention"` + Tracking Tracking `json:"tracking"` Deliverability Deliverability `json:"deliverability"` Notifications Notifications `json:"notifications"` } @@ -114,6 +170,7 @@ func Defaults() Document { }, Sync: DefaultSync(), Retention: DefaultRetention(), + Tracking: DefaultTracking(), Deliverability: DefaultDeliverability(), } } @@ -184,6 +241,7 @@ func (d *Document) Normalize() { } d.Sync.Normalize() d.Retention.Normalize() + d.Tracking.Normalize() d.Deliverability.Normalize() d.Notifications.Normalize() } @@ -254,6 +312,10 @@ type Patch struct { FormEventDays *int `json:"form_event_days"` AuditLogDays *int `json:"audit_log_days"` } `json:"retention"` + Tracking *struct { + MachineWindowOpenSeconds *int `json:"machine_window_open_seconds"` + MachineWindowClickSeconds *int `json:"machine_window_click_seconds"` + } `json:"tracking"` Deliverability *struct { EnforceDomainAuth *bool `json:"enforce_domain_auth"` AuthGraceHours *int `json:"auth_grace_hours"` @@ -309,6 +371,14 @@ func (p Patch) Apply(doc Document) Document { doc.Retention.AuditLogDays = *p.Retention.AuditLogDays } } + if p.Tracking != nil { + if p.Tracking.MachineWindowOpenSeconds != nil { + doc.Tracking.MachineWindowOpenSeconds = *p.Tracking.MachineWindowOpenSeconds + } + if p.Tracking.MachineWindowClickSeconds != nil { + doc.Tracking.MachineWindowClickSeconds = *p.Tracking.MachineWindowClickSeconds + } + } if p.Deliverability != nil { if p.Deliverability.EnforceDomainAuth != nil { doc.Deliverability.EnforceDomainAuth = *p.Deliverability.EnforceDomainAuth diff --git a/internal/app/instancesettings/document_test.go b/internal/app/instancesettings/document_test.go index c05c8da8..505edfcd 100644 --- a/internal/app/instancesettings/document_test.go +++ b/internal/app/instancesettings/document_test.go @@ -4,6 +4,8 @@ import ( "encoding/json" "testing" "time" + + "github.com/warmbly/warmbly/internal/config" ) func TestDeliverabilityDefaults(t *testing.T) { @@ -115,3 +117,89 @@ func TestPatchDeliverabilityZeroGraceClampsToFloor(t *testing.T) { t.Errorf("AuthGraceHours = %d, want the floor %d", got.Deliverability.AuthGraceHours, AuthGraceHoursMin) } } + +func TestTrackingDefaults(t *testing.T) { + tr := Defaults().Tracking + if tr.MachineWindowOpenSeconds != config.TrackingMachineWindowOpenSecondsDefault { + t.Errorf("MachineWindowOpenSeconds = %d, want %d", tr.MachineWindowOpenSeconds, config.TrackingMachineWindowOpenSecondsDefault) + } + if tr.MachineWindowClickSeconds != config.TrackingMachineWindowClickSecondsDefault { + t.Errorf("MachineWindowClickSeconds = %d, want %d", tr.MachineWindowClickSeconds, config.TrackingMachineWindowClickSecondsDefault) + } + // The click window is the tighter of the two on purpose: a misjudged + // click costs an automation, a misjudged open costs a metric. + if tr.ClickWindow() > tr.OpenWindow() { + t.Errorf("click window %v must not exceed the open window %v", tr.ClickWindow(), tr.OpenWindow()) + } +} + +func TestTrackingNormalize(t *testing.T) { + tests := []struct { + name string + seconds int + want int + }{ + // Zero is a document written before this section existed. It must + // resolve to the default, never to "nothing is ever automated". + {"zero resolves to the default", 0, config.TrackingMachineWindowOpenSecondsDefault}, + {"negative resolves to the default", -30, config.TrackingMachineWindowOpenSecondsDefault}, + {"in range is kept", 90, 90}, + {"at the floor is kept", config.TrackingMachineWindowSecondsMin, config.TrackingMachineWindowSecondsMin}, + {"at the ceiling is kept", config.TrackingMachineWindowSecondsMax, config.TrackingMachineWindowSecondsMax}, + {"above the ceiling clamps down", config.TrackingMachineWindowSecondsMax + 600, config.TrackingMachineWindowSecondsMax}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tr := Tracking{MachineWindowOpenSeconds: tt.seconds} + tr.Normalize() + if tr.MachineWindowOpenSeconds != tt.want { + t.Errorf("MachineWindowOpenSeconds = %d, want %d", tr.MachineWindowOpenSeconds, tt.want) + } + }) + } +} + +// A document stored before the tracking section existed must come back with +// the shipped windows rather than zero, which would read as "never automated" +// and let every delivery-time scan count as a person. +func TestDocumentUnmarshalOverDefaultsKeepsTracking(t *testing.T) { + doc := Defaults() + stored := []byte(`{"invitations":{"links_enabled":false,"ttl_hours":24}}`) + if err := json.Unmarshal(stored, &doc); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + doc.Normalize() + + if doc.Tracking.MachineWindowOpenSeconds != config.TrackingMachineWindowOpenSecondsDefault { + t.Errorf("MachineWindowOpenSeconds = %d, want the default %d to survive an older document", + doc.Tracking.MachineWindowOpenSeconds, config.TrackingMachineWindowOpenSecondsDefault) + } + if doc.Tracking.MachineWindowClickSeconds != config.TrackingMachineWindowClickSecondsDefault { + t.Errorf("MachineWindowClickSeconds = %d, want the default %d to survive an older document", + doc.Tracking.MachineWindowClickSeconds, config.TrackingMachineWindowClickSecondsDefault) + } +} + +func TestPatchTracking(t *testing.T) { + doc := Defaults() + open, click := 120, 45 + patch := Patch{Tracking: &struct { + MachineWindowOpenSeconds *int `json:"machine_window_open_seconds"` + MachineWindowClickSeconds *int `json:"machine_window_click_seconds"` + }{MachineWindowOpenSeconds: &open, MachineWindowClickSeconds: &click}} + + got := patch.Apply(doc) + got.Normalize() + if got.Tracking.MachineWindowOpenSeconds != open { + t.Errorf("MachineWindowOpenSeconds = %d, want %d", got.Tracking.MachineWindowOpenSeconds, open) + } + if got.Tracking.MachineWindowClickSeconds != click { + t.Errorf("MachineWindowClickSeconds = %d, want %d", got.Tracking.MachineWindowClickSeconds, click) + } + + // An absent section keeps what is stored rather than clearing it. + kept := Patch{}.Apply(got) + if kept.Tracking != got.Tracking { + t.Errorf("absent tracking section changed the document: %+v", kept.Tracking) + } +} diff --git a/internal/app/instancesettings/service.go b/internal/app/instancesettings/service.go index 05281af6..b3cd0023 100644 --- a/internal/app/instancesettings/service.go +++ b/internal/app/instancesettings/service.go @@ -44,6 +44,10 @@ type Service interface { // The retention sweeps read it on every pass, so an edit takes effect on // the next one rather than at the next restart. RetentionWindows(ctx context.Context) Retention + // TrackingPolicy is the engagement-classification section, already + // normalized. The tracking consumer reads it per event, so an edit takes + // effect without a restart, within the cacheTTL the read goes through. + TrackingPolicy(ctx context.Context) Tracking // DomainAuth is the sending-domain authentication gate: whether it is // enforced at all, and how long a domain must stay failing first. DomainAuth(ctx context.Context) (enforce bool, grace time.Duration) @@ -146,6 +150,12 @@ func (s *service) RetentionWindows(ctx context.Context) Retention { return r } +func (s *service) TrackingPolicy(ctx context.Context) Tracking { + t := s.Get(ctx).Tracking + t.Normalize() + return t +} + func (s *service) DomainAuth(ctx context.Context) (bool, time.Duration) { d := s.Get(ctx).Deliverability d.Normalize() diff --git a/internal/config/constants.go b/internal/config/constants.go index d9fcd1c5..4dbf4f97 100644 --- a/internal/config/constants.go +++ b/internal/config/constants.go @@ -153,13 +153,31 @@ const ( // it must stay well clear of a slow provider handshake. CampaignSendReclaimAfterMinutes = 30 - // TrackingMachineWindowSeconds is how soon after a step was dispatched an - // open or click is treated as automated rather than a person. The clock - // starts when the send is handed to the worker, before the provider has - // even accepted the message, so a person cannot plausibly have read and - // acted on it inside this window; security gateways that detonate every - // link at delivery time routinely do. - TrackingMachineWindowSeconds = 10 + // TrackingMachineWindowOpenSecondsDefault and + // TrackingMachineWindowClickSecondsDefault are how soon after a step was + // dispatched an open or a click is treated as automated rather than a + // person. Operator-editable under Instance settings. + // + // The clock starts when the send is handed to the worker, NOT when the + // recipient's server received it, so the window has to absorb the worker's + // SMTP handshake, the sending provider's outbound queue and the transit to + // the recipient's MX before the gateway that scans on arrival even starts. + // That is why these are not the "no human could read this fast" numbers + // they look like: against this anchor, ten seconds routinely expired before + // the scan it was meant to catch. + // + // Opens get the longer window. The two failure modes are not symmetric: a + // misjudged open costs a metric and an open-triggered branch, while a + // misjudged click costs an interested lead the automation behind it, and + // clicks have the scanner-network catalogue covering them as well. + TrackingMachineWindowOpenSecondsDefault = 60 + TrackingMachineWindowClickSecondsDefault = 30 + + // Bounds on both windows. One second is the floor rather than zero because + // it is effectively "off" while still keeping the rule's shape, and 15 + // minutes is past any plausible delivery lag. + TrackingMachineWindowSecondsMin = 1 + TrackingMachineWindowSecondsMax = 900 // TrackingClickBurstSeconds is the window inside which clicks on two // different links of the same email from the same source are treated as diff --git a/tracking/scanner-networks.txt b/tracking/scanner-networks.txt index 0be90e14..9839bd2d 100644 --- a/tracking/scanner-networks.txt +++ b/tracking/scanner-networks.txt @@ -49,3 +49,49 @@ # asn:8074 clicks microsoft # asn:12076 clicks microsoft # asn:15169 clicks google + +# Barracuda Email Gateway Defense. Published by Barracuda as the ranges its +# filtering layer connects to a customer's mail server from, one narrow block +# per region. Enabled for the same reason the EOP ranges are: these are the +# mail filtering tier itself, not the cloud it happens to sit in, and no +# recipient reads their mail from them. +3.24.133.128/25 all barracuda-egd +15.222.16.128/25 all barracuda-egd +35.157.190.224/27 all barracuda-egd +18.185.115.192/26 all barracuda-egd +18.184.203.224/27 all barracuda-egd +13.200.136.128/25 all barracuda-egd +35.176.92.96/27 all barracuda-egd +18.133.136.128/26 all barracuda-egd +18.133.136.96/27 all barracuda-egd +209.222.82.0/24 all barracuda-egd + +# Proofpoint, Mimecast and Cisco Secure Email, by ASN. Each of these is a pure +# mail security network with no consumer eyeball traffic, which is what makes +# them worth naming at all, and they are still NOT enabled by default. The +# reason is browser isolation, and it is worth understanding before turning +# one on: +# +# Proofpoint Isolation and Mimecast Browser Isolation render a clicked page +# in the vendor's own cloud and stream it to the recipient. When a policy +# sends a link to isolation, the GET on the click ticket comes from the +# vendor's network and a PERSON is on the other end of it. Isolation is +# usually scoped to uncategorised or suspicious URLs, which is precisely what +# a new cold-outreach domain looks like, so for this product it is not a +# rare edge case. +# +# So a whole-ASN entry here cannot distinguish the delivery-time scan from the +# isolated human click, and enabling one trades inflated click counts for lost +# automations. Turn them on when your recipients' scanner noise costs you more +# than that, and prefer leaving the machine-window rule to catch the +# delivery-time half: a scan runs seconds after the send, an isolated click +# runs whenever the person got to it. +# +# asn:22843 all proofpoint +# asn:26211 all proofpoint +# asn:52129 all proofpoint +# asn:30031 all mimecast +# asn:39588 all mimecast +# asn:42427 all mimecast +# asn:60492 all mimecast +# asn:16417 all cisco-ironport diff --git a/tracking/src/scanners.rs b/tracking/src/scanners.rs index 63d7b5d4..b2348f69 100644 --- a/tracking/src/scanners.rs +++ b/tracking/src/scanners.rs @@ -282,6 +282,70 @@ mod tests { assert_eq!(s.classify("13.107.128.5", &h, true, Request::Open), None); } + // Every CIDR in the shipped catalogue must already be its own network + // address. `insert` calls `trunc()`, so `209.222.82.9/24` would silently + // become `209.222.82.0/24` and a typo in the host part of a block would + // widen or shift it with nothing to show for it. + #[test] + fn shipped_networks_are_written_in_canonical_form() { + for line in BUILTIN_CATALOGUE.lines() { + let line = line.split('#').next().unwrap_or("").trim(); + let Some(source) = line.split_whitespace().next() else { + continue; + }; + if source.starts_with("asn:") { + continue; + } + let net: IpNet = source + .parse() + .unwrap_or_else(|_| panic!("{source} must parse")); + assert_eq!( + net, + net.trunc(), + "{source} has host bits set; write it as {}", + net.trunc() + ); + } + } + + // Barracuda's published filtering blocks are narrow, per-region and + // documented by the vendor as its own mail tier, so they ship enabled on + // both endpoints like the EOP ranges. + #[test] + fn barracuda_filtering_blocks_are_a_scanner_for_both() { + let s = builtins(); + for kind in [Request::Open, Request::Click] { + assert_eq!( + s.classify("209.222.82.10", &hdr(&[]), false, kind) + .as_deref(), + Some("barracuda-egd"), + "{kind:?} from Barracuda EGD should be labelled" + ); + } + } + + // Proofpoint, Mimecast and Cisco are catalogued by ASN and ship commented + // out. Proofpoint Isolation and Mimecast Browser Isolation render a + // clicked page in the vendor's own cloud, so a whole-ASN entry cannot tell + // the delivery-time scan from a person clicking through isolation, and + // enabling one is an operator's trade rather than a default. + #[test] + fn vendor_asns_are_not_on_by_default() { + let s = builtins(); + for asn in [ + "22843", "30031", "39588", "42427", "52129", "26211", "60492", "16417", + ] { + let h = hdr(&[("cf-asn", asn)]); + for kind in [Request::Open, Request::Click] { + assert_eq!( + s.classify("203.0.113.9", &h, true, kind), + None, + "AS{asn} must ship commented out for {kind:?}" + ); + } + } + } + // Outlook on the web is served from the Exchange Online ranges, which are // deliberately absent from the catalogue. #[test]