mirror of
https://github.com/warmbly/warmbly.git
synced 2026-09-05 16:02:48 +00:00
feat: merge main into the unsubscribe branch again, carrying the UTM campaign columns alongside unsubscribe_mode in every campaign scanner, keeping unsubscribe links out of the moved link tracker and its UTM tagging, and renumbering the opt-out migration to 000124 because main released 000123 for link clicks
This commit is contained in:
@@ -481,6 +481,8 @@ func main() {
|
||||
contactRepo,
|
||||
streamingPublisher,
|
||||
repository.NewTrackingDedupeRepository(primaryDB.Pool),
|
||||
repository.NewTrackedLinkRepository(primaryDB.Pool),
|
||||
repository.NewLinkClickRepository(primaryDB.Pool),
|
||||
advancedService,
|
||||
verificationEvidence,
|
||||
); terr != nil {
|
||||
|
||||
@@ -29,6 +29,7 @@ Returns the main dashboard overview for the active organization: aggregate stats
|
||||
"total_opens": 612,
|
||||
"machine_opens": 88,
|
||||
"total_clicks": 143,
|
||||
"machine_clicks": 6,
|
||||
"total_replies": 57,
|
||||
"total_bounces": 9,
|
||||
"open_rate": 49.35,
|
||||
@@ -239,6 +240,7 @@ Returns a single campaign's performance summary plus per-sequence-step stats. Th
|
||||
"unique_opens": 410,
|
||||
"machine_opens": 52,
|
||||
"unique_clicks": 99,
|
||||
"machine_clicks": 4,
|
||||
"replies": 41,
|
||||
"bounces": 5,
|
||||
"unsubscribes": 3,
|
||||
@@ -262,7 +264,7 @@ Returns a single campaign's performance summary plus per-sequence-step stats. Th
|
||||
}
|
||||
```
|
||||
|
||||
`machine_opens` is the subset of `unique_opens` from automated fetchers (Apple MPP prefetch, UA-less clients); human opens are `unique_opens` minus `machine_opens`.
|
||||
`machine_opens` is the subset of `unique_opens` from automated fetchers (Apple MPP prefetch, UA-less clients, opens inside ten seconds of the send); human opens are `unique_opens` minus `machine_opens`. `machine_clicks` counts steps whose only clicks were automated (a security gateway walking the links); those are not part of `unique_clicks` or `total_clicks`, which only ever count a person's click.
|
||||
|
||||
## Get campaign daily stats
|
||||
|
||||
|
||||
@@ -69,6 +69,10 @@ A `data` plus `pagination` envelope. `next_cursor` is the campaign id to resume
|
||||
"prioritize_new_leads": false,
|
||||
"tracking_domain": "",
|
||||
"tracking_domain_verified": false,
|
||||
"utm_tracking": true,
|
||||
"utm_source": "",
|
||||
"utm_medium": "",
|
||||
"utm_campaign": "",
|
||||
"updated_at": "2026-06-10T12:00:00Z",
|
||||
"created_at": "2026-06-01T09:00:00Z"
|
||||
}
|
||||
@@ -150,7 +154,11 @@ Create a campaign. Only `name` is required, every other field is optional and ap
|
||||
| `kind` | string | no | `sequence` (default) or `one_time`. A one-time email accepts at most one entry in `steps` here and refuses further email steps later; it is otherwise a normal campaign. Fixed at creation. |
|
||||
| `stop_on_reply` | boolean | no | Stop sending to a contact once they reply. |
|
||||
| `open_tracking` | boolean | no | Insert the open pixel. |
|
||||
| `link_tracking` | boolean | no | Rewrite links through the tracking ticket service. |
|
||||
| `link_tracking` | boolean | no | Rewrite links through the tracking ticket service. Each link gets its own ticket, so clicks are attributed per link. |
|
||||
| `utm_tracking` | boolean | no | Tag every link with `utm_source`, `utm_medium`, `utm_campaign` and a per-link `utm_content` at send time. Default `false`. Values already on a link are kept. |
|
||||
| `utm_source` | string | no | Overrides the default `warmbly`. Empty means the default. Up to 128 characters. |
|
||||
| `utm_medium` | string | no | Overrides the default `email`. Empty means the default. |
|
||||
| `utm_campaign` | string | no | Overrides the default, the campaign name as a slug. Empty means the default. |
|
||||
| `text_only` | boolean | no | Send plain text only: no HTML part, and open and click tracking are off regardless of their flags. |
|
||||
| `daily_limit` | integer | no | Per-campaign daily send cap. |
|
||||
| `unsubscribe_header` | boolean | no | Add the RFC 8058 one-click unsubscribe header. |
|
||||
@@ -236,7 +244,7 @@ Patch any subset of campaign fields. Omitted fields are left unchanged. The expl
|
||||
|
||||
### Request body
|
||||
|
||||
Every field is optional. Scalar fields use nullable pointers, so any field you send is applied. `start_date` and `end_date` additionally accept an explicit `null` to clear the stored date: a null `start_date` means "start now" and a null `end_date` means "run open-ended". Changing any schedule field (`start_date`, `end_date`, `timezone`, `days`, `start_time`, `end_time`, `schedule_windows`) on an active campaign reschedules its next send immediately, so clearing a future start date takes effect right away. Notable fields: `name`, `description`, `status`, `stop_on_reply`, `open_tracking`, `link_tracking`, `text_only`, `daily_limit`, `unsubscribe_header`, `risky_emails`, `cc`, `bcc`, `start_date`, `end_date`, `timezone`, `days`, `start_time`, `end_time`, `schedule_windows`, `email_tags`, `folders`, `contact_order_by`, `contact_order_dir`, `contact_order_field`, `sender_strategy`, `rotation_mode`, `ramp_enabled`, `ramp_start`, `ramp_increment`, `ramp_ceiling`, `esp_match_mode`, `max_new_leads_per_day`, `prioritize_new_leads`, `tracking_domain`.
|
||||
Every field is optional. Scalar fields use nullable pointers, so any field you send is applied. `start_date` and `end_date` additionally accept an explicit `null` to clear the stored date: a null `start_date` means "start now" and a null `end_date` means "run open-ended". Changing any schedule field (`start_date`, `end_date`, `timezone`, `days`, `start_time`, `end_time`, `schedule_windows`) on an active campaign reschedules its next send immediately, so clearing a future start date takes effect right away. Notable fields: `name`, `description`, `status`, `stop_on_reply`, `open_tracking`, `link_tracking`, `text_only`, `daily_limit`, `unsubscribe_header`, `risky_emails`, `cc`, `bcc`, `start_date`, `end_date`, `timezone`, `days`, `start_time`, `end_time`, `schedule_windows`, `email_tags`, `folders`, `contact_order_by`, `contact_order_dir`, `contact_order_field`, `sender_strategy`, `rotation_mode`, `ramp_enabled`, `ramp_start`, `ramp_increment`, `ramp_ceiling`, `esp_match_mode`, `max_new_leads_per_day`, `prioritize_new_leads`, `tracking_domain`, `utm_tracking`, `utm_source`, `utm_medium`, `utm_campaign`.
|
||||
|
||||
```json
|
||||
{
|
||||
|
||||
@@ -662,7 +662,7 @@ Returns a `data` array plus a `pagination` envelope.
|
||||
|
||||
`GET /contacts/:id/timeline`
|
||||
|
||||
Returns the merged activity feed for a contact: sends, opens, clicks, replies, bounces, deliverability and suppression events, notes, meeting bookings, and lifecycle events (the contact's creation with its first-touch source, and every time it joined or left a campaign or a category). Requires a selected organization (org-scoped events would otherwise be hidden), so a request with no organization returns `400`.
|
||||
Returns the merged activity feed for a contact: sends, opens, clicks (one per link, naming the link), replies, bounces, deliverability and suppression events, notes, meeting bookings, and lifecycle events (the contact's creation with its first-touch source, and every time it joined or left a campaign or a category). Requires a selected organization (org-scoped events would otherwise be hidden), so a request with no organization returns `400`.
|
||||
|
||||
Auth: **Scope** `READ_CONTACTS` · **Org permission** `view_contacts`
|
||||
|
||||
@@ -679,6 +679,28 @@ Returns a `data` array and a `has_more` flag (not a cursor envelope). Paginate b
|
||||
```json
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"type": "email_clicked",
|
||||
"at": "2026-06-09T11:42:00Z",
|
||||
"email_account_id": "e1...",
|
||||
"email_account_email": "rep@yourco.com",
|
||||
"campaign_id": "c1...",
|
||||
"campaign_name": "Q3 Outbound",
|
||||
"step_id": "s1...",
|
||||
"step_name": "Intro",
|
||||
"subject": "Quick question",
|
||||
"machine": false,
|
||||
"link": {
|
||||
"id": "7c0f...",
|
||||
"url": "https://yourco.com/pricing?utm_source=warmbly&utm_medium=email&utm_campaign=q3_outbound&utm_content=pricing",
|
||||
"label": "Pricing",
|
||||
"utm_source": "warmbly",
|
||||
"utm_medium": "email",
|
||||
"utm_campaign": "q3_outbound",
|
||||
"utm_content": "pricing",
|
||||
"user_agent": "Mozilla/5.0 ..."
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "email_replied",
|
||||
"at": "2026-06-09T11:02:00Z",
|
||||
@@ -716,6 +738,8 @@ Returns a `data` array and a `has_more` flag (not a cursor envelope). Paginate b
|
||||
|
||||
`type` is one of `email_sent`, `email_opened`, `email_clicked`, `email_replied`, `email_bounced`, `reply_received`, `deliverability`, `suppressed`, `note`, `meeting_booked`, `meeting_rescheduled`, `meeting_canceled`, `contact_created`, `campaign_added`, `campaign_removed`, `category_added`, or `category_removed`. Fields not relevant to an event type are omitted.
|
||||
|
||||
`email_opened` and `email_clicked` carry `machine`: `true` when an automated fetcher did it rather than the person (a mail privacy proxy, a fetch inside ten seconds of the send, several links followed within seconds). Per-link `email_clicked` events also carry `machine_reason` (`prefetch`, `instant` or `burst`); opens carry only the flag. Automated clicks are on the feed for the record but never count the step as clicked. An `email_clicked` event carries `link` with the link's `id`, `url`, `label` (its anchor text), the `utm_*` parameters the URL carried, and the `user_agent`; every link in an email is tracked on its own, so each link clicked is its own event. Clicks recorded before per-link attribution have no `link`.
|
||||
|
||||
Lifecycle events carry the name of what changed as it was at the time (`campaign_name`, or `category_id` plus `category_title`), so a later rename or deletion does not rewrite history. A `contact_created` event carries `source` (`manual`, `campaign`, `import`, `sheet_sync`, `api`, `ai_assistant`, or `unknown` for contacts that predate attribution) and `source_detail` (the file, campaign, sheet or API key name). The same values are on the contact itself as `source`, `source_detail` and `first_seen_at`, and never change after creation.
|
||||
|
||||
## Get a contact's campaign state
|
||||
|
||||
@@ -458,8 +458,8 @@ The full catalog is available at `GET /webhooks/event-types`. Below is the compl
|
||||
| `campaign.deleted` | A campaign is deleted. | |
|
||||
| `campaign.email_sent` | A campaign email is dispatched by a worker. | yes |
|
||||
| `campaign.email_delivered` | A campaign email is accepted by the recipient's mail server. | yes |
|
||||
| `campaign.email_opened` | A tracked open is recorded. | yes |
|
||||
| `campaign.email_clicked` | A tracked link click is recorded. | yes |
|
||||
| `campaign.email_opened` | A person opens a tracked email. Automated opens (mail privacy proxies, fetches inside seconds of the send) do not fire it. | yes |
|
||||
| `campaign.email_clicked` | A person clicks a tracked link. The payload carries the `url` and the link's `link_label`. Automated clicks (security gateways walking the links) do not fire it. | yes |
|
||||
| `campaign.email_bounced` | A campaign email bounces. | |
|
||||
| `campaign.reply_received` | A human reply lands for a campaign thread. | |
|
||||
| `campaign.unsubscribed` | A recipient opts out. `source` is `one_click` (the mail client's button), `link` (the link in the email), `reply` (a reply asking to stop) or `action` (a sequence's unsubscribe step). | |
|
||||
|
||||
@@ -74,13 +74,16 @@ Produced by the Rust tracking service when a pixel loads or a tracked link is cl
|
||||
"event_type": "EMAIL_OPENED",
|
||||
"task_id": "uuid",
|
||||
"original_url": "https://example.com/page",
|
||||
"link_id": "uuid",
|
||||
"timestamp": "2026-01-29T12:00:00Z",
|
||||
"user_agent": "Mozilla/5.0...",
|
||||
"ip_hash": "sha256..."
|
||||
}
|
||||
```
|
||||
|
||||
`event_type` is `EMAIL_OPENED` or `EMAIL_CLICKED`; `original_url` is set only for clicks; IPs are stored as hashes, never raw. The struct is `events.TrackingEvent` in `internal/events/schemas.go`, mirrored in `tracking/src/events.rs`.
|
||||
`event_type` is `EMAIL_OPENED` or `EMAIL_CLICKED`; `original_url` and `link_id` (the click ticket, which names the link's stored destination and anchor text) are set only for clicks; IPs are stored as hashes, never raw. The struct is `events.TrackingEvent` in `internal/events/schemas.go`, mirrored in `tracking/src/events.rs`.
|
||||
|
||||
The consumer classifies each event before it counts. An open is automated when the user agent is a mail privacy proxy or missing, or when it arrives within `TrackingMachineWindowSeconds` of the step's dispatch; it is recorded with `opened_machine` and upgraded by a later human open. A click is automated for a missing user agent, for arriving inside the same window, or when the same source clicked another link of the same email within `TrackingClickBurstSeconds`; every click is logged per link in `email_link_clicks` with its reason, and only a human click stamps `clicked_at`, fires instant actions, or emits a webhook. A burst is only recognisable from its second click, so a human click's stamp and log row are written at once but its effects (evidence, instant actions, webhook, realtime event) run after the burst window plus a second, on the classification the click has by then; a burst recognised meanwhile relabels the earlier click and withdraws the stamp when no human click remains. A consumer restart inside the window loses only those deferred effects, and routing still follows the clicked branch at the next step boundary.
|
||||
|
||||
Website page views do not ride this topic. The tracking service forwards each accepted view to the backend's internal API (`POST /api/v1/internal/page-hits`) instead, because the backend is where the user agent and IP are turned into device and location, and the IP must not sit in a durable stream on the way there.
|
||||
|
||||
|
||||
@@ -20,8 +20,9 @@ Four rules govern the counts:
|
||||
|
||||
- **Tracking must be on.** Opens and clicks need open or link tracking enabled on the campaign, and a tracking host on the install. A [custom tracking domain](/guides/mailboxes/#custom-tracking-domain) per mailbox is optional; without one they go through the shared host and still count. With tracking off they stay at zero while sends and replies still count.
|
||||
- **Replies are human replies.** Out-of-office and autoresponders never count, never stamp the contact as replied, and never trip stop-on-reply.
|
||||
- **Bots are filtered.** Crawlers, CLI agents, prefetches, chat link previews, and security gateways that open every link are served normally but never counted. Otherwise one corporate scanner would "click" every link seconds after delivery.
|
||||
- **Auto-opens are labeled, not hidden.** Privacy proxies like Apple Mail Privacy Protection still count (they confirm delivery) but are tagged and shown separately (`12 auto`). A later real open upgrades them to human. **Auto-opens never trigger opened-based branches or automations.**
|
||||
- **Bots are filtered.** Crawlers, CLI agents, prefetches, chat link previews, and security gateways that announce themselves are served normally but never counted. The ones that do not announce themselves are caught by what they do: an open or click inside ten seconds of the send (nobody reads that fast; the clock starts when the send is handed to the worker, before the mail has even been delivered) and clicks on two or more links of one email from the same source within five seconds (a scanner walking the message). Otherwise one corporate scanner would "click" every link seconds after delivery.
|
||||
- **Auto-opens are labeled, not hidden.** Privacy proxies like Apple Mail Privacy Protection, and instant opens, still count (they confirm delivery) but are tagged and shown separately (`12 auto`). A later real open upgrades them to human. **Auto-opens never trigger opened-based branches or automations.**
|
||||
- **Auto-clicks are kept but never counted.** A click classified as automated is logged on the contact's activity with the link it hit and the rule that caught it, and the campaign overview shows how many steps had only automated clicks (`3 auto`). It never makes the step clicked, never fires a clicked branch or automation, and never sends a webhook. A person clicking the same link later counts normally. A burst is only recognisable from its second click, so a click that looks human waits for the burst window plus one second (six seconds) before it fires anything (clicked branches, automations, webhooks, the live feed): if a second link follows inside that window, both clicks are relabeled, the stamp is withdrawn and nothing fires.
|
||||
|
||||
## Workspace dashboard
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ You need at least one active mailbox and a contact list. Warm new mailboxes befo
|
||||
|
||||
## Create a campaign
|
||||
|
||||
**Campaigns** > **New campaign** opens a wizard that first asks what you are sending: a **sequence** or a [one-time email](#one-time-emails). For a sequence the steps are **Basics** (name, description), **Schedule** (timezone, sending days, hours), **Sending** (mailbox tags, daily limit per mailbox, stop on reply, open and click tracking, unsubscribe header), and **First email** (subject, body, optional follow-ups). Follow-ups added here are connected in order, each waiting the number of days you set after the previous step; open the Steps tab to rearrange, branch, or change the waits. Only a name is required; the first email can be skipped and written in the full editor on the Steps tab, and everything else can be changed later from settings. Closing a wizard with unsaved edits asks before discarding them.
|
||||
**Campaigns** > **New campaign** opens a wizard that first asks what you are sending: a **sequence** or a [one-time email](#one-time-emails). For a sequence the steps are **Basics** (name, description), **Schedule** (timezone, sending days, hours), **Sending** (mailbox tags, daily limit per mailbox, stop on reply, open and click tracking, UTM parameters, unsubscribe header), and **First email** (subject, body, optional follow-ups). Follow-ups added here are connected in order, each waiting the number of days you set after the previous step; open the Steps tab to rearrange, branch, or change the waits. Only a name is required; the first email can be skipped and written in the full editor on the Steps tab, and everything else can be changed later from settings. Closing a wizard with unsaved edits asks before discarding them.
|
||||
|
||||
The campaign starts as a **draft**. Nothing sends until you start it.
|
||||
|
||||
@@ -95,7 +95,7 @@ A lead is **Processing** only while steps remain, so a finished campaign reads a
|
||||
|
||||
### Who opened, clicked and replied
|
||||
|
||||
Next to each lead's status, the Leads list shows three engagement columns: **Opened**, **Clicked** and **Replied**, each with the number of emails in the sequence the person engaged with. A dash means the lead was emailed and has not engaged; the cell is blank for a lead not emailed yet. An open counts only when a person opened the email. Mail clients that fetch every image automatically (Apple Mail Privacy Protection, for example) show as **auto** instead, the same opens the campaign overview reports as automatic, so they never pass for engagement.
|
||||
Next to each lead's status, the Leads list shows three engagement columns: **Opened**, **Clicked** and **Replied**, each with the number of emails in the sequence the person engaged with. A dash means the lead was emailed and has not engaged; the cell is blank for a lead not emailed yet. An open counts only when a person opened the email. Mail clients that fetch every image automatically (Apple Mail Privacy Protection, for example) show as **auto** instead, the same opens the campaign overview reports as automatic, so they never pass for engagement. Clicks are held to the same standard: a link followed within ten seconds of the send, or several links of one email followed within a few seconds of each other, is a security gateway scanning the message, not the recipient. Those clicks are kept on the contact's activity marked **auto**, counted apart on the campaign overview, and never make a lead **Clicked**, never fire a clicked branch or automation, and never send a webhook. See [Link tracking and UTM parameters](#link-tracking-and-utm-parameters).
|
||||
|
||||
The chips above the list are filters. Click a status chip (**Processing**, **Done**, **Replied**, **Queued**, **Bounced**, **Unsub**) or an engagement chip (**Opened**, **Not opened**, **Clicked**, **Not clicked**, **Replied**, **Not replied**) to show only those leads; click it again to clear. One status and one engagement chip can be active at once and both must match. The numbers on the chips are campaign-wide totals, and filtering happens on the server, so a scope shows every matching lead however long the list is. **Not opened**, **Not clicked** and **Not replied** only cover leads that have been sent at least one email: a queued lead has not had the chance. The same filters live in the **Filters** sheet under **Lead status** and **Engagement**.
|
||||
|
||||
@@ -120,6 +120,21 @@ A step counts as sent only once the sending worker has handed it to the mailbox
|
||||
|
||||
No contact is emailed the same step twice. Each step is recorded as attempted before the send is handed to a worker, so a crash, a restart, or a database hiccup in the moment between the two cannot make the step look unsent and send it again. The trade-off is a step whose outcome is genuinely unknown, when the worker stops responding mid-send: after 30 minutes with no answer the step is treated as a failed attempt, appears in **Needs attention**, and is retried like any other failure.
|
||||
|
||||
## Link tracking and UTM parameters
|
||||
|
||||
With **Link tracking** on, every link in the email body becomes its own tracked link, so Warmbly records which link was clicked, not only that one was. Open a contact's **Activity** tab and a click reads **Clicked Pricing**, with the campaign, step and sending mailbox on the row and, expanded, the link's text, its full URL, every UTM parameter it carried, the browser, and whether a person or a scanner followed it. The campaign's live feed names the link the same way, and the `campaign.email_clicked` webhook carries the URL and the link text.
|
||||
|
||||
**UTM parameters** (on by default for campaigns created in the dashboard, off for campaigns created through the API unless you send `utm_tracking`) tag every link when the email is sent, so nobody has to add them by hand and clicks show up attributed in your web analytics:
|
||||
|
||||
| Parameter | Value |
|
||||
|-----------|-------|
|
||||
| `utm_source` | `warmbly`, or the value you set on the campaign |
|
||||
| `utm_medium` | `email`, or the value you set |
|
||||
| `utm_campaign` | The campaign name as a slug (`q3_outbound`), or the value you set |
|
||||
| `utm_content` | The link's own text as a slug (`pricing_page`); a link with no text, such as an image, is numbered `link_1`, `link_2` in order |
|
||||
|
||||
A link that already carries one of these keeps the hand-written value; only the missing parameters are added. Existing query strings and fragments are left exactly as written, and the tags go on the real destination, so they survive the tracking redirect. Anchors, `mailto:` and `tel:` links, and anything that is not `http(s)` are never touched. UTM tagging works with link tracking off too: the tags are added and the link is otherwise left as written. Bare URLs in the plain-text part are tagged as well, numbered `link_1`, `link_2` since they have no text. Change the source, medium and campaign values under **Settings** > **Deliverability**; leave a field empty to use the default.
|
||||
|
||||
## Scheduling
|
||||
|
||||
Campaigns send only inside the **weekly sending windows** you define, in the campaign's timezone. Each day is independent, with different hours or several windows per day, set from presets like `Mon-Fri 9-5` or by dragging on the grid (which is Monday-first).
|
||||
|
||||
@@ -109,7 +109,9 @@ Only new contacts get a source. An import or API call that matches an existing c
|
||||
|
||||
The **Activity** tab of a contact is one feed, newest first, of everything Warmbly knows about them: every campaign email sent, opened, clicked, replied to or bounced (with the campaign, step, subject and sending mailbox), replies with their classified intent, deliverability and suppression events, notes, meetings, and the contact's lifecycle: when it was created and how, and each time it joined or left a campaign or a category.
|
||||
|
||||
Filter chips narrow the feed (**Emails**, **Replies**, **Deliv.**, **Notes**, **Meetings**, **Campaigns**, **Lifecycle**), the search box matches subjects, campaigns, steps, mailboxes, categories and reasons, and the date picker bounds it. Each row stays to one line until you click it; expanded, it shows every detail the event carries. The feed updates live as teammates and the schedulers write to it.
|
||||
A click names the link: the row reads **Clicked Pricing** and, expanded, shows the link's text, its full URL, the UTM source, medium, campaign and content it carried, and the browser. Every link in an email is tracked on its own, so two links clicked are two rows. Opens and clicks that came from a machine rather than the person (a mail privacy proxy, a security gateway that follows every link at delivery) carry an **auto** badge, and the expanded row says which rule caught them; see [Link tracking and UTM parameters](/guides/campaigns/#link-tracking-and-utm-parameters).
|
||||
|
||||
Filter chips narrow the feed (**Emails**, **Replies**, **Deliv.**, **Notes**, **Meetings**, **Campaigns**, **Lifecycle**), the search box matches subjects, campaigns, steps, mailboxes, categories, reasons and, for clicks, the link's text, URL and UTM values, and the date picker bounds it. Each row stays to one line until you click it; expanded, it shows every detail the event carries. The feed updates live as teammates and the schedulers write to it.
|
||||
|
||||
At the top of the tab sits the campaign panel: for each campaign the contact is in, its flow with this contact's progress, the lead status, and what the scheduler will do next. See [Campaigns](/guides/campaigns/) for how the next action is worked out and what its states mean.
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ The data is split into groups. Every export includes **Workspace**; the rest are
|
||||
|-------|----------|
|
||||
| Workspace | The organization, members, roles, teams, mailboxes, API keys, webhooks, and settings, including the website tracking site key. Always included |
|
||||
| Contacts | Contacts, categories, segments with their manual overrides, forms with their images, submissions, personalized link tickets and funnel events, notes, activities, and the suppression list |
|
||||
| Campaigns | Campaigns, sequences, senders, linked segments, attachments, and per-campaign settings |
|
||||
| Campaigns | Campaigns, sequences, senders, linked segments, attachments, per-campaign settings, and each lead's step progress with its per-link clicks |
|
||||
| CRM | Pipelines, deals, tasks, and meeting bookings |
|
||||
| Automations | Automations, connected integrations, and lead sync sources |
|
||||
| Assistant | Assistant sessions and messages, skills, MCP servers, and AI settings |
|
||||
|
||||
@@ -105,6 +105,10 @@ func (d Deps) registerCampaignTools(r *Registry) {
|
||||
"stop_on_reply": boolProp("Stop sequencing a lead once they reply."),
|
||||
"open_tracking": boolProp("Track opens."),
|
||||
"link_tracking": boolProp("Track link clicks."),
|
||||
"utm_tracking": boolProp("Tag every link with UTM parameters automatically."),
|
||||
"utm_source": strProp("utm_source override (empty means the default, warmbly)."),
|
||||
"utm_medium": strProp("utm_medium override (empty means the default, email)."),
|
||||
"utm_campaign": strProp("utm_campaign override (empty means the campaign name as a slug)."),
|
||||
"text_only": boolProp("Send plain text only."),
|
||||
"unsubscribe_header": boolProp("Send the List-Unsubscribe header (one-click unsubscribe)."),
|
||||
"unsubscribe_mode": strProp("In-body opt-out: inherit (workspace default), text (reply-to-opt-out line), link (unsubscribe link), off."),
|
||||
@@ -213,6 +217,10 @@ func (d Deps) updateCampaign(ctx context.Context, inv Invocation, args json.RawM
|
||||
StopOnReply *bool `json:"stop_on_reply"`
|
||||
OpenTracking *bool `json:"open_tracking"`
|
||||
LinkTracking *bool `json:"link_tracking"`
|
||||
UTMTracking *bool `json:"utm_tracking"`
|
||||
UTMSource *string `json:"utm_source"`
|
||||
UTMMedium *string `json:"utm_medium"`
|
||||
UTMCampaign *string `json:"utm_campaign"`
|
||||
TextOnly *bool `json:"text_only"`
|
||||
UnsubscribeHeader *bool `json:"unsubscribe_header"`
|
||||
UnsubscribeMode *string `json:"unsubscribe_mode"`
|
||||
@@ -235,6 +243,10 @@ func (d Deps) updateCampaign(ctx context.Context, inv Invocation, args json.RawM
|
||||
StopOnReply: in.StopOnReply,
|
||||
OpenTracking: in.OpenTracking,
|
||||
LinkTracking: in.LinkTracking,
|
||||
UTMTracking: in.UTMTracking,
|
||||
UTMSource: in.UTMSource,
|
||||
UTMMedium: in.UTMMedium,
|
||||
UTMCampaign: in.UTMCampaign,
|
||||
TextOnly: in.TextOnly,
|
||||
UnsubscribeHeader: in.UnsubscribeHeader,
|
||||
UnsubscribeMode: in.UnsubscribeMode,
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/warmbly/warmbly/internal/app/advanced"
|
||||
"github.com/warmbly/warmbly/internal/config"
|
||||
"github.com/warmbly/warmbly/internal/events"
|
||||
"github.com/warmbly/warmbly/internal/infrastructure/codec"
|
||||
"github.com/warmbly/warmbly/internal/infrastructure/eventbus"
|
||||
@@ -30,6 +31,11 @@ type TrackingConsumer struct {
|
||||
evidence advanced.EvidenceRecorder
|
||||
streamingPublisher *pubsub.StreamingPublisher
|
||||
dedupeRepo repository.TrackingDedupeRepository
|
||||
trackedLinks repository.TrackedLinkRepository
|
||||
linkClicks repository.LinkClickRepository
|
||||
// afterBurstWindow runs fn once the click burst window has passed, so a
|
||||
// human click's side effects wait for the burst rule's verdict.
|
||||
afterBurstWindow func(fn func())
|
||||
// advancedService fires INSTANT open/click action chains the moment a
|
||||
// tracking event lands (the open/click analog of the reply path in
|
||||
// ProcessIncomingReply). Best-effort and nil-safe: when unset, opens/clicks
|
||||
@@ -50,6 +56,8 @@ func NewTrackingConsumer(
|
||||
contactRepo repository.ContactRepository,
|
||||
streamingPublisher *pubsub.StreamingPublisher,
|
||||
dedupeRepo repository.TrackingDedupeRepository,
|
||||
trackedLinks repository.TrackedLinkRepository,
|
||||
linkClicks repository.LinkClickRepository,
|
||||
advancedService advanced.Service,
|
||||
evidence advanced.EvidenceRecorder,
|
||||
) (*TrackingConsumer, error) {
|
||||
@@ -62,10 +70,17 @@ func NewTrackingConsumer(
|
||||
contactRepo: contactRepo,
|
||||
streamingPublisher: streamingPublisher,
|
||||
dedupeRepo: dedupeRepo,
|
||||
advancedService: advancedService,
|
||||
evidence: evidence,
|
||||
topic: topic,
|
||||
group: group,
|
||||
trackedLinks: trackedLinks,
|
||||
linkClicks: linkClicks,
|
||||
afterBurstWindow: func(fn func()) {
|
||||
// One second past the window covers event-time skew between the
|
||||
// tracking service and the consumer.
|
||||
time.AfterFunc(time.Duration(config.TrackingClickBurstSeconds+1)*time.Second, fn)
|
||||
},
|
||||
advancedService: advancedService,
|
||||
evidence: evidence,
|
||||
topic: topic,
|
||||
group: group,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -88,7 +103,19 @@ func (tc *TrackingConsumer) receive(_ context.Context, msg eventbus.Message) err
|
||||
return tc.HandleTrackingEvent(context.Background(), &event)
|
||||
}
|
||||
|
||||
// HandleTrackingEvent processes a tracking event
|
||||
// HandleTrackingEvent processes a tracking event.
|
||||
//
|
||||
// Opens and clicks are classified before they count. The edge already drops
|
||||
// crawlers and security scanners it can name; here the ones it cannot are
|
||||
// caught by what they do: a fetch with no browser, a fetch inside the
|
||||
// machine window after dispatch (nobody reads that fast), and clicks on
|
||||
// several links of one email within seconds (a gateway walking the message).
|
||||
// A machine open is still recorded, labelled, because it proves delivery. A
|
||||
// machine click is logged per link with its reason but never stamps the step
|
||||
// as clicked, fires no automation, and sends no webhook: "clicked" keeps
|
||||
// meaning a person. Because a burst is only recognisable from its second
|
||||
// click, a human click's side effects wait out the burst window before
|
||||
// firing, on the classification the click has by then.
|
||||
func (tc *TrackingConsumer) HandleTrackingEvent(ctx context.Context, event *events.TrackingEvent) error {
|
||||
// Parse and validate task ID
|
||||
taskID, err := uuid.Parse(event.TaskID)
|
||||
@@ -97,37 +124,15 @@ func (tc *TrackingConsumer) HandleTrackingEvent(ctx context.Context, event *even
|
||||
return nil
|
||||
}
|
||||
|
||||
// Calculate URL hash for click event deduplication
|
||||
// Click dedupe identity: the ticket, so two links sharing a destination
|
||||
// are two clicks; the URL only for events from an older tracking build.
|
||||
urlHash := ""
|
||||
if event.EventType == events.EventTypeEmailClicked && event.OriginalURL != nil && *event.OriginalURL != "" {
|
||||
urlHash = hashURL(*event.OriginalURL)
|
||||
}
|
||||
|
||||
// Classify opens: machine fetches (Apple MPP prefetch, UA-less clients)
|
||||
// still count as delivery signal but are labeled, and must never fire
|
||||
// open-triggered automations (a prefetch is not intent).
|
||||
machineOpen := event.EventType == events.EventTypeEmailOpened && isMachineOpen(event.UserAgent)
|
||||
|
||||
// Check for duplicate at consumer level (belt and suspenders with Rust service)
|
||||
if tc.dedupeRepo != nil {
|
||||
processed, err := tc.dedupeRepo.IsProcessed(ctx, taskID, event.EventType, urlHash)
|
||||
if err != nil {
|
||||
// Log but continue - allow processing on dedupe errors
|
||||
log.Warn().Err(err).Str("task_id", event.TaskID).Msg("tracking dedupe check failed")
|
||||
} else if processed {
|
||||
// A HUMAN open after a machine-labeled one upgrades the label
|
||||
// (MPP prefetched at delivery; the person actually read it later
|
||||
// from another network). Quiet write only: the open was already
|
||||
// counted once, so no automations and no re-publish.
|
||||
if event.EventType == events.EventTypeEmailOpened && !machineOpen {
|
||||
if campaignTask, terr := tc.taskRepo.GetCampaignTask(ctx, taskID); terr == nil &&
|
||||
campaignTask != nil && campaignTask.CampaignID != nil &&
|
||||
campaignTask.ContactID != nil && campaignTask.SequenceID != nil {
|
||||
_ = tc.campaignProgressRepo.RecordEmailOpened(ctx,
|
||||
*campaignTask.CampaignID, *campaignTask.ContactID, *campaignTask.SequenceID, false)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
if event.EventType == events.EventTypeEmailClicked {
|
||||
switch {
|
||||
case event.LinkID != nil && *event.LinkID != "":
|
||||
urlHash = hashURL("link:" + *event.LinkID)
|
||||
case event.OriginalURL != nil && *event.OriginalURL != "":
|
||||
urlHash = hashURL(*event.OriginalURL)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -137,16 +142,56 @@ func (tc *TrackingConsumer) HandleTrackingEvent(ctx context.Context, event *even
|
||||
log.Warn().Err(err).Str("task_id", event.TaskID).Msg("failed to get campaign task for tracking event")
|
||||
return nil
|
||||
}
|
||||
if campaignTask == nil || campaignTask.CampaignID == nil || campaignTask.ContactID == nil || campaignTask.SequenceID == nil {
|
||||
// Task not found, not a campaign task, or missing its linkage: skip
|
||||
return nil
|
||||
}
|
||||
campaignID, contactID, sequenceID := *campaignTask.CampaignID, *campaignTask.ContactID, *campaignTask.SequenceID
|
||||
|
||||
if campaignTask == nil || campaignTask.CampaignID == nil {
|
||||
// Task not found or not a campaign task, skip
|
||||
at := eventTime(event.Timestamp)
|
||||
sentAt, err := tc.campaignProgressRepo.GetStepSentAt(ctx, campaignID, contactID, sequenceID)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Str("task_id", event.TaskID).Msg("failed to read step dispatch time; classifying by user agent only")
|
||||
sentAt = nil
|
||||
}
|
||||
|
||||
// Classify. Machine opens (Apple MPP prefetch, UA-less clients, a fetch
|
||||
// inside the machine window) still count as delivery signal but are
|
||||
// labelled, and must never fire open-triggered automations.
|
||||
var machine bool
|
||||
var reason string
|
||||
switch event.EventType {
|
||||
case events.EventTypeEmailOpened:
|
||||
machine = isMachineOpen(event.UserAgent) || isInstant(sentAt, at)
|
||||
case events.EventTypeEmailClicked:
|
||||
machine, reason = classifyClick(event.UserAgent, sentAt, at)
|
||||
default:
|
||||
// Unknown event type, skip
|
||||
return nil
|
||||
}
|
||||
|
||||
// Ensure we have contact_id and sequence_id
|
||||
if campaignTask.ContactID == nil || campaignTask.SequenceID == nil {
|
||||
// Missing required fields, skip
|
||||
return nil
|
||||
// Check for duplicate at consumer level (belt and suspenders with Rust service)
|
||||
if tc.dedupeRepo != nil {
|
||||
processed, err := tc.dedupeRepo.IsProcessed(ctx, taskID, event.EventType, urlHash)
|
||||
if err != nil {
|
||||
// Log but continue - allow processing on dedupe errors
|
||||
log.Warn().Err(err).Str("task_id", event.TaskID).Msg("tracking dedupe check failed")
|
||||
} else if processed {
|
||||
// A HUMAN engagement after a machine-labelled one upgrades the
|
||||
// label (a gateway scanned at delivery; the person acted later).
|
||||
// Quiet write only: the event was already counted once, so no
|
||||
// automations and no re-publish.
|
||||
if machine {
|
||||
return nil
|
||||
}
|
||||
switch event.EventType {
|
||||
case events.EventTypeEmailOpened:
|
||||
_ = tc.campaignProgressRepo.RecordEmailOpened(ctx, campaignID, contactID, sequenceID, false)
|
||||
case events.EventTypeEmailClicked:
|
||||
tc.upgradeClick(ctx, campaignTask, event, at)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Record the event, then fire any INSTANT open/click action chain for the
|
||||
@@ -155,33 +200,40 @@ func (tc *TrackingConsumer) HandleTrackingEvent(ctx context.Context, event *even
|
||||
// to the matcher's eventKind. Firing happens AFTER the Record* write so the
|
||||
// matcher reads the just-stamped opened_at / clicked_at off the progress row.
|
||||
var instantKind string
|
||||
var linkLabel string
|
||||
var deferred bool
|
||||
switch event.EventType {
|
||||
case events.EventTypeEmailOpened:
|
||||
err = tc.campaignProgressRepo.RecordEmailOpened(ctx,
|
||||
*campaignTask.CampaignID,
|
||||
*campaignTask.ContactID,
|
||||
*campaignTask.SequenceID,
|
||||
machineOpen)
|
||||
if !machineOpen {
|
||||
err = tc.campaignProgressRepo.RecordEmailOpened(ctx, campaignID, contactID, sequenceID, machine)
|
||||
if !machine {
|
||||
instantKind = "open"
|
||||
// A human open proves the mailbox is live; a prefetch proves
|
||||
// only that a proxy fetched an image.
|
||||
if tc.evidence != nil {
|
||||
tc.evidence.RecordEvidence(ctx, *campaignTask.ContactID, "opened", campaignTask.SequenceID.String(), "")
|
||||
tc.evidence.RecordEvidence(ctx, contactID, "opened", sequenceID.String(), "")
|
||||
}
|
||||
}
|
||||
case events.EventTypeEmailClicked:
|
||||
err = tc.campaignProgressRepo.RecordEmailClicked(ctx,
|
||||
*campaignTask.CampaignID,
|
||||
*campaignTask.ContactID,
|
||||
*campaignTask.SequenceID)
|
||||
instantKind = "click"
|
||||
if tc.evidence != nil {
|
||||
tc.evidence.RecordEvidence(ctx, *campaignTask.ContactID, "clicked", campaignTask.SequenceID.String(), "")
|
||||
var click *repository.LinkClick
|
||||
machine, reason, click, err = tc.recordClick(ctx, campaignTask, event, at, machine, reason)
|
||||
if click != nil {
|
||||
linkLabel = click.Label
|
||||
}
|
||||
if err == nil && !machine {
|
||||
// The stamp is stored state a burst can walk back; the effects
|
||||
// cannot be recalled, so they wait for the window to close.
|
||||
err = tc.campaignProgressRepo.RecordEmailClicked(ctx, campaignID, contactID, sequenceID)
|
||||
if err == nil && click != nil && tc.afterBurstWindow != nil {
|
||||
deferred = true
|
||||
task, ev, clickID, label := campaignTask, *event, click.ID, linkLabel
|
||||
tc.afterBurstWindow(func() { tc.finishHumanClick(task, ev, clickID, label) })
|
||||
} else if err == nil {
|
||||
instantKind = "click"
|
||||
if tc.evidence != nil {
|
||||
tc.evidence.RecordEvidence(ctx, contactID, "clicked", sequenceID.String(), "")
|
||||
}
|
||||
}
|
||||
}
|
||||
default:
|
||||
// Unknown event type, skip
|
||||
return nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
@@ -195,11 +247,7 @@ func (tc *TrackingConsumer) HandleTrackingEvent(ctx context.Context, event *even
|
||||
// opened/clicked branch at the next step boundary. Exactly-once per (step,
|
||||
// eventKind) is enforced inside FireInstantActions via ClaimInstantFire.
|
||||
if tc.advancedService != nil && instantKind != "" {
|
||||
tc.advancedService.FireInstantActions(ctx,
|
||||
*campaignTask.CampaignID,
|
||||
*campaignTask.ContactID,
|
||||
*campaignTask.SequenceID,
|
||||
instantKind)
|
||||
tc.advancedService.FireInstantActions(ctx, campaignID, contactID, sequenceID, instantKind)
|
||||
}
|
||||
|
||||
// Mark as processed for deduplication
|
||||
@@ -209,15 +257,187 @@ func (tc *TrackingConsumer) HandleTrackingEvent(ctx context.Context, event *even
|
||||
}
|
||||
}
|
||||
|
||||
// Publish to Pub/Sub for realtime updates
|
||||
tc.publishTrackingEvent(ctx, campaignTask, *event, machineOpen)
|
||||
if machine {
|
||||
log.Debug().Str("task_id", event.TaskID).Str("event_type", string(event.EventType)).Str("reason", reason).Msg("tracking event classified as machine")
|
||||
}
|
||||
|
||||
// Publish to Pub/Sub for realtime updates (a deferred human click
|
||||
// publishes once its verdict is final)
|
||||
if !deferred {
|
||||
tc.publishTrackingEvent(ctx, campaignTask, *event, machine, linkLabel)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// finishHumanClick runs the effects of a click that looked human when it
|
||||
// landed, once the burst window has passed: if a burst relabelled it in the
|
||||
// meantime it is announced as automated and nothing else fires. The stamp,
|
||||
// the click log and the dedupe mark were written up front, so a consumer
|
||||
// restart inside the window loses only these effects, and the scheduler
|
||||
// still routes the clicked branch at the next step boundary. The verdict is
|
||||
// re-read with retries and, when it cannot be read at all, nothing fires:
|
||||
// an automation for a scanner's click is worse than a missed one, and the
|
||||
// step boundary still routes on the stored stamp.
|
||||
func (tc *TrackingConsumer) finishHumanClick(task *repository.CampaignTask, event events.TrackingEvent, clickID uuid.UUID, label string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
var machine bool
|
||||
var err error
|
||||
for attempt := 0; attempt < 5; attempt++ {
|
||||
if machine, err = tc.linkClicks.IsMachine(ctx, clickID); err == nil {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(time.Duration(attempt+1) * 2 * time.Second):
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
log.Error().Err(err).Str("click_id", clickID.String()).Msg("could not re-read click classification; click effects skipped")
|
||||
return
|
||||
}
|
||||
if machine {
|
||||
tc.publishTrackingEvent(ctx, task, event, true, label)
|
||||
return
|
||||
}
|
||||
if tc.evidence != nil {
|
||||
tc.evidence.RecordEvidence(ctx, *task.ContactID, "clicked", task.SequenceID.String(), "")
|
||||
}
|
||||
if tc.advancedService != nil {
|
||||
tc.advancedService.FireInstantActions(ctx, *task.CampaignID, *task.ContactID, *task.SequenceID, "click")
|
||||
}
|
||||
tc.publishTrackingEvent(ctx, task, event, false, label)
|
||||
}
|
||||
|
||||
// resolveLink names the clicked link: the minted ticket when the event
|
||||
// carries one (destination and anchor text as stored at send time), else the
|
||||
// URL the event reports. nil ticket id means the click log row stands alone.
|
||||
func (tc *TrackingConsumer) resolveLink(ctx context.Context, event *events.TrackingEvent) (*uuid.UUID, string, string) {
|
||||
var destination string
|
||||
if event.OriginalURL != nil {
|
||||
destination = *event.OriginalURL
|
||||
}
|
||||
if event.LinkID == nil || tc.trackedLinks == nil {
|
||||
return nil, destination, ""
|
||||
}
|
||||
id, err := uuid.Parse(*event.LinkID)
|
||||
if err != nil {
|
||||
return nil, destination, ""
|
||||
}
|
||||
link, err := tc.trackedLinks.GetByID(ctx, id)
|
||||
if err != nil || link == nil {
|
||||
return nil, destination, ""
|
||||
}
|
||||
if link.Destination != "" {
|
||||
destination = link.Destination
|
||||
}
|
||||
return &link.ID, destination, link.Label
|
||||
}
|
||||
|
||||
// recordClick logs the click per link and applies the burst rule: a click on
|
||||
// a second link of the same email from the same source inside the burst
|
||||
// window turns this click AND the earlier ones into machine clicks. When
|
||||
// that leaves the step with no human click, the clicked stamp the first
|
||||
// click already wrote is walked back. Returns the final classification and
|
||||
// the logged row (nil when nothing could be logged).
|
||||
func (tc *TrackingConsumer) recordClick(ctx context.Context, task *repository.CampaignTask, event *events.TrackingEvent, at time.Time, machine bool, reason string) (bool, string, *repository.LinkClick, error) {
|
||||
if tc.linkClicks == nil {
|
||||
return machine, reason, nil, nil
|
||||
}
|
||||
linkID, destination, label := tc.resolveLink(ctx, event)
|
||||
if destination == "" {
|
||||
return machine, reason, nil, nil
|
||||
}
|
||||
ipHash := ""
|
||||
if event.IPHash != nil {
|
||||
ipHash = *event.IPHash
|
||||
}
|
||||
userAgent := ""
|
||||
if event.UserAgent != nil {
|
||||
userAgent = *event.UserAgent
|
||||
}
|
||||
|
||||
burstSince := at.Add(-time.Duration(config.TrackingClickBurstSeconds) * time.Second)
|
||||
burst := false
|
||||
if !machine && ipHash != "" {
|
||||
n, err := tc.linkClicks.CountRecentOtherLinks(ctx, task.TaskID, ipHash, linkID, destination, burstSince)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Str("task_id", task.TaskID.String()).Msg("burst check failed; treating click as a person's")
|
||||
} else if n > 0 {
|
||||
machine, reason, burst = true, repository.LinkClickReasonBurst, true
|
||||
}
|
||||
}
|
||||
|
||||
click := &repository.LinkClick{
|
||||
TrackedLinkID: linkID,
|
||||
TaskID: task.TaskID,
|
||||
CampaignID: *task.CampaignID,
|
||||
ContactID: *task.ContactID,
|
||||
SequenceID: *task.SequenceID,
|
||||
Destination: destination,
|
||||
Label: label,
|
||||
UserAgent: userAgent,
|
||||
IPHash: ipHash,
|
||||
Machine: machine,
|
||||
MachineReason: reason,
|
||||
ClickedAt: at,
|
||||
}
|
||||
if err := tc.linkClicks.Insert(ctx, click); err != nil {
|
||||
return machine, reason, click, err
|
||||
}
|
||||
|
||||
if burst {
|
||||
if _, err := tc.linkClicks.MarkBurst(ctx, task.TaskID, ipHash, burstSince); err != nil {
|
||||
log.Warn().Err(err).Str("task_id", task.TaskID.String()).Msg("failed to relabel burst clicks")
|
||||
}
|
||||
if err := tc.campaignProgressRepo.UnrecordEmailClicked(ctx, *task.CampaignID, *task.ContactID, *task.SequenceID); err != nil {
|
||||
log.Warn().Err(err).Str("task_id", task.TaskID.String()).Msg("failed to walk back the click stamp after a burst")
|
||||
}
|
||||
}
|
||||
return machine, reason, click, nil
|
||||
}
|
||||
|
||||
// upgradeClick handles a human click on a link this email was already
|
||||
// credited for: the step is stamped clicked if only machines had clicked so
|
||||
// far, and the click is logged once so the timeline shows the person's.
|
||||
func (tc *TrackingConsumer) upgradeClick(ctx context.Context, task *repository.CampaignTask, event *events.TrackingEvent, at time.Time) {
|
||||
_ = tc.campaignProgressRepo.RecordEmailClicked(ctx, *task.CampaignID, *task.ContactID, *task.SequenceID)
|
||||
if tc.linkClicks == nil {
|
||||
return
|
||||
}
|
||||
linkID, destination, label := tc.resolveLink(ctx, event)
|
||||
if destination == "" {
|
||||
return
|
||||
}
|
||||
if seen, err := tc.linkClicks.HasHumanClickOn(ctx, task.TaskID, linkID, destination); err != nil || seen {
|
||||
return
|
||||
}
|
||||
ipHash, userAgent := "", ""
|
||||
if event.IPHash != nil {
|
||||
ipHash = *event.IPHash
|
||||
}
|
||||
if event.UserAgent != nil {
|
||||
userAgent = *event.UserAgent
|
||||
}
|
||||
_ = tc.linkClicks.Insert(ctx, &repository.LinkClick{
|
||||
TrackedLinkID: linkID,
|
||||
TaskID: task.TaskID,
|
||||
CampaignID: *task.CampaignID,
|
||||
ContactID: *task.ContactID,
|
||||
SequenceID: *task.SequenceID,
|
||||
Destination: destination,
|
||||
Label: label,
|
||||
UserAgent: userAgent,
|
||||
IPHash: ipHash,
|
||||
ClickedAt: at,
|
||||
})
|
||||
}
|
||||
|
||||
// publishTrackingEvent publishes the tracking event to Pub/Sub for realtime UI
|
||||
// updates AND fans an opt-in firehose webhook (campaign.email_opened/clicked).
|
||||
func (tc *TrackingConsumer) publishTrackingEvent(ctx context.Context, task *repository.CampaignTask, event events.TrackingEvent, machine bool) {
|
||||
func (tc *TrackingConsumer) publishTrackingEvent(ctx context.Context, task *repository.CampaignTask, event events.TrackingEvent, machine bool, linkLabel string) {
|
||||
// Get campaign to find user ID + org
|
||||
campaign, err := tc.campaignRepo.GetByID(ctx, *task.CampaignID)
|
||||
if err != nil || campaign == nil {
|
||||
@@ -233,14 +453,14 @@ func (tc *TrackingConsumer) publishTrackingEvent(ctx context.Context, task *repo
|
||||
}
|
||||
}
|
||||
|
||||
// Fan an opt-in firehose webhook for the open/click (org-scoped). Human opens
|
||||
// only — a machine prefetch is not engagement intent — clicks always.
|
||||
if tc.advancedService != nil && campaign.OrganizationID != nil {
|
||||
// Fan an opt-in firehose webhook for the open/click (org-scoped). People
|
||||
// only: a prefetch or a gateway walking the links is not engagement.
|
||||
if tc.advancedService != nil && campaign.OrganizationID != nil && !machine {
|
||||
var whType models.WebhookEventType
|
||||
switch {
|
||||
case event.EventType == events.EventTypeEmailOpened && !machine:
|
||||
switch event.EventType {
|
||||
case events.EventTypeEmailOpened:
|
||||
whType = models.WebhookEventCampaignEmailOpened
|
||||
case event.EventType == events.EventTypeEmailClicked:
|
||||
case events.EventTypeEmailClicked:
|
||||
whType = models.WebhookEventCampaignEmailClicked
|
||||
}
|
||||
if whType != "" {
|
||||
@@ -252,6 +472,9 @@ func (tc *TrackingConsumer) publishTrackingEvent(ctx context.Context, task *repo
|
||||
}
|
||||
if event.EventType == events.EventTypeEmailClicked && event.OriginalURL != nil {
|
||||
data["url"] = *event.OriginalURL
|
||||
if linkLabel != "" {
|
||||
data["link_label"] = linkLabel
|
||||
}
|
||||
}
|
||||
tc.advancedService.EmitCampaignEvent(ctx, *campaign.OrganizationID, whType, data)
|
||||
}
|
||||
@@ -294,6 +517,7 @@ func (tc *TrackingConsumer) publishTrackingEvent(ctx context.Context, task *repo
|
||||
|
||||
if event.EventType == events.EventTypeEmailClicked && event.OriginalURL != nil {
|
||||
trackingPayload.OriginalURL = *event.OriginalURL
|
||||
trackingPayload.LinkLabel = linkLabel
|
||||
}
|
||||
|
||||
tc.streamingPublisher.PublishTrackingEvent(ctx, trackingPayload)
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
package jobs
|
||||
|
||||
import "strings"
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/warmbly/warmbly/internal/config"
|
||||
"github.com/warmbly/warmbly/internal/repository"
|
||||
)
|
||||
|
||||
// isMachineOpen reports whether an open event came from an automated fetcher
|
||||
// rather than a human-rendered view. The edge already filters crawlers and
|
||||
@@ -25,3 +31,41 @@ func isMachineOpen(userAgent *string) bool {
|
||||
}
|
||||
return strings.HasSuffix(ua, "(khtml, like gecko)")
|
||||
}
|
||||
|
||||
// isInstant reports whether an engagement arrived so soon after the step was
|
||||
// dispatched that no person could have read the email yet. Security
|
||||
// gateways (Safe Links, Proofpoint, Mimecast) open the pixel and walk every
|
||||
// link at delivery time with an ordinary browser UA, which is exactly what
|
||||
// the UA rules cannot see. An unknown dispatch time never counts as instant.
|
||||
func isInstant(sentAt *time.Time, at time.Time) bool {
|
||||
if sentAt == nil {
|
||||
return false
|
||||
}
|
||||
return at.Sub(*sentAt) < time.Duration(config.TrackingMachineWindowSeconds)*time.Second
|
||||
}
|
||||
|
||||
// classifyClick applies the per-event click rules (the burst rule needs the
|
||||
// click log and lives in the consumer). It returns whether the click is
|
||||
// automated and the reason recorded with it; an empty reason is a person.
|
||||
func classifyClick(userAgent *string, sentAt *time.Time, at time.Time) (bool, string) {
|
||||
if userAgent == nil || strings.TrimSpace(*userAgent) == "" {
|
||||
return true, repository.LinkClickReasonPrefetch
|
||||
}
|
||||
if isInstant(sentAt, at) {
|
||||
return true, repository.LinkClickReasonInstant
|
||||
}
|
||||
return false, ""
|
||||
}
|
||||
|
||||
// eventTime is when the tracking service saw the event, falling back to now
|
||||
// when the stamp is missing or unreadable, so consumer lag never turns a
|
||||
// delivery-time scan into a plausible human open.
|
||||
func eventTime(stamp string) time.Time {
|
||||
if t, err := time.Parse(time.RFC3339Nano, stamp); err == nil {
|
||||
return t
|
||||
}
|
||||
if t, err := time.Parse(time.RFC3339, stamp); err == nil {
|
||||
return t
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package jobs
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/warmbly/warmbly/internal/repository"
|
||||
)
|
||||
|
||||
func strp(s string) *string { return &s }
|
||||
|
||||
func TestIsInstantUsesTheDispatchClock(t *testing.T) {
|
||||
sent := time.Now()
|
||||
if !isInstant(&sent, sent.Add(3*time.Second)) {
|
||||
t.Fatal("three seconds after dispatch is a machine")
|
||||
}
|
||||
if isInstant(&sent, sent.Add(45*time.Second)) {
|
||||
t.Fatal("forty-five seconds after dispatch can be a person")
|
||||
}
|
||||
if isInstant(nil, sent) {
|
||||
t.Fatal("an unknown dispatch time must never count as instant")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyClick(t *testing.T) {
|
||||
sent := time.Now()
|
||||
chrome := strp("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36")
|
||||
|
||||
if m, r := classifyClick(nil, &sent, sent.Add(time.Minute)); !m || r != repository.LinkClickReasonPrefetch {
|
||||
t.Fatalf("no user agent = prefetch, got %v %q", m, r)
|
||||
}
|
||||
if m, r := classifyClick(chrome, &sent, sent.Add(2*time.Second)); !m || r != repository.LinkClickReasonInstant {
|
||||
t.Fatalf("a browser UA two seconds after dispatch = instant, got %v %q", m, r)
|
||||
}
|
||||
if m, r := classifyClick(chrome, &sent, sent.Add(time.Minute)); m || r != "" {
|
||||
t.Fatalf("a browser a minute later is a person, got %v %q", m, r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEventTimeFallsBackToNow(t *testing.T) {
|
||||
stamp := "2026-09-03T10:00:00Z"
|
||||
if got := eventTime(stamp); !got.Equal(time.Date(2026, 9, 3, 10, 0, 0, 0, time.UTC)) {
|
||||
t.Fatalf("unexpected parse: %v", got)
|
||||
}
|
||||
if d := time.Since(eventTime("garbage")); d < 0 || d > time.Minute {
|
||||
t.Fatalf("unreadable stamp should fall back to now, got %v ago", d)
|
||||
}
|
||||
}
|
||||
@@ -654,6 +654,13 @@ var Tables = []Table{
|
||||
Scope: `campaign_id IN ` + orgCampaigns,
|
||||
Note: "Click tickets already in the wild keep resolving after the move, provided the tracking domain follows.",
|
||||
},
|
||||
{
|
||||
// Campaign engagement like campaign_contact_progress, one row per
|
||||
// link. Sits below tracked_links because of the nullable ticket
|
||||
// reference, which the importer blanks when send history stays behind.
|
||||
Name: "email_link_clicks", Group: models.OrgDataGroupCampaigns,
|
||||
Scope: `campaign_id IN ` + orgCampaigns,
|
||||
},
|
||||
|
||||
// ---------- delivery events ----------
|
||||
{
|
||||
|
||||
@@ -132,6 +132,19 @@ const (
|
||||
// it must stay well clear of a slow provider handshake.
|
||||
CampaignSendReclaimAfterMinutes = 30
|
||||
|
||||
// TrackingMachineWindowSeconds is how soon after a step was dispatched an
|
||||
// open or click is treated as automated rather than a person. The clock
|
||||
// starts when the send is handed to the worker, before the provider has
|
||||
// even accepted the message, so a person cannot plausibly have read and
|
||||
// acted on it inside this window; security gateways that detonate every
|
||||
// link at delivery time routinely do.
|
||||
TrackingMachineWindowSeconds = 10
|
||||
|
||||
// TrackingClickBurstSeconds is the window inside which clicks on two
|
||||
// different links of the same email from the same source are treated as
|
||||
// a scanner walking the message. A person follows one link at a time.
|
||||
TrackingClickBurstSeconds = 5
|
||||
|
||||
// CampaignSendStampAttempts is how many times the control plane retries the
|
||||
// sent_at stamp after a send is already on the bus. The reservation is what
|
||||
// keeps the step from being re-sent, so a lost stamp is a pacing problem,
|
||||
|
||||
@@ -53,6 +53,7 @@ type TrackingEvent struct {
|
||||
EventType string `json:"event_type" avro:"event_type"` // EMAIL_OPENED or EMAIL_CLICKED
|
||||
TaskID string `json:"task_id" avro:"task_id"` // UUID string
|
||||
OriginalURL *string `json:"original_url" avro:"original_url"` // For click events only (nullable)
|
||||
LinkID *string `json:"link_id" avro:"link_id"` // Click ticket id (nullable; absent from older tracking builds)
|
||||
Timestamp string `json:"timestamp" avro:"timestamp"` // ISO8601 timestamp
|
||||
UserAgent *string `json:"user_agent" avro:"user_agent"` // Browser user agent (nullable)
|
||||
IPHash *string `json:"ip_hash" avro:"ip_hash"` // Hashed IP for privacy (nullable)
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
DROP TABLE IF EXISTS email_link_clicks;
|
||||
|
||||
ALTER TABLE tracked_links
|
||||
DROP COLUMN IF EXISTS label;
|
||||
|
||||
ALTER TABLE campaigns
|
||||
DROP COLUMN IF EXISTS utm_tracking,
|
||||
DROP COLUMN IF EXISTS utm_source,
|
||||
DROP COLUMN IF EXISTS utm_medium,
|
||||
DROP COLUMN IF EXISTS utm_campaign;
|
||||
@@ -0,0 +1,38 @@
|
||||
-- Per-link click attribution and automatic UTM tagging.
|
||||
--
|
||||
-- Every click on a tracked link is logged as its own row so the contact
|
||||
-- timeline can say WHICH link was clicked, not only that one was. A click
|
||||
-- that looks automated (a security gateway following every link seconds
|
||||
-- after delivery) is kept for the record but flagged, and it never stamps
|
||||
-- campaign_contact_progress.clicked_at, so "clicked" keeps meaning a person.
|
||||
|
||||
ALTER TABLE campaigns
|
||||
ADD COLUMN IF NOT EXISTS utm_tracking boolean NOT NULL DEFAULT false,
|
||||
ADD COLUMN IF NOT EXISTS utm_source text NOT NULL DEFAULT '',
|
||||
ADD COLUMN IF NOT EXISTS utm_medium text NOT NULL DEFAULT '',
|
||||
ADD COLUMN IF NOT EXISTS utm_campaign text NOT NULL DEFAULT '';
|
||||
|
||||
-- The anchor text the link was minted from ("Pricing"), so a click can be
|
||||
-- named without re-parsing the email.
|
||||
ALTER TABLE tracked_links
|
||||
ADD COLUMN IF NOT EXISTS label text NOT NULL DEFAULT '';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS email_link_clicks (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
tracked_link_id uuid REFERENCES tracked_links(id) ON DELETE SET NULL,
|
||||
task_id uuid NOT NULL,
|
||||
campaign_id uuid NOT NULL REFERENCES campaigns(id) ON DELETE CASCADE,
|
||||
contact_id uuid NOT NULL REFERENCES contacts(id) ON DELETE CASCADE,
|
||||
sequence_id uuid NOT NULL REFERENCES sequences(id) ON DELETE CASCADE,
|
||||
destination text NOT NULL,
|
||||
label text NOT NULL DEFAULT '',
|
||||
user_agent text NOT NULL DEFAULT '',
|
||||
ip_hash text NOT NULL DEFAULT '',
|
||||
machine boolean NOT NULL DEFAULT false,
|
||||
machine_reason text NOT NULL DEFAULT '' CHECK (machine_reason IN ('', 'prefetch', 'instant', 'burst')),
|
||||
clicked_at timestamp with time zone NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_email_link_clicks_contact ON email_link_clicks (contact_id, clicked_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_email_link_clicks_task ON email_link_clicks (task_id, clicked_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_email_link_clicks_step ON email_link_clicks (campaign_id, contact_id, sequence_id);
|
||||
@@ -217,8 +217,10 @@ type TrackingEventPayload struct {
|
||||
ContactEmail string `json:"contact_email,omitempty"`
|
||||
SequenceID string `json:"step_id,omitempty"`
|
||||
OriginalURL string `json:"original_url,omitempty"` // For click events
|
||||
// Machine marks an automated open (Apple MPP prefetch, UA-less fetcher)
|
||||
// so live views can badge it instead of presenting it as a human open.
|
||||
LinkLabel string `json:"link_label,omitempty"` // Anchor text of the clicked link
|
||||
// Machine marks an automated open or click (Apple MPP prefetch, UA-less
|
||||
// fetcher, a security gateway walking the links) so live views can badge
|
||||
// it instead of presenting it as a person's.
|
||||
Machine bool `json:"machine,omitempty"`
|
||||
}
|
||||
|
||||
|
||||
@@ -58,9 +58,13 @@ type CampaignSummary struct {
|
||||
// (Apple MPP prefetch, UA-less clients). Human opens = unique - machine.
|
||||
MachineOpens int `json:"machine_opens"`
|
||||
UniqueClicks int `json:"unique_clicks"`
|
||||
Replies int `json:"replies"`
|
||||
Bounces int `json:"bounces"`
|
||||
Unsubscribes int `json:"unsubscribes"`
|
||||
// MachineClicks counts steps whose only clicks came from automated
|
||||
// fetchers (security gateways walking the links). They are not part of
|
||||
// UniqueClicks, which only ever counts a person's click.
|
||||
MachineClicks int `json:"machine_clicks"`
|
||||
Replies int `json:"replies"`
|
||||
Bounces int `json:"bounces"`
|
||||
Unsubscribes int `json:"unsubscribes"`
|
||||
|
||||
OpenRate float64 `json:"open_rate"` // percentage
|
||||
ClickRate float64 `json:"click_rate"` // percentage
|
||||
@@ -249,8 +253,11 @@ type DashboardOverallStats struct {
|
||||
TotalEmailsSent int `json:"total_emails_sent"`
|
||||
TotalOpens int `json:"total_opens"`
|
||||
// MachineOpens is the subset of TotalOpens from automated fetchers.
|
||||
MachineOpens int `json:"machine_opens"`
|
||||
TotalClicks int `json:"total_clicks"`
|
||||
MachineOpens int `json:"machine_opens"`
|
||||
TotalClicks int `json:"total_clicks"`
|
||||
// MachineClicks counts steps clicked only by automated fetchers; they are
|
||||
// not part of TotalClicks.
|
||||
MachineClicks int `json:"machine_clicks"`
|
||||
TotalReplies int `json:"total_replies"`
|
||||
TotalBounces int `json:"total_bounces"`
|
||||
OpenRate float64 `json:"open_rate"`
|
||||
|
||||
@@ -187,6 +187,14 @@ type Campaign struct {
|
||||
TrackingDomainVerified bool `json:"tracking_domain_verified"`
|
||||
TrackingDomainVerifiedAt *time.Time `json:"tracking_domain_verified_at,omitempty"`
|
||||
|
||||
// Automatic UTM tagging of every link in the email body. Empty source,
|
||||
// medium and campaign values mean the defaults ("warmbly", "email", the
|
||||
// campaign name); utm_content is always the link's own text.
|
||||
UTMTracking bool `json:"utm_tracking"`
|
||||
UTMSource string `json:"utm_source"`
|
||||
UTMMedium string `json:"utm_medium"`
|
||||
UTMCampaign string `json:"utm_campaign"`
|
||||
|
||||
LastStatusChangeAt *time.Time `json:"last_status_change_at,omitempty"`
|
||||
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
@@ -313,6 +321,11 @@ type UpdateCampaign struct {
|
||||
PrioritizeNewLeads *bool `json:"prioritize_new_leads,omitempty"`
|
||||
TrackingDomain *string `json:"tracking_domain,omitempty"`
|
||||
|
||||
UTMTracking *bool `json:"utm_tracking,omitempty"`
|
||||
UTMSource *string `json:"utm_source,omitempty"`
|
||||
UTMMedium *string `json:"utm_medium,omitempty"`
|
||||
UTMCampaign *string `json:"utm_campaign,omitempty"`
|
||||
|
||||
// Auto-pause guardrails. GuardrailTrippedAt/Reason are server-owned and
|
||||
// are cleared when the campaign is started again, so they are not settable
|
||||
// here.
|
||||
@@ -385,6 +398,12 @@ type CreateCampaign struct {
|
||||
PrioritizeNewLeads *bool `json:"prioritize_new_leads,omitempty"`
|
||||
TrackingDomain *string `json:"tracking_domain,omitempty"`
|
||||
|
||||
// Automatic UTM tagging (off unless sent). Empty values keep the defaults.
|
||||
UTMTracking *bool `json:"utm_tracking,omitempty"`
|
||||
UTMSource *string `json:"utm_source,omitempty"`
|
||||
UTMMedium *string `json:"utm_medium,omitempty"`
|
||||
UTMCampaign *string `json:"utm_campaign,omitempty"`
|
||||
|
||||
// Initial sequences (in order) — caller can also create them after.
|
||||
Sequences []CreateSequenceInput `json:"steps,omitempty"`
|
||||
|
||||
|
||||
@@ -463,10 +463,36 @@ type ContactTimelineEvent struct {
|
||||
// Website page view (page_hit): URL, referrer, device, UTM, location.
|
||||
PageHit *WebsitePageHit `json:"page_hit,omitempty"`
|
||||
|
||||
// Engagement classification (email_opened / email_clicked). Machine is
|
||||
// true when the open or click came from an automated fetcher (a mail
|
||||
// privacy proxy, a security gateway walking the links) rather than a
|
||||
// person; MachineReason says which rule caught it.
|
||||
Machine *bool `json:"machine,omitempty"`
|
||||
MachineReason *string `json:"machine_reason,omitempty"`
|
||||
|
||||
// The exact link behind an email_clicked event, when the click was
|
||||
// logged per link (every click since link attribution shipped).
|
||||
Link *ContactLinkClick `json:"link,omitempty"`
|
||||
|
||||
// Author (notes, lifecycle events).
|
||||
UserID *uuid.UUID `json:"user_id,omitempty"`
|
||||
}
|
||||
|
||||
// ContactLinkClick names the link a contact clicked: where it went, the
|
||||
// anchor text it was minted from, and the UTM parameters the destination
|
||||
// carried (automatic or hand-written).
|
||||
type ContactLinkClick struct {
|
||||
ID uuid.UUID `json:"id"`
|
||||
URL string `json:"url"`
|
||||
Label string `json:"label,omitempty"`
|
||||
UTMSource string `json:"utm_source,omitempty"`
|
||||
UTMMedium string `json:"utm_medium,omitempty"`
|
||||
UTMCampaign string `json:"utm_campaign,omitempty"`
|
||||
UTMTerm string `json:"utm_term,omitempty"`
|
||||
UTMContent string `json:"utm_content,omitempty"`
|
||||
UserAgent string `json:"user_agent,omitempty"`
|
||||
}
|
||||
|
||||
type ContactTimelineResult struct {
|
||||
Data []ContactTimelineEvent `json:"data"`
|
||||
// True if we hit the per-call cap and the caller should paginate
|
||||
|
||||
@@ -405,7 +405,7 @@ func scanSuppressedRecipient(row pgx.Row) (*models.SuppressedRecipient, error) {
|
||||
// row first, else a row for its domain. Same predicate as the SQL function
|
||||
// recipient_suppressed() the send gates use, spelled out here because the
|
||||
// caller wants the row, not a boolean. Stored values are lowercase (every
|
||||
// write folds them; migration 000123 folded the rest), so the comparison is
|
||||
// write folds them; migration 000124 folded the rest), so the comparison is
|
||||
// an equality the unique index serves.
|
||||
func (r *advancedOutreachRepository) IsRecipientSuppressed(ctx context.Context, organizationID uuid.UUID, email string) (*models.SuppressedRecipient, error) {
|
||||
query := `
|
||||
|
||||
@@ -97,6 +97,13 @@ func (r *analyticsRepository) GetCampaignSummary(ctx context.Context, userID, ca
|
||||
COUNT(CASE WHEN ccp.opened_at IS NOT NULL THEN 1 END) as unique_opens,
|
||||
COUNT(CASE WHEN ccp.opened_at IS NOT NULL AND ccp.opened_machine THEN 1 END) as machine_opens,
|
||||
COUNT(CASE WHEN ccp.clicked_at IS NOT NULL THEN 1 END) as unique_clicks,
|
||||
COUNT(CASE WHEN ccp.clicked_at IS NULL AND EXISTS (
|
||||
SELECT 1 FROM email_link_clicks lc
|
||||
WHERE lc.campaign_id = ccp.campaign_id AND lc.contact_id = ccp.contact_id AND lc.sequence_id = ccp.sequence_id AND lc.machine
|
||||
) AND NOT EXISTS (
|
||||
SELECT 1 FROM email_link_clicks lc
|
||||
WHERE lc.campaign_id = ccp.campaign_id AND lc.contact_id = ccp.contact_id AND lc.sequence_id = ccp.sequence_id AND NOT lc.machine
|
||||
) THEN 1 END) as machine_clicks,
|
||||
COUNT(CASE WHEN ccp.replied_at IS NOT NULL THEN 1 END) as replies,
|
||||
COUNT(CASE WHEN ccp.bounced_at IS NOT NULL THEN 1 END) as bounces
|
||||
FROM campaign_contact_progress ccp
|
||||
@@ -114,6 +121,7 @@ func (r *analyticsRepository) GetCampaignSummary(ctx context.Context, userID, ca
|
||||
&summary.UniqueOpens,
|
||||
&summary.MachineOpens,
|
||||
&summary.UniqueClicks,
|
||||
&summary.MachineClicks,
|
||||
&summary.Replies,
|
||||
&summary.Bounces,
|
||||
)
|
||||
@@ -345,6 +353,13 @@ func (r *analyticsRepository) GetDashboardOverallStats(ctx context.Context, orgI
|
||||
COUNT(CASE WHEN ccp.opened_at IS NOT NULL AND ccp.sent_at >= $2 AND ccp.sent_at <= $3 THEN 1 END) as total_opens,
|
||||
COUNT(CASE WHEN ccp.opened_at IS NOT NULL AND ccp.opened_machine AND ccp.sent_at >= $2 AND ccp.sent_at <= $3 THEN 1 END) as machine_opens,
|
||||
COUNT(CASE WHEN ccp.clicked_at IS NOT NULL AND ccp.sent_at >= $2 AND ccp.sent_at <= $3 THEN 1 END) as total_clicks,
|
||||
COUNT(CASE WHEN ccp.clicked_at IS NULL AND ccp.sent_at >= $2 AND ccp.sent_at <= $3 AND EXISTS (
|
||||
SELECT 1 FROM email_link_clicks lc
|
||||
WHERE lc.campaign_id = ccp.campaign_id AND lc.contact_id = ccp.contact_id AND lc.sequence_id = ccp.sequence_id AND lc.machine
|
||||
) AND NOT EXISTS (
|
||||
SELECT 1 FROM email_link_clicks lc
|
||||
WHERE lc.campaign_id = ccp.campaign_id AND lc.contact_id = ccp.contact_id AND lc.sequence_id = ccp.sequence_id AND NOT lc.machine
|
||||
) THEN 1 END) as machine_clicks,
|
||||
COUNT(CASE WHEN ccp.replied_at IS NOT NULL AND ccp.sent_at >= $2 AND ccp.sent_at <= $3 THEN 1 END) as total_replies,
|
||||
COUNT(CASE WHEN ccp.bounced_at IS NOT NULL AND ccp.sent_at >= $2 AND ccp.sent_at <= $3 THEN 1 END) as total_bounces,
|
||||
(SELECT COUNT(*) FROM campaigns WHERE organization_id = $1 AND status = 'active') as active_campaigns,
|
||||
@@ -362,6 +377,7 @@ func (r *analyticsRepository) GetDashboardOverallStats(ctx context.Context, orgI
|
||||
&stats.TotalOpens,
|
||||
&stats.MachineOpens,
|
||||
&stats.TotalClicks,
|
||||
&stats.MachineClicks,
|
||||
&stats.TotalReplies,
|
||||
&stats.TotalBounces,
|
||||
&stats.ActiveCampaigns,
|
||||
@@ -397,9 +413,13 @@ func (r *analyticsRepository) GetRecentActivity(ctx context.Context, orgID uuid.
|
||||
|
||||
UNION ALL
|
||||
|
||||
-- Clicks
|
||||
-- Clicks (the first link a person clicked on the step, when logged per link)
|
||||
SELECT 'clicked' as type, ccp.campaign_id, c.name as campaign_name,
|
||||
co.email as contact_email, ccp.contact_id, ccp.clicked_at as timestamp, NULL as link
|
||||
co.email as contact_email, ccp.contact_id, ccp.clicked_at as timestamp,
|
||||
(SELECT lc.destination FROM email_link_clicks lc
|
||||
WHERE lc.campaign_id = ccp.campaign_id AND lc.contact_id = ccp.contact_id
|
||||
AND lc.sequence_id = ccp.sequence_id AND lc.machine = false
|
||||
ORDER BY lc.clicked_at LIMIT 1) as link
|
||||
FROM campaign_contact_progress ccp
|
||||
JOIN campaigns c ON c.id = ccp.campaign_id
|
||||
JOIN contacts co ON co.id = ccp.contact_id
|
||||
|
||||
@@ -146,6 +146,7 @@ const CAMPAIGN_SELECT = `id, name, description, status,
|
||||
guardrail_reply_rate_min, guardrail_min_sample, guardrail_window_days,
|
||||
guardrail_tripped_at, guardrail_reason,
|
||||
kind,
|
||||
utm_tracking, utm_source, utm_medium, utm_campaign,
|
||||
unsubscribe_mode`
|
||||
|
||||
func getCampaign(rows db.Scannable, campaign *models.Campaign, extra ...any) error {
|
||||
@@ -166,6 +167,7 @@ func getCampaign(rows db.Scannable, campaign *models.Campaign, extra ...any) err
|
||||
&campaign.GuardrailReplyRateMin, &campaign.GuardrailMinSample, &campaign.GuardrailWindowDays,
|
||||
&campaign.GuardrailTrippedAt, &campaign.GuardrailReason,
|
||||
&campaign.Kind,
|
||||
&campaign.UTMTracking, &campaign.UTMSource, &campaign.UTMMedium, &campaign.UTMCampaign,
|
||||
&campaign.UnsubscribeMode,
|
||||
}
|
||||
dest = append(dest, extra...)
|
||||
@@ -191,6 +193,7 @@ const CAMPAIGN_SELECT_FULL = `
|
||||
c.guardrail_reply_rate_min, c.guardrail_min_sample, c.guardrail_window_days,
|
||||
c.guardrail_tripped_at, c.guardrail_reason,
|
||||
c.kind,
|
||||
c.utm_tracking, c.utm_source, c.utm_medium, c.utm_campaign,
|
||||
c.unsubscribe_mode,
|
||||
COALESCE(array_agg(cet.tag_id) FILTER (WHERE cet.tag_id IS NOT NULL), '{}') AS email_tag_ids,
|
||||
COALESCE(array_agg(cec.folder_id) FILTER (WHERE cec.folder_id IS NOT NULL), '{}') AS email_folder_ids
|
||||
@@ -341,6 +344,26 @@ func (r *campaignRepository) Create(ctx context.Context, userID string, orgID *u
|
||||
unsubMode = *data.UnsubscribeMode
|
||||
}
|
||||
|
||||
utmTracking := false
|
||||
if data.UTMTracking != nil {
|
||||
utmTracking = *data.UTMTracking
|
||||
}
|
||||
utmSource, utmMedium, utmCampaign := "", "", ""
|
||||
if data.UTMSource != nil {
|
||||
utmSource = strings.TrimSpace(*data.UTMSource)
|
||||
}
|
||||
if data.UTMMedium != nil {
|
||||
utmMedium = strings.TrimSpace(*data.UTMMedium)
|
||||
}
|
||||
if data.UTMCampaign != nil {
|
||||
utmCampaign = strings.TrimSpace(*data.UTMCampaign)
|
||||
}
|
||||
for _, v := range []string{utmSource, utmMedium, utmCampaign} {
|
||||
if err := validate.CampaignUTMValue(v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// ── Net-new send controls. Defaults reproduce today's behavior exactly. ──
|
||||
senderStrategy := "tags"
|
||||
if data.SenderStrategy != nil {
|
||||
@@ -435,7 +458,9 @@ func (r *campaignRepository) Create(ctx context.Context, userID string, orgID *u
|
||||
sender_strategy, rotation_mode,
|
||||
ramp_enabled, ramp_start, ramp_increment, ramp_ceiling,
|
||||
esp_match_mode, max_new_leads_per_day, prioritize_new_leads,
|
||||
tracking_domain, kind, unsubscribe_mode,
|
||||
tracking_domain, kind,
|
||||
utm_tracking, utm_source, utm_medium, utm_campaign,
|
||||
unsubscribe_mode,
|
||||
created_at, updated_at
|
||||
) VALUES (
|
||||
gen_random_uuid(), $1, $2, $3, $4,
|
||||
@@ -446,7 +471,9 @@ func (r *campaignRepository) Create(ctx context.Context, userID string, orgID *u
|
||||
$20, $21,
|
||||
$22, $23, $24, $25,
|
||||
$26, $27, $28,
|
||||
$29, $30, $31,
|
||||
$29, $30,
|
||||
$31, $32, $33, $34,
|
||||
$35,
|
||||
NOW(), NOW()
|
||||
)
|
||||
RETURNING %s
|
||||
@@ -483,7 +510,11 @@ func (r *campaignRepository) Create(ctx context.Context, userID string, orgID *u
|
||||
prioritizeNewLeads, // $28
|
||||
trackingDomain, // $29
|
||||
kind, // $30
|
||||
unsubMode, // $31
|
||||
utmTracking, // $31
|
||||
utmSource, // $32
|
||||
utmMedium, // $33
|
||||
utmCampaign, // $34
|
||||
unsubMode, // $35
|
||||
}
|
||||
|
||||
row := tx.QueryRow(ctx, insertSQL, params...)
|
||||
@@ -1136,6 +1167,27 @@ func (r *campaignRepository) Update(ctx context.Context, userID, campaignID stri
|
||||
// the CNAME is re-resolved (only a verified override is honored).
|
||||
setClauses = append(setClauses, "tracking_domain_verified = false", "tracking_domain_verified_at = NULL")
|
||||
}
|
||||
if data.UTMTracking != nil {
|
||||
setClauses = append(setClauses, fmt.Sprintf("%s = $%d", "utm_tracking", argPos))
|
||||
args = append(args, *data.UTMTracking)
|
||||
argPos++
|
||||
}
|
||||
for col, val := range map[string]*string{
|
||||
"utm_source": data.UTMSource,
|
||||
"utm_medium": data.UTMMedium,
|
||||
"utm_campaign": data.UTMCampaign,
|
||||
} {
|
||||
if val == nil {
|
||||
continue
|
||||
}
|
||||
v := strings.TrimSpace(*val)
|
||||
if err := validate.CampaignUTMValue(v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
setClauses = append(setClauses, fmt.Sprintf("%s = $%d", col, argPos))
|
||||
args = append(args, v)
|
||||
argPos++
|
||||
}
|
||||
|
||||
// Auto-pause guardrails. Each rate is a percentage in [0,100] where 0 means
|
||||
// "this rule is off"; the DB CHECK mirrors these bounds, but rejecting here
|
||||
@@ -1293,6 +1345,7 @@ func (r *campaignRepository) GetByID(ctx context.Context, campaignID uuid.UUID)
|
||||
&campaign.GuardrailReplyRateMin, &campaign.GuardrailMinSample, &campaign.GuardrailWindowDays,
|
||||
&campaign.GuardrailTrippedAt, &campaign.GuardrailReason,
|
||||
&campaign.Kind,
|
||||
&campaign.UTMTracking, &campaign.UTMSource, &campaign.UTMMedium, &campaign.UTMCampaign,
|
||||
&campaign.UnsubscribeMode,
|
||||
&campaign.EmailTags, &campaign.Folders,
|
||||
)
|
||||
|
||||
@@ -104,6 +104,7 @@ func (r *campaignRepository) Duplicate(ctx context.Context, in DuplicateCampaign
|
||||
guardrail_enabled, guardrail_bounce_rate_max, guardrail_complaint_rate_max,
|
||||
guardrail_reply_rate_min, guardrail_min_sample, guardrail_window_days,
|
||||
guardrail_tripped_at, guardrail_reason,
|
||||
utm_tracking, utm_source, utm_medium, utm_campaign,
|
||||
last_status_change_at, updated_at, created_at, kind
|
||||
)
|
||||
SELECT
|
||||
@@ -122,6 +123,7 @@ func (r *campaignRepository) Duplicate(ctx context.Context, in DuplicateCampaign
|
||||
guardrail_enabled, guardrail_bounce_rate_max, guardrail_complaint_rate_max,
|
||||
guardrail_reply_rate_min, guardrail_min_sample, guardrail_window_days,
|
||||
NULL, '',
|
||||
utm_tracking, utm_source, utm_medium, utm_campaign,
|
||||
NULL, NOW(), NOW(), kind
|
||||
FROM campaigns
|
||||
WHERE id = $1
|
||||
|
||||
@@ -125,6 +125,14 @@ type CampaignProgressRepository interface {
|
||||
HasSentSteps(ctx context.Context, campaignID, contactID uuid.UUID) (bool, error)
|
||||
RecordEmailOpened(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID, machine bool) error
|
||||
RecordEmailClicked(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) error
|
||||
// UnrecordEmailClicked clears clicked_at when every logged click on the
|
||||
// step turned out to be automated (a burst recognised after the first
|
||||
// click already stamped it). clicked_at keeps meaning "a person clicked".
|
||||
UnrecordEmailClicked(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) error
|
||||
// GetStepSentAt returns when the step was dispatched (nil when it was not),
|
||||
// the reference point for telling an instant machine open or click from a
|
||||
// person's.
|
||||
GetStepSentAt(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) (*time.Time, error)
|
||||
RecordEmailReplied(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) error
|
||||
RecordEmailBounced(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) error
|
||||
RecordEmailComplained(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) error
|
||||
@@ -441,6 +449,52 @@ func (r *campaignProgressRepository) RecordEmailClicked(ctx context.Context, cam
|
||||
return err
|
||||
}
|
||||
|
||||
// UnrecordEmailClicked walks a click stamp back once no human click remains
|
||||
// on the step. Guarded by the click log so a concurrent human click is never
|
||||
// erased, and only a stamp written alongside a logged click is touched: a
|
||||
// stamp older than the step's earliest logged click predates per-link
|
||||
// logging, so it came from a person the log never saw.
|
||||
func (r *campaignProgressRepository) UnrecordEmailClicked(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) error {
|
||||
query := `
|
||||
UPDATE campaign_contact_progress ccp
|
||||
SET clicked_at = NULL
|
||||
WHERE ccp.campaign_id = $1
|
||||
AND ccp.contact_id = $2
|
||||
AND ccp.sequence_id = $3
|
||||
AND ccp.clicked_at IS NOT NULL
|
||||
AND ccp.clicked_at >= (
|
||||
SELECT MIN(lc.clicked_at) - INTERVAL '1 minute' FROM email_link_clicks lc
|
||||
WHERE lc.campaign_id = $1 AND lc.contact_id = $2 AND lc.sequence_id = $3
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM email_link_clicks lc
|
||||
WHERE lc.campaign_id = $1 AND lc.contact_id = $2 AND lc.sequence_id = $3 AND lc.machine = false
|
||||
)
|
||||
`
|
||||
|
||||
_, err := r.db.Exec(ctx, query, campaignID, contactID, sequenceID)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetStepSentAt returns the step's dispatch time, or nil when unsent/unknown.
|
||||
func (r *campaignProgressRepository) GetStepSentAt(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) (*time.Time, error) {
|
||||
query := `
|
||||
SELECT LEAST(dispatched_at, sent_at)
|
||||
FROM campaign_contact_progress
|
||||
WHERE campaign_id = $1 AND contact_id = $2 AND sequence_id = $3
|
||||
`
|
||||
|
||||
var sentAt *time.Time
|
||||
err := r.db.QueryRow(ctx, query, campaignID, contactID, sequenceID).Scan(&sentAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return sentAt, nil
|
||||
}
|
||||
|
||||
// RecordEmailReplied records that a contact replied
|
||||
func (r *campaignProgressRepository) RecordEmailReplied(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) error {
|
||||
query := `
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -3011,9 +3012,19 @@ func (r *contactRepository) ListTimeline(ctx context.Context, userID uuid.UUID,
|
||||
|
||||
// 1. Engagement events from campaign_contact_progress. One progress
|
||||
// row can emit up to 5 events (sent/opened/clicked/replied/bounced).
|
||||
// The coarse clicked stamp is emitted only when no logged click stands
|
||||
// for it (a stamp written before per-link logging); otherwise source 9
|
||||
// names the link. A logged click represents the stamp when it landed
|
||||
// within a minute of it, the stamp being written as the click is logged.
|
||||
progressQuery := `
|
||||
SELECT
|
||||
ccp.sent_at, ccp.opened_at, ccp.clicked_at, ccp.replied_at, ccp.bounced_at,
|
||||
ccp.opened_machine,
|
||||
(ccp.clicked_at IS NOT NULL AND EXISTS (
|
||||
SELECT 1 FROM email_link_clicks lc
|
||||
WHERE lc.campaign_id = ccp.campaign_id AND lc.contact_id = ccp.contact_id AND lc.sequence_id = ccp.sequence_id
|
||||
AND lc.clicked_at BETWEEN ccp.clicked_at - INTERVAL '1 minute' AND ccp.clicked_at + INTERVAL '1 minute'
|
||||
)) AS has_link_clicks,
|
||||
cam.id, cam.name,
|
||||
seq.id, seq.name, seq.subject,
|
||||
ea.id, ea.email, ea.name
|
||||
@@ -3050,10 +3061,12 @@ func (r *contactRepository) ListTimeline(ctx context.Context, userID uuid.UUID,
|
||||
}
|
||||
for prows.Next() {
|
||||
var sentAt, openedAt, clickedAt, repliedAt, bouncedAt *time.Time
|
||||
var openedMachine, hasLinkClicks bool
|
||||
var campID, seqID, eaID *uuid.UUID
|
||||
var campName, seqName, seqSubject, eaEmail, eaName *string
|
||||
if err := prows.Scan(
|
||||
&sentAt, &openedAt, &clickedAt, &repliedAt, &bouncedAt,
|
||||
&openedMachine, &hasLinkClicks,
|
||||
&campID, &campName,
|
||||
&seqID, &seqName, &seqSubject,
|
||||
&eaID, &eaEmail, &eaName,
|
||||
@@ -3081,16 +3094,91 @@ func (r *contactRepository) ListTimeline(ctx context.Context, userID uuid.UUID,
|
||||
if baseSubject != nil && *baseSubject != "" {
|
||||
ev.Subject = baseSubject
|
||||
}
|
||||
if ty == models.TimelineEmailOpened {
|
||||
machine := openedMachine
|
||||
ev.Machine = &machine
|
||||
}
|
||||
events = append(events, ev)
|
||||
}
|
||||
makeEvent(sentAt, models.TimelineEmailSent)
|
||||
makeEvent(openedAt, models.TimelineEmailOpened)
|
||||
makeEvent(clickedAt, models.TimelineEmailClicked)
|
||||
if !hasLinkClicks {
|
||||
makeEvent(clickedAt, models.TimelineEmailClicked)
|
||||
}
|
||||
makeEvent(repliedAt, models.TimelineEmailReplied)
|
||||
makeEvent(bouncedAt, models.TimelineEmailBounced)
|
||||
}
|
||||
prows.Close()
|
||||
|
||||
// 9. Per-link clicks: which link, where it went, and whether a person or
|
||||
// a scanner clicked it. Same campaign scope as the progress feed.
|
||||
clickQuery := `
|
||||
SELECT lc.id, lc.clicked_at, lc.destination, lc.label, lc.user_agent, lc.machine, lc.machine_reason,
|
||||
cam.id, cam.name,
|
||||
seq.id, seq.name, seq.subject,
|
||||
ea.id, ea.email, ea.name
|
||||
FROM email_link_clicks lc
|
||||
JOIN campaigns cam ON cam.id = lc.campaign_id
|
||||
JOIN sequences seq ON seq.id = lc.sequence_id
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT ea.id, ea.email, ea.name
|
||||
FROM tasks t
|
||||
JOIN email_accounts ea ON ea.id = t.email_account_id
|
||||
WHERE t.id = lc.task_id
|
||||
) ea ON TRUE
|
||||
WHERE lc.contact_id = $1
|
||||
AND cam.user_id = $2
|
||||
AND lc.clicked_at < $3
|
||||
ORDER BY lc.clicked_at DESC
|
||||
LIMIT $4
|
||||
`
|
||||
crows, err := r.DB.Query(ctx, clickQuery, contactID, userID, bound, limit)
|
||||
if err != nil {
|
||||
db.CaptureError(err, clickQuery, []any{contactID, userID, bound, limit}, "ListTimeline link clicks")
|
||||
return nil, errx.InternalError()
|
||||
}
|
||||
for crows.Next() {
|
||||
var link models.ContactLinkClick
|
||||
var at time.Time
|
||||
var machine bool
|
||||
var reason string
|
||||
var campID, seqID, eaID *uuid.UUID
|
||||
var campName, seqName, seqSubject, eaEmail, eaName *string
|
||||
if err := crows.Scan(
|
||||
&link.ID, &at, &link.URL, &link.Label, &link.UserAgent, &machine, &reason,
|
||||
&campID, &campName,
|
||||
&seqID, &seqName, &seqSubject,
|
||||
&eaID, &eaEmail, &eaName,
|
||||
); err != nil {
|
||||
crows.Close()
|
||||
db.CaptureError(err, "", nil, "ListTimeline link clicks scan")
|
||||
return nil, errx.InternalError()
|
||||
}
|
||||
fillUTM(&link)
|
||||
ev := models.ContactTimelineEvent{
|
||||
Type: models.TimelineEmailClicked,
|
||||
At: at,
|
||||
EmailAccountID: eaID,
|
||||
EmailAccountEmail: eaEmail,
|
||||
EmailAccountName: eaName,
|
||||
CampaignID: campID,
|
||||
CampaignName: campName,
|
||||
SequenceID: seqID,
|
||||
SequenceName: seqName,
|
||||
Machine: &machine,
|
||||
Link: &link,
|
||||
}
|
||||
if seqSubject != nil && *seqSubject != "" {
|
||||
ev.Subject = seqSubject
|
||||
}
|
||||
if reason != "" {
|
||||
r := reason
|
||||
ev.MachineReason = &r
|
||||
}
|
||||
events = append(events, ev)
|
||||
}
|
||||
crows.Close()
|
||||
|
||||
if orgID != nil {
|
||||
// 2. Reply intents (inbound replies with classification).
|
||||
replyQuery := `
|
||||
@@ -3393,6 +3481,21 @@ func (r *contactRepository) ListTimeline(ctx context.Context, userID uuid.UUID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// fillUTM reads the UTM parameters off a clicked link's destination, whether
|
||||
// the send path stamped them or the author wrote them by hand.
|
||||
func fillUTM(link *models.ContactLinkClick) {
|
||||
u, err := url.Parse(link.URL)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
q := u.Query()
|
||||
link.UTMSource = q.Get("utm_source")
|
||||
link.UTMMedium = q.Get("utm_medium")
|
||||
link.UTMCampaign = q.Get("utm_campaign")
|
||||
link.UTMTerm = q.Get("utm_term")
|
||||
link.UTMContent = q.Get("utm_content")
|
||||
}
|
||||
|
||||
// verificationFromRequest normalises a verdict a caller supplied with a
|
||||
// contact. Empty means none; an unrecognised value is the caller's error.
|
||||
func verificationFromRequest(status, provider string) (*models.ContactVerificationWrite, *errx.Error) {
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// LinkClick is one recorded click on one tracked link. Machine clicks (a
|
||||
// security gateway walking every link at delivery time) are kept for the
|
||||
// record with the reason they were flagged, but never count as engagement.
|
||||
type LinkClick struct {
|
||||
ID uuid.UUID
|
||||
TrackedLinkID *uuid.UUID
|
||||
TaskID uuid.UUID
|
||||
CampaignID uuid.UUID
|
||||
ContactID uuid.UUID
|
||||
SequenceID uuid.UUID
|
||||
Destination string
|
||||
Label string
|
||||
UserAgent string
|
||||
IPHash string
|
||||
Machine bool
|
||||
MachineReason string
|
||||
ClickedAt time.Time
|
||||
}
|
||||
|
||||
// Machine-click reasons stored in email_link_clicks.machine_reason.
|
||||
const (
|
||||
LinkClickReasonPrefetch = "prefetch" // no user agent: never a person's browser
|
||||
LinkClickReasonInstant = "instant" // arrived inside the machine window after dispatch
|
||||
LinkClickReasonBurst = "burst" // a second link of the same email from the same source within seconds
|
||||
)
|
||||
|
||||
// LinkClickRepository is the per-link click log behind the contact timeline
|
||||
// and the scanner heuristics. Only the tracking consumer writes it.
|
||||
type LinkClickRepository interface {
|
||||
Insert(ctx context.Context, click *LinkClick) error
|
||||
// CountRecentOtherLinks counts clicks from the same source on OTHER links
|
||||
// of the same email since the given time: the burst signal. A link is
|
||||
// identified by its ticket when known, else by destination (events from
|
||||
// an older tracking build).
|
||||
CountRecentOtherLinks(ctx context.Context, taskID uuid.UUID, ipHash string, linkID *uuid.UUID, destination string, since time.Time) (int, error)
|
||||
// MarkBurst flags the source's earlier human-labelled clicks on the email
|
||||
// since the given time as machine, once a burst is recognised.
|
||||
MarkBurst(ctx context.Context, taskID uuid.UUID, ipHash string, since time.Time) (int64, error)
|
||||
// HasHumanClick reports whether any click on the step is still labelled
|
||||
// as a person's.
|
||||
HasHumanClick(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) (bool, error)
|
||||
// IsMachine reads a logged click's current classification, which a burst
|
||||
// recognised after the fact may have changed.
|
||||
IsMachine(ctx context.Context, id uuid.UUID) (bool, error)
|
||||
// HasHumanClickOn reports whether a person already clicked this exact
|
||||
// link of the email, so a repeat is not logged twice. The ticket is the
|
||||
// identity when known (two links may share a destination); the
|
||||
// destination is the fallback for events from an older tracking build.
|
||||
HasHumanClickOn(ctx context.Context, taskID uuid.UUID, linkID *uuid.UUID, destination string) (bool, error)
|
||||
}
|
||||
|
||||
type linkClickRepository struct {
|
||||
db *pgxpool.Pool
|
||||
}
|
||||
|
||||
// NewLinkClickRepository creates a new link click repository.
|
||||
func NewLinkClickRepository(db *pgxpool.Pool) LinkClickRepository {
|
||||
return &linkClickRepository{db: db}
|
||||
}
|
||||
|
||||
func (r *linkClickRepository) Insert(ctx context.Context, c *LinkClick) error {
|
||||
if c.ID == uuid.Nil {
|
||||
c.ID = uuid.New()
|
||||
}
|
||||
if c.ClickedAt.IsZero() {
|
||||
c.ClickedAt = time.Now()
|
||||
}
|
||||
query := `
|
||||
INSERT INTO email_link_clicks
|
||||
(id, tracked_link_id, task_id, campaign_id, contact_id, sequence_id,
|
||||
destination, label, user_agent, ip_hash, machine, machine_reason, clicked_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
|
||||
`
|
||||
_, err := r.db.Exec(ctx, query,
|
||||
c.ID, c.TrackedLinkID, c.TaskID, c.CampaignID, c.ContactID, c.SequenceID,
|
||||
c.Destination, c.Label, c.UserAgent, c.IPHash, c.Machine, c.MachineReason, c.ClickedAt,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *linkClickRepository) CountRecentOtherLinks(ctx context.Context, taskID uuid.UUID, ipHash string, linkID *uuid.UUID, destination string, since time.Time) (int, error) {
|
||||
// A row is "another link" when both sides have a ticket and they differ;
|
||||
// a row without a ticket (older tracking build) is compared by
|
||||
// destination so a repeat click on one link never reads as a burst.
|
||||
query := `
|
||||
SELECT COUNT(*)
|
||||
FROM email_link_clicks
|
||||
WHERE task_id = $1
|
||||
AND ip_hash = $2
|
||||
AND clicked_at >= $3
|
||||
AND CASE
|
||||
WHEN tracked_link_id IS NOT NULL AND $4::uuid IS NOT NULL THEN tracked_link_id <> $4::uuid
|
||||
ELSE destination <> $5
|
||||
END
|
||||
`
|
||||
var n int
|
||||
err := r.db.QueryRow(ctx, query, taskID, ipHash, since, linkID, destination).Scan(&n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (r *linkClickRepository) MarkBurst(ctx context.Context, taskID uuid.UUID, ipHash string, since time.Time) (int64, error) {
|
||||
query := `
|
||||
UPDATE email_link_clicks
|
||||
SET machine = true, machine_reason = $4
|
||||
WHERE task_id = $1
|
||||
AND ip_hash = $2
|
||||
AND clicked_at >= $3
|
||||
AND machine = false
|
||||
`
|
||||
tag, err := r.db.Exec(ctx, query, taskID, ipHash, since, LinkClickReasonBurst)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
|
||||
func (r *linkClickRepository) HasHumanClick(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) (bool, error) {
|
||||
query := `
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM email_link_clicks
|
||||
WHERE campaign_id = $1 AND contact_id = $2 AND sequence_id = $3 AND machine = false
|
||||
)
|
||||
`
|
||||
var ok bool
|
||||
err := r.db.QueryRow(ctx, query, campaignID, contactID, sequenceID).Scan(&ok)
|
||||
return ok, err
|
||||
}
|
||||
|
||||
func (r *linkClickRepository) IsMachine(ctx context.Context, id uuid.UUID) (bool, error) {
|
||||
var machine bool
|
||||
err := r.db.QueryRow(ctx, `SELECT machine FROM email_link_clicks WHERE id = $1`, id).Scan(&machine)
|
||||
return machine, err
|
||||
}
|
||||
|
||||
func (r *linkClickRepository) HasHumanClickOn(ctx context.Context, taskID uuid.UUID, linkID *uuid.UUID, destination string) (bool, error) {
|
||||
query := `
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM email_link_clicks
|
||||
WHERE task_id = $1 AND destination = $2 AND machine = false
|
||||
)
|
||||
`
|
||||
args := []any{taskID, destination}
|
||||
if linkID != nil {
|
||||
query = `
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM email_link_clicks
|
||||
WHERE task_id = $1 AND tracked_link_id = $2 AND machine = false
|
||||
)
|
||||
`
|
||||
args = []any{taskID, *linkID}
|
||||
}
|
||||
var ok bool
|
||||
err := r.db.QueryRow(ctx, query, args...).Scan(&ok)
|
||||
return ok, err
|
||||
}
|
||||
@@ -16,7 +16,10 @@ type TrackedLink struct {
|
||||
TaskID uuid.UUID
|
||||
CampaignID uuid.UUID
|
||||
Destination string
|
||||
CreatedAt time.Time
|
||||
// Label is the anchor text the link was minted from ("Pricing"), so a
|
||||
// click can be named without re-parsing the email. Empty for image links.
|
||||
Label string
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
// TrackedLinkRepository is the server-side click-link store. Only the send
|
||||
@@ -47,12 +50,12 @@ func (r *trackedLinkRepository) CreateBatch(ctx context.Context, links []Tracked
|
||||
|
||||
rows := make([][]any, 0, len(links))
|
||||
for _, l := range links {
|
||||
rows = append(rows, []any{l.ID, l.TaskID, l.CampaignID, l.Destination})
|
||||
rows = append(rows, []any{l.ID, l.TaskID, l.CampaignID, l.Destination, l.Label})
|
||||
}
|
||||
|
||||
_, err := r.db.CopyFrom(ctx,
|
||||
pgx.Identifier{"tracked_links"},
|
||||
[]string{"id", "task_id", "campaign_id", "destination"},
|
||||
[]string{"id", "task_id", "campaign_id", "destination", "label"},
|
||||
pgx.CopyFromRows(rows),
|
||||
)
|
||||
return err
|
||||
@@ -61,13 +64,13 @@ func (r *trackedLinkRepository) CreateBatch(ctx context.Context, links []Tracked
|
||||
// GetByID resolves a ticket to its destination. nil, nil when unknown.
|
||||
func (r *trackedLinkRepository) GetByID(ctx context.Context, id uuid.UUID) (*TrackedLink, error) {
|
||||
query := `
|
||||
SELECT id, task_id, campaign_id, destination, created_at
|
||||
SELECT id, task_id, campaign_id, destination, label, created_at
|
||||
FROM tracked_links
|
||||
WHERE id = $1
|
||||
`
|
||||
|
||||
var l TrackedLink
|
||||
err := r.db.QueryRow(ctx, query, id).Scan(&l.ID, &l.TaskID, &l.CampaignID, &l.Destination, &l.CreatedAt)
|
||||
err := r.db.QueryRow(ctx, query, id).Scan(&l.ID, &l.TaskID, &l.CampaignID, &l.Destination, &l.Label, &l.CreatedAt)
|
||||
if err == pgx.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
@@ -11,6 +11,10 @@ type persona struct {
|
||||
Opens bool
|
||||
Clicks bool
|
||||
Replies bool
|
||||
// Scanned marks a contact behind a corporate security gateway that
|
||||
// opens the pixel and follows every link seconds after delivery with an
|
||||
// ordinary browser user agent, so the sandbox exercises machine detection.
|
||||
Scanned bool
|
||||
Flavor replyFlavor
|
||||
}
|
||||
|
||||
@@ -34,6 +38,7 @@ func personaFor(email string) persona {
|
||||
Opens: n%100 < 92,
|
||||
Clicks: n%7 < 4, // ~57% of openers
|
||||
Replies: n%11 < 5, // ~45% of openers
|
||||
Scanned: n%5 == 0, // ~20% sit behind a link-scanning gateway
|
||||
}
|
||||
switch n % 10 {
|
||||
case 0, 1, 2, 3:
|
||||
|
||||
@@ -212,9 +212,25 @@ func (s *simulator) actAsContact(ctx context.Context, c contactInfo, msg *mailpi
|
||||
body = msg.Text
|
||||
}
|
||||
|
||||
// A security gateway scans the message at delivery: pixel plus every
|
||||
// link, one after another, before anyone could have read it. The
|
||||
// consumer must label these as machine and never count them as clicks.
|
||||
if p.Scanned {
|
||||
s.sleep(ctx, 500*time.Millisecond, 2*time.Second)
|
||||
if task := firstMatch(pixelRe, body); task != "" {
|
||||
s.hitTracking(ctx, "/t/o/"+task+".png", c.Email)
|
||||
}
|
||||
for _, ticket := range allMatches(clickRe, body) {
|
||||
s.hitTracking(ctx, "/c/"+ticket, c.Email)
|
||||
}
|
||||
fmt.Printf("scanned %-34s %q\n", c.Email, msg.Subject)
|
||||
}
|
||||
|
||||
if p.Opens {
|
||||
if task := firstMatch(pixelRe, body); task != "" {
|
||||
s.sleep(ctx, 5*time.Second, 40*time.Second)
|
||||
// Never inside the machine window: a person needs the message
|
||||
// delivered, noticed and opened first.
|
||||
s.sleep(ctx, 15*time.Second, 40*time.Second)
|
||||
s.hitTracking(ctx, "/t/o/"+task+".png", c.Email)
|
||||
fmt.Printf("opened %-34s %q\n", c.Email, msg.Subject)
|
||||
}
|
||||
@@ -274,3 +290,17 @@ func firstMatch(re *regexp.Regexp, body string) string {
|
||||
}
|
||||
return m[1]
|
||||
}
|
||||
|
||||
// allMatches returns every distinct first capture group, in document order.
|
||||
func allMatches(re *regexp.Regexp, body string) []string {
|
||||
var out []string
|
||||
seen := map[string]bool{}
|
||||
for _, m := range re.FindAllStringSubmatch(body, -1) {
|
||||
if len(m) < 2 || seen[m[1]] {
|
||||
continue
|
||||
}
|
||||
seen[m[1]] = true
|
||||
out = append(out, m[1])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -565,18 +565,30 @@ func (s *tasksService) HandleCampaignTask(task *proto.ProcessTask) *errx.Error {
|
||||
bodyHTML = AddOpenTrackingPixel(bodyHTML, taskID, trackingDomain)
|
||||
}
|
||||
|
||||
if campaign.LinkTracking && bodyHTML != "" {
|
||||
wrapped, links := WrapLinksForTracking(bodyHTML, taskID, campaign.ID, trackingDomain)
|
||||
if bodyHTML != "" && (campaign.LinkTracking || campaign.UTMTracking) {
|
||||
linkOpts := LinkTracking{
|
||||
TaskID: taskID,
|
||||
CampaignID: campaign.ID,
|
||||
TrackingDomain: trackingDomain,
|
||||
Wrap: campaign.LinkTracking,
|
||||
UTM: CampaignUTM(campaign),
|
||||
}
|
||||
tracked, links := TrackLinks(bodyHTML, linkOpts)
|
||||
if len(links) == 0 {
|
||||
bodyHTML = wrapped
|
||||
bodyHTML = tracked
|
||||
} else if err := s.trackedLinkRepo.CreateBatch(ctx, links); err != nil {
|
||||
// Tracking is a nicety: ship the original working links rather
|
||||
// than tickets that would 404 at the tracking service.
|
||||
// than tickets that would 404 at the tracking service. UTM tags
|
||||
// need no ticket, so they still go on.
|
||||
log.Warn().Err(err).Str("campaign_id", campaign.ID.String()).Str("task_id", taskID.String()).Msg("Failed to store tracked links; sending untracked")
|
||||
bodyHTML, _ = TrackLinks(bodyHTML, LinkTracking{UTM: linkOpts.UTM})
|
||||
} else {
|
||||
bodyHTML = wrapped
|
||||
bodyHTML = tracked
|
||||
}
|
||||
}
|
||||
if campaign.UTMTracking && bodyPlain != "" {
|
||||
bodyPlain = TagPlainTextLinks(bodyPlain, CampaignUTM(campaign), trackingDomain)
|
||||
}
|
||||
|
||||
// STEP 12: Add signature
|
||||
if account.SignatureSync {
|
||||
|
||||
@@ -0,0 +1,300 @@
|
||||
package tasks
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"html"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/warmbly/warmbly/internal/config"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
"github.com/warmbly/warmbly/internal/repository"
|
||||
)
|
||||
|
||||
// UTMParams are the campaign-level UTM values stamped on every link.
|
||||
// utm_content is not here because it is per link: the link's own text.
|
||||
type UTMParams struct {
|
||||
Source string
|
||||
Medium string
|
||||
Campaign string
|
||||
}
|
||||
|
||||
// LinkTracking describes what the send path does to the links of one email:
|
||||
// rewrite them to click tickets, tag them with UTM parameters, or both.
|
||||
type LinkTracking struct {
|
||||
TaskID uuid.UUID
|
||||
CampaignID uuid.UUID
|
||||
// TrackingDomain hosts the click tickets. Empty means no ticket can be
|
||||
// resolved, so links are never wrapped even when Wrap is set.
|
||||
TrackingDomain string
|
||||
Wrap bool
|
||||
// UTM, when set, tags every link that does not already carry the
|
||||
// parameter. Hand-written UTM values always win.
|
||||
UTM *UTMParams
|
||||
}
|
||||
|
||||
// Defaults for automatic UTM tagging when the campaign leaves a value empty.
|
||||
const (
|
||||
utmDefaultSource = "warmbly"
|
||||
utmDefaultMedium = "email"
|
||||
utmMaxLabelRunes = 120
|
||||
utmMaxSlugRunes = 64
|
||||
)
|
||||
|
||||
// CampaignUTM resolves the campaign's UTM settings to the values the send
|
||||
// path stamps, filling defaults for anything left empty. nil when the
|
||||
// campaign does not tag links.
|
||||
func CampaignUTM(c *models.Campaign) *UTMParams {
|
||||
if c == nil || !c.UTMTracking {
|
||||
return nil
|
||||
}
|
||||
p := &UTMParams{
|
||||
Source: strings.TrimSpace(c.UTMSource),
|
||||
Medium: strings.TrimSpace(c.UTMMedium),
|
||||
Campaign: strings.TrimSpace(c.UTMCampaign),
|
||||
}
|
||||
if p.Source == "" {
|
||||
p.Source = utmDefaultSource
|
||||
}
|
||||
if p.Medium == "" {
|
||||
p.Medium = utmDefaultMedium
|
||||
}
|
||||
if p.Campaign == "" {
|
||||
p.Campaign = utmSlug(c.Name)
|
||||
}
|
||||
if p.Campaign == "" {
|
||||
p.Campaign = "campaign"
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// anchorTag matches one <a ...href=...>...</a>, capturing the href value
|
||||
// (double-quoted, single-quoted or bare) and the inner HTML the label is
|
||||
// read from. The attribute must follow whitespace so data-href and the like
|
||||
// never pass for it. Only anchors are touched: a <link href> in the head is
|
||||
// a stylesheet, and redirecting it through a click ticket breaks it.
|
||||
var anchorTag = regexp.MustCompile(`(?is)<a\b([^>]*?)\shref\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>` + "`" + `]+))([^>]*)>(.*?)</a>`)
|
||||
|
||||
var (
|
||||
htmlTag = regexp.MustCompile(`(?s)<[^>]*>`)
|
||||
imgAlt = regexp.MustCompile(`(?is)<img\b[^>]*\balt\s*=\s*(?:"([^"]*)"|'([^']*)')`)
|
||||
whitespace = regexp.MustCompile(`\s+`)
|
||||
)
|
||||
|
||||
// TrackLinks rewrites the anchors of an HTML body per opts and returns the
|
||||
// click tickets it minted (nil when nothing was wrapped). A wrapped link
|
||||
// carries only the opaque ticket (https://<domain>/c/<id>); the destination,
|
||||
// UTM parameters included, lives in the returned row. The caller MUST persist
|
||||
// the rows before using the rewritten body, and fall back on failure, so an
|
||||
// email can never ship dead tickets.
|
||||
//
|
||||
// Skipped, and left exactly as written: anchors, mailto: and tel: links,
|
||||
// data: and javascript: URLs, anything that is not http(s), and links that
|
||||
// already point at the tracking host.
|
||||
func TrackLinks(htmlBody string, opts LinkTracking) (string, []repository.TrackedLink) {
|
||||
trackingDomain := config.NormalizeTrackingHost(opts.TrackingDomain)
|
||||
wrap := opts.Wrap && trackingDomain != ""
|
||||
if !wrap && opts.UTM == nil {
|
||||
return htmlBody, nil
|
||||
}
|
||||
|
||||
var links []repository.TrackedLink
|
||||
position := 0
|
||||
|
||||
result := anchorTag.ReplaceAllStringFunc(htmlBody, func(match string) string {
|
||||
m := anchorTag.FindStringSubmatch(match)
|
||||
before, quoted, single, bare, after, inner := m[1], m[2], m[3], m[4], m[5], m[6]
|
||||
rawHref := quoted
|
||||
if rawHref == "" {
|
||||
rawHref = single
|
||||
}
|
||||
if rawHref == "" {
|
||||
rawHref = bare
|
||||
}
|
||||
dest := strings.TrimSpace(html.UnescapeString(rawHref))
|
||||
if !trackableURL(dest, trackingDomain) {
|
||||
return match
|
||||
}
|
||||
position++
|
||||
label := linkLabel(inner)
|
||||
|
||||
if opts.UTM != nil {
|
||||
dest = withUTM(dest, opts.UTM, utmContent(label, position))
|
||||
}
|
||||
|
||||
href := html.EscapeString(dest)
|
||||
if wrap {
|
||||
id := uuid.New()
|
||||
links = append(links, repository.TrackedLink{
|
||||
ID: id,
|
||||
TaskID: opts.TaskID,
|
||||
CampaignID: opts.CampaignID,
|
||||
Destination: dest,
|
||||
Label: label,
|
||||
})
|
||||
href = config.TrackingURL(trackingDomain, "/c/"+id.String())
|
||||
}
|
||||
return `<a` + before + ` href="` + href + `"` + after + `>` + inner + `</a>`
|
||||
})
|
||||
|
||||
return result, links
|
||||
}
|
||||
|
||||
// WrapLinksForTracking rewrites every external link to a click ticket and
|
||||
// returns the minted rows. Kept as the wrap-only form of TrackLinks.
|
||||
func WrapLinksForTracking(htmlBody string, taskID, campaignID uuid.UUID, trackingDomain string) (string, []repository.TrackedLink) {
|
||||
return TrackLinks(htmlBody, LinkTracking{
|
||||
TaskID: taskID,
|
||||
CampaignID: campaignID,
|
||||
TrackingDomain: trackingDomain,
|
||||
Wrap: true,
|
||||
})
|
||||
}
|
||||
|
||||
// trackableURL reports whether a destination is one a click ticket can
|
||||
// redirect to and a UTM tag makes sense on. A link already pointing at the
|
||||
// tracking host (its host, not merely a URL mentioning it) is left alone.
|
||||
func trackableURL(dest, trackingDomain string) bool {
|
||||
lower := strings.ToLower(dest)
|
||||
if !strings.HasPrefix(lower, "http://") && !strings.HasPrefix(lower, "https://") {
|
||||
return false
|
||||
}
|
||||
// An unsubscribe link is never a click to count or a redirect to bounce
|
||||
// through, and a UTM tag on it would only clutter the opt-out page URL.
|
||||
if strings.Contains(lower, unsubscribePathMarker) {
|
||||
return false
|
||||
}
|
||||
if trackingDomain == "" {
|
||||
return true
|
||||
}
|
||||
u, err := url.Parse(dest)
|
||||
if err != nil {
|
||||
return true
|
||||
}
|
||||
return config.NormalizeTrackingHost(u.Host) != trackingDomain
|
||||
}
|
||||
|
||||
// bareURL matches an http(s) URL in plain text, stopping at whitespace and
|
||||
// the characters that close it in prose.
|
||||
var bareURL = regexp.MustCompile(`(?i)https?://[^\s<>"'` + "`" + `]+`)
|
||||
|
||||
// TagPlainTextLinks appends UTM parameters to every bare URL of a plain-text
|
||||
// body. There is no anchor text, so utm_content numbers the links in order.
|
||||
// Trailing punctuation that belongs to the sentence stays outside the URL.
|
||||
func TagPlainTextLinks(body string, utm *UTMParams, trackingDomain string) string {
|
||||
if utm == nil || body == "" {
|
||||
return body
|
||||
}
|
||||
trackingDomain = config.NormalizeTrackingHost(trackingDomain)
|
||||
position := 0
|
||||
return bareURL.ReplaceAllStringFunc(body, func(match string) string {
|
||||
dest := strings.TrimRight(match, ".,;:!?)]}")
|
||||
trail := match[len(dest):]
|
||||
if !trackableURL(dest, trackingDomain) {
|
||||
return match
|
||||
}
|
||||
position++
|
||||
return withUTM(dest, utm, utmContent("", position)) + trail
|
||||
})
|
||||
}
|
||||
|
||||
// linkLabel is the anchor's visible text: tags stripped, entities decoded,
|
||||
// whitespace collapsed. An image link falls back to the image's alt text.
|
||||
func linkLabel(inner string) string {
|
||||
text := html.UnescapeString(htmlTag.ReplaceAllString(inner, " "))
|
||||
text = strings.TrimSpace(whitespace.ReplaceAllString(text, " "))
|
||||
if text == "" {
|
||||
if m := imgAlt.FindStringSubmatch(inner); m != nil {
|
||||
alt := m[1]
|
||||
if alt == "" {
|
||||
alt = m[2]
|
||||
}
|
||||
text = strings.TrimSpace(whitespace.ReplaceAllString(html.UnescapeString(alt), " "))
|
||||
}
|
||||
}
|
||||
return truncateRunes(text, utmMaxLabelRunes)
|
||||
}
|
||||
|
||||
// utmContent names the link inside the email: its text as a slug, or its
|
||||
// ordinal when it has none (a bare image or icon).
|
||||
func utmContent(label string, position int) string {
|
||||
if s := utmSlug(label); s != "" {
|
||||
return s
|
||||
}
|
||||
return fmt.Sprintf("link_%d", position)
|
||||
}
|
||||
|
||||
// utmSlug lowercases a label and joins its words with underscores
|
||||
// ("See our Pricing!" -> "see_our_pricing"), the shape analytics tools
|
||||
// expect in a utm value.
|
||||
func utmSlug(s string) string {
|
||||
var b strings.Builder
|
||||
pendingSep := false
|
||||
for _, r := range strings.ToLower(strings.TrimSpace(s)) {
|
||||
if unicode.IsLetter(r) || unicode.IsDigit(r) {
|
||||
if pendingSep && b.Len() > 0 {
|
||||
b.WriteByte('_')
|
||||
}
|
||||
pendingSep = false
|
||||
b.WriteRune(r)
|
||||
continue
|
||||
}
|
||||
pendingSep = true
|
||||
}
|
||||
return truncateRunes(b.String(), utmMaxSlugRunes)
|
||||
}
|
||||
|
||||
// withUTM appends the UTM parameters the destination does not already carry,
|
||||
// keeping the existing query and fragment exactly as written. The URL is
|
||||
// never re-encoded: a customer's signed or oddly-encoded link stays intact.
|
||||
func withUTM(dest string, p *UTMParams, content string) string {
|
||||
u, err := url.Parse(dest)
|
||||
if err != nil {
|
||||
return dest
|
||||
}
|
||||
existing := u.Query()
|
||||
pairs := [][2]string{
|
||||
{"utm_source", p.Source},
|
||||
{"utm_medium", p.Medium},
|
||||
{"utm_campaign", p.Campaign},
|
||||
{"utm_content", content},
|
||||
}
|
||||
var add []string
|
||||
for _, kv := range pairs {
|
||||
if kv[1] == "" || existing.Has(kv[0]) {
|
||||
continue
|
||||
}
|
||||
add = append(add, kv[0]+"="+url.QueryEscape(kv[1]))
|
||||
}
|
||||
if len(add) == 0 {
|
||||
return dest
|
||||
}
|
||||
|
||||
base, fragment, hasFragment := strings.Cut(dest, "#")
|
||||
sep := "?"
|
||||
if strings.Contains(base, "?") {
|
||||
sep = "&"
|
||||
if strings.HasSuffix(base, "?") || strings.HasSuffix(base, "&") {
|
||||
sep = ""
|
||||
}
|
||||
}
|
||||
out := base + sep + strings.Join(add, "&")
|
||||
if hasFragment {
|
||||
out += "#" + fragment
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func truncateRunes(s string, n int) string {
|
||||
if n <= 0 {
|
||||
return ""
|
||||
}
|
||||
r := []rune(s)
|
||||
if len(r) <= n {
|
||||
return s
|
||||
}
|
||||
return strings.TrimRight(string(r[:n]), "_ ")
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package tasks
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
)
|
||||
|
||||
func TestUnsubscribeLinkIsNeverTracked(t *testing.T) {
|
||||
body := `<p>Hi <a href="https://acme.com/pricing">pricing</a> and <a href="https://api.example.com/unsubscribe/abc123">unsubscribe</a></p>`
|
||||
out, links := WrapLinksForTracking(body, uuid.New(), uuid.New(), "t.example.com")
|
||||
if len(links) != 1 || links[0].Destination != "https://acme.com/pricing" {
|
||||
t.Fatalf("expected only the pricing link to be ticketed, got %+v", links)
|
||||
}
|
||||
if !strings.Contains(out, `href="https://api.example.com/unsubscribe/abc123"`) {
|
||||
t.Fatalf("unsubscribe link was rewritten: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOptOutFooter(t *testing.T) {
|
||||
text := models.UnsubscribeSettings{Mode: models.UnsubscribeModeText, Text: "Reply and I'll stop."}
|
||||
h, p := appendOptOut("<p>Hi</p>", "Hi", text, "")
|
||||
if !strings.Contains(h, "Reply and I'll stop.") || !strings.HasSuffix(p, "Reply and I'll stop.") {
|
||||
t.Fatalf("text footer missing: %q / %q", h, p)
|
||||
}
|
||||
|
||||
link := models.UnsubscribeSettings{Mode: models.UnsubscribeModeLink, Text: "fallback", LinkIntro: "Not interested?", LinkText: "Unsubscribe"}
|
||||
h, p = appendOptOut("<p>Hi</p>", "Hi", link, "https://api.example.com/unsubscribe/tok")
|
||||
if !strings.Contains(h, `href="https://api.example.com/unsubscribe/tok"`) || !strings.Contains(p, "Unsubscribe: https://api.example.com/unsubscribe/tok") {
|
||||
t.Fatalf("link footer missing: %q / %q", h, p)
|
||||
}
|
||||
// No link to mint: link mode degrades to the text line, never to nothing.
|
||||
h, _ = appendOptOut("<p>Hi</p>", "Hi", link, "")
|
||||
if !strings.Contains(h, "fallback") {
|
||||
t.Fatalf("link mode without a link should fall back to text: %q", h)
|
||||
}
|
||||
h, p = appendOptOut("<p>Hi</p>", "Hi", models.UnsubscribeSettings{Mode: models.UnsubscribeModeOff}, "x")
|
||||
if h != "<p>Hi</p>" || p != "Hi" {
|
||||
t.Fatalf("off mode changed the body: %q / %q", h, p)
|
||||
}
|
||||
}
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
"github.com/warmbly/warmbly/internal/pkg/tmplfuncs"
|
||||
"github.com/warmbly/warmbly/internal/pkg/warmpersona"
|
||||
"github.com/warmbly/warmbly/internal/repository"
|
||||
)
|
||||
|
||||
// Conversation represents a warmup conversation for AI generation
|
||||
@@ -294,63 +293,6 @@ func AddOpenTrackingPixel(htmlBody string, taskID uuid.UUID, trackingDomain stri
|
||||
return htmlBody + pixel
|
||||
}
|
||||
|
||||
// WrapLinksForTracking rewrites every external link to an opaque
|
||||
// click-tracking ticket (https://<domain>/c/<id>) and returns the minted
|
||||
// rows. The destination never travels inside the link, so there is nothing
|
||||
// to forge: the tracking service resolves tickets via the backend internal
|
||||
// API and 404s anything it does not know. The caller MUST persist the
|
||||
// returned rows before using the rewritten body (and fall back to the
|
||||
// original body on failure) so an email can never ship dead tickets.
|
||||
func WrapLinksForTracking(htmlBody string, taskID, campaignID uuid.UUID, trackingDomain string) (string, []repository.TrackedLink) {
|
||||
// No tracking host means no ticket can be resolved, and a wrapped link
|
||||
// would be a dead link in a real customer's email. Ship the originals.
|
||||
trackingDomain = config.NormalizeTrackingHost(trackingDomain)
|
||||
if trackingDomain == "" {
|
||||
return htmlBody, nil
|
||||
}
|
||||
|
||||
// Regex to find href attributes
|
||||
linkRegex := regexp.MustCompile(`href="([^"]+)"`)
|
||||
var links []repository.TrackedLink
|
||||
|
||||
result := linkRegex.ReplaceAllStringFunc(htmlBody, func(match string) string {
|
||||
// Extract the original URL
|
||||
originalURL := linkRegex.FindStringSubmatch(match)[1]
|
||||
|
||||
// Skip if already a tracking link, an unsubscribe link, or an anchor
|
||||
if strings.HasPrefix(originalURL, "#") ||
|
||||
strings.Contains(originalURL, trackingDomain) ||
|
||||
strings.Contains(originalURL, unsubscribePathMarker) ||
|
||||
strings.HasPrefix(originalURL, "mailto:") ||
|
||||
strings.HasPrefix(originalURL, "tel:") {
|
||||
return match
|
||||
}
|
||||
|
||||
// Skip data URLs and javascript links
|
||||
if strings.HasPrefix(originalURL, "data:") ||
|
||||
strings.HasPrefix(originalURL, "javascript:") {
|
||||
return match
|
||||
}
|
||||
|
||||
// Only http(s) destinations are storable redirect targets
|
||||
if !strings.HasPrefix(originalURL, "http://") && !strings.HasPrefix(originalURL, "https://") {
|
||||
return match
|
||||
}
|
||||
|
||||
id := uuid.New()
|
||||
links = append(links, repository.TrackedLink{
|
||||
ID: id,
|
||||
TaskID: taskID,
|
||||
CampaignID: campaignID,
|
||||
Destination: originalURL,
|
||||
})
|
||||
|
||||
return fmt.Sprintf(`href="%s"`, config.TrackingURL(trackingDomain, "/c/"+id.String()))
|
||||
})
|
||||
|
||||
return result, links
|
||||
}
|
||||
|
||||
// personaPick chooses from a mailbox's preferred subset of phrasing options so
|
||||
// each mailbox keeps a consistent "voice" while still varying message to
|
||||
// message. Falls back gracefully for tiny option sets.
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
)
|
||||
|
||||
func TestAddOpenTrackingPixelUsesTheConfiguredHost(t *testing.T) {
|
||||
@@ -68,3 +69,126 @@ func TestWrapLinksForTrackingSkipsItsOwnHostAndNonHTTP(t *testing.T) {
|
||||
t.Fatalf("body should be unchanged: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrackLinksReadsTheAnchorTextAsTheLabel(t *testing.T) {
|
||||
html := `<p>See <a href="https://example.com/pricing"><b>our</b> Pricing & plans</a> or <a href="https://example.com/x"><img src="cid:logo" alt="Logo"></a></p>`
|
||||
_, links := TrackLinks(html, LinkTracking{TaskID: uuid.New(), CampaignID: uuid.New(), TrackingDomain: "t.acme.com", Wrap: true})
|
||||
if len(links) != 2 {
|
||||
t.Fatalf("expected two tickets, got %d", len(links))
|
||||
}
|
||||
if links[0].Label != "our Pricing & plans" {
|
||||
t.Fatalf("label not read from the anchor text: %q", links[0].Label)
|
||||
}
|
||||
if links[1].Label != "Logo" {
|
||||
t.Fatalf("image link should fall back to alt text: %q", links[1].Label)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrackLinksTagsUTMAndStoresTheTaggedDestination(t *testing.T) {
|
||||
html := `<a href="https://example.com/pricing?ref=1#top">Pricing page</a>`
|
||||
utm := &UTMParams{Source: "warmbly", Medium: "email", Campaign: "q3_outbound"}
|
||||
out, links := TrackLinks(html, LinkTracking{TaskID: uuid.New(), CampaignID: uuid.New(), TrackingDomain: "t.acme.com", Wrap: true, UTM: utm})
|
||||
if len(links) != 1 {
|
||||
t.Fatalf("expected one ticket, got %d", len(links))
|
||||
}
|
||||
want := "https://example.com/pricing?ref=1&utm_source=warmbly&utm_medium=email&utm_campaign=q3_outbound&utm_content=pricing_page#top"
|
||||
if links[0].Destination != want {
|
||||
t.Fatalf("destination = %q, want %q", links[0].Destination, want)
|
||||
}
|
||||
if strings.Contains(out, "utm_") {
|
||||
t.Fatalf("the email must carry only the ticket, got: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrackLinksTagsUTMWithoutWrappingWhenLinkTrackingIsOff(t *testing.T) {
|
||||
html := `<a href='https://example.com/a?x=1&y=2'>Go</a>`
|
||||
out, links := TrackLinks(html, LinkTracking{UTM: &UTMParams{Source: "s", Medium: "m", Campaign: "c"}})
|
||||
if links != nil {
|
||||
t.Fatalf("no tickets expected, got %v", links)
|
||||
}
|
||||
want := `<a href="https://example.com/a?x=1&y=2&utm_source=s&utm_medium=m&utm_campaign=c&utm_content=go">Go</a>`
|
||||
if out != want {
|
||||
t.Fatalf("got %s\nwant %s", out, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrackLinksKeepsHandWrittenUTMValues(t *testing.T) {
|
||||
html := `<a href="https://example.com/?utm_source=newsletter&utm_content=hero">Hero</a>`
|
||||
_, links := TrackLinks(html, LinkTracking{TaskID: uuid.New(), CampaignID: uuid.New(), TrackingDomain: "t.acme.com", Wrap: true, UTM: &UTMParams{Source: "warmbly", Medium: "email", Campaign: "c"}})
|
||||
got := links[0].Destination
|
||||
if strings.Count(got, "utm_source=") != 1 || !strings.Contains(got, "utm_source=newsletter") || !strings.Contains(got, "utm_content=hero") {
|
||||
t.Fatalf("hand-written values must win: %s", got)
|
||||
}
|
||||
if !strings.Contains(got, "utm_medium=email") || !strings.Contains(got, "utm_campaign=c") {
|
||||
t.Fatalf("missing values must still be added: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrackLinksNumbersLinksWithoutText(t *testing.T) {
|
||||
html := `<a href="https://example.com/1"><img src="x"></a><a href="https://example.com/2"></a>`
|
||||
_, links := TrackLinks(html, LinkTracking{TaskID: uuid.New(), CampaignID: uuid.New(), TrackingDomain: "t.acme.com", Wrap: true, UTM: &UTMParams{Source: "s", Medium: "m", Campaign: "c"}})
|
||||
if !strings.HasSuffix(links[0].Destination, "utm_content=link_1") || !strings.HasSuffix(links[1].Destination, "utm_content=link_2") {
|
||||
t.Fatalf("unlabelled links should be numbered: %s / %s", links[0].Destination, links[1].Destination)
|
||||
}
|
||||
}
|
||||
|
||||
// A stylesheet or preload href in the head is not a link anyone clicks;
|
||||
// redirecting it through a ticket would break it.
|
||||
func TestTrackLinksOnlyTouchesAnchors(t *testing.T) {
|
||||
html := `<link rel="stylesheet" href="https://example.com/a.css"><a href="https://example.com/p">p</a>`
|
||||
out, links := WrapLinksForTracking(html, uuid.New(), uuid.New(), "t.acme.com")
|
||||
if len(links) != 1 || links[0].Destination != "https://example.com/p" {
|
||||
t.Fatalf("expected only the anchor wrapped: %v", links)
|
||||
}
|
||||
if !strings.Contains(out, `href="https://example.com/a.css"`) {
|
||||
t.Fatalf("stylesheet href must be untouched: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCampaignUTMDefaults(t *testing.T) {
|
||||
c := &models.Campaign{Name: "Q3 Outbound: Fintech!", UTMTracking: true}
|
||||
p := CampaignUTM(c)
|
||||
if p == nil || p.Source != "warmbly" || p.Medium != "email" || p.Campaign != "q3_outbound_fintech" {
|
||||
t.Fatalf("unexpected defaults: %+v", p)
|
||||
}
|
||||
c.UTMSource, c.UTMCampaign = " acme ", "launch"
|
||||
p = CampaignUTM(c)
|
||||
if p.Source != "acme" || p.Campaign != "launch" {
|
||||
t.Fatalf("overrides not honoured: %+v", p)
|
||||
}
|
||||
if CampaignUTM(&models.Campaign{Name: "x"}) != nil {
|
||||
t.Fatal("utm tagging off must yield nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrackLinksReadsBareHrefAndIgnoresDataHref(t *testing.T) {
|
||||
html := `<a data-href="https://tracker.example/x" href=https://example.com/bare target="_blank">Bare</a>`
|
||||
out, links := WrapLinksForTracking(html, uuid.New(), uuid.New(), "t.acme.com")
|
||||
if len(links) != 1 || links[0].Destination != "https://example.com/bare" || links[0].Label != "Bare" {
|
||||
t.Fatalf("expected the bare href wrapped: %+v", links)
|
||||
}
|
||||
if !strings.Contains(out, `data-href="https://tracker.example/x"`) || !strings.Contains(out, `target="_blank"`) {
|
||||
t.Fatalf("other attributes must survive: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrackableURLComparesTheHost(t *testing.T) {
|
||||
if trackableURL("https://t.acme.com/c/abc", "t.acme.com") {
|
||||
t.Fatal("a link already on the tracking host must be left alone")
|
||||
}
|
||||
if !trackableURL("https://example.com/?next=t.acme.com", "t.acme.com") {
|
||||
t.Fatal("a URL merely mentioning the host is a normal link")
|
||||
}
|
||||
if !trackableURL("https://not-t.acme.com/", "t.acme.com") {
|
||||
t.Fatal("a different host that ends with the tracking host is a normal link")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTagPlainTextLinks(t *testing.T) {
|
||||
body := "See https://example.com/pricing. Docs: https://example.com/docs?x=1\nSkip https://t.acme.com/c/abc"
|
||||
out := TagPlainTextLinks(body, &UTMParams{Source: "s", Medium: "m", Campaign: "c"}, "t.acme.com")
|
||||
want := "See https://example.com/pricing?utm_source=s&utm_medium=m&utm_campaign=c&utm_content=link_1. Docs: https://example.com/docs?x=1&utm_source=s&utm_medium=m&utm_campaign=c&utm_content=link_2\nSkip https://t.acme.com/c/abc"
|
||||
if out != want {
|
||||
t.Fatalf("got %s\nwant %s", out, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package validate
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/warmbly/warmbly/internal/bitmask"
|
||||
"github.com/warmbly/warmbly/internal/config"
|
||||
@@ -157,3 +158,18 @@ func CampaignTrackingDomain(host string) *errx.Error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CampaignUTMValue validates one of the campaign's UTM overrides. Empty means
|
||||
// "use the default". Values are query-string parameters, so they must be
|
||||
// short, single-line and printable; encoding is the send path's job.
|
||||
func CampaignUTMValue(v string) *errx.Error {
|
||||
if utf8.RuneCountInString(v) > 128 {
|
||||
return errx.New(errx.BadRequest, "utm values must be 128 characters or fewer")
|
||||
}
|
||||
for _, r := range v {
|
||||
if r < 0x20 || r == 0x7f {
|
||||
return errx.New(errx.BadRequest, "utm values cannot contain control characters")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -8,6 +8,9 @@ pub struct TrackingEvent {
|
||||
pub event_type: String,
|
||||
pub task_id: String,
|
||||
pub original_url: Option<String>,
|
||||
/// Click ticket id, so the consumer can name the link (destination and
|
||||
/// anchor text) without matching URLs.
|
||||
pub link_id: Option<String>,
|
||||
pub timestamp: String,
|
||||
pub user_agent: Option<String>,
|
||||
pub ip_hash: Option<String>,
|
||||
|
||||
@@ -164,6 +164,7 @@ pub async fn track_open(
|
||||
event_type: "EMAIL_OPENED".to_string(),
|
||||
task_id,
|
||||
original_url: None,
|
||||
link_id: None,
|
||||
timestamp: Utc::now().to_rfc3339(),
|
||||
user_agent,
|
||||
ip_hash,
|
||||
@@ -245,12 +246,14 @@ pub async fn track_click(
|
||||
// Publish event asynchronously (fire and forget)
|
||||
let producer = state.producer.clone();
|
||||
let destination = link.destination.clone();
|
||||
let ticket = link_id.clone();
|
||||
tokio::spawn(async move {
|
||||
producer
|
||||
.publish(TrackingEvent {
|
||||
event_type: "EMAIL_CLICKED".to_string(),
|
||||
task_id: link.task_id,
|
||||
original_url: Some(destination),
|
||||
link_id: Some(ticket),
|
||||
timestamp: Utc::now().to_rfc3339(),
|
||||
user_agent,
|
||||
ip_hash,
|
||||
|
||||
@@ -25,6 +25,7 @@ pub const TRACKING_EVENT_SCHEMA: &str = r#"
|
||||
{"name": "event_type", "type": "string", "avro.java.string": "String"},
|
||||
{"name": "task_id", "type": "string", "avro.java.string": "String"},
|
||||
{"name": "original_url", "type": ["null", "string"], "default": null},
|
||||
{"name": "link_id", "type": ["null", "string"], "default": null},
|
||||
{"name": "timestamp", "type": "string", "avro.java.string": "String"},
|
||||
{"name": "user_agent", "type": ["null", "string"], "default": null},
|
||||
{"name": "ip_hash", "type": ["null", "string"], "default": null}
|
||||
@@ -58,6 +59,13 @@ impl ToAvroValue for TrackingEvent {
|
||||
None => Value::Union(0, Box::new(Value::Null)),
|
||||
},
|
||||
),
|
||||
(
|
||||
"link_id",
|
||||
match &self.link_id {
|
||||
Some(id) => Value::Union(1, Box::new(Value::String(id.clone()))),
|
||||
None => Value::Union(0, Box::new(Value::Null)),
|
||||
},
|
||||
),
|
||||
("timestamp", Value::String(self.timestamp.clone())),
|
||||
(
|
||||
"user_agent",
|
||||
|
||||
@@ -29,6 +29,9 @@ import type { DitherTone } from "@/components/ui/dither";
|
||||
import AnalyticsShareButton from "@/components/app/analytics/AnalyticsShareButton";
|
||||
import useDashboard from "@/lib/api/hooks/app/analytics/useDashboard";
|
||||
|
||||
const AUTO_OPENS_TIP = "Auto-opens: pixel fetches from privacy proxies (e.g. Apple Mail) or within seconds of sending, not a person reading";
|
||||
const AUTO_CLICKS_TIP = "Auto-clicks: links followed by a security gateway scanning the email, not a person; not counted as clicks";
|
||||
|
||||
type Range = "7d" | "30d" | "90d";
|
||||
type Metric = "sent" | "opens" | "clicks" | "replies";
|
||||
|
||||
@@ -82,8 +85,8 @@ export default function AnalyticsPage() {
|
||||
|
||||
const breakdown = [
|
||||
{ label: "Sent", value: os?.total_emails_sent, icon: SendIcon, dot: "bg-slate-400" },
|
||||
{ label: "Opens", value: os?.total_opens, icon: MailCheckIcon, dot: "bg-emerald-500", note: os?.machine_opens ? `${num(os.machine_opens)} auto` : undefined },
|
||||
{ label: "Clicks", value: os?.total_clicks, icon: MousePointerClickIcon, dot: "bg-violet-500" },
|
||||
{ label: "Opens", value: os?.total_opens, icon: MailCheckIcon, dot: "bg-emerald-500", note: os?.machine_opens ? `${num(os.machine_opens)} auto` : undefined, noteTitle: AUTO_OPENS_TIP },
|
||||
{ label: "Clicks", value: os?.total_clicks, icon: MousePointerClickIcon, dot: "bg-violet-500", note: os?.machine_clicks ? `${num(os.machine_clicks)} auto` : undefined, noteTitle: AUTO_CLICKS_TIP },
|
||||
{ label: "Replies", value: os?.total_replies, icon: ReplyIcon, dot: "bg-amber-500" },
|
||||
{ label: "Bounces", value: os?.total_bounces, icon: TriangleAlertIcon, dot: "bg-rose-500" },
|
||||
];
|
||||
@@ -169,7 +172,7 @@ export default function AnalyticsPage() {
|
||||
{"note" in q && q.note && (
|
||||
<span
|
||||
className="text-[9.5px] text-slate-400 font-mono"
|
||||
title="Auto-opens: pixel fetches from privacy proxies (e.g. Apple Mail), not a person reading"
|
||||
title={q.noteTitle}
|
||||
>
|
||||
{q.note}
|
||||
</span>
|
||||
|
||||
@@ -19,6 +19,9 @@ import CampaignFormsPanel from "@/components/app/campaigns/CampaignFormsPanel";
|
||||
import AnimatedNumber from "@/components/ui/AnimatedNumber";
|
||||
import AdvisorStrip from "@/components/app/advisor/AdvisorStrip";
|
||||
|
||||
const AUTO_OPENS_TIP = "Auto-opens: pixel fetches from privacy proxies (e.g. Apple Mail) or within seconds of sending, not a person reading";
|
||||
const AUTO_CLICKS_TIP = "Auto-clicks: links followed by a security gateway scanning the email, not a person; not counted as clicks";
|
||||
|
||||
const pctFmt = (v: number) => `${v.toFixed(1)}%`;
|
||||
|
||||
type Metric = "sent" | "opens" | "clicks" | "replies";
|
||||
@@ -99,8 +102,8 @@ export default function CampaignOverview() {
|
||||
|
||||
const breakdown = [
|
||||
{ label: "Sent", value: summary?.emails_sent, icon: SendIcon, dot: "bg-slate-400" },
|
||||
{ label: "Opens", value: summary?.unique_opens, icon: MailCheckIcon, dot: "bg-emerald-500", note: summary?.machine_opens ? `${summary.machine_opens} auto` : undefined },
|
||||
{ label: "Clicks", value: summary?.unique_clicks, icon: MousePointerClickIcon, dot: "bg-violet-500" },
|
||||
{ label: "Opens", value: summary?.unique_opens, icon: MailCheckIcon, dot: "bg-emerald-500", note: summary?.machine_opens ? `${summary.machine_opens} auto` : undefined, noteTitle: AUTO_OPENS_TIP },
|
||||
{ label: "Clicks", value: summary?.unique_clicks, icon: MousePointerClickIcon, dot: "bg-violet-500", note: summary?.machine_clicks ? `${summary.machine_clicks} auto` : undefined, noteTitle: AUTO_CLICKS_TIP },
|
||||
{ label: "Replies", value: summary?.replies, icon: ReplyIcon, dot: "bg-amber-500" },
|
||||
{ label: "Bounces", value: summary?.bounces, icon: TriangleAlertIcon, dot: "bg-rose-500" },
|
||||
];
|
||||
@@ -281,7 +284,7 @@ export default function CampaignOverview() {
|
||||
{q.note && (
|
||||
<span
|
||||
className="text-[9.5px] text-slate-400 font-mono"
|
||||
title="Auto-opens: pixel fetches from privacy proxies (e.g. Apple Mail), not a person reading"
|
||||
title={q.noteTitle}
|
||||
>
|
||||
{q.note}
|
||||
</span>
|
||||
@@ -309,7 +312,7 @@ export default function CampaignOverview() {
|
||||
{q.note && (
|
||||
<span
|
||||
className="text-[9.5px] text-slate-400 font-mono"
|
||||
title="Auto-opens: pixel fetches from privacy proxies (e.g. Apple Mail), not a person reading"
|
||||
title={q.noteTitle}
|
||||
>
|
||||
{q.note}
|
||||
</span>
|
||||
|
||||
@@ -202,6 +202,10 @@ export default function CampaignPreferences() {
|
||||
...(newData.text_only !== campaign.text_only && { text_only: newData.text_only }),
|
||||
...(newData.open_tracking !== campaign.open_tracking && { open_tracking: newData.open_tracking }),
|
||||
...(newData.link_tracking !== campaign.link_tracking && { link_tracking: newData.link_tracking }),
|
||||
...(newData.utm_tracking !== campaign.utm_tracking && { utm_tracking: newData.utm_tracking }),
|
||||
...(newData.utm_source !== campaign.utm_source && { utm_source: newData.utm_source }),
|
||||
...(newData.utm_medium !== campaign.utm_medium && { utm_medium: newData.utm_medium }),
|
||||
...(newData.utm_campaign !== campaign.utm_campaign && { utm_campaign: newData.utm_campaign }),
|
||||
...(newData.unsubscribe_header !== campaign.unsubscribe_header && {
|
||||
unsubscribe_header: newData.unsubscribe_header,
|
||||
}),
|
||||
|
||||
@@ -114,6 +114,7 @@ type Draft = {
|
||||
stopOnReply: boolean;
|
||||
openTracking: boolean;
|
||||
linkTracking: boolean;
|
||||
utmTracking: boolean;
|
||||
unsubHeader: boolean;
|
||||
sequences: SequenceDraft[];
|
||||
// One-time only.
|
||||
@@ -136,6 +137,7 @@ const initialDraft = (timezone: string): Draft => ({
|
||||
stopOnReply: true,
|
||||
openTracking: true,
|
||||
linkTracking: true,
|
||||
utmTracking: true,
|
||||
unsubHeader: true,
|
||||
sequences: [newSequence(0)],
|
||||
segmentIds: [],
|
||||
@@ -316,6 +318,11 @@ export function NewCampaignDialog({ open, onClose }: Props) {
|
||||
draft.description.trim() !== "" ||
|
||||
draft.emailTagIds.length > 0 ||
|
||||
draft.segmentIds.length > 0 ||
|
||||
!draft.stopOnReply ||
|
||||
!draft.openTracking ||
|
||||
!draft.linkTracking ||
|
||||
!draft.utmTracking ||
|
||||
!draft.unsubHeader ||
|
||||
draft.sequences.some((s) => s.subject.trim() !== "" || s.body_plain.trim() !== "");
|
||||
|
||||
const isPending = create.isPending || submitting;
|
||||
@@ -374,6 +381,7 @@ export function NewCampaignDialog({ open, onClose }: Props) {
|
||||
daily_limit: draft.dailyLimit,
|
||||
open_tracking: draft.openTracking,
|
||||
link_tracking: draft.linkTracking,
|
||||
utm_tracking: draft.utmTracking,
|
||||
unsubscribe_header: draft.unsubHeader,
|
||||
email_tag_ids: draft.emailTagIds,
|
||||
steps: buildSteps(),
|
||||
@@ -1002,10 +1010,16 @@ function SendingStep({ draft, patch }: { draft: Draft; patch: (p: Partial<Draft>
|
||||
/>
|
||||
<SwitchRow
|
||||
label="Track clicks"
|
||||
description="Wrap links so click activity appears in your live feed."
|
||||
description="Wrap links so each click, and which link it was, appears in your live feed and the contact's activity."
|
||||
value={draft.linkTracking}
|
||||
onChange={(v) => patch({ linkTracking: v })}
|
||||
/>
|
||||
<SwitchRow
|
||||
label="Add UTM parameters"
|
||||
description="Tag every link with utm_source, utm_medium, utm_campaign and a per-link utm_content for your web analytics. Editable later in settings."
|
||||
value={draft.utmTracking}
|
||||
onChange={(v) => patch({ utmTracking: v })}
|
||||
/>
|
||||
<SwitchRow
|
||||
label="Unsubscribe header"
|
||||
description="Add List-Unsubscribe, which most providers require for bulk mail."
|
||||
|
||||
@@ -153,7 +153,7 @@ export function DeliverabilitySection({
|
||||
/>
|
||||
<SettingRow
|
||||
title="Link tracking"
|
||||
description="Track clicks on links to measure engagement (click-through rate)."
|
||||
description="Track clicks on links to measure engagement (click-through rate). Each link is tracked on its own, so a contact's activity shows exactly which link was clicked."
|
||||
control={
|
||||
<Toggle
|
||||
id="campaign-pref-link-tracking"
|
||||
@@ -163,6 +163,21 @@ export function DeliverabilitySection({
|
||||
/>
|
||||
}
|
||||
/>
|
||||
<SettingRow
|
||||
title="UTM parameters"
|
||||
description="Tag every link with utm_source, utm_medium, utm_campaign and utm_content (the link's own text) so clicks show up attributed in your web analytics. Links that already carry a UTM value keep it."
|
||||
control={
|
||||
<Toggle
|
||||
id="campaign-pref-utm-tracking"
|
||||
value={newCampaign.utm_tracking}
|
||||
disabled={newCampaign.text_only}
|
||||
onChange={(v) => setNewCampaign((bef) => ({ ...bef, utm_tracking: v }))}
|
||||
/>
|
||||
}
|
||||
/>
|
||||
{newCampaign.utm_tracking && !newCampaign.text_only && (
|
||||
<UTMFields campaign={newCampaign} setNewCampaign={setNewCampaign} />
|
||||
)}
|
||||
<SettingRow
|
||||
title="Unsubscribe header"
|
||||
description="Add a List-Unsubscribe header so mail clients can show their own one-click unsubscribe."
|
||||
@@ -193,3 +208,59 @@ export function DeliverabilitySection({
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/** The three campaign-level UTM values; utm_content is per link and not editable. */
|
||||
export function UTMFields({
|
||||
campaign,
|
||||
setNewCampaign,
|
||||
}: {
|
||||
campaign: Campaign;
|
||||
setNewCampaign: SetCampaign;
|
||||
}) {
|
||||
return (
|
||||
<div className="grid grid-cols-1 sm:grid-cols-3 gap-3 pl-0 sm:pl-4 sm:border-l-2 sm:border-slate-100">
|
||||
<div>
|
||||
<Label>utm_source</Label>
|
||||
<TextInput
|
||||
value={campaign.utm_source}
|
||||
placeholder="warmbly"
|
||||
onChange={(v) => setNewCampaign((bef) => ({ ...bef, utm_source: v }))}
|
||||
className="w-full"
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<Label>utm_medium</Label>
|
||||
<TextInput
|
||||
value={campaign.utm_medium}
|
||||
placeholder="email"
|
||||
onChange={(v) => setNewCampaign((bef) => ({ ...bef, utm_medium: v }))}
|
||||
className="w-full"
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<Label>utm_campaign</Label>
|
||||
<TextInput
|
||||
value={campaign.utm_campaign}
|
||||
placeholder={utmSlug(campaign.name) || "campaign"}
|
||||
onChange={(v) => setNewCampaign((bef) => ({ ...bef, utm_campaign: v }))}
|
||||
className="w-full"
|
||||
/>
|
||||
</div>
|
||||
<p className="sm:col-span-3 text-[11px] text-slate-400 -mt-1">
|
||||
Leave a field empty to use the placeholder default. utm_content is set per link from its text
|
||||
(for example <span className="font-mono">pricing</span>), so each link is attributed on its own.
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Mirrors the backend's slug: lowercase words joined with underscores.
|
||||
function utmSlug(s: string): string {
|
||||
return s
|
||||
.toLowerCase()
|
||||
.trim()
|
||||
.split(/[^\p{L}\p{N}]+/u)
|
||||
.filter(Boolean)
|
||||
.join("_")
|
||||
.slice(0, 64);
|
||||
}
|
||||
|
||||
@@ -658,6 +658,10 @@ function applyFilters(
|
||||
e.page_hit?.referrer_domain,
|
||||
e.page_hit?.utm_source,
|
||||
e.page_hit?.utm_campaign,
|
||||
e.link?.url,
|
||||
e.link?.label,
|
||||
e.link?.utm_content,
|
||||
e.link?.utm_campaign,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(" ")
|
||||
@@ -942,6 +946,12 @@ function EventRow({
|
||||
<span className="text-[12px] font-medium text-slate-900 shrink-0">
|
||||
{label}
|
||||
</span>
|
||||
{event.link && (
|
||||
<span className="text-[11.5px] text-slate-700 truncate">
|
||||
<Highlight text={event.link.label || linkHost(event.link.url)} q={highlight} />
|
||||
</span>
|
||||
)}
|
||||
{event.machine && <MachineBadge reason={event.machine_reason} />}
|
||||
{event.subject && (
|
||||
<span className="text-[11.5px] text-slate-600 truncate">
|
||||
· <Highlight text={event.subject} q={highlight} />
|
||||
@@ -1077,12 +1087,73 @@ function detailsFor(e: ContactTimelineEvent): [string, React.ReactNode][] {
|
||||
add("UTM content", h.utm_content);
|
||||
add("Session", <span className="font-mono">{h.session_key.slice(0, 8)}</span>);
|
||||
}
|
||||
if (e.link) {
|
||||
const l = e.link;
|
||||
add("Link text", l.label);
|
||||
add(
|
||||
"Link URL",
|
||||
safeHttpUrl(l.url) ? (
|
||||
<a
|
||||
href={l.url}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="text-sky-600 hover:text-sky-700 break-all"
|
||||
>
|
||||
{l.url}
|
||||
</a>
|
||||
) : (
|
||||
l.url
|
||||
),
|
||||
);
|
||||
add("UTM source", l.utm_source);
|
||||
add("UTM medium", l.utm_medium);
|
||||
add("UTM campaign", l.utm_campaign);
|
||||
add("UTM term", l.utm_term);
|
||||
add("UTM content", l.utm_content);
|
||||
add("Browser", l.user_agent);
|
||||
}
|
||||
if (e.type === "email_opened" || e.type === "email_clicked") {
|
||||
add("Classified as", e.machine ? machineLabel(e.machine_reason) : "A person");
|
||||
}
|
||||
add("Reason", e.reason);
|
||||
add("Content", e.content);
|
||||
if (e.task_id) add("Task", <span className="font-mono">{e.task_id}</span>);
|
||||
return out;
|
||||
}
|
||||
|
||||
// What the classifier caught, in words a reader can act on.
|
||||
function machineLabel(reason?: string | null): string {
|
||||
switch (reason) {
|
||||
case "instant":
|
||||
return "Automated: fetched within seconds of sending, before anyone could have read it";
|
||||
case "burst":
|
||||
return "Automated: several links followed within seconds, the way a security scanner walks an email";
|
||||
case "prefetch":
|
||||
return "Automated: fetched by a mail proxy or a client with no browser";
|
||||
default:
|
||||
return "Automated: a mail privacy proxy or scanner, not a person";
|
||||
}
|
||||
}
|
||||
|
||||
function linkHost(url: string): string {
|
||||
try {
|
||||
return new URL(url).host;
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
function MachineBadge({ reason }: { reason?: string | null }) {
|
||||
return (
|
||||
<span
|
||||
className="inline-flex items-center rounded-sm bg-amber-50 text-amber-700 border border-amber-200 px-1 text-[9.5px] font-medium uppercase tracking-[0.1em] shrink-0"
|
||||
title={machineLabel(reason)}
|
||||
>
|
||||
auto
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
function cap(s: string): string {
|
||||
if (!s || s === "unknown") return "";
|
||||
return s.charAt(0).toUpperCase() + s.slice(1);
|
||||
@@ -1233,6 +1304,13 @@ function EventMeta({
|
||||
</span>,
|
||||
);
|
||||
}
|
||||
if (event.link?.utm_content) {
|
||||
parts.push(
|
||||
<span key="utm">
|
||||
utm <Highlight text={event.link.utm_content} q={highlight} />
|
||||
</span>,
|
||||
);
|
||||
}
|
||||
if (event.intent) {
|
||||
parts.push(<span key="intent">intent: {event.intent}</span>);
|
||||
}
|
||||
@@ -1339,7 +1417,7 @@ function visualFor(e: ContactTimelineEvent): {
|
||||
case "email_opened":
|
||||
return { Icon: MailOpenIcon, label: "Opened" };
|
||||
case "email_clicked":
|
||||
return { Icon: MousePointerClickIcon, label: "Clicked link" };
|
||||
return { Icon: MousePointerClickIcon, label: e.link ? "Clicked" : "Clicked link" };
|
||||
case "email_replied":
|
||||
return { Icon: ReplyIcon, label: "Replied" };
|
||||
case "reply_received":
|
||||
|
||||
@@ -53,6 +53,10 @@ export interface CampaignRaw {
|
||||
stop_on_reply: boolean;
|
||||
open_tracking: boolean;
|
||||
link_tracking: boolean;
|
||||
utm_tracking: boolean;
|
||||
utm_source: string;
|
||||
utm_medium: string;
|
||||
utm_campaign: string;
|
||||
text_only: boolean;
|
||||
daily_limit: number;
|
||||
unsubscribe_header: boolean;
|
||||
|
||||
@@ -129,14 +129,23 @@ export function useCampaignChannel(campaignId: string): CampaignChannelState {
|
||||
break;
|
||||
}
|
||||
case 'EMAIL_CLICKED': {
|
||||
const data = payload as { contact_email?: string; original_url?: string };
|
||||
const data = payload as {
|
||||
contact_email?: string;
|
||||
original_url?: string;
|
||||
link_label?: string;
|
||||
machine?: boolean;
|
||||
};
|
||||
const who = data.contact_email || 'Unknown';
|
||||
const target = data.link_label || data.original_url;
|
||||
addActivity({
|
||||
id: nextId('click'),
|
||||
type: 'clicked',
|
||||
contactEmail: data.contact_email || 'Unknown',
|
||||
message: data.original_url
|
||||
? `Click from ${data.contact_email || 'Unknown'} → ${data.original_url}`
|
||||
: `Click from ${data.contact_email || 'Unknown'}`,
|
||||
contactEmail: who,
|
||||
message: data.machine
|
||||
? `Automated click on ${target ?? 'a link'} for ${who} (not counted)`
|
||||
: target
|
||||
? `Click from ${who} → ${target}`
|
||||
: `Click from ${who}`,
|
||||
timestamp: new Date(),
|
||||
});
|
||||
break;
|
||||
|
||||
@@ -15,6 +15,10 @@ export interface CreateCampaignInput {
|
||||
stop_on_reply?: boolean;
|
||||
open_tracking?: boolean;
|
||||
link_tracking?: boolean;
|
||||
utm_tracking?: boolean;
|
||||
utm_source?: string;
|
||||
utm_medium?: string;
|
||||
utm_campaign?: string;
|
||||
text_only?: boolean;
|
||||
daily_limit?: number;
|
||||
unsubscribe_header?: boolean;
|
||||
|
||||
@@ -17,6 +17,9 @@ export interface CampaignSummary {
|
||||
// Subset of unique_opens from automated fetchers (Apple MPP prefetch
|
||||
// and UA-less clients). Human opens = unique_opens - machine_opens.
|
||||
machine_opens: number
|
||||
// Steps whose only clicks came from automated fetchers (security
|
||||
// gateways walking the links). Not part of unique_clicks.
|
||||
machine_clicks: number
|
||||
unique_clicks: number
|
||||
replies: number
|
||||
bounces: number
|
||||
|
||||
@@ -8,6 +8,8 @@ export interface DashboardOverallStats {
|
||||
// Subset of total_opens from automated fetchers (auto-opens).
|
||||
machine_opens: number
|
||||
total_clicks: number
|
||||
// Steps clicked only by automated fetchers; not part of total_clicks.
|
||||
machine_clicks: number
|
||||
total_replies: number
|
||||
total_bounces: number
|
||||
open_rate: number
|
||||
|
||||
@@ -86,6 +86,14 @@ export default interface Campaign {
|
||||
tracking_domain_verified: boolean;
|
||||
tracking_domain_verified_at?: string | null;
|
||||
|
||||
// Automatic UTM tagging of every link. Empty source/medium/campaign keep
|
||||
// the defaults (warmbly / email / the campaign name); utm_content is
|
||||
// always the link's own text.
|
||||
utm_tracking: boolean;
|
||||
utm_source: string;
|
||||
utm_medium: string;
|
||||
utm_campaign: string;
|
||||
|
||||
updated_at: Date;
|
||||
created_at: Date;
|
||||
|
||||
|
||||
@@ -53,10 +53,33 @@ export interface ContactPageHit {
|
||||
city: string;
|
||||
}
|
||||
|
||||
// The exact link behind an email_clicked event: where it went, the anchor
|
||||
// text it was minted from, and the UTM parameters the destination carried.
|
||||
export interface ContactLinkClick {
|
||||
id: string;
|
||||
url: string;
|
||||
label?: string;
|
||||
utm_source?: string;
|
||||
utm_medium?: string;
|
||||
utm_campaign?: string;
|
||||
utm_term?: string;
|
||||
utm_content?: string;
|
||||
user_agent?: string;
|
||||
}
|
||||
|
||||
export default interface ContactTimelineEvent {
|
||||
type: ContactTimelineEventType;
|
||||
at: string;
|
||||
|
||||
// Engagement classification (email_opened / email_clicked): true when an
|
||||
// automated fetcher (mail privacy proxy, security gateway) did it rather
|
||||
// than a person; machine_reason names the rule (prefetch / instant / burst).
|
||||
machine?: boolean;
|
||||
machine_reason?: string | null;
|
||||
|
||||
// Per-link detail behind an email_clicked event.
|
||||
link?: ContactLinkClick | null;
|
||||
|
||||
email_account_id?: string | null;
|
||||
email_account_email?: string | null;
|
||||
email_account_name?: string | null;
|
||||
|
||||
Reference in New Issue
Block a user