feat: re-read stored opens and clicks by the live origin rules in batched, resumable consumer passes under an advisory lock instead of a boot-time SQL backfill, never read a proxy string on a click as Apple Mail or Gmail, keep recognising Outlook, Proton Mail, Yahoo Mail and HEY by name, show the mail app behind a click in the expanded timeline row, and note that other iOS mail apps count as Apple Mail

2026-10-03 08:02:04 +00:00 · 2026-09-22 22:31:27 -07:00
parent eca5416a6f
commit 19eb68ecd5
10 changed files with 358 additions and 143 deletions
@@ -89,13 +89,13 @@ What the request can tell depends on the mail client:
 | Gmail, on the web or in the Gmail apps | **Gmail · device hidden**, no location | Gmail loads every image through Google's servers, so the device and network belong to Google |
 | Yahoo Mail, HEY, Fastmail's web app, Seznam | The provider, device hidden, no location | The same: the provider fetches the image |
 | Apple Mail with Mail Privacy Protection | **Apple Mail · device hidden**, region only | Apple's relay hides the device and keeps only the rough region |
-| Apple Mail on an iPhone or iPad without it | **iPhone · Apple Mail app** | The request comes from the device |
+| Apple Mail on an iPhone or iPad without it | **iPhone · Apple Mail app** | The request comes from the device. Another iOS mail app that renders mail the same way and names nothing is counted here too; Spark, Edison and Canary block tracking images by default |
 | Classic Outlook for Windows, Outlook for Mac and mobile, Thunderbird, eM Client and other desktop apps | The device and **app** | The client names itself |
 | Mail on a Mac, or the new Outlook for Windows | **Apple Mail or Outlook app**, no device | Both send the same stripped header, which names neither the client nor the device |
 | An Android mail app | **Android phone · Mail app** | Android apps render mail in a web view that does not name the app |
 | Webmail in a browser that loads images directly | **Webmail in** the browser, with the device | A full browser loaded the image |

-A click names the browser the link opened in and the device it opened on, because the link leaves the mail client. Nothing is guessed: when the request cannot say something, the field is left empty rather than filled with the proxy's own details. Opens recorded before this was added were read again by the same rules when the instance upgraded, so older history reads the same way.
+A click names the browser the link opened in and the device it opened on, because the link leaves the mail client. Nothing is guessed: when the request cannot say something, the field is left empty rather than filled with the proxy's own details. Opens and clicks recorded before this was added are read again by the same rules in the background after the instance upgrades, a small batch at a time, so older history reads the same way once it finishes.

 These records back the engagement rows on a contact's [activity timeline](/guides/contacts-crm/#activity-timeline) and the per-campaign breakdown above.

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
+	"encoding/json"
 	"net/netip"
 	"strings"
 	"time"
@@ -157,6 +158,9 @@ func (tc *TrackingConsumer) Start(ctx context.Context) error {
 	if tc.linkClicks != nil {
 		go tc.sweepPendingClicks(ctx)
 	}
+	if tc.opens != nil {
+		go tc.backfillOrigins(ctx)
+	}
 	return tc.bus.Subscribe(ctx, []string{tc.topic}, tc.group, tc.receive)
 }

@@ -650,44 +654,79 @@ func (tc *TrackingConsumer) logOpen(ctx context.Context, task *repository.Campai
 // network, as far as a proxy in between leaves it meaningful. The network is
 // used here and dropped.
 func (tc *TrackingConsumer) originOf(event *events.TrackingEvent) models.EngagementOrigin {
-	var o models.EngagementOrigin
-	locate := mailclient.LocateFull
-	if event.UserAgent != nil {
-		r := mailclient.Detect(*event.UserAgent, event.EventType == events.EventTypeEmailClicked)
-		o.Client, o.ClientType, o.DeviceHidden = r.Client, r.ClientType, r.DeviceHidden
-		o.DeviceType, o.OS, o.Browser, o.BrowserVersion = r.DeviceType, r.OS, r.Browser, r.BrowserVersion
-		locate = r.Locate
-	}
-	if locate == mailclient.LocateNone || event.ClientIP == nil || tc.geo == nil {
-		return o
-	}
-	if addr, err := netip.ParseAddr(strings.TrimSpace(*event.ClientIP)); err == nil && !addr.IsPrivate() && !addr.IsLoopback() {
-		if info, err := tc.geo.Lookup(addr); err == nil && info != nil {
-			o.CountryCode = info.CountryCode
-			o.Region = info.Region
-			if info.City != "Unknown" && locate == mailclient.LocateFull {
-				o.City = info.City
+	var place models.EngagementOrigin
+	if event.ClientIP != nil && tc.geo != nil {
+		if addr, err := netip.ParseAddr(strings.TrimSpace(*event.ClientIP)); err == nil && !addr.IsPrivate() && !addr.IsLoopback() {
+			if info, err := tc.geo.Lookup(addr); err == nil && info != nil {
+				place.CountryCode, place.Region = info.CountryCode, info.Region
+				if info.City != "Unknown" {
+					place.City = info.City
+				}
 			}
 		}
 	}
+	ua := ""
+	if event.UserAgent != nil {
+		ua = *event.UserAgent
+	}
+	return deriveOrigin(ua, event.EventType == events.EventTypeEmailClicked, place)
+}
+
+// deriveOrigin is one open or click's origin from its user agent and the
+// place its network resolved to, keeping only as much of the place as a
+// proxy in between leaves meaningful. The backfill re-reads stored rows with it.
+func deriveOrigin(userAgent string, click bool, place models.EngagementOrigin) models.EngagementOrigin {
+	r := mailclient.Detect(userAgent, click)
+	o := models.EngagementOrigin{
+		Client: r.Client, ClientType: r.ClientType, DeviceHidden: r.DeviceHidden,
+		DeviceType: r.DeviceType, OS: r.OS, Browser: r.Browser, BrowserVersion: r.BrowserVersion,
+	}
+	switch r.Locate {
+	case mailclient.LocateFull:
+		o.CountryCode, o.Region, o.City = place.CountryCode, place.Region, place.City
+	case mailclient.LocateRegion:
+		o.CountryCode, o.Region = place.CountryCode, place.Region
+	}
 	return o
 }

-// originData is the origin as a plain map, the shape integration templates
-// and automations walk, with unknown fields left out as on the API.
-func originData(o models.EngagementOrigin) map[string]any {
-	out := map[string]any{}
-	for k, v := range map[string]string{
-		"client": o.Client, "client_type": o.ClientType, "device_type": o.DeviceType,
-		"os": o.OS, "browser": o.Browser, "browser_version": o.BrowserVersion,
-		"country_code": o.CountryCode, "region": o.Region, "city": o.City,
-	} {
-		if v != "" {
-			out[k] = v
+// backfillOrigins re-reads the opens and clicks logged before the current
+// origin rules, a batch at a time so it never holds the logs for long, and
+// stops for good once both are walked. Any consumer may run it; a batch
+// another holds is waited out.
+func (tc *TrackingConsumer) backfillOrigins(ctx context.Context) {
+	derive := func(ua string, click bool, stored models.EngagementOrigin) models.EngagementOrigin {
+		return deriveOrigin(ua, click, stored)
+	}
+	pause := 200 * time.Millisecond
+	for {
+		done, busy, err := tc.opens.BackfillOriginBatch(ctx, 500, derive)
+		switch {
+		case done:
+			log.Info().Msg("engagement origin backfill complete")
+			return
+		case err != nil:
+			log.Warn().Err(err).Msg("engagement origin backfill batch failed; retrying")
+			pause = time.Minute
+		case busy:
+			pause = 30 * time.Second
+		default:
+			pause = 200 * time.Millisecond
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(pause):
 		}
 	}
-	if o.DeviceHidden {
-		out["device_hidden"] = true
+}
+
+// originData is the origin as a plain map, the shape integration templates
+// and automations walk, with the same fields and omissions as the API.
+func originData(o models.EngagementOrigin) map[string]any {
+	out := map[string]any{}
+	if raw, err := json.Marshal(o); err == nil {
+		_ = json.Unmarshal(raw, &out)
 	}
 	return out
 }
@@ -364,3 +364,22 @@ func TestAProbableLabelNeverTurnsAnAutomatedEventHuman(t *testing.T) {
 		}
 	}
 }
+
+// A proxy's network is the provider's, so its place is dropped; Apple's relay
+// keeps the region; a direct fetch keeps everything.
+func TestDeriveOriginKeepsOnlyTheMeaningfulPlace(t *testing.T) {
+	place := models.EngagementOrigin{CountryCode: "US", Region: "California", City: "Mountain View"}
+
+	if o := deriveOrigin("Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0 (via ggpht.com GoogleImageProxy)", false, place); o.CountryCode != "" || o.City != "" || !o.DeviceHidden {
+		t.Fatalf("gmail proxy kept a place: %+v", o)
+	}
+	if o := deriveOrigin("Mozilla/5.0", false, place); o.CountryCode != "US" || o.Region != "California" || o.City != "" {
+		t.Fatalf("apple relay should keep the region only: %+v", o)
+	}
+	if o := deriveOrigin(chromeUA, false, place); o.City != "Mountain View" || o.ClientType != models.EngagementClientWebmail {
+		t.Fatalf("a direct fetch keeps its place: %+v", o)
+	}
+	if o := deriveOrigin("Mozilla/5.0", true, place); o.Client != "" || o.DeviceHidden || o.City != "Mountain View" {
+		t.Fatalf("a click is never read as a proxy: %+v", o)
+	}
+}
@@ -1,3 +0,0 @@
-- The backfill corrects rows in place from their stored user agent; there is
-- nothing to undo, and 000199's down drops the columns it filled.
-SELECT 1;
@@ -1,88 +0,0 @@
-- Re-reads the opens and clicks logged before 000199 by the rules the
-- consumer now applies (internal/pkg/mailclient). An image proxy's user
-- agent describes the proxy, so those rows lose the device, browser and, for
-- a data-centre proxy, the location they were given; installed mail apps
-- and webmail gain their client_type. Each statement only touches rows it
-- changes, so a rerun is a no-op.
-
-- Data-centre proxies: nothing about the reader survives.
-UPDATE email_opens
-SET client = CASE
-        WHEN user_agent ILIKE '%googleimageproxy%' OR user_agent ILIKE '%via ggpht.com%' THEN 'Gmail'
-        WHEN user_agent ILIKE '%yahoomailproxy%' THEN 'Yahoo Mail'
-        WHEN user_agent ILIKE '%hey.com/imageproxy%' THEN 'HEY'
-        WHEN user_agent ILIKE '%fastmailua%' THEN 'Fastmail'
-        ELSE 'Seznam Email' END,
-    client_type = '', device_hidden = true,
-    device_type = '', os = '', browser = '', browser_version = '',
-    country_code = '', region = '', city = ''
-WHERE NOT device_hidden
-  AND (user_agent ILIKE '%googleimageproxy%' OR user_agent ILIKE '%via ggpht.com%'
-       OR user_agent ILIKE '%yahoomailproxy%' OR user_agent ILIKE '%hey.com/imageproxy%'
-       OR user_agent ILIKE '%fastmailua%' OR user_agent ILIKE '%seznamemailproxy%');
-
-- Apple Mail Privacy Protection: the relay keeps the region, not the city.
-UPDATE email_opens
-SET client = 'Apple Mail', client_type = '', device_hidden = true,
-    device_type = '', os = '', browser = '', browser_version = '', city = ''
-WHERE NOT device_hidden AND lower(btrim(user_agent)) = 'mozilla/5.0';
-
-- The stripped WebKit signature: Mac Mail or the new Outlook, on a device
-- the string cannot name. The parser used to take its platform for a browser.
-UPDATE email_opens
-SET client = 'Apple Mail or Outlook', client_type = 'app', device_hidden = false,
-    device_type = '', os = '', browser = '', browser_version = ''
-WHERE client IS DISTINCT FROM 'Apple Mail or Outlook'
-  AND lower(btrim(user_agent)) LIKE '%applewebkit/%'
-  AND lower(btrim(user_agent)) LIKE '%(khtml, like gecko)';
-
-UPDATE email_link_clicks
-SET client = '', client_type = '', device_type = '', os = '', browser = '', browser_version = ''
-WHERE (client <> '' OR browser <> '' OR os <> '')
-  AND lower(btrim(user_agent)) LIKE '%applewebkit/%'
-  AND lower(btrim(user_agent)) LIKE '%(khtml, like gecko)';
-
-- Clients that name themselves are installed apps.
-UPDATE email_opens
-SET client = CASE
-        WHEN user_agent ILIKE '%thunderbird/%' THEN 'Thunderbird'
-        WHEN user_agent ILIKE '%em client%' THEN 'eM Client'
-        WHEN user_agent ILIKE '%mailbird/%' THEN 'Mailbird'
-        WHEN user_agent ILIKE '%mailspring/%' THEN 'Mailspring'
-        WHEN user_agent ILIKE '%bluemail/%' THEN 'BlueMail'
-        WHEN user_agent ILIKE '%superhuman%' THEN 'Superhuman'
-        ELSE 'Outlook' END,
-    client_type = 'app', browser = '', browser_version = '',
-    os = CASE WHEN os = '' AND (user_agent ILIKE '%microsoft outlook%' OR user_agent ILIKE '%ms-office%' OR user_agent ILIKE '%msoffice%') THEN 'Windows' ELSE os END,
-    device_type = CASE WHEN device_type IN ('', 'unknown') AND NOT user_agent ILIKE '%outlook-ios%' AND NOT user_agent ILIKE '%outlook-android%' AND NOT user_agent ILIKE '%bluemail/%' AND NOT user_agent ILIKE '%superhuman%' THEN 'desktop' ELSE device_type END
-WHERE client_type = '' AND NOT device_hidden
-  AND (user_agent ILIKE '%microsoft outlook%' OR user_agent ILIKE '%ms-office%' OR user_agent ILIKE '%msoffice%'
-       OR user_agent ILIKE '%macoutlook%' OR user_agent ILIKE '%outlook-ios%' OR user_agent ILIKE '%outlook-android%'
-       OR user_agent ILIKE '%thunderbird/%' OR user_agent ILIKE '%em client%' OR user_agent ILIKE '%mailbird/%'
-       OR user_agent ILIKE '%mailspring/%' OR user_agent ILIKE '%bluemail/%' OR user_agent ILIKE '%superhuman%');
-
-- Mail on an iPhone or iPad: iOS WebKit with the build token and nothing a
-- browser or another app adds.
-UPDATE email_opens
-SET client = 'Apple Mail', client_type = 'app', browser = '', browser_version = ''
-WHERE client_type = '' AND NOT device_hidden
-  AND user_agent ~* '\((iphone|ipad);'
-  AND user_agent ILIKE '%applewebkit/%' AND user_agent ILIKE '%mobile/%'
-  AND user_agent !~* '(safari/|crios|fxios|edgios|opios|gsa/|fban|fbav|instagram|line/|micromessenger|twitter|linkedinapp|snapchat|pinterest|duckduckgo)';
-
-- An Android app's embedded WebView: a mail app, unnamed.
-UPDATE email_opens
-SET client_type = 'app', browser = '', browser_version = ''
-WHERE client_type = '' AND NOT device_hidden AND client = ''
-  AND user_agent ILIKE '%android%' AND user_agent ILIKE '%; wv)%';
-
-- A full browser is webmail read in a tab.
-UPDATE email_opens
-SET client_type = 'webmail'
-WHERE client_type = '' AND NOT device_hidden AND client = ''
-  AND user_agent ILIKE 'mozilla/5.0%'
-  AND browser IN ('Chrome', 'Firefox', 'Safari', 'Edge', 'Opera', 'Vivaldi', 'Samsung Browser', 'Mobile Safari');
-
-- The device type no longer carries an "unknown" placeholder.
-UPDATE email_opens SET device_type = '' WHERE device_type = 'unknown';
-UPDATE email_link_clicks SET device_type = '' WHERE device_type = 'unknown';
@@ -6,13 +6,13 @@ import (
 	"strings"

 	"github.com/mileusna/useragent"
+	"github.com/warmbly/warmbly/internal/models"
 )

-// ClientType values, matching models.EngagementClientApp and
-// models.EngagementClientWebmail.
+// ClientType values, as the open and click logs store them.
 const (
-	App     = "app"
-	Webmail = "webmail"
+	App     = models.EngagementClientApp
+	Webmail = models.EngagementClientWebmail
 )

 // Locate is how far the request's network places the reader.
@@ -46,16 +46,20 @@ func Detect(userAgent string, click bool) Reading {
 	}
 	ua := strings.ToLower(raw)

-	// Proxies first: their user agents carry a fixed, fake browser.
+	// Proxies first: their user agents carry a fixed, fake browser. A proxy
+	// only loads images, so on a link the same string claims nothing.
 	for _, p := range proxies {
 		if p.match(ua) {
+			if click {
+				return Reading{}
+			}
 			return Reading{Client: p.client, DeviceHidden: true, Locate: p.locate}
 		}
 	}

 	parsed := useragent.Parse(raw)
 	r := Reading{
-		OS:             osName(parsed),
+		OS:             parsed.OS,
 		Browser:        parsed.Name,
 		BrowserVersion: parsed.Version,
 		DeviceType:     deviceType(parsed),
@@ -164,12 +168,16 @@ var apps = []app{
 	{client: "Outlook", tokens: []string{"outlook-ios"}, os: "iOS", device: "mobile"},
 	{client: "Outlook", tokens: []string{"macoutlook"}, os: "macOS", device: "desktop"},
 	{client: "Outlook", tokens: []string{"microsoft outlook", "ms-office", "msoffice"}, os: "Windows", device: "desktop"},
+	{client: "Outlook", tokens: []string{"outlook"}},
 	{client: "Thunderbird", tokens: []string{"thunderbird/"}, device: "desktop"},
 	{client: "eM Client", tokens: []string{"em client"}, device: "desktop"},
 	{client: "Mailbird", tokens: []string{"mailbird/"}, os: "Windows", device: "desktop"},
 	{client: "Mailspring", tokens: []string{"mailspring/"}, device: "desktop"},
 	{client: "BlueMail", tokens: []string{"bluemail/"}},
 	{client: "Superhuman", tokens: []string{"superhuman"}},
+	{client: "Proton Mail", tokens: []string{"protonmail", "proton mail"}},
+	{client: "Yahoo Mail", tokens: []string{"yahoo mail"}},
+	{client: "HEY", tokens: []string{"hey.com"}},
 }

 func namedApp(ua string) (app, bool) {
@@ -221,17 +229,6 @@ func isBrowser(ua string, parsed useragent.UserAgent) bool {
 	return false
 }

-// osName is the operating system as the dashboard names it.
-func osName(ua useragent.UserAgent) string {
-	switch ua.OS {
-	case useragent.CrOS:
-		return useragent.ChromeOS
-	case useragent.WindowsNT, useragent.WindowsPhoneOS:
-		return useragent.Windows
-	}
-	return ua.OS
-}
-
 // deviceType folds the parser's flags into desktop, mobile or tablet.
 func deviceType(ua useragent.UserAgent) string {
 	switch {
@@ -90,6 +90,11 @@ func TestDetectOpens(t *testing.T) {
 			ua:   "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15",
 			want: Reading{ClientType: Webmail, DeviceType: "desktop", OS: "macOS", Browser: "Safari", BrowserVersion: "17.5"},
 		},
+		{
+			name: "any other outlook build still names outlook",
+			ua:   "Mozilla/5.0 (Windows NT 10.0; Win64; x64) OutlookDesktop/1.2024",
+			want: Reading{Client: "Outlook", ClientType: App, DeviceType: "desktop", OS: "Windows"},
+		},
 		{
 			name: "empty says nothing",
 			ua:   "  ",
@@ -128,6 +133,14 @@ func TestDetectClicks(t *testing.T) {
 		t.Fatalf("browser click = %+v", got)
 	}

+	// A proxy only loads images: its string on a link, including Apple's bare
+	// product token from any HTTP library, claims nothing.
+	for _, ua := range []string{"Mozilla/5.0", "Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0 (via ggpht.com GoogleImageProxy)"} {
+		if got := Detect(ua, true); got != (Reading{}) {
+			t.Fatalf("proxy string on a click %q = %+v", ua, got)
+		}
+	}
+
 	// The stripped signature on a link names no browser the parser could
 	// read honestly (it takes the platform for one), so nothing is claimed.
 	if got := Detect("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko)", true); got != (Reading{}) {
@@ -7,6 +7,7 @@ import (

 	"github.com/google/uuid"
 	"github.com/warmbly/warmbly/internal/models"
+	"github.com/warmbly/warmbly/internal/pkg/mailclient"
 )

 // How a contact reads (issue #655): each person's open carries its client,
@@ -14,7 +15,7 @@ import (
 // reads the open log hands those back.
 func TestLiveEngagementOriginReachesEverySurface(t *testing.T) {
 	handle, pool := liveContactDB(t)
-	requireSchemaVersion(t, pool, 200)
+	requireSchemaVersion(t, pool, 199)
 	f := newSharedOrgFixture(t, pool)
 	ctx := context.Background()
 	step := uuid.New()
@@ -127,3 +128,96 @@ func TestLiveEngagementOriginReachesEverySurface(t *testing.T) {
 		t.Fatalf("surfaces = %+v, want one mobile_app open, one hidden open and one mobile click", breakdown.Surfaces)
 	}
 }
+
+// Rows logged before the current rules are re-read in id-ordered batches,
+// clicks after opens, and the walk records that it finished.
+func TestLiveOriginBackfillRereadsStoredRows(t *testing.T) {
+	_, pool := liveContactDB(t)
+	requireSchemaVersion(t, pool, 199)
+	f := newSharedOrgFixture(t, pool)
+	ctx := context.Background()
+	step := uuid.New()
+	if _, err := pool.Exec(ctx, `DELETE FROM admin_settings WHERE key = $1`, originBackfillKey); err != nil {
+		t.Fatalf("reset progress: %v", err)
+	}
+	t.Cleanup(func() {
+		_, _ = pool.Exec(context.Background(), `DELETE FROM admin_settings WHERE key = $1`, originBackfillKey)
+	})
+	if _, err := pool.Exec(ctx, `INSERT INTO sequences (id, campaign_id, organization_id, name, subject, body_plain, body_html)
+	      VALUES ($1, $2, $3, 'Intro', 'Hi', '', '')`, step, f.campaign, f.org); err != nil {
+		t.Fatalf("seed step: %v", err)
+	}
+	gmail := uuid.New()
+	if _, err := pool.Exec(ctx, `INSERT INTO email_opens (id, task_id, campaign_id, contact_id, sequence_id, opened_at, user_agent,
+	                               client, device_type, os, browser, browser_version, country_code, region, city)
+	      VALUES ($1, $2, $3, $4, $5, NOW(), 'Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0 (via ggpht.com GoogleImageProxy)',
+	              'Gmail', 'desktop', 'Windows', 'Firefox', '11.0', 'US', 'California', 'Mountain View')`,
+		gmail, uuid.New(), f.campaign, f.contact, step); err != nil {
+		t.Fatalf("seed open: %v", err)
+	}
+	click := uuid.New()
+	if _, err := pool.Exec(ctx, `INSERT INTO email_link_clicks (id, task_id, campaign_id, contact_id, sequence_id, destination, user_agent,
+	                                     client, os, browser)
+	      VALUES ($1, $2, $3, $4, $5, 'https://x', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko)',
+	              'Apple Mail', 'macOS', 'Intel Mac OS X 10_15_7')`,
+		click, uuid.New(), f.campaign, f.contact, step); err != nil {
+		t.Fatalf("seed click: %v", err)
+	}
+
+	derive := func(ua string, click bool, stored models.EngagementOrigin) models.EngagementOrigin {
+		r := mailclient.Detect(ua, click)
+		o := models.EngagementOrigin{Client: r.Client, ClientType: r.ClientType, DeviceHidden: r.DeviceHidden,
+			DeviceType: r.DeviceType, OS: r.OS, Browser: r.Browser, BrowserVersion: r.BrowserVersion}
+		if r.Locate == mailclient.LocateFull {
+			o.CountryCode, o.Region, o.City = stored.CountryCode, stored.Region, stored.City
+		}
+		return o
+	}
+
+	// Another consumer holding the lock makes this one wait, not race.
+	holder, err := pool.Begin(ctx)
+	if err != nil {
+		t.Fatalf("begin: %v", err)
+	}
+	if _, err := holder.Exec(ctx, `SELECT pg_advisory_xact_lock($1)`, originBackfillLock); err != nil {
+		t.Fatalf("lock: %v", err)
+	}
+	repo := NewEmailOpenRepository(pool)
+	if done, busy, err := repo.BackfillOriginBatch(ctx, 500, derive); err != nil || done || !busy {
+		t.Fatalf("with the lock held: done=%v busy=%v err=%v, want busy", done, busy, err)
+	}
+	_ = holder.Rollback(ctx)
+
+	for i := 0; ; i++ {
+		done, busy, err := repo.BackfillOriginBatch(ctx, 500, derive)
+		if err != nil || busy {
+			t.Fatalf("batch %d: busy=%v err=%v", i, busy, err)
+		}
+		if done {
+			break
+		}
+		if i > 10000 {
+			t.Fatal("backfill never finished")
+		}
+	}
+
+	var client, device, os, city string
+	var hidden bool
+	if err := pool.QueryRow(ctx, `SELECT client, device_hidden, device_type, os, city FROM email_opens WHERE id = $1`, gmail).
+		Scan(&client, &hidden, &device, &os, &city); err != nil {
+		t.Fatalf("read open: %v", err)
+	}
+	if client != "Gmail" || !hidden || device != "" || os != "" || city != "" {
+		t.Fatalf("gmail open after backfill = %q hidden=%v %q %q %q", client, hidden, device, os, city)
+	}
+	var browser string
+	if err := pool.QueryRow(ctx, `SELECT client, os, browser FROM email_link_clicks WHERE id = $1`, click).Scan(&client, &os, &browser); err != nil {
+		t.Fatalf("read click: %v", err)
+	}
+	if client != "" || os != "" || browser != "" {
+		t.Fatalf("bare webkit click after backfill = %q %q %q, want nothing claimed", client, os, browser)
+	}
+	if done, _, err := repo.BackfillOriginBatch(ctx, 500, derive); err != nil || !done {
+		t.Fatalf("a finished backfill answers done: done=%v err=%v", done, err)
+	}
+}
@@ -2,9 +2,12 @@ package repository

 import (
 	"context"
+	"encoding/json"
+	"errors"
 	"time"

 	"github.com/google/uuid"
+	"github.com/jackc/pgx/v5"
 	"github.com/jackc/pgx/v5/pgxpool"
 	"github.com/warmbly/warmbly/internal/models"
 )
@@ -43,6 +46,9 @@ type EmailOpenRepository interface {
 	HasHumanOpen(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID) (bool, error)
 	// Cleanup deletes opens older than the retention window.
 	Cleanup(ctx context.Context, olderThanDays int) (int64, error)
+	// BackfillOriginBatch re-reads the next batch of logged opens and clicks
+	// by the current origin rules.
+	BackfillOriginBatch(ctx context.Context, limit int, derive OriginDeriver) (done, busy bool, err error)
 }

 type emailOpenRepository struct {
@@ -100,3 +106,138 @@ func (r *emailOpenRepository) Cleanup(ctx context.Context, olderThanDays int) (i
 	}
 	return tag.RowsAffected(), nil
 }
+
+// OriginDeriver re-reads one logged event's origin from its stored user agent
+// and the location it was stored with.
+type OriginDeriver func(userAgent string, click bool, stored models.EngagementOrigin) models.EngagementOrigin
+
+// originBackfillKey is the admin_settings row holding the backfill's progress.
+const originBackfillKey = "engagement.origin_backfill"
+
+// originBackfillLock serialises backfill batches across consumers.
+const originBackfillLock int64 = 0x6f7269676e // "orign"
+
+type originBackfillState struct {
+	OpensAfter  *uuid.UUID `json:"opens_after,omitempty"`
+	OpensDone   bool       `json:"opens_done"`
+	ClicksAfter *uuid.UUID `json:"clicks_after,omitempty"`
+	ClicksDone  bool       `json:"clicks_done"`
+}
+
+// BackfillOriginBatch re-derives the origin of the next `limit` opens, then
+// clicks, in id order and records how far it got, in one transaction under an
+// advisory lock. busy reports another consumer holding the lock; done that
+// both logs have been walked, which every later call answers from one read.
+func (r *emailOpenRepository) BackfillOriginBatch(ctx context.Context, limit int, derive OriginDeriver) (done, busy bool, err error) {
+	tx, err := r.db.Begin(ctx)
+	if err != nil {
+		return false, false, err
+	}
+	defer func() { _ = tx.Rollback(ctx) }()
+
+	var locked bool
+	if err := tx.QueryRow(ctx, `SELECT pg_try_advisory_xact_lock($1)`, originBackfillLock).Scan(&locked); err != nil {
+		return false, false, err
+	}
+	if !locked {
+		return false, true, nil
+	}
+	var st originBackfillState
+	var raw []byte
+	switch err := tx.QueryRow(ctx, `SELECT value FROM admin_settings WHERE key = $1`, originBackfillKey).Scan(&raw); {
+	case errors.Is(err, pgx.ErrNoRows):
+	case err != nil:
+		return false, false, err
+	default:
+		if err := json.Unmarshal(raw, &st); err != nil {
+			return false, false, err
+		}
+	}
+	if st.OpensDone && st.ClicksDone {
+		return true, false, nil
+	}
+
+	table, after, click := "email_opens", st.OpensAfter, false
+	if st.OpensDone {
+		table, after, click = "email_link_clicks", st.ClicksAfter, true
+	}
+	// A nil cursor starts below every id.
+	cursor := uuid.Nil
+	if after != nil {
+		cursor = *after
+	}
+	rows, err := tx.Query(ctx, `
+		SELECT id, user_agent, client, client_type, device_hidden, device_type, os, browser, browser_version,
+		       country_code, region, city
+		FROM `+table+`
+		WHERE id > $1
+		ORDER BY id
+		LIMIT $2`, cursor, limit)
+	if err != nil {
+		return false, false, err
+	}
+	var ids []uuid.UUID
+	var cols [9][]string
+	var hidden []bool
+	scanned := 0
+	for rows.Next() {
+		var id uuid.UUID
+		var ua string
+		var o models.EngagementOrigin
+		if err := rows.Scan(&id, &ua, &o.Client, &o.ClientType, &o.DeviceHidden, &o.DeviceType, &o.OS, &o.Browser, &o.BrowserVersion,
+			&o.CountryCode, &o.Region, &o.City); err != nil {
+			rows.Close()
+			return false, false, err
+		}
+		scanned++
+		cursor = id
+		n := derive(ua, click, o)
+		if n == o {
+			continue
+		}
+		ids = append(ids, id)
+		for i, v := range []string{n.Client, n.ClientType, n.DeviceType, n.OS, n.Browser, n.BrowserVersion, n.CountryCode, n.Region, n.City} {
+			cols[i] = append(cols[i], v)
+		}
+		hidden = append(hidden, n.DeviceHidden)
+	}
+	seen := rows.Err()
+	rows.Close()
+	if seen != nil {
+		return false, false, seen
+	}
+
+	if len(ids) > 0 {
+		if _, err := tx.Exec(ctx, `
+			UPDATE `+table+` t
+			SET client = v.client, client_type = v.client_type, device_hidden = v.device_hidden,
+			    device_type = v.device_type, os = v.os, browser = v.browser, browser_version = v.browser_version,
+			    country_code = v.country_code, region = v.region, city = v.city
+			FROM unnest($1::uuid[], $2::text[], $3::text[], $4::bool[], $5::text[], $6::text[], $7::text[], $8::text[],
+			            $9::text[], $10::text[], $11::text[])
+			     AS v(id, client, client_type, device_hidden, device_type, os, browser, browser_version, country_code, region, city)
+			WHERE t.id = v.id`,
+			ids, cols[0], cols[1], hidden, cols[2], cols[3], cols[4], cols[5], cols[6], cols[7], cols[8]); err != nil {
+			return false, false, err
+		}
+	}
+
+	// A short batch reached the end of the log. Rows written after that are
+	// already read by the current rules.
+	last := scanned < limit
+	if click {
+		st.ClicksAfter, st.ClicksDone = &cursor, last
+	} else {
+		st.OpensAfter, st.OpensDone = &cursor, last
+	}
+	out, err := json.Marshal(st)
+	if err != nil {
+		return false, false, err
+	}
+	if _, err := tx.Exec(ctx, `
+		INSERT INTO admin_settings (key, value, updated_at) VALUES ($1, $2, now())
+		ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()`, originBackfillKey, out); err != nil {
+		return false, false, err
+	}
+	return false, false, tx.Commit(ctx)
+}
@@ -1228,8 +1228,11 @@ function detailsFor(e: ContactTimelineEvent): [string, React.ReactNode][] {
        if (kind === "open") add("Read in", readerLabel(o, "open"));
        add("Device", o.device_hidden ? hiddenReason(o) : deviceLabel(o));
        add("Operating system", o.os);
-        if (kind === "click" || o.client_type !== "app") {
-            add(kind === "click" ? "Opened in" : "Browser", [o.browser, o.browser_version].filter(Boolean).join(" "));
+        if (kind === "click") {
+            // A mail app that made the request itself names no browser.
+            add("Opened in", o.client || [o.browser, o.browser_version].filter(Boolean).join(" "));
+        } else if (o.client_type !== "app") {
+            add("Browser", [o.browser, o.browser_version].filter(Boolean).join(" "));
        }
        add("Location", [o.city, o.region, o.country_code].filter(Boolean).join(", "));
    }