diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b52a34747..7f3ec4c74 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -250,8 +250,8 @@ jobs: WARMBLY_TEST_DB: postgres://warmbly:warmbly@127.0.0.1:5432/warmbly_test?sslmode=disable run: | set -o pipefail - go test -race -v -p 1 ./internal/repository ./internal/app/analytics -run '^TestLive(WarmupPartnerCountsMatchIndividualCandidates|WarmupSummariesMatchIndividualReads|LinkedMailboxStatusBatchPreservesDetails|WarmupPartnerWeekCacheMatchesExactReadAndKeepsTheCapLive|WarmupPartnerWeekCacheRefreshesDetachedAndServesStale|WarmupPartnerCandidatesCarryReciprocityAndHonourTheCap|AccountStatusPageMatchesSingleReadsInBoundedQueries|WarmupOffMailboxReceivesNothing|WarmupPauseRetiresIncomingWork|WarmupInboxHasRoomMatchesTheReservation)$' -count=1 | tee linked-mailbox-batch.log - for test in TestLiveWarmupPartnerCountsMatchIndividualCandidates TestLiveWarmupSummariesMatchIndividualReads TestLiveLinkedMailboxStatusBatchPreservesDetails TestLiveWarmupPartnerWeekCacheMatchesExactReadAndKeepsTheCapLive TestLiveWarmupPartnerWeekCacheRefreshesDetachedAndServesStale TestLiveWarmupPartnerCandidatesCarryReciprocityAndHonourTheCap TestLiveAccountStatusPageMatchesSingleReadsInBoundedQueries TestLiveWarmupOffMailboxReceivesNothing TestLiveWarmupPauseRetiresIncomingWork TestLiveWarmupInboxHasRoomMatchesTheReservation; do + go test -race -v -p 1 ./internal/repository ./internal/app/analytics -run '^TestLive(WarmupPartnerCountsMatchIndividualCandidates|WarmupSummariesMatchIndividualReads|LinkedMailboxStatusBatchPreservesDetails|WarmupPartnerWeekCacheMatchesExactReadAndKeepsTheCapLive|WarmupPartnerWeekCacheRefreshesDetachedAndServesStale|WarmupPartnerCandidatesCarryReciprocityAndHonourTheCap|AccountStatusPageMatchesSingleReadsInBoundedQueries|WarmupOffMailboxReceivesNothing|WarmupPauseRetiresIncomingWork|WarmupReservationHoldsEachTierToItsShare|WarmupReplyIsAdmittedToAFullInbox|WarmupRepliesLeaveTheNewThreadCapUntouched|OwedRepliesContinueTheDeepestConversationFirst)$' -count=1 | tee linked-mailbox-batch.log + for test in TestLiveWarmupPartnerCountsMatchIndividualCandidates TestLiveWarmupSummariesMatchIndividualReads TestLiveLinkedMailboxStatusBatchPreservesDetails TestLiveWarmupPartnerWeekCacheMatchesExactReadAndKeepsTheCapLive TestLiveWarmupPartnerWeekCacheRefreshesDetachedAndServesStale TestLiveWarmupPartnerCandidatesCarryReciprocityAndHonourTheCap TestLiveAccountStatusPageMatchesSingleReadsInBoundedQueries TestLiveWarmupOffMailboxReceivesNothing TestLiveWarmupPauseRetiresIncomingWork TestLiveWarmupReservationHoldsEachTierToItsShare TestLiveWarmupReplyIsAdmittedToAFullInbox TestLiveWarmupRepliesLeaveTheNewThreadCapUntouched TestLiveOwedRepliesContinueTheDeepestConversationFirst; do grep -q -- "--- PASS: $test " linked-mailbox-batch.log done if grep -q -- '--- SKIP:' linked-mailbox-batch.log; then exit 1; fi diff --git a/AGENTS.md b/AGENTS.md index 245662afc..c6a128988 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -557,7 +557,7 @@ Migration 000156 guarantees exactly one pool per type on every instance, under ` Crossing tiers is an exchange and lives in one place: `WarmupPartnerCandidates` in `pg_warmup.go` returns a sender's own tier plus, when the sender has fewer recipients outside its own workspace than `max(WarmupPoolTierFallbackFloor, warmup_max)`, that many proven free mailboxes from other workspaces (healthy, never blocked, members for `WarmupPoolFallbackMinAgeDays`, workspace not restricted or suspended), best first (Google or Microsoft, then members for `WarmupPoolBorrowSeasonedDays`, then actively sending; random within a rank), and for a free sender that meets the same bar, the premium mailboxes that verifiably wrote to it within `WarmupPoolReturnVisitDays`. Siblings never count toward the floor: a customer with many mailboxes and few premium peers otherwise never borrows and warms against itself. `models.WarmupPoolBorrowsFrom` and `models.WarmupPoolReturnsTo` are the two directions and mirror each other, so nothing unsolicited from the free tier reaches a paying inbox: the paying side always opens the exchange. Before this half existed a thin premium tier sent into the free tier and received almost nothing (#633). Every candidate carries the pool it was drawn from (`PoolType`) and how it got there (`Origin`); the selector and the scheduler both read that one method, so they cannot disagree about who is reachable. A routing rule of weight 0 is an exclusion, not a weight: the candidate is dropped before the draw (and refused on the reply-back path), so a pool of one cannot smuggle it back, and a tick with nothing left ends in `errAllPartnersExcluded` rather than mailing an excluded address (#501). The selector draws its own tier's fresh partners before a borrowed one, but a return visit ranks with the own tier, or a free mailbox with a hundred fresh siblings would never pay a paying inbox back. Every candidate is gated with `CanParticipate` pinned to the pool it was drawn from; gating a borrowed recipient against the sender's pool is what made borrowing dead for months (#495). The reply-back (`directedWarmupPartner`) crosses tiers on the same terms (a free mailbox answers a paid one only while healthy, the bar the return cohort holds it to) and is refused when the free workspace is restricted. It crosses on `CanParticipate`'s `not_in_pool`, so that reason must stay distinct from an ineligible member, or every cross-tier reply is closed unsent. -Reciprocity is a weight and a cap, both read off the candidate. Each carries its verified sends and arrivals over seven days; `Starvation` (how far behind an inbox is on what it sent) multiplies its draw weight by up to `1 + reciprocityBoostK`, fading to nothing at parity, so the pool's traffic flows to whoever is owed the most and settles there instead of overshooting. `InboundDailyCap` (`WarmupInboundDailyMultiple` times daily sends, between `WarmupInboundDailyFloor` and `WarmupInboundDailyCeiling`) is applied inside `WarmupPartnerCandidates`, and a free sender sees every inbox as full at `WarmupFreeInboundSharePercent` of it, so the rest of each inbox's day is kept for premium senders. A recipient at its cap is offered to nobody for the rest of the day and the scheduler's per-day volume cap (`min(target, len(candidates))`) sees the same set. A sender with nobody left to write to sends nothing that tick and rechecks later; do not add a fallback that mails somebody anyway. +Reciprocity is a weight and a cap, both read off the candidate. Each carries its verified sends and arrivals over seven days; `Starvation` (how far behind an inbox is on what it sent) multiplies its draw weight by up to `1 + reciprocityBoostK`, fading to nothing at parity, so the pool's traffic flows to whoever is owed the most and settles there instead of overshooting. `InboundDailyCap` (`WarmupInboundDailyMultiple` times daily sends, between `WarmupInboundDailyFloor` and `WarmupInboundDailyCeiling`) is applied inside `WarmupPartnerCandidates`, and a free sender sees every inbox as full at `WarmupFreeInboundSharePercent` of it, so the rest of each inbox's day is kept for premium senders. The cap counts new threads only: a reply answers mail its recipient sent, so it is neither refused by the cap nor counted against it (`newThreadSQL`), and a reply to a full inbox is sent, not closed. A recipient at its cap is offered to nobody for the rest of the day and the scheduler's per-day volume cap (`min(target, len(candidates))`) sees the same set. A sender with nobody left to write to sends nothing that tick and rechecks later; do not add a fallback that mails somebody anyway. Reachability is read off the same candidate: `Delivered7d` is the pool's handed-off warmup mail to it (past `config.WarmupRecipientArrivalGraceHours`) and `Arrived7d` what its own sync verified (`warmup_tokens.consumed_at`). The draw multiplies by `ArrivalRate`, and `models.WarmupRecipientUnreachable` (none arrived, at least `WarmupRecipientUnreachableMinDelivered` from at least `WarmupRecipientUnreachableMinSenderOrgs` workspaces) drops the candidate in `capPartnerCandidates`, so the selector, the scheduler and the counts agree, and the Advisor's `warmup_mail_not_arriving` reads the same rule. A send the sender's provider accepted and later bounced looks like success to the worker, so arrival, not the send result, is the evidence. Keep the two-workspace floor: without it one broken sender takes its recipients out of the pool. diff --git a/docs/content/docs/api/realtime.mdx b/docs/content/docs/api/realtime.mdx index 05d5da14b..61bb5f6d8 100644 --- a/docs/content/docs/api/realtime.mdx +++ b/docs/content/docs/api/realtime.mdx @@ -61,7 +61,7 @@ After connecting, join one or more topics with a `phx_join` message: | `account:` | One mailbox's sync and warmup events | Requires `manage_emails` | | `bulk:` | Progress of one bulk operation | Import/export progress; only the user who started the operation receives its events | -Events arrive as channel messages whose event name is the event type, for example `EMAIL_SENT`, `EMAIL_OPENED`, `EMAIL_REPLIED`, `EMAIL_RECEIVED`, `AI_DRAFT_READY`, `CAMPAIGN_COMPLETED`, `CAMPAIGN_IDLE` (a continuous campaign ran out of leads and is waiting), `TASK_PROGRESS`, `ACCOUNT_HEALTH_CHANGED`, `ACCOUNT_SYNC_STATE`, `WARMUP_PLACEMENT` (a partner found one of the workspace's warmup emails; `status` is `inbox`, `tabs` or `spam`, or `rescue_moved`, `rescue_already_out`, `rescue_gone` or `rescue_failed` when the recipient's mailbox reports what a spam rescue came to), `PLACEMENT_TEST_UPDATED`, `AUDIT_CREATED`, `AUTOMATION_RUN`, `MEETING_BOOKED`, `NOTIFICATION_CREATED`, `PAGE_HIT`. Payloads always include `event_type` and a timestamp, plus the relevant ids (`campaign_id`, `contact_id`, `thread_id`, and so on). `EMAIL_OPENED` and `EMAIL_CLICKED` also carry `occurred_at` (when the tracking service saw it; `timestamp` is the publish time), `machine` (true for a mail client prefetch or a security scanner's fetch), and `client`, `client_type` (`app` or `webmail`), `device_hidden` (a mailbox provider's image proxy fetched it, so the device is unknowable), `device_type`, `os`, `browser`, `country_code` and `city` when the consumer could tell. +Events arrive as channel messages whose event name is the event type, for example `EMAIL_SENT`, `EMAIL_OPENED`, `EMAIL_REPLIED`, `EMAIL_RECEIVED`, `AI_DRAFT_READY`, `CAMPAIGN_COMPLETED`, `CAMPAIGN_IDLE` (a continuous campaign ran out of leads and is waiting), `TASK_PROGRESS`, `ACCOUNT_HEALTH_CHANGED`, `ACCOUNT_SYNC_STATE`, `WARMUP_PLACEMENT` (a partner found one of the workspace's warmup emails; `status` is `inbox`, `tabs` or `spam`, or `rescue_moved`, `rescue_already_out`, `rescue_gone` or `rescue_failed` when the recipient's mailbox reports what a spam rescue came to), `WARMUP_ACTIVITY` (one of the workspace's mailboxes sent or received warmup mail; `email_account_id` names it), `PLACEMENT_TEST_UPDATED`, `AUDIT_CREATED`, `AUTOMATION_RUN`, `MEETING_BOOKED`, `NOTIFICATION_CREATED`, `PAGE_HIT`. Payloads always include `event_type` and a timestamp, plus the relevant ids (`campaign_id`, `contact_id`, `thread_id`, and so on). `EMAIL_OPENED` and `EMAIL_CLICKED` also carry `occurred_at` (when the tracking service saw it; `timestamp` is the publish time), `machine` (true for a mail client prefetch or a security scanner's fetch), and `client`, `client_type` (`app` or `webmail`), `device_hidden` (a mailbox provider's image proxy fetched it, so the device is unknowable), `device_type`, `os`, `browser`, `country_code` and `city` when the consumer could tell. `AI_DRAFT_READY` fires when the [inbox agent](/guides/inbox-agent/) drafts a suggested reply awaiting review; it carries `thread_id` and `draft_id` and requires `access_unibox`. diff --git a/docs/content/docs/guides/warmup.mdx b/docs/content/docs/guides/warmup.mdx index 4c0394601..da28495c5 100644 --- a/docs/content/docs/guides/warmup.mdx +++ b/docs/content/docs/guides/warmup.mdx @@ -133,7 +133,7 @@ Premium access is to the safe partner set, not just to receivers with paid subsc The borrow is returned: a free-tier mailbox that meets the same bar (healthy now, a member for at least three days, in a workspace in good standing) may write to a paying mailbox that wrote to it in the last 14 days, and to no other paying mailbox. A visit is returned alongside the mailbox's own tier rather than after it, so a paying inbox is written back to even when the free tier is large. Nothing unsolicited from the free tier reaches a paying inbox: the paying side always opens the exchange, and a free mailbox on watch keeps warming in its own tier but stops calling on paying ones. -Every draw also favours the inbox that is owed the most. A mailbox that has sent to many partners and heard back from few is drawn up to four times as often as one in balance, and the preference fades as the pool pays it back, so traffic settles near parity instead of overshooting. The other way is bounded too: an inbox receives at most about twice what it sends in a day (never fewer than 10, never more than 60), and once it reaches that it is left out of every draw for the rest of the day. Free-tier senders stop at three quarters of that, so the rest of every inbox's day is kept for premium mailboxes. A mailbox with nobody left to write to sends nothing and tries again later rather than mailing the same partner twice. +Every draw also favours the inbox that is owed the most. A mailbox that has sent to many partners and heard back from few is drawn up to four times as often as one in balance, and the preference fades as the pool pays it back, so traffic settles near parity instead of overshooting. The other way is bounded too: an inbox is sent at most about twice as many new threads a day as it sends (never fewer than 10, never more than 60), and once it reaches that it is left out of every draw for the rest of the day. Free-tier senders stop at three quarters of that, so the rest of every inbox's day is kept for premium mailboxes. Replies are outside this count, because each answers mail the inbox sent itself. A mailbox with nobody left to write to sends nothing and tries again later rather than mailing the same partner twice. Partner/domain diversity broadens diagnostic coverage. It is not identity rotation to evade quotas or evidence of reputation efficacy. @@ -263,16 +263,16 @@ Replies continue the exact pinned thread and subject, each parent is answered on Replies thread properly with a real `Re:` subject and `In-Reply-To` header, and candidates must be between 45 minutes and seven days old, so nothing is answered instantly or revived indefinitely. -The **Reply rate** setting targets the share of a mailbox's own warmup sends that are replies over its last seven UTC days. With enough eligible received mail, a mailbox set to `30%` approaches three replies for every ten warmup emails. The Analytics tab reports the measured rate over its selected range, which defaults to the last 30 days, so the figures can differ. Replies never add volume: each one takes the place of a send the mailbox was going to make anyway. `0%` turns replies off. +The **Reply rate** setting targets the share of a mailbox's own warmup sends that are replies over its last seven UTC days. With enough eligible received mail, a mailbox set to `30%` approaches three replies for every ten warmup emails. The mailbox's **Reply target** card and the Analytics tab's **Replies** card both show the measured share over those same seven days, next to the target, and update as the mailbox sends and receives warmup mail. The reply count on the Analytics tab covers the last 30 days. Replies never add volume: each one takes the place of a send the mailbox was going to make anyway. `0%` turns replies off. Replies reach that share two ways: - **On arrival.** When a verified warmup email arrives, Warmbly includes the next send in its target calculation. If that leaves room for a reply, it sometimes points the next scheduled send back at whoever wrote, 45 minutes to 5 hours later and inside its own warmup hours. The chance is the mailbox's reply rate (raised for a sender in [recovery](#recovery-engagement)), drawn once when the reply is scheduled rather than again when it sends. Each mailbox has one warmup send queued at a time, so a reply-back moves that send earlier and aims it, and never pushes a send the mailbox had already planned sooner. If that send is due within 45 minutes of the arrival, or is already answering someone, the arrival is not answered then. -- **When a send goes out.** Warmbly calculates the replies needed to reach the target, including the next send. A deficit of one reply or more makes the send a reply when eligible mail is available; a fractional deficit becomes a reply chance. For example, a `25%` target with ten sends and two replies gives the next send a `75%` reply chance. Because the calculation includes the next send, it can allow a reply when the current share is slightly above target. A selected reply answers mail at least 45 minutes and at most seven days old that the mailbox has not answered. Mail from senders in [recovery](#recovery-engagement) is considered first, then newest first within each group. +- **When a send goes out.** Warmbly compares the mailbox's reply share over the last seven UTC days with its target and turns the difference into a reply chance for that send: the target itself when on track, more when behind and less when ahead, never more than one and a half times the target or less than half of it. For example, a `30%` target with no replies yet gives a `45%` chance, and one already replying to `80%` of its sends gives `15%`. A shortfall is made up over several days rather than in one burst of replies, and a mailbox ahead of its target keeps answering its conversations. A selected reply answers mail at least 45 minutes and at most seven days old that the mailbox has not answered. Mail from senders in [recovery](#recovery-engagement) is considered first, then replies in conversations already under way, deepest first, so replies build back-and-forth threads instead of answering many threads once; newest first breaks ties. Together these help the measured rate converge on the setting, including for busy mailboxes whose next send is almost always less than 45 minutes away. Small send counts and a recent setting change can make it differ temporarily. The rate can fall short when eligible received mail is unavailable, threads have reached their message caps, or the partners that wrote to it can no longer receive. Reply selection proceeds in batches with recovery priority, continuing past messages with unavailable content or partners until it finds a usable reply or exhausts the candidates. -A reply is held to the same daily receiving cap as any other warmup email: a reply to an inbox that is full for the day is not sent, and the mailbox goes on with another reply or a new thread, so one busy partner never stops a mailbox's warmup. Replies stop before a thread reaches its message cap so replies cannot answer replies indefinitely. A reply crosses the free and paid tiers like the mail it answers: a paid mailbox answers a free partner that wrote to it, and a free mailbox answers a paid one while it is healthy and its workspace is in good standing. +A reply is not held to the recipient's daily receiving cap and does not count against it. The cap limits new threads into an inbox; a reply answers mail that inbox sent, so its own sending already bounds how many replies it can get, and a free mailbox's replies never take the share of a paying inbox's day that is kept for premium senders. Replies stop before a thread reaches its message cap so replies cannot answer replies indefinitely. A reply crosses the free and paid tiers like the mail it answers: a paid mailbox answers a free partner that wrote to it, and a free mailbox answers a paid one while it is healthy and its workspace is in good standing. Scheduling respects windows, provider feedback, shared limits and spacing. Jitter distributes load, not human identities. Receiving participation allows probabilistic spam rescue, reads, importance and stars or flags on verified warmup messages. Filing and selected spam rescues are requested immediately; reads, importance and stars are durably scheduled after a randomized dwell. Current receiving permission and worker assignment are checked again at execution time. These actions are automation, not human feedback or a guaranteed reputation improvement. diff --git a/internal/app/consumer/event_new_email.go b/internal/app/consumer/event_new_email.go index fa3c254d6..dbf5b2a42 100644 --- a/internal/app/consumer/event_new_email.go +++ b/internal/app/consumer/event_new_email.go @@ -597,9 +597,13 @@ func (s *JobsService) acceptWarmupEmail(ctx context.Context, e *models.JobEventN return nil } -// recordWarmupPlacement adds the arrival to the sender's daily placement -// history and tells the sender's workspace it moved. +// recordWarmupPlacement tells the recipient's workspace the arrival landed, adds +// it to the sender's daily placement history and tells the sender's workspace. func (s *JobsService) recordWarmupPlacement(ctx context.Context, e *models.JobEventNewEmail, recipient *models.Email, landed string, rescued bool) { + // The receipt is already stored, so the recipient's counts moved whatever the placement claim does. + if s.StreamingPublisher != nil && recipient != nil && recipient.OrganizationID != nil { + s.StreamingPublisher.PublishWarmupActivity(ctx, recipient.OrganizationID.String(), recipient.UserID, recipient.ID.String(), recipient.Email) + } if s.WarmupPlacementRepo == nil || e.Message == nil { return } diff --git a/internal/app/warmupramp/reply.go b/internal/app/warmupramp/reply.go index b26154262..0ff2dcf72 100644 --- a/internal/app/warmupramp/reply.go +++ b/internal/app/warmupramp/reply.go @@ -22,10 +22,20 @@ func ReplyDeficit(target, sent, replied int) float64 { return float64(min(target, 100))/100*float64(sent+1) - float64(replied) } -// ReplyDue reports whether the next send should be a reply. roll is uniform in -// [0,1), so a fractional deficit is a chance and a whole one is certain. -func ReplyDue(deficit, roll float64) bool { - return deficit >= 1 || (deficit > 0 && roll < deficit) +// ReplyChance is the chance the next plain send answers owed mail: the target +// corrected by the window's drift, held within half the target either side so a +// shortfall is repaid over days instead of in one burst of replies, and a +// surplus slows replies without stopping conversations. +func ReplyChance(target, sent, replied int) float64 { + if target <= 0 { + return 0 + } + t := float64(min(target, 100)) / 100 + if sent <= 0 { + return t + } + share := float64(replied) / float64(sent) + return min(max(2*t-share, t/2), 1.5*t, 1) } // RecoverySignals are the reads that decide whether a sender gets recovery diff --git a/internal/app/warmupramp/reply_test.go b/internal/app/warmupramp/reply_test.go index bac5ae088..2ae991718 100644 --- a/internal/app/warmupramp/reply_test.go +++ b/internal/app/warmupramp/reply_test.go @@ -27,36 +27,65 @@ func TestReplyDeficit(t *testing.T) { } } -func TestReplyDue(t *testing.T) { - if ReplyDue(0, 0) || ReplyDue(-1, 0) { - t.Fatal("a mailbox at or ahead of its share was made to reply") - } - if !ReplyDue(1, 0.99) || !ReplyDue(2.5, 0.99) { - t.Fatal("a mailbox a whole reply behind was left a new thread") - } - if !ReplyDue(0.4, 0.39) || ReplyDue(0.4, 0.4) { - t.Fatal("a fractional deficit is not a chance of that size") +func TestReplyChance(t *testing.T) { + for _, tc := range []struct { + name string + target, sent, replied int + want float64 + }{ + {"replies off", 0, 40, 0, 0}, + {"fresh mailbox replies at its target", 30, 0, 0, 0.3}, + {"on target", 30, 10, 3, 0.3}, + {"behind is repaid at up to half again the target", 30, 100, 0, 0.45}, + {"slightly behind leans toward replies", 30, 100, 20, 0.4}, + {"ahead still answers at half the target", 30, 100, 80, 0.15}, + {"never above certain", 80, 100, 0, 1}, + {"a high target still slows when ahead", 80, 100, 100, 0.6}, + {"over 100 reads as 100", 150, 10, 10, 1}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := ReplyChance(tc.target, tc.sent, tc.replied); math.Abs(got-tc.want) > 1e-9 { + t.Fatalf("ReplyChance(%d,%d,%d)=%v, want %v", tc.target, tc.sent, tc.replied, got, tc.want) + } + }) } } -// The share converges on the setting whether arrivals answer most of the -// owed mail or, as at a high send rate, almost none of it. -func TestReplyShareConvergesOnTarget(t *testing.T) { +// Over the same seven-day window the scheduler reads, the share settles on the +// setting whether arrivals answer most of the owed mail or none of it, and a +// window with no replies in it is repaid without any day turning into a burst. +func TestReplyShareConvergesWithoutBursts(t *testing.T) { + const perDay, days = 30, 60 rng := rand.New(rand.NewSource(7)) - for _, target := range []int{0, 10, 30, 60, 100} { - for _, arrivalBinds := range []float64{0, 0.1, 0.9} { - sent, replied := 0, 0 - for range 2000 { - deficit := ReplyDeficit(target, sent, replied) - arrival := deficit > 0 && rng.Float64() < arrivalBinds*float64(target)/100 - if arrival || ReplyDue(deficit, rng.Float64()) { - replied++ - } - sent++ + for _, target := range []int{10, 30, 60, 100} { + for _, arrivalBinds := range []float64{0, 0.5, 0.9} { + sent, replied := make([]int, days+7), make([]int, days+7) + for i := range 7 { + sent[i] = perDay } - share := float64(replied) / float64(sent) * 100 - if math.Abs(share-float64(target)) > 0.5 { - t.Errorf("target %d%% with arrival binds %.0f%%: share %.2f%%", target, arrivalBinds*100, share) + tail := 0.0 + for d := 7; d < days+7; d++ { + for range perDay { + s, r := 0, 0 + for k := d - 6; k <= d; k++ { + s, r = s+sent[k], r+replied[k] + } + arrival := ReplyDeficit(target, s, r) > 0 && rng.Float64() < arrivalBinds*float64(target)/100 + if arrival || rng.Float64() < ReplyChance(target, s, r) { + replied[d]++ + } + sent[d]++ + } + day := float64(replied[d]) / perDay * 100 + if limit := min(2*float64(target)+12, 100); target < 100 && day > limit { + t.Errorf("target %d%% with arrival binds %.0f%%: day %d replied %.0f%% of its sends", target, arrivalBinds*100, d, day) + } + if d >= days { + tail += day / 7 + } + } + if math.Abs(tail-float64(target)) > 5 { + t.Errorf("target %d%% with arrival binds %.0f%%: settled at %.1f%%", target, arrivalBinds*100, tail) } } } diff --git a/internal/config/constants.go b/internal/config/constants.go index d07588df7..209980a75 100644 --- a/internal/config/constants.go +++ b/internal/config/constants.go @@ -612,7 +612,7 @@ const ( PoolLinkTrustUntrustedDivergence = 0.30 WarmupPoolFallbackMinAgeDays = 3 // a free mailbox must have been a pool member this long before premium may borrow it, or write back to one WarmupPoolReturnVisitDays = 14 // a proven free mailbox may write back to a paying mailbox that wrote to it this recently - // What one inbox may receive from the pool in a day: WarmupInboundDailyMultiple + // New threads one inbox may receive from the pool in a day (replies are not counted): WarmupInboundDailyMultiple // times its own daily sends, never below the floor (a recipient-only mailbox // sends nothing) and never above the ceiling. A recipient at its cap is left // out of every draw for the rest of the day, so paying the pool's debt to a diff --git a/internal/infrastructure/pubsub/audience_test.go b/internal/infrastructure/pubsub/audience_test.go index 31520199e..f8e79c661 100644 --- a/internal/infrastructure/pubsub/audience_test.go +++ b/internal/infrastructure/pubsub/audience_test.go @@ -175,3 +175,28 @@ func TestPersonalAndWorkspaceNotificationAudience(t *testing.T) { t.Fatalf("personal events must survive without resolver: %+v", bus.events) } } + +// Warmup events reach the mailbox's whole organization, never one checked against another. +func TestWarmupEventsReachTheOrganization(t *testing.T) { + org, user, mailbox := uuid.New(), uuid.New(), uuid.New() + for _, owner := range []uuid.UUID{org, uuid.New()} { + bus := &audienceBus{} + p := NewStreamingPublisher(bus, &audienceStub{org: owner, allowed: true}) + p.PublishWarmupActivity(context.Background(), org.String(), user.String(), mailbox.String(), "a@example.test") + p.PublishWarmupPlacement(context.Background(), org.String(), user.String(), mailbox.String(), "a@example.test", "inbox") + if owner != org { + if len(bus.events) != 0 { + t.Fatal("a warmup event was published for a mailbox outside its organization") + } + continue + } + if len(bus.events) != 2 { + t.Fatalf("published %d warmup events, want 2", len(bus.events)) + } + for i, e := range bus.events { + if e["org_id"] != org.String() || bus.attrs[i]["org_id"] != org.String() { + t.Fatalf("warmup event narrowed to its owner: event=%+v attrs=%+v", e, bus.attrs[i]) + } + } + } +} diff --git a/internal/infrastructure/pubsub/events.go b/internal/infrastructure/pubsub/events.go index 76d80991e..f2714f4a7 100644 --- a/internal/infrastructure/pubsub/events.go +++ b/internal/infrastructure/pubsub/events.go @@ -47,6 +47,8 @@ const ( EventAccountHealthChanged EventType = "ACCOUNT_HEALTH_CHANGED" // EventWarmupPlacement: a partner saw one of the org's warmup emails land. EventWarmupPlacement EventType = "WARMUP_PLACEMENT" + // EventWarmupActivity: a mailbox sent or received warmup mail, so its counts moved. + EventWarmupActivity EventType = "WARMUP_ACTIVITY" // EventAccountSyncState: a mailbox's import finished or fair use started // or stopped holding it. Status carries the backfill status, Reason the // throttle reason (empty when released). @@ -553,7 +555,7 @@ func (p *StreamingPublisher) PublishWarmupPlacement(ctx context.Context, orgID, if p == nil || p.client == nil || orgID == "" { return } - p.PublishAccountEvent(ctx, &AccountEvent{ + p.publishWarmupEvent(ctx, &AccountEvent{ BaseEvent: BaseEvent{EventType: EventWarmupPlacement, UserID: userID}, OrgID: orgID, EmailAccountID: accountID, @@ -562,6 +564,32 @@ func (p *StreamingPublisher) PublishWarmupPlacement(ctx context.Context, orgID, }) } +// PublishWarmupActivity tells a mailbox's organization its warmup counts moved, so its open views refresh live. +func (p *StreamingPublisher) PublishWarmupActivity(ctx context.Context, orgID, userID, accountID, email string) { + if p == nil || p.client == nil || orgID == "" { + return + } + p.publishWarmupEvent(ctx, &AccountEvent{ + BaseEvent: BaseEvent{EventType: EventWarmupActivity, UserID: userID}, + OrgID: orgID, + EmailAccountID: accountID, + Email: email, + }) +} + +// publishWarmupEvent sends a mailbox's warmup event to its whole organization; +// the user-events topic would narrow it to the mailbox's owner. +func (p *StreamingPublisher) publishWarmupEvent(ctx context.Context, event *AccountEvent) { + event.Timestamp = time.Now() + attrs := map[string]string{ + "user_id": event.UserID, + "org_id": event.OrgID, + "account_id": event.EmailAccountID, + "event_type": string(event.EventType), + } + _ = p.publish(ctx, TopicWarmupUpdate, event, attrs) +} + // AuditEvent signals that a new audit-trail entry was recorded. It is // org-scoped: org_id is set in both the body and the Pub/Sub attributes so the // realtime fanout delivers it to the org channel (owners/admins watching the diff --git a/internal/repository/pg_warmup_partner_load.go b/internal/repository/pg_warmup_partner_load.go index 4fec98da7..c3c6b4958 100644 --- a/internal/repository/pg_warmup_partner_load.go +++ b/internal/repository/pg_warmup_partner_load.go @@ -163,8 +163,8 @@ func scanPartnerWeeks(ctx context.Context, db sendResultDB, query string, args . return out, rows.Err() } -// partnerToday is what a mailbox received and was dispatched today, the two -// numbers its inbound cap is held against. +// partnerToday is the new threads a mailbox received and was dispatched today, +// the two numbers its inbound cap is held against. type partnerToday struct { received, inflight int } @@ -176,12 +176,13 @@ func (r *warmupRepository) partnerTodays(ctx context.Context, ids []uuid.UUID, e SELECT wr.email_account_id, COUNT(*), 0 FROM warmup_received wr WHERE wr.created_at >= date_trunc('day', NOW()) AND wr.email_account_id = ANY($1::uuid[]) + AND `+newThreadSQL("wr")+` GROUP BY 1 UNION ALL SELECT wt.recipient_account_id, 0, COUNT(*) FROM warmup_tokens wt WHERE wt.created_at >= date_trunc('day', NOW()) AND wt.recipient_account_id = ANY($1::uuid[]) - AND wt.task_id <> $2 + AND wt.task_id <> $2 AND `+newThreadSQL("wt")+` GROUP BY 1`, ids, excludeTask) if err != nil { return nil, err @@ -202,6 +203,13 @@ func (r *warmupRepository) partnerTodays(ctx context.Context, ids []uuid.UUID, e return out, rows.Err() } +// newThreadSQL is true for warmup mail (alias has task_id) that opens a thread. A +// reply answers mail its recipient sent, so it neither counts against the +// recipient's inbound cap nor uses the share a tier leaves to the other. +func newThreadSQL(alias string) string { + return `NOT EXISTS(SELECT 1 FROM warmup_tasks rw WHERE rw.task_id=` + alias + `.task_id AND rw.parent_task_id IS NOT NULL)` +} + // partnerInboundCap is how much one inbox may receive from the pool in a day, // scaled to the share the sender's tier may fill. warmupInboxRoomSQL holds the // same rule in SQL. @@ -211,20 +219,13 @@ func partnerInboundCap(sentWeek, sharePercent int) int { return daily * sharePercent / 100 } -// warmupInboxRoomSQL is true while recipient $1 can take $3 more warmup messages today at sender share $2, on the recipient's local day. +// warmupInboxRoomSQL is true while the new threads recipient $1 has today, on its local day, are within sender share $2 of its cap. var warmupInboxRoomSQL = `SELECT GREATEST( - (SELECT COUNT(*) FROM warmup_received wr WHERE wr.email_account_id=ea.id AND wr.created_at >= date_trunc('day',NOW() AT TIME ZONE COALESCE(NULLIF(ea.timezone,''),NULLIF(o.timezone,''),'UTC')) AT TIME ZONE COALESCE(NULLIF(ea.timezone,''),NULLIF(o.timezone,''),'UTC')), - (SELECT COUNT(*) FROM warmup_tokens wt WHERE wt.recipient_account_id=ea.id AND wt.created_at >= date_trunc('day',NOW() AT TIME ZONE COALESCE(NULLIF(ea.timezone,''),NULLIF(o.timezone,''),'UTC')) AT TIME ZONE COALESCE(NULLIF(ea.timezone,''),NULLIF(o.timezone,''),'UTC'))) - + $3::int <= LEAST(GREATEST((((SELECT COUNT(*) FROM warmup_tokens wt WHERE wt.sender_account_id=ea.id AND wt.sent_message_id<>'' AND wt.created_at>=NOW()-INTERVAL '7 days')+6)/7)*` + inboundDailyMultipleSQL + `,` + inboundDailyFloorSQL + `),` + inboundDailyCeilingSQL + `)*$2/100 + (SELECT COUNT(*) FROM warmup_received wr WHERE wr.email_account_id=ea.id AND wr.created_at >= date_trunc('day',NOW() AT TIME ZONE COALESCE(NULLIF(ea.timezone,''),NULLIF(o.timezone,''),'UTC')) AT TIME ZONE COALESCE(NULLIF(ea.timezone,''),NULLIF(o.timezone,''),'UTC') AND ` + newThreadSQL("wr") + `), + (SELECT COUNT(*) FROM warmup_tokens wt WHERE wt.recipient_account_id=ea.id AND wt.created_at >= date_trunc('day',NOW() AT TIME ZONE COALESCE(NULLIF(ea.timezone,''),NULLIF(o.timezone,''),'UTC')) AT TIME ZONE COALESCE(NULLIF(ea.timezone,''),NULLIF(o.timezone,''),'UTC') AND ` + newThreadSQL("wt") + `)) + <= LEAST(GREATEST((((SELECT COUNT(*) FROM warmup_tokens wt WHERE wt.sender_account_id=ea.id AND wt.sent_message_id<>'' AND wt.created_at>=NOW()-INTERVAL '7 days')+6)/7)*` + inboundDailyMultipleSQL + `,` + inboundDailyFloorSQL + `),` + inboundDailyCeilingSQL + `)*$2/100 FROM email_accounts ea JOIN organizations o ON o.id=ea.organization_id WHERE ea.id=$1` -// WarmupInboxHasRoom reports whether recipient can take one more warmup message today from a poolType sender, by the rule the send reservation enforces. -func (r *warmupRepository) WarmupInboxHasRoom(ctx context.Context, recipient uuid.UUID, poolType string) (bool, error) { - var room bool - err := resultDB(ctx, r.db).QueryRow(ctx, warmupInboxRoomSQL, recipient, inboundSharePercent(poolType), 1).Scan(&room) - return room, err -} - // inboundSharePercent: a paying sender may fill all of an inbox's daily cap; a // free sender stops short, so the rest of every inbox's day stays open to the // premium tier. diff --git a/internal/repository/send_admission.go b/internal/repository/send_admission.go index 552a249a1..840368a11 100644 --- a/internal/repository/send_admission.go +++ b/internal/repository/send_admission.go @@ -660,7 +660,8 @@ func (r *taskRepository) cancelOutbound(ctx context.Context, task, mailbox, work func checkWarmupRecipientReservation(ctx context.Context, tx pgx.Tx, task, sender uuid.UUID) error { var recipient uuid.UUID var pool string - if err := tx.QueryRow(ctx, `SELECT wt.recipient_account_id,wp.pool_type FROM warmup_tokens wt JOIN warmup_pool_participants wpp ON wpp.email_account_id=wt.sender_account_id JOIN warmup_pools wp ON wp.id=wpp.pool_id WHERE wt.task_id=$1 AND wt.sender_account_id=$2`, task, sender).Scan(&recipient, &pool); err != nil { + var reply bool + if err := tx.QueryRow(ctx, `SELECT wt.recipient_account_id,wp.pool_type,COALESCE(w.parent_task_id IS NOT NULL,false) FROM warmup_tokens wt JOIN warmup_pool_participants wpp ON wpp.email_account_id=wt.sender_account_id JOIN warmup_pools wp ON wp.id=wpp.pool_id LEFT JOIN warmup_tasks w ON w.task_id=wt.task_id WHERE wt.task_id=$1 AND wt.sender_account_id=$2`, task, sender).Scan(&recipient, &pool, &reply); err != nil { return err } if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,273))`, recipient.String()); err != nil { @@ -673,9 +674,13 @@ func checkWarmupRecipientReservation(ctx context.Context, tx pgx.Tx, task, sende if !receiving { return ErrSendAdmissionDenied } + // A reply answers mail the recipient sent, so its own sending bounds it; the cap and tier shares hold back new threads. + if reply { + return nil + } var allowed bool // The send's own token is already counted, so it needs no extra room. - err := tx.QueryRow(ctx, warmupInboxRoomSQL, recipient, inboundSharePercent(pool), 0).Scan(&allowed) + err := tx.QueryRow(ctx, warmupInboxRoomSQL, recipient, inboundSharePercent(pool)).Scan(&allowed) if err != nil { return err } diff --git a/internal/repository/warmup_lineage.go b/internal/repository/warmup_lineage.go index 8c703576a..96ba826e8 100644 --- a/internal/repository/warmup_lineage.go +++ b/internal/repository/warmup_lineage.go @@ -227,7 +227,7 @@ func (r *taskRepository) recoveringWarmupReplySenders(ctx context.Context, recip return priority, rows.Err() } -// OwedWarmupReplies applies recovery priority before the limit, then orders eligible messages newest first. +// OwedWarmupReplies applies recovery priority before the limit, then continues the deepest conversation, newest first, so replies build threads instead of answering each once. func (r *taskRepository) OwedWarmupReplies(ctx context.Context, recipient uuid.UUID, limit int, excluded WarmupReplyExclusions) ([]WarmupOwedReply, error) { priority, err := r.recoveringWarmupReplySenders(ctx, recipient, excluded) if err != nil { @@ -238,7 +238,7 @@ func (r *taskRepository) OwedWarmupReplies(ctx context.Context, recipient uuid.U COALESCE(wt.conversation_theme,''),COALESCE(wt.content_source,''),wt.conversation_id,wt.conversation_turn, COALESCE(original.reference_ids,'{}'::text[]),original.scenario_version,original.rendering_version,original.max_turns `+owedWarmupRepliesFromSQL+` - ORDER BY (wt.sender_account_id=ANY($5::uuid[])) DESC, wr.created_at DESC, wt.task_id LIMIT $4`, recipient, excluded.Parents, excluded.Senders, limit, priority) + ORDER BY (wt.sender_account_id=ANY($5::uuid[])) DESC, wt.conversation_turn DESC, wr.created_at DESC, wt.task_id LIMIT $4`, recipient, excluded.Parents, excluded.Senders, limit, priority) if err != nil { return nil, err } diff --git a/internal/repository/warmup_partner_load_live_test.go b/internal/repository/warmup_partner_load_live_test.go index a0935854d..d161c1467 100644 --- a/internal/repository/warmup_partner_load_live_test.go +++ b/internal/repository/warmup_partner_load_live_test.go @@ -2,6 +2,7 @@ package repository import ( "context" + "errors" "reflect" "testing" "time" @@ -231,49 +232,95 @@ func TestLiveWarmupPartnerCandidatesDropAnUnreachableRecipient(t *testing.T) { } } -// A reply's pre-check and the send reservation read one rule, so a reply the -// reservation would refuse is known to be refused before it holds the slot. -func TestLiveWarmupInboxHasRoomMatchesTheReservation(t *testing.T) { +// The reservation holds a new thread to its sender tier's share of the inbox's day. +func TestLiveWarmupReservationHoldsEachTierToItsShare(t *testing.T) { f := newPartnerOrgFixture(t) ctx := context.Background() - repo := &warmupRepository{db: f.pool} - room := func(pool string) bool { - t.Helper() - has, err := repo.WarmupInboxHasRoom(ctx, f.sibling, pool) - if err != nil { - t.Fatal(err) - } - return has - } reserve := func(pool string) bool { t.Helper() var allowed bool - if err := f.pool.QueryRow(ctx, warmupInboxRoomSQL, f.sibling, inboundSharePercent(pool), 0).Scan(&allowed); err != nil { + if err := f.pool.QueryRow(ctx, warmupInboxRoomSQL, f.sibling, inboundSharePercent(pool)).Scan(&allowed); err != nil { t.Fatal(err) } return allowed } freeCap := config.WarmupInboundDailyFloor * config.WarmupFreeInboundSharePercent / 100 - for i := 0; i < freeCap-1; i++ { + for range freeCap { f.received(t, f.sibling, f.outside, time.Now().Add(-time.Second)) } - if !room("free") || !room("premium") { - t.Fatal("an inbox one short of the free share has no room") - } - f.received(t, f.sibling, f.outside, time.Now().Add(-time.Second)) - if room("free") { - t.Fatal("a free sender sees room past its share of the inbox") - } if !reserve("free") { t.Fatal("the reservation refused a send already counted inside the free share") } - if !room("premium") { + f.received(t, f.sibling, f.outside, time.Now().Add(-time.Second)) + if reserve("free") { + t.Fatal("a free sender reserved past its share of the inbox") + } + if !reserve("premium") { t.Fatal("a paying sender lost the rest of the inbox's day") } - for i := freeCap; i < config.WarmupInboundDailyFloor; i++ { + for i := freeCap + 1; i <= config.WarmupInboundDailyFloor; i++ { f.received(t, f.sibling, f.outside, time.Now().Add(-time.Second)) } - if room("premium") || reserve("free") { + if reserve("premium") { t.Fatal("a full inbox still has room") } } + +// A reply answers mail its recipient sent, so it reaches an inbox whose day is +// already full of new threads; a new thread to the same inbox is still refused. +func TestLiveWarmupReplyIsAdmittedToAFullInbox(t *testing.T) { + for _, reply := range []bool{false, true} { + t.Run(map[bool]string{false: "new thread", true: "reply"}[reply], func(t *testing.T) { + f, r, worker, _ := newWarmupChargeFixture(t, reply) + ctx := t.Context() + if _, err := f.pool.Exec(ctx, `INSERT INTO warmup_received(email_account_id,internal_id,sender_account_id,message_id) + SELECT $1,gen_random_uuid(),$2,'' FROM generate_series(1,$3::int)`, f.recipient, f.sender, config.WarmupInboundDailyCeiling+1); err != nil { + t.Fatal(err) + } + address := "pl-" + f.recipient.String()[:8] + "@test.local" + _, err := r.ReserveOutbound(ctx, OutboundReservation{TaskID: f.task, MailboxID: f.sender, OrganizationID: f.org, WorkerID: worker, Provider: models.InboxProviderSMTPIMAP, Recipients: []string{address}}) + if reply && err != nil { + t.Fatal("a reply to a full inbox was refused", err) + } + if !reply && !errors.Is(err, ErrSendAdmissionDenied) { + t.Fatal("a new thread to a full inbox was admitted", err) + } + }) + } +} + +// Replies land outside an inbox's count, so a day full of them leaves every +// tier's share of new threads, in the draw and at the reservation, untouched. +func TestLiveWarmupRepliesLeaveTheNewThreadCapUntouched(t *testing.T) { + f := newPartnerOrgFixture(t) + ctx := context.Background() + repo := &warmupRepository{db: f.pool} + t.Cleanup(func() { + f.exec(`DELETE FROM warmup_received WHERE email_account_id = $1`, f.sibling) + f.exec(`DELETE FROM tasks WHERE email_account_id IN ($1, $2)`, f.sibling, f.outside) + }) + for range config.WarmupInboundDailyCeiling + 1 { + parent, reply := uuid.New(), uuid.New() + f.exec(`INSERT INTO tasks (id, task_type, email_account_id, status, message_id) VALUES ($1, 'warmup', $2, 'completed', '')`, parent, f.sibling) + f.exec(`INSERT INTO tasks (id, task_type, email_account_id, status, message_id) VALUES ($1, 'warmup', $2, 'completed', '')`, reply, f.outside) + f.exec(`INSERT INTO warmup_tasks (task_id, parent_task_id) VALUES ($1, $2)`, reply, parent) + f.exec(`INSERT INTO warmup_received (email_account_id, internal_id, message_id, sender_account_id, task_id) + VALUES ($1, gen_random_uuid(), $2, $3, $4)`, f.sibling, "<"+uuid.NewString()+"@test.local>", f.outside, reply) + } + for _, pool := range []string{"free", "premium"} { + var allowed bool + if err := f.pool.QueryRow(ctx, warmupInboxRoomSQL, f.sibling, inboundSharePercent(pool)).Scan(&allowed); err != nil { + t.Fatal(err) + } + if !allowed { + t.Fatalf("a %s sender lost an inbox's new-thread room to replies", pool) + } + } + got, err := repo.WarmupPartnerCandidates(ctx, "premium", f.sender) + if err != nil { + t.Fatal(err) + } + if _, ok := candidateByID(got, f.sibling); !ok { + t.Fatal("an inbox whose day is all replies was left out of the draw") + } +} diff --git a/internal/repository/warmup_reply_share_live_test.go b/internal/repository/warmup_reply_share_live_test.go index 4e6a7fb25..84112edc5 100644 --- a/internal/repository/warmup_reply_share_live_test.go +++ b/internal/repository/warmup_reply_share_live_test.go @@ -167,3 +167,36 @@ func TestLiveReplySharePrioritizesRecoveryBeforeCandidateLimit(t *testing.T) { }) } } + +// A reply in a running conversation is answered before newer mail that opens +// one, so replies build back-and-forth threads instead of one reply on each. +func TestLiveOwedRepliesContinueTheDeepestConversationFirst(t *testing.T) { + f, r := lineageFixture(t) + ctx := t.Context() + seed := func(turn int, age time.Duration) uuid.UUID { + t.Helper() + task := uuid.New() + messageID := task.String() + "@example.test" + for _, q := range []struct { + sql string + args []any + }{ + {`INSERT INTO tasks(id,task_type,email_account_id,status,message_id) VALUES($1,'warmup',$2,'completed','')`, []any{task, f.sender}}, + {`INSERT INTO warmup_tasks(task_id,lineage_version,subject,scenario_version,rendering_version,max_turns) VALUES($1,1,'Pinned diagnostic','diagnostic-v1','canonical-v1',6)`, []any{task}}, + {`INSERT INTO warmup_tokens(token,task_id,sender_account_id,recipient_account_id,conversation_id,conversation_turn,sent_message_id,expires_at) + VALUES($1,$2,$3,$4,$5,$6,$7,NOW()+INTERVAL '1 day')`, []any{uuid.New(), task, f.sender, f.recipient, uuid.New(), turn, "<" + messageID + ">"}}, + {`INSERT INTO warmup_received(email_account_id,internal_id,sender_account_id,message_id,task_id,created_at) VALUES($1,$2,$3,$4,$5,$6)`, []any{f.recipient, uuid.New(), f.sender, messageID, task, time.Now().Add(-age)}}, + } { + if _, err := f.pool.Exec(ctx, q.sql, q.args...); err != nil { + t.Fatal(err) + } + } + return task + } + opening := seed(0, time.Hour) + running := seed(2, 3*time.Hour) + owed, err := r.OwedWarmupReplies(ctx, f.recipient, 8, WarmupReplyExclusions{}) + if err != nil || len(owed) != 2 || owed[0].ParentTaskID != running || owed[1].ParentTaskID != opening { + t.Fatalf("owed replies %+v (%v), want the running conversation %v before the newer opening %v", owed, err, running, opening) + } +} diff --git a/internal/tasks/directed_partner_gate_test.go b/internal/tasks/directed_partner_gate_test.go index 48af23bfb..b7ef0cf15 100644 --- a/internal/tasks/directed_partner_gate_test.go +++ b/internal/tasks/directed_partner_gate_test.go @@ -215,49 +215,50 @@ func TestDirectedWarmupPartnerRefusesTwoSelfExecutedMailboxes(t *testing.T) { } } -// inboxRoomRepo answers the recipient-capacity check the reservation enforces. -type inboxRoomRepo struct { +// cappedCandidatesRepo is a draw whose new-thread cap has left the recipient out. +type cappedCandidatesRepo struct { standingRepo - - room bool - err error - asked []string } -func (r *inboxRoomRepo) WarmupInboxHasRoom(_ context.Context, _ uuid.UUID, poolType string) (bool, error) { - r.asked = append(r.asked, poolType) - return r.room, r.err +func (cappedCandidatesRepo) WarmupExecutionCandidates(context.Context, string, uuid.UUID, uuid.UUID) ([]models.WarmupPartnerCandidate, error) { + return nil, nil } -// A reply holds the mailbox's only pending task, so one to a full inbox closes -// instead of being refused by the reservation every few minutes for days. -func TestDirectedWarmupPartnerClosesAReplyToAFullInbox(t *testing.T) { - org := uuid.New() +// failingRuleRepo cannot read the organization's routing rules. +type failingRuleRepo struct { + repository.WarmupRoutingRepository +} + +func (failingRuleRepo) ListForOrganization(context.Context, uuid.UUID) ([]models.WarmupRoutingRule, error) { + return nil, errors.New("db down") +} + +// A reply is rechecked at execution by the rule that chose its partner, so an +// inbox whose day is full of new threads still gets it; a new thread does not. +func TestWarmupExecutionHoldsRepliesToTheReplyRule(t *testing.T) { + recipient, org := uuid.New(), uuid.New() for _, tc := range []struct { - name string - repo *inboxRoomRepo - want bool + name string + reply bool + pool string + routes repository.WarmupRoutingRepository + want bool }{ - {"a full inbox ends the thread", &inboxRoomRepo{room: false}, false}, - {"an inbox with room is answered", &inboxRoomRepo{room: true}, true}, - {"an unreadable capacity leaves the decision to the reservation", &inboxRoomRepo{err: errors.New("db down")}, true}, + {"a reply to a capped inbox goes out", true, "premium", ruleRepo{}, true}, + {"a new thread to a capped inbox does not", false, "premium", ruleRepo{}, false}, + {"a reply to a mailbox that left the pool does not", true, "", ruleRepo{}, false}, + {"a reply whose routing rules cannot be read does not", true, "premium", failingRuleRepo{}, false}, } { t.Run(tc.name, func(t *testing.T) { - target := uuid.New() - tc.repo.state = models.WarmupHealthHealthy s := &tasksService{ - taskRepo: &directedTaskRepo{target: target}, - emailRepo: directedEmailRepo{}, - warmupRepo: tc.repo, - warmupHealth: &pinnedGate{poolOf: map[uuid.UUID]string{target: "free"}}, - orgRiskRepo: riskRepo{state: models.OrgRiskTrusted}, + emailRepo: directedEmailRepo{}, + warmupRepo: cappedCandidatesRepo{standingRepo{state: models.WarmupHealthHealthy}}, + warmupHealth: &pinnedGate{poolOf: map[uuid.UUID]string{recipient: tc.pool}}, + warmupRoutingRepo: tc.routes, } - sender := &Email{ID: uuid.New(), OrganizationID: &org, WarmupPoolType: "free"} - if got := s.directedWarmupPartner(context.Background(), uuid.New(), sender, "free"); (got != nil) != tc.want { - t.Fatalf("partner = %v, want answered %v", got, tc.want) - } - if len(tc.repo.asked) != 1 || tc.repo.asked[0] != "free" { - t.Fatalf("capacity asked for pools %v, want the sender's free share", tc.repo.asked) + err := s.validateWarmupRecipient(context.Background(), uuid.New(), tc.reply, recipient, &Email{ID: uuid.New(), OrganizationID: &org}, "premium") + if (err == nil) != tc.want { + t.Fatalf("validate = %v, want allowed %v", err, tc.want) } }) } diff --git a/internal/tasks/email_task.go b/internal/tasks/email_task.go index c5e36ba0e..1f68cd4fd 100644 --- a/internal/tasks/email_task.go +++ b/internal/tasks/email_task.go @@ -1137,6 +1137,10 @@ func (s *tasksService) publishWarmupEmailSentEvent(ctx context.Context, task *Ta } } + if s.streamingPublisher != nil && account != nil && account.OrganizationID != nil { + s.streamingPublisher.PublishWarmupActivity(ctx, account.OrganizationID.String(), account.UserID, account.ID.String(), account.Email) + } + if s.advanced != nil && account != nil && account.OrganizationID != nil { s.advanced.EmitCampaignEvent(ctx, *account.OrganizationID, models.WebhookEventWarmupEmailSent, map[string]any{ "email_account_id": account.ID.String(), @@ -1173,11 +1177,11 @@ func (s *tasksService) directedWarmupPartner(ctx context.Context, taskID uuid.UU if err != nil || warmupTask == nil || warmupTask.TargetAccountID == nil { return nil } - return s.replyPartner(ctx, taskID, *warmupTask.TargetAccountID, account, poolType) + return s.replyPartner(ctx, *warmupTask.TargetAccountID, account, poolType) } // replyPartner is target when this mailbox may answer it now, or nil. -func (s *tasksService) replyPartner(ctx context.Context, taskID, target uuid.UUID, account *Email, poolType string) *Email { +func (s *tasksService) replyPartner(ctx context.Context, target uuid.UUID, account *Email, poolType string) *Email { if s.warmupHealth != nil { ok, reason, herr := s.warmupHealth.CanParticipate(ctx, target, poolType) if herr != nil { @@ -1211,15 +1215,6 @@ func (s *tasksService) replyPartner(ctx context.Context, taskID, target uuid.UUI return nil } } - // A reply holds the mailbox's only warmup task, so one to a full inbox ends the thread rather than waiting on the reservation's refusal. - if room, ok := s.warmupRepo.(interface { - WarmupInboxHasRoom(context.Context, uuid.UUID, string) (bool, error) - }); ok { - if has, err := room.WarmupInboxHasRoom(ctx, partner.ID, poolType); err == nil && !has { - log.Info().Str("task_id", taskID.String()).Str("email_account_id", account.ID.String()).Msg("warmup reply closed: the recipient's inbox is full today") - return nil - } - } return partner } diff --git a/internal/tasks/warmup_lineage.go b/internal/tasks/warmup_lineage.go index 00b894bb7..e982646d1 100644 --- a/internal/tasks/warmup_lineage.go +++ b/internal/tasks/warmup_lineage.go @@ -212,13 +212,55 @@ func (s *tasksService) ValidateWarmupExecution(ctx context.Context, taskID uuid. } else if _, ok := GenerateConversationReplyEmail(source, *account, current.ConversationTurn); !ok { return errors.New("diagnostic turn unavailable") } + pool := s.resolveWarmupPoolType(ctx, account) + if err := s.validateWarmupRecipient(ctx, taskID, warmup.ParentTaskID != nil, recipientID, account, pool); err != nil { + return err + } + if warmup.ParentTaskID != nil { + lineage, ok := s.taskRepo.(repository.WarmupLineageRepository) + if !ok { + return errors.New("warmup lineage authority unavailable") + } + parent, err := lineage.ExactWarmupParent(ctx, taskID) + if err != nil { + return err + } + account, xerr := s.emailRepo.GetByID(ctx, task.EmailAccountID) + if xerr != nil { + return xerr + } + if account == nil { + return errors.New("warmup mailbox unavailable") + } + if _, _, _, err = s.exactReplyContent(ctx, parent, *account); err != nil { + return err + } + } + return nil +} + +// validateWarmupRecipient re-checks the recipient at execution. A reply is held +// to the rule that chose its partner, because it answers mail its recipient +// sent and so is outside the new-thread cap the draw applies. +func (s *tasksService) validateWarmupRecipient(ctx context.Context, taskID uuid.UUID, reply bool, recipientID uuid.UUID, account *Email, pool string) error { + if reply && s.warmupHealth != nil { + // replyPartner reads unavailable routing rules as none, so the execution recheck fails closed on them first. + if pool == "premium" && s.warmupRoutingRepo != nil && account.OrganizationID != nil { + if _, err := s.warmupRoutingRepo.ListForOrganization(ctx, *account.OrganizationID); err != nil { + return err + } + } + if partner := s.replyPartner(ctx, recipientID, account, pool); partner == nil || partner.ID != recipientID { + return errors.New("warmup recipient no longer eligible") + } + return nil + } reader, ok := s.warmupRepo.(interface { WarmupExecutionCandidates(context.Context, string, uuid.UUID, uuid.UUID) ([]models.WarmupPartnerCandidate, error) }) if !ok { return errors.New("warmup recipient capacity unavailable") } - pool := s.resolveWarmupPoolType(ctx, account) candidates, err := reader.WarmupExecutionCandidates(ctx, pool, account.ID, taskID) if err != nil { return err @@ -246,25 +288,5 @@ func (s *tasksService) ValidateWarmupExecution(ctx context.Context, taskID uuid. return errors.New("warmup recipient excluded") } } - if warmup.ParentTaskID != nil { - lineage, ok := s.taskRepo.(repository.WarmupLineageRepository) - if !ok { - return errors.New("warmup lineage authority unavailable") - } - parent, err := lineage.ExactWarmupParent(ctx, taskID) - if err != nil { - return err - } - account, xerr := s.emailRepo.GetByID(ctx, task.EmailAccountID) - if xerr != nil { - return xerr - } - if account == nil { - return errors.New("warmup mailbox unavailable") - } - if _, _, _, err = s.exactReplyContent(ctx, parent, *account); err != nil { - return err - } - } return nil } diff --git a/internal/tasks/warmup_reply_share.go b/internal/tasks/warmup_reply_share.go index 9e429ebc4..5843030f0 100644 --- a/internal/tasks/warmup_reply_share.go +++ b/internal/tasks/warmup_reply_share.go @@ -15,8 +15,8 @@ import ( const owedReplyCandidates = 8 // owedWarmupReply makes a claimed plain send the reply to received warmup mail -// when the mailbox's reply share is behind its setting, answering recovering -// senders first. Nil leaves the send a new thread. +// at the mailbox's reply chance, answering recovering senders first, then the +// deepest conversation. Nil leaves the send a new thread. func (s *tasksService) owedWarmupReply(ctx context.Context, taskID uuid.UUID, account *Email, poolType string) *Email { if account == nil || account.WarmupReplyRate <= 0 || !account.IsWarmingActive() { return nil @@ -26,7 +26,7 @@ func (s *tasksService) owedWarmupReply(ctx context.Context, taskID uuid.UUID, ac return nil } sent, replied, err := share.WarmupReplyShare(ctx, account.ID) - if err != nil || !warmupramp.ReplyDue(warmupramp.ReplyDeficit(account.WarmupReplyRate, sent, replied), rand.Float64()) { + if err != nil || rand.Float64() >= warmupramp.ReplyChance(account.WarmupReplyRate, sent, replied) { return nil } excluded := repository.WarmupReplyExclusions{} @@ -43,7 +43,7 @@ func (s *tasksService) owedWarmupReply(ctx context.Context, taskID uuid.UUID, ac } partner, checked := partners[o.SenderID] if !checked { - partner = s.replyPartner(ctx, taskID, o.SenderID, account, poolType) + partner = s.replyPartner(ctx, o.SenderID, account, poolType) partners[o.SenderID] = partner if partner == nil { excluded.Senders = append(excluded.Senders, o.SenderID) diff --git a/internal/tasks/warmup_reply_share_live_test.go b/internal/tasks/warmup_reply_share_live_test.go index 23270c853..96a7d2a2d 100644 --- a/internal/tasks/warmup_reply_share_live_test.go +++ b/internal/tasks/warmup_reply_share_live_test.go @@ -13,7 +13,7 @@ import ( ) func TestLiveWarmupReplyShareContinuesPastUnavailableCandidates(t *testing.T) { - for _, reason := range []string{"pool eligibility", "routing exclusion", "full inbox", "missing content", "all unavailable"} { + for _, reason := range []string{"pool eligibility", "routing exclusion", "missing content", "all unavailable"} { t.Run(reason, func(t *testing.T) { f := newPartnerRoutingFixture(t) ctx := t.Context() @@ -82,9 +82,6 @@ func TestLiveWarmupReplyShareContinuesPastUnavailableCandidates(t *testing.T) { pool = "premium" exec(`UPDATE warmup_pool_participants SET pool_id=$2 WHERE email_account_id IN(SELECT id FROM email_accounts WHERE organization_id=$1)`, f.org, models.WarmupPoolPremiumID) f.svc.warmupRoutingRepo = ruleRepo{rules: excludeDomain("acme.test")} - case "full inbox": - exec(`INSERT INTO warmup_received(email_account_id,internal_id,sender_account_id,message_id) - SELECT $1,gen_random_uuid(),$2,'' FROM generate_series(1,1000)`, f.atWorkspace, f.sender.ID) case "missing content": exec(`UPDATE warmup_tokens SET conversation_id=$2 WHERE sender_account_id=$1`, f.atWorkspace, uuid.New()) } @@ -106,7 +103,7 @@ func TestLiveWarmupReplyShareContinuesPastUnavailableCandidates(t *testing.T) { if _, _, _, err := f.svc.exactReplyContent(ctx, &next[0].Parent, f.sender); err != nil { t.Fatalf("ordinary message cannot be rendered: %v", err) } - if f.svc.replyPartner(ctx, claimed, f.atSmallHost, &f.sender, pool) == nil { + if f.svc.replyPartner(ctx, f.atSmallHost, &f.sender, pool) == nil { t.Fatal("ordinary partner cannot receive") } } diff --git a/web/src/components/app/emails/InboxDetails.tsx b/web/src/components/app/emails/InboxDetails.tsx index 3c05cf168..08ea3702d 100644 --- a/web/src/components/app/emails/InboxDetails.tsx +++ b/web/src/components/app/emails/InboxDetails.tsx @@ -61,6 +61,7 @@ import type AccountStatusModel from "@/lib/api/models/app/analytics/AccountStatu import type { AccountError } from "@/lib/api/models/app/analytics/AccountStatus"; import useAccountStatus from "@/lib/api/hooks/app/analytics/useAccountStatus"; import useWarmupAnalytics from "@/lib/api/hooks/app/analytics/useWarmupAnalytics"; +import type WarmupAnalytics from "@/lib/api/models/app/analytics/WarmupAnalytics"; import useUpdateEmail from "@/lib/api/hooks/app/emails/useUpdateEmail"; import useRemoveEmail from "@/lib/api/hooks/app/emails/useRemoveEmail"; import useWarmupLifecycle from "@/lib/api/hooks/app/emails/useWarmupLifecycle"; @@ -99,7 +100,7 @@ import SendingBehaviorTab from "./SendingBehaviorTab"; import { useUserProfile } from "@/hooks/context/user"; import useCurrentOrganization from "@/lib/api/hooks/app/organizations/useCurrentOrganization"; import { timezoneOptions } from "@/lib/timezone"; -import { utcDay } from "@/lib/campaignPeriod"; +import { utcDay, utcToday } from "@/lib/campaignPeriod"; import SyncStatusCard from "./SyncStatusCard"; import ServerCertificatesCard from "./ServerCertificatesCard"; import CloudWarmupCard from "./CloudWarmupCard"; @@ -457,13 +458,9 @@ function Detail({ mailbox, onClose, initialTab, tabIntentKey, onTabIntentConsume usePresenceResource(mailbox.id ? `mailbox:${mailbox.id}` : null, "editing"); const status = useAccountStatus(mailbox.id); - const { from, to } = useMemo(() => { - const end = new Date(); - const start = new Date(); - start.setDate(start.getDate() - 29); - const fmt = (d: Date) => d.toISOString().slice(0, 10); - return { from: fmt(start), to: fmt(end) }; - }, []); + // Recomputed each render so a drawer left open past UTC midnight moves on to the new day. + const to = utcToday(); + const from = utcDay(new Date(Date.now() - 29 * 86_400_000)); const warmup = useWarmupAnalytics(mailbox.id, from, to); const pool = useCloudPool(); const mutation = useUpdateEmail(mailbox.id, pool.connected && pool.isEnrolled(mailbox.id) ? cloudOwnedWarmupFields : undefined); @@ -574,9 +571,9 @@ function Detail({ mailbox, onClose, initialTab, tabIntentKey, onTabIntentConsume {/* Body */}
- {tab === "overview" && } + {tab === "overview" && } {tab === "deliverability" && } - {tab === "analytics" && void warmup.refetch()} />} + {tab === "analytics" && void warmup.refetch()} />} {tab === "warmup" && } {tab === "sending" && } {tab === "settings" && } @@ -735,10 +732,11 @@ function DelegatedGrantNotice({ mailbox, inError = false }: { mailbox: Inbox; in ); } -function OverviewTab({ status, loading, mailbox }: { status?: import("@/lib/api/models/app/analytics/AccountStatus").default; loading: boolean; mailbox: Inbox }) { +function OverviewTab({ status, loading, mailbox, warmup }: { status?: import("@/lib/api/models/app/analytics/AccountStatus").default; loading: boolean; mailbox: Inbox; warmup?: WarmupAnalytics }) { const health = status?.health; const usage = status?.daily_usage; const ws = status?.warmup_status; + const replyShare = weekReplyShare(warmup); // When a sending-behaviour profile is on, TODAY'S rolled plan is the real // ceiling and spacing — not the mailbox's fixed cap and gap. Showing the @@ -825,7 +823,7 @@ function OverviewTab({ status, loading, mailbox }: { status?: import("@/lib/api/ } /> - +
@@ -915,9 +913,21 @@ function OverviewTab({ status, loading, mailbox }: { status?: import("@/lib/api/ ); } +// The reply target is held over the last seven UTC days, so that window is the one to compare it with. +function weekReplyShare(warmup?: WarmupAnalytics): number | null { + const since = utcDay(new Date(Date.now() - 6 * 86_400_000)); + let sent = 0, replied = 0; + for (const d of warmup?.daily_stats ?? []) { + if (d.date < since) continue; + sent += d.emails_sent; + replied += d.emails_replied; + } + return sent > 0 ? Math.round((replied / sent) * 100) : null; +} + /* ── Analytics ─────────────────────── */ -function AnalyticsTab({ warmup, loading, failed, retry }: { warmup?: import("@/lib/api/models/app/analytics/WarmupAnalytics").default; loading: boolean; failed: boolean; retry: () => void }) { +function AnalyticsTab({ warmup, replyTarget, loading, failed, retry }: { warmup?: WarmupAnalytics; replyTarget: number; loading: boolean; failed: boolean; retry: () => void }) { // Tap affordance for touch devices, where the native title tooltip never fires. const [selectedDay, setSelectedDay] = useState(null); if (loading) { @@ -944,6 +954,11 @@ function AnalyticsTab({ warmup, loading, failed, retry }: { warmup?: import("@/l ); } const s = warmup.summary; + // The range runs 30 days but the target is held over 7, so the share is read against the target on that window. + const weekShare = weekReplyShare(warmup); + const replySub = weekShare === null + ? `${s.reply_rate.toFixed(1)}% reply rate` + : replyTarget > 0 ? `${weekShare}% last 7d · target ${replyTarget}%` : `${weekShare}% last 7 days`; const chartData = warmup.daily_stats.map((d) => ({ key: d.date, value: d.emails_sent, @@ -959,7 +974,7 @@ function AnalyticsTab({ warmup, loading, failed, retry }: { warmup?: import("@/l
- +
diff --git a/web/src/hooks/useRealtimeEvents.ts b/web/src/hooks/useRealtimeEvents.ts index 42ffa7c19..5cf26eed8 100644 --- a/web/src/hooks/useRealtimeEvents.ts +++ b/web/src/hooks/useRealtimeEvents.ts @@ -367,6 +367,13 @@ export function useRealtimeEvents() { return } + // A mailbox sent or received warmup mail: only its own counts moved. + if (event === 'WARMUP_ACTIVITY') { + const accountId = getString('email_account_id') + if (accountId) accountRefresh.add([['analytics', 'warmup', accountId], ['analytics', 'accounts', accountId]]) + return + } + if (includes('ACCOUNT', 'EMAIL_STATUS', 'EMAIL_ERROR', 'WARMUP')) { // ACCOUNT_SYNC_STATE: the mailbox's import finished or fair use // started/stopped holding it; the drawer's sync card refetches.