Files
warmbly/internal/tasks/email_task.go
T

1300 lines
47 KiB
Go

package tasks
import (
"context"
"errors"
"fmt"
"math/rand"
"strings"
"time"
"github.com/google/uuid"
"github.com/rs/zerolog/log"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/observability/errs"
"github.com/warmbly/warmbly/internal/pkg/generation"
"github.com/warmbly/warmbly/internal/pkg/mailhost"
"github.com/warmbly/warmbly/internal/repository"
"github.com/warmbly/warmbly/internal/scheduler"
"github.com/warmbly/warmbly/internal/tasks/proto"
)
func (s *tasksService) HandleEmailTask(task *proto.ProcessTask) *errx.Error {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
// STEP 1: Parse task ID
taskID, err := uuid.Parse(task.TaskId)
if err != nil {
errs.CaptureException(err)
return errx.New(errx.BadRequest, "invalid task ID")
}
executionKey := "warmup:" + taskID.String()
executionStatus := "failed"
// lane distinguishes a normal user-enabled warmup send ("warmup") from a
// health-check send kept flowing only because the mailbox backs a live
// campaign ("health_check"). Set once the account is loaded; the defer
// reads it at completion time.
lane := "warmup"
// STEP 2: Load task record
taskRecord, err := s.taskRepo.GetTask(ctx, taskID)
if err != nil {
errs.CaptureException(err)
return errx.InternalError()
}
if taskRecord == nil {
return errx.ErrNotFound
}
if taskRecord.ScheduledAt != nil && taskRecord.ScheduledAt.After(time.Now()) {
return nil
}
lineage, ok := s.taskRepo.(repository.WarmupLineageRepository)
if !ok {
return errx.InternalError()
}
if taskRecord.Status != "pending" {
log.Info().
Str("task_id", taskID.String()).
Str("status", taskRecord.Status).
Msg("warmup task skipped: task not in pending state")
executionStatus = "completed"
return nil
}
// STEP 3: Load email account
account, xerr := s.emailRepo.GetByID(ctx, taskRecord.EmailAccountID)
if xerr != nil {
return xerr
}
if account == nil {
return errx.ErrNotFound
}
// A mailbox enrolled in Warmbly Cloud is warmed there; the local chain ends here.
if !account.TestSendingAllowed() {
_ = s.taskRepo.UpdateTaskStatus(ctx, taskID, "cancelled")
executionStatus = "skipped_diagnostic_stopped"
return nil
}
if s.cloudLink != nil && s.cloudLink.IsEnrolled(ctx, account.ID) {
_ = s.taskRepo.UpdateTaskStatus(ctx, taskID, "cancelled")
executionStatus = "skipped_cloud_warmup"
return nil
}
// Keep the warmup chain alive while the mailbox is actively warming OR
// while it backs a live campaign (the low-volume health-check lane). Once
// neither holds, the chain is allowed to wind down.
activelyWarming := account.IsWarmingActive()
inCampaign := false
if !activelyWarming {
inCampaign = s.accountInActiveCampaign(ctx, account.ID)
}
if account.Status != "active" || (!activelyWarming && !inCampaign) {
if s.warmupHealth != nil {
if account.Status == "active" && account.OrganizationID != nil && s.featureGate != nil {
canWarmup, xerr := s.featureGate.CanUseWarmup(ctx, *account.OrganizationID)
switch {
case xerr != nil:
// Entitlement unknown: leave the membership alone rather than let a blip evict.
case canWarmup:
_ = s.warmupHealth.EnsurePoolMembershipWithRole(ctx, account.ID, s.resolveWarmupPoolType(ctx, account), "recipient_only")
default:
_ = s.warmupHealth.RemoveFromAllPools(ctx, account.ID)
}
} else {
_ = s.warmupHealth.RemoveFromAllPools(ctx, account.ID)
}
}
_ = s.taskRepo.UpdateTaskStatus(ctx, taskID, "cancelled")
executionStatus = "completed"
return nil
}
if !activelyWarming && inCampaign {
lane = "health_check"
log.Info().
Str("task_id", taskID.String()).
Str("email_account_id", account.ID.String()).
Msg("warmup health-check send (mailbox in active campaign, warmup off)")
}
// Tenancy gate: warmup entitlement, pool tier and the shared-pool safety
// signals are all org-scoped, so a mailbox with no workspace cannot be
// checked against any of them. Fail closed — it leaves the pool and stops
// warming rather than warming as if it were entitled.
if account.OrganizationID == nil {
if s.warmupHealth != nil {
_ = s.warmupHealth.RemoveFromAllPools(ctx, account.ID)
}
errs.CaptureException(fmt.Errorf("email account %s reached warmup with no organization", account.ID))
log.Error().Str("task_id", taskID.String()).Str("email_account_id", account.ID.String()).
Msg("warmup blocked: mailbox has no organization, entitlement and pool policy cannot be checked")
_ = s.taskRepo.UpdateTaskStatus(ctx, taskID, "skipped_no_warmup_access")
executionStatus = "completed"
return nil
}
// STEP 3.5: Check if organization can use warmup (only paid orgs)
if s.featureGate != nil {
canWarmup, entitlementErr := s.featureGate.CanUseWarmup(ctx, *account.OrganizationID)
if !canWarmup {
// Only a definite "not entitled" evicts; the send is skipped either way.
if s.warmupHealth != nil && entitlementErr == nil {
_ = s.warmupHealth.RemoveFromAllPools(ctx, account.ID)
}
// Organization cannot use warmup - skip this task
s.taskRepo.UpdateTaskStatus(ctx, taskID, "skipped_no_warmup_access")
executionStatus = "completed"
return nil
}
}
poolType := s.resolveWarmupPoolType(ctx, account)
// STEP 3.6: Sending-domain authentication. Pool reputation is shared, so an
// unauthenticated sender damages every other participant, not just itself.
// Demoted to recipient_only rather than removed: receiving harms nobody and
// the mailbox promotes straight back once its DNS is fixed.
if s.domainAuthBlocked(ctx, account) {
if s.warmupHealth != nil {
_ = s.warmupHealth.EnsurePoolMembershipWithRole(ctx, account.ID, poolType, "recipient_only")
}
log.Warn().
Str("task_id", taskID.String()).
Str("email_account_id", account.ID.String()).
Str("email", account.Email).
Msg("warmup send skipped: sending domain fails authentication")
_ = s.taskRepo.UpdateTaskStatus(ctx, taskID, "skipped_domain_auth")
// Re-seed the chain, or the mailbox never warms again even after its
// DNS is fixed: this task has no successor and nothing else creates
// one. The recheck cadence is the partner-exhaustion one (4-8h), which
// is the right order of magnitude for a DNS change to propagate.
if createErr := s.createWarmupTask(ctx, account.ID, warmupPartnerRecheckTime()); createErr != nil {
log.Warn().Err(createErr).Str("task_id", taskID.String()).Str("email_account_id", account.ID.String()).Msg("Failed to reschedule warmup task after domain-auth gate")
}
executionStatus = "completed"
return nil
}
// A suspended workspace stops warming too: warmup is outbound mail from the
// same domains, so leaving it running would keep spending the reputation
// the suspension exists to protect.
if s.orgBlocksSending(ctx, account.OrganizationID) {
_ = s.taskRepo.UpdateTaskStatus(ctx, taskID, "skipped_org_suspended")
executionStatus = "completed"
return nil
}
// A legacy mailbox whose credential stayed on its instance ends its chain; handing the credential to Cloud restarts it.
if executed, err := s.instanceExecuted(ctx, account.ID); err != nil {
return errx.InternalError()
} else if executed {
_ = s.taskRepo.UpdateTaskStatus(ctx, taskID, "cancelled")
executionStatus = "skipped_credential_required"
return nil
}
if s.warmupHealth != nil {
canParticipate, err := s.warmupRepo.IsPoolEligible(ctx, account.ID, poolType, true)
if err != nil {
return errx.InternalError()
}
if !canParticipate {
_ = s.taskRepo.UpdateTaskStatus(ctx, taskID, "skipped_warmup_protected")
s.scheduleWarmupRecovery(ctx, account.ID, poolType)
executionStatus = "completed"
return nil
}
}
// A reply whose parent expired is closed here; the gates below would only keep rescheduling it.
if wt, err := s.taskRepo.GetWarmupTask(ctx, taskID); err == nil && wt != nil && wt.ParentTaskID != nil {
if parent, err := lineage.ExactWarmupParent(ctx, taskID); err == nil && parent == nil {
return s.closeWarmupThread(ctx, taskID, account.ID)
}
}
if err := s.sendAdmission(ctx, account); err != nil {
if errors.Is(err, repository.ErrSendAdmissionDenied) {
if err := lineage.RescheduleWarmupTask(ctx, taskID, time.Now().Add(5*time.Minute)); err != nil {
return errx.InternalError()
}
return nil
}
return errx.InternalError()
}
capable, ok := s.emailSender.(interface {
WarmupWorkerReady(context.Context, models.Email) (bool, error)
})
if !ok {
return errx.InternalError()
}
ready, capErr := capable.WarmupWorkerReady(ctx, *account)
if capErr != nil {
return errx.InternalError()
}
if !ready {
if err := lineage.RescheduleWarmupTask(ctx, taskID, time.Now().Add(5*time.Minute)); err != nil {
return errx.InternalError()
}
return nil
}
if gate, ok := s.scheduler.(interface {
WarmupDispatchNotBefore(context.Context, uuid.UUID, time.Time) (time.Time, error)
}); ok {
at, err := gate.WarmupDispatchNotBefore(ctx, account.ID, time.Now())
if errors.Is(err, scheduler.ErrWarmupTimeWindowInvalid) {
if err := lineage.RescheduleWarmupTask(ctx, taskID, time.Now().Add(5*time.Minute)); err != nil {
return errx.InternalError()
}
return nil
}
if err != nil {
return errx.InternalError()
}
if at.After(time.Now()) {
// The gate answers with the window's opening; spread deferred sends past it, unless that would leave a short window.
jittered := at.Add(time.Duration(1+rand.Intn(20*60)) * time.Second)
if again, gerr := gate.WarmupDispatchNotBefore(ctx, account.ID, jittered); gerr == nil && !again.After(jittered) {
at = jittered
}
if err := lineage.RescheduleWarmupTask(ctx, taskID, at); err != nil {
return errx.InternalError()
}
return nil
}
} else {
return errx.InternalError()
}
claimed, err := lineage.ClaimWarmupTask(ctx, taskID, time.Now())
if err != nil {
return errx.InternalError()
}
if !claimed {
return nil
}
defer func() {
releaseCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_ = lineage.ReleaseUnpublishedWarmupTask(releaseCtx, taskID, time.Now().Add(5*time.Minute))
}()
if s.advanced != nil {
_, xerr := s.advanced.StartTaskExecution(ctx, taskID, executionKey, map[string]interface{}{"task_type": "warmup"})
if xerr != nil {
return xerr
}
defer func() {
_ = s.advanced.CompleteTaskExecution(ctx, taskID, executionKey, executionStatus, map[string]interface{}{"task_type": "warmup", "lane": lane})
}()
}
// STEP 5: Select warmup partner. A task directed at one partner is a
// reply-back: someone mailed this mailbox and it is answering them, which
// is what makes a thread read as a conversation instead of two monologues.
// The directed partner still passes the same health gate as a drawn one.
partner := s.directedWarmupPartner(ctx, taskID, account, poolType)
warmupTask, err := s.taskRepo.GetWarmupTask(ctx, taskID)
if err != nil || warmupTask == nil {
return errx.InternalError()
}
if warmupTask.TargetAccountID != nil && (warmupTask.ParentTaskID == nil || partner == nil) {
return s.closeWarmupThread(ctx, taskID, account.ID)
}
// A plain send answers owed mail while the mailbox's reply share is behind its setting.
if partner == nil {
partner = s.owedWarmupReply(ctx, taskID, account, poolType)
}
// A directed task IS the reply; the reply-rate draw already happened when
// it was scheduled. Rolling again here would square the rate, so a 30%
// reply rate would answer 9% of the time.
directedReply := partner != nil
if partner == nil {
partner, err = s.selectWarmupPartner(ctx, *account)
}
if err != nil {
_ = s.taskRepo.UpdateTaskStatus(ctx, taskID, "skipped_warmup_protected")
nextTime, scheduleErr := s.scheduler.CalculateNextWarmupTime(ctx, account.ID)
if scheduleErr != nil {
nextTime = warmupPartnerRecheckTime()
}
if nextTime.Before(time.Now().Add(4 * time.Hour)) {
nextTime = warmupPartnerRecheckTime()
}
if createErr := s.createWarmupTask(ctx, account.ID, nextTime); createErr != nil {
log.Warn().Err(createErr).Str("task_id", taskID.String()).Str("email_account_id", account.ID.String()).Msg("Failed to reschedule warmup task after partner exhaustion")
}
executionStatus = "completed"
return nil
}
// STEP 6: Determine if this should be a reply or a new warmup email.
// Replies advance the exact cached conversation plan used by the parent.
shouldReply := directedReply
var subject, emailBody, conversationTheme, contentSource string
var conversationID *uuid.UUID
var conversationTurn int
var inReplyTo string
var references []string
var threadID string
var content warmupContent
if shouldReply {
candidate, replyErr := lineage.ExactWarmupParent(ctx, taskID)
if replyErr != nil {
return errx.InternalError()
}
content, inReplyTo, references, replyErr = s.exactReplyContent(ctx, candidate, *account)
if replyErr != nil {
return s.closeWarmupThread(ctx, taskID, account.ID)
}
conversationTurn = candidate.ConversationTurn + 1
if account.Provider == string(models.InboxProviderGoogle) && candidate.ThreadID != nil {
threadID = *candidate.ThreadID
}
}
// STEP 7: Draw content neither mailbox has seen recently; only a database failure retries.
if !shouldReply {
var contentErr error
content, contentErr = s.pickNewWarmupContent(ctx, *account, partner, taskID)
if contentErr != nil {
if !errors.Is(contentErr, errWarmupContentUnavailable) {
errs.CaptureException(contentErr)
return errx.InternalError()
}
log.Warn().Str("email_account_id", account.ID.String()).Msg("Warmup has no content to send; the static library is empty")
return s.closeWarmupThread(ctx, taskID, account.ID)
}
}
subject, emailBody, conversationTheme, contentSource, conversationID = content.subject, content.body, content.theme, content.contentSource, content.conversationID
if err := lintDiagnosticContent(subject, emailBody, shouldReply, *account); err != nil {
return s.closeWarmupThread(ctx, taskID, account.ID)
}
if err := lineage.SaveWarmupLineage(ctx, taskID, repository.WarmupLineage{Subject: subject, References: references, ScenarioVersion: content.scenarioVersion, RenderingVersion: content.renderingVersion, MaxTurns: content.maxTurns}); err != nil {
return errx.InternalError()
}
// Added after linting: the recipient's opt-in filter tag is the one marker its owner asked for.
if tagger, ok := s.warmupRepo.(warmupFilterTagReader); ok {
tag, tagErr := tagger.RecipientWarmupFilterTag(ctx, partner.ID)
if tagErr != nil {
return errx.InternalError()
}
emailBody, content.bodyHTML = models.AppendWarmupFilterTag(emailBody, content.bodyHTML, tag)
}
// STEP 9: Generate Message-ID
messageID := generateMessageID(account.Email)
if err := s.taskRepo.UpdateTaskMessageID(ctx, taskID, messageID); err != nil {
return errx.InternalError()
}
// STEP 9.5: Generate warmup verification token
warmupToken := uuid.New()
tokenRecord := &models.WarmupToken{
Token: warmupToken,
TaskID: taskID,
SenderAccountID: account.ID,
RecipientAccountID: partner.ID,
ConversationTheme: conversationTheme,
ContentSource: contentSource,
ConversationID: conversationID,
ConversationTurn: conversationTurn,
SequenceID: content.sequenceID,
VariantID: content.variantID,
// The recipient falls back to matching on the subject when the verify
// header does not survive delivery (Graph strips it).
Subject: subject,
ExpiresAt: time.Now().Add(7 * 24 * time.Hour),
}
if err := s.warmupRepo.CreateWarmupToken(ctx, tokenRecord); err != nil {
// Without a token the partner cannot tell this is warmup, so it would
// land in their unibox as ordinary mail. Skip the send, keep the chain.
log.Warn().Err(err).Str("task_id", taskID.String()).Str("email_account_id", account.ID.String()).Msg("Failed to create warmup token; send skipped")
_ = s.taskRepo.RecordTaskFailure(ctx, taskID, "Warmup token not created", err.Error())
nextTime, scheduleErr := s.scheduler.CalculateNextWarmupTime(ctx, account.ID)
if scheduleErr != nil {
nextTime = warmupPartnerRecheckTime()
}
if createErr := s.createWarmupTask(ctx, account.ID, nextTime); createErr != nil {
log.Warn().Err(createErr).Str("task_id", taskID.String()).Str("email_account_id", account.ID.String()).Msg("Failed to reschedule warmup task after token failure")
}
executionStatus = "completed"
return nil
}
warmupTokenStr := warmupToken.String()
// STEP 10: Send warmup email to worker via Kafka
emailMsg := EmailMessage{
From: account.Email,
To: []string{partner.Email},
Subject: subject,
BodyHTML: content.bodyHTML, // only campaign copy carries HTML
BodyPlain: emailBody,
UnsubscribeURL: content.unsubscribeURL,
InReplyTo: inReplyTo,
References: references,
ThreadID: threadID,
MessageID: messageID,
IsWarmup: true,
Tracking: content.tracking, // campaign copy keeps its campaign's tracking
WarmupToken: warmupTokenStr,
}
if err := s.validateWarmupPair(ctx, account, partner, poolType, shouldReply); err != nil {
return s.closeWarmupThread(ctx, taskID, account.ID)
}
if err := s.sendAdmission(ctx, account); err != nil {
return errx.InternalError()
}
dispatch, ok := s.taskRepo.(repository.WarmupDispatchRepository)
if !ok || account.WorkerID == nil {
return errx.InternalError()
}
nonce, err := dispatch.AuthorizeWarmupDispatch(ctx, taskID, account.ID, *account.WorkerID)
if err != nil {
return errx.InternalError()
}
emailMsg.DispatchNonce = nonce.String()
if err := s.emailSender.Send(ctx, taskID, emailMsg, *account); err != nil {
if !errors.Is(err, ErrSendDispatchUnknown) && !errors.Is(err, ErrWarmupDispatchDeferred) {
result := models.SendEmailResult{TaskID: taskID, Error: &models.EmailSendError{Failure: &errx.SendFailure{Protocol: "internal", Stage: "prepare", Disposition: errx.SendRetry, Scope: "mailbox", ObservedAt: time.Now().UTC()}}, SentAt: time.Now().UTC()}
recovery := s.taskRepo.(repository.SendResultRecovery)
if applyErr := recovery.ApplySendResult(ctx, result, func(ctx context.Context) error {
return s.warmupRepo.FailWarmupSend(ctx, account.ID, taskID, result.SentAt, "Send not published", "Worker authority unavailable")
}); applyErr != nil {
return errx.InternalError()
}
}
return nil
}
// STEP 11: Update task record
taskRecord.MessageID = messageID
taskRecord.Status = "completed"
// STEP 14: Publish events
s.publishWarmupEmailSentEvent(ctx, taskRecord, account, partner, shouldReply)
// STEP 15: Calculate next warmup time and create new task
nextTime, err := s.scheduler.CalculateNextWarmupTime(ctx, account.ID)
if err != nil {
log.Warn().Err(err).Str("task_id", taskID.String()).Str("email_account_id", account.ID.String()).Msg("Failed to calculate next warmup time")
return nil
}
if err := s.createWarmupTask(ctx, account.ID, nextTime); err != nil {
log.Warn().Err(err).Str("task_id", taskID.String()).Str("email_account_id", account.ID.String()).Msg("Failed to create next warmup task")
}
executionStatus = "completed"
return nil
}
// accountInActiveCampaign reports whether the mailbox currently backs at
// least one active campaign. Failing closed on error keeps the health-check
// lane conservative — a transient DB error just lets the chain wind down if
// warmup is otherwise off.
func (s *tasksService) accountInActiveCampaign(ctx context.Context, accountID uuid.UUID) bool {
if s.campaignRepo == nil {
return false
}
count, err := s.campaignRepo.CountActiveCampaignsForAccount(ctx, accountID)
if err != nil {
return false
}
return count > 0
}
// recentPartnerWindow controls how long a partner stays excluded from
// re-selection after being used. Bumped from 24h to 72h so a small pool
// does not rotate through the same handful of mailboxes every day.
const recentPartnerWindow = 72 * time.Hour
// recentDomainWindow controls the lookback for the domain-distribution
// histogram. A week is long enough to smooth out daily randomness while
// still reflecting current behaviour.
const recentDomainWindow = 7 * 24 * time.Hour
// smallPoolWarnThreshold defines the participant count below which we log
// a warning. Tiny pools force partner reuse and create obvious patterns
// that mailbox providers can cluster on.
const smallPoolWarnThreshold = 8
// partnerDiversityWindow / partnerMaxSharedWindow set the explicit
// partner-diversity target: within partnerDiversityWindow, a sender should not
// send to the same partner more than partnerMaxSharedWindow times. Over-used
// partners are demoted to the fallback tier so warmup traffic spreads across
// many partners rather than forming a tight reciprocal pair (a closed-loop
// graph signal). This is a soft target — it only applies while enough other
// partners remain available.
const (
partnerDiversityWindow = 7 * 24 * time.Hour
partnerMaxSharedWindow = 3
)
var (
errNoWarmupPartners = errors.New("no warmup partners available")
errNoEligibleWarmupPartners = errors.New("no eligible warmup partners after health gate")
errAllPartnersExcluded = errors.New("every warmup partner is excluded by a routing rule")
)
func warmupPartnerRecheckTime() time.Time {
return time.Now().Add(time.Duration(240+rand.Intn(240)) * time.Minute)
}
// selectWarmupPartner selects a warmup partner from the pool, preferring
// partners on under-represented recipient domains to avoid concentrating
// traffic on a single provider (e.g. all-Gmail warmup loops).
func (s *tasksService) selectWarmupPartner(ctx context.Context, account Email) (*Email, error) {
poolType := s.resolveWarmupPoolType(ctx, &account)
candidates, err := s.warmupRepo.WarmupPartnerCandidates(ctx, poolType, account.ID)
if err != nil {
return nil, err
}
if len(candidates) == 0 {
return nil, errNoWarmupPartners
}
if len(candidates) < smallPoolWarnThreshold {
log.Warn().
Int("participants", len(candidates)).
Str("pool", poolType).
Str("email_account_id", account.ID.String()).
Msg("Warmup pool below diversity threshold; partner reuse likely")
}
// Customer routing rules apply to the premium pool only; a borrowed
// candidate answers to them like any other, since its address is known.
routingRules := s.warmupRoutingRules(ctx, &account, poolType)
// Every per-recipient signal derives from the address, so no candidate can
// be missing from a map and score a silent neutral default. A rule of
// weight 0 is an exclusion, so it drops the candidate here rather than
// weighting it: a weight cannot survive a pool of one (#501).
eligible := make([]models.WarmupPartnerCandidate, 0, len(candidates))
domainsByID := make(map[uuid.UUID]string, len(candidates))
hostsByID := make(map[uuid.UUID]string, len(candidates))
ruleWeight := make(map[uuid.UUID]float64, len(candidates))
starvation := make(map[uuid.UUID]float64, len(candidates))
filterJunk := make(map[uuid.UUID]float64)
arrival := make(map[uuid.UUID]float64)
poolOf := make(map[uuid.UUID]string, len(candidates))
excluded := 0
for _, c := range candidates {
weight := 1.0
if len(routingRules) > 0 {
weight = routingMultiplier(routingRules, account.Email, c.Email)
}
if weight <= 0 {
excluded++
continue
}
domain := strings.ToLower(models.EmailDomain(c.Email))
domainsByID[c.ID] = domain
hostsByID[c.ID] = partnerHost(c)
ruleWeight[c.ID] = weight
starvation[c.ID] = c.Starvation()
if junk := c.FilterJunkRate(); junk > 0 {
filterJunk[c.ID] = junk
}
if rate := c.ArrivalRate(); rate < 1 {
arrival[c.ID] = rate
}
poolOf[c.ID] = c.PoolType
eligible = append(eligible, c)
}
if len(eligible) == 0 {
// The rules are the customer's, so this is obeyed rather than worked
// around, and said out loud because nothing else surfaces it yet.
log.Warn().
Int("excluded", excluded).
Str("pool", poolType).
Str("email_account_id", account.ID.String()).
Msg("warmup: every available partner is excluded by a routing rule; no warmup mail this tick")
return nil, errAllPartnersExcluded
}
candidates = eligible
// This sender's recent record per recipient mail host. Best-effort: a lookup
// error leaves the map empty and weighting degrades to domain diversity.
placementByHost, placeErr := s.warmupRepo.SenderPlacementByHost(ctx, account.ID, time.Now().Add(-hostPlacementWindow))
if placeErr != nil {
placementByHost = nil
}
recentPartnerSet := map[uuid.UUID]struct{}{}
recentPartnerIDs, err := s.warmupRepo.GetRecentlyUsedPartners(ctx, account.ID, time.Now().Add(-recentPartnerWindow))
if err == nil {
for _, pid := range recentPartnerIDs {
recentPartnerSet[pid] = struct{}{}
}
}
todayPartnerSet := map[uuid.UUID]struct{}{}
todayPartnerIDs, err := s.warmupRepo.GetRecentlyUsedPartners(ctx, account.ID, time.Now().Truncate(24*time.Hour))
if err == nil {
for _, pid := range todayPartnerIDs {
todayPartnerSet[pid] = struct{}{}
}
}
domainCounts, err := s.warmupRepo.GetRecentPartnerDomainCounts(ctx, account.ID, time.Now().Add(-recentDomainWindow))
if err != nil {
domainCounts = nil
}
// Explicit partner-diversity target: how many times each partner has been
// used in the diversity window, so partners the sender already leans on
// heavily get demoted out of the preferred tier.
partnerCounts, err := s.warmupRepo.GetRecentPartnerCounts(ctx, account.ID, time.Now().Add(-partnerDiversityWindow))
if err != nil {
partnerCounts = nil
}
// Rank outside workspaces first while retaining siblings as a self-host fallback.
var buckets [4][]uuid.UUID
foreign, own := 0, 0
for _, c := range candidates {
if _, usedToday := todayPartnerSet[c.ID]; usedToday {
continue
}
rank := 0
if sameOrganization(account.OrganizationID, c.OrganizationID) {
rank += 2
own++
} else {
foreign++
}
if _, recentlyUsed := recentPartnerSet[c.ID]; recentlyUsed || partnerCounts[c.ID] >= partnerMaxSharedWindow {
rank++
}
buckets[rank] = append(buckets[rank], c.ID)
}
if foreign == 0 && own > 0 {
// Surface intentional sibling fallback without warning on local-only pools.
log.Info().
Int("participants", len(candidates)).
Str("pool", poolType).
Str("email_account_id", account.ID.String()).
Msg("warmup: no partner outside this workspace is available; pairing within the organization")
}
sig := partnerSignals{
domainsByID: domainsByID,
domainCounts: domainCounts,
hostsByID: hostsByID,
placementByHost: placementByHost,
ruleWeight: ruleWeight,
starvation: starvation,
filterJunk: filterJunk,
arrival: arrival,
warmWhereJunked: account.ConsumerMailbox(),
}
// A pick that fails the gate is dropped and the draw repeats; an emptied
// bucket falls through to the next, so a stale own-tier row cannot hide a
// healthy borrowed one. The gate re-evaluates a just-unblocked recipient
// (probation), matching the CLAUDE.md re-entry policy on the recipient surface.
// Every failure removes a candidate, so the loop ends by exhaustion.
var available []uuid.UUID
next := 0
for {
for len(available) == 0 && next < len(buckets) {
available = buckets[next]
next++
}
if len(available) == 0 {
break
}
partnerID := pickWeightedPartner(available, sig)
if s.warmupHealth != nil {
// Pinned to the pool it was drawn from, so a row that moved pools
// between the read and the gate is not accepted elsewhere (#495).
gatePool := poolOf[partnerID]
if gatePool == "" {
gatePool = poolType
}
if ok, _, _ := s.warmupHealth.CanParticipate(ctx, partnerID, gatePool); !ok {
available = removePartnerID(available, partnerID)
continue
}
}
partner, err := s.emailRepo.GetByID(ctx, partnerID)
if err != nil {
return nil, err
}
return partner, nil
}
return nil, errNoEligibleWarmupPartners
}
// partnerHost is who runs a candidate's mail, the key its sender's placement
// record is kept under. Never the domain: a custom domain can be on any host.
func partnerHost(c models.WarmupPartnerCandidate) string {
return string(mailhost.ForMailbox(c.MailHost, c.Provider, c.Email))
}
// sameOrganization treats unknown ownership as outside rather than guessing.
func sameOrganization(a, b *uuid.UUID) bool {
return a != nil && b != nil && *a == *b
}
// removePartnerID returns ids without the first occurrence of target. Used to
// drop a partner that failed the health gate before re-picking.
func removePartnerID(ids []uuid.UUID, target uuid.UUID) []uuid.UUID {
out := make([]uuid.UUID, 0, len(ids))
for _, id := range ids {
if id != target {
out = append(out, id)
}
}
return out
}
const (
// weight *= 1/(1 + k*rate). Never an exclusion: a sender that stops mailing
// a host cannot discover it recovered there.
hostPlacementPenaltyK = 4.0
// weight *= 1 + k*rate at Google, Microsoft and Yahoo for a free Gmail or
// Outlook.com sender: its mail is rescued at the recovery rate, so warming
// where it is junked is what repairs its standing there.
junkedHostBoostK = 1.0
// Below this many verified deliveries a host's rate is noise, not a pattern.
hostPlacementMinDelivered = 5
hostPlacementWindow = 7 * 24 * time.Hour
// weight *= 1 + k*starvation. An inbox that has received nothing back for
// what it sent is drawn this many times more often than one in balance,
// and the boost fades as the pool pays it back, so traffic settles near
// parity instead of overshooting.
reciprocityBoostK = 3.0
// weight *= 1/(1 + k*junk). A small host whose own filter junks what the
// pool sends it earns no sender anything, so it is drawn less, never excluded.
recipientFilterPenaltyK = 3.0
)
// partnerSignals are the per-pick inputs to partner weighting.
type partnerSignals struct {
domainsByID map[uuid.UUID]string
domainCounts map[string]int
// Keyed by who RUNS the recipient's mail (mailhost.ForMailbox), so a custom
// domain on Workspace and one on a small host are told apart.
hostsByID map[uuid.UUID]string
placementByHost map[string]repository.HostPlacementStat
// ruleWeight is the customer's routing multiplier per candidate. An
// exclusion never appears here: it removed the candidate (#501).
ruleWeight map[uuid.UUID]float64
// starvation is how far behind each candidate is on what it sent (0..1).
starvation map[uuid.UUID]float64
// filterJunk is how much warmup mail a small-host candidate's own filter
// junks (0..1); absent when nothing is known or the host is a major one.
filterJunk map[uuid.UUID]float64
// arrival is the share of the pool's mail to a candidate that verifiably
// arrived; absent below the sample or when nothing was lost.
arrival map[uuid.UUID]float64
// warmWhereJunked turns the major-provider host penalty into a boost.
warmWhereJunked bool
}
// arrivalWeight draws a recipient in proportion to the chance the mail
// arrives. 1.0 when nothing is known.
func (sig partnerSignals) arrivalWeight(partnerID uuid.UUID) float64 {
if rate, ok := sig.arrival[partnerID]; ok {
return rate
}
return 1.0
}
// filterPenalty draws a recipient whose own filter junks warmup mail less
// often. 1.0 when nothing is known.
func (sig partnerSignals) filterPenalty(partnerID uuid.UUID) float64 {
return 1.0 / (1.0 + recipientFilterPenaltyK*sig.filterJunk[partnerID])
}
// reciprocityBoost favours the inbox that is owed the most. 1.0 for one in
// balance, and on every candidate the signal is missing for.
func (sig partnerSignals) reciprocityBoost(partnerID uuid.UUID) float64 {
starved, ok := sig.starvation[partnerID]
if !ok || starved <= 0 {
return 1.0
}
if starved > 1 {
starved = 1
}
return 1.0 + reciprocityBoostK*starved
}
// hostPenalty weights sending to one mail host by how this sender has
// recently landed there. 1.0 on too small a sample, and on every error path.
func (sig partnerSignals) hostPenalty(partnerID uuid.UUID) float64 {
host, ok := sig.hostsByID[partnerID]
if !ok || len(sig.placementByHost) == 0 {
return 1.0
}
stat, ok := sig.placementByHost[host]
if !ok || stat.Delivered < hostPlacementMinDelivered {
return 1.0
}
if sig.warmWhereJunked && models.WarmupRecipientGroup(host, "") != models.WarmupRecipientOther {
return 1.0 + junkedHostBoostK*stat.Rate()
}
return 1.0 / (1.0 + hostPlacementPenaltyK*stat.Rate())
}
// pickWeightedPartner picks a partner ID using a composite weight:
// - balanced host coverage with inverse-frequency domains within each host
// - this sender's recent junk rate at the partner's mail host (feedback)
// - how far behind the partner is on what it sent (reciprocity)
// - how much a small-host partner's own filter junks (recipient quality)
// - how much of the pool's mail to the partner arrives (reachability)
// - customer-defined routing rule multipliers (preference)
//
// Every candidate here is one the customer allows; an excluded pair was
// dropped before the draw. Falls back to uniform when no signals are available.
func pickWeightedPartner(candidates []uuid.UUID, sig partnerSignals) uuid.UUID {
if len(candidates) == 1 {
return candidates[0]
}
if len(sig.hostsByID) == 0 && len(sig.domainsByID) == 0 && len(sig.ruleWeight) == 0 && len(sig.placementByHost) == 0 && len(sig.starvation) == 0 && len(sig.filterJunk) == 0 && len(sig.arrival) == 0 {
return candidates[rand.Intn(len(candidates))]
}
weights := make([]float64, len(candidates))
hostTotals := make(map[string]float64)
groupHosts := make(map[string]map[string]struct{})
// Normalize only the coverage base (equal per provider group, then per host
// within it); quality and customer rules still change the shares.
for _, id := range candidates {
if host := sig.hostsByID[id]; host != "" {
hostTotals[host] += 1.0 / float64(1+sig.domainCounts[sig.domainsByID[id]])
group := models.WarmupRecipientGroup(host, "")
if groupHosts[group] == nil {
groupHosts[group] = make(map[string]struct{})
}
groupHosts[group][host] = struct{}{}
}
}
var total float64
for i, id := range candidates {
domain := sig.domainsByID[id]
// Diversity base weight.
w := 1.0 / float64(1+sig.domainCounts[domain])
host := sig.hostsByID[id]
if hostTotal := hostTotals[host]; hostTotal > 0 {
w /= hostTotal * float64(len(groupHosts[models.WarmupRecipientGroup(host, "")]))
}
// Per-host placement feedback.
w *= sig.hostPenalty(id)
// The pool's debt to this inbox.
w *= sig.reciprocityBoost(id)
// A recipient whose own filter junks what it is sent.
w *= sig.filterPenalty(id)
// A recipient the pool's mail does not reliably reach.
w *= sig.arrivalWeight(id)
// Routing rule multiplier (premium pool only, when configured).
if rw, ok := sig.ruleWeight[id]; ok {
w *= rw
}
weights[i] = w
total += w
}
if total <= 0 {
// Nothing an exclusion can reach: those candidates are already gone.
// Only an underflow of the positive terms lands here, so draw uniformly.
return candidates[rand.Intn(len(candidates))]
}
r := rand.Float64() * total
var cum float64
for i, w := range weights {
cum += w
if r <= cum {
return candidates[i]
}
}
return candidates[len(candidates)-1]
}
// routingMultiplier returns the weight multiplier from the first matching
// rule in priority order (lowest priority value wins). 1.0 when no rule
// matches — neutral, so unmatched pairs neither preferred nor penalized.
func routingMultiplier(rules []models.WarmupRoutingRule, senderEmail, recipientEmail string) float64 {
for i := range rules {
if rules[i].Matches(senderEmail, recipientEmail) {
return rules[i].Weight
}
}
return 1.0
}
func (s *tasksService) resolveWarmupPoolType(ctx context.Context, account *Email) string {
if account == nil {
return "premium"
}
return warmupPoolTypeFor(account.OrganizationID, account.WarmupPoolType,
func(org uuid.UUID) bool { return s.orgSuspendedOrRestricted(ctx, org) },
func(org uuid.UUID) (bool, bool) { return s.hasPremiumWarmup(ctx, org) })
}
// warmupPoolTypeFor is the pool a mailbox belongs in, given lookups for its organization.
func warmupPoolTypeFor(orgID *uuid.UUID, stored string, restricted func(uuid.UUID) bool, premium func(uuid.UUID) (paid, known bool)) string {
// No organization means no entitlement to check, so the mailbox gets the
// lower-trust pool rather than defaulting into the paid one.
if orgID == nil {
return "free"
}
// A restricted organization leaves the paid pool whatever it pays. Checked
// before the stored tier, which is never empty and would short-circuit it.
if restricted(*orgID) {
return "free"
}
if stored != "" {
return stored
}
if paid, known := premium(*orgID); known && !paid {
return "free"
}
return "premium"
}
// hasPremiumWarmup reports the organization's paid warmup, and whether it could be read.
func (s *tasksService) hasPremiumWarmup(ctx context.Context, orgID uuid.UUID) (bool, bool) {
if s.featureGate == nil {
return false, false
}
isPaid, xerr := s.featureGate.HasPremiumWarmup(ctx, orgID)
return isPaid, xerr == nil
}
func (s *tasksService) scheduleWarmupRecovery(ctx context.Context, accountID uuid.UUID, poolType string) {
if s.warmupRepo == nil {
return
}
health, err := s.warmupRepo.GetParticipantHealth(ctx, accountID, poolType)
if err != nil || health == nil || health.BlockedUntil == nil {
return
}
if !health.BlockedUntil.After(time.Now()) {
return
}
if err := s.createWarmupTask(ctx, accountID, *health.BlockedUntil); err != nil {
log.Warn().Err(err).Str("email_account_id", accountID.String()).Str("pool_type", poolType).Msg("Failed to create warmup recovery task")
}
}
// EnsureWarmupScheduled makes sure the mailbox has a pending warmup task so
// its warmup / health-check chain is running. Safe to call repeatedly: the
// per-mailbox advisory lock in CreateWarmupTaskWithLock means a duplicate
// pending task is never created. Returns ErrWarmupNotEnabled (benign) when
// the mailbox is neither warming nor backing a live campaign.
func (s *tasksService) EnsureWarmupScheduled(ctx context.Context, accountID uuid.UUID) error {
account, xerr := s.emailRepo.GetByID(ctx, accountID)
if xerr != nil {
return xerr
}
if account == nil {
return errors.New("mailbox unavailable")
}
if s.warmupHealth != nil {
pool := s.resolveWarmupPoolType(ctx, account)
_, xerr := s.warmupHealth.MovePoolMembership(ctx, accountID, pool)
if xerr != nil {
return xerr
}
health, err := s.warmupRepo.GetParticipantHealth(ctx, accountID, pool)
if err != nil {
return err
}
if health == nil {
if xerr := s.warmupHealth.EnsurePoolMembershipWithRole(ctx, accountID, pool, "sender_receiver"); xerr != nil {
return xerr
}
}
}
nextTime, err := s.scheduler.CalculateNextWarmupTime(ctx, accountID)
if err != nil {
return err
}
return s.createWarmupTask(ctx, accountID, nextTime)
}
// ReplaceWarmupSchedule re-places a mailbox's parked warmup wakeup under its
// current hours, so a schedule change takes effect now and not after a slot
// computed under the old one. A protection hold's recovery wakeup is left alone.
func (s *tasksService) ReplaceWarmupSchedule(ctx context.Context, accountID uuid.UUID) error {
mover, ok := s.taskRepo.(interface {
ReplaceParkedWarmupTask(context.Context, uuid.UUID, time.Time) (bool, error)
})
if !ok {
return nil
}
account, xerr := s.emailRepo.GetByID(ctx, accountID)
if xerr != nil {
return xerr
}
if account == nil || (s.cloudLink != nil && s.cloudLink.IsEnrolled(ctx, account.ID)) {
return nil
}
if s.warmupRepo != nil {
eligible, err := s.warmupRepo.IsPoolEligible(ctx, accountID, s.resolveWarmupPoolType(ctx, account), true)
if err != nil || !eligible {
return err
}
}
nextTime, err := s.scheduler.CalculateNextWarmupTime(ctx, accountID)
if errors.Is(err, scheduler.ErrWarmupNotEnabled) {
return nil
}
if err != nil {
return err
}
_, err = mover.ReplaceParkedWarmupTask(ctx, accountID, nextTime)
return err
}
func (s *tasksService) ReplaceWorkspaceWarmupSchedules(ctx context.Context, orgID uuid.UUID) {
lister, ok := s.emailRepo.(interface {
WarmingOnWorkspaceClock(context.Context, uuid.UUID) ([]uuid.UUID, error)
})
if !ok {
return
}
ids, err := lister.WarmingOnWorkspaceClock(ctx, orgID)
if err != nil {
return
}
for _, id := range ids {
if ctx.Err() != nil {
return
}
_ = s.ReplaceWarmupSchedule(ctx, id)
}
}
// createWarmupTask creates a new warmup task in GCP Cloud Tasks
func (s *tasksService) createWarmupTask(ctx context.Context, accountID uuid.UUID, scheduleTime time.Time) error {
// Create task in database
newTaskID := uuid.New()
newTask := &Task{
ID: newTaskID,
TaskType: "warmup",
EmailAccountID: accountID,
Status: "pending",
ScheduledAt: &scheduleTime,
}
// Create warmup task entry
warmupTask := &WarmupTask{
TaskID: newTaskID,
}
created, err := s.taskRepo.CreateWarmupTaskWithLock(ctx, newTask, warmupTask)
if err != nil {
return err
}
if !created {
return nil
}
// Create GCP Cloud Task
processTask := &proto.ProcessTask{
TaskId: newTaskID.String(),
}
cloudTaskName, err := s.tasksClient.CreateTask(ctx, processTask, scheduleTime)
if err != nil {
return err
}
// Update task with cloud task name
lineage, ok := s.taskRepo.(repository.WarmupLineageRepository)
if !ok {
return errors.New("warmup queue authority unavailable")
}
acked, err := lineage.AckWarmupQueue(ctx, repository.WarmupQueueRevision{TaskID: newTaskID, At: scheduleTime, Revision: 1}, cloudTaskName)
if err != nil {
return err
}
if !acked {
_ = s.tasksClient.DeleteTask(ctx, cloudTaskName)
}
return nil
}
// publishWarmupEmailSentEvent publishes warmup email sent event (realtime) and
// an opt-in firehose webhook (warmup.email_sent).
func (s *tasksService) publishWarmupEmailSentEvent(ctx context.Context, task *Task, account *Email, partner *Email, isReply bool) {
if s.eventsPublisher != nil {
if err := s.eventsPublisher.PublishWarmupEmailSent(ctx, task, account, partner, isReply); err != nil {
log.Warn().Err(err).Str("task_id", task.ID.String()).Msg("Failed to publish warmup email sent event")
}
}
if s.streamingPublisher != nil && account != nil && account.OrganizationID != nil {
s.streamingPublisher.PublishWarmupActivity(ctx, account.OrganizationID.String(), account.UserID, account.ID.String(), account.Email)
}
if s.advanced != nil && account != nil && account.OrganizationID != nil {
s.advanced.EmitCampaignEvent(ctx, *account.OrganizationID, models.WebhookEventWarmupEmailSent, map[string]any{
"email_account_id": account.ID.String(),
"is_reply": isReply,
})
}
}
// staticConvID derives a stable id for a static-library conversation from its content.
func staticConvID(theme, description string) uuid.UUID {
return uuid.NewSHA1(uuid.NameSpaceOID, []byte("warmup-static:"+theme+"|"+description))
}
// warmupConversations is the hand-written library that keeps warmup sending when no AI bank is configured.
func warmupConversations() []Conversation {
conversations := make([]Conversation, 0, len(warmupLibraryWork)+len(warmupLibraryLife)+len(warmupLibraryCraft))
conversations = append(conversations, warmupLibraryWork...)
conversations = append(conversations, warmupLibraryLife...)
conversations = append(conversations, warmupLibraryCraft...)
for i := range conversations {
conversations[i].ID = staticConvID(conversations[i].Theme, conversations[i].Description)
conversations[i].Version = generation.WarmupScenarioVersion
}
return conversations
}
// directedWarmupPartner resolves a task's explicit reply-back target. Nil for
// an ordinary task, or when the target is no longer eligible, in which case the
// caller draws a partner as usual. A reply may cross tiers, because the other
// side started the thread by borrowing or by returning a visit; a restricted
// workspace still may not answer into a paying inbox.
func (s *tasksService) directedWarmupPartner(ctx context.Context, taskID uuid.UUID, account *Email, poolType string) *Email {
warmupTask, err := s.taskRepo.GetWarmupTask(ctx, taskID)
if err != nil || warmupTask == nil || warmupTask.TargetAccountID == nil {
return nil
}
return s.replyPartner(ctx, *warmupTask.TargetAccountID, account, poolType)
}
// replyPartner is target when this mailbox may answer it now, or nil.
func (s *tasksService) replyPartner(ctx context.Context, target uuid.UUID, account *Email, poolType string) *Email {
if s.warmupHealth != nil {
ok, reason, herr := s.warmupHealth.CanParticipate(ctx, target, poolType)
if herr != nil {
return nil
}
if !ok && reason == "not_in_pool" && s.replyMayCrossTiers(ctx, account, poolType) {
if ok, _, herr = s.warmupHealth.CanParticipate(ctx, target, otherPoolType(poolType)); herr != nil {
return nil
}
}
if !ok {
return nil
}
}
partner, xerr := s.emailRepo.GetByID(ctx, target)
if xerr != nil || partner == nil {
return nil
}
// An exclusion is about who this mailbox may mail, so it holds for a reply
// too; the thread exists because the other side started it (#501).
if rules := s.warmupRoutingRules(ctx, account, poolType); len(rules) > 0 {
if routingMultiplier(rules, account.Email, partner.Email) <= 0 {
return nil
}
}
return partner
}
// warmupRoutingRules are the sender organization's rules, premium pool only:
// a free-tier mailbox has no provider-shape preferences to express.
func (s *tasksService) warmupRoutingRules(ctx context.Context, account *Email, poolType string) []models.WarmupRoutingRule {
if poolType != "premium" || s.warmupRoutingRepo == nil || account == nil || account.OrganizationID == nil {
return nil
}
rules, err := s.warmupRoutingRepo.ListForOrganization(ctx, *account.OrganizationID)
if err != nil {
return nil
}
return rules
}
// replyMayCrossTiers: a paid mailbox may always answer a borrowed partner; a
// free one may answer a paid mailbox only while it is healthy, from a
// workspace whose standing is known to be good. Unlike the other risk reads
// this fails closed: the downside is one unanswered warmup thread, not free
// mail in a paying inbox.
func (s *tasksService) replyMayCrossTiers(ctx context.Context, account *Email, poolType string) bool {
if poolType == "premium" {
return true
}
if s.orgRiskRepo == nil || account == nil || account.OrganizationID == nil {
return false
}
states, err := s.orgRiskRepo.GetOrgRiskStates(ctx, []uuid.UUID{*account.OrganizationID})
if err != nil {
return false
}
state, known := states[*account.OrganizationID]
if !known || state.ForcesFreeWarmupPool() || s.warmupRepo == nil {
return false
}
// The same bar the return-visit cohort holds the sender to, or the dispatch
// check defers the reply until its token expires.
own, err := s.warmupRepo.GetParticipantHealth(ctx, account.ID, poolType)
return err == nil && own != nil && own.HealthState == models.WarmupHealthHealthy && own.BlockedAt == nil
}
// orgSuspendedOrRestricted reports whether the workspace's posture bars the
// paid warmup pool. Fails open.
func (s *tasksService) orgSuspendedOrRestricted(ctx context.Context, orgID uuid.UUID) bool {
if s.orgRiskRepo == nil {
return false
}
states, err := s.orgRiskRepo.GetOrgRiskStates(ctx, []uuid.UUID{orgID})
if err != nil {
return false
}
return states[orgID].ForcesFreeWarmupPool()
}
// orgBlocksSending reports whether the workspace is suspended. Fails open, for
// the same reason every other risk read does.
func (s *tasksService) orgBlocksSending(ctx context.Context, orgID *uuid.UUID) bool {
if s.orgRiskRepo == nil || orgID == nil {
return false
}
states, err := s.orgRiskRepo.GetOrgRiskStates(ctx, []uuid.UUID{*orgID})
if err != nil {
return false
}
return states[*orgID].BlocksSending()
}
// otherPoolType is the tier on the far side of a borrowed exchange.
func otherPoolType(poolType string) string {
if poolType == "premium" {
return "free"
}
return "premium"
}
// warmupFilterTagReader is implemented by the Postgres warmup repository.
type warmupFilterTagReader interface {
RecipientWarmupFilterTag(ctx context.Context, accountID uuid.UUID) (string, error)
}