diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b08b1dfa..7ec3ed782 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -464,9 +464,9 @@ jobs: run: | set -o pipefail go test -race ./internal/repository -run '^TestLive(Monitoring|LastWarmupSendFailure|PersistentMailboxLoadingFailure|DispatchRetry|LineageQueueRevisions|ReplyShare|AdminPlatformOverview|WarmupRecoversOldPreparation|WarmupPreparationRecoveryRetains|WarmupFilingRequiresAcknowledgement|WarmupFilingAuthority|WarmupTimeWindow|WarmupInvalidWindow|StoppedRecipientIsNeverAddressed)' -count=1 - go test -race -v ./internal/tasks -run '^Test(LiveWarmup(SafetyHoldReschedules|Recovery|UnknownRetirement|TaskDoesNotApplyResults|DispatchDefersInvalidHours|ReplyShare)|WarmupSenderDistinguishes)' -count=1 | tee warmup-recovery.log + go test -race -v ./internal/tasks -run '^Test(LiveWarmup(SafetyHoldReschedules|Recovery|UnknownRetirement|TaskDoesNotApplyResults|DispatchDefersInvalidHours|ReplyShare|ReplySubject)|WarmupSenderDistinguishes)' -count=1 | tee warmup-recovery.log if grep -q -- '--- SKIP: TestLiveWarmupUnknownRetirement' warmup-recovery.log; then exit 1; fi - for test in TestLiveWarmupUnknownRetirementPreservesEvidenceAndFencesReplayOnce TestLiveWarmupUnknownRetirementAllowsDistinctNewSendAndKeepsCooldown TestLiveWarmupUnknownRetirementAndDefinitiveResultSerialize TestLiveWarmupUnknownRetirementSkipsProtectedRowsWithoutStarvingLaterMailboxes TestLiveWarmupUnknownRetirementReconcilerSeedsOnceAndClearsActiveWarnings; do + for test in TestLiveWarmupUnknownRetirementPreservesEvidenceAndFencesReplayOnce TestLiveWarmupUnknownRetirementAllowsDistinctNewSendAndKeepsCooldown TestLiveWarmupUnknownRetirementAndDefinitiveResultSerialize TestLiveWarmupUnknownRetirementSkipsProtectedRowsWithoutStarvingLaterMailboxes TestLiveWarmupUnknownRetirementReconcilerSeedsOnceAndClearsActiveWarnings TestLiveWarmupReplySubjectReadsAsAReply; do grep -q -- "--- PASS: $test " warmup-recovery.log || exit 1 done go test -race -v ./internal/infrastructure/db -run '^TestLiveWarmupUnknownRetirementUpgrade' -count=1 | tee warmup-retirement-upgrade.log diff --git a/docs/content/docs/guides/warmup.mdx b/docs/content/docs/guides/warmup.mdx index 5d589a181..72243e3a2 100644 --- a/docs/content/docs/guides/warmup.mdx +++ b/docs/content/docs/guides/warmup.mdx @@ -108,7 +108,7 @@ Recovery does not bypass paused participation, provider limits or deliverability ### Where the content comes from -Warmup mail reads like ordinary correspondence: a short, specific subject, an opening and up to five alternating replies between two people who know each other loosely. It never says it is automated, generated or a warmup, and any turn that does is refused before it sends. Topics, tone, structure and length vary from thread to thread, and replies keep the same subject and the real sender names. +Warmup mail reads like ordinary correspondence: a short, specific subject, an opening and up to five alternating replies between two people who know each other loosely. It never says it is automated, generated or a warmup, and any turn that does is refused before it sends. Topics, tone, structure and length vary from thread to thread, and replies answer under the thread's subject with "Re:" in front, as a mail client would, with the real sender names. Every instance ships with a hand-written library of about seventy threads, so warmup works with no AI provider at all. With an OpenAI key configured and `AI_PROVIDER` unset or set to `openai`, background batches add generated threads. Each passes structural and wording checks, then a separate review of the complete thread (TypeSafe when configured, otherwise OpenAI) before it can be used. An unavailable, incomplete or uncertain review never approves content. diff --git a/internal/app/aitools/tools_inbox_send.go b/internal/app/aitools/tools_inbox_send.go index adea6aa2e..5bda75369 100644 --- a/internal/app/aitools/tools_inbox_send.go +++ b/internal/app/aitools/tools_inbox_send.go @@ -90,7 +90,7 @@ func (d Deps) replyTarget(ctx context.Context, inv Invocation, threadID string) if to == "" { return "", "", uuid.Nil, ErrInvalidArgs } - return to, replySubject(latest.Subject), latest.ID, nil + return to, mailhdr.ReplySubject(latest.Subject), latest.ID, nil } // previewSendReply resolves the reply's recipient and sender, and pins both in the args the member approves. diff --git a/internal/app/aitools/tools_unibox.go b/internal/app/aitools/tools_unibox.go index bea44fdcc..ecd9eaefb 100644 --- a/internal/app/aitools/tools_unibox.go +++ b/internal/app/aitools/tools_unibox.go @@ -9,6 +9,7 @@ import ( "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/pkg/generation" + "github.com/warmbly/warmbly/internal/pkg/mailhdr" ) func (d Deps) registerUniboxTools(r *Registry) { @@ -213,7 +214,7 @@ func (d Deps) draftReply(ctx context.Context, inv Invocation, args json.RawMessa if len(latest.FromAddr) > 0 { to = latest.FromAddr[0] } - subject = replySubject(latest.Subject) + subject = mailhdr.ReplySubject(latest.Subject) } return jsonResult(map[string]any{ "draft": map[string]any{ @@ -225,15 +226,3 @@ func (d Deps) draftReply(ctx context.Context, inv Invocation, args json.RawMessa "note": "This is a draft only. It has not been sent; the user must review and send it.", }) } - -// replySubject prefixes "Re: " unless already present. -func replySubject(s string) string { - s = strings.TrimSpace(s) - if s == "" { - return "Re:" - } - if strings.HasPrefix(strings.ToLower(s), "re:") { - return s - } - return "Re: " + s -} diff --git a/internal/app/inboxagent/service.go b/internal/app/inboxagent/service.go index a8ccb70db..2542ecc4f 100644 --- a/internal/app/inboxagent/service.go +++ b/internal/app/inboxagent/service.go @@ -21,6 +21,7 @@ import ( "github.com/warmbly/warmbly/internal/errx" "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/pkg/generation" + "github.com/warmbly/warmbly/internal/pkg/mailhdr" "github.com/warmbly/warmbly/internal/repository" ) @@ -227,7 +228,7 @@ func (s *service) draft(ctx context.Context, r models.InboxAgentReply) { ThreadID: r.ThreadID, SourceMessageID: src, ToAddr: r.Counterpart, - Subject: replySubject(r.Subject), + Subject: mailhdr.ReplySubject(r.Subject), InReplyTo: r.InReplyTo, Body: strings.TrimSpace(res.Text), IntentClass: r.IntentClass, @@ -331,15 +332,3 @@ func (s *service) contactGrounding(ctx context.Context, contactID uuid.UUID) str } return strings.TrimSpace(b.String()) } - -// replySubject prefixes "Re: " once, mirroring a normal reply subject. -func replySubject(subject string) string { - s := strings.TrimSpace(subject) - if s == "" { - return "Re:" - } - if strings.HasPrefix(strings.ToLower(s), "re:") { - return s - } - return "Re: " + s -} diff --git a/internal/pkg/mailhdr/mailhdr.go b/internal/pkg/mailhdr/mailhdr.go index 9bfe4f512..73a4076af 100644 --- a/internal/pkg/mailhdr/mailhdr.go +++ b/internal/pkg/mailhdr/mailhdr.go @@ -26,6 +26,18 @@ func Subject(s string) string { return mime.QEncoding.Encode("utf-8", s) } +// ReplySubject is the subject a reply carries: "Re: " once, never stacked. +func ReplySubject(subject string) string { + s := strings.TrimSpace(subject) + if s == "" { + return "Re:" + } + if strings.HasPrefix(strings.ToLower(s), "re:") { + return s + } + return "Re: " + s +} + // Address encodes one address entry, which may be bare ("a@b.com") or carry a // display name ("Ana Rodríguez "). Anything that is not exactly one // address encodes to "", so it never reaches a header. diff --git a/internal/pkg/mailhdr/mailhdr_test.go b/internal/pkg/mailhdr/mailhdr_test.go index 5286bf6e7..28545b338 100644 --- a/internal/pkg/mailhdr/mailhdr_test.go +++ b/internal/pkg/mailhdr/mailhdr_test.go @@ -108,3 +108,17 @@ func TestSubjectRoundTrip(t *testing.T) { t.Fatalf("round trip lost content: %q", got) } } + +func TestReplySubjectPrefixesOnce(t *testing.T) { + for in, want := range map[string]string{ + "Coffee or tea at work": "Re: Coffee or tea at work", + " Podcast habits ": "Re: Podcast habits", + "Re: Podcast habits": "Re: Podcast habits", + "RE: Podcast habits": "RE: Podcast habits", + "": "Re:", + } { + if got := ReplySubject(in); got != want { + t.Errorf("ReplySubject(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/repository/pg_ai_draft.go b/internal/repository/pg_ai_draft.go index 4e7c52114..ed7380807 100644 --- a/internal/repository/pg_ai_draft.go +++ b/internal/repository/pg_ai_draft.go @@ -326,7 +326,7 @@ func (r *aiDraftRepository) upsertPendingDraftOnce(ctx context.Context, orgID uu contact_id = $10, campaign_id = $11, email_account_id = $12, owner_user_id = $13, updated_at = NOW() WHERE id = $1 AND organization_id = $14 AND status = 'pending' RETURNING `+aiDraftCols, cur.ID, p.Body, p.Model, p.IntentClass, p.Confidence, - in.id, in.sender, draftReplySubject(in.subject), in.messageID, contactID, campaignID, in.emailID, in.ownerID, orgID), &d); err != nil { + in.id, in.sender, mailhdr.ReplySubject(in.subject), in.messageID, contactID, campaignID, in.emailID, in.ownerID, orgID), &d); err != nil { return nil, false, err } return &d, false, tx.Commit(ctx) @@ -340,7 +340,7 @@ func (r *aiDraftRepository) upsertPendingDraftOnce(ctx context.Context, orgID uu ThreadID: p.ThreadID, SourceMessageID: &in.id, ToAddr: in.sender, - Subject: draftReplySubject(in.subject), + Subject: mailhdr.ReplySubject(in.subject), InReplyTo: in.messageID, Body: p.Body, IntentClass: p.IntentClass, @@ -457,15 +457,3 @@ func campaignForDraft(ctx context.Context, tx pgx.Tx, orgID, contactID uuid.UUID } return &id } - -// draftReplySubject prefixes "Re: " once, as a normal reply subject. -func draftReplySubject(subject string) string { - s := strings.TrimSpace(subject) - if s == "" { - return "Re:" - } - if strings.HasPrefix(strings.ToLower(s), "re:") { - return s - } - return "Re: " + s -} diff --git a/internal/tasks/email_task.go b/internal/tasks/email_task.go index 7caed4681..27dbf6fd1 100644 --- a/internal/tasks/email_task.go +++ b/internal/tasks/email_task.go @@ -14,6 +14,7 @@ import ( "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/observability/errs" "github.com/warmbly/warmbly/internal/pkg/generation" + "github.com/warmbly/warmbly/internal/pkg/mailhdr" "github.com/warmbly/warmbly/internal/pkg/mailhost" "github.com/warmbly/warmbly/internal/repository" "github.com/warmbly/warmbly/internal/scheduler" @@ -386,6 +387,12 @@ func (s *tasksService) HandleEmailTask(task *proto.ProcessTask) *errx.Error { emailBody, content.bodyHTML = models.AppendWarmupFilterTag(emailBody, content.bodyHTML, tag) } + // Lineage keeps the thread's subject; what goes out reads as a reply. + wireSubject := subject + if shouldReply { + wireSubject = mailhdr.ReplySubject(subject) + } + // STEP 9: Generate Message-ID messageID := generateMessageID(account.Email) if err := s.taskRepo.UpdateTaskMessageID(ctx, taskID, messageID); err != nil { @@ -407,7 +414,7 @@ func (s *tasksService) HandleEmailTask(task *proto.ProcessTask) *errx.Error { VariantID: content.variantID, // The recipient falls back to matching on the subject when the verify // header does not survive delivery (Graph strips it). - Subject: subject, + Subject: wireSubject, ExpiresAt: time.Now().Add(7 * 24 * time.Hour), } if err := s.warmupRepo.CreateWarmupToken(ctx, tokenRecord); err != nil { @@ -431,7 +438,7 @@ func (s *tasksService) HandleEmailTask(task *proto.ProcessTask) *errx.Error { emailMsg := EmailMessage{ From: account.Email, To: []string{partner.Email}, - Subject: subject, + Subject: wireSubject, BodyHTML: content.bodyHTML, // only campaign copy carries HTML BodyPlain: emailBody, UnsubscribeURL: content.unsubscribeURL, diff --git a/internal/tasks/warmup_reply_subject_live_test.go b/internal/tasks/warmup_reply_subject_live_test.go new file mode 100644 index 000000000..9aec3afcd --- /dev/null +++ b/internal/tasks/warmup_reply_subject_live_test.go @@ -0,0 +1,101 @@ +package tasks + +import ( + "context" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + warmupapp "github.com/warmbly/warmbly/internal/app/warmup" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/pkg/generation" + "github.com/warmbly/warmbly/internal/repository" + "github.com/warmbly/warmbly/internal/tasks/proto" +) + +type capturingWarmupSender struct { + preparationDeferringSender + pool *pgxpool.Pool + sent []EmailMessage + tokenSubject string + lineageSubject string +} + +// Send reads the token before deferring, because a deferral retires it. +func (s *capturingWarmupSender) Send(ctx context.Context, task uuid.UUID, msg EmailMessage, account models.Email) error { + s.sent = append(s.sent, msg) + if err := s.pool.QueryRow(ctx, `SELECT wt.subject,w.subject FROM warmup_tokens wt JOIN warmup_tasks w ON w.task_id=wt.task_id WHERE wt.task_id=$1`, task).Scan(&s.tokenSubject, &s.lineageSubject); err != nil { + return err + } + return s.preparationDeferringSender.Send(ctx, task, msg, account) +} + +func TestLiveWarmupReplySubjectReadsAsAReply(t *testing.T) { + f := newPartnerRoutingFixture(t) + ctx := t.Context() + worker, task := uuid.New(), uuid.New() + exec := func(query string, args ...any) { + t.Helper() + if _, err := f.pool.Exec(ctx, query, args...); err != nil { + t.Fatal(err) + } + } + t.Cleanup(func() { + for _, query := range []string{ + `DELETE FROM warmup_received WHERE email_account_id IN(SELECT id FROM email_accounts WHERE organization_id=$1)`, + `DELETE FROM warmup_tokens WHERE sender_account_id IN(SELECT id FROM email_accounts WHERE organization_id=$1)`, + `DELETE FROM tasks WHERE email_account_id IN(SELECT id FROM email_accounts WHERE organization_id=$1)`, + } { + if _, err := f.pool.Exec(context.Background(), query, f.org); err != nil { + t.Error(err) + } + } + }) + exec(`INSERT INTO fleet_nodes(id,role,active,last_seen_at,warmup_send_protocol) VALUES($1,'worker',true,NOW(),2)`, worker) + exec(`INSERT INTO workers(id) VALUES($1)`, worker) + t.Cleanup(func() { _, _ = f.pool.Exec(context.Background(), `DELETE FROM fleet_nodes WHERE id=$1`, worker) }) + exec(`UPDATE email_accounts SET worker_id=$1,warmup=NOW()-INTERVAL '30 days',warmup_days=127,warmup_pool_type='free',warmup_reply_rate=100 WHERE organization_id=$2`, worker, f.org) + f.svc.taskRepo = repository.NewTaskRepository(f.pool) + f.svc.warmupHealth = warmupapp.NewService(f.svc.warmupRepo) + f.svc.scheduler = preparationScheduler{} + + // The partner opened a thread to this mailbox, which verified its arrival. + conv := warmupConversations()[0] + parent, receipt := uuid.New(), uuid.New() + parentMessageID := parent.String() + "@shop.test" + exec(`INSERT INTO tasks(id,task_type,email_account_id,status,message_id) VALUES($1,'warmup',$2,'completed','')`, parent, f.atSmallHost) + exec(`INSERT INTO warmup_tasks(task_id,lineage_version,subject,scenario_version,rendering_version,max_turns) VALUES($1,1,$2,$3,$4,$5)`, + parent, conv.Subject, conv.Version, generation.CanonicalRenderingVersion, len(conv.Messages)+1) + exec(`INSERT INTO warmup_tokens(token,task_id,sender_account_id,recipient_account_id,subject,conversation_id,conversation_turn,sent_message_id,content_source) + VALUES($1,$2,$3,$4,$5,$6,0,$7,'static')`, uuid.New(), parent, f.atSmallHost, f.sender.ID, conv.Subject, conv.ID, "<"+parentMessageID+">") + exec(`INSERT INTO warmup_received(email_account_id,internal_id,sender_account_id,message_id,created_at) VALUES($1,$2,$3,$4,NOW()-INTERVAL '1 hour')`, + f.sender.ID, receipt, f.atSmallHost, parentMessageID) + partner, xerr := f.svc.emailRepo.GetByID(ctx, f.atSmallHost) + if xerr != nil { + t.Fatal(xerr) + } + lineage := f.svc.taskRepo.(repository.WarmupLineageRepository) + if err := lineage.RecordVerifiedWarmupParent(ctx, parent, f.sender.ID, receipt, "thread", repository.WarmupReceiptProof{SenderAddress: partner.Email}); err != nil { + t.Fatal(err) + } + + exec(`INSERT INTO tasks(id,email_account_id,task_type,status,message_id,scheduled_at) VALUES($1,$2,'warmup','pending','',NOW()-INTERVAL '1 minute')`, task, f.sender.ID) + exec(`INSERT INTO warmup_tasks(task_id) VALUES($1)`, task) + sender := &capturingWarmupSender{preparationDeferringSender: preparationDeferringSender{repo: f.svc.taskRepo.(repository.WarmupDispatchRepository)}, pool: f.pool} + f.svc.emailSender = sender + if xerr := f.svc.HandleEmailTask(&proto.ProcessTask{TaskId: task.String()}); xerr != nil { + t.Fatal(xerr) + } + if len(sender.sent) != 1 { + t.Fatalf("expected one send, got %d", len(sender.sent)) + } + msg := sender.sent[0] + want := "Re: " + conv.Subject + if msg.InReplyTo != parentMessageID || msg.Subject != want { + t.Fatalf("reply went out as subject=%q in_reply_to=%q, want %q answering %q", msg.Subject, msg.InReplyTo, want, parentMessageID) + } + // The recipient matches the token on the subject it receives; the thread keeps its own. + if sender.tokenSubject != want || sender.lineageSubject != conv.Subject { + t.Fatalf("token subject=%q lineage subject=%q", sender.tokenSubject, sender.lineageSubject) + } +}