diff --git a/docs/content/docs/guides/inbox-agent.mdx b/docs/content/docs/guides/inbox-agent.mdx index bdb48159e..04b28c96d 100644 --- a/docs/content/docs/guides/inbox-agent.mdx +++ b/docs/content/docs/guides/inbox-agent.mdx @@ -32,6 +32,7 @@ A new draft shows up two ways: - In the conversation itself: open the thread in the unibox and the draft appears below the messages, in an editable box, with **Approve and send**, and **Discard**. - In the scope rail: the **Agent drafts** filter lists every conversation with a draft waiting, with a live count next to it. The count and list update the moment a new draft lands, no refresh needed. +- In the **Drafts** folder: the same conversations are listed first, under **Agent drafts**, above the drafts saved in your mailboxes, and the Drafts count includes them. From the draft box you can: diff --git a/docs/content/docs/guides/unibox.mdx b/docs/content/docs/guides/unibox.mdx index 95d90e154..a55884129 100644 --- a/docs/content/docs/guides/unibox.mdx +++ b/docs/content/docs/guides/unibox.mdx @@ -113,7 +113,7 @@ The rail is two short groups and then your mailboxes. The first group is where y | Folder | Group | Shows | | --- | --- | --- | | Inbox | First | Inbound mail, plus anything filed in a custom folder at the provider, unless that folder is [excluded from sync](/guides/mailboxes/#folders-you-do-not-want-synced) | -| Drafts | Second | Messages sitting in a mailbox's drafts folder | +| Drafts | Second | Conversations with an [inbox agent](/guides/inbox-agent/) reply waiting for review, listed first under **Agent drafts**, then messages sitting in a mailbox's drafts folder. The count includes both | | Sent | Second | Outbound mail, campaign and manual | | Archive | Second | Archived (Gmail's All Mail, the Archive folder elsewhere) | | Spam | Second | Junked at the provider | @@ -127,6 +127,8 @@ Each message starts in the folder the provider has it in: IMAP special-use folde Spam and Trash stay out of every other view, and so does Archive: filing a conversation is how you take it out of the way, so it leaves Unread, Awaiting reply, every mailbox, label and tag view, and the unread badge. **All mail** and the **Archive** folder are the two places it stays, which is where you go to find it again. +Account notices a mailbox receives from its own provider never reach the inbox. Connecting a mailbox signs in to it, so Google and Microsoft send it security alerts, 2-Step Verification notices, welcome tips and service notices. Mail from `no-reply@accounts.google.com`, `mail-noreply@google.com`, `workspace-noreply@google.com`, `account-security-noreply@accountprotection.microsoft.com`, `msonlineservicesteam@microsoftonline.com` and `no-reply@microsoft.com` is filed straight into **Archive**, marked read, and left out of All mail, every other view and every count. It never counts as a reply, never starts a webhook, Slack post or agent draft, and it is never deleted: the **Archive** folder and a search in All mail still find it. The match is on the sender's exact address, so a display name or a lookalike domain is ordinary mail. + The initial import that runs when a mailbox is first connected covers Inbox, Sent, Archive, and Drafts. It deliberately skips the existing contents of Spam and Trash, whose history would consume the import budget that belongs to real conversations. Both folders fill normally from the moment the mailbox is connected, so they show what arrives from then on rather than what was already there. diff --git a/internal/app/consumer/event_new_email.go b/internal/app/consumer/event_new_email.go index 97731d648..feb5a7c2b 100644 --- a/internal/app/consumer/event_new_email.go +++ b/internal/app/consumer/event_new_email.go @@ -126,6 +126,10 @@ func (s *JobsService) ingestNewEmail(ctx context.Context, e *models.JobEventNewE CaptureError(e.UserID, e.Message.EmailID, err) return err } + // A provider's notice about the mailbox itself is stored archived and read; it is nobody's reply, so nothing below runs. + if repository.IsSystemNotice(e.Message) { + return nil + } if e.Message != nil { finish = arrivaldiag.Begin(ctx, arrivaldiag.Realtime) inbox := s.emailInboxEvent(ctx, e.UserID, e.Message) diff --git a/internal/app/consumer/event_new_email_test.go b/internal/app/consumer/event_new_email_test.go index e0c63fe9a..801c8ae78 100644 --- a/internal/app/consumer/event_new_email_test.go +++ b/internal/app/consumer/event_new_email_test.go @@ -74,3 +74,40 @@ func TestHandleNewEmailDoesNotProcessSentMailAsReply(t *testing.T) { }) } } + +func TestHandleNewEmailRunsNothingForProviderNotices(t *testing.T) { + for _, tc := range []struct { + name, from string + wantCalls int + }{ + {name: "google security alert", from: "Google "}, + {name: "microsoft sign-in alert", from: "Microsoft account team "}, + {name: "display name spoof", from: "Google ", wantCalls: 1}, + {name: "lookalike domain", from: "no-reply@accounts.google.com.evil.io", wantCalls: 1}, + } { + t.Run(tc.name, func(t *testing.T) { + advancedService := &replyRecordingAdvanced{} + service := &JobsService{ + UniboxRepository: newEmailInboxRepo{}, + EmailRepository: newEmailAccountRepo{}, + AdvancedService: advancedService, + } + err := service.HandleNewEmail(context.Background(), &models.JobEventNewEmail{ + UserID: uuid.New(), + Message: &models.EmailMessageStoreData{ + ID: uuid.New(), EmailID: uuid.New(), Folder: models.FolderInbox, + FromAddr: []string{tc.from}, + ToAddr: []string{"recipient@example.test"}, + MessageID: "", + Subject: "Security alert", + }, + }) + if err != nil { + t.Fatal(err) + } + if advancedService.calls != tc.wantCalls { + t.Fatalf("ProcessIncomingReply calls = %d, want %d", advancedService.calls, tc.wantCalls) + } + }) + } +} diff --git a/internal/infrastructure/db/migrations/000331_unibox_system_notices.down.sql b/internal/infrastructure/db/migrations/000331_unibox_system_notices.down.sql new file mode 100644 index 000000000..c0e72c875 --- /dev/null +++ b/internal/infrastructure/db/migrations/000331_unibox_system_notices.down.sql @@ -0,0 +1,2 @@ +-- Archived rows stay archived: which a person filed is not recorded; Archive and search still list them. +ALTER TABLE unibox_emails DROP COLUMN IF EXISTS system_notice; diff --git a/internal/infrastructure/db/migrations/000331_unibox_system_notices.up.sql b/internal/infrastructure/db/migrations/000331_unibox_system_notices.up.sql new file mode 100644 index 000000000..efb9962fd --- /dev/null +++ b/internal/infrastructure/db/migrations/000331_unibox_system_notices.up.sql @@ -0,0 +1,40 @@ +-- Provider notices to a mailbox about itself (sign-in alerts, 2-Step Verification, tips); old releases ignore the column. +ALTER TABLE unibox_emails ADD COLUMN IF NOT EXISTS system_notice boolean NOT NULL DEFAULT false; + +-- Backfill every workspace with ingest's treatment; senders mirror internal/pkg/systemmail, re-running changes nothing. +UPDATE unibox_emails ue +SET system_notice = true, + automated = true, + seen = true, + folder = CASE WHEN ue.folder = 'inbox' THEN 'archive' ELSE ue.folder END, + updated_at = NOW() +WHERE ( + public.unibox_participants(ue.from_addr, ue.to_addr, ue.cc) ILIKE '%no-reply@accounts.google.com%' + OR public.unibox_participants(ue.from_addr, ue.to_addr, ue.cc) ILIKE '%mail-noreply@google.com%' + OR public.unibox_participants(ue.from_addr, ue.to_addr, ue.cc) ILIKE '%workspace-noreply@google.com%' + OR public.unibox_participants(ue.from_addr, ue.to_addr, ue.cc) ILIKE '%account-security-noreply@accountprotection.microsoft.com%' + OR public.unibox_participants(ue.from_addr, ue.to_addr, ue.cc) ILIKE '%msonlineservicesteam@microsoftonline.com%' + OR public.unibox_participants(ue.from_addr, ue.to_addr, ue.cc) ILIKE '%no-reply@microsoft.com%' + ) + AND ue.folder NOT IN ('sent', 'drafts') + AND ue.provider_folder NOT IN ('sent', 'drafts') + AND ( + SELECT lower(coalesce( + substring(f.addr from '<([^<>]*)>[^<>]*$'), + substring(f.addr from '^[[:space:]]*([^[:space:]()<>"]+@[^[:space:]()<>"]+)[[:space:]]*\('), + substring(f.addr from '\(([^()]*)\)\s*$'), + btrim(f.addr) + )) + FROM unnest(ue.from_addr) WITH ORDINALITY AS f(addr, n) + WHERE f.addr LIKE '%@%' + ORDER BY f.n + LIMIT 1 + ) IN ( + 'no-reply@accounts.google.com', + 'mail-noreply@google.com', + 'workspace-noreply@google.com', + 'account-security-noreply@accountprotection.microsoft.com', + 'msonlineservicesteam@microsoftonline.com', + 'no-reply@microsoft.com' + ) + AND NOT (ue.system_notice AND ue.automated AND ue.seen AND ue.folder <> 'inbox'); diff --git a/internal/pkg/systemmail/systemmail.go b/internal/pkg/systemmail/systemmail.go new file mode 100644 index 000000000..a39baf1f9 --- /dev/null +++ b/internal/pkg/systemmail/systemmail.go @@ -0,0 +1,43 @@ +// Package systemmail recognises a provider's notices to a mailbox about itself (sign-in alerts, 2-Step Verification, tips). +package systemmail + +import ( + "slices" + "strings" + + "github.com/warmbly/warmbly/internal/pkg/mailhdr" +) + +// providerNoticeSenders is matched exactly on the parsed address, never a display name or suffix. +// Migration 000331 backfills from a copy of this list; TestMigrationCoversEverySender keeps them in step. +var providerNoticeSenders = map[string]struct{}{ + "no-reply@accounts.google.com": {}, // Google sign-in, security and 2-Step Verification alerts + "mail-noreply@google.com": {}, // Gmail welcome and tips + "workspace-noreply@google.com": {}, // Google Workspace service and legal notices + "account-security-noreply@accountprotection.microsoft.com": {}, // Microsoft account security and sign-in alerts + "msonlineservicesteam@microsoftonline.com": {}, // Microsoft 365 service notices + "no-reply@microsoft.com": {}, // Microsoft account notices +} + +// Senders returns the recognised addresses, sorted. +func Senders() []string { + out := make([]string, 0, len(providerNoticeSenders)) + for addr := range providerNoticeSenders { + out = append(out, addr) + } + slices.Sort(out) + return out +} + +// IsProviderNotice reports whether the first From entry that parses to an address is a provider notice sender. +func IsProviderNotice(from []string) bool { + for _, raw := range from { + addr := strings.ToLower(strings.TrimSpace(mailhdr.Bare(raw))) + if !strings.Contains(addr, "@") { + continue + } + _, ok := providerNoticeSenders[addr] + return ok + } + return false +} diff --git a/internal/pkg/systemmail/systemmail_test.go b/internal/pkg/systemmail/systemmail_test.go new file mode 100644 index 000000000..2c75b81d3 --- /dev/null +++ b/internal/pkg/systemmail/systemmail_test.go @@ -0,0 +1,54 @@ +package systemmail + +import ( + "os" + "strings" + "testing" +) + +func TestIsProviderNoticeMatchesTheParsedAddressOnly(t *testing.T) { + for _, tc := range []struct { + name string + from []string + want bool + }{ + {"google security alert", []string{"Google "}, true}, + {"gmail team", []string{"Gmail Team "}, true}, + {"workspace team", []string{`"The Google Workspace Team" `}, true}, + {"bare and uppercase", []string{"NO-REPLY@ACCOUNTS.GOOGLE.COM"}, true}, + {"legacy parenthesised form", []string{"Google (no-reply@accounts.google.com)"}, true}, + {"microsoft account security", []string{"Microsoft account team "}, true}, + {"microsoft 365", []string{"Microsoft Online Services Team "}, true}, + {"microsoft no-reply", []string{"Microsoft "}, true}, + {"empty first entry skipped", []string{"", "Google "}, true}, + {"lookalike domain", []string{"Google "}, false}, + {"lookalike prefix", []string{"Google "}, false}, + {"display name spoof", []string{"Google "}, false}, + {"address quoted in display name", []string{`"no-reply@accounts.google.com" `}, false}, + {"other google sender", []string{"Google Calendar "}, false}, + {"only first sender counts", []string{"them@example.com", "Google "}, false}, + {"no sender", nil, false}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := IsProviderNotice(tc.from); got != tc.want { + t.Fatalf("IsProviderNotice(%q) = %v, want %v", tc.from, got, tc.want) + } + }) + } +} + +func TestMigrationCoversEverySender(t *testing.T) { + raw, err := os.ReadFile("../../infrastructure/db/migrations/000331_unibox_system_notices.up.sql") + if err != nil { + t.Fatal(err) + } + sql := string(raw) + for _, addr := range Senders() { + if !strings.Contains(sql, "'%"+addr+"%'") || !strings.Contains(sql, "'"+addr+"'") { + t.Errorf("migration 000331 does not backfill %s", addr) + } + } + if got := strings.Count(sql, "ILIKE '%"); got != len(Senders()) { + t.Errorf("migration prefilters %d senders, the list has %d", got, len(Senders())) + } +} diff --git a/internal/repository/pg_inbox_tag.go b/internal/repository/pg_inbox_tag.go index 10631a007..ba24b1814 100644 --- a/internal/repository/pg_inbox_tag.go +++ b/internal/repository/pg_inbox_tag.go @@ -168,6 +168,7 @@ func (r *inboxTagRepository) Save(ctx context.Context, res *InboxTagResult) erro WHERE ue.email_id = saved.email_account_id AND ue.message_id = saved.message_id AND ue.automated IS DISTINCT FROM saved.automated + AND NOT ue.system_notice ` answers := res.Answers if len(answers) == 0 { @@ -419,6 +420,7 @@ func (r *inboxTagRepository) Reopen(ctx context.Context, orgID uuid.UUID, messag WHERE ue.email_id = dropped.email_account_id AND ue.message_id = dropped.message_id AND ue.automated + AND NOT ue.system_notice ) SELECT labels FROM dropped `, orgID, messageID, kind).Scan(&labels) diff --git a/internal/repository/pg_unibox.go b/internal/repository/pg_unibox.go index 96113964f..343d981a5 100644 --- a/internal/repository/pg_unibox.go +++ b/internal/repository/pg_unibox.go @@ -13,6 +13,7 @@ import ( "github.com/jackc/pgx/v5" "github.com/warmbly/warmbly/internal/infrastructure/db" "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/pkg/systemmail" ) type UpdateUniboxEntry struct { @@ -258,27 +259,39 @@ func (r *uniboxRepository) CreateEntry(ctx context.Context, userID uuid.UUID, e flags, bcc, cc, from_addr, in_reply_to, reply_to, to_addr, subject, size, internal_date, sent_date, snippet, seen, created_at, updated_at, body_text, folder, - provider_folder, automated, filed_forward + provider_folder, automated, filed_forward, system_notice ) VALUES ( $1, $2, $3, $4, $5, COALESCE((SELECT thread_id FROM fwd), $6), $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, - $28, + $29, -- A row re-created by a sync keeps the verdict its message already has. - EXISTS ( + $30 OR EXISTS ( SELECT 1 FROM inbox_tag_results r WHERE r.organization_id = (SELECT organization_id FROM email_accounts WHERE id = $3) AND r.message_id = $7 AND $7 <> '' AND r.automated ), - EXISTS (SELECT 1 FROM fwd) + EXISTS (SELECT 1 FROM fwd), + $30 ) ON CONFLICT (id) DO NOTHING RETURNING thread_id ` + // A message starts where the provider put it; a system notice is filed to Archive, read, while provider_folder keeps the provider's placement for sync. + providerFolder := models.NormalizeFolder(e.Folder, e.Flags) + folder, seen := providerFolder, e.Seen + notice := IsSystemNotice(e) + if notice { + seen = true + if folder == models.FolderInbox { + folder = models.FolderArchive + } + } + // The array columns are NOT NULL. A nil Go slice binds as SQL NULL, so a // message with no In-Reply-To (any thread root) would fail the insert. var threadID string @@ -288,10 +301,8 @@ func (r *uniboxRepository) CreateEntry(ctx context.Context, userID uuid.UUID, e textArray(e.Flags), textArray(e.BCC), textArray(e.CC), textArray(e.FromAddr), textArray(e.InReplyTo), textArray(e.ReplyTo), textArray(e.ToAddr), e.Subject, e.Size, e.InternalDate, e.SentDate, - e.Snippet, e.Seen, e.CreatedAt, e.UpdatedAt, e.BodyText, - // $28 is both columns: a message starts out where the provider put it, - // and only diverges once someone files it in Warmbly. - models.NormalizeFolder(e.Folder, e.Flags), + e.Snippet, seen, e.CreatedAt, e.UpdatedAt, e.BodyText, + folder, providerFolder, notice, ).Scan(&threadID) if errors.Is(err, pgx.ErrNoRows) { // Already stored: the conflict leaves the row as it was. @@ -304,6 +315,11 @@ func (r *uniboxRepository) CreateEntry(ctx context.Context, userID uuid.UUID, e return nil } +// IsSystemNotice reports whether an arriving message is inbound mail from a provider notice sender (systemmail). +func IsSystemNotice(e *models.EmailMessageStoreData) bool { + return e.MayBeInbound() && systemmail.IsProviderNotice(e.FromAddr) +} + // FiledSentForward is a stored copy FileSentForward moved, with its mailbox owner. type FiledSentForward struct { UserID uuid.UUID @@ -745,6 +761,14 @@ func uniboxSearchQuery(orgID uuid.UUID, params *models.MailSearchParams) (string inner += ` AND ue.folder NOT IN ` + foldersOutsideWorkingViews } + // System notices show only in their own folder (never Inbox) and in a search. + searching := (params.Subject != nil && *params.Subject != "") || + (params.Sender != nil && *params.Sender != "") || + (params.Address != nil && *params.Address != "") + if !searching && (params.Folder == nil || *params.Folder == models.FolderInbox) { + inner += ` AND NOT ue.system_notice` + } + // Snooze handling. nil = exclude snoozed (the inbox default), so // threads with an active snooze never appear unless asked for. switch { @@ -981,6 +1005,7 @@ func (r *uniboxRepository) GetUnseenCount(ctx context.Context, orgID uuid.UUID, WHERE ue.email_id IN (SELECT id FROM email_accounts WHERE organization_id = $1) AND ue.seen = FALSE AND ue.folder = '` + models.FolderInbox + `' + AND NOT ue.system_notice AND NOT EXISTS ( SELECT 1 FROM unibox_snoozes s WHERE s.organization_id = $1 AND s.user_id = ue.user_id @@ -1006,6 +1031,7 @@ func (r *uniboxRepository) UnseenCountForMailboxes(ctx context.Context, orgID uu AND ue.email_id = ANY($2::uuid[]) AND ue.seen = FALSE AND ue.folder = '` + models.FolderInbox + `' + AND NOT ue.system_notice AND NOT EXISTS ( SELECT 1 FROM unibox_snoozes s WHERE s.organization_id = $1 AND s.user_id = ue.user_id @@ -1754,6 +1780,7 @@ func (r *uniboxRepository) overview(ctx context.Context, orgID uuid.UUID, accoun FROM unibox_emails e WHERE e.email_id IN `+mailboxes+` AND e.folder NOT IN `+foldersOutsideAllMail+` + AND NOT e.system_notice ), threads AS ( SELECT @@ -1819,6 +1846,7 @@ func (r *uniboxRepository) overview(ctx context.Context, orgID uuid.UUID, accoun AND s.snoozed_until > NOW() ) AND NOT (e.folder = 'inbox' AND `+automatedThreadSQL("e.thread_id", "e.automated", "$1")+`) + AND NOT e.system_notice GROUP BY e.folder `, orgID) if err != nil { @@ -1861,6 +1889,7 @@ func (r *uniboxRepository) overview(ctx context.Context, orgID uuid.UUID, accoun FROM email_accounts ea LEFT JOIN unibox_emails ue ON ue.email_id = ea.id AND ue.user_id = ea.user_id AND ue.folder NOT IN `+foldersOutsideWorkingViews+` + AND NOT ue.system_notice AND NOT `+automatedThreadSQL("ue.thread_id", "ue.automated", "$1")+` WHERE ea.organization_id = $1`+onlyMailboxes("ea.id")+` GROUP BY ea.id, ea.email, ea.name diff --git a/internal/repository/unibox_automated_live_test.go b/internal/repository/unibox_automated_live_test.go index af643d3a6..75d0117b1 100644 --- a/internal/repository/unibox_automated_live_test.go +++ b/internal/repository/unibox_automated_live_test.go @@ -45,7 +45,7 @@ func TestLiveUniboxAutomatedLeavesTheInbox(t *testing.T) { ours := f.mailboxAddress(t, repo) now := time.Now().UTC() - alert := f.scopedMessage(t, repo, "thread-alert", "Google ", models.FolderInbox, now) + alert := f.scopedMessage(t, repo, "thread-alert", "Acme Billing ", models.FolderInbox, now) f.judge(t, handle.Pool, alert, "thread-alert", true) f.scopedMessage(t, repo, "thread-person", "them@example.com", models.FolderInbox, now) diff --git a/internal/repository/unibox_system_notice_live_test.go b/internal/repository/unibox_system_notice_live_test.go new file mode 100644 index 000000000..bee782a93 --- /dev/null +++ b/internal/repository/unibox_system_notice_live_test.go @@ -0,0 +1,229 @@ +package repository + +import ( + "context" + "net/url" + "os" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "github.com/warmbly/warmbly/internal/infrastructure/db" + "github.com/warmbly/warmbly/internal/models" +) + +// Provider notices to a mailbox about itself land archived, read and out of every working view and counter: +// +// WARMBLY_TEST_DB=postgres://warmbly@127.0.0.1:15432/warmbly_dev?sslmode=disable \ +// go test ./internal/repository/ -run LiveUniboxSystemNotice -v + +// liveFreshUniboxDB is a freshly migrated database of its own, so the backfill can run against it. +func liveFreshUniboxDB(t *testing.T) *db.DB { + t.Helper() + dsn := os.Getenv("WARMBLY_TEST_DB") + if dsn == "" { + t.Skip("WARMBLY_TEST_DB not set") + } + u, err := url.Parse(dsn) + if err != nil || (u.Hostname() != "localhost" && u.Hostname() != "127.0.0.1") { + t.Fatal("system notice fixture requires local PostgreSQL") + } + admin, err := pgx.Connect(t.Context(), dsn) + if err != nil { + t.Fatal(err) + } + name := "system_notice_" + uuid.New().String()[:8] + quoted := pgx.Identifier{name}.Sanitize() + if _, err := admin.Exec(t.Context(), "CREATE DATABASE "+quoted); err != nil { + _ = admin.Close(context.Background()) + t.Fatal(err) + } + u.Path = "/" + name + if err := db.RunMigrations(u.String()); err != nil { + t.Fatal(err) + } + handle, err := db.New(t.Context(), u.String()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + handle.Pool.Close() + if _, err := admin.Exec(context.Background(), "DROP DATABASE "+quoted); err != nil { + t.Error(err) + } + _ = admin.Close(context.Background()) + }) + return handle +} + +type storedPlacement struct { + folder, provider string + seen, automated, systemNotice bool + updatedAt time.Time +} + +func readPlacement(t *testing.T, handle *db.DB, id uuid.UUID) storedPlacement { + t.Helper() + var p storedPlacement + err := handle.Pool.QueryRow(context.Background(), ` + SELECT folder, provider_folder, seen, automated, system_notice, updated_at + FROM unibox_emails WHERE id = $1`, id). + Scan(&p.folder, &p.provider, &p.seen, &p.automated, &p.systemNotice, &p.updatedAt) + if err != nil { + t.Fatalf("read %s: %v", id, err) + } + return p +} + +func TestLiveUniboxSystemNoticeLandsArchivedAndOutOfEveryView(t *testing.T) { + handle := liveFreshUniboxDB(t) + f := newUniboxFolderFixture(t, handle.Pool) + repo := NewUniboxRepository(handle) + ctx := context.Background() + now := time.Now().UTC() + + alert := f.scopedMessage(t, repo, "thread-alert", "Google ", models.FolderInbox, now) + ms := f.scopedMessage(t, repo, "thread-ms", "Microsoft account team ", models.FolderInbox, now) + spoof := f.scopedMessage(t, repo, "thread-spoof", "Google ", models.FolderInbox, now) + f.scopedMessage(t, repo, "thread-person", "them@example.com", models.FolderInbox, now) + + for _, id := range []uuid.UUID{alert, ms} { + p := readPlacement(t, handle, id) + if p.folder != models.FolderArchive || p.provider != models.FolderInbox || !p.seen || !p.automated || !p.systemNotice { + t.Fatalf("notice stored as %+v, want archived, provider inbox, seen, automated, system_notice", p) + } + } + if p := readPlacement(t, handle, spoof); p.folder != models.FolderInbox || p.seen || p.systemNotice { + t.Fatalf("display-name spoof stored as %+v, want an unread inbox message", p) + } + + // The provider still reports the notice in its inbox; sync must not read that as a move back. + if folder, _, moved := models.ResolveFolderSync(models.FolderArchive, models.FolderInbox, models.FolderInbox); moved || folder != models.FolderArchive { + t.Fatalf("sync moved the notice to %q", folder) + } + + search := func(p models.MailSearchParams) *models.MailSearchResult { + t.Helper() + p.PageSize = 50 + res, err := repo.Search(ctx, f.org, &p) + if err != nil { + t.Fatalf("Search: %v", err) + } + return res + } + no, yes := false, true + inbox, archive := models.FolderInbox, models.FolderArchive + hidden := func(label string, res *models.MailSearchResult) { + t.Helper() + if listsThread(res, "thread-alert") || listsThread(res, "thread-ms") { + t.Fatalf("%s = %v, want no system notice", label, threadIDs(res)) + } + } + hidden("inbox", search(models.MailSearchParams{Folder: &inbox, Automated: &no})) + hidden("all mail", search(models.MailSearchParams{IncludeArchived: &yes})) + hidden("unread", search(models.MailSearchParams{Unseen: &yes, Automated: &no})) + hidden("automated", search(models.MailSearchParams{Automated: &yes})) + hidden("awaiting reply", search(models.MailSearchParams{AwaitingReply: &yes})) + if got := search(models.MailSearchParams{IncludeArchived: &yes}); !listsThread(got, "thread-spoof") || !listsThread(got, "thread-person") { + t.Fatalf("all mail = %v, want the spoof and the person", threadIDs(got)) + } + if got := search(models.MailSearchParams{Folder: &archive}); !listsThread(got, "thread-alert") || !listsThread(got, "thread-ms") { + t.Fatalf("archive = %v, want both notices", threadIDs(got)) + } + query := "Scope" + if got := search(models.MailSearchParams{IncludeArchived: &yes, Subject: &query}); !listsThread(got, "thread-alert") { + t.Fatalf("all mail search = %v, want the notice findable", threadIDs(got)) + } + + ov, err := repo.Overview(ctx, f.org) + if err != nil { + t.Fatalf("Overview: %v", err) + } + if ov.Total != 2 || ov.Unread != 2 || ov.Automated != 0 { + t.Fatalf("overview total=%d unread=%d automated=%d, want 2, 2, 0", ov.Total, ov.Unread, ov.Automated) + } + for _, fo := range ov.Folders { + if fo.Folder == models.FolderArchive && fo.Total != 0 { + t.Fatalf("archive folder total = %d, want 0", fo.Total) + } + } + if n, err := repo.GetUnseenCount(ctx, f.org, nil); err != nil || n != 2 { + t.Fatalf("unread badge = %d (%v), want 2", n, err) + } + + // A later classifier verdict never turns a notice back into a person's mail. + err = NewInboxTagRepository(handle.Pool).Save(ctx, &InboxTagResult{ + OrganizationID: f.org, EmailAccountID: f.mailbox, + MessageID: "<" + alert.String() + "@test.local>", ThreadID: "thread-alert", + Kind: "human_reply", KindConfidence: 0.9, KindSource: "model", Priority: "whenever", + }) + if err != nil { + t.Fatalf("Save verdict: %v", err) + } + if p := readPlacement(t, handle, alert); !p.automated { + t.Fatal("a human verdict cleared automated on a system notice") + } +} + +func TestLiveUniboxSystemNoticeBackfillIsIdempotent(t *testing.T) { + handle := liveFreshUniboxDB(t) + f := newUniboxFolderFixture(t, handle.Pool) + repo := NewUniboxRepository(handle) + ctx := context.Background() + now := time.Now().UTC() + + // Rows stored before the rule: written as ordinary mail, then given the sender they arrived with. + legacy := func(thread, from, folder string) uuid.UUID { + t.Helper() + id := f.scopedMessage(t, repo, thread, "them@example.com", folder, now) + if _, err := handle.Pool.Exec(ctx, `UPDATE unibox_emails SET from_addr = $2 WHERE id = $1`, id, []string{from}); err != nil { + t.Fatal(err) + } + return id + } + google := legacy("t-google", "Google ", models.FolderInbox) + gmail := legacy("t-gmail", "Gmail Team ", models.FolderInbox) + spam := legacy("t-spam", "Microsoft ", models.FolderSpam) + lookalike := legacy("t-look", "Google ", models.FolderInbox) + spoof := legacy("t-spoof", "Google ", models.FolderInbox) + cc := legacy("t-cc", "them@example.com", models.FolderInbox) + if _, err := handle.Pool.Exec(ctx, `UPDATE unibox_emails SET cc = ARRAY['no-reply@accounts.google.com'] WHERE id = $1`, cc); err != nil { + t.Fatal(err) + } + + up, err := os.ReadFile("../infrastructure/db/migrations/000331_unibox_system_notices.up.sql") + if err != nil { + t.Fatal(err) + } + if _, err := handle.Pool.Exec(ctx, string(up)); err != nil { + t.Fatalf("backfill: %v", err) + } + + for _, id := range []uuid.UUID{google, gmail} { + if p := readPlacement(t, handle, id); p.folder != models.FolderArchive || p.provider != models.FolderInbox || !p.seen || !p.automated || !p.systemNotice { + t.Fatalf("backfilled notice = %+v, want archived, seen, automated, system_notice", p) + } + } + if p := readPlacement(t, handle, spam); p.folder != models.FolderSpam || !p.systemNotice || !p.seen { + t.Fatalf("notice in spam = %+v, want left in spam, marked", p) + } + for _, id := range []uuid.UUID{lookalike, spoof, cc} { + if p := readPlacement(t, handle, id); p.folder != models.FolderInbox || p.seen || p.systemNotice { + t.Fatalf("non-notice touched: %+v", p) + } + } + + before := readPlacement(t, handle, google) + tag, err := handle.Pool.Exec(ctx, string(up)) + if err != nil { + t.Fatalf("backfill rerun: %v", err) + } + if tag.RowsAffected() != 0 { + t.Fatalf("rerun updated %d rows, want 0", tag.RowsAffected()) + } + if after := readPlacement(t, handle, google); !after.updatedAt.Equal(before.updatedAt) { + t.Fatal("rerun touched an already treated row") + } +} diff --git a/web/src/components/app/unibox/AgentDraftCard.tsx b/web/src/components/app/unibox/AgentDraftCard.tsx index 7fe02f03e..05639659f 100644 --- a/web/src/components/app/unibox/AgentDraftCard.tsx +++ b/web/src/components/app/unibox/AgentDraftCard.tsx @@ -30,6 +30,7 @@ export default function AgentDraftCard({ threadId }: { threadId: string }) { const refresh = () => { void queryClient.invalidateQueries({ queryKey: ["unibox", "agent-drafts"] }); void queryClient.invalidateQueries({ queryKey: ["unibox", "overview"] }); + void queryClient.invalidateQueries({ queryKey: ["unibox", "search"] }); void queryClient.invalidateQueries({ queryKey: ["unibox", "thread", threadId] }); }; diff --git a/web/src/components/app/unibox/ConversationList.tsx b/web/src/components/app/unibox/ConversationList.tsx index ea99d0f2b..33418f5c7 100644 --- a/web/src/components/app/unibox/ConversationList.tsx +++ b/web/src/components/app/unibox/ConversationList.tsx @@ -32,15 +32,18 @@ import { cn } from "@/lib/utils"; import type { UniboxSearchParams } from "@/lib/api/models/app/unibox/UniboxSearch"; import { Checkbox } from "@/components/ui/checkbox"; -type Bucket = "today" | "yesterday" | "week" | "earlier"; +type Bucket = "agent" | "today" | "yesterday" | "week" | "earlier"; const BUCKET_LABELS: Record = { + agent: "Agent drafts", today: "Today", yesterday: "Yesterday", week: "This week", earlier: "Earlier", }; +const NO_ROWS: ReturnType["emails"] = []; + // Views where an empty list means the work is done rather than that nothing // ever landed there. const CAUGHT_UP_SCOPES = new Set(["folder:inbox", "unread", "awaiting", "all"]); @@ -121,7 +124,26 @@ export function ConversationList({ const emptyScope = params.accountIds?.length === 0 || params.categoryIds?.length === 0; const result = useUniboxSearch(merged, scopeKey, !emptyScope); const q = emptyScope ? { ...result, emails: [], isPending: false, isFetching: false, hasNextPage: false } : result; - const emails = q.emails; + + // Drafts lists threads with a pending inbox-agent draft above the mailbox's own drafts. + const draftsFolder = params.folder === "drafts" && !emptyScope; + const agentParams = React.useMemo( + () => ({ ...merged, folder: undefined, agentDrafts: true }), + [merged], + ); + const agent = useUniboxSearch(agentParams, `${scopeKey}:agent`, draftsFolder); + const agentRows = draftsFolder ? agent.emails : NO_ROWS; + const agentKeys = React.useMemo( + () => new Set(agentRows.map((r) => r.thread_id || r.id)), + [agentRows], + ); + const emails = React.useMemo( + () => + agentRows.length === 0 + ? q.emails + : [...agentRows, ...q.emails.filter((r) => !agentKeys.has(r.thread_id || r.id))], + [agentRows, agentKeys, q.emails], + ); const totalShown = emails.length; const activeFilters = countUserFilters(params, baseParams); @@ -265,13 +287,13 @@ export function ConversationList({ )[] = []; let last: Bucket | null = null; for (const e of emails) { - const b = bucketFor(new Date(e.internal_date)); + const b = agentKeys.has(rowKey(e)) ? "agent" : bucketFor(new Date(e.internal_date)); if (b !== last) out.push({ kind: "header", key: `bucket:${b}`, bucket: b }); last = b; out.push({ kind: "row", key: rowKey(e), row: e }); } return out; - }, [emails, rowKey]); + }, [emails, rowKey, agentKeys]); // Which items were on screen last time, and a generation per key that // left. A key that comes back gets a fresh presence key: AnimatePresence diff --git a/web/src/components/app/unibox/ScopeRail.tsx b/web/src/components/app/unibox/ScopeRail.tsx index 841629ff8..f70ed6a91 100644 --- a/web/src/components/app/unibox/ScopeRail.tsx +++ b/web/src/components/app/unibox/ScopeRail.tsx @@ -274,14 +274,15 @@ export function ScopeRail({ scope, onChange, browseKey = "desktop" }: ScopeRailP const folderRow = (folder: UniboxFolder, label: string, icon: React.ReactNode): RailRow => { const counts = folderCounts.get(folder); - // Drafts reads better as a total; everywhere else the number is unread. - const count = folder === "drafts" ? counts?.total : counts?.unread; + // Drafts reads better as a total, pending agent drafts included; everywhere else the number is unread. + const agentDrafts = folder === "drafts" ? (data?.awaiting_agent_draft ?? 0) : 0; + const count = folder === "drafts" ? (counts?.total ?? 0) + agentDrafts : counts?.unread; return { key: `folder:${folder}`, label, icon, count: count || undefined, - accent: folder !== "drafts" && !!count, + accent: folder === "drafts" ? agentDrafts > 0 : !!count, noun: "folder", onOpen: () => onChange({ kind: "folder", folder }), onMarkAllRead: readOnly ? undefined : () => markSeen.mutate({ folder, seen: true }), diff --git a/web/src/components/app/unibox/conversationListDrafts.test.tsx b/web/src/components/app/unibox/conversationListDrafts.test.tsx new file mode 100644 index 000000000..14628b91e --- /dev/null +++ b/web/src/components/app/unibox/conversationListDrafts.test.tsx @@ -0,0 +1,111 @@ +// The Drafts folder lists threads with a pending inbox-agent draft above the mailbox's own drafts. + +import React from "react"; +import { afterEach, expect, it, vi } from "vitest"; +import { cleanup, render, screen } from "@testing-library/react"; +import type { UniboxSearchParams } from "@/lib/api/models/app/unibox/UniboxSearch"; +import { ConversationList } from "./ConversationList"; + +const search = vi.hoisted(() => ({ calls: [] as { params: UniboxSearchParams; enabled: boolean }[] })); + +function row(id: string, thread: string, subject: string) { + return { + id, + thread_id: thread, + email_id: "m1", + from_addr: ["Lead "], + to_addr: ["me@example.com"], + subject, + snippet: subject, + internal_date: new Date().toISOString(), + has_unread: false, + message_count: 1, + labels: [], + }; +} + +vi.mock("@/lib/api/hooks/app/unibox/useUniboxSearch", () => ({ + default: (params: UniboxSearchParams, _scopeKey: string, enabled = true) => { + search.calls.push({ params, enabled }); + const emails = !enabled + ? [] + : params.agentDrafts + ? [row("a1", "t-agent", "Agent reply to a lead"), row("a2", "t-both", "Agent reply on a drafted thread")] + : [row("d1", "t-mine", "My own draft"), row("d2", "t-both", "Provider copy of a drafted thread")]; + return { + emails, + isPending: false, + isFetching: false, + isPlaceholderData: false, + isError: false, + hasNextPage: false, + isFetchingNextPage: false, + isFetchNextPageError: false, + fetchNextPage: () => Promise.resolve(), + refetch: () => Promise.resolve(), + }; + }, +})); +vi.mock("./ConversationItem", () => ({ + ConversationItem: ({ email }: { email: { subject: string } }) =>
{email.subject}
, +})); +vi.mock("./AnimatedRow", () => ({ + AnimatedRow: ({ children }: { children: React.ReactNode }) => <>{children}, +})); +vi.mock("./SelectionBar", () => ({ SelectionBar: () => null })); +vi.mock("./UniboxFilterPopover", () => ({ + countUserFilters: () => 0, + UniboxFilterButton: () => null, + UniboxFilterChips: () => null, +})); +vi.mock("@/components/app/unibox/compose/ComposeButton", () => ({ default: () => null })); +vi.mock("@/hooks/useConversationActions", () => ({ useConversationActions: () => ({}) })); +vi.mock("@/hooks/useShortcutActions", () => ({ useShortcutActions: () => {} })); +vi.mock("@/hooks/usePermission", () => ({ useAccessRestricted: () => false })); +vi.mock("@/hooks/useBrowseDebouncedValue", () => ({ default: (v: unknown) => v })); +vi.mock("@/hooks/useScrollMemory", () => ({ useScrollMemory: () => {} })); +vi.mock("framer-motion", async (importOriginal) => ({ + ...(await importOriginal>()), + AnimatePresence: ({ children }: { children: React.ReactNode }) => <>{children}, +})); + +afterEach(() => { + cleanup(); + search.calls = []; +}); + +function mount(params: UniboxSearchParams, scopeKey: string) { + return render( + {}} + search="" + setSearch={() => {}} + />, + ); +} + +it("lists pending agent drafts first under their own header, then the mailbox drafts", () => { + mount({ sortBy: "newest", folder: "drafts" }, "folder:drafts"); + + expect(screen.getByText("Agent drafts")).toBeTruthy(); + expect(screen.getAllByTestId("row").map((r) => r.textContent)).toEqual([ + "Agent reply to a lead", + "Agent reply on a drafted thread", + "My own draft", + ]); + const agent = search.calls.find((c) => c.params.agentDrafts); + expect(agent?.enabled).toBe(true); + expect(agent?.params.folder).toBeUndefined(); +}); + +it("leaves every other folder alone", () => { + mount({ sortBy: "newest", folder: "inbox", automated: false }, "folder:inbox"); + + expect(screen.queryByText("Agent drafts")).toBeNull(); + expect(search.calls.filter((c) => c.params.agentDrafts).every((c) => !c.enabled)).toBe(true); + expect(screen.getAllByTestId("row")).toHaveLength(2); +}); diff --git a/web/src/components/app/unibox/scopeRail.test.tsx b/web/src/components/app/unibox/scopeRail.test.tsx index d18a055cd..d2d029e21 100644 --- a/web/src/components/app/unibox/scopeRail.test.tsx +++ b/web/src/components/app/unibox/scopeRail.test.tsx @@ -1139,3 +1139,19 @@ describe("sanitizeUniboxRailFavorites", () => { } }); }); + +it("counts pending agent drafts on Drafts and highlights them", () => { + const saved = overview.data; + overview.data = { + ...saved, + awaiting_agent_draft: 3, + folders: [...saved.folders, { folder: "drafts", unread: 0, total: 1 }], + }; + try { + mountRail(); + const rows = Array.from(document.querySelectorAll("[data-rail-row]")).map((r) => r.textContent ?? ""); + expect(rows.find((r) => r.startsWith("Drafts"))).toBe("Drafts4"); + } finally { + overview.data = saved; + } +});