mirror of
https://github.com/warmbly/warmbly.git
synced 2026-11-18 08:19:19 +00:00
feat: file provider account notices (Google and Microsoft sign-in, security and service mail) into Archive as read automated system notices kept out of every unibox view, count and the reply pipeline, backfill them in migration 000331, and list pending inbox-agent drafts at the top of the Drafts folder with their count
This commit is contained in:
@@ -32,6 +32,7 @@ A new draft shows up two ways:
|
||||
|
||||
- In the conversation itself: open the thread in the unibox and the draft appears below the messages, in an editable box, with **Approve and send**, and **Discard**.
|
||||
- In the scope rail: the **Agent drafts** filter lists every conversation with a draft waiting, with a live count next to it. The count and list update the moment a new draft lands, no refresh needed.
|
||||
- In the **Drafts** folder: the same conversations are listed first, under **Agent drafts**, above the drafts saved in your mailboxes, and the Drafts count includes them.
|
||||
|
||||
From the draft box you can:
|
||||
|
||||
|
||||
@@ -113,7 +113,7 @@ The rail is two short groups and then your mailboxes. The first group is where y
|
||||
| Folder | Group | Shows |
|
||||
| --- | --- | --- |
|
||||
| Inbox | First | Inbound mail, plus anything filed in a custom folder at the provider, unless that folder is [excluded from sync](/guides/mailboxes/#folders-you-do-not-want-synced) |
|
||||
| Drafts | Second | Messages sitting in a mailbox's drafts folder |
|
||||
| Drafts | Second | Conversations with an [inbox agent](/guides/inbox-agent/) reply waiting for review, listed first under **Agent drafts**, then messages sitting in a mailbox's drafts folder. The count includes both |
|
||||
| Sent | Second | Outbound mail, campaign and manual |
|
||||
| Archive | Second | Archived (Gmail's All Mail, the Archive folder elsewhere) |
|
||||
| Spam | Second | Junked at the provider |
|
||||
@@ -127,6 +127,8 @@ Each message starts in the folder the provider has it in: IMAP special-use folde
|
||||
|
||||
Spam and Trash stay out of every other view, and so does Archive: filing a conversation is how you take it out of the way, so it leaves Unread, Awaiting reply, every mailbox, label and tag view, and the unread badge. **All mail** and the **Archive** folder are the two places it stays, which is where you go to find it again.
|
||||
|
||||
Account notices a mailbox receives from its own provider never reach the inbox. Connecting a mailbox signs in to it, so Google and Microsoft send it security alerts, 2-Step Verification notices, welcome tips and service notices. Mail from `no-reply@accounts.google.com`, `mail-noreply@google.com`, `workspace-noreply@google.com`, `account-security-noreply@accountprotection.microsoft.com`, `msonlineservicesteam@microsoftonline.com` and `no-reply@microsoft.com` is filed straight into **Archive**, marked read, and left out of All mail, every other view and every count. It never counts as a reply, never starts a webhook, Slack post or agent draft, and it is never deleted: the **Archive** folder and a search in All mail still find it. The match is on the sender's exact address, so a display name or a lookalike domain is ordinary mail.
|
||||
|
||||
<Callout type="info" title="Spam and Trash start from when you connect">
|
||||
The initial import that runs when a mailbox is first connected covers Inbox, Sent, Archive, and Drafts. It deliberately skips the existing contents of Spam and Trash, whose history would consume the import budget that belongs to real conversations. Both folders fill normally from the moment the mailbox is connected, so they show what arrives from then on rather than what was already there.
|
||||
</Callout>
|
||||
|
||||
@@ -126,6 +126,10 @@ func (s *JobsService) ingestNewEmail(ctx context.Context, e *models.JobEventNewE
|
||||
CaptureError(e.UserID, e.Message.EmailID, err)
|
||||
return err
|
||||
}
|
||||
// A provider's notice about the mailbox itself is stored archived and read; it is nobody's reply, so nothing below runs.
|
||||
if repository.IsSystemNotice(e.Message) {
|
||||
return nil
|
||||
}
|
||||
if e.Message != nil {
|
||||
finish = arrivaldiag.Begin(ctx, arrivaldiag.Realtime)
|
||||
inbox := s.emailInboxEvent(ctx, e.UserID, e.Message)
|
||||
|
||||
@@ -74,3 +74,40 @@ func TestHandleNewEmailDoesNotProcessSentMailAsReply(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleNewEmailRunsNothingForProviderNotices(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, from string
|
||||
wantCalls int
|
||||
}{
|
||||
{name: "google security alert", from: "Google <no-reply@accounts.google.com>"},
|
||||
{name: "microsoft sign-in alert", from: "Microsoft account team <account-security-noreply@accountprotection.microsoft.com>"},
|
||||
{name: "display name spoof", from: "Google <attacker@x.com>", wantCalls: 1},
|
||||
{name: "lookalike domain", from: "no-reply@accounts.google.com.evil.io", wantCalls: 1},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
advancedService := &replyRecordingAdvanced{}
|
||||
service := &JobsService{
|
||||
UniboxRepository: newEmailInboxRepo{},
|
||||
EmailRepository: newEmailAccountRepo{},
|
||||
AdvancedService: advancedService,
|
||||
}
|
||||
err := service.HandleNewEmail(context.Background(), &models.JobEventNewEmail{
|
||||
UserID: uuid.New(),
|
||||
Message: &models.EmailMessageStoreData{
|
||||
ID: uuid.New(), EmailID: uuid.New(), Folder: models.FolderInbox,
|
||||
FromAddr: []string{tc.from},
|
||||
ToAddr: []string{"recipient@example.test"},
|
||||
MessageID: "<alert@example.test>",
|
||||
Subject: "Security alert",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if advancedService.calls != tc.wantCalls {
|
||||
t.Fatalf("ProcessIncomingReply calls = %d, want %d", advancedService.calls, tc.wantCalls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
-- Archived rows stay archived: which a person filed is not recorded; Archive and search still list them.
|
||||
ALTER TABLE unibox_emails DROP COLUMN IF EXISTS system_notice;
|
||||
@@ -0,0 +1,40 @@
|
||||
-- Provider notices to a mailbox about itself (sign-in alerts, 2-Step Verification, tips); old releases ignore the column.
|
||||
ALTER TABLE unibox_emails ADD COLUMN IF NOT EXISTS system_notice boolean NOT NULL DEFAULT false;
|
||||
|
||||
-- Backfill every workspace with ingest's treatment; senders mirror internal/pkg/systemmail, re-running changes nothing.
|
||||
UPDATE unibox_emails ue
|
||||
SET system_notice = true,
|
||||
automated = true,
|
||||
seen = true,
|
||||
folder = CASE WHEN ue.folder = 'inbox' THEN 'archive' ELSE ue.folder END,
|
||||
updated_at = NOW()
|
||||
WHERE (
|
||||
public.unibox_participants(ue.from_addr, ue.to_addr, ue.cc) ILIKE '%no-reply@accounts.google.com%'
|
||||
OR public.unibox_participants(ue.from_addr, ue.to_addr, ue.cc) ILIKE '%mail-noreply@google.com%'
|
||||
OR public.unibox_participants(ue.from_addr, ue.to_addr, ue.cc) ILIKE '%workspace-noreply@google.com%'
|
||||
OR public.unibox_participants(ue.from_addr, ue.to_addr, ue.cc) ILIKE '%account-security-noreply@accountprotection.microsoft.com%'
|
||||
OR public.unibox_participants(ue.from_addr, ue.to_addr, ue.cc) ILIKE '%msonlineservicesteam@microsoftonline.com%'
|
||||
OR public.unibox_participants(ue.from_addr, ue.to_addr, ue.cc) ILIKE '%no-reply@microsoft.com%'
|
||||
)
|
||||
AND ue.folder NOT IN ('sent', 'drafts')
|
||||
AND ue.provider_folder NOT IN ('sent', 'drafts')
|
||||
AND (
|
||||
SELECT lower(coalesce(
|
||||
substring(f.addr from '<([^<>]*)>[^<>]*$'),
|
||||
substring(f.addr from '^[[:space:]]*([^[:space:]()<>"]+@[^[:space:]()<>"]+)[[:space:]]*\('),
|
||||
substring(f.addr from '\(([^()]*)\)\s*$'),
|
||||
btrim(f.addr)
|
||||
))
|
||||
FROM unnest(ue.from_addr) WITH ORDINALITY AS f(addr, n)
|
||||
WHERE f.addr LIKE '%@%'
|
||||
ORDER BY f.n
|
||||
LIMIT 1
|
||||
) IN (
|
||||
'no-reply@accounts.google.com',
|
||||
'mail-noreply@google.com',
|
||||
'workspace-noreply@google.com',
|
||||
'account-security-noreply@accountprotection.microsoft.com',
|
||||
'msonlineservicesteam@microsoftonline.com',
|
||||
'no-reply@microsoft.com'
|
||||
)
|
||||
AND NOT (ue.system_notice AND ue.automated AND ue.seen AND ue.folder <> 'inbox');
|
||||
@@ -0,0 +1,43 @@
|
||||
// Package systemmail recognises a provider's notices to a mailbox about itself (sign-in alerts, 2-Step Verification, tips).
|
||||
package systemmail
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/warmbly/warmbly/internal/pkg/mailhdr"
|
||||
)
|
||||
|
||||
// providerNoticeSenders is matched exactly on the parsed address, never a display name or suffix.
|
||||
// Migration 000331 backfills from a copy of this list; TestMigrationCoversEverySender keeps them in step.
|
||||
var providerNoticeSenders = map[string]struct{}{
|
||||
"no-reply@accounts.google.com": {}, // Google sign-in, security and 2-Step Verification alerts
|
||||
"mail-noreply@google.com": {}, // Gmail welcome and tips
|
||||
"workspace-noreply@google.com": {}, // Google Workspace service and legal notices
|
||||
"account-security-noreply@accountprotection.microsoft.com": {}, // Microsoft account security and sign-in alerts
|
||||
"msonlineservicesteam@microsoftonline.com": {}, // Microsoft 365 service notices
|
||||
"no-reply@microsoft.com": {}, // Microsoft account notices
|
||||
}
|
||||
|
||||
// Senders returns the recognised addresses, sorted.
|
||||
func Senders() []string {
|
||||
out := make([]string, 0, len(providerNoticeSenders))
|
||||
for addr := range providerNoticeSenders {
|
||||
out = append(out, addr)
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// IsProviderNotice reports whether the first From entry that parses to an address is a provider notice sender.
|
||||
func IsProviderNotice(from []string) bool {
|
||||
for _, raw := range from {
|
||||
addr := strings.ToLower(strings.TrimSpace(mailhdr.Bare(raw)))
|
||||
if !strings.Contains(addr, "@") {
|
||||
continue
|
||||
}
|
||||
_, ok := providerNoticeSenders[addr]
|
||||
return ok
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package systemmail
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIsProviderNoticeMatchesTheParsedAddressOnly(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
from []string
|
||||
want bool
|
||||
}{
|
||||
{"google security alert", []string{"Google <no-reply@accounts.google.com>"}, true},
|
||||
{"gmail team", []string{"Gmail Team <mail-noreply@google.com>"}, true},
|
||||
{"workspace team", []string{`"The Google Workspace Team" <workspace-noreply@google.com>`}, true},
|
||||
{"bare and uppercase", []string{"NO-REPLY@ACCOUNTS.GOOGLE.COM"}, true},
|
||||
{"legacy parenthesised form", []string{"Google (no-reply@accounts.google.com)"}, true},
|
||||
{"microsoft account security", []string{"Microsoft account team <account-security-noreply@accountprotection.microsoft.com>"}, true},
|
||||
{"microsoft 365", []string{"Microsoft Online Services Team <msonlineservicesteam@microsoftonline.com>"}, true},
|
||||
{"microsoft no-reply", []string{"Microsoft <no-reply@microsoft.com>"}, true},
|
||||
{"empty first entry skipped", []string{"", "Google <no-reply@accounts.google.com>"}, true},
|
||||
{"lookalike domain", []string{"Google <no-reply@accounts.google.com.evil.io>"}, false},
|
||||
{"lookalike prefix", []string{"Google <xno-reply@accounts.google.com>"}, false},
|
||||
{"display name spoof", []string{"Google <attacker@x.com>"}, false},
|
||||
{"address quoted in display name", []string{`"no-reply@accounts.google.com" <attacker@x.com>`}, false},
|
||||
{"other google sender", []string{"Google Calendar <calendar-notification@google.com>"}, false},
|
||||
{"only first sender counts", []string{"them@example.com", "Google <no-reply@accounts.google.com>"}, false},
|
||||
{"no sender", nil, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := IsProviderNotice(tc.from); got != tc.want {
|
||||
t.Fatalf("IsProviderNotice(%q) = %v, want %v", tc.from, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrationCoversEverySender(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../infrastructure/db/migrations/000331_unibox_system_notices.up.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sql := string(raw)
|
||||
for _, addr := range Senders() {
|
||||
if !strings.Contains(sql, "'%"+addr+"%'") || !strings.Contains(sql, "'"+addr+"'") {
|
||||
t.Errorf("migration 000331 does not backfill %s", addr)
|
||||
}
|
||||
}
|
||||
if got := strings.Count(sql, "ILIKE '%"); got != len(Senders()) {
|
||||
t.Errorf("migration prefilters %d senders, the list has %d", got, len(Senders()))
|
||||
}
|
||||
}
|
||||
@@ -168,6 +168,7 @@ func (r *inboxTagRepository) Save(ctx context.Context, res *InboxTagResult) erro
|
||||
WHERE ue.email_id = saved.email_account_id
|
||||
AND ue.message_id = saved.message_id
|
||||
AND ue.automated IS DISTINCT FROM saved.automated
|
||||
AND NOT ue.system_notice
|
||||
`
|
||||
answers := res.Answers
|
||||
if len(answers) == 0 {
|
||||
@@ -419,6 +420,7 @@ func (r *inboxTagRepository) Reopen(ctx context.Context, orgID uuid.UUID, messag
|
||||
WHERE ue.email_id = dropped.email_account_id
|
||||
AND ue.message_id = dropped.message_id
|
||||
AND ue.automated
|
||||
AND NOT ue.system_notice
|
||||
)
|
||||
SELECT labels FROM dropped
|
||||
`, orgID, messageID, kind).Scan(&labels)
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/warmbly/warmbly/internal/infrastructure/db"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
"github.com/warmbly/warmbly/internal/pkg/systemmail"
|
||||
)
|
||||
|
||||
type UpdateUniboxEntry struct {
|
||||
@@ -258,27 +259,39 @@ func (r *uniboxRepository) CreateEntry(ctx context.Context, userID uuid.UUID, e
|
||||
flags, bcc, cc, from_addr, in_reply_to, reply_to,
|
||||
to_addr, subject, size, internal_date, sent_date,
|
||||
snippet, seen, created_at, updated_at, body_text, folder,
|
||||
provider_folder, automated, filed_forward
|
||||
provider_folder, automated, filed_forward, system_notice
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, COALESCE((SELECT thread_id FROM fwd), $6), $7,
|
||||
$8, $9, $10, $11,
|
||||
$12, $13, $14, $15, $16, $17,
|
||||
$18, $19, $20, $21, $22,
|
||||
$23, $24, $25, $26, $27, $28,
|
||||
$28,
|
||||
$29,
|
||||
-- A row re-created by a sync keeps the verdict its message already has.
|
||||
EXISTS (
|
||||
$30 OR EXISTS (
|
||||
SELECT 1 FROM inbox_tag_results r
|
||||
WHERE r.organization_id = (SELECT organization_id FROM email_accounts WHERE id = $3)
|
||||
AND r.message_id = $7 AND $7 <> ''
|
||||
AND r.automated
|
||||
),
|
||||
EXISTS (SELECT 1 FROM fwd)
|
||||
EXISTS (SELECT 1 FROM fwd),
|
||||
$30
|
||||
)
|
||||
ON CONFLICT (id) DO NOTHING
|
||||
RETURNING thread_id
|
||||
`
|
||||
|
||||
// A message starts where the provider put it; a system notice is filed to Archive, read, while provider_folder keeps the provider's placement for sync.
|
||||
providerFolder := models.NormalizeFolder(e.Folder, e.Flags)
|
||||
folder, seen := providerFolder, e.Seen
|
||||
notice := IsSystemNotice(e)
|
||||
if notice {
|
||||
seen = true
|
||||
if folder == models.FolderInbox {
|
||||
folder = models.FolderArchive
|
||||
}
|
||||
}
|
||||
|
||||
// The array columns are NOT NULL. A nil Go slice binds as SQL NULL, so a
|
||||
// message with no In-Reply-To (any thread root) would fail the insert.
|
||||
var threadID string
|
||||
@@ -288,10 +301,8 @@ func (r *uniboxRepository) CreateEntry(ctx context.Context, userID uuid.UUID, e
|
||||
textArray(e.Flags), textArray(e.BCC), textArray(e.CC), textArray(e.FromAddr),
|
||||
textArray(e.InReplyTo), textArray(e.ReplyTo), textArray(e.ToAddr),
|
||||
e.Subject, e.Size, e.InternalDate, e.SentDate,
|
||||
e.Snippet, e.Seen, e.CreatedAt, e.UpdatedAt, e.BodyText,
|
||||
// $28 is both columns: a message starts out where the provider put it,
|
||||
// and only diverges once someone files it in Warmbly.
|
||||
models.NormalizeFolder(e.Folder, e.Flags),
|
||||
e.Snippet, seen, e.CreatedAt, e.UpdatedAt, e.BodyText,
|
||||
folder, providerFolder, notice,
|
||||
).Scan(&threadID)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
// Already stored: the conflict leaves the row as it was.
|
||||
@@ -304,6 +315,11 @@ func (r *uniboxRepository) CreateEntry(ctx context.Context, userID uuid.UUID, e
|
||||
return nil
|
||||
}
|
||||
|
||||
// IsSystemNotice reports whether an arriving message is inbound mail from a provider notice sender (systemmail).
|
||||
func IsSystemNotice(e *models.EmailMessageStoreData) bool {
|
||||
return e.MayBeInbound() && systemmail.IsProviderNotice(e.FromAddr)
|
||||
}
|
||||
|
||||
// FiledSentForward is a stored copy FileSentForward moved, with its mailbox owner.
|
||||
type FiledSentForward struct {
|
||||
UserID uuid.UUID
|
||||
@@ -745,6 +761,14 @@ func uniboxSearchQuery(orgID uuid.UUID, params *models.MailSearchParams) (string
|
||||
inner += ` AND ue.folder NOT IN ` + foldersOutsideWorkingViews
|
||||
}
|
||||
|
||||
// System notices show only in their own folder (never Inbox) and in a search.
|
||||
searching := (params.Subject != nil && *params.Subject != "") ||
|
||||
(params.Sender != nil && *params.Sender != "") ||
|
||||
(params.Address != nil && *params.Address != "")
|
||||
if !searching && (params.Folder == nil || *params.Folder == models.FolderInbox) {
|
||||
inner += ` AND NOT ue.system_notice`
|
||||
}
|
||||
|
||||
// Snooze handling. nil = exclude snoozed (the inbox default), so
|
||||
// threads with an active snooze never appear unless asked for.
|
||||
switch {
|
||||
@@ -981,6 +1005,7 @@ func (r *uniboxRepository) GetUnseenCount(ctx context.Context, orgID uuid.UUID,
|
||||
WHERE ue.email_id IN (SELECT id FROM email_accounts WHERE organization_id = $1)
|
||||
AND ue.seen = FALSE
|
||||
AND ue.folder = '` + models.FolderInbox + `'
|
||||
AND NOT ue.system_notice
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM unibox_snoozes s
|
||||
WHERE s.organization_id = $1 AND s.user_id = ue.user_id
|
||||
@@ -1006,6 +1031,7 @@ func (r *uniboxRepository) UnseenCountForMailboxes(ctx context.Context, orgID uu
|
||||
AND ue.email_id = ANY($2::uuid[])
|
||||
AND ue.seen = FALSE
|
||||
AND ue.folder = '` + models.FolderInbox + `'
|
||||
AND NOT ue.system_notice
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM unibox_snoozes s
|
||||
WHERE s.organization_id = $1 AND s.user_id = ue.user_id
|
||||
@@ -1754,6 +1780,7 @@ func (r *uniboxRepository) overview(ctx context.Context, orgID uuid.UUID, accoun
|
||||
FROM unibox_emails e
|
||||
WHERE e.email_id IN `+mailboxes+`
|
||||
AND e.folder NOT IN `+foldersOutsideAllMail+`
|
||||
AND NOT e.system_notice
|
||||
),
|
||||
threads AS (
|
||||
SELECT
|
||||
@@ -1819,6 +1846,7 @@ func (r *uniboxRepository) overview(ctx context.Context, orgID uuid.UUID, accoun
|
||||
AND s.snoozed_until > NOW()
|
||||
)
|
||||
AND NOT (e.folder = 'inbox' AND `+automatedThreadSQL("e.thread_id", "e.automated", "$1")+`)
|
||||
AND NOT e.system_notice
|
||||
GROUP BY e.folder
|
||||
`, orgID)
|
||||
if err != nil {
|
||||
@@ -1861,6 +1889,7 @@ func (r *uniboxRepository) overview(ctx context.Context, orgID uuid.UUID, accoun
|
||||
FROM email_accounts ea
|
||||
LEFT JOIN unibox_emails ue ON ue.email_id = ea.id AND ue.user_id = ea.user_id
|
||||
AND ue.folder NOT IN `+foldersOutsideWorkingViews+`
|
||||
AND NOT ue.system_notice
|
||||
AND NOT `+automatedThreadSQL("ue.thread_id", "ue.automated", "$1")+`
|
||||
WHERE ea.organization_id = $1`+onlyMailboxes("ea.id")+`
|
||||
GROUP BY ea.id, ea.email, ea.name
|
||||
|
||||
@@ -45,7 +45,7 @@ func TestLiveUniboxAutomatedLeavesTheInbox(t *testing.T) {
|
||||
ours := f.mailboxAddress(t, repo)
|
||||
now := time.Now().UTC()
|
||||
|
||||
alert := f.scopedMessage(t, repo, "thread-alert", "Google <no-reply@accounts.google.com>", models.FolderInbox, now)
|
||||
alert := f.scopedMessage(t, repo, "thread-alert", "Acme Billing <billing@acme.example>", models.FolderInbox, now)
|
||||
f.judge(t, handle.Pool, alert, "thread-alert", true)
|
||||
|
||||
f.scopedMessage(t, repo, "thread-person", "them@example.com", models.FolderInbox, now)
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/url"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"github.com/warmbly/warmbly/internal/infrastructure/db"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
)
|
||||
|
||||
// Provider notices to a mailbox about itself land archived, read and out of every working view and counter:
|
||||
//
|
||||
// WARMBLY_TEST_DB=postgres://warmbly@127.0.0.1:15432/warmbly_dev?sslmode=disable \
|
||||
// go test ./internal/repository/ -run LiveUniboxSystemNotice -v
|
||||
|
||||
// liveFreshUniboxDB is a freshly migrated database of its own, so the backfill can run against it.
|
||||
func liveFreshUniboxDB(t *testing.T) *db.DB {
|
||||
t.Helper()
|
||||
dsn := os.Getenv("WARMBLY_TEST_DB")
|
||||
if dsn == "" {
|
||||
t.Skip("WARMBLY_TEST_DB not set")
|
||||
}
|
||||
u, err := url.Parse(dsn)
|
||||
if err != nil || (u.Hostname() != "localhost" && u.Hostname() != "127.0.0.1") {
|
||||
t.Fatal("system notice fixture requires local PostgreSQL")
|
||||
}
|
||||
admin, err := pgx.Connect(t.Context(), dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
name := "system_notice_" + uuid.New().String()[:8]
|
||||
quoted := pgx.Identifier{name}.Sanitize()
|
||||
if _, err := admin.Exec(t.Context(), "CREATE DATABASE "+quoted); err != nil {
|
||||
_ = admin.Close(context.Background())
|
||||
t.Fatal(err)
|
||||
}
|
||||
u.Path = "/" + name
|
||||
if err := db.RunMigrations(u.String()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handle, err := db.New(t.Context(), u.String())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
handle.Pool.Close()
|
||||
if _, err := admin.Exec(context.Background(), "DROP DATABASE "+quoted); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
_ = admin.Close(context.Background())
|
||||
})
|
||||
return handle
|
||||
}
|
||||
|
||||
type storedPlacement struct {
|
||||
folder, provider string
|
||||
seen, automated, systemNotice bool
|
||||
updatedAt time.Time
|
||||
}
|
||||
|
||||
func readPlacement(t *testing.T, handle *db.DB, id uuid.UUID) storedPlacement {
|
||||
t.Helper()
|
||||
var p storedPlacement
|
||||
err := handle.Pool.QueryRow(context.Background(), `
|
||||
SELECT folder, provider_folder, seen, automated, system_notice, updated_at
|
||||
FROM unibox_emails WHERE id = $1`, id).
|
||||
Scan(&p.folder, &p.provider, &p.seen, &p.automated, &p.systemNotice, &p.updatedAt)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", id, err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func TestLiveUniboxSystemNoticeLandsArchivedAndOutOfEveryView(t *testing.T) {
|
||||
handle := liveFreshUniboxDB(t)
|
||||
f := newUniboxFolderFixture(t, handle.Pool)
|
||||
repo := NewUniboxRepository(handle)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
|
||||
alert := f.scopedMessage(t, repo, "thread-alert", "Google <no-reply@accounts.google.com>", models.FolderInbox, now)
|
||||
ms := f.scopedMessage(t, repo, "thread-ms", "Microsoft account team <account-security-noreply@accountprotection.microsoft.com>", models.FolderInbox, now)
|
||||
spoof := f.scopedMessage(t, repo, "thread-spoof", "Google <attacker@x.com>", models.FolderInbox, now)
|
||||
f.scopedMessage(t, repo, "thread-person", "them@example.com", models.FolderInbox, now)
|
||||
|
||||
for _, id := range []uuid.UUID{alert, ms} {
|
||||
p := readPlacement(t, handle, id)
|
||||
if p.folder != models.FolderArchive || p.provider != models.FolderInbox || !p.seen || !p.automated || !p.systemNotice {
|
||||
t.Fatalf("notice stored as %+v, want archived, provider inbox, seen, automated, system_notice", p)
|
||||
}
|
||||
}
|
||||
if p := readPlacement(t, handle, spoof); p.folder != models.FolderInbox || p.seen || p.systemNotice {
|
||||
t.Fatalf("display-name spoof stored as %+v, want an unread inbox message", p)
|
||||
}
|
||||
|
||||
// The provider still reports the notice in its inbox; sync must not read that as a move back.
|
||||
if folder, _, moved := models.ResolveFolderSync(models.FolderArchive, models.FolderInbox, models.FolderInbox); moved || folder != models.FolderArchive {
|
||||
t.Fatalf("sync moved the notice to %q", folder)
|
||||
}
|
||||
|
||||
search := func(p models.MailSearchParams) *models.MailSearchResult {
|
||||
t.Helper()
|
||||
p.PageSize = 50
|
||||
res, err := repo.Search(ctx, f.org, &p)
|
||||
if err != nil {
|
||||
t.Fatalf("Search: %v", err)
|
||||
}
|
||||
return res
|
||||
}
|
||||
no, yes := false, true
|
||||
inbox, archive := models.FolderInbox, models.FolderArchive
|
||||
hidden := func(label string, res *models.MailSearchResult) {
|
||||
t.Helper()
|
||||
if listsThread(res, "thread-alert") || listsThread(res, "thread-ms") {
|
||||
t.Fatalf("%s = %v, want no system notice", label, threadIDs(res))
|
||||
}
|
||||
}
|
||||
hidden("inbox", search(models.MailSearchParams{Folder: &inbox, Automated: &no}))
|
||||
hidden("all mail", search(models.MailSearchParams{IncludeArchived: &yes}))
|
||||
hidden("unread", search(models.MailSearchParams{Unseen: &yes, Automated: &no}))
|
||||
hidden("automated", search(models.MailSearchParams{Automated: &yes}))
|
||||
hidden("awaiting reply", search(models.MailSearchParams{AwaitingReply: &yes}))
|
||||
if got := search(models.MailSearchParams{IncludeArchived: &yes}); !listsThread(got, "thread-spoof") || !listsThread(got, "thread-person") {
|
||||
t.Fatalf("all mail = %v, want the spoof and the person", threadIDs(got))
|
||||
}
|
||||
if got := search(models.MailSearchParams{Folder: &archive}); !listsThread(got, "thread-alert") || !listsThread(got, "thread-ms") {
|
||||
t.Fatalf("archive = %v, want both notices", threadIDs(got))
|
||||
}
|
||||
query := "Scope"
|
||||
if got := search(models.MailSearchParams{IncludeArchived: &yes, Subject: &query}); !listsThread(got, "thread-alert") {
|
||||
t.Fatalf("all mail search = %v, want the notice findable", threadIDs(got))
|
||||
}
|
||||
|
||||
ov, err := repo.Overview(ctx, f.org)
|
||||
if err != nil {
|
||||
t.Fatalf("Overview: %v", err)
|
||||
}
|
||||
if ov.Total != 2 || ov.Unread != 2 || ov.Automated != 0 {
|
||||
t.Fatalf("overview total=%d unread=%d automated=%d, want 2, 2, 0", ov.Total, ov.Unread, ov.Automated)
|
||||
}
|
||||
for _, fo := range ov.Folders {
|
||||
if fo.Folder == models.FolderArchive && fo.Total != 0 {
|
||||
t.Fatalf("archive folder total = %d, want 0", fo.Total)
|
||||
}
|
||||
}
|
||||
if n, err := repo.GetUnseenCount(ctx, f.org, nil); err != nil || n != 2 {
|
||||
t.Fatalf("unread badge = %d (%v), want 2", n, err)
|
||||
}
|
||||
|
||||
// A later classifier verdict never turns a notice back into a person's mail.
|
||||
err = NewInboxTagRepository(handle.Pool).Save(ctx, &InboxTagResult{
|
||||
OrganizationID: f.org, EmailAccountID: f.mailbox,
|
||||
MessageID: "<" + alert.String() + "@test.local>", ThreadID: "thread-alert",
|
||||
Kind: "human_reply", KindConfidence: 0.9, KindSource: "model", Priority: "whenever",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Save verdict: %v", err)
|
||||
}
|
||||
if p := readPlacement(t, handle, alert); !p.automated {
|
||||
t.Fatal("a human verdict cleared automated on a system notice")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLiveUniboxSystemNoticeBackfillIsIdempotent(t *testing.T) {
|
||||
handle := liveFreshUniboxDB(t)
|
||||
f := newUniboxFolderFixture(t, handle.Pool)
|
||||
repo := NewUniboxRepository(handle)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
|
||||
// Rows stored before the rule: written as ordinary mail, then given the sender they arrived with.
|
||||
legacy := func(thread, from, folder string) uuid.UUID {
|
||||
t.Helper()
|
||||
id := f.scopedMessage(t, repo, thread, "them@example.com", folder, now)
|
||||
if _, err := handle.Pool.Exec(ctx, `UPDATE unibox_emails SET from_addr = $2 WHERE id = $1`, id, []string{from}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
google := legacy("t-google", "Google <no-reply@accounts.google.com>", models.FolderInbox)
|
||||
gmail := legacy("t-gmail", "Gmail Team <mail-noreply@google.com>", models.FolderInbox)
|
||||
spam := legacy("t-spam", "Microsoft <no-reply@microsoft.com>", models.FolderSpam)
|
||||
lookalike := legacy("t-look", "Google <no-reply@accounts.google.com.evil.io>", models.FolderInbox)
|
||||
spoof := legacy("t-spoof", "Google <attacker@x.com>", models.FolderInbox)
|
||||
cc := legacy("t-cc", "them@example.com", models.FolderInbox)
|
||||
if _, err := handle.Pool.Exec(ctx, `UPDATE unibox_emails SET cc = ARRAY['no-reply@accounts.google.com'] WHERE id = $1`, cc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
up, err := os.ReadFile("../infrastructure/db/migrations/000331_unibox_system_notices.up.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := handle.Pool.Exec(ctx, string(up)); err != nil {
|
||||
t.Fatalf("backfill: %v", err)
|
||||
}
|
||||
|
||||
for _, id := range []uuid.UUID{google, gmail} {
|
||||
if p := readPlacement(t, handle, id); p.folder != models.FolderArchive || p.provider != models.FolderInbox || !p.seen || !p.automated || !p.systemNotice {
|
||||
t.Fatalf("backfilled notice = %+v, want archived, seen, automated, system_notice", p)
|
||||
}
|
||||
}
|
||||
if p := readPlacement(t, handle, spam); p.folder != models.FolderSpam || !p.systemNotice || !p.seen {
|
||||
t.Fatalf("notice in spam = %+v, want left in spam, marked", p)
|
||||
}
|
||||
for _, id := range []uuid.UUID{lookalike, spoof, cc} {
|
||||
if p := readPlacement(t, handle, id); p.folder != models.FolderInbox || p.seen || p.systemNotice {
|
||||
t.Fatalf("non-notice touched: %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
before := readPlacement(t, handle, google)
|
||||
tag, err := handle.Pool.Exec(ctx, string(up))
|
||||
if err != nil {
|
||||
t.Fatalf("backfill rerun: %v", err)
|
||||
}
|
||||
if tag.RowsAffected() != 0 {
|
||||
t.Fatalf("rerun updated %d rows, want 0", tag.RowsAffected())
|
||||
}
|
||||
if after := readPlacement(t, handle, google); !after.updatedAt.Equal(before.updatedAt) {
|
||||
t.Fatal("rerun touched an already treated row")
|
||||
}
|
||||
}
|
||||
@@ -30,6 +30,7 @@ export default function AgentDraftCard({ threadId }: { threadId: string }) {
|
||||
const refresh = () => {
|
||||
void queryClient.invalidateQueries({ queryKey: ["unibox", "agent-drafts"] });
|
||||
void queryClient.invalidateQueries({ queryKey: ["unibox", "overview"] });
|
||||
void queryClient.invalidateQueries({ queryKey: ["unibox", "search"] });
|
||||
void queryClient.invalidateQueries({ queryKey: ["unibox", "thread", threadId] });
|
||||
};
|
||||
|
||||
|
||||
@@ -32,15 +32,18 @@ import { cn } from "@/lib/utils";
|
||||
import type { UniboxSearchParams } from "@/lib/api/models/app/unibox/UniboxSearch";
|
||||
import { Checkbox } from "@/components/ui/checkbox";
|
||||
|
||||
type Bucket = "today" | "yesterday" | "week" | "earlier";
|
||||
type Bucket = "agent" | "today" | "yesterday" | "week" | "earlier";
|
||||
|
||||
const BUCKET_LABELS: Record<Bucket, string> = {
|
||||
agent: "Agent drafts",
|
||||
today: "Today",
|
||||
yesterday: "Yesterday",
|
||||
week: "This week",
|
||||
earlier: "Earlier",
|
||||
};
|
||||
|
||||
const NO_ROWS: ReturnType<typeof useUniboxSearch>["emails"] = [];
|
||||
|
||||
// Views where an empty list means the work is done rather than that nothing
|
||||
// ever landed there.
|
||||
const CAUGHT_UP_SCOPES = new Set(["folder:inbox", "unread", "awaiting", "all"]);
|
||||
@@ -121,7 +124,26 @@ export function ConversationList({
|
||||
const emptyScope = params.accountIds?.length === 0 || params.categoryIds?.length === 0;
|
||||
const result = useUniboxSearch(merged, scopeKey, !emptyScope);
|
||||
const q = emptyScope ? { ...result, emails: [], isPending: false, isFetching: false, hasNextPage: false } : result;
|
||||
const emails = q.emails;
|
||||
|
||||
// Drafts lists threads with a pending inbox-agent draft above the mailbox's own drafts.
|
||||
const draftsFolder = params.folder === "drafts" && !emptyScope;
|
||||
const agentParams = React.useMemo<UniboxSearchParams>(
|
||||
() => ({ ...merged, folder: undefined, agentDrafts: true }),
|
||||
[merged],
|
||||
);
|
||||
const agent = useUniboxSearch(agentParams, `${scopeKey}:agent`, draftsFolder);
|
||||
const agentRows = draftsFolder ? agent.emails : NO_ROWS;
|
||||
const agentKeys = React.useMemo(
|
||||
() => new Set(agentRows.map((r) => r.thread_id || r.id)),
|
||||
[agentRows],
|
||||
);
|
||||
const emails = React.useMemo(
|
||||
() =>
|
||||
agentRows.length === 0
|
||||
? q.emails
|
||||
: [...agentRows, ...q.emails.filter((r) => !agentKeys.has(r.thread_id || r.id))],
|
||||
[agentRows, agentKeys, q.emails],
|
||||
);
|
||||
const totalShown = emails.length;
|
||||
const activeFilters = countUserFilters(params, baseParams);
|
||||
|
||||
@@ -265,13 +287,13 @@ export function ConversationList({
|
||||
)[] = [];
|
||||
let last: Bucket | null = null;
|
||||
for (const e of emails) {
|
||||
const b = bucketFor(new Date(e.internal_date));
|
||||
const b = agentKeys.has(rowKey(e)) ? "agent" : bucketFor(new Date(e.internal_date));
|
||||
if (b !== last) out.push({ kind: "header", key: `bucket:${b}`, bucket: b });
|
||||
last = b;
|
||||
out.push({ kind: "row", key: rowKey(e), row: e });
|
||||
}
|
||||
return out;
|
||||
}, [emails, rowKey]);
|
||||
}, [emails, rowKey, agentKeys]);
|
||||
|
||||
// Which items were on screen last time, and a generation per key that
|
||||
// left. A key that comes back gets a fresh presence key: AnimatePresence
|
||||
|
||||
@@ -274,14 +274,15 @@ export function ScopeRail({ scope, onChange, browseKey = "desktop" }: ScopeRailP
|
||||
|
||||
const folderRow = (folder: UniboxFolder, label: string, icon: React.ReactNode): RailRow => {
|
||||
const counts = folderCounts.get(folder);
|
||||
// Drafts reads better as a total; everywhere else the number is unread.
|
||||
const count = folder === "drafts" ? counts?.total : counts?.unread;
|
||||
// Drafts reads better as a total, pending agent drafts included; everywhere else the number is unread.
|
||||
const agentDrafts = folder === "drafts" ? (data?.awaiting_agent_draft ?? 0) : 0;
|
||||
const count = folder === "drafts" ? (counts?.total ?? 0) + agentDrafts : counts?.unread;
|
||||
return {
|
||||
key: `folder:${folder}`,
|
||||
label,
|
||||
icon,
|
||||
count: count || undefined,
|
||||
accent: folder !== "drafts" && !!count,
|
||||
accent: folder === "drafts" ? agentDrafts > 0 : !!count,
|
||||
noun: "folder",
|
||||
onOpen: () => onChange({ kind: "folder", folder }),
|
||||
onMarkAllRead: readOnly ? undefined : () => markSeen.mutate({ folder, seen: true }),
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
// The Drafts folder lists threads with a pending inbox-agent draft above the mailbox's own drafts.
|
||||
|
||||
import React from "react";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { cleanup, render, screen } from "@testing-library/react";
|
||||
import type { UniboxSearchParams } from "@/lib/api/models/app/unibox/UniboxSearch";
|
||||
import { ConversationList } from "./ConversationList";
|
||||
|
||||
const search = vi.hoisted(() => ({ calls: [] as { params: UniboxSearchParams; enabled: boolean }[] }));
|
||||
|
||||
function row(id: string, thread: string, subject: string) {
|
||||
return {
|
||||
id,
|
||||
thread_id: thread,
|
||||
email_id: "m1",
|
||||
from_addr: ["Lead <lead@example.com>"],
|
||||
to_addr: ["me@example.com"],
|
||||
subject,
|
||||
snippet: subject,
|
||||
internal_date: new Date().toISOString(),
|
||||
has_unread: false,
|
||||
message_count: 1,
|
||||
labels: [],
|
||||
};
|
||||
}
|
||||
|
||||
vi.mock("@/lib/api/hooks/app/unibox/useUniboxSearch", () => ({
|
||||
default: (params: UniboxSearchParams, _scopeKey: string, enabled = true) => {
|
||||
search.calls.push({ params, enabled });
|
||||
const emails = !enabled
|
||||
? []
|
||||
: params.agentDrafts
|
||||
? [row("a1", "t-agent", "Agent reply to a lead"), row("a2", "t-both", "Agent reply on a drafted thread")]
|
||||
: [row("d1", "t-mine", "My own draft"), row("d2", "t-both", "Provider copy of a drafted thread")];
|
||||
return {
|
||||
emails,
|
||||
isPending: false,
|
||||
isFetching: false,
|
||||
isPlaceholderData: false,
|
||||
isError: false,
|
||||
hasNextPage: false,
|
||||
isFetchingNextPage: false,
|
||||
isFetchNextPageError: false,
|
||||
fetchNextPage: () => Promise.resolve(),
|
||||
refetch: () => Promise.resolve(),
|
||||
};
|
||||
},
|
||||
}));
|
||||
vi.mock("./ConversationItem", () => ({
|
||||
ConversationItem: ({ email }: { email: { subject: string } }) => <div data-testid="row">{email.subject}</div>,
|
||||
}));
|
||||
vi.mock("./AnimatedRow", () => ({
|
||||
AnimatedRow: ({ children }: { children: React.ReactNode }) => <>{children}</>,
|
||||
}));
|
||||
vi.mock("./SelectionBar", () => ({ SelectionBar: () => null }));
|
||||
vi.mock("./UniboxFilterPopover", () => ({
|
||||
countUserFilters: () => 0,
|
||||
UniboxFilterButton: () => null,
|
||||
UniboxFilterChips: () => null,
|
||||
}));
|
||||
vi.mock("@/components/app/unibox/compose/ComposeButton", () => ({ default: () => null }));
|
||||
vi.mock("@/hooks/useConversationActions", () => ({ useConversationActions: () => ({}) }));
|
||||
vi.mock("@/hooks/useShortcutActions", () => ({ useShortcutActions: () => {} }));
|
||||
vi.mock("@/hooks/usePermission", () => ({ useAccessRestricted: () => false }));
|
||||
vi.mock("@/hooks/useBrowseDebouncedValue", () => ({ default: (v: unknown) => v }));
|
||||
vi.mock("@/hooks/useScrollMemory", () => ({ useScrollMemory: () => {} }));
|
||||
vi.mock("framer-motion", async (importOriginal) => ({
|
||||
...(await importOriginal<Record<string, unknown>>()),
|
||||
AnimatePresence: ({ children }: { children: React.ReactNode }) => <>{children}</>,
|
||||
}));
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
search.calls = [];
|
||||
});
|
||||
|
||||
function mount(params: UniboxSearchParams, scopeKey: string) {
|
||||
return render(
|
||||
<ConversationList
|
||||
scopeKey={scopeKey}
|
||||
scopeLabel="Drafts"
|
||||
params={params}
|
||||
baseParams={params}
|
||||
setParams={() => {}}
|
||||
search=""
|
||||
setSearch={() => {}}
|
||||
/>,
|
||||
);
|
||||
}
|
||||
|
||||
it("lists pending agent drafts first under their own header, then the mailbox drafts", () => {
|
||||
mount({ sortBy: "newest", folder: "drafts" }, "folder:drafts");
|
||||
|
||||
expect(screen.getByText("Agent drafts")).toBeTruthy();
|
||||
expect(screen.getAllByTestId("row").map((r) => r.textContent)).toEqual([
|
||||
"Agent reply to a lead",
|
||||
"Agent reply on a drafted thread",
|
||||
"My own draft",
|
||||
]);
|
||||
const agent = search.calls.find((c) => c.params.agentDrafts);
|
||||
expect(agent?.enabled).toBe(true);
|
||||
expect(agent?.params.folder).toBeUndefined();
|
||||
});
|
||||
|
||||
it("leaves every other folder alone", () => {
|
||||
mount({ sortBy: "newest", folder: "inbox", automated: false }, "folder:inbox");
|
||||
|
||||
expect(screen.queryByText("Agent drafts")).toBeNull();
|
||||
expect(search.calls.filter((c) => c.params.agentDrafts).every((c) => !c.enabled)).toBe(true);
|
||||
expect(screen.getAllByTestId("row")).toHaveLength(2);
|
||||
});
|
||||
@@ -1139,3 +1139,19 @@ describe("sanitizeUniboxRailFavorites", () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it("counts pending agent drafts on Drafts and highlights them", () => {
|
||||
const saved = overview.data;
|
||||
overview.data = {
|
||||
...saved,
|
||||
awaiting_agent_draft: 3,
|
||||
folders: [...saved.folders, { folder: "drafts", unread: 0, total: 1 }],
|
||||
};
|
||||
try {
|
||||
mountRail();
|
||||
const rows = Array.from(document.querySelectorAll("[data-rail-row]")).map((r) => r.textContent ?? "");
|
||||
expect(rows.find((r) => r.startsWith("Drafts"))).toBe("Drafts4");
|
||||
} finally {
|
||||
overview.data = saved;
|
||||
}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user