diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d66683b9a..f0325745a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -148,6 +148,10 @@ jobs: run: | go test -race -coverprofile=coverage.out -covermode=atomic ./... + # The Avro codec's registry handling only compiles with the kafka tag. + - name: Test the Avro codec + run: go test -tags kafka ./internal/infrastructure/codec/ + - name: Upload coverage uses: codecov/codecov-action@v4 with: diff --git a/AGENTS.md b/AGENTS.md index 3ad68442a..1066a2c7d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -768,7 +768,7 @@ Signals used: - every warmup email carries a verification token, minted by the platform, single-use, bound to its recipient - no inbound token is evidence against the mailbox that received it. It did not present the token; its worker synced whatever landed in its inbox, and inbound mail is attacker-controlled: every pool member holds tokens naming itself and a partner, and forwarding three to another member used to block that member for 30 days. The recipient check already makes a token worthless anywhere but its own destination, so nothing is charged on that path (#468, #481). Do not reintroduce a charge there, whether gated by a window, a folder check, a clock or by which pair the token names; each of those was tried and each was a way to be wrong (#477, #480) -- tampering with warmup mail a mailbox verifiably received (deleting it, flagging it as spam) is attributed to that mailbox, because only its owner can do it. It is a ladder, not a first-strike ban: `evaluateMetrics` weighs a deletion as one strike and a spam flag as two over the seven-day window, and warns at one, quarantines at two and blocks at four (`tampering*Strikes` in `internal/app/warmup/service.go`). One deletion is someone tidying the folder by hand until proven otherwise (#635). `RecordTampering` only records the event and re-evaluates, so a sweep reaches the same answer; a tampering block carries a term like every other band and never requires review +- tampering with warmup mail a mailbox verifiably received (deleting it, flagging it as spam) is attributed to that mailbox, because only its owner can do it. It is a ladder, not a first-strike ban: `evaluateMetrics` counts a deletion and a spam move as one strike each over the seven-day window, and warns at one, quarantines at two and blocks at four. No provider names who moved a message into spam (Microsoft's ZAP, Workspace post-delivery scanning and client junk filters look exactly like a user's report), so a spam move is never charged on sight. A spam label on mail that arrived in spam is the filter's (`warmup_received.landed_spam`; Gmail can report it as a later label change) and is not even held. Any other move is held in `warmup_spam_moves` and `attributeSpamMove` (`internal/app/consumer/warmup_spam_attribution.go`) decides it after `config.WarmupSpamMoveSettleMinutes`: provider when the same sender was junked in another workspace within a day (which also withdraws owner verdicts it explains) or the move came straight after arrival with nobody there; owner when `mailbox_owner_activity` shows the owner at the mailbox around it, or a mailbox in use keeps junking many senders nobody else does; nobody otherwise. Owner activity is only a read, unread or star change the provider reported that our store did not already hold, on mail past its arrival grace, so Warmbly's own echoes and a filter finishing delivery never count. Only an owner verdict strikes the recipient and files a complaint against the sender; the rest are placement (`tampering*Strikes` in `internal/app/warmup/service.go`). One deletion is someone tidying the folder by hand until proven otherwise (#635). `RecordTampering` only records the event and re-evaluates, so a sweep reaches the same answer; a tampering block carries a term like every other band and never requires review - a deletion is a strike only inside `config.WarmupDeletionStrikeHours` of arrival (`warmupDeletionCounts` in `internal/app/consumer/event_remove_email.go`), and never for a receipt the retention sweep has retired. The engagement a message earns happens in its first hours; after that the platform deletes it itself (#637), so a later removal, whichever of the owner, Gmail's Trash purge, a server retention rule or our own sweep did it, is housekeeping. Gmail's Delete arrives as the `TRASH` label and is judged there on the same rule, because the `messagesDeleted` history record only comes when Trash is emptied, weeks later and in a burst. Do not widen the window or count a removal past it: every mailbox on a fixed quota has to be able to clear the folder - a removal is never a strike on its own. Graph reports a move exactly like a delete, and a provider's filter, a mailbox rule, another Warmbly instance syncing the same mailbox (a self-hosted instance warming in Warmbly Cloud) or our own filing can all move warmup mail. A fresh removal publishes a `verify_removal` warmup action; the worker searches the whole mailbox by Message-ID (`internal/app/worker/event_warmup_verify.go`) and answers `WARMUP_REMOVAL_CHECKED`, and `HandleWarmupRemovalChecked` (`internal/app/consumer/warmup_removal_check.go`) records a strike only for a message in the trash or gone. Found anywhere else withdraws any strike for it (`WithdrawTampering`), and a tampering pause or block is re-decided on the strikes left in the seven days before it was imposed (never on today's window, which old strikes have aged out of), then lowered, shortened from its original decision time, or lifted. The revision lands on the pool row, or on the address's ledger row when no mailbox with the address is in a pool, the one write to `warmup_reputation_ledger` outside its trigger, so a withdrawn hold is not seeded back on rejoin. A search that fails or cannot tell (IMAP only sees synced folders) charges nothing. `warmup_tampering_events.verified_at` is NULL only on strikes from before the search; `StartWarmupTamperingRecheck` asks for those (`Recheck`), stamps `verify_requested_at` and asks again after six hours while unanswered. A recheck never adds a strike: present or retired-by-retention withdraws it, anything else stamps `verified_at` and it stands. Tampering events are kept at least `config.WarmupTamperingKeepDays` so the strikes behind a live hold are there to re-decide it. Gmail's `TRASH` label needs no search, since it is the message entering the trash. Do not add another path that records a deletion without the search; the self-move marker is only a shortcut that skips it for our own filing - warmup mail is retained by the platform, not the owner: `StartWarmupMailRetention` (`internal/app/consumer/warmup_mail_retention.go`) retires every received copy and every sender's copy past the mailbox's window (`email_accounts.warmup_retention_days`, else `retention.warmup_mail_days`) and publishes `WarmupActionDelete` to the worker, which trashes it on Gmail, deletes it on Graph, expunges it on IMAP and drops the stored body. The row is retired only after the action is on the bus, so a failed publish is re-offered. The same loop prunes tokens, receipts, tampering events and spam reports past `retention.warmup_event_days`; `warmup_statistics` carries the analytics and is never pruned diff --git a/Makefile b/Makefile index 15ef214ac..96da2830c 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ PROTOC_GEN_GO_GRPC_VERSION ?= v1.6.1 PROTO_DIR := internal/tasks/proto PROTO_GEN_FILES := $(PROTO_DIR)/tasks.pb.go -.PHONY: poollink-dev poollink-dev-down poollink-dev-reset setup-tools fmt lint check-migrations join-check split-cloud-check pages-check kafka-check proto check-proto \ +.PHONY: poollink-dev poollink-dev-down poollink-dev-reset setup-tools fmt schemas lint check-migrations join-check split-cloud-check pages-check kafka-check proto check-proto \ up upgrade claim doctor cli seed-demo seed seed-plan sandbox sandbox-seed sandbox-simulate reset logs status stop down test-seed \ restart restart-go restart-all infra infra-down app app-down app-logs \ backend forms forms-web consumer worker run dev tracking realtime web \ @@ -82,6 +82,11 @@ cli-check: fmt: gofmt -w ./cmd ./internal +# Record the bus schemas the next release publishes. CI refuses a change the +# registry would refuse, and a compatible one until it is recorded here. +schemas: + go test ./internal/app/eventschemas -run TestPublishedSchemasStayCompatible -update + lint: check-migrations join-check split-cloud-check pages-check check-dockerfiles ./scripts/check-forms-mirror.sh $(GO_BIN)/golangci-lint run --timeout=5m diff --git a/admin/src/app/dashboard/configuration/SettingsTab.tsx b/admin/src/app/dashboard/configuration/SettingsTab.tsx index cad5b90fa..e7d64c326 100644 --- a/admin/src/app/dashboard/configuration/SettingsTab.tsx +++ b/admin/src/app/dashboard/configuration/SettingsTab.tsx @@ -230,6 +230,38 @@ const PLACEMENT_FIELDS = [ max: 10000, help: "What a test costs in credits once a workspace has used its free tests for the month. The workspace agrees to the price before each paid test, and a test that delivers no copy is refunded. 0 turns paid tests off, so a workspace waits for next month instead. Only a hosted (DEPLOYMENT_MODE=cloud) instance charges.", }, + { + key: "batchSendersMax", + setting: "batch_senders_max", + label: "Most senders in one batch", + min: 1, + max: 100000, + help: "The largest sender list one placement batch may hold. It limits how big a batch can be, not how many senders run at once, so it can sit well above any fleet a workspace runs.", + }, + { + key: "batchSenderConcurrency", + setting: "batch_sender_concurrency", + label: "Batch senders sending at once", + min: 1, + max: 500, + help: "How many of one workspace's batch senders may be sending their copies at the same time, across all its batches. The next sender starts when one has sent every copy. Each mailbox still keeps its own daily limit and spacing.", + }, + { + key: "batchInstanceConcurrency", + setting: "batch_instance_concurrency", + label: "Batch senders sending at once, instance-wide", + min: 1, + max: 5000, + help: "The same limit across every workspace together. It bounds how much batch mail the instance seed panel receives at once, so a seed never takes in enough in an hour to trip the sync flood rule that deactivates a mailbox.", + }, + { + key: "batchStartsPerMinute", + setting: "batch_starts_per_minute", + label: "Batch senders started per minute", + min: 1, + max: 600, + help: "How fast one batch starts its senders, so a large batch ramps up instead of starting its whole concurrency at once.", + }, ] as const; type PlacementFieldKey = (typeof PLACEMENT_FIELDS)[number]["key"]; @@ -241,6 +273,10 @@ const PLACEMENT_DEFAULTS: InstanceSettings["placement"] = { seeds_per_test: 20, spacing_seconds: 60, credits_per_test: 25, + batch_senders_max: 10000, + batch_sender_concurrency: 20, + batch_instance_concurrency: 200, + batch_starts_per_minute: 10, }; interface FormState { @@ -285,6 +321,10 @@ function toForm(s: InstanceSettings): FormState { seedsPerTest: String(placement.seeds_per_test), spacingSeconds: String(placement.spacing_seconds), creditsPerTest: String(placement.credits_per_test ?? PLACEMENT_DEFAULTS.credits_per_test), + batchSendersMax: String(placement.batch_senders_max ?? PLACEMENT_DEFAULTS.batch_senders_max), + batchSenderConcurrency: String(placement.batch_sender_concurrency ?? PLACEMENT_DEFAULTS.batch_sender_concurrency), + batchInstanceConcurrency: String(placement.batch_instance_concurrency ?? PLACEMENT_DEFAULTS.batch_instance_concurrency), + batchStartsPerMinute: String(placement.batch_starts_per_minute ?? PLACEMENT_DEFAULTS.batch_starts_per_minute), }, enforceDomainAuth: s.deliverability.enforce_domain_auth, authGraceHours: String(s.deliverability.auth_grace_hours), @@ -349,7 +389,7 @@ export function SettingsTab({ onDirtyChange, onSwitchTab }: SettingsTabProps) { !!form && PLACEMENT_FIELDS.some( (f) => - form.placement[f.key] !== String((server.placement ?? PLACEMENT_DEFAULTS)[f.setting]), + form.placement[f.key] !== String(server.placement?.[f.setting] ?? PLACEMENT_DEFAULTS[f.setting]), ); const dirty = !!server && @@ -463,6 +503,10 @@ export function SettingsTab({ onDirtyChange, onSwitchTab }: SettingsTabProps) { seeds_per_test: Number(form.placement.seedsPerTest), spacing_seconds: Number(form.placement.spacingSeconds), credits_per_test: Number(form.placement.creditsPerTest), + batch_senders_max: Number(form.placement.batchSendersMax), + batch_sender_concurrency: Number(form.placement.batchSenderConcurrency), + batch_instance_concurrency: Number(form.placement.batchInstanceConcurrency), + batch_starts_per_minute: Number(form.placement.batchStartsPerMinute), }, }); } diff --git a/admin/src/app/dashboard/placement/format.ts b/admin/src/app/dashboard/placement/format.ts index 7d7153d50..6891e15f4 100644 --- a/admin/src/app/dashboard/placement/format.ts +++ b/admin/src/app/dashboard/placement/format.ts @@ -18,6 +18,7 @@ export const ORIGIN_LABEL: Record = { monitor: "Monitor", admin: "Admin", remote: "Linked instance", + batch: "Batch", }; // The backend's `error` field is the HTTP status text; the sentence worth diff --git a/admin/src/lib/api/client/admin/instance.ts b/admin/src/lib/api/client/admin/instance.ts index f247f14cd..b38b34408 100644 --- a/admin/src/lib/api/client/admin/instance.ts +++ b/admin/src/lib/api/client/admin/instance.ts @@ -162,6 +162,14 @@ export interface InstanceSettings { spacing_seconds: number; /** Credits a test past the monthly allowance costs; 0 turns paid tests off. */ credits_per_test: number; + /** Most senders one placement batch may hold. */ + batch_senders_max: number; + /** A workspace's batch senders sending probes at the same time. */ + batch_sender_concurrency: number; + /** Batch senders sending at once across every workspace. */ + batch_instance_concurrency: number; + /** Senders one batch starts per minute. */ + batch_starts_per_minute: number; }; // Operator notification channels. Targets and secrets are redacted on // read: a chat webhook URL is a bearer credential, so the server returns a diff --git a/admin/src/lib/api/client/admin/placement.ts b/admin/src/lib/api/client/admin/placement.ts index 4b1ea4ff8..617371ee3 100644 --- a/admin/src/lib/api/client/admin/placement.ts +++ b/admin/src/lib/api/client/admin/placement.ts @@ -5,7 +5,7 @@ import { Request } from "@/lib/api/client"; export type PlacementPanel = "instance" | "workspace" | "cloud"; -export type PlacementOrigin = "manual" | "monitor" | "admin" | "remote"; +export type PlacementOrigin = "manual" | "monitor" | "admin" | "remote" | "batch"; export type PlacementStatus = "running" | "completed" | "cancelled" | "failed"; export type PlacementTracking = "campaign" | "on" | "off" | "compare"; export type PlacementFolder = diff --git a/cmd/backend/main.go b/cmd/backend/main.go index 19e960b6f..fd327784d 100644 --- a/cmd/backend/main.go +++ b/cmd/backend/main.go @@ -56,6 +56,7 @@ import ( "github.com/warmbly/warmbly/internal/app/email" "github.com/warmbly/warmbly/internal/app/emailsend" emailverifyapp "github.com/warmbly/warmbly/internal/app/emailverify" + "github.com/warmbly/warmbly/internal/app/eventschemas" "github.com/warmbly/warmbly/internal/app/feature" "github.com/warmbly/warmbly/internal/app/fleet" "github.com/warmbly/warmbly/internal/app/fleetnode" @@ -149,6 +150,7 @@ import ( "github.com/warmbly/warmbly/internal/tasks" "github.com/warmbly/warmbly/internal/tasks/proto" "github.com/warmbly/warmbly/internal/tasksched" + "github.com/warmbly/warmbly/internal/version" "golang.org/x/oauth2" "golang.org/x/oauth2/google" ) @@ -596,6 +598,15 @@ func main() { errs.CaptureFatal(err) log.Fatal(err) } + // Register this release's bus schemas before any node publishes them. + go func() { + rctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + if err := eventschemas.Register(rctx, codecImpl); err != nil { + errs.CaptureException(err) + log.Printf("event schemas: %v", err) + } + }() bus, err := eventbus.FromEnv(kafkaBootstrapServers, kafkaSaslConfig) if err != nil { @@ -1154,6 +1165,7 @@ func main() { GithubRepo: getenvDefault("RELEASES_GITHUB_REPO", "warmbly/warmbly"), WorkerImageRepo: getenvDefault("RELEASES_WORKER_IMAGE_REPO", "ghcr.io/warmbly/warmbly/worker"), GithubToken: os.Getenv("RELEASES_GITHUB_TOKEN"), + SchemaGate: eventschemas.Gate(codecImpl, version.Version), }, fleetSettingsRepo, ) @@ -2004,6 +2016,7 @@ func main() { Notifier: notificationService, Mailboxes: emailService, Pauser: guardrailService, + Batches: repository.NewPlacementBatchRepository(primaryDB), } if streamingPublisher != nil { placementDeps.Publisher = streamingPublisher diff --git a/cmd/cli/specs.go b/cmd/cli/specs.go index f094512a9..050712646 100644 --- a/cmd/cli/specs.go +++ b/cmd/cli/specs.go @@ -318,6 +318,46 @@ the hold. Resuming a lead that is not held succeeds and changes nothing.`, {Name: "contact", Help: "The contact's id"}, }, }, + { + Name: "lead-cc", Short: "Contacts copied on one lead's emails", + Method: http.MethodGet, Path: "/campaigns/{id}/leads/{contact}/cc", + Args: []argSpec{ + {Name: "id", Help: "The campaign's id"}, + {Name: "contact", Help: "The lead's contact id"}, + }, + Table: output.Table{Root: "cc", Columns: []output.Column{ + col("CONTACT", "contact_id"), col("EMAIL", "email"), col("STATUS", "status"), + }, Empty: "This lead copies nobody."}, + }, + { + Name: "set-lead-cc", Short: "Replace the contacts copied on one lead's emails", + Long: `Copy up to two contacts on every email this campaign sends one lead, follow-ups +included, so colleagues at one company share a single thread. The list replaces +the current one. A copied contact who is also a lead of the campaign has their +own sequence held while any lead copies them, so they never get two threads.`, + Example: " $ warmbly campaign set-lead-cc CAMPAIGN_ID CONTACT_ID --cc COLLEAGUE_ID\n" + + " $ warmbly campaign set-lead-cc CAMPAIGN_ID CONTACT_ID --input '{\"contact_ids\":[]}' # copy nobody", + Method: http.MethodPut, Path: "/campaigns/{id}/leads/{contact}/cc", Body: bodyRequired, + Args: []argSpec{ + {Name: "id", Help: "The campaign's id"}, + {Name: "contact", Help: "The lead's contact id"}, + }, + Flag: []flagSpec{ + {Name: "cc", Help: "A contact id to copy (repeatable, at most 2)", Kind: flagStrings, Key: "contact_ids"}, + }, + Success: "Lead CC replaced.", + }, + { + Name: "lead-cc-suggestions", Short: "The lead's likely colleagues to copy", + Method: http.MethodGet, Path: "/campaigns/{id}/leads/{contact}/cc/suggestions", + Args: []argSpec{ + {Name: "id", Help: "The campaign's id"}, + {Name: "contact", Help: "The lead's contact id"}, + }, + Table: output.Table{Root: "data", Columns: []output.Column{ + col("CONTACT", "contact_id"), col("EMAIL", "email"), col("COMPANY", "company"), col("MATCH", "reason"), + }, Empty: "No contacts share the lead's company or email domain."}, + }, { Name: "logs", Short: "The campaign's send log", Method: http.MethodGet, Path: "/campaigns/{id}/logs", Paginate: true, @@ -1506,6 +1546,47 @@ func placementSpec() resource { }, Empty: "No placement tests yet. Start one with `warmbly placement test --mailbox MAILBOX_ID --campaign CAMPAIGN_ID`.", } + batchTable := output.Table{ + Root: "data", + Columns: []output.Column{ + col("ID", "id"), + colt("SUBJECT", "subject", 30), + col("MAILBOXES", "sender_count"), + col("STATUS", "status"), + col("DONE", "progress.completed"), + col("SKIPPED", "progress.skipped"), + col("INBOX", "summary.inbox"), + col("SPAM", "summary.spam"), + colf("CREATED", "created_at", "time"), + }, + Empty: "No placement batches yet. Start one with `warmbly placement batch-start --scope campaign --scope-campaign CAMPAIGN_ID --campaign CAMPAIGN_ID`.", + } + batchFlags := []flagSpec{ + {Name: "mailbox", Help: "Test from this mailbox's id (repeatable); instead of --scope", Kind: flagStrings, Key: "sender_account_ids"}, + {Name: "scope", Help: "campaign (a campaign's mailboxes) or workspace (every mailbox)", Key: "sender_scope[type]"}, + {Name: "scope-campaign", Help: "The campaign whose mailboxes to test, with --scope campaign", Key: "sender_scope[campaign_id]"}, + {Name: "only-provider", Help: "Only mailboxes hosted by this provider, e.g. google_workspace (repeatable)", Kind: flagStrings, Key: "sender_scope[providers]"}, + {Name: "only-domain", Help: "Only mailboxes sending from this domain (repeatable)", Kind: flagStrings, Key: "sender_scope[domains]"}, + {Name: "only-tag", Help: "Only mailboxes with this tag id (repeatable)", Kind: flagStrings, Key: "sender_scope[tag_ids]"}, + {Name: "include-inactive", Help: "Keep disconnected mailboxes", Kind: flagBool, Key: "sender_scope[include_inactive]"}, + {Name: "untested-days", Help: "Only mailboxes with no finished placement test in this many days", Kind: flagInt, Key: "sender_scope[untested_days]"}, + {Name: "sample", Help: "all, random, percent, per_domain or per_provider", Key: "sample[mode]"}, + {Name: "sample-count", Help: "Mailboxes for random, or per group for per_domain and per_provider", Kind: flagInt, Key: "sample[count]"}, + {Name: "sample-percent", Help: "Share of mailboxes for percent, 1 to 100", Kind: flagInt, Key: "sample[percent]"}, + {Name: "stratify", Help: "Spread a random or percent sample across provider or domain", Key: "sample[stratify]"}, + {Name: "campaign", Help: "Test this campaign's copy", Key: "campaign_id"}, + {Name: "step", Help: "The step to test; with --campaign", Key: "sequence_id"}, + {Name: "contact", Help: "Render the copy for this contact's id", Key: "contact_id"}, + {Name: "subject", Help: "Subject of an ad-hoc template", Key: "subject"}, + {Name: "text", Help: "Plain-text body of an ad-hoc template", Key: "body_plain"}, + {Name: "html", Help: "HTML body of an ad-hoc template", Key: "body_html"}, + {Name: "tracking", Help: "campaign, on, off or compare (sends twice, with and without)", Key: "tracking"}, + {Name: "panel", Help: "instance, workspace or cloud", Key: "panel"}, + {Name: "seed", Help: "Only this seed inbox's id, with --panel workspace (repeatable)", Kind: flagStrings, Key: "seed_ids"}, + {Name: "provider", Help: "Only seeds at this provider, e.g. gmail or outlook (repeatable)", Kind: flagStrings, Key: "families"}, + {Name: "on-unavailable", Help: "defer (retry a mailbox that cannot send yet, the default) or skip", Key: "on_unavailable"}, + {Name: "max-credits", Help: "Pay up to this many credits in total for tests past this month's free ones", Kind: flagInt, Key: "max_credits"}, + } return resource{ Name: "placement", Aliases: []string{"placement-test", "placement-tests"}, @@ -1515,7 +1596,8 @@ func placementSpec() resource { seed inboxes and see where each copy landed. Every copy is a real send from that mailbox, counted against its daily limit, -so starting a test asks before it does it.`, +so starting a test asks before it does it. A batch runs the same test from many +mailboxes, a few at a time.`, Endpoints: []endpoint{ {Name: "overview", Short: "The seed panels you can test on and this month's allowance", Method: http.MethodGet, Path: "/placement/overview"}, { @@ -1598,6 +1680,56 @@ so starting a test asks before it does it.`, Args: []argSpec{{Name: "id", Help: "The campaign's id"}}, Success: "Placement monitor removed.", }, + { + Name: "batches", Short: "List placement batches: one test run from many mailboxes", + Method: http.MethodGet, Path: "/placement/batches", Paginate: true, + Flag: withPaging(), + Table: batchTable, + }, + { + Name: "batch", Aliases: []string{"batch-view"}, Short: "Show a batch with its placement by domain and provider", + Method: http.MethodGet, Path: "/placement/batches/{id}", + Args: []argSpec{{Name: "id", Help: "The batch's id"}}, + }, + { + Name: "batch-preview", Short: "Count the mailboxes, tests, copies and credits a batch would take", + Method: http.MethodPost, Path: "/placement/batches/preview", Body: bodyRequired, + Example: " $ warmbly placement batch-preview --scope campaign --scope-campaign CAMPAIGN_ID --sample percent --sample-percent 10 --stratify provider --campaign CAMPAIGN_ID --step STEP_ID", + Flag: batchFlags, + }, + { + Name: "batch-start", Aliases: []string{"batch-test"}, Short: "Run a placement test from many mailboxes, a few at a time", + Method: http.MethodPost, Path: "/placement/batches", Body: bodyRequired, Sends: true, Idempotent: true, + Example: " $ warmbly placement batch-start --scope campaign --scope-campaign CAMPAIGN_ID --campaign CAMPAIGN_ID --step STEP_ID\n" + + " $ warmbly placement batch-start --scope workspace --untested-days 30 --sample per_domain --sample-count 2 --subject \"Quick question\" --text \"Hi there\"", + Flag: batchFlags, + Table: batchTable, + }, + { + Name: "batch-senders", Short: "List a batch's mailboxes, worst inbox rate first", + Method: http.MethodGet, Path: "/placement/batches/{id}/senders", Paginate: true, + Args: []argSpec{{Name: "id", Help: "The batch's id"}}, + Flag: withPaging( + flagSpec{Name: "sort", Help: "worst (default), best, email or status", Query: true}, + flagSpec{Name: "status", Help: "Only mailboxes in this status: queued, deferred, running, completed, skipped, failed or cancelled", Query: true}, + flagSpec{Name: "search", Help: "Only addresses containing this text", Query: true, Key: "q"}, + ), + Table: output.Table{Root: "data", Columns: []output.Column{ + colt("MAILBOX", "sender_email", 34), col("PROVIDER", "sender_family_label"), col("STATUS", "status"), + col("INBOX", "summary.inbox"), col("SPAM", "summary.spam"), col("MISSING", "summary.missing"), + colt("REASON", "detail", 40), + }, Empty: "No mailboxes match."}, + }, + { + Name: "batch-cancel", Short: "Stop a batch; copies already sent keep being classified", + Method: http.MethodPost, Path: "/placement/batches/{id}/cancel", Body: bodyOptional, + Args: []argSpec{{Name: "id", Help: "The batch's id"}}, + Success: "Placement batch cancelled.", + }, + { + Name: "coverage", Short: "How many mailboxes finished a placement test in the last 7 and 30 days", + Method: http.MethodGet, Path: "/placement/coverage", + }, }, } } diff --git a/cmd/consumer/main.go b/cmd/consumer/main.go index e78e19928..0911f2d18 100644 --- a/cmd/consumer/main.go +++ b/cmd/consumer/main.go @@ -525,6 +525,8 @@ func main() { // Searches the mailbox for deletion strikes recorded before removals were // checked, withdrawing any whose message is still there. go jobsService.StartWarmupTamperingRecheck(ctx) + // Attributes each warmup email moved to spam once the activity around it settles. + go jobsService.StartWarmupSpamMoveAttribution(ctx) go jobsService.StartWarmupPlacementSweep(ctx) go jobsService.StartPendingWarmupVerification(ctx) // Re-offers inbound mail that reply processing never claimed, so a diff --git a/cmd/warmblyctl/api_resources.go b/cmd/warmblyctl/api_resources.go index 68e01dae8..c9bdc2eeb 100644 --- a/cmd/warmblyctl/api_resources.go +++ b/cmd/warmblyctl/api_resources.go @@ -72,6 +72,11 @@ var apiSpecs = []apiSpec{ {name: "campaign lead-hold", summary: "Whether one lead's flow is held", method: "GET", path: "/campaigns/{id}/leads/{child}/hold", child: "contact"}, {name: "campaign pause-lead", summary: "Hold one lead's flow until a date, or until resumed", method: "POST", path: "/campaigns/{id}/leads/{child}/pause", body: bodyOptional, child: "contact"}, {name: "campaign resume-lead", summary: "Lift one lead's hold now", method: "POST", path: "/campaigns/{id}/leads/{child}/resume", child: "contact"}, + // Contacts copied on every email to one lead. --data carries + // {"contact_ids": ["", ...]}, at most two; [] copies nobody. + {name: "campaign lead-cc", summary: "Contacts copied on one lead's emails", method: "GET", path: "/campaigns/{id}/leads/{child}/cc", child: "contact"}, + {name: "campaign set-lead-cc", summary: "Replace the contacts copied on one lead's emails", method: "PUT", path: "/campaigns/{id}/leads/{child}/cc", body: bodyRequired, child: "contact"}, + {name: "campaign lead-cc-suggestions", summary: "The lead's likely colleagues to copy", method: "GET", path: "/campaigns/{id}/leads/{child}/cc/suggestions", child: "contact"}, // Contacts. {name: "contact list", summary: "List or search contacts; --data carries the filter body", method: "POST", path: "/contacts/search", body: bodyOptional, query: []string{"limit", "cursor"}}, diff --git a/docs/content/docs/api/cli.mdx b/docs/content/docs/api/cli.mdx index 2fe074378..a1f22ab0b 100644 --- a/docs/content/docs/api/cli.mdx +++ b/docs/content/docs/api/cli.mdx @@ -227,7 +227,7 @@ $ warmbly campaign start 6f1c… ! `warmbly campaign start` sends real mail. Continue? [y/N] ``` -`--yes` is the only way past it, which makes it the flag to grep for in a script review. The commands that behave this way are `campaign start`, `campaign test`, `placement test`, `mailbox send`, `inbox reply`, `inbox compose` and `inbox approve-draft`. +`--yes` is the only way past it, which makes it the flag to grep for in a script review. The commands that behave this way are `campaign start`, `campaign test`, `placement test`, `placement batch-start`, `mailbox send`, `inbox reply`, `inbox compose` and `inbox approve-draft`. ## Commands @@ -238,7 +238,7 @@ Run `warmbly --help` for the flags, and `warmbly --help` for the flags, and `warmbly ` is still accepted as the older spelling. Exactly one column must map to `email`. A `verification_status` column is read in the vocabulary named by `verification_provider` (see [Create contacts](#create-contacts)), or recognised value by value when it is omitted; a cell nobody recognises leaves that contact unverified rather than failing the row. The preview suggests this target itself when a column's header or values look like another service's results. | +| `mapping` | array | Yes | Column mappings: `{ "index", "target", "custom_key", "verification_provider" }`. `target` is `ignore`, `email`, `first_name`, `last_name`, `company`, `phone`, `subscribed`, `categories` (label names), `verification_status`, or `custom` with the name in `custom_key`. `custom:` is still accepted as the older spelling. Exactly one column must map to `email`. A `verification_status` column is read in the vocabulary named by `verification_provider` (see [Create contacts](#create-contacts)), or recognised value by value when it is omitted; a cell nobody recognises leaves that contact unverified rather than failing the row. The preview suggests this target itself when a column's header or values look like another service's results. | | `dedup` | string | Yes | `skip`, `update`, or `create_duplicate` for rows whose email matches an existing contact. | | `has_header` | boolean | Yes | Whether the first row is a header. | -| `category_ids` | string[] | No | Categories to assign to imported contacts. | +| `category_ids` | string[] | No | Label IDs to put on imported contacts. | | `campaign_ids` | string[] | No | Campaigns to add imported contacts to. | | `segment_ids` | string[] | No | Segments to pin imported contacts into, as a manual include override. Applies to imported, updated, and skipped-but-linked contacts alike. An id that is not a valid UUID, or that names no segment in the organization, is rejected with `400` before any row is written. | | `subscribed_default` | boolean | No | Subscription state for new contacts when no subscribed column is mapped. Defaults to true. | @@ -523,7 +523,7 @@ Reads a draft under a mapping and writes nothing. `409` once the import has star } ``` -Every row lands in exactly one of `new`, `existing`, `duplicates_in_file`, `invalid`, or `conflicts` (an address the caller holds as a contact in another organization). `invalid_samples` lists up to 25 of the invalid and conflicting rows. `problem`, when present, is why starting would be refused as a whole: the plan's contact limit, or too many distinct categories. +Every row lands in exactly one of `new`, `existing`, `duplicates_in_file`, `invalid`, or `conflicts` (an address the caller holds as a contact in another organization). `invalid_samples` lists up to 25 of the invalid and conflicting rows. `problem`, when present, is why starting would be refused as a whole: the plan's contact limit, or too many distinct labels. ### Start an import @@ -739,7 +739,7 @@ When the contact is suppressed, `suppression` is an object: `{ "id", "kind", "va `PATCH /contacts/:id` -Partially updates a single contact. Only the fields present are changed. Campaign and category lists can be set wholesale or adjusted with diff-style add/remove. +Partially updates a single contact. Only the fields present are changed. Campaign and label lists (`categories`) can be set wholesale or adjusted with diff-style add/remove. `email` replaces the contact's address. It is stored lowercased, a display name (`Dana Reyes `) is reduced to the address inside it, and anything that is not an address answers `400`. The address has to be free: one another contact already holds answers `409` with `code` `contact_email_taken` rather than merging the two. A changed address drops the contact's verification verdict back to `unknown`, clears the delivery evidence behind it and forgets its `mail_host` and `esp_provider`, because all of them belonged to the old mailbox; the next verification pass checks the new address and the provider is read again from the new domain. Steps already sent went to the old address and keep their history. @@ -761,7 +761,7 @@ Auth: **Scope** `WRITE_CONTACTS` · **Org permission** `manage_contacts` | `custom_fields` | object | No | Replaces the custom-fields map. | | `subscribed` | boolean | No | Subscription status. | | `campaigns` | string[] | No | Set the full campaign membership (nil leaves as-is). | -| `categories` | string[] | No | Set the full category list (nil leaves as-is). | +| `categories` | string[] | No | Set the full list of label IDs (nil leaves as-is). | | `add_categories` | string[] | No | Diff-style add (ignored when `categories` is set). | | `remove_categories` | string[] | No | Diff-style remove (ignored when `categories` is set). | @@ -865,7 +865,7 @@ Returns a `data` array plus a `pagination` envelope. `GET /contacts/:id/timeline` -Returns the selected organization's merged activity feed for a contact: sends, opens, clicks (one per link, naming the link), replies, bounces, deliverability and suppression events, notes, meeting bookings, and lifecycle events (the contact's creation with its first-touch source, and every time it joined or left a campaign or a category). Every member with permission to view contacts receives the same timeline, regardless of who created the contact or its campaigns. A request with no selected organization returns `400`, and a contact outside the selected organization returns `404`. +Returns the selected organization's merged activity feed for a contact: sends, opens, clicks (one per link, naming the link), replies, bounces, deliverability and suppression events, notes, meeting bookings, and lifecycle events (the contact's creation with its first-touch source, and every time it joined or left a campaign or gained or lost a label). Every member with permission to view contacts receives the same timeline, regardless of who created the contact or its campaigns. A request with no selected organization returns `400`, and a contact outside the selected organization returns `404`. Auth: **Scope** `READ_CONTACTS` · **Org permission** `view_contacts` @@ -947,7 +947,7 @@ Returns a `data` array and the standard `pagination` envelope. Paginate by passi A `form_submitted` event carries `form_id` and `form_name`. A `page_hit` event is a page view on your own site from a browser tied to the contact through an email-link ticket (see [Website tracking](/guides/website-tracking/)); `subject` is the page title, or its path when the page has none, and `page_hit` carries the full view: `url`, `path`, `title`, `referrer`, `referrer_domain`, `landing` (the first view of a session), the `utm_*` parameters, `device_type`, `os`, `browser`, `browser_version`, `device_brand`, `language`, `timezone`, `screen_width`, `screen_height`, and `country_code`, `region`, `city` when known. -Lifecycle events carry the name of what changed as it was at the time (`campaign_name`, or `category_id` plus `category_title`), so a later rename or deletion does not rewrite history. A `contact_created` event carries `source` (`manual`, `campaign`, `import`, `sheet_sync`, `api`, `form`, `automation`, `ai_assistant`, or `unknown` for contacts that predate attribution) and `source_detail` (the file, campaign, sheet, form, automation or API key name). The same values are on the contact itself as `source`, `source_detail` and `first_seen_at`, and never change after creation. +Lifecycle events carry the name of what changed as it was at the time (`campaign_name`, or, for a label, `category_id` plus `category_title`), so a later rename or deletion does not rewrite history. A `contact_created` event carries `source` (`manual`, `campaign`, `import`, `sheet_sync`, `api`, `form`, `automation`, `ai_assistant`, or `unknown` for contacts that predate attribution) and `source_detail` (the file, campaign, sheet, form, automation or API key name). The same values are on the contact itself as `source`, `source_detail` and `first_seen_at`, and never change after creation. ## Get a contact's campaign state @@ -1008,7 +1008,7 @@ Auth: **Scope** `READ_CONTACTS` · **Org permission** `view_contacts` `sender_id` and `sender_email` are the mailbox this lead's whole sequence sends from. Rotation picks it when the first email goes out and every follow-up keeps it, so the contact only ever hears from one address; both fields are absent until that first email. They change only when that mailbox can no longer send for the campaign. -`lead_status` uses the same values as the campaign Leads view: `pending`, `active`, `completed`, `replied`, `bounced`, `failed`, `paused`, `unsubscribed`, or `undeliverable`. A held lead also carries a `hold` object (`since`, `until`, `reason`, `source`) and keeps its `next` action, with the hold as the reason it is waiting; see [pause a lead](/api/reference/campaigns/#pause-a-lead). Each step carries whichever of `sent_at`, `opened_at`, `clicked_at`, `replied_at`, `bounced_at` and `failed_at` apply, plus `attempts` and `in_flight` (reserved for a worker whose result has not come back). `opened_at` is a person's open, as it is in the Leads view: a step a mail client prefetched or a security gateway scanned carries no `opened_at`. While a branch condition is undecided, `next.step_id` is absent and `next.step_label` says the step depends on the contact's response. +`lead_status` uses the same values as the campaign Leads view: `pending`, `active`, `completed`, `replied`, `bounced`, `failed`, `paused`, `unsubscribed`, or `undeliverable`. A held lead also carries a `hold` object (`since`, `until`, `reason`, `source`) and keeps its `next` action, with the hold as the reason it is waiting; see [pause a lead](/api/reference/campaigns/#pause-a-lead). `cc` lists the contacts copied on every email to the contact in that campaign, empty when none; see [get a lead's CC](/api/reference/campaigns/#get-a-leads-cc). Each step carries whichever of `sent_at`, `opened_at`, `clicked_at`, `replied_at`, `bounced_at` and `failed_at` apply, plus `attempts` and `in_flight` (reserved for a worker whose result has not come back). `opened_at` is a person's open, as it is in the Leads view: a step a mail client prefetched or a security gateway scanned carries no `opened_at`. While a branch condition is undecided, `next.step_id` is absent and `next.step_label` says the step depends on the contact's response. ## List a contact's activities @@ -1257,7 +1257,7 @@ Each condition names a `field`, an `operator`, and either a `value` (scalar oper | bool | `subscribed`, `suppressed`, `is_catch_all` | `is_true`, `is_false` | none | | date | `created_at`, `updated_at`, `last_sent_at`, `last_opened_at`, `last_clicked_at`, `last_replied_at` | `within_days`, `not_within_days` (`value` is a day count, 1 to 3650); `before`, `after` (`value` is `YYYY-MM-DD` or RFC 3339); `is_empty`, `is_not_empty` | see operators | | number | `campaign_count`, `emails_sent`, `emails_opened`, `emails_clicked`, `emails_replied`, `emails_bounced` | `equals`, `not_equals`, `gt`, `gte`, `lt`, `lte` | `value`, a whole number | -| category | `category` | `in`, `not_in`, `is_empty`, `is_not_empty` | `values`, category ids | +| category | `category` | `in`, `not_in`, `is_empty`, `is_not_empty` | `values`, label ids | | campaign | `campaign` | `in`, `not_in`, `is_empty`, `is_not_empty` | `values`, campaign ids | | segment | `segment` | `in`, `not_in` | `values`, segment ids; at most five levels deep, no loops | @@ -1287,7 +1287,7 @@ Counts the contacts an unsaved definition would match. Send `match` and `conditi { "contacts": ["…", "…"], "mode": "include" } ``` -`mode` is `include` (pin in), `exclude` (pin out) or `auto` (clear the override). Up to 1,000 contact ids per call; ids outside the organization are ignored. Returns `{ "updated": n }`. The body also takes a [filter selection](#selecting-contacts-for-a-bulk-action) (`all`, `filters`, `exclude`) instead of `contacts`, for pinning everything a search matches. +`mode` is `include` (pin in), `exclude` (pin out) or `auto` (clear the override). Up to 10,000 contact ids per call, or a filter selection of up to 250,000; ids outside the organization are ignored. Returns `{ "updated": n }`. The body also takes a [filter selection](#selecting-contacts-for-a-bulk-action) (`all`, `filters`, `exclude`) instead of `contacts`, for pinning everything a search matches. `POST /segments/:id/members/lookup` takes `{ "contacts": [...] }` and returns `{ "data": { "": "include" | "exclude" } }` for the contacts that carry an override. diff --git a/docs/content/docs/api/reference/integrations.mdx b/docs/content/docs/api/reference/integrations.mdx index f89f494e2..8da268263 100644 --- a/docs/content/docs/api/reference/integrations.mdx +++ b/docs/content/docs/api/reference/integrations.mdx @@ -1123,7 +1123,7 @@ Auth: **Scope** `WRITE_CONTACTS` · **Org permission** `manage_contacts`. | `column_mapping` | array | Yes | Column-to-target mappings ([same shape as contact import](/api/reference/contacts/)). Validated on save, not on the next sync: a mapping with no `email` column, or a custom field whose name Warmbly cannot use, is a `400` here. | | `dedup` | string | No | Collision strategy: `skip`, `update`, or `create_duplicate`. | | `target_campaign_id` | uuid | No | Enrol new/updated leads into this campaign on each sync. | -| `category_ids` | string[] | No | Categories to assign to synced leads. | +| `category_ids` | string[] | No | Label IDs to put on synced leads. | | `segment_ids` | string[] | No | Segments every synced row is pinned into as a manual include override, on every run. Each id must name a segment in the organization or the save is a `400`; a segment deleted later is dropped from the run instead of failing it. | | `subscribed_default` | boolean | No | Default subscription state for new contacts. | | `label` | string | No | Friendly name. | @@ -1215,7 +1215,7 @@ Auth: **Scope** `WRITE_CONTACTS` · **Org permission** `manage_contacts`. | `dedup` | string | No | Collision strategy. | | `target_campaign_id` | uuid | No | New target campaign. | | `clear_campaign` | boolean | No | When `true`, unsets the target campaign. | -| `category_ids` | string[] | No | Replacement category set. | +| `category_ids` | string[] | No | Replacement set of label IDs. | | `segment_ids` | string[] | No | Replacement segment target set. An unknown id is a `400`; an empty array clears the targets. | | `subscribed_default` | boolean | No | Default subscription state. | | `label` | string | No | New label. | diff --git a/docs/content/docs/api/reference/placement.mdx b/docs/content/docs/api/reference/placement.mdx index 953a1024b..f595f5bdf 100644 --- a/docs/content/docs/api/reference/placement.mdx +++ b/docs/content/docs/api/reference/placement.mdx @@ -1,13 +1,13 @@ --- title: Inbox placement tests -description: Start placement tests, read where each copy landed, manage the workspace's seed inboxes, and schedule a campaign's placement monitor. +description: Start placement tests and batches, read where each copy landed, manage the workspace's seed inboxes, and schedule a campaign's placement monitor. --- An inbox placement test sends a template or a campaign step from one of the workspace's mailboxes to a panel of seed inboxes and reports where each copy landed: the primary inbox, a Gmail tab, spam, or nowhere. The [placement tests guide](/guides/placement-tests/) explains how copies are rendered, sent and found, and how to read a result. Every route is scoped to the selected organization (API keys are always bound to one), and every id in a request is checked against it: a sending mailbox, campaign, step or contact from another workspace is a `404`. Errors follow the shared `{error, message, code, request_id}` envelope; the codes these routes add are listed under [placement test refusals](/api/error-codes/#placement-test-refusals). -Changes are pushed live as a `PLACEMENT_TEST_UPDATED` event on the organization's [realtime](/api/realtime/) channel when a test starts, on every new verdict, and when it finishes or is cancelled. +Changes are pushed live as a `PLACEMENT_TEST_UPDATED` event on the organization's [realtime](/api/realtime/) channel when a test starts, on every new verdict, and when it finishes or is cancelled. A batch sends the same event with `batch_id` instead of `test_id` when it starts, skips or defers a mailbox, and finishes. ## Get the overview @@ -163,6 +163,8 @@ Auth: **Scope** `READ_ANALYTICS` · **Org permission** `view_analytics` | `cursor` | query | string | Opaque cursor from a previous `pagination.next_cursor`. One this API did not issue is a `400` | | `campaign_id` | query | uuid | Only tests of this campaign | +Tests a [batch](#start-a-batch) started are left out; read them through their batch. + ### Response ```json @@ -202,7 +204,8 @@ Each test carries the fields shown under [start a test](#start-a-test); the exam | Field | Description | |-------|-------------| | `status` | `running`, `completed`, `cancelled` or `failed`. A test fails when no copy was delivered at all, with the reason in `error` | -| `origin` | `manual` (started by a person or a key), `monitor` (a campaign's placement monitor), `admin` (an operator) or `remote` | +| `origin` | `manual` (started by a person or a key), `monitor` (a campaign's placement monitor), `batch` (a [placement batch](#start-a-batch), named by `batch_id`), `admin` (an operator) or `remote` | +| `batch_id` | The batch that started the test, `null` otherwise | | `pace` | `spaced` or `quick` | | `credits_charged` | Credits the test cost past the month's free tests, `0` for a free test | | `credits_refunded` | Credits a paid test that delivered no copy got back. Monthly credits whose allowance has reset since the charge are not returned, so this can be less than `credits_charged` | @@ -272,6 +275,235 @@ Auth: **Scope** `SEND_CAMPAIGNS` · **Org permission** `send_campaigns` `200 OK` with the test, in the list shape. A test that has already finished answers `409` with `placement_not_running`. +## Start a batch + +`POST /placement/batches` + +Runs the same test from many sending mailboxes. The request is validated as a whole (the copy, panel, tracking, allowance and credits agreed), the senders are resolved and written down, and the batch is answered `queued` right away. Nothing is sent by the request itself: the backend starts the senders a few at a time, each with its own test, checking each one's daily limit, connection and worker when its turn comes. See [testing a fleet with batches](/guides/placement-tests/#testing-a-fleet-with-batches). Supports `Idempotency-Key`. + +Auth: **Scope** `SEND_CAMPAIGNS` · **Org permission** `send_campaigns` + +### Request body + +The copy fields are the ones [start a test](#start-a-test) takes: `campaign_id`, `sequence_id`, `contact_id`, `subject`, `body_html`, `body_plain`, `tracking`, `panel`, `families` and `seed_ids`. `pace` is always `spaced` for a batch; `quick` is a `400`. The copy is resolved once, when the batch is created, so every sender tests the same email. Instead of `sender_account_id` a batch takes exactly one of: + +| Field | Type | Description | +|-------|------|-------------| +| `sender_account_ids` | uuid[] | These mailboxes. Duplicates are dropped; one that is not in the workspace, or is a seed inbox, is a `404`, and one an API key with a mailbox allowlist may not use is a `403`. A disconnected one is kept and deferred or skipped when its turn comes | +| `sender_scope` | object | Mailboxes resolved on the server, so a fleet of thousands needs no id list | +| `sender_scope.type` | string | `campaign` (every mailbox `campaign_id`'s scheduler sends from) or `workspace` (every mailbox of the workspace) | +| `sender_scope.campaign_id` | uuid | Required with `type` `campaign` | +| `sender_scope.providers` | string[] | Only mailboxes hosted by these families, such as `google_workspace`, `gmail`, `microsoft365`, `outlook` or `other` | +| `sender_scope.domains` | string[] | Only mailboxes sending from these domains | +| `sender_scope.tag_ids` | uuid[] | Only mailboxes carrying one of these tags | +| `sender_scope.include_inactive` | boolean | Keep disconnected mailboxes. Default `false` | +| `sender_scope.untested_days` | integer | Only mailboxes with no finished placement test in this many days, `1` to `365` | + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `sample` | object | no | Which of the resolved mailboxes to keep. Default `{"mode": "all"}` | +| `sample.mode` | string | no | `all`, `random` (`count` mailboxes), `percent` (`percent` of them, rounded up), `per_domain` or `per_provider` (up to `count` from each) | +| `sample.stratify` | string | no | With `random` or `percent` only: `provider` or `domain` splits the sample in proportion to each group's share | +| `on_unavailable` | string | no | `defer` (default: retry a mailbox that cannot send when its turn comes, for up to seven days) or `skip` | +| `max_credits` | integer | no | The most credits you agree to pay across the whole batch for tests past the month's free ones. Needed only when the batch needs more tests than are free: the request is refused with `placement_quota_exceeded`, naming the most it can cost, when it is lower. The batch never spends more | + +```json +{ + "sender_scope": { "type": "campaign", "campaign_id": "b1f2c3d4-0000-0000-0000-000000000001", "providers": ["google_workspace"] }, + "sample": { "mode": "percent", "percent": 10, "stratify": "domain" }, + "campaign_id": "b1f2c3d4-0000-0000-0000-000000000001", + "sequence_id": "c7d8e9f0-0000-0000-0000-000000000002", + "tracking": "compare" +} +``` + +### Response + +`201 Created` with the batch. + +```json +{ + "data": { + "id": "a9b8c7d6-0000-0000-0000-000000000030", + "created_by": "9c2a0000-0000-0000-0000-000000000001", + "campaign_id": "b1f2c3d4-0000-0000-0000-000000000001", + "sequence_id": "c7d8e9f0-0000-0000-0000-000000000002", + "contact_id": "e5f6a7b8-0000-0000-0000-000000000004", + "subject": "Quick question about outbound at Acme", + "tracking": "compare", + "panel": "instance", + "pace": "spaced", + "families": [], + "seed_ids": [], + "on_unavailable": "defer", + "selection": { + "sender_scope": { "type": "campaign", "campaign_id": "b1f2c3d4-0000-0000-0000-000000000001", "providers": ["google_workspace"] }, + "sample": { "mode": "percent", "percent": 10, "stratify": "domain" }, + "matched": 1126 + }, + "sender_count": 113, + "max_credits": 0, + "credits_spent": 0, + "status": "queued", + "retry_until": "2026-10-02T14:00:00Z", + "created_at": "2026-09-25T14:00:00Z", + "started_at": null, + "finished_at": null, + "progress": { "total": 113, "queued": 113, "deferred": 0, "running": 0, "completed": 0, "skipped": 0, "failed": 0, "cancelled": 0 }, + "summary": { "total": 0, "pending": 0, "inbox": 0, "promotions": 0, "other": 0, "spam": 0, "missing": 0, "failed": 0, "cancelled": 0, "delivered": 0, "inbox_rate": null, "tabs_rate": null, "spam_rate": null, "missing_rate": null } + } +} +``` + +| Field | Description | +|-------|-------------| +| `status` | `queued` until the first mailbox starts, then `running`; at the end `completed`, `completed_with_warnings` (some mailboxes were skipped or failed), `failed` (none finished a test, with `error`) or `cancelled` | +| `selection` | How the mailboxes were chosen, and how many the scope `matched` before sampling. The mailboxes themselves are the snapshot under [list a batch's senders](#list-a-batchs-senders) | +| `progress` | The batch's mailboxes by status. `running` is a mailbox whose test has not finished yet | +| `summary` | Where the batch's copies landed so far, over the copy the campaign really sends: the tracked half of a tracking comparison | +| `retry_until` | Deferred mailboxes are retried until then, then skipped | + +## Preview a batch + +`POST /placement/batches/preview` + +Takes the body of [start a batch](#start-a-batch) and returns what it would come to, without writing anything or sending anything. The copy fields may be left out to count the mailboxes alone; once any is given, the copy is checked as on start. A random sample is drawn again when the batch starts, so the counts are the same but the mailboxes can differ. + +Auth: **Scope** `SEND_CAMPAIGNS` · **Org permission** `send_campaigns` + +### Response + +```json +{ + "data": { + "matched": 1347, "selected": 135, "inactive": 2, "domains": 41, + "providers": [ + { "key": "google_workspace", "label": "Google Workspace", "senders": 80 }, + { "key": "other", "label": "Other provider", "senders": 40 }, + { "key": "microsoft365", "label": "Microsoft 365", "senders": 15 } + ], + "variants": 2, "tests": 270, "seeds_per_test": 20, "max_sends": 5400, + "metered": true, "free_tests": 40, "paid_tests": 230, "credits": 5750, + "usage": { "used": 0, "limit": 40, "credits_per_test": 25, "credit_balance": 9000, "period_start": "2026-09-01T00:00:00Z", "period_end": "2026-10-01T00:00:00Z" }, + "senders_max": 10000, "concurrency": 20 + } +} +``` + +| Field | Description | +|-------|-------------| +| `variants` | `2` for a tracking comparison, which doubles `tests` and `max_sends` | +| `max_sends` | The most copies the batch sends: `tests` times `seeds_per_test`. Each mailbox's daily limit can only make it fewer | +| `free_tests`, `paid_tests`, `credits` | How `tests` splits against the month's free tests, and the most the paid ones cost. All `0` on an unmetered panel | +| `concurrency` | How many of the workspace's batch mailboxes send at once | + +## List batches + +`GET /placement/batches` + +Returns the workspace's batches, newest first, in the shape above. Takes `limit` (`1` to `100`, default `25`) and an opaque `cursor`, and returns `data` plus `pagination`. + +Auth: **Scope** `READ_ANALYTICS` · **Org permission** `view_analytics` + +## Get a batch + +`GET /placement/batches/:id` + +Returns one batch with its placement grouped across the fleet. + +Auth: **Scope** `READ_ANALYTICS` · **Org permission** `view_analytics` + +### Response + +The batch's fields as above, plus: + +| Field | Description | +|-------|-------------| +| `untracked` | On a tracking comparison only: the untracked half's counts. `summary` is the tracked half | +| `domains` | Per sending domain, worst inbox rate first: `key` (the domain), `senders` in the batch, `tested` (senders that finished a test) and `counts` | +| `providers` | The same per sending provider, `key` a host family and `label` its name | +| `recipients` | Counts per recipient provider family, like a test's `families` | +| `matrix` | Per sending domain, in the order of `domains`, its counts per recipient provider in the order of `recipients` | +| `content` | The content score of the batch's copy | + +The groups read the same copy as `summary`. They are not broken down by worker: a worker signs in to the mailbox provider, which delivers from its own servers, so a recipient's filter never sees it. + +## List a batch's senders + +`GET /placement/batches/:id/senders` + +Returns the batch's mailboxes with where each one's copies landed. + +Auth: **Scope** `READ_ANALYTICS` · **Org permission** `view_analytics` + +| Parameter | In | Type | Description | +|-----------|----|------|-------------| +| `limit` | query | integer | `1` to `100`, default `25` | +| `cursor` | query | string | Opaque cursor from a previous `pagination.next_cursor` | +| `sort` | query | string | `worst` (default: lowest inbox rate first, untested last), `best`, `email` or `status` | +| `status` | query | string | Only mailboxes in this status: `queued`, `deferred`, `running`, `completed`, `skipped`, `failed` or `cancelled` | +| `q` | query | string | Only addresses containing this text | + +### Response + +```json +{ + "data": [ + { + "id": "c0c1c2c3-0000-0000-0000-000000000040", + "batch_id": "a9b8c7d6-0000-0000-0000-000000000030", + "email_account_id": "a0a1a2a3-0000-0000-0000-000000000003", + "sender_email": "john@domain2.com", + "sender_domain": "domain2.com", + "sender_family": "google_workspace", + "sender_family_label": "Google Workspace", + "status": "completed", + "attempts": 1, + "next_attempt_at": "2026-09-25T14:00:00Z", + "started_at": "2026-09-25T14:02:00Z", + "finished_at": "2026-09-25T16:40:00Z", + "summary": { "total": 20, "pending": 0, "inbox": 8, "promotions": 0, "other": 0, "spam": 12, "missing": 0, "failed": 0, "cancelled": 0, "delivered": 20, "inbox_rate": 0.4, "tabs_rate": 0, "spam_rate": 0.6, "missing_rate": 0 }, + "test_ids": ["d1e2f3a4-0000-0000-0000-000000000011", "d1e2f3a4-0000-0000-0000-000000000012"] + } + ], + "pagination": { "total": 113, "has_more": true, "next_cursor": "MjU" } +} +``` + +| Field | Description | +|-------|-------------| +| `status` | `queued`, `deferred` (waiting to retry at `next_attempt_at`), `running`, `completed`, `skipped`, `failed` or `cancelled` | +| `reason`, `detail` | Why a mailbox was deferred, skipped or failed: a code from [placement test refusals](/api/error-codes/#placement-test-refusals) and its sentence | +| `test_ids` | The mailbox's tests, readable with [get a test](#get-a-test): the untracked half first on a comparison | + +## Cancel a batch + +`POST /placement/batches/:id/cancel` + +Stops the batch: no mailbox starts again, copies not sent yet are cancelled, and copies already sent keep being classified. + +Auth: **Scope** `SEND_CAMPAIGNS` · **Org permission** `send_campaigns` + +### Response + +`200 OK` with the batch. A batch that has already finished answers `409` with `placement_batch_not_running`. + +## Get fleet coverage + +`GET /placement/coverage` + +How many of the workspace's connected sending mailboxes finished a placement test recently. + +Auth: **Scope** `READ_ANALYTICS` · **Org permission** `view_analytics` + +### Response + +```json +{ + "data": { "mailboxes": 1347, "tested_7d": 312, "tested_30d": 947, "never_tested": 126 } +} +``` + ## List seed inboxes `GET /placement/seeds` diff --git a/docs/content/docs/api/reference/unibox.mdx b/docs/content/docs/api/reference/unibox.mdx index 760ade95b..d35236a0f 100644 --- a/docs/content/docs/api/reference/unibox.mdx +++ b/docs/content/docs/api/reference/unibox.mdx @@ -201,7 +201,7 @@ The response is a `data` plus `pagination` envelope. Each item is a full message `GET /unibox/thread/labels` -Returns the conversation labels (the workspace's categories) attached to a thread. Auth: **Scope** `READ_UNIBOX` · **Org permission** `access_unibox`. +Returns the labels attached to a thread, drawn from the same workspace list as contact labels (managed through `/categories`). Auth: **Scope** `READ_UNIBOX` · **Org permission** `access_unibox`. | Parameter | In | Type | Description | | --- | --- | --- | --- | @@ -223,14 +223,14 @@ The response wraps the labels in a `data` array. `PUT /unibox/thread/labels` -Replaces the full conversation-label set on a thread, for the whole workspace. The body's `category_ids` is the desired set, so the call is idempotent and retries are naturally safe. Only the workspace's own categories are attached; an id belonging to another organization is dropped. Auth: **Scope** `WRITE_UNIBOX` · **Org permission** `access_unibox`. +Replaces the full conversation-label set on a thread, for the whole workspace. The body's `category_ids` is the desired set, so the call is idempotent and retries are naturally safe. Only the workspace's own labels are attached; an id belonging to another organization is dropped. Auth: **Scope** `WRITE_UNIBOX` · **Org permission** `access_unibox`. ### Request body | Field | Type | Required | Description | | --- | --- | --- | --- | | `thread_id` | string | Yes | The thread to label. | -| `category_ids` | string[] | No | The full desired set of category UUIDs. An empty array clears all labels. | +| `category_ids` | string[] | No | The full desired set of label UUIDs. An empty array clears all labels. | ```json { diff --git a/docs/content/docs/development/architecture.mdx b/docs/content/docs/development/architecture.mdx index bf68694fb..5cb261191 100644 --- a/docs/content/docs/development/architecture.mdx +++ b/docs/content/docs/development/architecture.mdx @@ -169,6 +169,8 @@ An empty `desired_version` means "no opinion" and must never be read as "downgra The backend is deliberately excluded from this. It is what tells everyone else their version, and a self-update that goes wrong leaves nothing to recover with. +On an instance that encodes the bus with Avro, the fleet also waits for the backend. A booting backend registers every schema its release publishes, and the fleet only moves to a release once the backend runs it and the Schema Registry has accepted those schemas. A refusal holds the fleet on the release it has and shows in the Fleet section, instead of reaching a worker that would then fail every event it reports. + The release check runs once when the backend boots, and only with `RELEASES_ENABLED=true`: it is off by default, so a self-hosted fleet is never rolled onto a vendor image. An operator moves the fleet by hand from the admin panel's Fleet section or with `warmblyctl fleet version` and `warmblyctl fleet channel`, which set the channel or pin a tag directly. Configuration is env-driven (`RELEASES_ENABLED`, `RELEASES_GITHUB_REPO`, `RELEASES_WORKER_IMAGE_REPO`, `RELEASES_GITHUB_TOKEN`) so self-hosters can point at their own fork and registry. See `internal/app/releases/service.go`. diff --git a/docs/content/docs/development/configuration.mdx b/docs/content/docs/development/configuration.mdx index 935c90b85..e47e18521 100644 --- a/docs/content/docs/development/configuration.mdx +++ b/docs/content/docs/development/configuration.mdx @@ -312,7 +312,7 @@ A worker's command topic is named after the node id issued when it joined, so th -`json` needs nothing and is the default. `avro` resolves every event against `SCHEMA_REGISTRY_URL` and ships only in the `-kafka` images. Producers and consumers have to agree on one: there is no in-band marker, so a consumer on the other codec cannot read what is already on the bus. Change it by draining the bus, not in place. `PUBSUB_ENABLED` must agree across backend, consumer and realtime. +`json` needs nothing and is the default. `avro` resolves every event against `SCHEMA_REGISTRY_URL` and ships only in the `-kafka` images. Producers and consumers have to agree on one: there is no in-band marker, so a consumer on the other codec cannot read what is already on the bus. Change it by draining the bus, not in place. The envelope subjects need `BACKWARD` compatibility, because a new event type adds a union branch; a publisher that meets `FORWARD` or `FULL` sets its subject to `BACKWARD` itself. `PUBSUB_ENABLED` must agree across backend, consumer and realtime. `KAFKA_TRACKING_TOPIC` is read by the Rust publisher and the Go subscriber. Override it in one place only and opens and clicks stop being consumed, with no error anywhere. @@ -797,6 +797,10 @@ These are the only settings a browser can change, and no environment variable ow | `placement.seeds_per_test` | integer, 1 to 100 | `20` | The most seed inboxes one test sends to. Every copy is a send from the sending mailbox's daily limit, so this is also how much of that limit one test takes. A test is sized down to what the mailbox has left that day, and refused below five seeds | | `placement.spacing_seconds` | integer, 5 to 600 | `60` | The average gap between two copies of a test from one mailbox, varied by up to a third either way. A test started as quick uses the smaller of this and 8 seconds | | `placement.credits_per_test` | integer, 0 to 10000 | `25` | What a test on the instance panel costs in credits once a paid workspace has used its free tests for the month. The workspace agrees to the price before each paid test, and a test that delivers no copy is refunded. `0` turns paid tests off. Only an instance running with `DEPLOYMENT_MODE=cloud` meters, so a self-hosted instance never charges | +| `placement.batch_senders_max` | integer, 1 to 100000 | `10000` | The most sending mailboxes one [placement batch](/guides/placement-tests/#testing-a-fleet-with-batches) may hold. An infrastructure safeguard on batch size only; how many run at once is the next setting | +| `placement.batch_sender_concurrency` | integer, 1 to 500 | `20` | How many of one workspace's batch mailboxes may be sending their copies at the same time, across all its batches. The next mailbox starts when one has sent every copy; each mailbox keeps its own daily limit and minimum gap | +| `placement.batch_instance_concurrency` | integer, 1 to 5000 | `200` | The same limit across every workspace together. It bounds how much batch mail the instance seed panel takes in at once, so a seed never receives enough in an hour to trip the sync flood rule, which deactivates a mailbox | +| `placement.batch_starts_per_minute` | integer, 1 to 600 | `10` | How many mailboxes one batch starts per minute, so a large batch ramps up instead of starting its whole concurrency at once | The four `sync.*` values are read by the backend when a mailbox is loaded onto a worker (on connect, on reassignment, and by the reconciler's periodic republish), so a change reaches every mailbox within 30 minutes without a restart (the republish gives each mailbox one turn per 30 minutes, spread out so no worker gets them all at once). The fixed pacing numbers around them (burst per five minutes, hourly, backfill pace, the flood threshold and the chronic-overage rule) are compiled constants listed under **Instance > Configuration > Effective limits**; see [Mailboxes](/guides/mailboxes/#what-gets-synced) for how the budgets behave. The four `sync.*` values also have a companion read view on **Operations > Sync**, which shows each mailbox's backfill progress and fair-use throttle against them and can clear a throttle or restart a backfill. @@ -812,7 +816,7 @@ It is applied only while the settings row has never been written, so from the fi The two `deliverability.*` values are read on every scheduling pass and every warmup send, so turning the gate off releases blocked mailboxes within the 30 second cache window. Turning it back on does not stop anything retroactively: a domain still has to spend its whole grace window failing first. Only a sustained failure gates, so a domain reading `unknown` (never checked, DNS could not answer, or a special-use domain that cannot resolve) always sends. Manual sends and unibox replies are never gated; see [domain authentication](/guides/deliverability/#domain-authentication). -The four `placement.*` values are read when a test starts, so a change applies to the next test and never to one already sending. A value of zero or below resolves to the default, and one above the range to its ceiling. Which mailboxes form the instance seed panel is not a setting: an administrator marks them from the admin panel. +The `placement.*` allowance, seed and spacing values are read when a test starts, so a change applies to the next test and never to one already sending. The four `placement.batch_*` values are read on every pass of the batch runner, about every 30 seconds, so a change reaches running batches within a minute; lowering the concurrency never stops a mailbox already sending, it only delays the next start. A value of zero or below resolves to the default, and one above the range to its ceiling. Which mailboxes form the instance seed panel is not a setting: an administrator marks them from the admin panel. Changing them is audited, and every value is validated and clamped server side on write as well as on read, so a row written by an older version still resolves. diff --git a/docs/content/docs/development/data-control.mdx b/docs/content/docs/development/data-control.mdx index 77ab8c89a..9aa8c31ec 100644 --- a/docs/content/docs/development/data-control.mdx +++ b/docs/content/docs/development/data-control.mdx @@ -98,7 +98,7 @@ See [Automatic inbox tagging](/guides/inbox-tagging/), [Advisor](/guides/advisor | `retention.form_event_days` | 180 | Form funnel events: views, starts, field-level drop-off | | `retention.audit_log_days` | 90 | The audit trail: actor, IP address, user agent, change payload | | `retention.warmup_mail_days` | 30 | Warmup mail in the mailboxes themselves, and the stored copy of each message's body. Deleted by the platform once older than this, wherever the mailbox files it; a mailbox can set its own window | -| `retention.warmup_event_days` | 365 | Per-message warmup records: tokens, receipts, tampering events, spam reports. The daily sent and received counts behind the analytics are kept | +| `retention.warmup_event_days` | 365 | Per-message warmup records: tokens, receipts, tampering events, spam moves and their attribution, spam reports. The daily sent and received counts behind the analytics are kept. When each mailbox's owner last acted in it (five-minute marks, no message content) is kept 30 days whatever this is set to | The first three are between 1 and 3,650 days, and they are the settings a retention or privacy policy applies to, because each window is also how long the personal data in that log is held. The warmup mail window starts at 3 days and the warmup records window at 30, the least the engagement legs and the pool health bands need. diff --git a/docs/content/docs/development/events.mdx b/docs/content/docs/development/events.mdx index 5abe3cdc9..24f58021d 100644 --- a/docs/content/docs/development/events.mdx +++ b/docs/content/docs/development/events.mdx @@ -20,13 +20,17 @@ Message encoding is orthogonal to transport, selected by `CODEC_PROVIDER`: Schemas are derived from the Go structs (`internal/models/event_schema.go`), and each field is given the Avro default matching its zero value. This is what makes adding a field safe: a reader on the new schema can still decode data written under the schema registered before it, filling the field it did not carry from the default, so the registry accepts the new version. A newly added field with no default is rejected under `BACKWARD` compatibility, and because the publisher registers before it serializes, the rejection stops every publish on that topic rather than degrading one field. -Two things follow. The registered document is the marshalled schema, not `Schema.String()`, which omits defaults and would throw the guarantee away. And the registry stays on `BACKWARD` rather than `FORWARD`: adding a field is safe under both, but adding a new event type adds a union branch, which `BACKWARD` accepts and `FORWARD` refuses. +Two things follow. The registered document is the marshalled schema, not `Schema.String()`, which omits defaults and would throw the guarantee away. And the registry stays on `BACKWARD` rather than `FORWARD`: adding a field is safe under both, but adding a new event type adds a union branch, which `BACKWARD` accepts and `FORWARD` refuses. When a registration is refused and the subject's effective level is `FORWARD` or `FULL` (or their transitive forms), the publisher sets that subject to `BACKWARD` and registers again, so the registry key needs permission to change a subject's compatibility. Any other level, and any other refusal, is left as it is. + +Nothing in this reaches production untested. `internal/app/eventschemas` lists every schema a release publishes and keeps the one the registry last accepted under `testdata/`. CI decodes the recorded schema with the new one, which is the registry's own `BACKWARD` check, and fails a change the registry would refuse. It also fails a compatible change that has not been recorded, so a schema change is always in the diff: run `make schemas` and commit the result. On release the backend registers every listed schema when it boots, and the fleet does not move to the release until that has succeeded. + +The registry check covers the schema, not the rollout. Readers decode with the schema id each message carries, so a worker still on the previous release reads new fields fine, but a command type it has never heard of is logged and skipped. The backend rolls out before the fleet, so a new worker command type must be safe to lose while the fleet catches up: send it only to a worker whose reported version handles it, or have the backend retry it until one does. The Rust tracking service follows both switches. It speaks Kafka only when compiled with its `kafka` cargo feature, which is what the `tracking:*-kafka` image is, and it encodes with `CODEC_PROVIDER` on either transport. So a Kafka deployment on `CODEC_PROVIDER=json` needs no Schema Registry at all, and `avro` there is refused at boot without `SCHEMA_REGISTRY_URL` rather than failing at the first published event. -The consumer decodes `jobs.worker-events` and `tracking-events` with the same `CODEC_PROVIDER`, and worker envelopes cannot be Avro. That makes `json` the only value a working deployment uses, and it is why the tracking publisher has to read the setting rather than always writing Avro: a JSON consumer handed Avro drops every open and click with nothing but a deserialize warning. +The consumer decodes `jobs.worker-events` and `tracking-events` with the same `CODEC_PROVIDER`, which is why the tracking publisher has to read the setting rather than always writing Avro: a JSON consumer handed Avro drops every open and click with nothing but a deserialize warning. ## Topics @@ -77,12 +81,14 @@ Workers publish the same envelope shape back on `jobs.worker-events`: } ``` -Types: `NEW_EMAIL`, `INBOUND_BOUNCE`, `REMOVE_EMAIL`, `FLAGS_ADD`, `FLAGS_REMOVE`, `UPDATE_EMAIL`, `UPDATE_MAILBOX`, `DELETE_MAILBOX`, `TOKEN_UPDATE`, `HISTORY_ID_UPDATE`, `GRAPH_DELTA_UPDATE`, `SYNC_STATE`, `EMAIL_SENT`, `EMAIL_FAILED`, `EMAIL_AUTH_ERROR`, `EMAIL_DISABLED`, `EMAIL_RATE_LIMITED`, `EMAIL_SERVER_ERROR`, `WORKER_HEALTH`. +Types: `NEW_EMAIL`, `INBOUND_BOUNCE`, `REMOVE_EMAIL`, `FLAGS_ADD`, `FLAGS_REMOVE`, `UPDATE_EMAIL`, `UPDATE_FOLDER`, `UPDATE_MAILBOX`, `DELETE_MAILBOX`, `TOKEN_UPDATE`, `HISTORY_ID_UPDATE`, `GRAPH_DELTA_UPDATE`, `SYNC_STATE`, `EMAIL_SENT`, `EMAIL_FAILED`, `EMAIL_AUTH_ERROR`, `EMAIL_DISABLED`, `EMAIL_RATE_LIMITED`, `EMAIL_SERVER_ERROR`, `WORKER_HEALTH`. The consumer registers one handler per type (`internal/app/consumer/events.go`); unregistered types are logged and acknowledged rather than redelivered. Every `SEND_EMAIL` is answered with exactly one per-task result: `EMAIL_SENT` (the provider accepted the message; the consumer records the wire Message-ID on the task) or `EMAIL_FAILED` (a `SendEmailResult` carrying the error code and message). The control plane stamps a campaign step sent when it hands the send to the worker, so `EMAIL_FAILED` is what walks that back: the consumer marks the task failed, clears the step's `sent_at` and counts the attempt on `campaign_contact_progress`, gives the send back to the campaign's daily counters, writes the failure to the campaign activity log, and reopens a campaign that completed while the send was in flight. After `CampaignSendMaxAttempts` (5) the lead is marked failed and routing drops it; a `RECIPIENT_REJECTED` result (refused at RCPT) skips the retries and is ingested as a bounce instead. Account-level conditions (`EMAIL_AUTH_ERROR`, `EMAIL_DISABLED`, `EMAIL_RATE_LIMITED`, `EMAIL_SERVER_ERROR`) are raised in addition to, never instead of, the per-task result; they carry an `EmailErrorEvent` and act on the mailbox. A worker that does not hold the mailbox yet leaves the send for a few redeliveries before reporting it failed, and the backend never publishes a send to a worker that is not heartbeating. +`UPDATE_FOLDER` reports the canonical folder a provider now has a message in, without the rest of the message. Gmail sends it when a history record adds or removes `INBOX`, `TRASH`, `SPAM`, `SENT` or `DRAFT`, with the folder recomputed from the message's labels, and from a reconciliation that runs every six hours and once per mailbox load: the worker lists the Gmail inbox and asks the backend (`GET /api/v1/internal/sync/provider-folder-messages`) which rows it believes are in inbox, archive, spam or trash, then reports the ones Gmail has elsewhere. The consumer resolves it like `UPDATE_EMAIL`: against the folder the provider last reported, so a message filed in Warmbly stays filed unless the provider itself moved it. + `SYNC_STATE` is the worker's relay of a mailbox's sync state (backfill progress and cursor, fair-use throttle, last-synced time). It carries the full state rather than a delta, so a lost event is repaired by the next one; the consumer writes it to `email_sync_state` and the backend hands it back inside `ADD_EMAIL` on the next load, which is what lets a replaced worker resume an import instead of restarting it. ## Tracking events diff --git a/docs/content/docs/development/local-development.mdx b/docs/content/docs/development/local-development.mdx index 7f1acf978..ffd943cf5 100644 --- a/docs/content/docs/development/local-development.mdx +++ b/docs/content/docs/development/local-development.mdx @@ -179,7 +179,7 @@ Baseline (always loads): The dev user's org always loads as a mid-flight workspace, not an empty shell: - 4 warmed mailboxes on the shared worker, all in the premium warmup pool -- Folders, tags, and categories with real bindings (mailbox tags, campaign folders, contact categories, inbox thread labels) +- Folders, tags, and labels with real bindings (mailbox tags, campaign folders, labels on contacts and inbox threads) - ~30 contacts with titles and companies, a few unsubscribed or suppressed - An active 3-step campaign with 24 leads spread across the funnel (sent, opened, replied, bounced, queued), plus a draft campaign - 14 days of stats history (campaign sends, warmup volume, per-mailbox counts), always including sends today diff --git a/docs/content/docs/development/sandbox.mdx b/docs/content/docs/development/sandbox.mdx index 0b21e66a6..7e32d2c0b 100644 --- a/docs/content/docs/development/sandbox.mdx +++ b/docs/content/docs/development/sandbox.mdx @@ -44,7 +44,7 @@ Two other entry points: - a CRM pipeline with five stages, five deals across them, tasks, contact notes, and per-contact activity timelines - the analytics tables behind the deliverability and reporting views: deliverability events (opens, clicks, replies, bounces, complaints, unsubscribes), reply-intent classifications, a suppression list, and resolved and open mailbox errors - an org audit trail (campaign created and started, mailbox connected, member invited, deal created, key created) so the audit log has depth -- reply templates, notifications (some unread), and labels everywhere they appear: folders (Outbound, Nurture), mailbox tags (VIP, Cold, Agency), and categories (Lead, Customer, Churn risk) bound to the campaigns, senders, several contacts, and inbox threads +- reply templates, notifications (some unread), and labels everywhere they appear: folders (Outbound, Nurture), mailbox tags (VIP, Cold, Agency), and workspace labels (Lead, Customer, Churn risk) bound to the campaigns, senders, several contacts, and inbox threads - working credentials for every `smtp_imap` account in the database, including the older `make seed` fixtures, sealed with `CREDENTIALS_ENCRYPTION_KEY` so the worker can decrypt and use them - an [Advisor](/guides/advisor/) showcase (see below), and one evaluation run at the end of seeding so the recommendations exist before you log in diff --git a/docs/content/docs/guides/ai-assistant.mdx b/docs/content/docs/guides/ai-assistant.mdx index 51cf5ad62..1c7ebc4d0 100644 --- a/docs/content/docs/guides/ai-assistant.mdx +++ b/docs/content/docs/guides/ai-assistant.mdx @@ -32,7 +32,7 @@ The panel resizes by dragging its inner edge, which also takes the keyboard once ## What you can ask -- Find, read, create, and delete contacts, edit fields, tags, subscription, and notes, and bulk-edit +- Find, read, create, and delete contacts, edit fields, labels, subscription, and notes, and bulk-edit - Read a contact's activity timeline and the emails you sent them - List campaigns and stats, and list leads by status, which is how it finds leads that went cold - Create, edit, or delete campaigns, manage sender mailboxes and tracking domains, read logs, and edit sequence steps diff --git a/docs/content/docs/guides/ai-steps-in-automations.mdx b/docs/content/docs/guides/ai-steps-in-automations.mdx index abb6c419e..fc8d99059 100644 --- a/docs/content/docs/guides/ai-steps-in-automations.mdx +++ b/docs/content/docs/guides/ai-steps-in-automations.mdx @@ -16,11 +16,11 @@ Every mode takes a plain-language **instruction**, templated so you can drop eve | Classify | Picks exactly one of your labels | `ai_class` | | Extract fields | Pulls the fields you name out of the event text | One variable per field | -**Agent** is the default. Check which reversible actions it may take: add or remove a tag, label the email, create a task, create a deal, move a deal stage, set variables, and unsubscribe. It decides which fit, can chain several, and writes the details itself (task title, deal name, variable value). It only ever takes the actions you enable, and **never sends email or replies**. Billed one credit per step it takes. +**Agent** is the default. Check which reversible actions it may take: add or remove a label, label the conversation, create a task, create a deal, move a deal stage, set variables, and unsubscribe. It decides which fit, can chain several, and writes the details itself (task title, deal name, variable value). It only ever takes the actions you enable, and **never sends email or replies**. Billed one credit per step it takes. -For tag and label actions, an optional **pool** limits which tags it may choose. An empty pool lets it use any of yours, and optionally create a new one when nothing fits. +For the label actions, an optional **pool** limits which labels it may choose: the labels it can add or remove on the contact, and the conversation labels it can apply. An empty pool lets it use any of yours, and optionally create a new one when nothing fits. -On a Reply received trigger, an instruction like "If they ask about pricing, tag them and open a follow-up task; if they ask to stop, unsubscribe them" with those three actions enabled handles all three outcomes in one step. +On a Reply received trigger, an instruction like "If they ask about pricing, label them and open a follow-up task; if they ask to stop, unsubscribe them" with those three actions enabled handles all three outcomes in one step. **Classify** needs at least two labels and returns exactly one, stored in `ai_class`. A non-exact answer resolves to the closest match; if none is close, the raw answer is stored so you can see what happened in run history. diff --git a/docs/content/docs/guides/automations.mdx b/docs/content/docs/guides/automations.mdx index f3ab7902f..8dec66c86 100644 --- a/docs/content/docs/guides/automations.mdx +++ b/docs/content/docs/guides/automations.mdx @@ -3,7 +3,7 @@ title: Automations description: "A visual flow builder: a trigger event connected to action steps across your integrations." --- -An automation is a flow on a canvas: one **trigger** (a reply arrives, a meeting is booked) runs one or more **actions** (post to Slack, push to your CRM, tag a contact), with **IF conditions** in between to branch. No code, though an advanced mode accepts a free-form condition. +An automation is a flow on a canvas: one **trigger** (a reply arrives, a meeting is booked) runs one or more **actions** (post to Slack, push to your CRM, label a contact), with **IF conditions** in between to branch. No code, though an advanced mode accepts a free-form condition. `Dana@Acme.com` and `dana@acme.com` are the same contact, and a contact belongs to the workspace, so an address a teammate already added counts as one you have. Choose **Leave as is** (their details stay untouched) or **Update details** (empty details are filled in from the file). The API also accepts `create_duplicate`, which updates too, since one address is one contact per workspace. -Skipping still adds the contact to the campaigns, segments, and categories the import targets: it leaves their fields alone, it does not leave them out of the list. If the same address appears twice in one file it becomes one contact, and the extra rows count as skipped with the line they repeat. +Skipping still adds the contact to the campaigns, segments, and labels the import targets: it leaves their fields alone, it does not leave them out of the list. If the same address appears twice in one file it becomes one contact, and the extra rows count as skipped with the line they repeat. An address you already hold as a contact in another workspace you belong to cannot be added to this one, and that row fails with a reason saying so; the other workspace's contact is never changed by an import here. @@ -56,7 +56,7 @@ Problems with the mapping itself, an unnamed custom field or a name Warmbly cann On the **Review** step, **Campaigns** enrols everyone in the file as leads of the campaigns you pick (started from a campaign's Leads tab, that campaign is shown as the fixed target), and **Segments** pins them into the [segments](/guides/segments/) you pick as manual includes, so they stay members whatever the segment's conditions say. A segment created from the picker has no conditions, so it holds exactly the contacts pinned into it. Started from a segment, that segment is always applied and shown as a fixed row; the picker below it adds more. -**From Google Sheets**, a sheet is a reusable **sync source** rather than a one-time upload. Connect it once (Warmbly reads only the tab you choose and never writes back), paste the spreadsheet ID from the URL between `/d/` and `/edit`, pick the tab, map columns, then set duplicate handling, a label, and optionally a campaign to enroll into, categories to apply and segments to pin into. **Nothing syncs automatically**: press **Sync now**, or save and sync immediately. +**From Google Sheets**, a sheet is a reusable **sync source** rather than a one-time upload. Connect it once (Warmbly reads only the tab you choose and never writes back), paste the spreadsheet ID from the URL between `/d/` and `/edit`, pick the tab, map columns, then set duplicate handling, a label, and optionally a campaign to enroll into, labels to apply and segments to pin into. **Nothing syncs automatically**: press **Sync now**, or save and sync immediately. Segment targets apply on every run, so a sheet you keep adding rows to keeps feeding the same audience. Opening **Sync sources** from a segment's member list lists only the sources feeding it and pre-targets a new one to it. A segment deleted later is dropped from the run rather than stopping it. @@ -100,7 +100,7 @@ See [Personalization & expressions](/guides/expressions/) for the full templatin ## Filtering the list -A filter bar sits above the contact list. **Category**, **Segment**, **Status** and **Campaign** are always there; open one, tick values, and the list updates immediately with the matching count next to the bar. **Add filter** adds a custom-field condition (field, contains/is/starts with/ends with, value), a date-added or last-updated range, a number-of-campaigns range, the address verification verdict, the [email provider](#email-provider) and, on a campaign's Leads tab, lead status and engagement. Each active filter is a pill you can reopen to change or remove with its cross; **Clear** drops them all, and **Save as segment** turns the current set into a [segment](/guides/segments/). Free-text search, sort and the column chooser stay in the toolbar. +A filter bar sits above the contact list. **Label**, **Segment**, **Status** and **Campaign** are always there; open one, tick values, and the list updates immediately with the matching count next to the bar. **Add filter** adds a custom-field condition (field, contains/is/starts with/ends with, value), a date-added or last-updated range, a number-of-campaigns range, the address verification verdict, the [email provider](#email-provider) and, on a campaign's Leads tab, lead status and engagement. Each active filter is a pill you can reopen to change or remove with its cross; **Clear** drops them all, and **Save as segment** turns the current set into a [segment](/guides/segments/). Free-text search, sort and the column chooser stay in the toolbar. Free-text search matches first name, last name, email, company and phone. Every word you type has to match one of those, so `Test Demo` finds the contact whose first name is Test and last name is Demo, and `Demo Acme` finds everyone named Demo at Acme. Words can be in any order, and only the first six count. @@ -122,7 +122,7 @@ Each contact's avatar carries a small badge with the logo of whoever hosts their A contact on a company domain shows that company's logo in place of their initials, and the **Company** column shows it beside the company name; when no company is on file, the column shows the domain their address is on. Personal inboxes (Gmail, Outlook.com, Yahoo and the like) keep their initials. Logos are on for Warmbly Cloud and off on a self-hosted instance unless its operator turns them on, because the browser fetches each one from DuckDuckGo; see [data control](/development/data-control/#outbound-calls). -It shows on the contacts page and on a campaign's Leads tab alike. You can sort by it from **Sort**, filter with **Add filter** > **Email provider** (including **Unknown or not checked yet**), select every match and act on them in bulk (add them to a campaign, tag them, export them), or save the filter as a segment. Campaign [ESP matching](/guides/campaigns/) uses the same data to pair each lead with a same-provider mailbox. +It shows on the contacts page and on a campaign's Leads tab alike. You can sort by it from **Sort**, filter with **Add filter** > **Email provider** (including **Unknown or not checked yet**), select every match and act on them in bulk (add them to a campaign, label them, export them), or save the filter as a segment. Campaign [ESP matching](/guides/campaigns/) uses the same data to pair each lead with a same-provider mailbox. ## Selecting rows @@ -130,35 +130,35 @@ Tick a row's checkbox to select it, or the one in the table header to select eve When more contacts match than are loaded, a bar appears under the header: **Select all N matching**. Clicking it hands the whole filtered set to the next action, however many pages that is, and the selection bar counts the full number rather than the loaded rows. Unticking a row afterwards takes just that contact out and the count follows. **Clear selection** in the bar, or the header checkbox, drops back to nothing. -The set is the one the list is showing: search, filters, the subscription facet, and the campaign or segment the list is scoped to all narrow it. Changing any of them clears the selection, because it would no longer mean what it did when you made it. A selection past `50,000` contacts is refused rather than half-applied; add a filter and work through it in parts. +The set is the one the list is showing: search, filters, the subscription facet, and the campaign or segment the list is scoped to all narrow it. Changing any of them clears the selection, because it would no longer mean what it did when you made it. A selection past `250,000` contacts is refused rather than half-applied; add a filter and work through it in parts. -Every bulk action reads the selection: **Edit**, **Segment**, **Remove from segment**, **Remove from campaign**, **Research**, **Verify**, **Mark deliverable** and **Delete**. **Push to CRM** is the exception: it calls the CRM once per contact while you wait, so it stays capped at `500` at a time. +Every bulk action reads the selection: **Edit**, **Segment**, **Remove from segment**, **Remove from campaign**, **Research**, **Verify**, **Mark deliverable** and **Delete**. **Push to CRM** and **Research** are the exceptions: a push calls the CRM once per contact while you wait, and each research run spends AI credits, so both stay capped at `500` at a time. **Select all matching** is in the **From contacts** picker too, on a campaign's Leads tab and a segment page, so a whole search can be added as leads or members in one step. ## Editing one contact -Clicking a row opens the contact's panel. Its **Details** tab edits the fields the contact is made of: name, email address, company, phone, subscription, campaigns, categories and custom fields. Nothing is sent until **Save**, **Discard** puts the panel back to the stored values, and closing it with unsaved edits asks first. +Clicking a row opens the contact's panel. Its **Details** tab edits the fields the contact is made of: name, email address, company, phone, subscription, campaigns, labels and custom fields. Nothing is sent until **Save**, **Discard** puts the panel back to the stored values, and closing it with unsaved edits asks first. The email address can be changed here, which is the right move when someone's address was mistyped on import or they moved to a new domain. It is stored lowercased and stripped of any display name, and it has to be free: an address another contact already holds is refused, because merging two people's campaign history is not something the edit could undo. A new address also clears the contact's [verification](/guides/deliverability/#address-verification) verdict and everything the platform had observed about the old mailbox, so the background check starts the new address from scratch on its next pass, and the recipient provider Warmbly matches senders against is worked out again from the new domain. Emails already sent went to the old address and stay in the timeline as they happened. ## Editing many contacts at once -Tick rows in any contact list, a segment's members or a campaign's leads, and the selection bar's **Edit** opens a bulk panel. It can add or remove campaigns, add or remove categories, force a subscription state, and queue custom-field operations (add, edit, delete, rename) that run on every selected contact (the key box suggests your existing fields as you type), including a **Select all matching** selection that reaches past the loaded pages. +Tick rows in any contact list, a segment's members or a campaign's leads, and the selection bar's **Edit** opens a bulk panel. It can add or remove campaigns, add or remove labels, force a subscription state, and queue custom-field operations (add, edit, delete, rename) that run on every selected contact (the key box suggests your existing fields as you type), including a **Select all matching** selection that reaches past the loaded pages. Nothing is applied while you build it up. The header names the list the selection came from, and a **Will apply** strip above the buttons lists every queued change as a chip you can take back one at a time before pressing Apply. A field operation missing its key, or its value where one is needed, is marked and skipped rather than sent half-finished. -## Categories +## Labels -Colored labels that group and filter contacts (`Warm lead`, `Conference 2026`, `Enterprise`), behaving like tags. The same picker appears in bulk edit, a contact's Details tab, the new-contact dialog, the filters, and the import and sync wizards. +Colored labels that group and filter contacts (`Warm lead`, `Conference 2026`, `Enterprise`). The same list labels [inbox conversations](/guides/unibox/#labels) and is what [forms](/guides/forms/) file their submissions under. The same picker appears in bulk edit, a contact's Details tab, the new-contact dialog, the filters, and the import and sync wizards. -It supports type-ahead search, and typing an unmatched name offers **Create** to add and select it in one step. Each category keeps its color everywhere its chip appears. +It supports type-ahead search, and typing an unmatched name offers **Create** to add and select it in one step. Each label keeps its color everywhere its chip appears. -Categories belong to the workspace, not to whoever made them: every teammate sees the same list and can file contacts under it. **Contacts > Categories** lists every category with a live contact count. From there you can create one, rename it, change its color, delete it (contacts are kept; the label is removed from them and from inbox threads), or click a row to open the contact list filtered to it. +Labels belong to the workspace, not to whoever made them: every teammate sees the same list and can label contacts with it. **Contacts > Labels** lists every label with a live contact count. From there you can create one, rename it, change its color, delete it (contacts are kept; the label is removed from them and from inbox threads), or click a row to open the contact list filtered to it. -Categories also drive automation: a sequence can run **Add tag** or **Remove tag** as a contact moves through a flow, so a label can be applied automatically on a positive reply. **Add to segment** and **Remove from segment** do the same for [segments](/guides/segments/). +Labels also drive automation: a sequence can run **Add label** or **Remove label** as a contact moves through a flow, so a label can be applied automatically on a positive reply. **Add to segment** and **Remove from segment** do the same for [segments](/guides/segments/). -To turn labels and activity into a reusable audience, build a [segment](/guides/segments/): a saved set of conditions over contacts (categories, fields, campaign activity, engagement) that you can browse and add to a campaign in one step. +To turn labels and activity into a reusable audience, build a [segment](/guides/segments/): a saved set of conditions over contacts (labels, fields, campaign activity, engagement) that you can browse and add to a campaign in one step. ## Where a contact came from @@ -182,11 +182,11 @@ A new contact is also an event. `contact.created` goes to your [webhooks](/guide ## Activity timeline -The **Activity** tab of a contact is one workspace-wide feed, newest first, of everything Warmbly knows about them. Every member with permission to view contacts sees the same timeline, regardless of who created the contact or its campaigns. It includes every campaign email sent, opened, clicked, replied to or bounced (with the campaign, step, subject and sending mailbox), replies with their classified intent, deliverability and suppression events, notes, meetings, and the contact's lifecycle: when it was created and how, and each time it joined or left a campaign or a category. +The **Activity** tab of a contact is one workspace-wide feed, newest first, of everything Warmbly knows about them. Every member with permission to view contacts sees the same timeline, regardless of who created the contact or its campaigns. It includes every campaign email sent, opened, clicked, replied to or bounced (with the campaign, step, subject and sending mailbox), replies with their classified intent, deliverability and suppression events, notes, meetings, and the contact's lifecycle: when it was created and how, and each time it joined or left a campaign or gained or lost a label. Opens appear once per event, not once per email: a second open from another device is its own row. Opens show the device and mail client they were read on, such as **iPhone · Apple Mail app** or **Gmail · device hidden**, and clicks the device and browser the link opened in, with the city and country when known (see [How an open was read](/guides/analytics/#how-an-open-was-read)); expanded, the operating system, browser and full location, and why a device is hidden when a mail provider's image proxy fetched the email. The Overview tab sums this up under **How they read**: each client and device the contact's opens came from, with the count and the latest. A click names the link: the row reads **Clicked Pricing** and, expanded, shows the link's text, its full URL, the UTM source, medium, campaign and content it carried, and the browser. Every link in an email is tracked on its own, so two links clicked are two rows. Opens and clicks that came from a machine rather than the person (a mail privacy proxy, a security gateway that follows every link at delivery) carry an **auto** badge, and the expanded row says which rule caught them; see [Link tracking and UTM parameters](/guides/campaigns/#link-tracking-and-utm-parameters). -Filter chips narrow the feed (**Emails**, **Replies**, **Deliv.**, **Notes**, **Meetings**, **Campaigns**, **Lifecycle**), the search box matches subjects, campaigns, steps, mailboxes, categories, reasons and, for clicks, the link's text, URL and UTM values, and the date picker bounds it. Each row stays to one line until you click it; expanded, it shows every detail the event carries. The feed updates live as teammates and the schedulers write to it. +Filter chips narrow the feed (**Emails**, **Replies**, **Deliv.**, **Notes**, **Meetings**, **Campaigns**, **Lifecycle**), the search box matches subjects, campaigns, steps, mailboxes, labels, reasons and, for clicks, the link's text, URL and UTM values, and the date picker bounds it. Each row stays to one line until you click it; expanded, it shows every detail the event carries. The feed updates live as teammates and the schedulers write to it. At the top of the tab sits the campaign panel: for each campaign the contact is in, its flow with this contact's progress, the lead status, and what the scheduler will do next. See [Campaigns](/guides/campaigns/) for how the next action is worked out and what its states mean. diff --git a/docs/content/docs/guides/deliverability.mdx b/docs/content/docs/guides/deliverability.mdx index a76130892..a57196b62 100644 --- a/docs/content/docs/guides/deliverability.mdx +++ b/docs/content/docs/guides/deliverability.mdx @@ -42,7 +42,7 @@ Each mailbox sits in a band, shown as a colored chip. The band controls how Warm | Watch | `>= 10%` | `>= 0.03%` | n/a | n/a | | Throttled | `>= 20%` | n/a | n/a | n/a | | Quarantine | n/a | `>= 0.10%` | `>= 5%` | Repeated tampering with received warmup mail | -| Blocked | n/a | `>= 0.30%` | `>= 10%` | Clear abuse signals (repeated spam flags on received warmup mail) | +| Blocked | n/a | `>= 0.30%` | `>= 10%` | Clear abuse signals (four or more deletions or spam moves of received warmup mail in 7 days) | A mailbox enters a band by crossing any single threshold. What each band does: diff --git a/docs/content/docs/guides/forms.mdx b/docs/content/docs/guides/forms.mdx index e5da556ac..5026cde43 100644 --- a/docs/content/docs/guides/forms.mdx +++ b/docs/content/docs/guides/forms.mdx @@ -3,7 +3,7 @@ title: "Forms" description: "Build a hosted lead-capture form, style it to match your site, embed it anywhere, and turn every submission into a contact." --- -Forms turn website visitors into contacts. You build a form in the dashboard, publish it to a hosted page, and either share the link or embed the form on any website. Every submission is stored, and when it carries an email address it creates or updates a contact, files it under the categories you chose, and can drop it straight into a campaign. +Forms turn website visitors into contacts. You build a form in the dashboard, publish it to a hosted page, and either share the link or embed the form on any website. Every submission is stored, and when it carries an email address it creates or updates a contact, applies the labels you chose, and can drop it straight into a campaign. Open **Forms** in the sidebar. Viewing takes the same permission as viewing contacts; building and publishing take the manage-contacts permission. @@ -58,7 +58,7 @@ The **Design** tab styles the form while the canvas updates live. The defaults a The **Settings** tab controls what a submission does: - **Success message** is shown after submitting, or set a **redirect URL** to send the visitor to your own thank-you page instead. -- **Add to categories** files every submitted contact under the categories you pick, for example "Website leads". +- **Add to labels** puts the labels you pick on every submitted contact, for example "Website leads". - **Add to campaign** enrolls new contacts as leads in the campaign you pick. Sending still follows the campaign's own schedule, limits and windows; a form never causes immediate mail. Picking a campaign turns on its **Keep running for new leads** setting, so it waits between submissions instead of finishing; a campaign that had already finished restarts when a lead arrives, through the usual launch checks, and a refused restart is noted in its activity log while the lead waits. - **Spam protection** and **allowed embed domains** are covered below. diff --git a/docs/content/docs/guides/make.mdx b/docs/content/docs/guides/make.mdx index ea236cfc7..c7012d674 100644 --- a/docs/content/docs/guides/make.mdx +++ b/docs/content/docs/guides/make.mdx @@ -37,7 +37,7 @@ Triggers poll on your scenario's schedule. Make remembers what it has seen, so a | **Campaigns** | Create, Update, Start, Stop, Delete | | **Templates** | Create, Update, Delete, Render Reply Template | | **Meetings** | Log Meeting, Delete Meeting | -| **Organization** | Create Contact Category, Mailbox Tag, or Campaign Folder | +| **Organization** | Create Contact Category (a contact label), Mailbox Tag, or Campaign Folder | Campaign, mailbox, pipeline, stage, template, and contact fields use live dropdowns from your workspace, so you pick real records instead of pasting ids. @@ -56,7 +56,7 @@ Pair a search with an action for idempotent flows: Find Contact, then Create or Facebook and Instagram Lead Ads, LinkedIn Lead Gen Forms and TikTok Lead Generation all deliver new form submissions to Make in real time, which makes Make the shortest route from an ad to a follow-up sent from your own mailbox. 1. **Trigger:** *Facebook Lead Ads: Watch Leads* (or the LinkedIn or TikTok equivalent), picking the Page and form. -2. **Module:** Warmbly *Create or Update Contact*. Map the form's email and name fields, put every other question into a custom field, and pick the categories. Then *Add to Campaign* with the campaign that follows up. Re-running the same lead updates the contact rather than duplicating it. +2. **Module:** Warmbly *Create or Update Contact*. Map the form's email and name fields, put every other question into a custom field, and pick the labels. Then *Add to Campaign* with the campaign that follows up. Re-running the same lead updates the contact rather than duplicating it. Prefer to keep the mapping inside Warmbly? Use the *HTTP: Make a request* module to POST the lead as JSON to an [inbound webhook automation](/guides/automations/#lead-intake) instead. The automation's **Create or update contact** action maps the JSON keys onto contact fields with templates, and the same flow can tag, notify Slack and open a task. diff --git a/docs/content/docs/guides/n8n.mdx b/docs/content/docs/guides/n8n.mdx index 39356e5e4..3e9a933f1 100644 --- a/docs/content/docs/guides/n8n.mdx +++ b/docs/content/docs/guides/n8n.mdx @@ -22,7 +22,7 @@ Both deliver the same JSON body: a delivery id, the event type and the full even Two routes, depending on where you want the field mapping to live. -**Mapping in n8n.** An HTTP Request node calling `POST /v1/contacts` with the contact's fields, `categories` and `campaigns`. The write is an upsert by email, so re-running a workflow updates the contact instead of duplicating it. +**Mapping in n8n.** An HTTP Request node calling `POST /v1/contacts` with the contact's fields, its labels (`categories`) and `campaigns`. The write is an upsert by email, so re-running a workflow updates the contact instead of duplicating it. **Mapping in Warmbly.** Create an automation with the **Inbound webhook** trigger, copy its URL, and POST the raw payload to it from an HTTP Request node. The automation's **Create or update contact** action maps the JSON keys onto contact fields with templates (`{{.email}}`, `{{.answers.company}}`), tags the contact and enrols it in a campaign, and the same flow can notify Slack or open a task. See [Lead intake](/guides/automations/#lead-intake). This route needs no API key: the URL is the credential. diff --git a/docs/content/docs/guides/placement-tests.mdx b/docs/content/docs/guides/placement-tests.mdx index 453cd2fd2..d73f5a460 100644 --- a/docs/content/docs/guides/placement-tests.mdx +++ b/docs/content/docs/guides/placement-tests.mdx @@ -1,6 +1,6 @@ --- title: "Inbox placement tests" -description: "Send a campaign step or a template from one of your mailboxes to seed inboxes at the major providers, and see where each copy landed." +description: "Send a campaign step or a template from one of your mailboxes, or from a whole fleet at once, to seed inboxes at the major providers, and see where each copy landed." --- A placement test answers one question before your prospects do: when this mailbox sends this email, where does it land? Warmbly sends a copy from a real mailbox of yours to a panel of seed inboxes spread across providers, then reads each seed's mail to see whether the copy reached the primary inbox, a Gmail tab, spam, or nowhere at all. @@ -168,6 +168,78 @@ A monitor only tests a campaign that is running. When a run cannot start (the ca **When a monitor's test lands below its threshold**, everyone in the workspace who can view campaigns gets a **Placement monitor alert**, with the result and a link to the test. With **Pause on alert** on, the campaign is also paused automatically, with the reason in its activity log. Like every [auto-pause](/guides/campaigns/#auto-pause-guardrails), it does not resume on its own: find out what changed, then start the campaign again yourself. +## Testing a fleet with batches + +A test answers where one mailbox lands. With hundreds or thousands of sending mailboxes the question is different: is the fleet healthy, and if not, is the problem one mailbox, one domain, or one provider? A **placement batch** runs the same test from many mailboxes and reads the results together. Start one with **New batch** on the placement tests page, with **Test sender pool** on a campaign's settings, or with **Test untested mailboxes** from the coverage line. + +Each mailbox in a batch gets its own ordinary placement test with its own results, so everything above about how a test runs, what it costs and how to read it applies to every one of them. The batch adds shared settings, one progress view, results grouped across the fleet, and one cancel button. + +### Choosing the senders + +| Senders | What it selects | +|---------|-----------------| +| **Choose mailboxes** | The mailboxes you tick, by hand (`sender_account_ids` in the API) | +| **A campaign's senders** | Every mailbox the campaign sends from, the same set its scheduler uses (`sender_scope.type` `campaign`) | +| **All mailboxes** | Every mailbox in the workspace (`sender_scope.type` `workspace`) | + +A campaign's senders and all mailboxes can be narrowed by provider (Google Workspace, Microsoft 365, SMTP and the rest), by sending domain, by mailbox tag, and to mailboxes with no finished placement test in the last so many days. Disconnected mailboxes are left out unless you include them; a mailbox you tick by hand is always kept, and is reported by name if it cannot send when its turn comes. Seed inboxes never send. + +The senders are resolved on the server and written down when the batch starts. A mailbox tagged or added to the campaign afterwards does not join a batch that is already running, and one removed from it does not leave. + +### Sampling + +Testing every mailbox is thorough but spends a test and a day's sends from each. A sample tests part of the fleet instead: + +| Sample | Result | +|--------|--------| +| **All** | Every selected mailbox | +| **Random** | That many mailboxes, picked at random | +| **Percent** | That share of the mailboxes, rounded up. **Spread across providers** splits the sample in proportion to each provider's share of the fleet, so 10% of 800 Google Workspace, 400 SMTP and 147 Microsoft 365 mailboxes is 80, 40 and 15, rather than 135 picked at random | +| **Per domain** | Up to that many mailboxes from every sending domain, so a large domain is not over-represented | +| **Per provider** | Up to that many mailboxes from every sending provider | + +### How a batch runs + +A batch can hold thousands of mailboxes, but only a few send at once. Warmbly starts the next mailbox when one has sent every copy of its test, so at most `20` of a workspace's batch mailboxes are sending at the same time, a batch starts at most `10` mailboxes a minute, and the instance as a whole runs at most `200` batch mailboxes at once, which keeps the shared seed panel from receiving a burst (the operator can change all three). Every batch test runs at the spaced pace; quick is for a single test. Consecutive starts rotate across sending providers and domains, so a batch never works through one provider's mailboxes back to back. The number of mailboxes in a batch has no effect on how many run together; the operator's ceiling on batch size, `10,000` by default, exists only to protect the instance. + +Nothing is reserved when the batch starts. Each mailbox is checked when its turn comes, against its own daily limit, minimum gap, connection, worker and the workspace's standing, exactly as a single test is. So campaigns keep sending while a batch works through the fleet, and a batch never gets around a safeguard a single test would hit. + +A mailbox that cannot run when its turn comes is handled by the batch's policy: + +| Policy | A mailbox without room today, disconnected or offline | +|--------|------------------------------------------------------| +| **Retry until every sender has been tested** (default, `defer`) | Waits and is tried again: early the next UTC day when its daily limit ran out, an hour later when it was disconnected. The batch keeps going with the other mailboxes, and retries for up to seven days from when it started | +| **Skip senders that can't send right away** (`skip`) | Skipped, with the reason, and the batch moves on | + +A mailbox still sending another test is always retried a quarter of an hour later. A mailbox with no seed it can reach (every seed is on its own domain) is skipped under either policy. One mailbox failing never fails the batch: every mailbox keeps its own result or its own reason, such as its daily limit, its connection or its worker. + +A batch ends **completed** when every mailbox finished a test, **completed with warnings** when some were skipped or failed, **failed** when none finished, or **cancelled**. When the free tests or the credits you agreed to run out partway, or the campaign or step being tested is deleted, the mailboxes not yet started are skipped with that reason and the batch ends with the results it has. + +Cancelling a batch stops it starting mailboxes, cancels the copies not sent yet, and leaves the copies already sent to be classified, so a cancelled batch keeps every result it had. + +### Reading a batch + +The batch page shows its progress and the overall placement, then four views: + +- **Mailboxes**, worst inbox rate first, with the reason for any that was skipped, deferred or failed. Each opens its own test. +- **Domains**, each sending domain's placement across its mailboxes, worst first. One bad mailbox on an otherwise healthy domain is a mailbox problem; a whole domain low is that domain's reputation or authentication. +- **Providers**, the same by sending provider, which shows a problem with one kind of mailbox. +- **Recipient providers**, a table of sending domains against the providers that received the copies, which shows a domain that only one provider is filtering. + +For a tracking comparison the overall figure is the tracked half, the copy the campaign really sends, with the untracked half beside it. + +The batch does not group by worker. A worker signs in to your mailbox provider, and the provider delivers the mail from its own servers, so recipients' filters never see which worker handled a mailbox. + +### What a batch costs + +Before a batch starts, the dialog states the workload: how many tests, the most copies it can send (seeds per test times mailboxes, doubled for a tracking comparison), and how many mailboxes send at once. Each mailbox's daily limit can only make the real number smaller. + +Every mailbox's test counts as one test against the monthly free tests (two for a comparison), exactly like a single test. When a batch needs more tests than are free, you agree up front to the most the rest can cost, which the API takes as `max_credits` for the whole batch; the batch never spends more than that, and a test that delivers nothing is refunded as usual. When the tests cannot be paid for (a trial, or a panel without paid tests), a batch larger than the free tests left is refused; take a smaller sample or test on your own seed inboxes, which are never counted. + +### Fleet coverage + +The placement tests page shows how much of your connected fleet finished a placement test in the last 30 days, and how many mailboxes never did. **Test untested mailboxes** starts a batch of exactly those. + ## Reading the results honestly A placement test is a signal, not a verdict. Read it with its limits in mind: @@ -184,16 +256,22 @@ Tests also feed the seed placement figures on the [Deliverability](/guides/deliv | Notification | Who gets it | Default | |--------------|-------------|---------| -| Placement test finished | Whoever started the test by hand, once every copy has a verdict. A tracking comparison reports once, with both halves | On, in-app and push, no email | +| Placement test finished | Whoever started the test by hand, once every copy has a verdict. A tracking comparison reports once, with both halves. For a batch, whoever started it, once, when the whole batch ends | On, in-app and push, no email | | Placement monitor alert | Everyone who can view campaigns, when a monitor's test lands below its threshold | On, including email | -Both open the test. Change them under **Settings > Notifications**; see [notifications](/guides/notifications/). +Both open the test, or the batch. Change them under **Settings > Notifications**; see [notifications](/guides/notifications/). ## Limits | Limit | Value | |-------|-------| -| Tests running at once per workspace | `3` (a comparison counts as two) | +| Tests running at once per workspace | `3` (a comparison counts as two). A batch's tests are paced by the batch instead | +| Mailboxes in one batch | up to `10,000` by default, at most `100,000` (operator setting) | +| Batch mailboxes sending at once per workspace | `20` by default, at most `500` (operator setting) | +| Batch mailboxes sending at once across the instance | `200` by default, at most `5,000` (operator setting) | +| Batch mailboxes started per minute | `10` per batch by default (operator setting) | +| Batches running at once per workspace | `5` | +| How long a batch retries a deferred mailbox | `7` days from when it started | | Tests sending from one mailbox at once | `1` | | Seeds per test | up to `20` by default, at most `100` (operator setting). Seeds you chose by hand all get a copy, up to `50` | | Your own seed inboxes | `50` per workspace | @@ -202,7 +280,7 @@ Both open the test. Change them under **Settings > Notifications**; see [notific ## Moving a workspace -Placement tests and their results travel in a [workspace export](/guides/workspace-export-import/) as a record, under Delivery events (without what a paid test cost, since credits do not travel), and campaign placement monitors travel with the campaigns. Which mailboxes are seed inboxes does not travel, so mark them again on the destination. A test that is still sending leaves its unsent copies behind. +Placement tests and their results travel in a [workspace export](/guides/workspace-export-import/) as a record, under Delivery events (without what a paid test cost, since credits do not travel), and so do placement batches with their mailboxes and results. A batch still running when it is exported arrives cancelled, so the destination never starts sending it. Campaign placement monitors travel with the campaigns. Which mailboxes are seed inboxes does not travel, so mark them again on the destination. A test that is still sending leaves its unsent copies behind. ## Related guides diff --git a/docs/content/docs/guides/segments.mdx b/docs/content/docs/guides/segments.mdx index 667ffc6af..c76ad4d8e 100644 --- a/docs/content/docs/guides/segments.mdx +++ b/docs/content/docs/guides/segments.mdx @@ -1,11 +1,11 @@ --- title: Segments -description: Save reusable audiences built from contact fields, categories, campaign activity and email engagement, then add them to campaigns in one step. +description: Save reusable audiences built from contact fields, labels, campaign activity and email engagement, then add them to campaigns in one step. --- -A segment is a saved audience: a set of conditions over your contacts plus any contacts you pin in or out by hand. A segment says which contacts belong together; a campaign says what happens to them. Membership is evaluated live, so a contact that starts matching (a new category, a reply, a bounce) is in the segment the next time anyone looks, with no rebuild step. +A segment is a saved audience: a set of conditions over your contacts plus any contacts you pin in or out by hand. A segment says which contacts belong together; a campaign says what happens to them. Membership is evaluated live, so a contact that starts matching (a new label, a reply, a bounce) is in the segment the next time anyone looks, with no rebuild step. -Segments live under **Contacts > Segments**, next to **All contacts** and **Categories**, and need the same permissions as contacts: **View contacts** to browse, **Manage contacts** to create, edit or delete. +Segments live under **Contacts > Segments**, next to **All contacts** and **Labels**, and need the same permissions as contacts: **View contacts** to browse, **Manage contacts** to create, edit or delete. ## Building a segment @@ -19,7 +19,7 @@ Give the segment a name and a color, then add conditions. Each condition is a fi | Contact | Subscribed, on the suppression list, catch-all domain | is yes, is no | | Contact | Source, verification status, email provider, email provider family | is any of, is none of | | Contact | Created, updated | in the last N days, not in the last N days, after, before | -| Contact | Category | has any of, has none of, has none, has any | +| Contact | Label | has any of, has none of, has none, has any | | Company | Company name | the text operators above | | Campaign activity | In campaign | is in any of, is in none of, is in no campaign, is in a campaign | | Campaign activity | Number of campaigns | is, is not, more than, at least, less than, at most | @@ -52,7 +52,7 @@ A segment with no conditions at all is a plain list: it holds exactly the contac ## Using a segment -- **Browse**: a segment page lists its current members with the same table, filters, detail drawer and bulk actions as the contacts page, including [**Select all matching**](/guides/contacts-crm/#selecting-rows) so an action covers every member and not only the rows loaded so far, for selections up to `50,000` contacts. +- **Browse**: a segment page lists its current members with the same table, filters, detail drawer and bulk actions as the contacts page, including [**Select all matching**](/guides/contacts-crm/#selecting-rows) so an action covers every member and not only the rows loaded so far, for selections up to `250,000` contacts. - **Add to campaign**: enrols every current member as a lead of the campaign you pick, from the segment page or with **From segment** on a campaign's Leads tab. Contacts already in that campaign are skipped, and a running campaign wakes up to schedule the new leads. This is a snapshot: contacts who join the segment later are not added until you run it again. To keep a campaign fed automatically, [link the segment](#linking-a-segment-to-a-campaign) instead. - **Link to campaign**: attaches the segment to a campaign as a live audience, so contacts who join the segment later become leads on their own. See [below](#linking-a-segment-to-a-campaign). - **New campaign**: pick the segment as a lead list on the first step of **Campaigns** > **New campaign**, which shows how many leads it holds and how long the mailbox pool needs to reach them. See [create a campaign](/guides/campaigns/#create-a-campaign). @@ -93,15 +93,15 @@ Everything above can be driven from the [API](/api/reference/contacts/#segments) API calls address a segment by its ID. It is shown at the bottom of the segment page header (click it to copy), in the **Copy segment ID** entry of a segment's row menu on the Segments tab, and in every segment the API returns. Reads take the `READ_CONTACTS` key scope, writes `WRITE_CONTACTS`, and enrolling into a campaign `WRITE_CAMPAIGNS`. - -Categories are labels you put on a contact. Segments are rules that read those labels (and everything else) to decide who belongs. Use a category to mark a fact about a contact, and a segment to describe an audience. + +Labels are something you put on a contact. Segments are rules that read those labels (and everything else) to decide who belongs. Use a label to mark a fact about a contact, and a segment to describe an audience. ## Limits - 200 segments per workspace - 50 conditions per segment, 200 values per list condition -- 1,000 contacts per manual add or remove request +- 10,000 ticked contacts, or `250,000` through **Select all matching**, per manual add or remove ## Where to go next diff --git a/docs/content/docs/guides/sequences.mdx b/docs/content/docs/guides/sequences.mdx index 3aad34283..ae297dbc6 100644 --- a/docs/content/docs/guides/sequences.mdx +++ b/docs/content/docs/guides/sequences.mdx @@ -127,10 +127,10 @@ Upload attachments for a step below its composer by dragging and dropping files ### Action steps -A step can perform an action instead of sending: add or remove a tag, label email, create a task, create a deal or move its stage, unsubscribe the contact, notify (fires your webhooks and integrations), run an automation, or switch. They connect like email steps and work best at the end of a reply branch, for example creating a deal and notifying your team on a positive reply. +A step can perform an action instead of sending: add or remove a label, label the conversation, create a task, create a deal or move its stage, unsubscribe the contact, notify (fires your webhooks and integrations), run an automation, or switch. They connect like email steps and work best at the end of a reply branch, for example creating a deal and notifying your team on a positive reply. -**Label email** is reply-only. It labels the conversation the contact replied on, so place it on a reply branch; anywhere else it is a no-op because there is no thread to label. +**Label conversation** is reply-only. It labels the conversation the contact replied on, so place it on a reply branch; anywhere else it is a no-op because there is no thread to label. ### Switch steps diff --git a/docs/content/docs/guides/team-roles.mdx b/docs/content/docs/guides/team-roles.mdx index 1d8a076d8..4ca73b9b3 100644 --- a/docs/content/docs/guides/team-roles.mdx +++ b/docs/content/docs/guides/team-roles.mdx @@ -55,7 +55,7 @@ Roles are workspace data. Every workspace starts with three seeded roles that ar | Area | Capability | Allows | | --- | --- | --- | | **Data** | View / manage campaigns | Read settings, sequences, analytics / create, edit, archive | -| | View / manage contacts | Read contacts, segments, tags / create, edit, delete | +| | View / manage contacts | Read contacts, segments, labels / create, edit, delete | | | Manage sequences | Edit step content and spacing | | | View analytics | Deliverability and engagement reports | | | Use integrations | Push contacts and deals to connected tools | diff --git a/docs/content/docs/guides/unibox.mdx b/docs/content/docs/guides/unibox.mdx index a2735f1bc..8ed6f9b7a 100644 --- a/docs/content/docs/guides/unibox.mdx +++ b/docs/content/docs/guides/unibox.mdx @@ -1,6 +1,6 @@ --- title: "Unibox" -description: "A unified inbox across all connected mailboxes, with categories and threading." +description: "A unified inbox across all connected mailboxes, with labels and threading." --- One inbox for every connected mailbox. Read, sort, and reply from a single screen instead of logging into each account. Three columns: a **scope rail** for picking what to look at, a **conversation list**, and a **thread view** where you read and reply. @@ -16,6 +16,7 @@ The columns are yours to size. Every choice here is remembered in the browser yo | The divider between the list and the thread | Drag it to widen the conversation list so long subjects fit, or to give the space back to the thread. It also takes the keyboard once it has focus: arrow keys nudge it (hold `Shift` for a bigger step), `Home` and `End` go to the narrowest and widest the window allows, `Enter` (or a double-click) puts it back to the default. | | **Collapse** at the bottom of the left navigation, or `b` | Shrinks Warmbly's own navigation to an icon rail and gives the width to the page. Row labels become tooltips; unread mail keeps its count badge and an open [Advisor](/guides/advisor/) finding shows as a coloured dot on the icon. The same control expands it again. | | The section headers in the left navigation (Email, CRM, Resources) | Click one to fold its section away, and again to bring it back. Like the other layout choices, it is remembered in this browser. A folded section still shows the page you are on, and a coloured dot beside its header flags an open [Advisor](/guides/advisor/) finding on one of the pages it hides. The icon rail follows the same choice. | +| The section headers in the scope rail (Mail, Views, Mailboxes, Labels, Tags) | Click one to fold its section away, and again to bring it back. A folded section still shows the scope you are looking at, and a small blue dot beside its header means a row it folds away has a highlighted count (rows you hid never raise it). The **…** beside a header (or a right-click on it) folds every other section, unfolds them all, or moves the section up or down the rail. The pencil on **Mail** and **Views** puts the rows in edit mode: untick a row to hide it, drag it by its handle (or focus the handle and use the arrow keys) to reorder, and press **Done**, `Escape`, or click anywhere else. **Reset** puts that section back the way it shipped. Each row also has its own **…** menu and right-click menu to move it, hide it (with an undo), or mark a folder read, and `Alt` with an arrow key moves the focused row. A small "2 hidden" beside the pencil tells you rows are off the rail. Hiding a row only takes it off the rail, so its shortcuts and links keep working, and the scope you are in always stays visible. All of it is remembered in this browser. | | The contact button in the thread header | Shows or hides the contact panel on the right. It starts closed when no preference is saved. Opening or closing it sticks across conversations, and existing preferences are preserved. | How wide the list can get depends on the window: the thread always keeps enough room to read a message, and the contact panel counts toward that when it is open, so the widest setting is narrower on a laptop than on a large monitor. @@ -58,7 +59,7 @@ The rail is two short groups and then your mailboxes. The first group is where y Opening **Inbox** from the main navigation starts in the Inbox folder. Sent messages live in **Sent**; choose **All mail** when you want inbound and outbound messages together. An Inbox conversation still shows its complete history, including your replies, when you open it. -Each message starts in the folder the provider has it in: IMAP special-use folder attributes, Gmail labels, and Outlook well-known folders all map to the same six. Moves at the provider (junking a message, clearing it out of spam) follow on the next sync. Drafts are also reconciled: a draft the mailbox no longer holds is removed here too, so the fresh copy Gmail saves on every autosave does not leave the previous ones behind. You can also file a conversation yourself, which moves it here without moving it at the provider; see [filing a conversation](#filing-a-conversation). The active row is highlighted grey, unread counts sit on the right in blue, and hovering a folder reveals a three-dot menu with **Mark all as read**. +Each message starts in the folder the provider has it in: IMAP special-use folder attributes, Gmail labels, and Outlook well-known folders all map to the same six. Moves at the provider (archiving, deleting, junking a message, clearing it out of spam, moving it back to the inbox) follow on the next sync. In Gmail each of those is a label change, and Warmbly files the message by the labels it ends up with, so a conversation archived in Gmail leaves Inbox here too. Gmail mailboxes are also rechecked against Gmail every few hours, which catches a move the live sync missed. Drafts are also reconciled: a draft the mailbox no longer holds is removed here too, so the fresh copy Gmail saves on every autosave does not leave the previous ones behind. You can also file a conversation yourself, which moves it here without moving it at the provider; see [filing a conversation](#filing-a-conversation). The active row is highlighted grey, unread counts sit on the right in blue, and hovering a folder reveals a three-dot menu with **Mark all as read**. Spam and Trash stay out of every other view, and so does Archive: filing a conversation is how you take it out of the way, so it leaves Unread, Awaiting reply, every mailbox, label and tag view, and the unread badge. **All mail** and the **Archive** folder are the two places it stays, which is where you go to find it again. @@ -108,6 +109,8 @@ Each row is a conversation, not a message, with a count next to the sender when The list loads more conversations as you reach the end of it, and it keeps its place: opening a conversation, or leaving the inbox and coming back, returns you to the row you were on rather than the top of the list. +Picking another folder or view closes the open conversation, since it may not be in the new list. To close it yourself, use the **X** at the right of the conversation header on a wide screen, the back link above it on a phone (it names the list you came from), or `Escape`. + Threading applies on both sides. A reply carries an `In-Reply-To` header naming the last message in the conversation, which is what nests it for the recipient. From a Gmail mailbox that holds the conversation it also carries Gmail's thread id, which nests it in your own mailbox. The recipient's mail client only ever sees the header: a thread id means nothing outside the mailbox that issued it, so a reply sent [from another mailbox](#choosing-the-sending-mailbox) goes without one. @@ -130,14 +133,14 @@ Recognized quoted history collapses behind **Show quoted text**. Click it to rea Every open message has a details toggle on its recipient line, the `to` under the sender's name, and the sender's name and address open the same panel. It shows the message envelope in place: every From, Reply-To, To, Cc and Bcc address, when it was sent and, if that differs, when the mailbox received it, both in your own time zone, the mailbox and folder it lives in, its size, and the Message-ID and In-Reply-To headers. Click any address or identifier to copy it. The info icon next to Reply and Forward opens the same panel, including on a message that is still collapsed. -## Categories and labels +## Labels -Categories are the workspace's conversation labels, shared with the rest of Warmbly, so `Interested` means the same thing on a contact as it does here. They are shared with the rest of the team too: a category anyone creates is available to everyone, and a conversation one member labels shows that label to the next. Label with the tag button in the conversation header or `c`: search existing categories, tick what applies, or type a name and **Create**. A conversation can carry several. +Labels are one workspace list, used on [contacts](/guides/contacts-crm/#labels), on conversations here and by [forms](/guides/forms/), so `Interested` means the same thing on a contact as it does on a conversation. The list is shared with the rest of the team too: a label anyone creates is available to everyone, and a conversation one member labels shows that label to the next. Label with the tag button in the conversation header or `c`: search existing labels, tick what applies, or type a name and **Create**. A conversation can carry several. -Labeled conversations show colored chips on the row and in the header, and each category appears in the rail with its own count for one-click filtering. Labels you apply are never touched by the system. [Automatic inbox tagging](/guides/inbox-tagging/) adds its own labels alongside them, additively, and never removes one a person applied. +Labeled conversations show colored chips on the row and in the header, and each label appears in the rail with its own count for one-click filtering. Labels you apply are never touched by the system. [Automatic inbox tagging](/guides/inbox-tagging/) adds its own labels alongside them, additively, and never removes one a person applied. - -Categories label conversations. Tags label the mailboxes themselves (grouping accounts by client or domain). Both filter from the rail, but they describe different things. + +Labels mark conversations and contacts. Mailbox tags mark the mailboxes themselves (grouping accounts by client or domain). Both filter from the rail, but they describe different things. ## Read state @@ -173,7 +176,7 @@ Each row has a tick box in its left gutter, in place of the unread dot. Hover a With anything ticked, a bar appears along the bottom of the screen with the count and the actions that apply to all of them: **Mark read**, **Mark unread**, **Snooze**, **Archive** (or **Move to inbox**) and **Delete**. Each one is a single request, so filing a screenful is as quick as filing one. **Clear** drops the selection, and so does `Escape`; changing view drops it too, since the rows it applied to are no longer the rows on screen. -Unlike read state, Archive and Delete are Warmbly's own filing. The message keeps its place in Gmail, Outlook, or whatever mail client the mailbox belongs to, and deleting a conversation here never deletes mail there. The next sync will not undo your filing either: Warmbly records where the provider has each message separately from where you filed it, and follows the provider only when the provider itself moves the message. So junking a message in Gmail still reaches Warmbly, and an ordinary sync pass does not. +Unlike read state, Archive and Delete are Warmbly's own filing. The message keeps its place in Gmail, Outlook, or whatever mail client the mailbox belongs to, and deleting a conversation here never deletes mail there. The next sync will not undo your filing either: Warmbly records where the provider has each message separately from where you filed it, and follows the provider only when the provider itself moves the message. So archiving or junking a message in Gmail still reaches Warmbly, and an ordinary sync pass does not. ## Replying @@ -194,7 +197,7 @@ The composer shows the message it will forward under your note; expand **Forward ### Pausing their follow-ups -The contact panel lists every campaign the sender is a lead of, with their status in it and what happens next: the next step and when, the hold and when it lifts, or why the flow ended. A campaign with a step still to send has **Pause**, which opens the same dialog as the campaign's Leads list (until a date, or until you resume them, with an optional note). A held lead has **Resume** instead. Neither appears on a lead whose flow has ended, and both need the **Manage campaigns** permission. See [Out of office and pausing one lead](/guides/campaigns/#out-of-office-and-pausing-one-lead). +The contact panel lists every campaign the sender is a lead of, with their status in it and what happens next: the next step and when, the hold and when it lifts, or why the flow ended. A campaign with a step still to send has **Pause**, which opens the same dialog as the campaign's Leads list (until a date, or until you resume them, with an optional note). A held lead has **Resume** instead; for a contact held because they are [copied on another lead's emails](/guides/campaigns/#copying-colleagues-on-one-lead) it asks first, since it starts a second thread to them. Neither appears on a lead whose flow has ended, and both need the **Manage campaigns** permission. See [Out of office and pausing one lead](/guides/campaigns/#out-of-office-and-pausing-one-lead). The reply composer offers the same thing for the moment you answer. When the recipient still has a follow-up queued, **Pause follow-ups** next to **Schedule** holds them for 3 days, 1 week, 2 weeks, 30 days or until you resume them. When they are a lead of more than one such campaign, the menu lists each one, all ticked, and you untick the ones to leave running (at least one stays ticked). Nothing is paused until the reply is accepted, and a send that fails pauses nothing. A scheduled reply pauses the follow-ups straight away and counts the length from when the reply goes out, so a follow-up cannot overtake it. Undoing or cancelling the reply does not lift the pause, so use **Resume** in the contact panel if you change your mind. A lead already on hold, for example after an away message arrived while you were writing, keeps that hold. diff --git a/docs/content/docs/guides/warmup.mdx b/docs/content/docs/guides/warmup.mdx index 8f2c32a1a..ca63b41ec 100644 --- a/docs/content/docs/guides/warmup.mdx +++ b/docs/content/docs/guides/warmup.mdx @@ -233,19 +233,29 @@ Partner selection favours the recipients where warmup earns something. A small-h Quarantined and blocked mailboxes are selected as neither sender nor recipient. Mail arriving in a mailbox is never held against it: every warmup message carries a single-use token bound to its recipient, so a token cannot be replayed or redirected, and one that lands where it does not belong is simply filed as ordinary mail. -What a mailbox does to warmup mail it received is held against it, on a ladder rather than at once. Deleting a warmup email within a day of its arrival counts as one strike and marking one as spam counts as two, over the last seven days. One strike puts the mailbox on watch with the reason shown in its drawer, two pause it from the pool for seven days, and four block it for thirty. So deleting one fresh warmup message is a warning, not a ban, while flagging pool mail as spam twice is a block. +What a mailbox does to warmup mail it received is held against it, on a ladder rather than at once. Deleting a warmup email within a day of its arrival counts as one strike, and so does moving one to spam, over the last seven days. One strike puts the mailbox on watch with the reason shown in its drawer, two pause it from the pool for seven days, and four block it for thirty. So one deleted or junked warmup message is a warning, not a ban. Only a fresh deletion counts, because that is the one that costs the pool something: the engagement a warmup message earns happens in its first hours, and removing it before then takes that signal away. A warmup message deleted later is housekeeping, whether by you, by Gmail emptying its Trash, by a retention rule on your mail server or by Warmbly's own [retention](#retention), and it is never held against the mailbox. On Gmail, pressing Delete is what is judged, not the purge from Trash weeks later. A mailbox's own filing is never counted either: Warmbly moving a warmup email into its folder, marking it read, rescuing it from spam or deleting it once its window has passed is the platform acting, not the owner. Moving a warmup email is not deleting it. Outlook and Microsoft 365 report a message moved to another folder exactly as they report one deleted, and a mailbox's own rules, its provider's filter or a second Warmbly instance syncing the same mailbox can all move mail. So before a removal counts, Warmbly searches the whole mailbox for the message: found in any folder other than Trash (Deleted Items on Outlook), it was filed, and nothing is held against the mailbox. Only a message in Trash or gone for good is a strike. A search that cannot run, or cannot tell, counts as nothing. Deletion strikes recorded before this search existed are searched for in the same way. A strike whose message is still in the mailbox is withdrawn, and the pause or block it caused is decided again on the strikes that remain, so it is lifted or shortened only when those no longer earn it. +No provider says who moved a message into spam. Gmail, Microsoft 365 and IMAP report it the same way whether you pressed "Report spam", the provider re-filed it after delivery (Microsoft's zero-hour auto purge, Google Workspace's post-delivery scanning), a desktop mail client's junk filter moved it, or a security tool did. So Warmbly charges a move to spam only when the evidence points at a person, and waits 30 minutes after the move to see it: + +- **Arrived in spam.** A warmup email that was in spam when it arrived is never a strike. The label on it is the provider's filter, and it counts as spam placement against the sender. +- **The provider at work.** When the same sender's warmup mail was moved to spam in another workspace within a day, the provider is re-judging that sender, and nobody is charged. If a mailbox was already charged for one of those, the strike is withdrawn and any pause it caused is decided again. A move within 15 minutes of arrival while nobody is using the mailbox is the filter catching up, and is not charged either. +- **You at the mailbox.** A move while someone was reading, marking unread or starring mail in the mailbox, within 30 minutes either side, is taken as yours. Only changes made at the provider count: reading a message in Warmbly's inbox is never mistaken for it. +- **A pattern.** In a mailbox someone has used in the last 14 days, unexplained moves of warmup mail from three or more different senders in a week, which no other workspace sees, are the mailbox's own filtering and are charged. +- **Anything else charges nobody**, including every move in a mailbox nobody has used for 14 days. The sender still has it counted as spam placement, which only slows sending down. + +Only a move charged to you files a complaint against the sender. A move attributed to the provider, or to nobody, is spam placement for the sender instead. On Outlook, Microsoft 365 and IMAP mailboxes, a move to Junk is found by the search above and is not charged at all. + Warmbly intervenes well before providers would penalize a mailbox. Complaints, bounces and tampering take a mailbox out of the pool early. A mailbox landing in spam at the major providers is slowed down instead, so it keeps warming, which is how it recovers. Getting back in requires requalifying, not just waiting: healthy authentication, no recent complaints or hard-bounce spikes, and spam placement back to a low level. Return is gradual, not a jump back to the old ceiling. -A quarantine or a block also holds for its full term. The signals behind it age out of their windows long before it ends, and that does not release it early; a more serious finding can still replace it. A throttle is different and lifts as soon as the mailbox recovers, as the table says. +A quarantine or a block also holds for its full term. The signals behind it age out of their windows long before it ends, and that does not release it early; a more serious finding can still replace it. A throttle is different and lifts as soon as the mailbox recovers, as the table says. When support lifts a pause or block, or approves an appeal against one, the strikes behind it are cleared with it, so the next evaluation does not reimpose it. On a self-hosted instance linked to [Warmbly Cloud](/guides/warmbly-cloud/#safety-and-enforcement), a mailbox the cloud warms is judged there, and the instance applies the cloud's standing to its own campaigns with the same effects as this table. diff --git a/docs/content/docs/guides/workspace-export-import.mdx b/docs/content/docs/guides/workspace-export-import.mdx index e72a8a9e2..f3d65be1a 100644 --- a/docs/content/docs/guides/workspace-export-import.mdx +++ b/docs/content/docs/guides/workspace-export-import.mdx @@ -18,15 +18,15 @@ The data is split into groups. Every export includes **Workspace**; the rest are | Group | Contents | |-------|----------| | Workspace | The organization, members, roles, teams, mailboxes and their profile photos, mailbox tags, the column mappings saved by [mailbox imports](/guides/mailbox-import/), inbox vendor connections, [root redirects](/guides/sending-domains/#root-redirects), API keys, webhooks, and settings, including the website tracking site key. Always included | -| Contacts | Contacts, categories, the column mappings saved by [contact imports](/guides/contacts-crm/#importing), segments with their manual overrides, forms with their images, submissions, personalized link tickets and funnel events, notes, activities, and the suppression list | -| Campaigns | Campaigns, folders, sequences, senders, linked segments, attachments, the email image library, per-campaign settings, each lead's step progress with its per-link clicks and per-event opens, [placement monitors](/guides/placement-tests/#campaign-placement-monitors), and the [unsubscribe links](/guides/unsubscribe/) already in recipients' inboxes | +| Contacts | Contacts, labels, the column mappings saved by [contact imports](/guides/contacts-crm/#importing), segments with their manual overrides, forms with their images, submissions, personalized link tickets and funnel events, notes, activities, and the suppression list | +| Campaigns | Campaigns, folders, sequences, senders, linked segments, attachments, the email image library, per-campaign settings, each lead's step progress with its per-link clicks and per-event opens, the colleagues [copied on each lead](/guides/campaigns/#copying-colleagues-on-one-lead), [placement monitors](/guides/placement-tests/#campaign-placement-monitors), and the [unsubscribe links](/guides/unsubscribe/) already in recipients' inboxes | | CRM | Pipelines, deals, tasks, and meeting bookings | | Automations | Automations, connected integrations, and lead sync sources | | Assistant | Assistant sessions and messages, skills, MCP servers, and AI settings | | Warmup | Warmup participation, routing rules, statistics and inbox placement history, appeals, and the standing of every penalised address, current or removed, so a move is not a way past a block | | Inbox | Unified inbox threads, message bodies, conversation labels, mailbox sync state, and completed automatic-tagging verdicts with their raw probabilities | | Send history | Queued and completed send tasks with their payloads | -| Delivery events | Bounces, complaints, opens, clicks, [placement tests](/guides/placement-tests/) with where each copy landed, and website page views with the browser records that tie them to contacts | +| Delivery events | Bounces, complaints, opens, clicks, [placement tests](/guides/placement-tests/) and batches with where each copy landed, and website page views with the browser records that tie them to contacts | | Verification evidence | What real mail showed about each contact's address (deliveries, opens, replies, bounces), so verdicts and confidence survive the move | | Logs | Audit log, campaign logs, and notifications | | Billing history | Subscription, credit ledger, and referral records | @@ -97,7 +97,7 @@ Some things belong to an instance rather than to a workspace, so they are not ap | Cold rotation state | Whether a mailbox is resting or held in reserve is this instance's decision about sending it watched. Every mailbox arrives in normal rotation and earns its way out again | | Cold sending ramps | How far a mailbox had eased into cold volume raises its cap, and the destination never watched it send. Mailboxes re-graduate from their warmup maturity, which costs a few days and errs toward sending less | | Seed inboxes | Which mailboxes are placement seed inboxes is this instance's choice of test inboxes, and an archive must not add mailboxes to another instance's seed panel. Every mailbox arrives as an ordinary one; mark your own seed inboxes again on the destination | -| Placement test sends | Copies of a test that had not been sent yet stay behind, so the destination never sends to the source instance's seeds. So does the copy a running tracking comparison rendered for each seed. Tests and their results arrive as a record, without what a test cost in credits, since the credit ledger does not travel. A campaign's placement monitor keeps its settings and next run, but not the record of its last run | +| Placement test sends | Copies of a test that had not been sent yet stay behind, so the destination never sends to the source instance's seeds. So does the copy a running tracking comparison rendered for each seed. Tests and their results arrive as a record, without what a test cost in credits, since the credit ledger does not travel. A campaign's placement monitor keeps its settings and next run, but not the record of its last run. A placement batch that was still running arrives cancelled, with the results it had, so it never starts sending from the destination | | Scheduled deletions | A pending deletion from the source must never follow the workspace to its new home | | Failure and delivery counters | A webhook endpoint's failure streak and auto-disable state, and whether a notification's email already went out, describe what happened on the source. They start fresh, so an endpoint is not pre-disabled on the new instance and a notification is not re-sent | | Sends still in flight | A campaign step handed to a worker on the source has no worker on the destination to report back, so it arrives queued and is sent there instead of waiting forever. Steps already sent keep their history | diff --git a/docs/content/docs/guides/zapier.mdx b/docs/content/docs/guides/zapier.mdx index 6d8f3d25d..6fa308368 100644 --- a/docs/content/docs/guides/zapier.mdx +++ b/docs/content/docs/guides/zapier.mdx @@ -37,7 +37,7 @@ Triggers poll on Zapier's schedule (frequency depends on your plan). Zapier reme | **Campaigns** | Create, Update, Start, Stop, Delete | | **Templates** | Create, Update, Delete, Render Reply Template | | **Meetings** | Log Meeting, Delete Meeting | -| **Organization** | Create Contact Category, Mailbox Tag, or Campaign Folder | +| **Organization** | Create Contact Category (a contact label), Mailbox Tag, or Campaign Folder | Campaign, mailbox, pipeline, stage, and contact fields use live dropdowns from your workspace, so you pick real records instead of pasting ids. @@ -56,7 +56,7 @@ Pair a search with an action for idempotent flows: Find Contact, then Create or Facebook and Instagram Lead Ads, LinkedIn Lead Gen Forms and TikTok Lead Generation all deliver new form submissions to Zapier in real time, which makes Zapier the shortest route from an ad to a follow-up sent from your own mailbox. 1. **Trigger:** *Facebook Lead Ads: New Lead* (or the LinkedIn or TikTok equivalent), picking the Page and form. -2. **Action:** Warmbly *Create or Update Contact*. Map the form's email and name fields, put every other question into a custom field, and pick the categories. Then *Add to Campaign* with the campaign that follows up. Re-running the same lead updates the contact rather than duplicating it. +2. **Action:** Warmbly *Create or Update Contact*. Map the form's email and name fields, put every other question into a custom field, and pick the labels. Then *Add to Campaign* with the campaign that follows up. Re-running the same lead updates the contact rather than duplicating it. Prefer to keep the mapping inside Warmbly? Use *Webhooks by Zapier: POST* to an [inbound webhook automation](/guides/automations/#lead-intake) instead, sending the lead as JSON. The automation's **Create or update contact** action maps the JSON keys onto contact fields with templates, and the same flow can tag, notify Slack and open a task. diff --git a/docs/public/openapi.json b/docs/public/openapi.json index 7abab5d76..9cbcdf5db 100644 --- a/docs/public/openapi.json +++ b/docs/public/openapi.json @@ -5823,6 +5823,295 @@ } } }, + "/campaigns/{id}/leads/{contactId}/cc": { + "get": { + "operationId": "campaigns_get_lead_cc", + "summary": "Get a lead's CC", + "description": "List the contacts copied on every email this campaign sends one lead. Scope READ_CAMPAIGNS and READ_CONTACTS, org permission view_campaigns and view_contacts.", + "tags": [ + "campaigns" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "name": "contactId", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "200": { + "description": "OK.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CampaignLeadCCResult" + } + } + } + }, + "401": { + "description": "Unauthorized.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "403": { + "description": "Forbidden.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "404": { + "description": "The campaign is not the caller's organization's, or the contact is not a lead of it.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "429": { + "description": "Rate limited.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + }, + "put": { + "operationId": "campaigns_set_lead_cc", + "summary": "Set a lead's CC", + "description": "Replace the contacts copied on every email this campaign sends one lead, follow-ups included. An empty list removes them all. A copied contact who is also a lead of the campaign has their own sequence held (source cc) while any lead copies them. The body is the whole list, so retries are safe without an Idempotency-Key. Scope WRITE_CAMPAIGNS and READ_CONTACTS, org permission manage_campaigns and view_contacts.", + "tags": [ + "campaigns" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "name": "contactId", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CampaignLeadCCRequest" + } + } + } + }, + "responses": { + "200": { + "description": "OK.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CampaignLeadCCResult" + } + } + } + }, + "400": { + "description": "bad_request: a contact_ids entry is not a uuid; lead_cc_limit: more than 2 contacts; lead_cc_self: the lead is in its own list.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "401": { + "description": "Unauthorized.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "403": { + "description": "Forbidden.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "404": { + "description": "not_found: the campaign is not the caller's organization's, or the contact is not a lead of it; lead_cc_contact_not_found: a contact to copy is not in the workspace.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "409": { + "description": "lead_cc_lead_is_copied: the lead is copied on another lead in this campaign; lead_cc_has_copies: a contact to copy has copies of their own in this campaign.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "429": { + "description": "Rate limited.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } + }, + "/campaigns/{id}/leads/{contactId}/cc/suggestions": { + "get": { + "operationId": "campaigns_suggest_lead_cc", + "summary": "Suggest colleagues to CC", + "description": "Up to eight contacts who look like the lead's colleagues: the same company name, or the same email domain when it belongs to a company rather than a personal mail service. Unsubscribed contacts and ones already copied are left out. Scope READ_CAMPAIGNS and READ_CONTACTS, org permission view_campaigns and view_contacts.", + "tags": [ + "campaigns" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "name": "contactId", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "200": { + "description": "OK.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CampaignLeadCCSuggestions" + } + } + } + }, + "401": { + "description": "Unauthorized.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "403": { + "description": "Forbidden.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "404": { + "description": "The campaign is not the caller's organization's.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "429": { + "description": "Rate limited.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } + }, "/campaigns/{id}/logs": { "get": { "operationId": "campaigns_list_logs", @@ -6967,7 +7256,7 @@ ], "operationId": "contacts_bulk_update", "summary": "Bulk update contacts", - "description": "Applies one set of edits across a selection of contacts (up to 1000 by ID, or everything a filter matches, up to 50000): add/remove campaigns and categories, custom-field operations, and subscription. Scope `BULK_CONTACTS`.", + "description": "Applies one set of edits across a selection of contacts (up to 10000 by ID, or everything a filter matches, up to 250000): add/remove campaigns and categories, custom-field operations, and subscription. Scope `BULK_CONTACTS`.", "security": [ { "bearerAuth": [] @@ -7003,7 +7292,7 @@ } }, "400": { - "description": "A selection that names nothing, an explicit list over 1000 or an exclusion list over 50000 (`too_many_contacts`), a select-all with no filters, a filter matching no contacts, or a filter matching more than 50000 (`selection_too_large`).", + "description": "A selection that names nothing, an explicit list over 10000 or an exclusion list over 250000 (`too_many_contacts`), a select-all with no filters, a filter matching no contacts, or a filter matching more than 250000 (`selection_too_large`).", "content": { "application/json": { "schema": { @@ -7050,7 +7339,7 @@ ], "operationId": "contacts_bulk_delete", "summary": "Bulk delete contacts", - "description": "Deletes a selection of contacts: up to 1000 by ID, or everything a filter matches (up to 50000). Scope `BULK_CONTACTS`.", + "description": "Deletes a selection of contacts: up to 10000 by ID, or everything a filter matches (up to 250000). Scope `BULK_CONTACTS`.", "security": [ { "bearerAuth": [] @@ -7063,7 +7352,7 @@ ], "requestBody": { "required": true, - "description": "A JSON array of contact ID strings (1 to 1000), or a ContactSelection object naming a filter.", + "description": "A JSON array of contact ID strings (1 to 10000), or a ContactSelection object naming a filter.", "content": { "application/json": { "schema": { @@ -7071,7 +7360,7 @@ { "type": "array", "minItems": 1, - "maxItems": 1000, + "maxItems": 10000, "items": { "type": "string", "format": "uuid" @@ -7090,7 +7379,7 @@ "description": "Contacts deleted." }, "400": { - "description": "A selection that names nothing, an explicit list over 1000 or an exclusion list over 50000 (`too_many_contacts`), a select-all with no filters, a filter matching no contacts, or a filter matching more than 50000 (`selection_too_large`).", + "description": "A selection that names nothing, an explicit list over 10000 or an exclusion list over 250000 (`too_many_contacts`), a select-all with no filters, a filter matching no contacts, or a filter matching more than 250000 (`selection_too_large`).", "content": { "application/json": { "schema": { @@ -15725,6 +16014,785 @@ } } }, + "/placement/batches": { + "get": { + "operationId": "placement_batches_list", + "tags": [ + "placement" + ], + "summary": "List placement batches", + "description": "The workspace's placement batches, newest first, with progress and headline placement. Scope READ_ANALYTICS, org permission view_analytics.", + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "limit", + "in": "query", + "required": false, + "description": "Page size, 1 to 100. Default 25.", + "schema": { + "type": "integer", + "minimum": 1, + "maximum": 100, + "default": 25 + } + }, + { + "name": "cursor", + "in": "query", + "required": false, + "description": "Opaque cursor from a previous pagination.next_cursor.", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "A page of batches.", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "data", + "pagination" + ], + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/components/schemas/PlacementBatch" + } + }, + "pagination": { + "$ref": "#/components/schemas/Pagination" + } + } + } + } + } + }, + "400": { + "description": "Invalid cursor or limit.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "401": { + "description": "Missing or invalid credentials.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "403": { + "description": "Insufficient permissions.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "429": { + "description": "Rate limited.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + }, + "post": { + "operationId": "placement_batches_create", + "tags": [ + "placement" + ], + "summary": "Start a placement batch", + "description": "Run the same placement test from many sending mailboxes, chosen by id or resolved on the server from a campaign or the whole workspace, optionally sampled. The senders are written down and the batch is answered queued; the backend starts them a few at a time, checking each mailbox's daily limit, connection and worker when its turn comes. Nothing is sent by the request. Scope SEND_CAMPAIGNS, org permission send_campaigns.", + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PlacementBatchRequest" + } + } + } + }, + "responses": { + "201": { + "description": "The queued batch.", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "data" + ], + "properties": { + "data": { + "$ref": "#/components/schemas/PlacementBatch" + } + } + } + } + } + }, + "400": { + "description": "Malformed request, placement_batch_empty, placement_batch_too_large, placement_invalid_seeds or placement_invalid_tracking.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "401": { + "description": "Missing or invalid credentials.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "402": { + "description": "placement_not_entitled, placement_quota_exceeded or insufficient_credits.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "403": { + "description": "Insufficient permissions, or an API key naming a mailbox it may not use.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "404": { + "description": "A sending mailbox, campaign, step or contact is not in the workspace.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "409": { + "description": "placement_no_seeds or placement_panel_unavailable.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "429": { + "description": "Rate limited, or placement_too_many_batches.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } + }, + "/placement/batches/preview": { + "post": { + "operationId": "placement_batches_preview", + "tags": [ + "placement" + ], + "summary": "Preview a placement batch", + "description": "What a batch request would come to: senders matched and selected, tests, the most copies sent, free and paid tests and credits. Writes and sends nothing. Scope SEND_CAMPAIGNS, org permission send_campaigns.", + "security": [ + { + "bearerAuth": [] + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PlacementBatchRequest" + } + } + } + }, + "responses": { + "200": { + "description": "The counts.", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "data" + ], + "properties": { + "data": { + "$ref": "#/components/schemas/PlacementBatchPreview" + } + } + } + } + } + }, + "400": { + "description": "Malformed request.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "401": { + "description": "Missing or invalid credentials.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "402": { + "description": "placement_not_entitled.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "403": { + "description": "Insufficient permissions.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "404": { + "description": "A sending mailbox, campaign, step or contact is not in the workspace.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "409": { + "description": "placement_no_seeds or placement_panel_unavailable.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "429": { + "description": "Rate limited.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } + }, + "/placement/batches/{id}": { + "get": { + "operationId": "placement_batches_get", + "tags": [ + "placement" + ], + "summary": "Get a placement batch", + "description": "One batch with its placement overall and grouped by sending domain, sending provider and recipient provider, worst first. Scope READ_ANALYTICS, org permission view_analytics.", + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "description": "Placement batch id.", + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "200": { + "description": "The batch.", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "data" + ], + "properties": { + "data": { + "$ref": "#/components/schemas/PlacementBatchDetail" + } + } + } + } + } + }, + "400": { + "description": "Invalid id.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "401": { + "description": "Missing or invalid credentials.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "403": { + "description": "Insufficient permissions.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "404": { + "description": "Not found.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "429": { + "description": "Rate limited.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } + }, + "/placement/batches/{id}/senders": { + "get": { + "operationId": "placement_batches_senders", + "tags": [ + "placement" + ], + "summary": "List a placement batch's senders", + "description": "The batch's mailboxes with where each one's copies landed and why any was deferred, skipped or failed. Scope READ_ANALYTICS, org permission view_analytics.", + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "description": "Placement batch id.", + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "name": "limit", + "in": "query", + "required": false, + "description": "Page size, 1 to 100. Default 25.", + "schema": { + "type": "integer", + "minimum": 1, + "maximum": 100, + "default": 25 + } + }, + { + "name": "cursor", + "in": "query", + "required": false, + "description": "Opaque cursor from a previous pagination.next_cursor.", + "schema": { + "type": "string" + } + }, + { + "name": "sort", + "in": "query", + "required": false, + "description": "worst (default, lowest inbox rate first), best, email or status.", + "schema": { + "type": "string", + "enum": [ + "worst", + "best", + "email", + "status" + ], + "default": "worst" + } + }, + { + "name": "status", + "in": "query", + "required": false, + "description": "Only mailboxes in this status.", + "schema": { + "type": "string", + "enum": [ + "queued", + "deferred", + "running", + "completed", + "skipped", + "failed", + "cancelled" + ] + } + }, + { + "name": "q", + "in": "query", + "required": false, + "description": "Only addresses containing this text.", + "schema": { + "type": "string", + "maxLength": 200 + } + } + ], + "responses": { + "200": { + "description": "A page of senders.", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "data", + "pagination" + ], + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/components/schemas/PlacementBatchSender" + } + }, + "pagination": { + "$ref": "#/components/schemas/Pagination" + } + } + } + } + } + }, + "400": { + "description": "Invalid id, cursor, limit, sort or status.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "401": { + "description": "Missing or invalid credentials.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "403": { + "description": "Insufficient permissions.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "404": { + "description": "Not found.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "429": { + "description": "Rate limited.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } + }, + "/placement/batches/{id}/cancel": { + "post": { + "operationId": "placement_batches_cancel", + "tags": [ + "placement" + ], + "summary": "Cancel a placement batch", + "description": "Stop the batch: no mailbox starts again, copies not sent yet are cancelled, copies already sent keep being classified. Scope SEND_CAMPAIGNS, org permission send_campaigns.", + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "description": "Placement batch id.", + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ], + "responses": { + "200": { + "description": "The cancelled batch.", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "data" + ], + "properties": { + "data": { + "$ref": "#/components/schemas/PlacementBatch" + } + } + } + } + } + }, + "400": { + "description": "Invalid id.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "401": { + "description": "Missing or invalid credentials.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "403": { + "description": "Insufficient permissions.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "404": { + "description": "Not found.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "409": { + "description": "placement_batch_not_running.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "429": { + "description": "Rate limited.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } + }, + "/placement/coverage": { + "get": { + "operationId": "placement_coverage", + "tags": [ + "placement" + ], + "summary": "Get placement fleet coverage", + "description": "How many of the workspace's connected sending mailboxes finished a placement test in the last 7 and 30 days, and how many never did. Scope READ_ANALYTICS, org permission view_analytics.", + "security": [ + { + "bearerAuth": [] + } + ], + "responses": { + "200": { + "description": "The coverage.", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "data" + ], + "properties": { + "data": { + "$ref": "#/components/schemas/PlacementCoverage" + } + } + } + } + } + }, + "401": { + "description": "Missing or invalid credentials.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "403": { + "description": "Insufficient permissions.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "429": { + "description": "Rate limited.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } + }, "/placement/seeds": { "get": { "operationId": "placement_seeds_list", @@ -18022,7 +19090,7 @@ } }, "400": { - "description": "A selection that names nothing, an exclusion list over 50000 or one resolving to more than 500 contacts (`too_many_contacts`), a select-all with no filters, a filter matching no contacts, or a filter matching more than 50000 (`selection_too_large`).", + "description": "A selection that names nothing, an exclusion list over 250000 or one resolving to more than 500 contacts (`too_many_contacts`), a select-all with no filters, a filter matching no contacts, or a filter matching more than 250000 (`selection_too_large`).", "content": { "application/json": { "schema": { @@ -30087,6 +31155,13 @@ ], "nullable": true, "description": "The live per-lead hold: an out-of-office auto-reply parked the contact, or a member paused them. Absent when the lead is not held." + }, + "cc": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CampaignLeadCC" + }, + "description": "Contacts copied on every email to this lead in this campaign. Absent when none." } } }, @@ -30597,7 +31672,7 @@ }, "ContactSelection": { "type": "object", - "description": "Names the contacts a bulk action applies to: either an explicit id list, or every contact matching a search (all + filters) minus the ids in exclude. The filter form lets one call cover far more contacts than a page, and is refused with selection_too_large past 50000 matches.", + "description": "Names the contacts a bulk action applies to: either an explicit id list, or every contact matching a search (all + filters) minus the ids in exclude. The filter form lets one call cover far more contacts than a page, and is refused with selection_too_large past 250000 matches.", "oneOf": [ { "required": [ @@ -30629,12 +31704,12 @@ "contacts": { "type": "array", "minItems": 1, - "maxItems": 1000, + "maxItems": 10000, "items": { "type": "string", "format": "uuid" }, - "description": "Contact ids (1 to 1000). Required unless all is set." + "description": "Contact ids (1 to 10000). Required unless all is set." }, "all": { "type": "boolean", @@ -30654,7 +31729,7 @@ "type": "string", "format": "uuid" }, - "maxItems": 50000, + "maxItems": 250000, "description": "Contact ids to drop from the resolved set. Ignored unless all is set." } } @@ -30693,12 +31768,12 @@ "contacts": { "type": "array", "minItems": 1, - "maxItems": 1000, + "maxItems": 10000, "items": { "type": "string", "format": "uuid" }, - "description": "Contact IDs to edit (1 to 1000). Required unless all is set." + "description": "Contact IDs to edit (1 to 10000). Required unless all is set." }, "all": { "type": "boolean", @@ -30718,7 +31793,7 @@ "type": "string", "format": "uuid" }, - "maxItems": 50000, + "maxItems": 250000, "description": "Contact ids to drop from the resolved set. Ignored unless all is set." }, "add_campaigns": { @@ -36835,7 +37910,7 @@ "type": "string", "format": "uuid" }, - "maxItems": 50000, + "maxItems": 250000, "description": "Contact ids to drop from the resolved set. Ignored unless all is set." } }, @@ -38875,9 +39950,11 @@ "type": "string", "enum": [ "manual", - "out_of_office" + "out_of_office", + "inbox_tagging", + "cc" ], - "description": "What wrote it." + "description": "What wrote it. cc holds a contact's own lead while they are copied on another lead's emails in the campaign; its reason is that lead's address." } }, "required": [ @@ -38927,6 +40004,140 @@ } } }, + "CampaignLeadCC": { + "type": "object", + "description": "A contact copied on every email one campaign sends one lead.", + "properties": { + "contact_id": { + "type": "string", + "format": "uuid" + }, + "email": { + "type": "string", + "format": "email" + }, + "first_name": { + "type": "string" + }, + "last_name": { + "type": "string" + }, + "company": { + "type": "string" + }, + "status": { + "type": "string", + "enum": [ + "active", + "unsubscribed", + "bounced", + "undeliverable" + ], + "description": "Whether the next email copies them: only active does." + }, + "bounced_at": { + "type": "string", + "format": "date-time", + "nullable": true, + "description": "Set when a bounce was attributed to this copy on this lead's thread." + } + }, + "required": [ + "contact_id", + "email", + "first_name", + "last_name", + "status" + ] + }, + "CampaignLeadCCResult": { + "type": "object", + "properties": { + "campaign_id": { + "type": "string", + "format": "uuid" + }, + "contact_id": { + "type": "string", + "format": "uuid" + }, + "cc": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CampaignLeadCC" + } + } + }, + "required": [ + "campaign_id", + "contact_id", + "cc" + ] + }, + "CampaignLeadCCRequest": { + "type": "object", + "properties": { + "contact_ids": { + "type": "array", + "maxItems": 2, + "items": { + "type": "string", + "format": "uuid" + }, + "description": "Contacts of the workspace to copy, at most 2. Duplicates are ignored; an empty list removes every copy." + } + }, + "required": [ + "contact_ids" + ] + }, + "CampaignLeadCCSuggestions": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "type": "object", + "properties": { + "contact_id": { + "type": "string", + "format": "uuid" + }, + "email": { + "type": "string", + "format": "email" + }, + "first_name": { + "type": "string" + }, + "last_name": { + "type": "string" + }, + "company": { + "type": "string" + }, + "reason": { + "type": "string", + "enum": [ + "company", + "domain" + ] + } + }, + "required": [ + "contact_id", + "email", + "first_name", + "last_name", + "reason" + ] + } + } + }, + "required": [ + "data" + ] + }, "MailboxSendAsIdentity": { "type": "object", "properties": { @@ -40388,6 +41599,14 @@ ], "format": "uuid" }, + "batch_id": { + "type": [ + "string", + "null" + ], + "format": "uuid", + "description": "The placement batch that started the test." + }, "subject": { "type": "string" }, @@ -40419,7 +41638,8 @@ "manual", "monitor", "admin", - "remote" + "remote", + "batch" ] }, "panel": { @@ -40832,6 +42052,660 @@ } } }, + "PlacementSenderScope": { + "type": "object", + "required": [ + "type" + ], + "description": "Mailboxes resolved on the server.", + "properties": { + "type": { + "type": "string", + "enum": [ + "campaign", + "workspace" + ] + }, + "campaign_id": { + "type": "string", + "format": "uuid", + "description": "Required with type campaign." + }, + "providers": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Only mailboxes hosted by these families, such as google_workspace or microsoft365." + }, + "domains": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Only mailboxes sending from these domains." + }, + "tag_ids": { + "type": "array", + "items": { + "type": "string", + "format": "uuid" + }, + "description": "Only mailboxes carrying one of these tags." + }, + "include_inactive": { + "type": "boolean", + "default": false, + "description": "Keep disconnected mailboxes." + }, + "untested_days": { + "type": "integer", + "minimum": 0, + "maximum": 365, + "description": "Only mailboxes with no finished placement test in this many days." + } + } + }, + "PlacementSample": { + "type": "object", + "description": "Which resolved mailboxes to keep.", + "properties": { + "mode": { + "type": "string", + "enum": [ + "all", + "random", + "percent", + "per_domain", + "per_provider" + ], + "default": "all" + }, + "count": { + "type": "integer", + "minimum": 1, + "description": "Mailboxes for random; per group for per_domain and per_provider." + }, + "percent": { + "type": "integer", + "minimum": 1, + "maximum": 100, + "description": "Share for percent, rounded up." + }, + "stratify": { + "type": "string", + "enum": [ + "provider", + "domain" + ], + "description": "With random or percent only: split the sample in proportion to each group's share." + } + } + }, + "PlacementBatchRequest": { + "type": "object", + "description": "A placement batch: the copy fields of a single test, and the senders by id or by scope.", + "properties": { + "sender_account_ids": { + "type": "array", + "items": { + "type": "string", + "format": "uuid" + }, + "description": "These mailboxes. Exactly one of sender_account_ids and sender_scope. Duplicates are dropped." + }, + "sender_scope": { + "$ref": "#/components/schemas/PlacementSenderScope" + }, + "sample": { + "$ref": "#/components/schemas/PlacementSample" + }, + "campaign_id": { + "type": "string", + "format": "uuid", + "description": "Render with this campaign's opt-out line, tracking and attachments." + }, + "sequence_id": { + "type": "string", + "format": "uuid", + "description": "A step of campaign_id; its subject and body are used when the request carries neither." + }, + "contact_id": { + "type": "string", + "format": "uuid", + "description": "The contact to render merge fields and AI blocks for. Defaults to the campaign's first lead." + }, + "subject": { + "type": "string", + "description": "Required when there is no step to take it from." + }, + "body_html": { + "type": "string" + }, + "body_plain": { + "type": "string" + }, + "tracking": { + "type": "string", + "enum": [ + "campaign", + "on", + "off", + "compare" + ], + "default": "campaign" + }, + "panel": { + "type": "string", + "enum": [ + "instance", + "workspace", + "cloud" + ], + "default": "instance" + }, + "seed_ids": { + "type": "array", + "items": { + "type": "string", + "format": "uuid" + }, + "description": "Only with panel workspace: send to these seed inboxes of the workspace and no others. Each must be connected and running, and each gets a copy; seeds_per_test does not cap the list, and a sender whose daily limit cannot pay for all of them is refused with placement_daily_budget. Omit to let the test pick." + }, + "families": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Only seeds at these provider families, as the overview's panels[].families[].family lists them. The way to narrow a test on the instance and cloud panels, whose addresses are masked." + }, + "pace": { + "type": "string", + "enum": [ + "spaced", + "quick" + ], + "default": "spaced", + "description": "A batch always sends spaced; quick is refused with a 400." + }, + "on_unavailable": { + "type": "string", + "enum": [ + "defer", + "skip" + ], + "default": "defer", + "description": "defer retries a mailbox that cannot send when its turn comes, for up to seven days; skip skips it." + }, + "max_credits": { + "type": "integer", + "minimum": 0, + "description": "The most credits you agree to pay across the whole batch for tests past the month's free ones. Needed only when the batch needs more tests than are free; the batch never spends more." + } + } + }, + "PlacementBatch": { + "type": "object", + "description": "One placement batch with its progress and headline placement.", + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "created_by": { + "type": [ + "string", + "null" + ], + "format": "uuid" + }, + "campaign_id": { + "type": [ + "string", + "null" + ], + "format": "uuid" + }, + "sequence_id": { + "type": [ + "string", + "null" + ], + "format": "uuid" + }, + "contact_id": { + "type": [ + "string", + "null" + ], + "format": "uuid" + }, + "subject": { + "type": "string" + }, + "tracking": { + "type": "string", + "enum": [ + "campaign", + "on", + "off", + "compare" + ] + }, + "panel": { + "type": "string", + "enum": [ + "instance", + "workspace", + "cloud" + ] + }, + "pace": { + "type": "string", + "enum": [ + "spaced", + "quick" + ] + }, + "families": { + "type": "array", + "items": { + "type": "string" + } + }, + "seed_ids": { + "type": "array", + "items": { + "type": "string", + "format": "uuid" + } + }, + "on_unavailable": { + "type": "string", + "enum": [ + "defer", + "skip" + ] + }, + "selection": { + "type": "object", + "description": "How the mailboxes were chosen.", + "properties": { + "sender_account_ids": { + "type": "integer", + "description": "How many ids were named." + }, + "sender_scope": { + "$ref": "#/components/schemas/PlacementSenderScope" + }, + "sample": { + "$ref": "#/components/schemas/PlacementSample" + }, + "matched": { + "type": "integer", + "description": "Mailboxes the scope resolved to before sampling." + } + } + }, + "sender_count": { + "type": "integer" + }, + "max_credits": { + "type": "integer" + }, + "credits_spent": { + "type": "integer" + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "completed", + "completed_with_warnings", + "cancelled", + "failed" + ] + }, + "error": { + "type": "string" + }, + "retry_until": { + "type": "string", + "format": "date-time", + "description": "Deferred mailboxes are retried until then, then skipped." + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "started_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "finished_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "progress": { + "type": "object", + "properties": { + "total": { + "type": "integer" + }, + "queued": { + "type": "integer" + }, + "deferred": { + "type": "integer" + }, + "running": { + "type": "integer" + }, + "completed": { + "type": "integer" + }, + "skipped": { + "type": "integer" + }, + "failed": { + "type": "integer" + }, + "cancelled": { + "type": "integer" + } + } + }, + "summary": { + "$ref": "#/components/schemas/PlacementCounts", + "description": "The headline copy: the tracked half of a tracking comparison." + } + } + }, + "PlacementBatchDetail": { + "description": "A placement batch with its placement grouped across the fleet.", + "allOf": [ + { + "$ref": "#/components/schemas/PlacementBatch" + }, + { + "type": "object", + "properties": { + "untracked": { + "$ref": "#/components/schemas/PlacementCounts", + "description": "The untracked half of a tracking comparison." + }, + "domains": { + "type": "array", + "items": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "label": { + "type": "string" + }, + "senders": { + "type": "integer" + }, + "tested": { + "type": "integer", + "description": "Senders that finished a test." + }, + "counts": { + "$ref": "#/components/schemas/PlacementCounts" + } + } + }, + "description": "Per sending domain, worst inbox rate first." + }, + "providers": { + "type": "array", + "items": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "label": { + "type": "string" + }, + "senders": { + "type": "integer" + }, + "tested": { + "type": "integer", + "description": "Senders that finished a test." + }, + "counts": { + "$ref": "#/components/schemas/PlacementCounts" + } + } + }, + "description": "Per sending provider, worst inbox rate first." + }, + "recipients": { + "type": "array", + "items": { + "$ref": "#/components/schemas/PlacementFamilyCounts" + } + }, + "matrix": { + "type": "array", + "items": { + "type": "object", + "properties": { + "domain": { + "type": "string" + }, + "recipients": { + "type": "array", + "items": { + "$ref": "#/components/schemas/PlacementFamilyCounts" + } + } + } + } + }, + "content": { + "type": "object", + "properties": { + "score": { + "type": "integer" + }, + "issues": { + "type": [ + "array", + "null" + ], + "items": { + "type": "object" + } + } + } + } + } + } + ] + }, + "PlacementBatchSender": { + "type": "object", + "description": "One mailbox of a batch.", + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "batch_id": { + "type": "string", + "format": "uuid" + }, + "email_account_id": { + "type": [ + "string", + "null" + ], + "format": "uuid" + }, + "sender_email": { + "type": "string" + }, + "sender_domain": { + "type": "string" + }, + "sender_family": { + "type": "string" + }, + "sender_family_label": { + "type": "string" + }, + "status": { + "type": "string", + "enum": [ + "queued", + "deferred", + "running", + "completed", + "skipped", + "failed", + "cancelled" + ] + }, + "reason": { + "type": "string", + "description": "Why the mailbox was deferred, skipped or failed, as an error code." + }, + "detail": { + "type": "string" + }, + "attempts": { + "type": "integer" + }, + "next_attempt_at": { + "type": "string", + "format": "date-time" + }, + "started_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "finished_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "summary": { + "$ref": "#/components/schemas/PlacementCounts" + }, + "test_ids": { + "type": "array", + "items": { + "type": "string", + "format": "uuid" + }, + "description": "The mailbox's tests, the untracked half first on a comparison." + } + } + }, + "PlacementBatchPreview": { + "type": "object", + "description": "What a batch request comes to.", + "properties": { + "matched": { + "type": "integer" + }, + "selected": { + "type": "integer" + }, + "inactive": { + "type": "integer" + }, + "domains": { + "type": "integer" + }, + "providers": { + "type": "array", + "items": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "label": { + "type": "string" + }, + "senders": { + "type": "integer" + } + } + } + }, + "variants": { + "type": "integer", + "description": "2 for a tracking comparison." + }, + "tests": { + "type": "integer" + }, + "seeds_per_test": { + "type": "integer" + }, + "max_sends": { + "type": "integer", + "description": "The most copies the batch sends; daily limits only make it fewer." + }, + "metered": { + "type": "boolean" + }, + "free_tests": { + "type": "integer" + }, + "paid_tests": { + "type": "integer" + }, + "credits": { + "type": "integer" + }, + "usage": { + "$ref": "#/components/schemas/PlacementUsage" + }, + "senders_max": { + "type": "integer" + }, + "concurrency": { + "type": "integer" + } + } + }, + "PlacementCoverage": { + "type": "object", + "properties": { + "mailboxes": { + "type": "integer" + }, + "tested_7d": { + "type": "integer" + }, + "tested_30d": { + "type": "integer" + }, + "never_tested": { + "type": "integer" + } + } + }, "WarmupPlacementCounts": { "type": "object", "description": "Where a set of verified warmup deliveries landed.", diff --git a/internal/api/handler/campaign_lead_cc.go b/internal/api/handler/campaign_lead_cc.go new file mode 100644 index 000000000..bfa7ca966 --- /dev/null +++ b/internal/api/handler/campaign_lead_cc.go @@ -0,0 +1,85 @@ +package handler + +import ( + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" +) + +// leadCCResponse is what the read and the write answer with. +type leadCCResponse struct { + CampaignID string `json:"campaign_id"` + ContactID string `json:"contact_id"` + CC []models.CampaignLeadCC `json:"cc"` +} + +func leadCCOK(c *gin.Context, campaignID, contactID uuid.UUID, cc []models.CampaignLeadCC) { + if cc == nil { + cc = []models.CampaignLeadCC{} + } + c.JSON(http.StatusOK, leadCCResponse{CampaignID: campaignID.String(), ContactID: contactID.String(), CC: cc}) +} + +// GetCampaignLeadCC lists the contacts copied on every email to one lead. +// +// GET /campaigns/:id/leads/:contactId/cc +func (h *Handler) GetCampaignLeadCC(c *gin.Context) { + orgID, campaignID, contactID, ok := leadHoldParams(c) + if !ok { + return + } + cc, xerr := h.CampaignService.ListLeadCC(c.Request.Context(), orgID, campaignID, contactID) + if xerr != nil { + errx.JSON(c, xerr) + return + } + leadCCOK(c, campaignID, contactID, cc) +} + +// SetCampaignLeadCC replaces the contacts copied on one lead. +// +// No Idempotency-Key: the body is the whole list, so a retry lands on the +// same state. +// +// PUT /campaigns/:id/leads/:contactId/cc +func (h *Handler) SetCampaignLeadCC(c *gin.Context) { + orgID, campaignID, contactID, ok := leadHoldParams(c) + if !ok { + return + } + var req models.SetCampaignLeadCC + if err := c.ShouldBindJSON(&req); err != nil { + errx.JSON(c, errx.InvalidBody(err)) + return + } + cc, xerr := h.CampaignService.SetLeadCC(c.Request.Context(), orgID, campaignID, contactID, req.ContactIDs) + if xerr != nil { + errx.JSON(c, xerr) + return + } + h.auditOrg(c, models.AuditActionUpdate, models.AuditEntityCampaignLead, &contactID, nil, map[string]string{ + "campaign_id": campaignID.String(), + "cc": strconv.Itoa(len(cc)), + }) + leadCCOK(c, campaignID, contactID, cc) +} + +// SuggestCampaignLeadCC offers the lead's likely colleagues to copy. +// +// GET /campaigns/:id/leads/:contactId/cc/suggestions +func (h *Handler) SuggestCampaignLeadCC(c *gin.Context) { + orgID, campaignID, contactID, ok := leadHoldParams(c) + if !ok { + return + } + out, xerr := h.CampaignService.SuggestLeadCC(c.Request.Context(), orgID, campaignID, contactID) + if xerr != nil { + errx.JSON(c, xerr) + return + } + c.JSON(http.StatusOK, gin.H{"data": out}) +} diff --git a/internal/api/handler/contact.go b/internal/api/handler/contact.go index a7a9dd0e7..49c1d1f25 100644 --- a/internal/api/handler/contact.go +++ b/internal/api/handler/contact.go @@ -14,8 +14,6 @@ import ( "github.com/warmbly/warmbly/internal/models" ) -const maxBulkOperationSize = 1000 - func (h *Handler) AddContacts(c *gin.Context) { userIDStr := middleware.GetUserID(c) diff --git a/internal/api/handler/contact_selection.go b/internal/api/handler/contact_selection.go index 2922529c6..ba06b4375 100644 --- a/internal/api/handler/contact_selection.go +++ b/internal/api/handler/contact_selection.go @@ -20,9 +20,9 @@ func (h *Handler) resolveContactSelection(c *gin.Context, orgID uuid.UUID, sel m errx.Handle(c, errx.New(errx.BadRequest, "no contacts provided")) return nil, false } - if len(sel.Contacts) > maxBulkOperationSize { + if len(sel.Contacts) > models.MaxContactBatchIDs { errx.Handle(c, errx.NewWithIdentifier(errx.BadRequest, "too_many_contacts", - fmt.Sprintf("too many contacts, maximum is %d per batch", maxBulkOperationSize))) + fmt.Sprintf("too many contacts, maximum is %d per batch", models.MaxContactBatchIDs))) return nil, false } return sel.Contacts, true diff --git a/internal/api/handler/internal_sync.go b/internal/api/handler/internal_sync.go index bbe4a120d..971cc1f9d 100644 --- a/internal/api/handler/internal_sync.go +++ b/internal/api/handler/internal_sync.go @@ -7,6 +7,8 @@ import ( "github.com/gin-gonic/gin" "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/observability/errs" "github.com/warmbly/warmbly/internal/repository" ) @@ -86,3 +88,58 @@ func (h *Handler) InternalSyncFolderMessages(c *gin.Context) { } c.JSON(http.StatusOK, gin.H{"messages": messages}) } + +// maxProviderFolderMessages bounds one provider-folder listing. +const maxProviderFolderMessages = 1000 + +// InternalSyncProviderFolderMessages answers the Gmail folder reconciliation: +// which rows does the platform still believe the provider has in these +// folders? The worker checks each against Gmail's labels and reports the ones +// that moved. +// +// GET /api/v1/internal/sync/provider-folder-messages?user_id=&email_id=&folders=inbox,archive&limit= +// -> 200 {"messages":[{"id":"...","provider_id":"...","provider_folder":"inbox","internal_date":"..."}]} +func (h *Handler) InternalSyncProviderFolderMessages(c *gin.Context) { + if h.EmailSyncState == nil { + c.JSON(http.StatusOK, gin.H{"messages": []repository.ProviderFolderMessage{}}) + return + } + userID, err := uuid.Parse(c.Query("user_id")) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid user_id"}) + return + } + emailID, err := uuid.Parse(c.Query("email_id")) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid email_id"}) + return + } + var folders []string + for _, raw := range strings.Split(c.Query("folders"), ",") { + f := strings.TrimSpace(raw) + if f == "" { + continue + } + if !models.ValidFolder(f) { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid folder"}) + return + } + folders = append(folders, f) + } + if len(folders) == 0 { + c.JSON(http.StatusBadRequest, gin.H{"error": "folders required"}) + return + } + limit, err := strconv.Atoi(c.Query("limit")) + if err != nil || limit < 1 || limit > maxProviderFolderMessages { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid limit"}) + return + } + messages, err := h.EmailSyncState.ListProviderFolderMessages(c.Request.Context(), userID, emailID, folders, limit) + if err != nil { + errs.CaptureException(err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "could not list stored messages"}) + return + } + c.JSON(http.StatusOK, gin.H{"messages": messages}) +} diff --git a/internal/api/handler/placement_batch.go b/internal/api/handler/placement_batch.go new file mode 100644 index 000000000..c4f8cd79d --- /dev/null +++ b/internal/api/handler/placement_batch.go @@ -0,0 +1,262 @@ +package handler + +import ( + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/api/middleware" + "github.com/warmbly/warmbly/internal/app/placement" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +type placementBatchRequest struct { + SenderAccountIDs []uuid.UUID `json:"sender_account_ids"` + SenderScope *models.PlacementSenderScope `json:"sender_scope"` + Sample models.PlacementSample `json:"sample"` + CampaignID string `json:"campaign_id"` + SequenceID string `json:"sequence_id"` + ContactID string `json:"contact_id"` + Subject string `json:"subject"` + BodyHTML string `json:"body_html"` + BodyPlain string `json:"body_plain"` + Tracking string `json:"tracking"` + Panel string `json:"panel"` + Pace string `json:"pace"` + Families []string `json:"families"` + SeedIDs []uuid.UUID `json:"seed_ids"` + OnUnavailable string `json:"on_unavailable"` + MaxCredits int `json:"max_credits"` +} + +// placementBatchInput binds a batch request; false after answering the error. +func (h *Handler) placementBatchInput(c *gin.Context) (placement.BatchInput, bool) { + var in placement.BatchInput + orgID := middleware.GetOrganizationID(c) + if orgID == nil { + errx.JSON(c, errx.New(errx.BadRequest, "no organization selected")) + return in, false + } + if !h.placementReady(c) { + return in, false + } + var req placementBatchRequest + if err := c.ShouldBindJSON(&req); err != nil { + errx.JSON(c, errx.InvalidBody(err)) + return in, false + } + campaignID, ok := optionalUUID(c, req.CampaignID, "campaign_id") + if !ok { + return in, false + } + sequenceID, ok := optionalUUID(c, req.SequenceID, "sequence_id") + if !ok { + return in, false + } + contactID, ok := optionalUUID(c, req.ContactID, "contact_id") + if !ok { + return in, false + } + in = placement.BatchInput{ + OrgID: *orgID, + SenderAccountIDs: req.SenderAccountIDs, + Scope: req.SenderScope, + Sample: req.Sample, + CampaignID: campaignID, + SequenceID: sequenceID, + ContactID: contactID, + Subject: req.Subject, + BodyHTML: req.BodyHTML, + BodyPlain: req.BodyPlain, + Tracking: req.Tracking, + Panel: req.Panel, + Pace: req.Pace, + Families: req.Families, + SeedIDs: req.SeedIDs, + OnUnavailable: req.OnUnavailable, + MaxCredits: req.MaxCredits, + } + // A key limited to some mailboxes only ever tests from those. + if middleware.GetAuthType(c) == middleware.AuthTypeAPIKey { + if allowed := middleware.GetAPIKeyAllowedEmailAccounts(c); len(allowed) > 0 { + in.AllowedSenders = allowed + } + } + if id, err := middleware.GetUserUUID(c); err == nil && id != uuid.Nil { + in.UserID = &id + } + return in, true +} + +// PreviewPlacementBatch reports how many senders, tests, sends and credits a +// batch would come to. It starts nothing. +func (h *Handler) PreviewPlacementBatch(c *gin.Context) { + in, ok := h.placementBatchInput(c) + if !ok { + return + } + preview, xerr := h.PlacementService.PreviewBatch(c.Request.Context(), in) + if xerr != nil { + errx.JSON(c, xerr) + return + } + c.JSON(http.StatusOK, gin.H{"data": preview}) +} + +// CreatePlacementBatch snapshots the senders and queues the batch; the +// backend starts its senders a few at a time. +func (h *Handler) CreatePlacementBatch(c *gin.Context) { + in, ok := h.placementBatchInput(c) + if !ok { + return + } + batch, xerr := h.PlacementService.CreateBatch(c.Request.Context(), in) + if xerr != nil { + errx.JSON(c, xerr) + return + } + id := batch.ID + h.auditOrg(c, models.AuditActionCreate, models.AuditEntityPlacementBatch, &id, nil, map[string]string{ + "senders": strconv.Itoa(batch.SenderCount), + "panel": batch.Panel, + }) + c.JSON(http.StatusCreated, gin.H{"data": batch}) +} + +// ListPlacementBatches lists the workspace's batches, newest first. +func (h *Handler) ListPlacementBatches(c *gin.Context) { + orgID := middleware.GetOrganizationID(c) + if orgID == nil { + errx.JSON(c, errx.New(errx.BadRequest, "no organization selected")) + return + } + if !h.placementReady(c) { + return + } + offset, ok := decodeOffsetCursor(c.Query("cursor")) + if !ok { + errx.JSON(c, errx.New(errx.BadRequest, "invalid cursor")) + return + } + limit, ok := placementLimit(c) + if !ok { + return + } + batches, total, xerr := h.PlacementService.ListBatches(c.Request.Context(), *orgID, limit, offset) + if xerr != nil { + errx.JSON(c, xerr) + return + } + c.JSON(http.StatusOK, gin.H{"data": batches, "pagination": pageMetaFor(offset, limit, len(batches), total)}) +} + +// GetPlacementBatch returns one batch with its placement overall and grouped +// by sending domain, sending provider and recipient provider. +func (h *Handler) GetPlacementBatch(c *gin.Context) { + orgID := middleware.GetOrganizationID(c) + if orgID == nil { + errx.JSON(c, errx.New(errx.BadRequest, "no organization selected")) + return + } + if !h.placementReady(c) { + return + } + id, err := uuid.Parse(c.Param("id")) + if err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid id")) + return + } + detail, xerr := h.PlacementService.GetBatch(c.Request.Context(), *orgID, id) + if xerr != nil { + errx.JSON(c, xerr) + return + } + c.JSON(http.StatusOK, gin.H{"data": detail}) +} + +// ListPlacementBatchSenders lists a batch's senders with where each one's +// copies landed, worst inbox rate first unless sorted otherwise. +func (h *Handler) ListPlacementBatchSenders(c *gin.Context) { + orgID := middleware.GetOrganizationID(c) + if orgID == nil { + errx.JSON(c, errx.New(errx.BadRequest, "no organization selected")) + return + } + if !h.placementReady(c) { + return + } + id, err := uuid.Parse(c.Param("id")) + if err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid id")) + return + } + offset, ok := decodeOffsetCursor(c.Query("cursor")) + if !ok { + errx.JSON(c, errx.New(errx.BadRequest, "invalid cursor")) + return + } + limit, ok := placementLimit(c) + if !ok { + return + } + senders, total, xerr := h.PlacementService.ListBatchSenders(c.Request.Context(), *orgID, id, repository.PlacementBatchSenderFilter{ + Status: c.Query("status"), + Search: c.Query("q"), + Sort: c.Query("sort"), + Limit: limit, + Offset: offset, + }) + if xerr != nil { + errx.JSON(c, xerr) + return + } + c.JSON(http.StatusOK, gin.H{"data": senders, "pagination": pageMetaFor(offset, limit, len(senders), total)}) +} + +// CancelPlacementBatch stops a batch. Copies already sent keep being +// classified. +func (h *Handler) CancelPlacementBatch(c *gin.Context) { + orgID := middleware.GetOrganizationID(c) + if orgID == nil { + errx.JSON(c, errx.New(errx.BadRequest, "no organization selected")) + return + } + if !h.placementReady(c) { + return + } + id, err := uuid.Parse(c.Param("id")) + if err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid id")) + return + } + view, xerr := h.PlacementService.CancelBatch(c.Request.Context(), *orgID, id) + if xerr != nil { + errx.JSON(c, xerr) + return + } + h.auditOrg(c, models.AuditActionStop, models.AuditEntityPlacementBatch, &id, nil, nil) + c.JSON(http.StatusOK, gin.H{"data": view}) +} + +// GetPlacementCoverage reports how much of the workspace's connected fleet +// delivered a placement test in the last 7 and 30 days. +func (h *Handler) GetPlacementCoverage(c *gin.Context) { + orgID := middleware.GetOrganizationID(c) + if orgID == nil { + errx.JSON(c, errx.New(errx.BadRequest, "no organization selected")) + return + } + if !h.placementReady(c) { + return + } + cov, xerr := h.PlacementService.Coverage(c.Request.Context(), *orgID) + if xerr != nil { + errx.JSON(c, xerr) + return + } + c.JSON(http.StatusOK, gin.H{"data": cov}) +} diff --git a/internal/api/routes.go b/internal/api/routes.go index 6c95e06f7..d9dc56510 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -191,6 +191,10 @@ func Run( // folder, so the worker can drop the rows the server no longer reports. internal.GET("/sync/folder-messages", h.InternalSyncFolderMessages) + // Gmail folder reconciliation: the rows the platform believes Gmail + // has in a folder, so the worker can report the ones that moved. + internal.GET("/sync/provider-folder-messages", h.InternalSyncProviderFolderMessages) + // Worker bootstrap config + heartbeat. Workers POST their identity // on boot (worker_id + bind_ip + tag) and pull their runtime config // instead of carrying it all in the install-time env file. @@ -704,6 +708,14 @@ func Run( campaigns.POST("/:id/leads/:contactId/pause", m.RequireOrganization(), m.RequireAccess(models.PermManageCampaigns, models.APIPermWriteCampaigns), h.PauseCampaignLead) campaigns.POST("/:id/leads/:contactId/resume", m.RequireOrganization(), m.RequireAccess(models.PermManageCampaigns, models.APIPermWriteCampaigns), h.ResumeCampaignLead) + // Contacts copied on every email to one lead, so colleagues + // share one thread. The answers carry contact details, hence + // the contacts gate too. PUT replaces the list, so retries are + // safe. + campaigns.GET("/:id/leads/:contactId/cc", m.RequireOrganization(), m.RequireAccess(models.PermViewCampaigns, models.APIPermReadCampaigns), m.RequireAccess(models.PermViewContacts, models.APIPermReadContacts), h.GetCampaignLeadCC) + campaigns.PUT("/:id/leads/:contactId/cc", m.RequireOrganization(), m.RequireAccess(models.PermManageCampaigns, models.APIPermWriteCampaigns), m.RequireAccess(models.PermViewContacts, models.APIPermReadContacts), h.SetCampaignLeadCC) + campaigns.GET("/:id/leads/:contactId/cc/suggestions", m.RequireOrganization(), m.RequireAccess(models.PermViewCampaigns, models.APIPermReadCampaigns), m.RequireAccess(models.PermViewContacts, models.APIPermReadContacts), h.SuggestCampaignLeadCC) + sequences := campaigns.Group("/:id/steps") { sequences.GET("", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadCampaigns), h.GetSequences) @@ -1011,6 +1023,14 @@ func Run( placementTests.GET("/tests/:id", m.RateLimitMiddleware(models.RateLimitAnalytics), m.RequireAccess(models.PermViewAnalytics, models.APIPermReadAnalytics), h.GetPlacementTest) placementTests.POST("/tests", m.RequireAccess(models.PermSendCampaigns, models.APIPermSendCampaigns), h.CreatePlacementTest) placementTests.POST("/tests/:id/cancel", m.RequireAccess(models.PermSendCampaigns, models.APIPermSendCampaigns), h.CancelPlacementTest) + // Batches: one test run from many senders, started a few at a time. + placementTests.GET("/batches", m.RateLimitMiddleware(models.RateLimitAnalytics), m.RequireAccess(models.PermViewAnalytics, models.APIPermReadAnalytics), h.ListPlacementBatches) + placementTests.GET("/batches/:id", m.RateLimitMiddleware(models.RateLimitAnalytics), m.RequireAccess(models.PermViewAnalytics, models.APIPermReadAnalytics), h.GetPlacementBatch) + placementTests.GET("/batches/:id/senders", m.RateLimitMiddleware(models.RateLimitAnalytics), m.RequireAccess(models.PermViewAnalytics, models.APIPermReadAnalytics), h.ListPlacementBatchSenders) + placementTests.POST("/batches/preview", m.RateLimitMiddleware(models.RateLimitAnalytics), m.RequireAccess(models.PermSendCampaigns, models.APIPermSendCampaigns), h.PreviewPlacementBatch) + placementTests.POST("/batches", m.RequireAccess(models.PermSendCampaigns, models.APIPermSendCampaigns), h.CreatePlacementBatch) + placementTests.POST("/batches/:id/cancel", m.RequireAccess(models.PermSendCampaigns, models.APIPermSendCampaigns), h.CancelPlacementBatch) + placementTests.GET("/coverage", m.RateLimitMiddleware(models.RateLimitAnalytics), m.RequireAccess(models.PermViewAnalytics, models.APIPermReadAnalytics), h.GetPlacementCoverage) placementTests.GET("/seeds", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadEmails), h.ListPlacementSeeds) placementTests.PUT("/seeds/:id", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.SetPlacementSeed) } diff --git a/internal/app/advanced/inbound_bounce.go b/internal/app/advanced/inbound_bounce.go index d8749f2a7..eb41beb9d 100644 --- a/internal/app/advanced/inbound_bounce.go +++ b/internal/app/advanced/inbound_bounce.go @@ -7,6 +7,7 @@ import ( "github.com/google/uuid" "github.com/warmbly/warmbly/internal/errx" "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/pkg/mailhdr" ) // RecordInboundBounce turns a permanent NDR the worker parsed into a bounce @@ -60,9 +61,19 @@ func (s *service) RecordInboundBounce(ctx context.Context, emailAccountID uuid.U if ct, cerr := s.taskRepo.GetCampaignTask(ctx, task.ID); cerr == nil && ct != nil { req.CampaignID = ct.CampaignID req.ContactID = ct.ContactID - if req.RecipientEmail == "" && ct.ContactID != nil { + if ct.ContactID != nil { if contact, cerr := s.contactRepo.GetByID(ctx, *ct.ContactID); cerr == nil && contact != nil { - req.RecipientEmail = contact.Email + switch { + case req.RecipientEmail == "": + req.RecipientEmail = contact.Email + case ct.CampaignID != nil && !strings.EqualFold(mailhdr.Bare(req.RecipientEmail), strings.TrimSpace(contact.Email)): + if owner, isCopy := s.copyBounceOwner(ctx, *ct.CampaignID, *ct.ContactID, req.RecipientEmail); isCopy { + // A copy bounced, not the lead: the lead keeps its + // sequence, and the copy's own NDR is its own event. + req.ContactID = owner + req.IdempotencyKey += ":" + strings.ToLower(mailhdr.Bare(req.RecipientEmail)) + } + } } } } @@ -144,3 +155,26 @@ func (s *service) RecordInboundComplaint(ctx context.Context, emailAccountID uui return s.IngestDeliverabilityEvent(ctx, *account.OrganizationID, req) } + +// copyBounceOwner tells a bounced copy apart from the lead. A contact copied on +// the lead is marked bounced there and owns the event; an address from the +// campaign's own CC or BCC owns it with no contact. Any other address (a +// forward, an alias) stays the lead's, as before. +func (s *service) copyBounceOwner(ctx context.Context, campaignID, leadID uuid.UUID, address string) (*uuid.UUID, bool) { + bare := mailhdr.Bare(address) + if s.campaignProgressRepo != nil { + if id, err := s.campaignProgressRepo.MarkLeadCCBounced(ctx, campaignID, leadID, bare); err == nil && id != nil { + return id, true + } + } + if campaign, err := s.campaignRepo.GetByID(ctx, campaignID); err == nil && campaign != nil { + for _, list := range [][]string{campaign.CC, campaign.BCC} { + for _, a := range list { + if strings.EqualFold(mailhdr.Bare(a), bare) { + return nil, true + } + } + } + } + return &leadID, false +} diff --git a/internal/app/advanced/inbound_bounce_test.go b/internal/app/advanced/inbound_bounce_test.go index 502ec9c2c..9c9ce56af 100644 --- a/internal/app/advanced/inbound_bounce_test.go +++ b/internal/app/advanced/inbound_bounce_test.go @@ -83,3 +83,49 @@ func TestRecordInboundBounceStillAttributesRealSends(t *testing.T) { t.Fatal("a campaign NDR was dropped by the warmup gate") } } + +type copyBounceProgress struct { + repository.CampaignProgressRepository + copies map[string]uuid.UUID +} + +func (r copyBounceProgress) MarkLeadCCBounced(_ context.Context, _, _ uuid.UUID, address string) (*uuid.UUID, error) { + if id, ok := r.copies[address]; ok { + return &id, nil + } + return nil, nil +} + +type copyBounceCampaigns struct { + repository.CampaignRepository + cc, bcc []string +} + +func (r copyBounceCampaigns) GetByID(_ context.Context, id uuid.UUID) (*models.Campaign, error) { + return &models.Campaign{ID: id, CC: r.cc, BCC: r.bcc}, nil +} + +// A DSN naming someone copied on the send bounces that copy, not the lead; an +// address nobody copied (a forward, an alias) stays the lead's as before. +func TestCopyBounceOwnerTellsACopyFromTheLead(t *testing.T) { + lead, copied := uuid.New(), uuid.New() + s := &service{ + campaignProgressRepo: copyBounceProgress{copies: map[string]uuid.UUID{"jonas@acme.test": copied}}, + campaignRepo: copyBounceCampaigns{cc: []string{"Boss "}, bcc: []string{"crm@acme.test"}}, + } + for _, tc := range []struct { + address string + owner *uuid.UUID + isCopy bool + }{ + {"jonas@acme.test", &copied, true}, + {"BOSS@acme.test", nil, true}, + {"crm@acme.test", nil, true}, + {"forwarded@elsewhere.test", &lead, false}, + } { + owner, isCopy := s.copyBounceOwner(context.Background(), uuid.New(), lead, tc.address) + if isCopy != tc.isCopy || (owner == nil) != (tc.owner == nil) || (owner != nil && *owner != *tc.owner) { + t.Errorf("%s: owner %v copy %v, want %v %v", tc.address, owner, isCopy, tc.owner, tc.isCopy) + } + } +} diff --git a/internal/app/advanced/incoming_reply_test.go b/internal/app/advanced/incoming_reply_test.go index 0e6ee36a0..02b520c12 100644 --- a/internal/app/advanced/incoming_reply_test.go +++ b/internal/app/advanced/incoming_reply_test.go @@ -84,6 +84,17 @@ type incomingReplyProgressRepo struct { receivingSent bool completeErr error advanced *incomingReplyAdvancedRepo + copies []models.CampaignLeadCC + copiedLead *repository.CopiedLeadRef + copiesErr error +} + +func (r *incomingReplyProgressRepo) ListLeadCC(context.Context, uuid.UUID, uuid.UUID) ([]models.CampaignLeadCC, error) { + return r.copies, r.copiesErr +} + +func (r *incomingReplyProgressRepo) LeadForCopiedReply(context.Context, uuid.UUID, uuid.UUID) (*repository.CopiedLeadRef, error) { + return r.copiedLead, nil } func (r *incomingReplyProgressRepo) IsInboundReplySource(context.Context, uuid.UUID, uuid.UUID) (bool, error) { diff --git a/internal/app/advanced/lead_copy_reply_test.go b/internal/app/advanced/lead_copy_reply_test.go new file mode 100644 index 000000000..4415bce26 --- /dev/null +++ b/internal/app/advanced/lead_copy_reply_test.go @@ -0,0 +1,185 @@ +package advanced + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +type copyReplyAdvancedRepo struct { + *incomingReplyAdvancedRepo + suppressed []string +} + +func (r *copyReplyAdvancedRepo) UpsertSuppressedRecipient(_ context.Context, s *models.SuppressedRecipient) error { + r.suppressed = append(r.suppressed, strings.ToLower(s.Email)) + return nil +} + +type copyReplyContactRepo struct { + incomingReplyContactRepo +} + +func (copyReplyContactRepo) SetSubscribedByEmail(context.Context, uuid.UUID, string, bool) error { + return nil +} + +type copyReplyProgressRepo struct { + *incomingReplyProgressRepo + heldEverywhere int +} + +func (r *copyReplyProgressRepo) HoldLeadEverywhere(context.Context, uuid.UUID, *time.Time, string, string) ([]uuid.UUID, error) { + r.heldEverywhere++ + return nil, nil +} + +// newCopyReplyService is the incoming-reply harness with the lead copying +// jonas@acme.test, answering in the lead's thread. +func newCopyReplyService(t *testing.T) (*service, *copyReplyProgressRepo, *copyReplyAdvancedRepo, uuid.UUID) { + t.Helper() + orgID, accountID, leadID := uuid.New(), uuid.New(), uuid.New() + account := &models.Email{ID: accountID, OrganizationID: &orgID, Email: "sender@example.test"} + svc, progress := newIncomingReplyService(account, nil, leadID) + progress.copies = []models.CampaignLeadCC{{ContactID: uuid.New(), Email: "jonas@acme.test", Status: models.LeadCCStatusActive}} + adv := ©ReplyAdvancedRepo{incomingReplyAdvancedRepo: progress.advanced} + wrapped := ©ReplyProgressRepo{incomingReplyProgressRepo: progress} + svc.repo = adv + svc.campaignProgressRepo = wrapped + svc.contactRepo = copyReplyContactRepo{svc.contactRepo.(incomingReplyContactRepo)} + return svc, wrapped, adv, accountID +} + +func copyReply(accountID uuid.UUID, subject, body string, inReplyTo []string) *models.EmailMessageStoreData { + return &models.EmailMessageStoreData{ + ID: uuid.New(), EmailID: accountID, Folder: models.FolderInbox, + FromAddr: []string{"Jonas "}, + ToAddr: []string{"sender@example.test"}, + InReplyTo: inReplyTo, + Subject: subject, + Snippet: body, + BodyText: body, + } +} + +// A copy asking to stop is about the copy: the lead they were copied on is +// not suppressed with them. +func TestCopyOptOutSuppressesOnlyTheCopy(t *testing.T) { + svc, _, adv, accountID := newCopyReplyService(t) + if xerr := svc.ProcessIncomingReply(context.Background(), accountID, + copyReply(accountID, "Re: Hello", "Please remove me from your list.", []string{""})); xerr != nil { + t.Fatal(xerr) + } + if len(adv.suppressed) != 1 || adv.suppressed[0] != "jonas@acme.test" { + t.Fatalf("suppressed %v, want only the copy", adv.suppressed) + } +} + +// A copy's away message says nothing about the lead's desk. +func TestCopyOutOfOfficeDoesNotHoldTheLead(t *testing.T) { + svc, progress, _, accountID := newCopyReplyService(t) + if xerr := svc.ProcessIncomingReply(context.Background(), accountID, + copyReply(accountID, "Automatic reply: Hello", "I am out of the office until Monday.", []string{""})); xerr != nil { + t.Fatal(xerr) + } + if progress.heldEverywhere != 0 { + t.Fatalf("the lead was held %d times for a copy's away message", progress.heldEverywhere) + } +} + +// A copy answering further down the thread names no message of ours; the +// mailbox that wrote to the lead ties it back, and it is the lead's reply. +func TestCopyReplyDownThreadCountsForTheLead(t *testing.T) { + svc, progress, _, accountID := newCopyReplyService(t) + copyContact := &models.Contact{ID: uuid.New(), Email: "jonas@acme.test"} + svc.taskRepo = incomingReplyTaskRepo{} + cr := svc.contactRepo.(copyReplyContactRepo) + cr.senderContact = copyContact + svc.contactRepo = cr + progress.copiedLead = &repository.CopiedLeadRef{CampaignID: uuid.New(), ContactID: uuid.New(), SequenceID: uuid.New()} + + if xerr := svc.ProcessIncomingReply(context.Background(), accountID, + copyReply(accountID, "Re: Hello", "Sounds good, let's talk Tuesday.", []string{""})); xerr != nil { + t.Fatal(xerr) + } + if progress.replied != 1 { + t.Fatalf("RecordEmailReplied calls = %d, want the copy's reply counted for the lead", progress.replied) + } +} + +// The link in a copied message cannot say who used it, so it opts out the +// lead and everyone copied; a sequence action is about the lead alone. +func TestUnsubscribeLinkOptsOutTheLeadsCopies(t *testing.T) { + for _, tc := range []struct { + name string + run func(s *service, org, campaign, lead uuid.UUID) *errx.Error + want []string + }{ + {"link", func(s *service, org, campaign, lead uuid.UUID) *errx.Error { + return s.UnsubscribeFromLink(context.Background(), org, campaign, lead, "one_click") + }, []string{"task-contact@example.test", "jonas@acme.test"}}, + {"sequence action", func(s *service, _, campaign, lead uuid.UUID) *errx.Error { + return s.Unsubscribe(context.Background(), campaign, lead) + }, []string{"task-contact@example.test"}}, + } { + svc, _, adv, _ := newCopyReplyService(t) + campaign := svc.campaignRepo.(incomingReplyCampaignRepo).campaign + lead := svc.contactRepo.(copyReplyContactRepo).taskContact.ID + if xerr := tc.run(svc, *campaign.OrganizationID, campaign.ID, lead); xerr != nil { + t.Fatalf("%s: %v", tc.name, xerr) + } + if strings.Join(adv.suppressed, ",") != strings.Join(tc.want, ",") { + t.Fatalf("%s: suppressed %v, want %v", tc.name, adv.suppressed, tc.want) + } + } +} + +// An unreadable copy list is an error, never "not a copy": guessing would +// suppress or hold the lead for what a copy did. +func TestCopyLookupFailureIsNotReadAsTheLead(t *testing.T) { + svc, progress, adv, accountID := newCopyReplyService(t) + progress.copiesErr = errors.New("database unavailable") + if xerr := svc.ProcessIncomingReply(context.Background(), accountID, + copyReply(accountID, "Re: Hello", "Please remove me from your list.", []string{""})); xerr == nil { + t.Fatal("a failed copy lookup was processed as if the sender were not a copy") + } + if len(adv.suppressed) != 0 { + t.Fatalf("suppressed %v on a failed lookup", adv.suppressed) + } +} + +// A fresh message from a copy is not a reply to anything, so it credits no lead. +func TestCopyFreshMessageCreditsNoLead(t *testing.T) { + svc, progress, _, accountID := newCopyReplyService(t) + svc.taskRepo = incomingReplyTaskRepo{} + cr := svc.contactRepo.(copyReplyContactRepo) + cr.senderContact = &models.Contact{ID: uuid.New(), Email: "jonas@acme.test"} + svc.contactRepo = cr + progress.copiedLead = &repository.CopiedLeadRef{CampaignID: uuid.New(), ContactID: uuid.New(), SequenceID: uuid.New()} + + if xerr := svc.ProcessIncomingReply(context.Background(), accountID, + copyReply(accountID, "Quick question", "Unrelated: are you at the fair next week?", nil)); xerr != nil { + t.Fatal(xerr) + } + if progress.replied != 0 { + t.Fatalf("RecordEmailReplied calls = %d, want none for a message that replies to nothing", progress.replied) + } +} + +// The opt-out is not acknowledged while a copy is still sendable. +func TestUnsubscribeFailsWhenCopiesCannotBeRead(t *testing.T) { + svc, progress, _, _ := newCopyReplyService(t) + progress.copiesErr = errors.New("database unavailable") + campaign := svc.campaignRepo.(incomingReplyCampaignRepo).campaign + lead := svc.contactRepo.(copyReplyContactRepo).taskContact.ID + if xerr := svc.UnsubscribeFromLink(context.Background(), *campaign.OrganizationID, campaign.ID, lead, "link"); xerr == nil { + t.Fatal("the unsubscribe succeeded without reaching the lead's copies") + } +} diff --git a/internal/app/advanced/service.go b/internal/app/advanced/service.go index 927299464..10f0e8228 100644 --- a/internal/app/advanced/service.go +++ b/internal/app/advanced/service.go @@ -568,7 +568,7 @@ func (s *service) ListCategories(ctx context.Context, orgID uuid.UUID) ([]models // creator is nil: an automation has no human behind it. func (s *service) CreateCategory(ctx context.Context, orgID uuid.UUID, title, color string) (models.MiniCategory, error) { if s.categoryRepo == nil { - return models.MiniCategory{}, errx.New(errx.BadRequest, "categories are not available") + return models.MiniCategory{}, errx.New(errx.BadRequest, "labels are not available") } if strings.TrimSpace(color) == "" { color = "#64748b" @@ -641,6 +641,72 @@ func (s *service) unsubscribe(ctx context.Context, expectOrg *uuid.UUID, campaig "contact_email": contact.Email, "source": via, }) + + // The link in a message is the same for everyone it copied and cannot say + // who used it, so it opts all of them out. A sequence action is about the + // lead alone. + if via != "action" { + return s.unsubscribeLeadCopies(ctx, *campaign.OrganizationID, campaignID, contactID, contact.Email, via, reason) + } + return nil +} + +// isLeadCopy reports whether sender is one of the contacts copied on the +// lead's emails rather than the lead answering from another address. A failed +// read is an error, never "not a copy", which would charge the lead. +func (s *service) isLeadCopy(ctx context.Context, campaignID, contactID uuid.UUID, leadEmail, sender string) (bool, error) { + if s.campaignProgressRepo == nil || sender == "" || strings.EqualFold(leadEmail, sender) { + return false, nil + } + copies, err := s.campaignProgressRepo.ListLeadCC(ctx, campaignID, contactID) + if err != nil { + return false, err + } + for _, cp := range copies { + if strings.EqualFold(strings.TrimSpace(cp.Email), sender) { + return true, nil + } + } + return false, nil +} + +// unsubscribeLeadCopies suppresses every contact copied on one lead's emails. +// A failure fails the request, so the opt-out is retried rather than +// acknowledged with a copy still sendable; the upserts are idempotent. +func (s *service) unsubscribeLeadCopies(ctx context.Context, orgID, campaignID, contactID uuid.UUID, leadEmail, via, reason string) *errx.Error { + if s.campaignProgressRepo == nil { + return nil + } + copies, err := s.campaignProgressRepo.ListLeadCC(ctx, campaignID, contactID) + if err != nil { + return toErrx(err) + } + for _, cp := range copies { + addr := strings.ToLower(strings.TrimSpace(cp.Email)) + if addr == "" { + continue + } + if err := s.repo.UpsertSuppressedRecipient(ctx, &models.SuppressedRecipient{ + OrganizationID: orgID, + Email: addr, + Kind: models.SuppressionKindEmail, + Reason: reason + " on an email copied to them", + Source: models.DeliverabilityEventUnsubscribe, + CampaignID: &campaignID, + Metadata: map[string]interface{}{"via": via, "copied_on": leadEmail}, + }); err != nil { + return toErrx(err) + } + if err := s.contactRepo.SetSubscribedByEmail(ctx, orgID, addr, false); err != nil { + log.Warn().Err(err).Str("contact_id", cp.ContactID.String()).Msg("unsubscribe: could not clear a copied contact's subscription flag") + } + s.emit(ctx, orgID, models.WebhookEventCampaignUnsubscribed, map[string]any{ + "campaign_id": campaignID.String(), + "contact_id": cp.ContactID.String(), + "contact_email": cp.Email, + "source": via, + }) + } return nil } @@ -1173,6 +1239,10 @@ func (s *service) ProcessIncomingReply(ctx context.Context, emailAccountID uuid. // contactEmail is the address we mailed, which is not always the one // that answered; an opt-out has to reach both. var contactEmail string + // senderIsCopy is a reply from a contact copied on the lead's emails. It + // counts as the lead's reply, but the copy's own away message or opt-out + // is about the copy, not the lead. + var senderIsCopy bool // First, try exact message threading via In-Reply-To. for _, mid := range msg.InReplyTo { @@ -1223,6 +1293,11 @@ func (s *service) ProcessIncomingReply(ctx context.Context, emailAccountID uuid. campaignID = ct.CampaignID contactID = ct.ContactID sequenceID = ct.SequenceID + isCopy, cerr := s.isLeadCopy(ctx, *ct.CampaignID, *ct.ContactID, contactEmail, sender) + if cerr != nil { + return toErrx(cerr) + } + senderIsCopy = isCopy break } @@ -1245,6 +1320,29 @@ func (s *service) ProcessIncomingReply(ctx context.Context, emailAccountID uuid. } } + // A copied contact answering further down the thread (to the lead's own + // reply, say) names no message of ours; the mailbox that wrote to the + // lead is the evidence, and the reply is the lead's. A fresh message with + // no parent is not a reply to anything and credits nobody. + if campaignID == nil && contactID != nil && !referencesCampaignThread && len(msg.InReplyTo) > 0 { + ref, err := s.campaignProgressRepo.LeadForCopiedReply(ctx, *contactID, emailAccountID) + if err != nil { + return toErrx(err) + } + if ref != nil { + lead, lerr := s.contactRepo.GetByID(ctx, ref.ContactID) + if lerr != nil { + return lerr + } + if lead != nil { + campaignID, sequenceID = &ref.CampaignID, &ref.SequenceID + contactID = &ref.ContactID + contactEmail = strings.TrimSpace(lead.Email) + senderIsCopy = true + } + } + } + if campaignID != nil && contactID != nil && sequenceID != nil { cID, ctID, sID := *campaignID, *contactID, *sequenceID @@ -1416,7 +1514,7 @@ func (s *service) ProcessIncomingReply(ctx context.Context, emailAccountID uuid. } var held *time.Time - if campaignID != nil && contactID != nil && verdict.Class == replyclassify.ClassOutOfOffice && settings.ReplyIntent.HoldOnOutOfOffice { + if campaignID != nil && contactID != nil && !senderIsCopy && verdict.Class == replyclassify.ClassOutOfOffice && settings.ReplyIntent.HoldOnOutOfOffice { held = s.holdForOutOfOffice(ctx, *account.OrganizationID, *contactID, settings.ReplyIntent, msg) } @@ -1446,8 +1544,9 @@ func (s *service) ProcessIncomingReply(ctx context.Context, emailAccountID uuid. if err := s.contactRepo.SetSubscribedByEmail(ctx, *account.OrganizationID, sender, false); err != nil { log.Warn().Err(err).Msg("reply opt-out: could not clear the contact's subscription flag") } - // Answered from another address: the one we mailed asked to stop too. - if contactEmail != "" && !strings.EqualFold(contactEmail, sender) { + // Answered from another address: the one we mailed asked to stop too, + // unless it was a copy asking for themselves. + if contactEmail != "" && !senderIsCopy && !strings.EqualFold(contactEmail, sender) { _ = s.repo.UpsertSuppressedRecipient(ctx, &models.SuppressedRecipient{ OrganizationID: *account.OrganizationID, Email: strings.ToLower(contactEmail), diff --git a/internal/app/advisor/service.go b/internal/app/advisor/service.go index 347e5c0f0..0f8d25999 100644 --- a/internal/app/advisor/service.go +++ b/internal/app/advisor/service.go @@ -162,6 +162,10 @@ func WithCopyJudge(asker typesafe.Asker, cache repository.CopyJudgmentRepository // least important cards their rewrite, and the next run picks them up. const maxNarrationsPerRun = 12 +// narrationBudget bounds the completions of one run so the summary and run +// record still land inside the caller's deadline. +const narrationBudget = 60 * time.Second + func (s *service) Evaluate(ctx context.Context, orgID uuid.UUID, trigger string) (*models.AdvisorSummary, error) { settings, err := s.repo.GetSettings(ctx, orgID) if err != nil { @@ -219,7 +223,9 @@ func (s *service) Evaluate(ctx context.Context, orgID uuid.UUID, trigger string) // rather than reporting problems that no longer exist. fixed := s.autopilot(ctx, orgID, settings, stored) - narrated := s.narrateBatch(ctx, orgID, stored) + nctx, cancelNarrate := context.WithTimeout(ctx, narrationBudget) + narrated := s.narrateBatch(nctx, orgID, stored) + cancelNarrate() summary, err := s.repo.Summary(ctx, orgID) if err != nil { @@ -303,7 +309,7 @@ func (s *service) narrateBatch(ctx context.Context, orgID uuid.UUID, findings [] // Detect, so the cap spends the budget on what matters. count := 0 for _, f := range findings { - if count >= maxNarrationsPerRun { + if count >= maxNarrationsPerRun || ctx.Err() != nil { break } if f.Narrated { diff --git a/internal/app/aitools/tools_contacts.go b/internal/app/aitools/tools_contacts.go index 4dbce9e1d..93d632e54 100644 --- a/internal/app/aitools/tools_contacts.go +++ b/internal/app/aitools/tools_contacts.go @@ -25,7 +25,7 @@ func (d Deps) registerContactTools(r *Registry) { r.Register(Tool{ Name: "get_contact", - Description: "Get one contact by id, including custom fields, categories, subscription state, and engagement summary.", + Description: "Get one contact by id, including custom fields, labels (categories), subscription state, and engagement summary.", InputSchema: objectSchema(map[string]any{ "contact_id": strProp("The contact's UUID."), }, "contact_id"), @@ -55,10 +55,10 @@ func (d Deps) registerContactTools(r *Registry) { r.Register(Tool{ Name: "add_tag", - Description: "Add a category (tag) to a contact. The category_id comes from a contact's categories in get_contact/search results.", + Description: "Add a label to a contact. Labels are called categories in the API; the category_id comes from a contact's categories in get_contact/search results.", InputSchema: objectSchema(map[string]any{ "contact_id": strProp("The contact's UUID."), - "category_id": strProp("The category (tag) UUID to add."), + "category_id": strProp("The label (category) UUID to add."), }, "contact_id", "category_id"), Risk: generation.RiskWrite, RequiredOrgPerm: models.PermManageContacts, @@ -68,10 +68,10 @@ func (d Deps) registerContactTools(r *Registry) { r.Register(Tool{ Name: "remove_tag", - Description: "Remove a category (tag) from a contact.", + Description: "Remove a label (category) from a contact.", InputSchema: objectSchema(map[string]any{ "contact_id": strProp("The contact's UUID."), - "category_id": strProp("The category (tag) UUID to remove."), + "category_id": strProp("The label (category) UUID to remove."), }, "contact_id", "category_id"), Risk: generation.RiskWrite, RequiredOrgPerm: models.PermManageContacts, @@ -114,8 +114,8 @@ func (d Deps) registerContactTools(r *Registry) { Description: "Apply the same change (add/remove tags, set subscription) to many contacts at once.", InputSchema: objectSchema(map[string]any{ "contact_ids": arrProp("Contact UUIDs to edit (required).", strProp("Contact UUID.")), - "add_categories": arrProp("Category (tag) UUIDs to add to each contact.", strProp("Category UUID.")), - "remove_categories": arrProp("Category (tag) UUIDs to remove from each contact.", strProp("Category UUID.")), + "add_categories": arrProp("Label (category) UUIDs to add to each contact.", strProp("Label UUID.")), + "remove_categories": arrProp("Label (category) UUIDs to remove from each contact.", strProp("Label UUID.")), "subscribe": boolProp("Set subscription state on each contact."), }, "contact_ids"), Risk: generation.RiskWrite, diff --git a/internal/app/aitools/tools_inbox.go b/internal/app/aitools/tools_inbox.go index 4b12d94fd..00c3f6e7c 100644 --- a/internal/app/aitools/tools_inbox.go +++ b/internal/app/aitools/tools_inbox.go @@ -30,7 +30,7 @@ func (d Deps) registerInboxActionTools(r *Registry) { r.Register(Tool{ Name: "set_thread_labels", - Description: "Replace a conversation thread's label (category) set. Pass the full desired set; an empty list clears labels.", + Description: "Replace a conversation thread's label set (labels are called categories in the API). Pass the full desired set; an empty list clears labels.", InputSchema: objectSchema(map[string]any{ "thread_id": strProp("The thread id."), "category_ids": arrProp("Category (label) UUIDs to apply.", strProp("Category UUID.")), diff --git a/internal/app/aitools/tools_placement.go b/internal/app/aitools/tools_placement.go index 983fe0adc..559435e85 100644 --- a/internal/app/aitools/tools_placement.go +++ b/internal/app/aitools/tools_placement.go @@ -3,6 +3,8 @@ package aitools import ( "context" "encoding/json" + "errors" + "strconv" "github.com/google/uuid" @@ -18,6 +20,9 @@ type PlacementTests interface { CreateTests(ctx context.Context, in placement.CreateInput) ([]placement.TestView, *errx.Error) ListTests(ctx context.Context, orgID *uuid.UUID, campaignID *uuid.UUID, limit, offset int) ([]placement.TestView, int, *errx.Error) GetTest(ctx context.Context, orgID *uuid.UUID, id uuid.UUID) (*placement.TestDetail, *errx.Error) + CreateBatch(ctx context.Context, in placement.BatchInput) (*placement.BatchView, *errx.Error) + ListBatches(ctx context.Context, orgID uuid.UUID, limit, offset int) ([]placement.BatchView, int, *errx.Error) + GetBatch(ctx context.Context, orgID, id uuid.UUID) (*placement.BatchDetail, *errx.Error) } type placementTools struct { @@ -78,6 +83,195 @@ func RegisterPlacementTools(r *Registry, svc PlacementTests, auditSvc audit.Audi RequiredAPIPerm: models.APIPermSendCampaigns, Handler: p.run, }) + + r.Register(Tool{ + Name: "list_placement_batches", + Description: "List the workspace's placement batches (one placement test run from many sending mailboxes), newest first, with each batch's progress and overall inbox placement.", + InputSchema: objectSchema(map[string]any{ + "limit": intProp("Maximum batches to return (default 10, max 50)."), + }), + Risk: generation.RiskRead, + RequiredOrgPerm: models.PermViewAnalytics, + RequiredAPIPerm: models.APIPermReadAnalytics, + Handler: p.listBatches, + }) + + r.Register(Tool{ + Name: "get_placement_batch", + Description: "Read one placement batch: progress, overall placement, and placement by sending domain, by sending provider and by recipient provider, worst first. " + + "Use it to tell one bad mailbox from a whole domain or provider losing reputation.", + InputSchema: objectSchema(map[string]any{ + "batch_id": strProp("The placement batch's UUID."), + }, "batch_id"), + Risk: generation.RiskRead, + RequiredOrgPerm: models.PermViewAnalytics, + RequiredAPIPerm: models.APIPermReadAnalytics, + Handler: p.getBatch, + }) + + r.Register(Tool{ + Name: "run_placement_batch", + Description: "Start a placement batch: run the same placement test from many of the workspace's mailboxes (a campaign's senders or the whole workspace, optionally sampled). " + + "Every copy is a real send counted against its mailbox's daily limit, and the backend starts a few senders at a time, so a large batch takes hours. Only run it when the user asked for a fleet-wide placement test.", + InputSchema: objectSchema(map[string]any{ + "scope": enumProp("Whose mailboxes to test: a campaign's senders or every mailbox in the workspace.", "campaign", "workspace"), + "campaign_id": strProp("The campaign whose senders to test and, with sequence_id, whose copy to send (UUID)."), + "sequence_id": strProp("The campaign step to test (UUID); requires campaign_id."), + "subject": strProp("Subject of an ad-hoc template, when not testing a campaign step."), + "body_plain": strProp("Plain-text body of an ad-hoc template."), + "sample": enumProp("all (default), random (sample_count mailboxes), percent (sample_percent of them, spread across providers), per_domain or per_provider (sample_count each).", "all", "random", "percent", "per_domain", "per_provider"), + "sample_count": intProp("Mailboxes for a random sample, or per group for per_domain and per_provider."), + "sample_percent": intProp("Share of mailboxes for a percent sample, 1 to 100."), + "panel": enumProp("Which seed inboxes to test on (default instance).", "instance", "workspace", "cloud"), + "tracking": enumProp("Open and click tracking on the copies (default campaign).", "campaign", "on", "off", "compare"), + }, "scope"), + Risk: generation.RiskSend, + RequiredOrgPerm: models.PermSendCampaigns, + RequiredAPIPerm: models.APIPermSendCampaigns, + Handler: p.runBatch, + }) +} + +func batchSummary(v placement.BatchView) map[string]any { + out := map[string]any{ + "batch_id": v.ID.String(), + "status": v.Status, + "subject": v.Subject, + "panel": v.Panel, + "tracking": v.Tracking, + "senders": v.SenderCount, + "progress": v.Progress, + "summary": v.Summary, + "created_at": v.CreatedAt, + } + if v.CampaignID != nil { + out["campaign_id"] = v.CampaignID.String() + } + if v.Error != "" { + out["error"] = v.Error + } + return out +} + +func (p placementTools) listBatches(ctx context.Context, inv Invocation, args json.RawMessage) (string, error) { + in, err := decodeArgs[struct { + Limit int `json:"limit"` + }](args) + if err != nil { + return "", err + } + limit := in.Limit + if limit <= 0 { + limit = 10 + } + batches, total, xerr := p.svc.ListBatches(ctx, inv.OrgID, min(limit, 50), 0) + if xerr != nil { + return "", fromErrx(xerr) + } + out := make([]map[string]any, 0, len(batches)) + for _, b := range batches { + out = append(out, batchSummary(b)) + } + return jsonResult(map[string]any{"batches": out, "total": total}) +} + +func (p placementTools) getBatch(ctx context.Context, inv Invocation, args json.RawMessage) (string, error) { + in, err := decodeArgs[struct { + BatchID string `json:"batch_id"` + }](args) + if err != nil { + return "", err + } + id, err := parseUUIDArg(in.BatchID) + if err != nil { + return "", err + } + d, xerr := p.svc.GetBatch(ctx, inv.OrgID, id) + if xerr != nil { + return "", fromErrx(xerr) + } + out := batchSummary(d.BatchView) + if d.Untracked != nil { + out["untracked_summary"] = d.Untracked + } + // The worst groups carry the answer; the rest are on the batch page. + out["by_sending_domain"] = d.Domains[:min(len(d.Domains), 25)] + out["by_sending_provider"] = d.Providers + out["by_recipient_provider"] = d.Recipients + out["content_score"] = d.Content.Score + return jsonResult(out) +} + +func (p placementTools) runBatch(ctx context.Context, inv Invocation, args json.RawMessage) (string, error) { + // An API key may be limited to some mailboxes, which this path cannot + // see; the REST endpoint applies that limit. + if inv.IsAPIKey { + return "", errors.New("start a placement batch with POST /placement/batches, which applies this key's mailbox limits") + } + in, err := decodeArgs[struct { + Scope string `json:"scope"` + CampaignID string `json:"campaign_id"` + SequenceID string `json:"sequence_id"` + Subject string `json:"subject"` + BodyPlain string `json:"body_plain"` + Sample string `json:"sample"` + SampleCount int `json:"sample_count"` + SamplePercent int `json:"sample_percent"` + Panel string `json:"panel"` + Tracking string `json:"tracking"` + }](args) + if err != nil { + return "", err + } + optional := func(s string) (*uuid.UUID, error) { + if s == "" { + return nil, nil + } + id, err := parseUUIDArg(s) + return &id, err + } + campaignID, err := optional(in.CampaignID) + if err != nil { + return "", err + } + sequenceID, err := optional(in.SequenceID) + if err != nil { + return "", err + } + scope := &models.PlacementSenderScope{Type: in.Scope, CampaignID: campaignID} + sample := models.PlacementSample{Mode: in.Sample, Count: in.SampleCount, Percent: in.SamplePercent} + if sample.Mode == models.PlacementSamplePercent { + sample.Stratify = "provider" + } + var userID *uuid.UUID + if inv.UserID != uuid.Nil { + u := inv.UserID + userID = &u + } + batch, xerr := p.svc.CreateBatch(ctx, placement.BatchInput{ + OrgID: inv.OrgID, + UserID: userID, + Scope: scope, + Sample: sample, + CampaignID: campaignID, + SequenceID: sequenceID, + Subject: in.Subject, + BodyPlain: in.BodyPlain, + Panel: in.Panel, + Tracking: in.Tracking, + }) + if xerr != nil { + return "", fromErrx(xerr) + } + id := batch.ID + if p.audit != nil { + p.audit.LogAction(ctx, inv.OrgID, inv.UserID, models.AuditActionCreate, models.AuditEntityPlacementBatch, &id, + inv.IP, inv.UserAgent, nil, map[string]string{"senders": strconv.Itoa(batch.SenderCount), "panel": batch.Panel}) + } + return jsonResult(map[string]any{ + "batch": batchSummary(*batch), + "note": "Senders start a few at a time; read progress with get_placement_batch.", + }) } func placementSummary(v placement.TestView) map[string]any { diff --git a/internal/app/campaign/lead_cc.go b/internal/app/campaign/lead_cc.go new file mode 100644 index 000000000..fabc74e17 --- /dev/null +++ b/internal/app/campaign/lead_cc.go @@ -0,0 +1,111 @@ +package campaign + +import ( + "context" + "errors" + "fmt" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +// leadCCSuggestionLimit bounds the colleagues offered when picking copies. +const leadCCSuggestionLimit = 8 + +func (s *campaignService) ListLeadCC(ctx context.Context, orgID, campaignID, contactID uuid.UUID) ([]models.CampaignLeadCC, *errx.Error) { + if s.campaignProgressRepo == nil { + return nil, errx.InternalError() + } + if xerr := s.ownedCampaign(ctx, orgID, campaignID); xerr != nil { + return nil, xerr + } + if _, err := s.campaignProgressRepo.GetLeadHold(ctx, campaignID, contactID); err != nil { + if errors.Is(err, repository.ErrLeadNotInCampaign) { + return nil, errx.New(errx.NotFound, "contact is not a lead of this campaign") + } + return nil, errx.InternalError() + } + cc, err := s.campaignProgressRepo.ListLeadCC(ctx, campaignID, contactID) + if err != nil { + return nil, errx.InternalError() + } + return cc, nil +} + +func (s *campaignService) SetLeadCC(ctx context.Context, orgID, campaignID, contactID uuid.UUID, contactIDs []string) ([]models.CampaignLeadCC, *errx.Error) { + if s.campaignProgressRepo == nil { + return nil, errx.InternalError() + } + if xerr := s.ownedCampaign(ctx, orgID, campaignID); xerr != nil { + return nil, xerr + } + ids := make([]uuid.UUID, 0, len(contactIDs)) + seen := map[uuid.UUID]bool{} + for _, raw := range contactIDs { + id, err := uuid.Parse(raw) + if err != nil { + return nil, errx.New(errx.BadRequest, "contact_ids must be contact ids") + } + if !seen[id] { + seen[id] = true + ids = append(ids, id) + } + } + if len(ids) > config.CampaignLeadMaxCC { + return nil, errx.NewWithIdentifier(errx.BadRequest, "lead_cc_limit", + fmt.Sprintf("A lead can have at most %d contacts copied on their emails", config.CampaignLeadMaxCC)) + } + + before, err := s.campaignProgressRepo.ListLeadCC(ctx, campaignID, contactID) + if err != nil { + return nil, errx.InternalError() + } + switch err := s.campaignProgressRepo.SetLeadCC(ctx, orgID, campaignID, contactID, ids); { + case err == nil: + case errors.Is(err, repository.ErrLeadNotInCampaign): + return nil, errx.New(errx.NotFound, "contact is not a lead of this campaign") + case errors.Is(err, repository.ErrLeadCCSelf): + return nil, errx.NewWithIdentifier(errx.BadRequest, "lead_cc_self", "A lead cannot be copied on their own emails") + case errors.Is(err, repository.ErrLeadCCContactNotFound): + return nil, errx.NewWithIdentifier(errx.NotFound, "lead_cc_contact_not_found", "A contact to copy was not found in this workspace") + case errors.Is(err, repository.ErrLeadCCLeadIsCopied): + return nil, errx.NewWithIdentifier(errx.Conflict, "lead_cc_lead_is_copied", + "This lead is copied on another lead's emails in this campaign, so it sends none of its own to copy anyone on") + case errors.Is(err, repository.ErrLeadCCHasCopies): + return nil, errx.NewWithIdentifier(errx.Conflict, "lead_cc_has_copies", + "A contact to copy has contacts copied on their own emails in this campaign; remove those first") + default: + return nil, errx.InternalError() + } + + // A removed copy's own lead is released and may be due now. + for _, b := range before { + if !seen[b.ContactID] { + s.WakeCampaigns(ctx, orgID, []string{campaignID.String()}) + break + } + } + + cc, err := s.campaignProgressRepo.ListLeadCC(ctx, campaignID, contactID) + if err != nil { + return nil, errx.InternalError() + } + return cc, nil +} + +func (s *campaignService) SuggestLeadCC(ctx context.Context, orgID, campaignID, contactID uuid.UUID) ([]models.CampaignLeadCCSuggestion, *errx.Error) { + if s.campaignProgressRepo == nil { + return nil, errx.InternalError() + } + if xerr := s.ownedCampaign(ctx, orgID, campaignID); xerr != nil { + return nil, xerr + } + out, err := s.campaignProgressRepo.SuggestLeadCC(ctx, orgID, campaignID, contactID, leadCCSuggestionLimit) + if err != nil { + return nil, errx.InternalError() + } + return out, nil +} diff --git a/internal/app/campaign/service.go b/internal/app/campaign/service.go index 72722845a..2b6b6cdbe 100644 --- a/internal/app/campaign/service.go +++ b/internal/app/campaign/service.go @@ -89,6 +89,14 @@ type CampaignService interface { ResumeLead(ctx context.Context, orgID, campaignID, contactID uuid.UUID) *errx.Error // GetLeadHold reads the live hold on one lead (nil when it is not held). GetLeadHold(ctx context.Context, orgID, campaignID, contactID uuid.UUID) (*models.LeadHold, *errx.Error) + + // ListLeadCC reads the contacts copied on every email to one lead. + ListLeadCC(ctx context.Context, orgID, campaignID, contactID uuid.UUID) ([]models.CampaignLeadCC, *errx.Error) + // SetLeadCC replaces the contacts copied on one lead and returns the new + // list. A copied contact's own lead in the campaign is held meanwhile. + SetLeadCC(ctx context.Context, orgID, campaignID, contactID uuid.UUID, contactIDs []string) ([]models.CampaignLeadCC, *errx.Error) + // SuggestLeadCC offers the lead's likely colleagues to copy. + SuggestLeadCC(ctx context.Context, orgID, campaignID, contactID uuid.UUID) ([]models.CampaignLeadCCSuggestion, *errx.Error) } // Bounds on a manual lead hold. A hold in the past would lift the moment it diff --git a/internal/app/consumer/event_flags_update.go b/internal/app/consumer/event_flags_update.go index c010621ed..4322a3150 100644 --- a/internal/app/consumer/event_flags_update.go +++ b/internal/app/consumer/event_flags_update.go @@ -8,6 +8,7 @@ import ( "time" "github.com/google/uuid" + "github.com/rs/zerolog/log" "github.com/warmbly/warmbly/internal/config" "github.com/warmbly/warmbly/internal/models" @@ -16,19 +17,22 @@ import ( func (s *JobsService) HandleFlagsAdd(ctx context.Context, e *models.JobEventFlags) error { // Tampering check first: verified warmup mail is NOT in the unibox, so we - // detect it via the warmup_received record. If the recipient marked a - // warmup email as spam, that harms the pool — penalise the sender for the - // spam signal AND ban the harmer (they can appeal). Warmup mail isn't - // tracked in the unibox, so there's nothing else to do for it. + // detect it via the warmup_received record. A spam label names no actor on + // any provider: on mail that arrived in spam it is the filter's own, and + // any other move is held and attributed on the evidence around it + // (attributeSpamMove) before anyone is charged. if s.WarmupRepo != nil { if rec, _ := s.WarmupRepo.GetWarmupReceived(ctx, e.EmailID, e.ID); rec != nil { switch { case s.WarmupService == nil: + case containsSpamFlag(e.Flags) && (rec.LandedSpam || rec.MessageID == ""): case containsSpamFlag(e.Flags): - hSender, _ := s.WarmupService.ApplySpamReport(ctx, e.EmailID, rec.SenderAccountID, rec.MessageID, "user_complaint") - s.markRiskBandFromWarmupHealth(ctx, rec.SenderAccountID, hSender) - hHarmer, _ := s.WarmupService.RecordTampering(ctx, e.EmailID, rec.MessageID, "spam_flag") - s.markRiskBandFromWarmupHealth(ctx, e.EmailID, hHarmer) + if _, err := s.WarmupRepo.RecordWarmupSpamMove(ctx, repository.WarmupSpamMove{ + EmailAccountID: e.EmailID, MessageID: rec.MessageID, + SenderAccountID: rec.SenderAccountID, ReceivedAt: rec.CreatedAt, + }); err != nil { + return fmt.Errorf("hold warmup spam move: %w", err) + } case containsTrashFlag(e.Flags) && warmupDeletionCounts(rec, time.Now()): // Gmail reports Delete as gaining the TRASH label and only // reports the message gone when Trash is emptied, weeks later. @@ -98,6 +102,9 @@ func (s *JobsService) HandleFlagsAdd(ctx context.Context, e *models.JobEventFlag if !slices.Contains(email.Flags, e.Flags[i]) { email.Flags = append(email.Flags, e.Flags[i]) updated = true + if e.Flags[i] == models.FlagFlagged { + s.noteOwnerActivity(ctx, e.EmailID, email.InternalDate) + } } } @@ -111,6 +118,7 @@ func (s *JobsService) HandleFlagsAdd(ctx context.Context, e *models.JobEventFlag update.Seen = &seen email.Seen = true updated = true + s.noteOwnerActivity(ctx, e.EmailID, email.InternalDate) } if !updated { @@ -175,6 +183,9 @@ func (s *JobsService) HandleFlagsRemove(ctx context.Context, e *models.JobEventF // Losing \Seen is the provider reporting the message back to unread, and // that is the column the inbox reads, not the flag array. unread := models.SeenFromFlags(e.Flags) && email.Seen + if unread || (slices.Contains(e.Flags, models.FlagFlagged) && slices.Contains(email.Flags, models.FlagFlagged)) { + s.noteOwnerActivity(ctx, e.EmailID, email.InternalDate) + } if len(email.Flags) == 0 && !unread { return nil @@ -228,6 +239,22 @@ func (s *JobsService) HandleFlagsRemove(ctx context.Context, e *models.JobEventF return nil } +// ownerActivityArrivalGrace is how long after arrival a filter may still be labelling a message. +const ownerActivityArrivalGrace = 2 * time.Minute + +// noteOwnerActivity records the owner acting on their own mail at the +// provider. Callers pass only changes our store did not already hold, so a +// change made in Warmbly and echoed back by the sync never counts, and a +// change to mail that only just arrived may be a filter finishing delivery. +func (s *JobsService) noteOwnerActivity(ctx context.Context, accountID uuid.UUID, arrived time.Time) { + if s.WarmupRepo == nil || arrived.IsZero() || time.Since(arrived) < ownerActivityArrivalGrace { + return + } + if err := s.WarmupRepo.RecordOwnerActivity(ctx, accountID, time.Now()); err != nil { + log.Warn().Err(err).Str("email_id", accountID.String()).Msg("owner activity not recorded") + } +} + // containsTrashFlag reports the transition Gmail emits for Delete: the TRASH // label, passed through untranslated by the worker. func containsTrashFlag(flags []string) bool { diff --git a/internal/app/consumer/event_new_email.go b/internal/app/consumer/event_new_email.go index df46505b7..fb02a9db6 100644 --- a/internal/app/consumer/event_new_email.go +++ b/internal/app/consumer/event_new_email.go @@ -45,6 +45,7 @@ func (s *JobsService) ingestNewEmail(ctx context.Context, e *models.JobEventNewE log.Warn().Msg("NEW_EMAIL event without a message body, dropping") return nil } + e.Message.ValidText() warmupToken := warmupTokenFromMessage(e.Message) if warmupToken != "" { handled, err := s.handleWarmupEmail(ctx, e, warmupToken) @@ -446,18 +447,19 @@ func firstSenderAddress(from []string) string { func (s *JobsService) acceptWarmupEmail(ctx context.Context, e *models.JobEventNewEmail, token *models.WarmupToken) { s.WarmupRepo.ConsumeWarmupToken(ctx, token.Token) + landed := models.ClassifyWarmupLanding(e.Message.Folder, e.Message.Flags) + // Record the receipt so a later deletion or spam-flag of THIS message can be // attributed back to warmup and to the sender. Verified warmup mail is not // stored in the unibox, so this is the only record that the message was a // warmup email. if e.Message != nil { - if err := s.WarmupRepo.RecordWarmupReceived(ctx, e.Message.EmailID, e.Message.ID, e.Message.MessageID, token.SenderAccountID); err != nil { + if err := s.WarmupRepo.RecordWarmupReceived(ctx, e.Message.EmailID, e.Message.ID, e.Message.MessageID, token.SenderAccountID, landed == models.WarmupLandedSpam); err != nil { log.Warn().Err(err).Str("email_id", e.Message.EmailID.String()).Msg("Failed to record warmup receipt") } } recipient := s.recipientAccount(ctx, e.Message.EmailID) - landed := models.ClassifyWarmupLanding(e.Message.Folder, e.Message.Flags) // If the warmup mail arrived in a Junk/Spam state, record a // spam_placement event against the sender. This is distinct from a diff --git a/internal/app/consumer/event_send_result.go b/internal/app/consumer/event_send_result.go index 0d6eb3ae2..fa137dc59 100644 --- a/internal/app/consumer/event_send_result.go +++ b/internal/app/consumer/event_send_result.go @@ -15,6 +15,7 @@ import ( "github.com/warmbly/warmbly/internal/errx" "github.com/warmbly/warmbly/internal/infrastructure/pubsub" "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/pkg/mailhdr" "github.com/warmbly/warmbly/internal/repository" ) @@ -167,7 +168,7 @@ func (s *JobsService) HandleEmailFailed(ctx context.Context, result models.SendE switch task.TaskType { case "campaign": - return s.failCampaignSend(ctx, task, reason, code, nil) + return s.failCampaignSend(ctx, task, reason, code, refusedRecipient(result), nil) case "email": s.notifyUserSendFailed(ctx, task, reason) case "placement": @@ -194,7 +195,7 @@ func (s *JobsService) failWarmupSend(ctx context.Context, task *repository.Task, // day the send was counted against, for giving the daily counters back; nil // reads it off the task, which is right for a worker result but not for the // reclaimer, whose sends can be counted on an earlier day. -func (s *JobsService) failCampaignSend(ctx context.Context, task *repository.Task, reason, code string, countedOn *time.Time) error { +func (s *JobsService) failCampaignSend(ctx context.Context, task *repository.Task, reason, code, refused string, countedOn *time.Time) error { ct, err := s.TaskRepo.GetCampaignTask(ctx, task.ID) if err != nil { return err @@ -216,6 +217,11 @@ func (s *JobsService) failCampaignSend(ctx context.Context, task *repository.Tas } } + // A copy the server refused is not the lead's bounce: it is dropped from + // later emails and the lead's step is retried without it. + copyRefused := code == string(errx.MailErrorCodeRecipientRejected) && refused != "" && + recipient != "" && !strings.EqualFold(mailhdr.Bare(refused), recipient) + // A refusal on the SENDING DOMAIN's authentication is not about this lead: // the recipient received nothing, and every retry from that domain fails // identically until its DNS is fixed. Give the reservation back without @@ -234,14 +240,22 @@ func (s *JobsService) failCampaignSend(ctx context.Context, task *repository.Tas } // Only permanent recipient refusals are bounce evidence; deferrals may use the same wording. - if s.Evidence != nil && ct.ContactID != nil && ct.SequenceID != nil && + if s.Evidence != nil && ct.ContactID != nil && ct.SequenceID != nil && !copyRefused && code != string(errx.MailErrorCodeServerUnreachable) && emailverify.NamesRecipient(reason) { s.Evidence.RecordEvidence(ctx, *ct.ContactID, models.Step(&campaignID, ct.SequenceID), "bounced_recipient", "send:"+ct.SequenceID.String(), reason) } + // A refused copy that the retry will leave off costs the lead no attempt; + // one that could not be recorded is counted, so it cannot loop forever. + copyExcluded := copyRefused && s.recordRefusedCopy(ctx, task, ct, campaign, refused, reason) + attempts, exhausted, rolledBack := 0, false, false if ct.ContactID != nil && ct.SequenceID != nil && s.CampaignProgressRepo != nil { - attempts, exhausted, rolledBack, err = s.CampaignProgressRepo.RecordSendFailure(ctx, campaignID, *ct.ContactID, *ct.SequenceID, reason) + if copyExcluded { + attempts, exhausted, rolledBack, err = s.CampaignProgressRepo.WalkBackSend(ctx, campaignID, *ct.ContactID, *ct.SequenceID, reason, false) + } else { + attempts, exhausted, rolledBack, err = s.CampaignProgressRepo.RecordSendFailure(ctx, campaignID, *ct.ContactID, *ct.SequenceID, reason) + } if err != nil { return err } @@ -271,7 +285,7 @@ func (s *JobsService) failCampaignSend(ctx context.Context, task *repository.Tas // reputation, so it goes through the bounce pipeline (progress, optional // suppression, guardrails, warmup health, webhooks) and the lead is // dropped as bounced instead of being offered again. - if rolledBack && code == string(errx.MailErrorCodeRecipientRejected) { + if rolledBack && !copyRefused && code == string(errx.MailErrorCodeRecipientRejected) { if s.recordSynchronousBounce(ctx, task, ct, campaign, recipient, reason) { s.logCampaignSendFailure(ctx, campaignID, ct, recipient, reason, code, attempts, false, false, false) s.publishCampaignUpdated(ctx, campaign, campaignID, "") @@ -335,6 +349,52 @@ func (s *JobsService) recordSynchronousBounce(ctx context.Context, task *reposit return true } +// recordRefusedCopy feeds a copied address the server refused at RCPT into +// the bounce pipeline under its own name, and reports whether the next send +// is sure to leave it off: a lead's copy through its bounced mark, a +// campaign-wide one through the recorded bounce the send path reads. +func (s *JobsService) recordRefusedCopy(ctx context.Context, task *repository.Task, ct *repository.CampaignTask, campaign *models.Campaign, refused, reason string) bool { + if campaign == nil || campaign.OrganizationID == nil || ct.CampaignID == nil || ct.ContactID == nil { + return false + } + address := strings.ToLower(mailhdr.Bare(refused)) + var owner *uuid.UUID + if s.CampaignProgressRepo != nil { + id, err := s.CampaignProgressRepo.MarkLeadCCBounced(ctx, *ct.CampaignID, *ct.ContactID, address) + if err != nil { + log.Warn().Err(err).Str("task_id", task.ID.String()).Msg("could not mark a refused copy bounced") + } + owner = id + } + if s.AdvancedService == nil { + return owner != nil + } + taskID := task.ID + req := &models.IngestDeliverabilityEventRequest{ + EventType: models.DeliverabilityEventBounce, + Provider: "smtp_reject", + TaskID: &taskID, + CampaignID: ct.CampaignID, + ContactID: owner, + RecipientEmail: address, + Reason: reason, + IdempotencyKey: "reject:" + taskID.String() + ":" + address, + } + if xerr := s.AdvancedService.IngestDeliverabilityEvent(ctx, *campaign.OrganizationID, req); xerr != nil { + log.Warn().Str("task_id", taskID.String()).Str("error", xerr.Message).Msg("could not record a refused copy as a bounce") + return owner != nil + } + return true +} + +// refusedRecipient is the address the server refused, when the worker knew. +func refusedRecipient(result models.SendEmailResult) string { + if result.Error == nil { + return "" + } + return result.Error.Recipient +} + // publishCampaignUpdated pulses the campaign for every teammate (status "" keeps // the dashboard's status as is). func (s *JobsService) publishCampaignUpdated(ctx context.Context, campaign *models.Campaign, campaignID uuid.UUID, status string) { diff --git a/internal/app/consumer/event_update_email.go b/internal/app/consumer/event_update_email.go index 3e3d5bab6..66a364001 100644 --- a/internal/app/consumer/event_update_email.go +++ b/internal/app/consumer/event_update_email.go @@ -45,6 +45,7 @@ func (s *JobsService) HandleUpdateEmail(ctx context.Context, e *models.JobEventE // provider: mail read in the customer's own client is read here too. if seen := models.SeenFromFlags(e.Flags); seen != email.Seen { updateData.Seen = &seen + s.noteOwnerActivity(ctx, e.EmailID, email.InternalDate) } if email.UID != e.UID { updateData.UID = &e.UID @@ -87,6 +88,43 @@ func (s *JobsService) HandleUpdateEmail(ctx context.Context, e *models.JobEventE return nil } +// HandleFolderUpdate files a message where the provider moved it. Like a full +// rescan, it only moves the stored folder when the provider's own placement +// changed, so a message filed in Warmbly stays filed. +func (s *JobsService) HandleFolderUpdate(ctx context.Context, e *models.JobEventFolderUpdate) error { + if !models.ValidFolder(e.Folder) { + return nil + } + email, err := s.emailForSyncUpdate(ctx, e.UserID, e.ID, func(message *models.EmailMessageStoreData) { + // A pending row is not visible yet, so nothing has filed it locally. + message.Folder = e.Folder + }) + if err != nil { + CaptureError(e.UserID, e.EmailID, fmt.Errorf("Email (%s): %w", e.ID.String(), err)) + return err + } + if email == nil { + return nil + } + + folder, provider, providerMoved := models.ResolveFolderSync(email.Folder, email.ProviderFolder, e.Folder) + if !providerMoved { + return nil + } + update := repository.UpdateUniboxEntry{ProviderFolder: &provider} + if folder != email.Folder { + update.Folder = &folder + } + if err := s.UniboxRepository.UpdateEntry(ctx, e.UserID, e.EmailID, e.ID, &update); err != nil { + return err + } + + email.Folder = folder + email.ProviderFolder = provider + s.publishEmailUpdated(ctx, e.UserID, email) + return nil +} + // emailForSyncUpdate rechecks visible mail if verification won the pending-row lock. func (s *JobsService) emailForSyncUpdate(ctx context.Context, userID, id uuid.UUID, updatePending func(*models.EmailMessageStoreData)) (*models.EmailMessageStoreData, error) { message, err := s.UniboxRepository.GetByID(ctx, userID, id) diff --git a/internal/app/consumer/events.go b/internal/app/consumer/events.go index 833cfb11f..38fb23e51 100644 --- a/internal/app/consumer/events.go +++ b/internal/app/consumer/events.go @@ -33,6 +33,7 @@ func (w *JobsService) InitEvents() { Register(w, models.JobEventTypeRemoveEmail, w.HandleRemoveEmail) Register(w, models.JobEventTypeFlagsAdd, w.HandleFlagsAdd) Register(w, models.JobEventTypeFlagsRemove, w.HandleFlagsRemove) + Register(w, models.JobEventTypeFolderUpdate, w.HandleFolderUpdate) Register(w, models.JobEventTypeMailboxUpdate, w.HandleMailboxUpdate) Register(w, models.JobEventTypeMailboxDelete, w.HandleMailboxDelete) Register(w, models.JobEventTypeMailboxRename, w.HandleMailboxRename) diff --git a/internal/app/consumer/refused_copy_test.go b/internal/app/consumer/refused_copy_test.go new file mode 100644 index 000000000..c4f8f085b --- /dev/null +++ b/internal/app/consumer/refused_copy_test.go @@ -0,0 +1,138 @@ +package jobs + +import ( + "context" + "testing" + "time" + + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/app/advanced" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +type copyContactRepo struct { + repository.ContactRepository + email string +} + +func (r copyContactRepo) GetByID(_ context.Context, id uuid.UUID) (*models.Contact, *errx.Error) { + return &models.Contact{ID: id, Email: r.email}, nil +} + +type copyCampaignRepo struct { + repository.CampaignRepository + org uuid.UUID +} + +func (r copyCampaignRepo) GetByID(_ context.Context, id uuid.UUID) (*models.Campaign, error) { + return &models.Campaign{ID: id, OrganizationID: &r.org, Status: "active"}, nil +} + +func (copyCampaignRepo) DecrementCampaignDailySend(context.Context, uuid.UUID, time.Time, bool) error { + return nil +} + +type copyProgressRepo struct { + repository.CampaignProgressRepository + copyID uuid.UUID + bounced []string + counted []bool +} + +func (r *copyProgressRepo) RecordSendFailure(context.Context, uuid.UUID, uuid.UUID, uuid.UUID, string) (int, bool, bool, error) { + r.counted = append(r.counted, true) + return 1, false, true, nil +} + +func (r *copyProgressRepo) WalkBackSend(_ context.Context, _, _, _ uuid.UUID, _ string, count bool) (int, bool, bool, error) { + r.counted = append(r.counted, count) + return 0, false, true, nil +} + +func (*copyProgressRepo) HasSentSteps(context.Context, uuid.UUID, uuid.UUID) (bool, error) { + return false, nil +} + +func (r *copyProgressRepo) MarkLeadCCBounced(_ context.Context, _, _ uuid.UUID, address string) (*uuid.UUID, error) { + r.bounced = append(r.bounced, address) + return &r.copyID, nil +} + +type copyAdvanced struct { + advanced.Service + events []*models.IngestDeliverabilityEventRequest +} + +func (a *copyAdvanced) IngestDeliverabilityEvent(_ context.Context, _ uuid.UUID, req *models.IngestDeliverabilityEventRequest) *errx.Error { + a.events = append(a.events, req) + return nil +} + +// A copy the server refused at RCPT bounces the copy, never the lead: no +// address evidence against the lead, and the bounce event names the copy. +func TestRefusedCopyIsNotTheLeadsBounce(t *testing.T) { + for _, tc := range []struct { + name string + refused string + wantCopy bool + }{ + {"a copy", "Jonas ", true}, + {"the lead", "ana@acme.test", false}, + } { + campaign, lead, step, taskID := uuid.New(), uuid.New(), uuid.New(), uuid.New() + progress := ©ProgressRepo{copyID: uuid.New()} + adv := ©Advanced{} + ev := &recordingEvidence{} + s := &JobsService{ + TaskRepo: &evidenceTaskRepo{ + task: &repository.Task{ID: taskID, TaskType: "campaign", EmailAccountID: uuid.New(), Status: "completed"}, + ct: &repository.CampaignTask{TaskID: taskID, CampaignID: &campaign, ContactID: &lead, SequenceID: &step}, + }, + CampaignRepo: copyCampaignRepo{org: uuid.New()}, + CampaignProgressRepo: progress, + ContactRepo: copyContactRepo{email: "ana@acme.test"}, + AdvancedService: adv, + Evidence: ev, + } + err := s.HandleEmailFailed(context.Background(), models.SendEmailResult{ + TaskID: taskID, + Error: &models.EmailSendError{ + Code: string(errx.MailErrorCodeRecipientRejected), + Message: `The mail server rejected the recipient: 550 "5.1.1 no such user"`, + Recipient: tc.refused, + }, + }) + if err != nil { + t.Fatalf("%s: %v", tc.name, err) + } + if len(adv.events) != 1 { + t.Fatalf("%s: %d bounce events, want 1", tc.name, len(adv.events)) + } + got := adv.events[0] + if tc.wantCopy { + if len(ev.kinds) != 0 { + t.Fatalf("%s: evidence %v recorded against the lead", tc.name, ev.kinds) + } + if got.RecipientEmail != "jonas@acme.test" || got.ContactID == nil || *got.ContactID != progress.copyID { + t.Fatalf("%s: bounce event %+v, want it on the copy", tc.name, got) + } + if len(progress.bounced) != 1 { + t.Fatalf("%s: copy marked bounced %v times, want once", tc.name, progress.bounced) + } + // The retry leaves the copy off, so the lead is not charged an attempt. + if len(progress.counted) != 1 || progress.counted[0] { + t.Fatalf("%s: walk-backs %v, want one that counts no attempt", tc.name, progress.counted) + } + continue + } + if len(progress.counted) != 1 || !progress.counted[0] { + t.Fatalf("%s: walk-backs %v, want the lead's attempt counted", tc.name, progress.counted) + } + if got.RecipientEmail != "ana@acme.test" || got.ContactID == nil || *got.ContactID != lead || len(progress.bounced) != 0 { + t.Fatalf("%s: bounce event %+v (copies marked %v), want the lead's own bounce", tc.name, got, progress.bounced) + } + } +} diff --git a/internal/app/consumer/stuck_send_reclaimer.go b/internal/app/consumer/stuck_send_reclaimer.go index d2c66cfbd..084525709 100644 --- a/internal/app/consumer/stuck_send_reclaimer.go +++ b/internal/app/consumer/stuck_send_reclaimer.go @@ -109,7 +109,7 @@ func (s *JobsService) reclaimStuckSend(ctx context.Context, d repository.StuckDi return "", err } dispatchedAt := d.DispatchedAt - if err := s.failCampaignSend(ctx, task, reason, "SEND_OUTCOME_LOST", &dispatchedAt); err != nil { + if err := s.failCampaignSend(ctx, task, reason, "SEND_OUTCOME_LOST", "", &dispatchedAt); err != nil { return "", err } return "reclaimed", nil diff --git a/internal/app/consumer/unibox_folder_sync_test.go b/internal/app/consumer/unibox_folder_sync_test.go new file mode 100644 index 000000000..bd77abb0f --- /dev/null +++ b/internal/app/consumer/unibox_folder_sync_test.go @@ -0,0 +1,53 @@ +package jobs + +import ( + "context" + "testing" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/models" +) + +// A folder the provider reports is resolved against where the provider last +// had the message, so Gmail's own moves land and Warmbly's filing survives. +func TestFolderUpdateFollowsOnlyProviderMoves(t *testing.T) { + for _, tc := range []struct { + name string + folder string + providerFolder string + reported string + wantFolder string // "" means nothing written + }{ + {"archived in Gmail", models.FolderInbox, models.FolderInbox, models.FolderArchive, models.FolderArchive}, + {"trashed in Gmail", models.FolderInbox, models.FolderInbox, models.FolderTrash, models.FolderTrash}, + {"moved back to the inbox in Gmail", models.FolderArchive, models.FolderArchive, models.FolderInbox, models.FolderInbox}, + {"filed in Warmbly, still in the Gmail inbox", models.FolderArchive, models.FolderInbox, models.FolderInbox, ""}, + {"unknown folder", models.FolderInbox, models.FolderInbox, "elsewhere", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + s, repo := seenSyncService(&models.EmailMessageStoreData{Folder: tc.folder, ProviderFolder: tc.providerFolder}) + if err := s.HandleFolderUpdate(context.Background(), &models.JobEventFolderUpdate{ + UserID: uuid.New(), EmailID: uuid.New(), ID: uuid.New(), + Folder: tc.reported, + }); err != nil { + t.Fatal(err) + } + if tc.wantFolder == "" { + if len(repo.updates) != 0 { + t.Fatalf("wrote %+v, want nothing", repo.updates) + } + return + } + if len(repo.updates) != 1 { + t.Fatalf("wrote %d updates, want 1", len(repo.updates)) + } + u := repo.updates[0] + if u.ProviderFolder == nil || *u.ProviderFolder != tc.reported { + t.Errorf("provider_folder = %v, want %q", u.ProviderFolder, tc.reported) + } + if u.Folder == nil || *u.Folder != tc.wantFolder { + t.Errorf("folder = %v, want %q", u.Folder, tc.wantFolder) + } + }) + } +} diff --git a/internal/app/consumer/warmup_retention_test.go b/internal/app/consumer/warmup_retention_test.go index 6c7973fe2..0f4ce575b 100644 --- a/internal/app/consumer/warmup_retention_test.go +++ b/internal/app/consumer/warmup_retention_test.go @@ -19,17 +19,24 @@ import ( // handlers make. type retentionWarmupRepo struct { repository.WarmupRepository - rec *repository.WarmupReceived + rec *repository.WarmupReceived + held *[]repository.WarmupSpamMove } func (r retentionWarmupRepo) GetWarmupReceived(context.Context, uuid.UUID, uuid.UUID) (*repository.WarmupReceived, error) { return r.rec, nil } +func (r retentionWarmupRepo) RecordWarmupSpamMove(_ context.Context, m repository.WarmupSpamMove) (bool, error) { + *r.held = append(*r.held, m) + return true, nil +} + // retentionWarmupService records which strikes the handlers asked for. type retentionWarmupService struct { warmupapp.Service strikes []string + held []repository.WarmupSpamMove fail bool } @@ -54,7 +61,7 @@ func (s *retentionWarmupService) ApplySpamReport(context.Context, uuid.UUID, uui func retentionService(rec *repository.WarmupReceived) (*JobsService, *retentionWarmupService) { svc := &retentionWarmupService{} return &JobsService{ - WarmupRepo: retentionWarmupRepo{rec: rec}, + WarmupRepo: retentionWarmupRepo{rec: rec, held: &svc.held}, WarmupService: svc, EmailRepository: warmupInboxEmailRepo{}, }, svc @@ -275,19 +282,24 @@ func TestRecheckTamperingSearchesEachOldStrike(t *testing.T) { } // Gmail reports Delete as gaining the TRASH label. That is the owner's act -// and is judged on the same freshness rule; a spam flag is still the graver -// strike and is never subject to the window. +// and is judged on the same freshness rule. A spam label charges nobody on +// sight: a move after arrival is held for attribution, and the label on mail +// that arrived in spam is the filter's own and is not even held. func TestFlagsAddJudgesGmailTrashOnFreshness(t *testing.T) { + landedSpam := receivedAgo(time.Second) + landedSpam.LandedSpam = true cases := []struct { name string rec *repository.WarmupReceived flags []string want []string + held int }{ - {"trashed an hour after arrival", receivedAgo(time.Hour), []string{"TRASH"}, []string{"deletion"}}, - {"trashed a month after arrival", receivedAgo(30 * 24 * time.Hour), []string{"TRASH"}, nil}, - {"flagged as spam a month after arrival", receivedAgo(30 * 24 * time.Hour), []string{"SPAM"}, []string{"spam_report", "spam_flag"}}, - {"read is not a strike", receivedAgo(time.Hour), []string{models.FlagSeen}, nil}, + {"trashed an hour after arrival", receivedAgo(time.Hour), []string{"TRASH"}, []string{"deletion"}, 0}, + {"trashed a month after arrival", receivedAgo(30 * 24 * time.Hour), []string{"TRASH"}, nil, 0}, + {"moved to spam a month after arrival is held", receivedAgo(30 * 24 * time.Hour), []string{"SPAM"}, nil, 1}, + {"spam label on mail that arrived in spam", landedSpam, []string{"SPAM"}, nil, 0}, + {"read is not a strike", receivedAgo(time.Hour), []string{models.FlagSeen}, nil, 0}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -300,6 +312,9 @@ func TestFlagsAddJudgesGmailTrashOnFreshness(t *testing.T) { if len(svc.strikes) != len(tc.want) { t.Fatalf("strikes = %v, want %v", svc.strikes, tc.want) } + if len(svc.held) != tc.held { + t.Fatalf("held %d moves, want %d", len(svc.held), tc.held) + } for i := range tc.want { if svc.strikes[i] != tc.want[i] { t.Fatalf("strikes = %v, want %v", svc.strikes, tc.want) diff --git a/internal/app/consumer/warmup_spam_attribution.go b/internal/app/consumer/warmup_spam_attribution.go new file mode 100644 index 000000000..61d19cbad --- /dev/null +++ b/internal/app/consumer/warmup_spam_attribution.go @@ -0,0 +1,167 @@ +package jobs + +import ( + "context" + "fmt" + "slices" + "time" + + "github.com/rs/zerolog/log" + + "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/jobrun" + "github.com/warmbly/warmbly/internal/repository" +) + +const ( + spamMoveAttributionInterval = 5 * time.Minute + spamMoveAttributionBatch = 200 + // spamMoveClaimLease outlives one move's work; a consumer that dies holding it frees it on expiry. + spamMoveClaimLease = 5 * time.Minute +) + +// Evidence names stored with each verdict, so an operator can read why. +const ( + spamMoveCorrelated = "correlated" + spamMoveOnArrival = "on_arrival" + spamMoveOwnerActive = "owner_active" + spamMoveRepeated = "repeated" + spamMoveDormant = "dormant" + spamMoveNoOwnerTrace = "no_owner_activity" +) + +// attributeSpamMove decides who moved a received warmup email into spam. No +// provider says, so the verdict rests on what the pool and the mailbox show: +// the provider when other workspaces saw the same sender junked or the move +// came straight after arrival with nobody there, the owner when they were +// active in the mailbox around it or keep junking pool mail nobody else does, +// and nobody otherwise. Only the owner is charged. +func attributeSpamMove(m repository.WarmupSpamMove, ev repository.WarmupSpamMoveEvidence) (string, []string) { + quick := m.ObservedAt.Sub(m.ReceivedAt) < time.Duration(config.WarmupSpamMoveQuickMinutes)*time.Minute + switch { + case ev.CorrelatedElsewhere > 0: + return repository.SpamMoveProvider, []string{spamMoveCorrelated} + case quick && !ev.OwnerActiveNear: + return repository.SpamMoveProvider, []string{spamMoveOnArrival} + case ev.OwnerActiveNear: + return repository.SpamMoveOwner, []string{spamMoveOwnerActive} + case !ev.OwnerActiveRecently: + return repository.SpamMoveUnattributed, []string{spamMoveDormant} + case ev.PatternSenders >= config.WarmupSpamMovePatternSenders: + return repository.SpamMoveOwner, []string{spamMoveRepeated} + } + return repository.SpamMoveUnattributed, []string{spamMoveNoOwnerTrace} +} + +// StartWarmupSpamMoveAttribution decides each held spam move once it has settled. +func (s *JobsService) StartWarmupSpamMoveAttribution(ctx context.Context) { + if s.WarmupRepo == nil || s.WarmupService == nil { + return + } + jobrun.Loop(ctx, "warmup_spam_move_attribution", spamMoveAttributionInterval, true, func(ctx context.Context) error { + batchCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + return s.attributeSpamMoves(batchCtx, time.Now()) + }) +} + +func (s *JobsService) attributeSpamMoves(ctx context.Context, now time.Time) error { + settled := now.Add(-time.Duration(config.WarmupSpamMoveSettleMinutes) * time.Minute) + moves, err := s.WarmupRepo.ListSettledWarmupSpamMoves(ctx, settled, spamMoveAttributionBatch) + if err != nil { + return fmt.Errorf("list warmup spam moves: %w", err) + } + for _, m := range moves { + if err := s.attributeOneSpamMove(ctx, m); err != nil { + return err + } + } + return nil +} + +// attributeOneSpamMove claims the move, fixes its verdict once and applies it. +// Effects are idempotent and the move is completed only after all of them, so +// a failure part way re-applies the same verdict on a later pass. +func (s *JobsService) attributeOneSpamMove(ctx context.Context, m repository.WarmupSpamMove) error { + claimed, err := s.WarmupRepo.ClaimWarmupSpamMove(ctx, m.EmailAccountID, m.MessageID, spamMoveClaimLease) + if err != nil { + return fmt.Errorf("claim warmup spam move: %w", err) + } + if !claimed { + return nil + } + + verdict, signals := m.Verdict, m.Signals + if verdict == repository.SpamMovePending { + ev, err := s.WarmupRepo.WarmupSpamMoveEvidence(ctx, m) + if err != nil { + return fmt.Errorf("warmup spam move evidence: %w", err) + } + verdict, signals = attributeSpamMove(m, ev) + fixed, err := s.WarmupRepo.FixWarmupSpamMoveVerdict(ctx, m.EmailAccountID, m.MessageID, verdict, signals) + if err != nil { + return fmt.Errorf("fix warmup spam move verdict: %w", err) + } + if !fixed { + return nil + } + } + + if verdict == repository.SpamMoveOwner { + hSender, xerr := s.WarmupService.ApplySpamReport(ctx, m.EmailAccountID, m.SenderAccountID, m.MessageID, "user_complaint") + if xerr != nil { + return fmt.Errorf("record warmup spam complaint: %w", xerr) + } + s.markRiskBandFromWarmupHealth(ctx, m.SenderAccountID, hSender) + hOwner, xerr := s.WarmupService.RecordTampering(ctx, m.EmailAccountID, m.MessageID, "spam_flag") + if xerr != nil { + return fmt.Errorf("record warmup spam strike: %w", xerr) + } + s.markRiskBandFromWarmupHealth(ctx, m.EmailAccountID, hOwner) + } else { + // The provider junked the sender's mail, or may have: a placement + // reading against the sender, which only ever slows it down. + provider, domain := recipientProviderDomain(s.recipientAccount(ctx, m.EmailAccountID)) + hSender, xerr := s.WarmupService.RecordSpamPlacement(ctx, m.EmailAccountID, m.SenderAccountID, m.MessageID, "", provider, domain) + if xerr != nil { + return fmt.Errorf("record warmup spam placement: %w", xerr) + } + s.markRiskBandFromWarmupHealth(ctx, m.SenderAccountID, hSender) + } + + if slices.Contains(signals, spamMoveCorrelated) { + if err := s.withdrawCorrelatedOwnerMoves(ctx, m); err != nil { + return err + } + } + + if err := s.WarmupRepo.CompleteWarmupSpamMove(ctx, m.EmailAccountID, m.MessageID); err != nil { + return fmt.Errorf("complete warmup spam move: %w", err) + } + log.Info().Str("email_id", m.EmailAccountID.String()).Str("verdict", verdict).Strs("signals", signals). + Msg("warmup spam move attributed") + return nil +} + +// withdrawCorrelatedOwnerMoves takes back the strikes charged for the same +// sender's mail in other workspaces before this move showed the provider at work. +func (s *JobsService) withdrawCorrelatedOwnerMoves(ctx context.Context, m repository.WarmupSpamMove) error { + owners, err := s.WarmupRepo.CorrelatedOwnerSpamMoves(ctx, m.SenderAccountID, m.EmailAccountID, m.ObservedAt) + if err != nil { + return fmt.Errorf("correlated spam moves: %w", err) + } + for _, o := range owners { + health, xerr := s.WarmupService.WithdrawTampering(ctx, o.EmailAccountID, o.MessageID, "spam_flag") + if xerr != nil { + return fmt.Errorf("withdraw correlated spam strike: %w", xerr) + } + s.markRiskBandFromWarmupHealth(ctx, o.EmailAccountID, health) + } + if len(owners) == 0 { + return nil + } + if err := s.WarmupRepo.ReattributeOwnerSpamMoves(ctx, m.SenderAccountID, m.EmailAccountID, m.ObservedAt); err != nil { + return fmt.Errorf("reattribute correlated spam moves: %w", err) + } + return nil +} diff --git a/internal/app/consumer/warmup_spam_attribution_test.go b/internal/app/consumer/warmup_spam_attribution_test.go new file mode 100644 index 000000000..ab5314e4c --- /dev/null +++ b/internal/app/consumer/warmup_spam_attribution_test.go @@ -0,0 +1,238 @@ +package jobs + +import ( + "context" + "slices" + "testing" + "time" + + "github.com/google/uuid" + + warmupapp "github.com/warmbly/warmbly/internal/app/warmup" + "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +func spamMoveAfter(sinceArrival time.Duration) repository.WarmupSpamMove { + observed := time.Date(2026, 9, 28, 14, 18, 36, 0, time.UTC) + return repository.WarmupSpamMove{ + EmailAccountID: uuid.New(), MessageID: "", SenderAccountID: uuid.New(), + ReceivedAt: observed.Add(-sinceArrival), ObservedAt: observed, Verdict: repository.SpamMovePending, + } +} + +func TestAttributeSpamMove(t *testing.T) { + quick := time.Duration(config.WarmupSpamMoveQuickMinutes)*time.Minute - time.Second + later := 6 * time.Hour + pattern := config.WarmupSpamMovePatternSenders + cases := []struct { + name string + since time.Duration + ev repository.WarmupSpamMoveEvidence + want string + }{ + {"the filter catching up with nobody there", quick, repository.WarmupSpamMoveEvidence{OwnerActiveRecently: true}, repository.SpamMoveProvider}, + {"the owner at the mailbox right after arrival", quick, repository.WarmupSpamMoveEvidence{OwnerActiveNear: true, OwnerActiveRecently: true}, repository.SpamMoveOwner}, + {"the owner at the mailbox later", later, repository.WarmupSpamMoveEvidence{OwnerActiveNear: true, OwnerActiveRecently: true}, repository.SpamMoveOwner}, + {"other workspaces junked the same sender", later, repository.WarmupSpamMoveEvidence{OwnerActiveNear: true, OwnerActiveRecently: true, CorrelatedElsewhere: 1}, repository.SpamMoveProvider}, + {"a mailbox nobody uses", later, repository.WarmupSpamMoveEvidence{PatternSenders: pattern + 5}, repository.SpamMoveUnattributed}, + {"one unexplained move in a used mailbox", later, repository.WarmupSpamMoveEvidence{OwnerActiveRecently: true, PatternSenders: 1}, repository.SpamMoveUnattributed}, + {"a used mailbox junking many senders nobody else does", later, repository.WarmupSpamMoveEvidence{OwnerActiveRecently: true, PatternSenders: pattern}, repository.SpamMoveOwner}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, signals := attributeSpamMove(spamMoveAfter(tc.since), tc.ev) + if got != tc.want { + t.Fatalf("verdict = %s (%v), want %s", got, signals, tc.want) + } + if len(signals) == 0 { + t.Fatal("a verdict carries no evidence for an operator to read") + } + }) + } +} + +// attributionRepo serves settled moves and their evidence, and records what was decided. +type attributionRepo struct { + repository.WarmupRepository + moves []repository.WarmupSpamMove + evidence repository.WarmupSpamMoveEvidence + correlated []repository.WarmupSpamMove + claimLost bool + log *[]string +} + +func (r attributionRepo) ClaimWarmupSpamMove(context.Context, uuid.UUID, string, time.Duration) (bool, error) { + return !r.claimLost, nil +} + +func (r attributionRepo) FixWarmupSpamMoveVerdict(_ context.Context, _ uuid.UUID, _, verdict string, _ []string) (bool, error) { + *r.log = append(*r.log, "fix:"+verdict) + return true, nil +} + +func (r attributionRepo) CompleteWarmupSpamMove(context.Context, uuid.UUID, string) error { + *r.log = append(*r.log, "complete") + return nil +} + +func (r attributionRepo) ListSettledWarmupSpamMoves(context.Context, time.Time, int) ([]repository.WarmupSpamMove, error) { + return r.moves, nil +} + +func (r attributionRepo) WarmupSpamMoveEvidence(context.Context, repository.WarmupSpamMove) (repository.WarmupSpamMoveEvidence, error) { + return r.evidence, nil +} + +func (r attributionRepo) CorrelatedOwnerSpamMoves(context.Context, uuid.UUID, uuid.UUID, time.Time) ([]repository.WarmupSpamMove, error) { + return r.correlated, nil +} + +func (r attributionRepo) ReattributeOwnerSpamMoves(context.Context, uuid.UUID, uuid.UUID, time.Time) error { + *r.log = append(*r.log, "reattribute") + return nil +} + +// attributionService records the effects a verdict has, in order. +type attributionService struct { + warmupapp.Service + senderFails bool + log *[]string +} + +func (s attributionService) ApplySpamReport(_ context.Context, _, _ uuid.UUID, _, reportType string) (*models.WarmupParticipantHealth, *errx.Error) { + if s.senderFails { + return nil, errx.InternalError() + } + *s.log = append(*s.log, "sender:"+reportType) + return nil, nil +} + +func (s attributionService) RecordSpamPlacement(context.Context, uuid.UUID, uuid.UUID, string, string, string, string) (*models.WarmupParticipantHealth, *errx.Error) { + if s.senderFails { + return nil, errx.InternalError() + } + *s.log = append(*s.log, "sender:spam_placement") + return nil, nil +} + +func (s attributionService) RecordTampering(_ context.Context, _ uuid.UUID, _, kind string) (*models.WarmupParticipantHealth, *errx.Error) { + *s.log = append(*s.log, "strike:"+kind) + return nil, nil +} + +func (s attributionService) WithdrawTampering(_ context.Context, _ uuid.UUID, _, kind string) (*models.WarmupParticipantHealth, *errx.Error) { + *s.log = append(*s.log, "withdraw:"+kind) + return nil, nil +} + +// Only an owner verdict charges the recipient; the rest read as placement +// against the sender. The verdict is fixed before any effect and the move is +// completed after all of them, so a failure part way re-applies the same +// verdict, and a correlation takes back the owner verdicts it explains first. +func TestAttributeSpamMovesAppliesTheVerdict(t *testing.T) { + owner := repository.WarmupSpamMoveEvidence{OwnerActiveNear: true, OwnerActiveRecently: true} + fixedProvider := spamMoveAfter(6 * time.Hour) + fixedProvider.Verdict, fixedProvider.Signals = repository.SpamMoveProvider, []string{spamMoveOnArrival} + cases := []struct { + name string + move repository.WarmupSpamMove + ev repository.WarmupSpamMoveEvidence + correlated int + claimLost bool + senderFails bool + wantErr bool + want []string + }{ + {name: "owner", move: spamMoveAfter(6 * time.Hour), ev: owner, + want: []string{"fix:owner", "sender:user_complaint", "strike:spam_flag", "complete"}}, + {name: "provider on arrival", move: spamMoveAfter(time.Minute), + want: []string{"fix:provider", "sender:spam_placement", "complete"}}, + {name: "unattributed", move: spamMoveAfter(6 * time.Hour), ev: repository.WarmupSpamMoveEvidence{OwnerActiveRecently: true, PatternSenders: 1}, + want: []string{"fix:unattributed", "sender:spam_placement", "complete"}}, + {name: "correlated withdraws earlier owner verdicts", move: spamMoveAfter(6 * time.Hour), ev: repository.WarmupSpamMoveEvidence{CorrelatedElsewhere: 2}, correlated: 2, + want: []string{"fix:provider", "sender:spam_placement", "withdraw:spam_flag", "withdraw:spam_flag", "reattribute", "complete"}}, + {name: "a move another consumer holds is left alone", move: spamMoveAfter(6 * time.Hour), ev: owner, claimLost: true, + want: nil}, + {name: "a fixed verdict is re-applied, not decided again", move: fixedProvider, ev: owner, + want: []string{"sender:spam_placement", "complete"}}, + {name: "a failed sender write leaves the move to retry", move: spamMoveAfter(6 * time.Hour), ev: owner, senderFails: true, wantErr: true, + want: []string{"fix:owner"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var log []string + repo := attributionRepo{moves: []repository.WarmupSpamMove{tc.move}, evidence: tc.ev, claimLost: tc.claimLost, log: &log} + for range tc.correlated { + repo.correlated = append(repo.correlated, spamMoveAfter(6*time.Hour)) + } + s := &JobsService{WarmupRepo: repo, WarmupService: attributionService{log: &log, senderFails: tc.senderFails}, EmailRepository: warmupInboxEmailRepo{}} + err := s.attributeSpamMoves(context.Background(), time.Now()) + if (err != nil) != tc.wantErr { + t.Fatalf("err = %v, want error %v", err, tc.wantErr) + } + if !slices.Equal(log, tc.want) { + t.Fatalf("effects = %v, want %v", log, tc.want) + } + }) + } +} + +// activityRepo is a mailbox with no warmup receipt that records owner activity. +type activityRepo struct { + repository.WarmupRepository + noted *int +} + +func (activityRepo) GetWarmupReceived(context.Context, uuid.UUID, uuid.UUID) (*repository.WarmupReceived, error) { + return nil, nil +} + +func (r activityRepo) RecordOwnerActivity(context.Context, uuid.UUID, time.Time) error { + *r.noted++ + return nil +} + +// Owner activity is a change the provider reports that our store did not +// already hold: a read made in Warmbly and echoed back is not the owner at the +// mailbox, and a label on mail that only just arrived may be a filter. +func TestOwnerActivityIsOnlyTheProvidersOwnChange(t *testing.T) { + old := time.Now().Add(-time.Hour) + cases := []struct { + name string + stored models.EmailMessageStoreData + add bool + flags []string + want int + }{ + {"read at the provider", models.EmailMessageStoreData{Flags: []string{}, InternalDate: old}, true, []string{models.FlagSeen}, 1}, + {"a read made in Warmbly echoed back", models.EmailMessageStoreData{Flags: []string{models.FlagSeen}, Seen: true, InternalDate: old}, true, []string{models.FlagSeen}, 0}, + {"read by a filter on arrival", models.EmailMessageStoreData{Flags: []string{}, InternalDate: time.Now()}, true, []string{models.FlagSeen}, 0}, + {"starred at the provider", models.EmailMessageStoreData{Flags: []string{}, Seen: true, InternalDate: old}, true, []string{models.FlagFlagged}, 1}, + {"marked unread at the provider", models.EmailMessageStoreData{Flags: []string{models.FlagSeen}, Seen: true, InternalDate: old}, false, []string{models.FlagSeen}, 1}, + {"a label no person sets", models.EmailMessageStoreData{Flags: []string{}, Seen: true, InternalDate: old}, true, []string{`\Important`}, 0}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + stored := tc.stored + s, _ := seenSyncService(&stored) + noted := 0 + s.WarmupRepo = activityRepo{noted: ¬ed} + ev := &models.JobEventFlags{UserID: uuid.New(), EmailID: uuid.New(), ID: uuid.New(), Flags: tc.flags} + var err error + if tc.add { + err = s.HandleFlagsAdd(context.Background(), ev) + } else { + err = s.HandleFlagsRemove(context.Background(), ev) + } + if err != nil { + t.Fatal(err) + } + if noted != tc.want { + t.Fatalf("noted %d owner activities, want %d", noted, tc.want) + } + }) + } +} diff --git a/internal/app/contact/campaign_state.go b/internal/app/contact/campaign_state.go index 12e273796..54b9b1600 100644 --- a/internal/app/contact/campaign_state.go +++ b/internal/app/contact/campaign_state.go @@ -114,8 +114,15 @@ func holdCopy(hold *models.LeadHold) string { return "Paused for this contact" } what := "Paused for this contact" - if hold.Source == models.LeadHoldSourceOutOfOffice { + switch hold.Source { + case models.LeadHoldSourceOutOfOffice: what = "Out of office" + case models.LeadHoldSourceCC: + // The reason is the lead they are copied on; no end is expected. + if r := strings.TrimSpace(hold.Reason); r != "" { + return "Copied on the emails to " + r + ", so none of their own are sent" + } + return "Copied on another lead's emails, so none of their own are sent" } if r := strings.TrimSpace(hold.Reason); r != "" { what += ": " + r diff --git a/internal/app/contact/export.go b/internal/app/contact/export.go index 9de2b1190..158980a58 100644 --- a/internal/app/contact/export.go +++ b/internal/app/contact/export.go @@ -194,7 +194,7 @@ func fieldHeader(f string) string { case models.ContactExportFieldSubscribed: return "Subscribed" case models.ContactExportFieldCategories: - return "Categories" + return "Labels" case models.ContactExportFieldCampaigns: return "Campaigns" case models.ContactExportFieldCreatedAt: diff --git a/internal/app/eventschemas/eventschemas.go b/internal/app/eventschemas/eventschemas.go new file mode 100644 index 000000000..6077f51b1 --- /dev/null +++ b/internal/app/eventschemas/eventschemas.go @@ -0,0 +1,67 @@ +// Package eventschemas registers a release's bus schemas from the control +// plane, which rolls out first, and holds the fleet off a release the schema +// registry refused, so a refusal never reaches a worker mid-send. +package eventschemas + +import ( + "context" + "fmt" + "strings" + + "github.com/hamba/avro/v2" + "github.com/warmbly/warmbly/internal/events" + "github.com/warmbly/warmbly/internal/infrastructure/codec" + "github.com/warmbly/warmbly/internal/infrastructure/kafka" + "github.com/warmbly/warmbly/internal/models" +) + +// WorkerCommands names every per-node command topic (w.). +const WorkerCommands = "w.*" + +// Published is every schema this build publishes, by topic. +func Published() map[string]avro.Schema { + return map[string]avro.Schema{ + kafka.TopicWorkerEvents: models.JobEvent{}.Schema(), + WorkerCommands: models.WorkerEvent{}.Schema(), + events.TopicEmailEvents: events.EmailSentEvent{}.Schema(), + events.TopicWarmupEvents: events.WarmupEmailSentEvent{}.Schema(), + } +} + +// Register registers every published schema when c resolves against a +// registry. A codec without one has nothing to refuse. +func Register(ctx context.Context, c codec.Codec) error { + r, ok := c.(codec.SchemaRegistrar) + if !ok { + return nil + } + return r.RegisterSchemas(ctx, Published()) +} + +// Gate answers whether the fleet may move to tag. On a registry, the control +// plane has to be running that release and its schemas have to register; +// running is this binary's stamped version, and a dev build is not held. +func Gate(c codec.Codec, running string) func(ctx context.Context, tag string) error { + return func(ctx context.Context, tag string) error { + if _, ok := c.(codec.SchemaRegistrar); !ok { + return nil + } + if release(running) && release(tag) && base(running) != base(tag) { + return fmt.Errorf("the control plane runs %s, so the fleet waits for it to run %s and register its schemas", running, tag) + } + if err := Register(ctx, c); err != nil { + return fmt.Errorf("schema registry refused this release's event schemas: %w", err) + } + return nil + } +} + +// release is a tagged version rather than a dev or unstamped build. +func release(v string) bool { + return len(v) > 1 && v[0] == 'v' && v[1] >= '0' && v[1] <= '9' +} + +// base drops the image variant, so v1.2.3-kafka and v1.2.3 are one release. +func base(v string) string { + return strings.TrimSuffix(v, "-kafka") +} diff --git a/internal/app/eventschemas/eventschemas_test.go b/internal/app/eventschemas/eventschemas_test.go new file mode 100644 index 000000000..e126b52e7 --- /dev/null +++ b/internal/app/eventschemas/eventschemas_test.go @@ -0,0 +1,121 @@ +package eventschemas + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "flag" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/hamba/avro/v2" + "github.com/warmbly/warmbly/internal/infrastructure/codec" + "github.com/warmbly/warmbly/internal/infrastructure/kafka" + "github.com/warmbly/warmbly/internal/models" +) + +var update = flag.Bool("update", false, "record the current schemas as the snapshot (make schemas)") + +// snapshotPath is the recorded schema for topic, as the registry last accepted it. +func snapshotPath(topic string) string { + name := topic + if topic == WorkerCommands { + name = "worker-commands" + } + return filepath.Join("testdata", name+".avsc") +} + +func document(t *testing.T, s avro.Schema) []byte { + t.Helper() + doc, err := models.SchemaDocument(s) + if err != nil { + t.Fatal(err) + } + var out bytes.Buffer + if err := json.Indent(&out, doc, "", " "); err != nil { + t.Fatal(err) + } + return append(out.Bytes(), '\n') +} + +// TestPublishedSchemasStayCompatible is the registry's check, run before merge: +// a reader on the new schema has to decode what the recorded one wrote. +func TestPublishedSchemasStayCompatible(t *testing.T) { + for topic, schema := range Published() { + t.Run(topic, func(t *testing.T) { + path := snapshotPath(topic) + current := document(t, schema) + recorded, err := os.ReadFile(path) + if err == nil { + old, perr := avro.Parse(string(recorded)) + if perr != nil { + t.Fatalf("%s does not parse: %v", path, perr) + } + if cerr := avro.NewSchemaCompatibility().Compatible(schema, old); cerr != nil { + t.Fatalf("the %s schema is not BACKWARD compatible with %s, so the registry would refuse it and every publish on the topic would stop: %v", topic, path, cerr) + } + } else if !os.IsNotExist(err) || !*update { + t.Fatalf("no recorded schema for %s; run make schemas: %v", topic, err) + } + if *update { + if err := os.WriteFile(path, current, 0o644); err != nil { + t.Fatal(err) + } + return + } + if !bytes.Equal(current, recorded) { + t.Fatalf("the %s schema changed compatibly; run make schemas to record it in %s", topic, path) + } + }) + } +} + +type registry struct { + codec.Codec + err error + registered map[string]avro.Schema +} + +func (r *registry) RegisterSchemas(_ context.Context, s map[string]avro.Schema) error { + r.registered = s + return r.err +} + +func TestGate(t *testing.T) { + ctx := context.Background() + for _, tc := range []struct { + name, running, tag string + err error + held bool + }{ + {"the control plane runs the release", "v1.2.3", "v1.2.3", nil, false}, + {"the kafka image is the same release", "v1.2.3-kafka", "v1.2.3", nil, false}, + {"the control plane is behind", "v1.2.3", "v1.2.4", nil, true}, + {"a dev build still registers", "dev-abc", "v1.2.4", nil, false}, + {"the registry refuses", "v1.2.3", "v1.2.3", errors.New("409"), true}, + } { + t.Run(tc.name, func(t *testing.T) { + r := ®istry{err: tc.err} + err := Gate(r, tc.running)(ctx, tc.tag) + if (err != nil) != tc.held { + t.Fatalf("held = %v, want %v", err, tc.held) + } + if !tc.held && len(r.registered) != len(Published()) { + t.Fatalf("registered %d schemas, want %d", len(r.registered), len(Published())) + } + }) + } + + if err := Gate(codec.NewJSON(), "v1.2.3")(ctx, "v9.9.9"); err != nil { + t.Fatalf("a codec with no registry held the fleet: %v", err) + } +} + +func TestWorkerCommandsNamesThePerNodeTopics(t *testing.T) { + if !strings.HasPrefix(kafka.GetWorkerTopic("node"), strings.TrimSuffix(WorkerCommands, "*")) { + t.Fatalf("WorkerCommands %q does not match w.", WorkerCommands) + } +} diff --git a/internal/app/eventschemas/testdata/email-events.avsc b/internal/app/eventschemas/testdata/email-events.avsc new file mode 100644 index 000000000..981e0a395 --- /dev/null +++ b/internal/app/eventschemas/testdata/email-events.avsc @@ -0,0 +1,59 @@ +{ + "fields": [ + { + "default": "", + "name": "event_type", + "type": "string" + }, + { + "default": "", + "name": "task_id", + "type": "string" + }, + { + "default": "", + "name": "account_id", + "type": "string" + }, + { + "default": "", + "name": "campaign_id", + "type": "string" + }, + { + "default": "", + "name": "contact_id", + "type": "string" + }, + { + "default": "", + "name": "sequence_id", + "type": "string" + }, + { + "default": "", + "name": "message_id", + "type": "string" + }, + { + "default": "", + "name": "recipient", + "type": "string" + }, + { + "default": "", + "name": "subject", + "type": "string" + }, + { + "default": 0, + "name": "sent_at", + "type": { + "logicalType": "timestamp-millis", + "type": "long" + } + } + ], + "name": "warmbly.events.EmailSentEvent", + "type": "record" +} diff --git a/internal/app/eventschemas/testdata/jobs.worker-events.avsc b/internal/app/eventschemas/testdata/jobs.worker-events.avsc new file mode 100644 index 000000000..2c9cb36ee --- /dev/null +++ b/internal/app/eventschemas/testdata/jobs.worker-events.avsc @@ -0,0 +1,1087 @@ +{ + "fields": [ + { + "name": "type", + "type": "string" + }, + { + "name": "body", + "type": [ + { + "fields": [ + { + "default": "", + "name": "task_id", + "type": "string" + }, + { + "default": "", + "name": "email_account_id", + "type": "string" + }, + { + "default": "", + "name": "user_id", + "type": "string" + }, + { + "default": "", + "name": "error_code", + "type": "string" + }, + { + "default": "", + "name": "error_type", + "type": "string" + }, + { + "default": "", + "name": "resolve_method", + "type": "string" + }, + { + "default": "", + "name": "message", + "type": "string" + }, + { + "default": false, + "name": "user_visible", + "type": "boolean" + }, + { + "default": "", + "name": "user_title", + "type": "string" + }, + { + "default": "", + "name": "user_message", + "type": "string" + }, + { + "default": "", + "name": "action_required", + "type": "string" + }, + { + "default": 0, + "name": "timestamp", + "type": "long" + } + ], + "name": "warmbly.events.EmailErrorEvent", + "type": "record" + }, + { + "fields": [ + { + "default": "", + "name": "user_id", + "type": "string" + }, + { + "default": "", + "name": "email_id", + "type": "string" + }, + { + "default": "", + "name": "id", + "type": "string" + }, + { + "default": 0, + "name": "uid", + "type": "long" + }, + { + "default": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "name": "mod_seq", + "type": { + "name": "warmbly.events.uint64", + "size": 8, + "type": "fixed" + } + }, + { + "default": 0, + "name": "mailbox", + "type": "long" + }, + { + "default": "", + "name": "folder_path", + "type": "string" + }, + { + "default": "", + "name": "folder", + "type": "string" + }, + { + "default": [], + "name": "flags", + "type": { + "items": "string", + "type": "array" + } + } + ], + "name": "warmbly.events.JobEventEmailUpdate", + "type": "record" + }, + { + "fields": [ + { + "default": "", + "name": "user_id", + "type": "string" + }, + { + "default": "", + "name": "email_id", + "type": "string" + }, + { + "default": "", + "name": "id", + "type": "string" + }, + { + "default": [], + "name": "flags", + "type": { + "items": "string", + "type": "array" + } + } + ], + "name": "warmbly.events.JobEventFlags", + "type": "record" + }, + { + "fields": [ + { + "default": "", + "name": "user_id", + "type": "string" + }, + { + "default": "", + "name": "email_id", + "type": "string" + }, + { + "default": "", + "name": "id", + "type": "string" + }, + { + "default": "", + "name": "folder", + "type": "string" + } + ], + "name": "warmbly.events.JobEventFolderUpdate", + "type": "record" + }, + { + "fields": [ + { + "default": "", + "name": "user_id", + "type": "string" + }, + { + "default": "", + "name": "email_id", + "type": "string" + }, + { + "default": "", + "name": "folder", + "type": "string" + }, + { + "default": "", + "name": "delta_link", + "type": "string" + } + ], + "name": "warmbly.events.JobEventGraphDeltaUpdate", + "type": "record" + }, + { + "fields": [ + { + "default": "", + "name": "user_id", + "type": "string" + }, + { + "default": "", + "name": "email_id", + "type": "string" + }, + { + "default": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "name": "history_id", + "type": "warmbly.events.uint64" + } + ], + "name": "warmbly.events.JobEventHistoryIDUpdate", + "type": "record" + }, + { + "fields": [ + { + "default": "", + "name": "user_id", + "type": "string" + }, + { + "default": "", + "name": "email_id", + "type": "string" + }, + { + "default": "", + "name": "original_message_id", + "type": "string" + }, + { + "default": "", + "name": "failed_recipient", + "type": "string" + }, + { + "default": "", + "name": "reason", + "type": "string" + } + ], + "name": "warmbly.events.JobEventInboundBounce", + "type": "record" + }, + { + "fields": [ + { + "default": "", + "name": "user_id", + "type": "string" + }, + { + "default": "", + "name": "email_id", + "type": "string" + }, + { + "default": "", + "name": "original_message_id", + "type": "string" + }, + { + "default": "", + "name": "complained_recipient", + "type": "string" + }, + { + "default": "", + "name": "provider", + "type": "string" + } + ], + "name": "warmbly.events.JobEventInboundComplaint", + "type": "record" + }, + { + "fields": [ + { + "default": "", + "name": "user_id", + "type": "string" + }, + { + "default": "", + "name": "email_id", + "type": "string" + }, + { + "default": "", + "name": "mailbox", + "type": "string" + }, + { + "default": 0, + "name": "uid_validity", + "type": "long" + }, + { + "default": false, + "name": "skipped", + "type": "boolean" + } + ], + "name": "warmbly.events.JobEventMailboxDelete", + "type": "record" + }, + { + "fields": [ + { + "default": "", + "name": "user_id", + "type": "string" + }, + { + "default": "", + "name": "email_id", + "type": "string" + }, + { + "default": "", + "name": "from", + "type": "string" + }, + { + "default": "", + "name": "to", + "type": "string" + } + ], + "name": "warmbly.events.JobEventMailboxRename", + "type": "record" + }, + { + "fields": [ + { + "default": "", + "name": "user_id", + "type": "string" + }, + { + "default": "", + "name": "email_id", + "type": "string" + }, + { + "default": null, + "name": "data", + "type": [ + "null", + { + "fields": [ + { + "default": "", + "name": "name", + "type": "string" + }, + { + "default": [], + "name": "attributes", + "type": { + "items": "string", + "type": "array" + } + }, + { + "default": 0, + "name": "uid_validity", + "type": "long" + }, + { + "default": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "name": "highestmodseq", + "type": "warmbly.events.uint64" + }, + { + "default": 0, + "name": "uid_next", + "type": "long" + }, + { + "default": "", + "name": "delim", + "type": "string" + }, + { + "default": 0, + "name": "messages", + "type": "long" + }, + { + "default": 0, + "name": "updated_at", + "type": { + "logicalType": "timestamp-millis", + "type": "long" + } + } + ], + "name": "warmbly.events.Mailbox", + "type": "record" + } + ] + } + ], + "name": "warmbly.events.JobEventMailboxUpdate", + "type": "record" + }, + { + "fields": [ + { + "default": "", + "name": "user_id", + "type": "string" + }, + { + "default": null, + "name": "message", + "type": [ + "null", + { + "fields": [ + { + "default": "", + "name": "id", + "type": "string" + }, + { + "default": "", + "name": "email_id", + "type": "string" + }, + { + "default": 0, + "name": "mailbox", + "type": "long" + }, + { + "default": "", + "name": "folder_path", + "type": "string" + }, + { + "default": "", + "name": "folder", + "type": "string" + }, + { + "default": "", + "name": "provider_folder", + "type": "string" + }, + { + "default": "", + "name": "thread_id", + "type": "string" + }, + { + "default": "", + "name": "message_id", + "type": "string" + }, + { + "default": "", + "name": "gmail_id", + "type": "string" + }, + { + "default": "", + "name": "parent_id", + "type": "string" + }, + { + "default": 0, + "name": "uid", + "type": "long" + }, + { + "default": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "name": "mod_seq", + "type": "warmbly.events.uint64" + }, + { + "default": [], + "name": "flags", + "type": { + "items": "string", + "type": "array" + } + }, + { + "default": [], + "name": "bcc", + "type": { + "items": "string", + "type": "array" + } + }, + { + "default": [], + "name": "cc", + "type": { + "items": "string", + "type": "array" + } + }, + { + "default": [], + "name": "from_addr", + "type": { + "items": "string", + "type": "array" + } + }, + { + "default": [], + "name": "in_reply_to", + "type": { + "items": "string", + "type": "array" + } + }, + { + "default": [], + "name": "reply_to", + "type": { + "items": "string", + "type": "array" + } + }, + { + "default": [], + "name": "to_addr", + "type": { + "items": "string", + "type": "array" + } + }, + { + "default": "", + "name": "subject", + "type": "string" + }, + { + "default": 0, + "name": "size", + "type": "long" + }, + { + "default": 0, + "name": "internal_date", + "type": { + "logicalType": "timestamp-millis", + "type": "long" + } + }, + { + "default": 0, + "name": "sent_date", + "type": { + "logicalType": "timestamp-millis", + "type": "long" + } + }, + { + "default": "", + "name": "snippet", + "type": "string" + }, + { + "default": false, + "name": "seen", + "type": "boolean" + }, + { + "default": "", + "name": "body_text", + "type": "string" + }, + { + "default": 0, + "name": "updated_at", + "type": { + "logicalType": "timestamp-millis", + "type": "long" + } + }, + { + "default": 0, + "name": "created_at", + "type": { + "logicalType": "timestamp-millis", + "type": "long" + } + } + ], + "name": "warmbly.events.EmailMessageStoreData", + "type": "record" + } + ] + }, + { + "default": "", + "name": "report_original_message_id", + "type": "string" + } + ], + "name": "warmbly.events.JobEventNewEmail", + "type": "record" + }, + { + "fields": [ + { + "default": "", + "name": "user_id", + "type": "string" + }, + { + "default": "", + "name": "email_id", + "type": "string" + }, + { + "default": "", + "name": "id", + "type": "string" + }, + { + "default": "", + "name": "skipped_folder", + "type": "string" + } + ], + "name": "warmbly.events.JobEventRemoveEmail", + "type": "record" + }, + { + "fields": [ + { + "default": "", + "name": "user_id", + "type": "string" + }, + { + "default": "", + "name": "email_id", + "type": "string" + }, + { + "default": { + "backfill_completed_at": null, + "backfill_cursor": { + "folders": {}, + "page_token": "" + }, + "backfill_since": null, + "backfill_started_at": null, + "backfill_status": "", + "backfill_synced": 0, + "deferred": 0, + "folders_skipped_cap": 0, + "folders_skipped_conflict": 0, + "last_synced_at": null, + "throttle_reason": "", + "throttled_until": null + }, + "name": "state", + "type": { + "fields": [ + { + "default": "", + "name": "backfill_status", + "type": "string" + }, + { + "default": { + "folders": {}, + "page_token": "" + }, + "name": "backfill_cursor", + "type": { + "fields": [ + { + "default": "", + "name": "page_token", + "type": "string" + }, + { + "default": {}, + "name": "folders", + "type": { + "type": "map", + "values": { + "fields": [ + { + "default": "", + "name": "next", + "type": "string" + }, + { + "default": 0, + "name": "uid", + "type": "long" + }, + { + "default": false, + "name": "done", + "type": "boolean" + } + ], + "name": "warmbly.events.SyncFolderCursor", + "type": "record" + } + } + } + ], + "name": "warmbly.events.SyncCursor", + "type": "record" + } + }, + { + "default": 0, + "name": "backfill_synced", + "type": "int" + }, + { + "default": null, + "name": "backfill_since", + "type": [ + "null", + { + "logicalType": "timestamp-millis", + "type": "long" + } + ] + }, + { + "default": null, + "name": "backfill_started_at", + "type": [ + "null", + { + "logicalType": "timestamp-millis", + "type": "long" + } + ] + }, + { + "default": null, + "name": "backfill_completed_at", + "type": [ + "null", + { + "logicalType": "timestamp-millis", + "type": "long" + } + ] + }, + { + "default": null, + "name": "throttled_until", + "type": [ + "null", + { + "logicalType": "timestamp-millis", + "type": "long" + } + ] + }, + { + "default": "", + "name": "throttle_reason", + "type": "string" + }, + { + "default": 0, + "name": "deferred", + "type": "int" + }, + { + "default": 0, + "name": "folders_skipped_cap", + "type": "int" + }, + { + "default": 0, + "name": "folders_skipped_conflict", + "type": "int" + }, + { + "default": null, + "name": "last_synced_at", + "type": [ + "null", + { + "logicalType": "timestamp-millis", + "type": "long" + } + ] + } + ], + "name": "warmbly.events.SyncState", + "type": "record" + } + } + ], + "name": "warmbly.events.JobEventSyncState", + "type": "record" + }, + { + "fields": [ + { + "default": "", + "name": "user_id", + "type": "string" + }, + { + "default": "", + "name": "email_id", + "type": "string" + }, + { + "default": "", + "name": "access_token", + "type": "string" + }, + { + "default": "", + "name": "refresh_token", + "type": "string" + }, + { + "default": 0, + "name": "expires_at", + "type": { + "logicalType": "timestamp-millis", + "type": "long" + } + } + ], + "name": "warmbly.events.JobEventTokenUpdate", + "type": "record" + }, + { + "fields": [ + { + "default": "", + "name": "user_id", + "type": "string" + }, + { + "default": "", + "name": "email_id", + "type": "string" + }, + { + "default": "", + "name": "rfc_message_id", + "type": "string" + }, + { + "default": "", + "name": "outcome", + "type": "string" + }, + { + "default": false, + "name": "recheck", + "type": "boolean" + } + ], + "name": "warmbly.events.JobEventWarmupRemovalChecked", + "type": "record" + }, + { + "fields": [ + { + "default": "", + "name": "task_id", + "type": "string" + }, + { + "default": false, + "name": "success", + "type": "boolean" + }, + { + "default": "", + "name": "message_id", + "type": "string" + }, + { + "default": "", + "name": "provider_msg_id", + "type": "string" + }, + { + "default": "", + "name": "thread_id", + "type": "string" + }, + { + "default": 0, + "name": "sent_at", + "type": { + "logicalType": "timestamp-millis", + "type": "long" + } + }, + { + "default": null, + "name": "error", + "type": [ + "null", + { + "fields": [ + { + "default": "", + "name": "code", + "type": "string" + }, + { + "default": "", + "name": "type", + "type": "string" + }, + { + "default": "", + "name": "message", + "type": "string" + }, + { + "default": "", + "name": "resolve_method", + "type": "string" + }, + { + "default": false, + "name": "user_visible", + "type": "boolean" + }, + { + "default": "", + "name": "user_title", + "type": "string" + }, + { + "default": "", + "name": "user_message", + "type": "string" + }, + { + "default": "", + "name": "action_required", + "type": "string" + }, + { + "default": "", + "name": "recipient", + "type": "string" + } + ], + "name": "warmbly.events.EmailSendError", + "type": "record" + } + ] + }, + { + "default": "", + "name": "legacy_error", + "type": "string" + } + ], + "name": "warmbly.events.SendEmailResult", + "type": "record" + }, + { + "fields": [ + { + "default": "", + "name": "worker_id", + "type": "string" + }, + { + "default": 0, + "name": "observed_at", + "type": { + "logicalType": "timestamp-millis", + "type": "long" + } + }, + { + "default": 0, + "name": "assigned_count", + "type": "int" + }, + { + "default": 0, + "name": "imap_idle_count", + "type": "int" + }, + { + "default": 0, + "name": "memory_mb", + "type": "int" + }, + { + "default": 0, + "name": "goroutine_count", + "type": "int" + }, + { + "default": 0, + "name": "sends_attempted", + "type": "int" + }, + { + "default": 0, + "name": "sends_succeeded", + "type": "int" + }, + { + "default": 0, + "name": "bounces_hard", + "type": "int" + }, + { + "default": 0, + "name": "bounces_soft", + "type": "int" + }, + { + "default": 0, + "name": "complaints", + "type": "int" + }, + { + "default": 0, + "name": "auth_errors", + "type": "int" + }, + { + "default": 0, + "name": "rate_limit_errors", + "type": "int" + }, + { + "default": 0, + "name": "smtp_latency_p50_ms", + "type": "int" + }, + { + "default": 0, + "name": "smtp_latency_p99_ms", + "type": "int" + } + ], + "name": "warmbly.events.WorkerHealthSample", + "type": "record" + } + ] + } + ], + "name": "warmbly.events.JobEvent", + "type": "record" +} diff --git a/internal/app/eventschemas/testdata/warmup-events.avsc b/internal/app/eventschemas/testdata/warmup-events.avsc new file mode 100644 index 000000000..6e86fd745 --- /dev/null +++ b/internal/app/eventschemas/testdata/warmup-events.avsc @@ -0,0 +1,44 @@ +{ + "fields": [ + { + "default": "", + "name": "event_type", + "type": "string" + }, + { + "default": "", + "name": "task_id", + "type": "string" + }, + { + "default": "", + "name": "sender_account_id", + "type": "string" + }, + { + "default": "", + "name": "target_account_id", + "type": "string" + }, + { + "default": "", + "name": "message_id", + "type": "string" + }, + { + "default": false, + "name": "is_reply", + "type": "boolean" + }, + { + "default": 0, + "name": "sent_at", + "type": { + "logicalType": "timestamp-millis", + "type": "long" + } + } + ], + "name": "warmbly.events.WarmupEmailSentEvent", + "type": "record" +} diff --git a/internal/app/eventschemas/testdata/worker-commands.avsc b/internal/app/eventschemas/testdata/worker-commands.avsc new file mode 100644 index 000000000..59ae48a14 --- /dev/null +++ b/internal/app/eventschemas/testdata/worker-commands.avsc @@ -0,0 +1,839 @@ +{ + "fields": [ + { + "name": "type", + "type": "string" + }, + { + "name": "body", + "type": [ + { + "fields": [ + { + "default": "", + "name": "id", + "type": "string" + }, + { + "default": "", + "name": "user_id", + "type": "string" + }, + { + "default": null, + "name": "organization_id", + "type": [ + "null", + "string" + ] + }, + { + "default": false, + "name": "imap_sync", + "type": "boolean" + }, + { + "default": null, + "name": "save_to_sent", + "type": [ + "null", + "boolean" + ] + }, + { + "default": "", + "name": "email", + "type": "string" + }, + { + "default": "", + "name": "first_name", + "type": "string" + }, + { + "default": "", + "name": "last_name", + "type": "string" + }, + { + "default": "", + "name": "type", + "type": "string" + }, + { + "default": null, + "name": "google", + "type": [ + "null", + { + "fields": [ + { + "default": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "name": "last_history_id", + "type": { + "name": "warmbly.events.uint64", + "size": 8, + "type": "fixed" + } + }, + { + "default": null, + "name": "token", + "type": [ + "null", + { + "fields": [ + { + "default": "", + "name": "AccessToken", + "type": "string" + }, + { + "default": "", + "name": "TokenType", + "type": "string" + }, + { + "default": "", + "name": "RefreshToken", + "type": "string" + }, + { + "default": 0, + "name": "Expiry", + "type": { + "logicalType": "timestamp-millis", + "type": "long" + } + }, + { + "default": 0, + "name": "ExpiresIn", + "type": "long" + } + ], + "name": "warmbly.events.Token", + "type": "record" + } + ] + } + ], + "name": "warmbly.events.AddWorkerEmailGoogleData", + "type": "record" + } + ] + }, + { + "default": null, + "name": "smtp_imap", + "type": [ + "null", + { + "fields": [ + { + "default": [], + "name": "mailboxes", + "type": { + "items": { + "fields": [ + { + "default": "", + "name": "name", + "type": "string" + }, + { + "default": [], + "name": "attributes", + "type": { + "items": "string", + "type": "array" + } + }, + { + "default": 0, + "name": "uid_validity", + "type": "long" + }, + { + "default": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", + "name": "highestmodseq", + "type": "warmbly.events.uint64" + }, + { + "default": 0, + "name": "uid_next", + "type": "long" + }, + { + "default": "", + "name": "delim", + "type": "string" + }, + { + "default": 0, + "name": "messages", + "type": "long" + }, + { + "default": 0, + "name": "updated_at", + "type": { + "logicalType": "timestamp-millis", + "type": "long" + } + } + ], + "name": "warmbly.events.Mailbox", + "type": "record" + }, + "type": "array" + } + }, + { + "default": null, + "name": "token", + "type": [ + "null", + "warmbly.events.Token" + ] + }, + { + "default": null, + "name": "credentials", + "type": [ + "null", + { + "fields": [ + { + "default": null, + "name": "smtp", + "type": [ + "null", + { + "fields": [ + { + "default": "", + "name": "username", + "type": "string" + }, + { + "default": "", + "name": "password", + "type": "string" + }, + { + "default": "", + "name": "host", + "type": "string" + }, + { + "default": 0, + "name": "port", + "type": "int" + }, + { + "default": "", + "name": "security", + "type": "string" + } + ], + "name": "warmbly.events.Service", + "type": "record" + } + ] + }, + { + "default": null, + "name": "imap", + "type": [ + "null", + "warmbly.events.Service" + ] + } + ], + "name": "warmbly.events.SmtpImap", + "type": "record" + } + ] + } + ], + "name": "warmbly.events.AddWorkerEmailSmtpImapData", + "type": "record" + } + ] + }, + { + "default": null, + "name": "graph", + "type": [ + "null", + { + "fields": [ + { + "default": null, + "name": "token", + "type": [ + "null", + "warmbly.events.Token" + ] + }, + { + "default": {}, + "name": "delta_links", + "type": { + "type": "map", + "values": "string" + } + }, + { + "default": "", + "name": "user", + "type": "string" + } + ], + "name": "warmbly.events.AddWorkerEmailGraphData", + "type": "record" + } + ] + }, + { + "default": null, + "name": "sync", + "type": [ + "null", + { + "fields": [ + { + "default": { + "backfill_days": 0, + "backfill_messages": 0, + "daily_messages": 0, + "org_daily_messages": 0, + "skip_folders": [] + }, + "name": "policy", + "type": { + "fields": [ + { + "default": 0, + "name": "backfill_days", + "type": "int" + }, + { + "default": 0, + "name": "backfill_messages", + "type": "int" + }, + { + "default": 0, + "name": "daily_messages", + "type": "int" + }, + { + "default": 0, + "name": "org_daily_messages", + "type": "int" + }, + { + "default": [], + "name": "skip_folders", + "type": { + "items": "string", + "type": "array" + } + } + ], + "name": "warmbly.events.SyncPolicy", + "type": "record" + } + }, + { + "default": null, + "name": "state", + "type": [ + "null", + { + "fields": [ + { + "default": "", + "name": "backfill_status", + "type": "string" + }, + { + "default": { + "folders": {}, + "page_token": "" + }, + "name": "backfill_cursor", + "type": { + "fields": [ + { + "default": "", + "name": "page_token", + "type": "string" + }, + { + "default": {}, + "name": "folders", + "type": { + "type": "map", + "values": { + "fields": [ + { + "default": "", + "name": "next", + "type": "string" + }, + { + "default": 0, + "name": "uid", + "type": "long" + }, + { + "default": false, + "name": "done", + "type": "boolean" + } + ], + "name": "warmbly.events.SyncFolderCursor", + "type": "record" + } + } + } + ], + "name": "warmbly.events.SyncCursor", + "type": "record" + } + }, + { + "default": 0, + "name": "backfill_synced", + "type": "int" + }, + { + "default": null, + "name": "backfill_since", + "type": [ + "null", + { + "logicalType": "timestamp-millis", + "type": "long" + } + ] + }, + { + "default": null, + "name": "backfill_started_at", + "type": [ + "null", + { + "logicalType": "timestamp-millis", + "type": "long" + } + ] + }, + { + "default": null, + "name": "backfill_completed_at", + "type": [ + "null", + { + "logicalType": "timestamp-millis", + "type": "long" + } + ] + }, + { + "default": null, + "name": "throttled_until", + "type": [ + "null", + { + "logicalType": "timestamp-millis", + "type": "long" + } + ] + }, + { + "default": "", + "name": "throttle_reason", + "type": "string" + }, + { + "default": 0, + "name": "deferred", + "type": "int" + }, + { + "default": 0, + "name": "folders_skipped_cap", + "type": "int" + }, + { + "default": 0, + "name": "folders_skipped_conflict", + "type": "int" + }, + { + "default": null, + "name": "last_synced_at", + "type": [ + "null", + { + "logicalType": "timestamp-millis", + "type": "long" + } + ] + } + ], + "name": "warmbly.events.SyncState", + "type": "record" + } + ] + } + ], + "name": "warmbly.events.AddWorkerEmailSyncData", + "type": "record" + } + ] + }, + { + "default": false, + "name": "brokered", + "type": "boolean" + } + ], + "name": "warmbly.events.AddWorkerEmail", + "type": "record" + }, + { + "fields": [ + { + "default": "", + "name": "org_id", + "type": "string" + }, + { + "default": "", + "name": "process_id", + "type": "string" + }, + { + "default": null, + "name": "credentials", + "type": [ + "null", + "warmbly.events.SmtpImap" + ] + } + ], + "name": "warmbly.events.EventWorkerEmailValidation", + "type": "record" + }, + { + "fields": [ + { + "default": "", + "name": "email_id", + "type": "string" + }, + { + "default": "", + "name": "process_id", + "type": "string" + }, + { + "default": false, + "name": "want_signature", + "type": "boolean" + }, + { + "default": "", + "name": "signature_for", + "type": "string" + } + ], + "name": "warmbly.events.EventWorkerMailboxIdentity", + "type": "record" + }, + { + "fields": [ + { + "default": "", + "name": "email_id", + "type": "string" + }, + { + "default": false, + "name": "seen", + "type": "boolean" + }, + { + "default": [], + "name": "messages", + "type": { + "items": { + "fields": [ + { + "default": "", + "name": "provider_id", + "type": "string" + }, + { + "default": 0, + "name": "uid", + "type": "long" + }, + { + "default": "", + "name": "folder", + "type": "string" + }, + { + "default": "", + "name": "rfc_message_id", + "type": "string" + } + ], + "name": "warmbly.events.MessageSeenRef", + "type": "record" + }, + "type": "array" + } + } + ], + "name": "warmbly.events.MessageSeenAction", + "type": "record" + }, + { + "fields": [ + { + "default": "", + "name": "user_id", + "type": "string" + }, + { + "default": "", + "name": "email_id", + "type": "string" + } + ], + "name": "warmbly.events.RemoveWorkerEmail", + "type": "record" + }, + { + "fields": [ + { + "default": "", + "name": "task_id", + "type": "string" + }, + { + "default": "", + "name": "email_id", + "type": "string" + }, + { + "default": "", + "name": "org_id", + "type": "string" + }, + { + "default": [], + "name": "to", + "type": { + "items": "string", + "type": "array" + } + }, + { + "default": [], + "name": "cc", + "type": { + "items": "string", + "type": "array" + } + }, + { + "default": [], + "name": "bcc", + "type": { + "items": "string", + "type": "array" + } + }, + { + "default": "", + "name": "subject", + "type": "string" + }, + { + "default": "", + "name": "body_s3_key", + "type": "string" + }, + { + "default": "", + "name": "message_id", + "type": "string" + }, + { + "default": "", + "name": "in_reply_to", + "type": "string" + }, + { + "default": null, + "name": "parent", + "type": [ + "null", + { + "fields": [ + { + "default": "", + "name": "id", + "type": "string" + }, + { + "default": "", + "name": "message_id", + "type": "string" + }, + { + "default": "", + "name": "thread_id", + "type": "string" + } + ], + "name": "warmbly.events.EmailParent", + "type": "record" + } + ] + }, + { + "default": false, + "name": "is_warmup", + "type": "boolean" + }, + { + "default": null, + "name": "tracking_info", + "type": [ + "null", + { + "fields": [ + { + "default": false, + "name": "open_tracking", + "type": "boolean" + }, + { + "default": false, + "name": "link_tracking", + "type": "boolean" + }, + { + "default": "", + "name": "tracking_domain", + "type": "string" + } + ], + "name": "warmbly.events.TrackingInfo", + "type": "record" + } + ] + }, + { + "default": "", + "name": "warmup_token", + "type": "string" + }, + { + "default": "", + "name": "unsubscribe_url", + "type": "string" + } + ], + "name": "warmbly.events.SendEmail", + "type": "record" + }, + { + "fields": [ + { + "default": "", + "name": "user_id", + "type": "string" + }, + { + "default": "", + "name": "email_id", + "type": "string" + }, + { + "default": "", + "name": "gmail_id", + "type": "string" + }, + { + "default": 0, + "name": "uid", + "type": "long" + }, + { + "default": 0, + "name": "mailbox_uid_validity", + "type": "long" + }, + { + "default": "", + "name": "mailbox_folder", + "type": "string" + }, + { + "default": "", + "name": "rfc_message_id", + "type": "string" + }, + { + "default": [], + "name": "actions", + "type": { + "items": "string", + "type": "array" + } + }, + { + "default": "", + "name": "placement", + "type": "string" + }, + { + "default": "", + "name": "target_folder", + "type": "string" + }, + { + "default": "", + "name": "internal_id", + "type": "string" + }, + { + "default": false, + "name": "recheck", + "type": "boolean" + }, + { + "default": 0, + "name": "delay_seconds", + "type": "int" + } + ], + "name": "warmbly.events.WarmupEmailAction", + "type": "record" + } + ] + } + ], + "name": "warmbly.events.WorkerEvent", + "type": "record" +} diff --git a/internal/app/instancesettings/document.go b/internal/app/instancesettings/document.go index 59d65e95c..43f4aca26 100644 --- a/internal/app/instancesettings/document.go +++ b/internal/app/instancesettings/document.go @@ -202,6 +202,17 @@ type Placement struct { // CreditsPerTest is what a test past the monthly allowance costs in // credits. Zero turns paid tests off; nil is the compiled default. CreditsPerTest *int `json:"credits_per_test"` + // BatchSendersMax is the most senders one batch may hold, an + // infrastructure safeguard separate from how many run at once. + BatchSendersMax int `json:"batch_senders_max"` + // BatchSenderConcurrency is how many of one workspace's batch senders + // may be sending probes at the same time. + BatchSenderConcurrency int `json:"batch_sender_concurrency"` + // BatchInstanceConcurrency caps batch senders sending at once across + // every workspace, which bounds what the shared seed panel receives. + BatchInstanceConcurrency int `json:"batch_instance_concurrency"` + // BatchStartsPerMinute paces how fast one batch starts senders. + BatchStartsPerMinute int `json:"batch_starts_per_minute"` } // CreditPrice is the resolved price of a paid test, zero when off. @@ -221,6 +232,11 @@ func DefaultPlacement() Placement { SeedsPerTest: config.PlacementSeedsPerTestDefault, SpacingSeconds: config.PlacementSpacingSecondsDefault, CreditsPerTest: &price, + + BatchSendersMax: config.PlacementBatchSendersMaxDefault, + BatchSenderConcurrency: config.PlacementBatchSenderConcurrencyDefault, + BatchInstanceConcurrency: config.PlacementBatchInstanceConcurrencyDefault, + BatchStartsPerMinute: config.PlacementBatchStartsPerMinuteDefault, } } @@ -248,6 +264,16 @@ func (p *Placement) Normalize() { v = max(0, min(*p.CreditsPerTest, config.PlacementCreditsPerTestMax)) } p.CreditsPerTest = &v + clamp := func(v, def, ceiling int) int { + if v <= 0 { + return def + } + return min(v, ceiling) + } + p.BatchSendersMax = clamp(p.BatchSendersMax, config.PlacementBatchSendersMaxDefault, config.PlacementBatchSendersMaxCeiling) + p.BatchSenderConcurrency = clamp(p.BatchSenderConcurrency, config.PlacementBatchSenderConcurrencyDefault, config.PlacementBatchSenderConcurrencyMax) + p.BatchInstanceConcurrency = clamp(p.BatchInstanceConcurrency, config.PlacementBatchInstanceConcurrencyDefault, config.PlacementBatchInstanceConcurrencyMax) + p.BatchStartsPerMinute = clamp(p.BatchStartsPerMinute, config.PlacementBatchStartsPerMinuteDefault, config.PlacementBatchStartsPerMinuteMax) } // QuickSpacing is the gap between two probes of a quick test. @@ -453,6 +479,11 @@ type Patch struct { SeedsPerTest *int `json:"seeds_per_test"` SpacingSeconds *int `json:"spacing_seconds"` CreditsPerTest *int `json:"credits_per_test"` + + BatchSendersMax *int `json:"batch_senders_max"` + BatchSenderConcurrency *int `json:"batch_sender_concurrency"` + BatchInstanceConcurrency *int `json:"batch_instance_concurrency"` + BatchStartsPerMinute *int `json:"batch_starts_per_minute"` } `json:"placement"` // Channels replaces the whole list when present. A channel that comes back // with a masked target or secret keeps the stored value, so the admin panel @@ -552,6 +583,18 @@ func (p Patch) Apply(doc Document) Document { v := *p.Placement.CreditsPerTest doc.Placement.CreditsPerTest = &v } + if p.Placement.BatchSendersMax != nil { + doc.Placement.BatchSendersMax = *p.Placement.BatchSendersMax + } + if p.Placement.BatchSenderConcurrency != nil { + doc.Placement.BatchSenderConcurrency = *p.Placement.BatchSenderConcurrency + } + if p.Placement.BatchInstanceConcurrency != nil { + doc.Placement.BatchInstanceConcurrency = *p.Placement.BatchInstanceConcurrency + } + if p.Placement.BatchStartsPerMinute != nil { + doc.Placement.BatchStartsPerMinute = *p.Placement.BatchStartsPerMinute + } } if p.Notifications != nil && p.Notifications.Channels != nil { doc.Notifications.Channels = mergeChannels(doc.Notifications.Channels, *p.Notifications.Channels) diff --git a/internal/app/orgtransfer/spec.go b/internal/app/orgtransfer/spec.go index 8f08a2e95..db34354f6 100644 --- a/internal/app/orgtransfer/spec.go +++ b/internal/app/orgtransfer/spec.go @@ -286,7 +286,7 @@ var Tables = []Table{ { Name: "categories", Group: models.OrgDataGroupContacts, Scope: scopeOrg, - Note: "The whole category registry travels, including ones no contact or conversation carries yet.", + Note: "The whole label registry travels, including ones no contact or conversation carries yet.", }, { Name: "contacts", Group: models.OrgDataGroupContacts, @@ -462,6 +462,12 @@ var Tables = []Table{ Name: "campaign_segments", Group: models.OrgDataGroupCampaigns, Scope: `campaign_id IN ` + orgCampaigns, }, + { + // Both contacts are in the contacts group, which campaigns require. + Name: "campaign_lead_cc", Group: models.OrgDataGroupCampaigns, + Scope: `campaign_id IN ` + orgCampaigns, + Note: "Must travel with the leads, or a copied contact held on their own lead is released into a second sequence.", + }, { Name: "campaign_lead_removals", Group: models.OrgDataGroupCampaigns, Scope: `campaign_id IN ` + orgCampaigns, @@ -767,6 +773,17 @@ var Tables = []Table{ Name: "deliverability_events", Group: models.OrgDataGroupEvents, Scope: scopeOrg, }, + { + // A batch travels as a record of its senders and results. It lands + // inactive, so the destination never resumes sending it. + Name: "placement_batches", Group: models.OrgDataGroupEvents, + Scope: scopeOrg, + ResetOnImport: []string{"active", "lease_until", "last_tick_at"}, + }, + { + Name: "placement_batch_senders", Group: models.OrgDataGroupEvents, + Scope: `batch_id IN (SELECT id FROM placement_batches WHERE organization_id = $1)`, + }, { // The results travel as a record. The link to a cloud-run test and the // seeds on the source instance's panel do not, and neither does a @@ -888,6 +905,8 @@ var ExcludedTables = map[string]string{ "oauth_authorization_codes": "Single-use authorization codes, valid for seconds.", "scheduled_deletions": "Instance lifecycle state. Importing a pending deletion would schedule the destination workspace for destruction.", "dedicated_worker_assignments": "Worker topology, which is a property of the instance rather than the workspace.", + "warmup_spam_moves": "Per-message attribution evidence for warmup mail this instance synced, kept only to decide recent tampering; the destination judges its own.", + "mailbox_owner_activity": "Five-minute buckets of sync-observed owner activity on this instance, read only to attribute recent spam moves.", "warmup_pools": "Instance-global pool definitions shared by every workspace on the instance.", "pool_link_codes": "In-flight link handshakes between a self-hosted instance and this cloud, valid for minutes.", "cli_auth_codes": "In-flight `warmbly auth login` handshakes, valid for minutes. The API key an approval mints does travel, with the api_keys rows.", diff --git a/internal/app/placement/batch.go b/internal/app/placement/batch.go new file mode 100644 index 000000000..1f535a242 --- /dev/null +++ b/internal/app/placement/batch.go @@ -0,0 +1,1055 @@ +package placement + +import ( + "context" + "math" + "math/rand/v2" + "slices" + "sort" + "strconv" + "strings" + + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/observability/errs" + "github.com/warmbly/warmbly/internal/pkg/mailhost" + "github.com/warmbly/warmbly/internal/repository" +) + +// BatchInput is one request to run a placement test from many senders. +// Exactly one of SenderAccountIDs and Scope chooses the senders. +type BatchInput struct { + OrgID uuid.UUID + UserID *uuid.UUID + SenderAccountIDs []uuid.UUID + Scope *models.PlacementSenderScope + Sample models.PlacementSample + // AllowedSenders is an API key's mailbox restriction; nil allows every one. + AllowedSenders []uuid.UUID + + CampaignID *uuid.UUID + SequenceID *uuid.UUID + ContactID *uuid.UUID + Subject string + BodyHTML string + BodyPlain string + Tracking string + Panel string + Pace string + Families []string + SeedIDs []uuid.UUID + OnUnavailable string + // MaxCredits is the most the caller agreed to pay across the whole batch + // for tests past the monthly free allowance. + MaxCredits int +} + +// BatchGroupCount is how many selected senders share a provider. +type BatchGroupCount struct { + Key string `json:"key"` + Label string `json:"label"` + Senders int `json:"senders"` +} + +// BatchPreview is what a batch would do, before it is started. +type BatchPreview struct { + // Matched is how many senders the scope resolved to, Selected how many + // the sample kept. + Matched int `json:"matched"` + Selected int `json:"selected"` + Inactive int `json:"inactive"` + Domains int `json:"domains"` + Providers []BatchGroupCount `json:"providers"` + // Variants is two for a tracking comparison, which doubles every count. + Variants int `json:"variants"` + Tests int `json:"tests"` + SeedsPerTest int `json:"seeds_per_test"` + // MaxSends is the most probes the batch sends; each sender's own daily + // limit can only make it fewer. + MaxSends int `json:"max_sends"` + // FreeTests and PaidTests split Tests against the monthly allowance, and + // Credits is the most the paid ones cost. Unmetered panels leave all three + // zero. + Metered bool `json:"metered"` + FreeTests int `json:"free_tests"` + PaidTests int `json:"paid_tests"` + Credits int `json:"credits"` + Usage models.PlacementUsage `json:"usage"` + SendersMax int `json:"senders_max"` + // Concurrency is how many senders send at once in this workspace. + Concurrency int `json:"concurrency"` +} + +// BatchView is a batch with its progress and headline placement. +type BatchView struct { + models.PlacementBatch + Progress models.PlacementBatchProgress `json:"progress"` + Summary models.PlacementCounts `json:"summary"` +} + +// BatchGroup is a batch's placement for one sending domain or provider. +type BatchGroup struct { + Key string `json:"key"` + Label string `json:"label"` + Senders int `json:"senders"` + Tested int `json:"tested"` + Counts models.PlacementCounts `json:"counts"` +} + +// BatchMatrixRow is one sending domain's placement per recipient provider. +type BatchMatrixRow struct { + Domain string `json:"domain"` + Recipients []models.PlacementFamilyCounts `json:"recipients"` +} + +// BatchDetail is one batch in full. +type BatchDetail struct { + BatchView + // Untracked is the untracked half of a tracking comparison; Summary is + // the tracked half, the copy the campaign really sends. + Untracked *models.PlacementCounts `json:"untracked,omitempty"` + Domains []BatchGroup `json:"domains"` + Providers []BatchGroup `json:"providers"` + Recipients []models.PlacementFamilyCounts `json:"recipients"` + Matrix []BatchMatrixRow `json:"matrix"` + Content ContentCheck `json:"content"` +} + +// BatchSenderView is one sender of a batch with where its copies landed. +type BatchSenderView struct { + models.PlacementBatchSender + SenderFamilyLabel string `json:"sender_family_label"` + Summary models.PlacementCounts `json:"summary"` + TestIDs []uuid.UUID `json:"test_ids"` +} + +// batchCandidate is a resolved sender with the groups sampling reads. +type batchCandidate struct { + repository.PlacementBatchCandidate + family string + domain string +} + +func (s *service) batchesReady() *errx.Error { + if s.Batches == nil { + return errx.New(errx.NotImplemented, "placement batches are not configured") + } + return nil +} + +// validateBatch normalizes everything a batch request carries except the +// senders, and resolves its copy once so every sender tests the same email. +func (s *service) validateBatch(ctx context.Context, in *BatchInput, copyGiven bool) ([]variant, *errx.Error) { + if in.Panel == "" { + in.Panel = models.PlacementPanelInstance + } + if !models.ValidPlacementPanel(in.Panel) { + return nil, errx.New(errx.BadRequest, "panel must be instance, workspace or cloud") + } + if len(in.SeedIDs) > 0 && in.Panel != models.PlacementPanelWorkspace { + return nil, errx.New(errx.BadRequest, "seed_ids needs panel workspace") + } + if len(in.SeedIDs) > config.PlacementSeedsPerWorkspaceMax { + return nil, errx.New(errx.BadRequest, "seed_ids has more entries than a workspace can have seed inboxes") + } + // A batch runs for hours anyway; quick copies from many senders at once + // would only reach the shared seeds as a burst. + if in.Pace == "" { + in.Pace = models.PlacementPaceSpaced + } + if in.Pace != models.PlacementPaceSpaced { + return nil, errx.New(errx.BadRequest, "a batch always sends spaced; pace quick is for a single test") + } + if in.Tracking == "" { + in.Tracking = models.PlacementTrackingCampaign + } + switch in.Tracking { + case models.PlacementTrackingCampaign, models.PlacementTrackingOn, models.PlacementTrackingOff, models.PlacementTrackingCompare: + default: + return nil, errx.New(errx.BadRequest, "tracking must be campaign, on, off or compare") + } + if in.OnUnavailable == "" { + in.OnUnavailable = models.PlacementUnavailableDefer + } + if in.OnUnavailable != models.PlacementUnavailableDefer && in.OnUnavailable != models.PlacementUnavailableSkip { + return nil, errx.New(errx.BadRequest, "on_unavailable must be skip or defer") + } + if in.MaxCredits < 0 { + return nil, errx.New(errx.BadRequest, "max_credits cannot be negative") + } + families, xerr := normalizeFamilies(in.Families) + if xerr != nil { + return nil, xerr + } + in.Families = families + + if s.Gate != nil && !config.SelfHosted() { + if ok, _ := s.Gate.CanSendCampaignEmail(ctx, in.OrgID); !ok { + return nil, placementErr(errx.PaymentRequired, "placement_not_entitled", "Placement tests need an active trial or subscription.") + } + } + + // A preview asked before the copy is chosen counts senders alone; the + // copy is checked once any of it is given, and always on create. + var variants []variant + if copyGiven || in.CampaignID != nil || in.SequenceID != nil || strings.TrimSpace(in.Subject) != "" || in.BodyHTML != "" || in.BodyPlain != "" { + spec := copySpec{ + CampaignID: in.CampaignID, SequenceID: in.SequenceID, ContactID: in.ContactID, + Subject: in.Subject, BodyHTML: in.BodyHTML, BodyPlain: in.BodyPlain, Panel: in.Panel, + } + if xerr := s.resolveCopy(ctx, in.OrgID, &spec); xerr != nil { + return nil, xerr + } + in.ContactID, in.Subject, in.BodyHTML, in.BodyPlain = spec.ContactID, spec.Subject, spec.BodyHTML, spec.BodyPlain + if variants, xerr = trackingVariants(in.Tracking, spec.campaign); xerr != nil { + return nil, xerr + } + } else if variants, xerr = trackingVariants(in.Tracking, nil); xerr != nil { + return nil, xerr + } + + // A panel no sender could test on refuses the batch, not every sender. + switch in.Panel { + case models.PlacementPanelCloud: + if s.Cloud == nil { + return nil, placementErr(errx.Conflict, "placement_panel_unavailable", "Link this instance to Warmbly Cloud to test on its seed panel.") + } + default: + scope, orgFilter := models.SeedScopeInstance, (*uuid.UUID)(nil) + if in.Panel == models.PlacementPanelWorkspace { + scope, orgFilter = models.SeedScopeWorkspace, &in.OrgID + } + all, err := s.Repo.ListSeeds(ctx, scope, orgFilter, false) + if err != nil { + errs.CaptureException(err) + return nil, errx.InternalError() + } + if len(in.SeedIDs) > 0 { + known := map[uuid.UUID]bool{} + for _, r := range all { + known[r.ID] = true + } + for _, id := range in.SeedIDs { + if !known[id] { + return nil, placementErr(errx.BadRequest, "placement_invalid_seeds", + "Every chosen seed inbox has to be a seed inbox of this workspace.") + } + } + } + if len(inFamilies(all, families)) == 0 { + return nil, placementErr(errx.Conflict, "placement_no_seeds", noSeedsMessage(in.Panel)) + } + } + return variants, nil +} + +// resolveSenders turns the request's senders into the candidates the batch +// will snapshot, before sampling, and reports how many matched. +func (s *service) resolveSenders(ctx context.Context, in BatchInput) ([]batchCandidate, *errx.Error) { + if (len(in.SenderAccountIDs) > 0) == (in.Scope != nil) { + return nil, errx.New(errx.BadRequest, "choose the senders with either sender_account_ids or sender_scope") + } + allowed := func(id uuid.UUID) bool { return in.AllowedSenders == nil || slices.Contains(in.AllowedSenders, id) } + + var filter repository.PlacementCandidateFilter + var providers, domains []string + if len(in.SenderAccountIDs) > 0 { + ids := uniqueUUIDs(in.SenderAccountIDs) + if len(ids) > config.PlacementBatchSendersMaxCeiling { + return nil, placementErr(errx.BadRequest, "placement_batch_too_large", "sender_account_ids names more mailboxes than any batch may hold.") + } + for _, id := range ids { + if !allowed(id) { + return nil, errx.New(errx.Forbidden, "this API key cannot send from one of those mailboxes") + } + } + // Chosen by hand, so a disconnected one is kept and reported by name + // when its turn comes. + filter = repository.PlacementCandidateFilter{IDs: ids, IncludeInactive: true} + rows, err := s.Batches.ListBatchCandidates(ctx, in.OrgID, filter) + if err != nil { + errs.CaptureException(err) + return nil, errx.InternalError() + } + if len(rows) != len(ids) { + return nil, errx.New(errx.NotFound, "a sending mailbox was not found in this workspace, or is a seed inbox") + } + return candidates(rows), nil + } + + sc := in.Scope + if len(sc.Providers) > config.PlacementFamiliesMax { + return nil, errx.New(errx.BadRequest, "sender_scope.providers names too many providers") + } + if len(sc.Domains) > 1000 { + return nil, errx.New(errx.BadRequest, "sender_scope.domains names too many domains") + } + if len(sc.TagIDs) > 200 { + return nil, errx.New(errx.BadRequest, "sender_scope.tag_ids names too many tags") + } + if sc.UntestedDays < 0 || sc.UntestedDays > 365 { + return nil, errx.New(errx.BadRequest, "sender_scope.untested_days must be between 0 and 365") + } + for _, p := range sc.Providers { + p = strings.ToLower(strings.TrimSpace(p)) + if p != "" { + providers = append(providers, p) + } + } + for _, d := range sc.Domains { + d = strings.ToLower(strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(d), "@"))) + if d != "" { + domains = append(domains, d) + } + } + filter = repository.PlacementCandidateFilter{TagIDs: sc.TagIDs, IncludeInactive: sc.IncludeInactive} + if sc.UntestedDays > 0 { + since := s.now().AddDate(0, 0, -sc.UntestedDays) + filter.UntestedSince = &since + } + switch sc.Type { + case models.PlacementScopeWorkspace: + case models.PlacementScopeCampaign: + if sc.CampaignID == nil { + return nil, errx.New(errx.BadRequest, "sender_scope.campaign_id is required for a campaign scope") + } + campaign, xerr := s.ownedCampaign(ctx, in.OrgID, *sc.CampaignID) + if xerr != nil { + return nil, xerr + } + pool, xerr := repository.ResolveCampaignSenderPool(ctx, s.Emails, campaign) + if xerr != nil { + return nil, xerr + } + filter.IDs = make([]uuid.UUID, 0, len(pool.Accounts)) + for _, a := range pool.Accounts { + filter.IDs = append(filter.IDs, a.ID) + } + default: + return nil, errx.New(errx.BadRequest, "sender_scope.type must be campaign or workspace") + } + rows, err := s.Batches.ListBatchCandidates(ctx, in.OrgID, filter) + if err != nil { + errs.CaptureException(err) + return nil, errx.InternalError() + } + out := make([]batchCandidate, 0, len(rows)) + for _, c := range candidates(rows) { + if !allowed(c.ID) { + continue + } + if len(providers) > 0 && !slices.Contains(providers, c.family) { + continue + } + if len(domains) > 0 && !slices.Contains(domains, c.domain) { + continue + } + out = append(out, c) + } + return out, nil +} + +func candidates(rows []repository.PlacementBatchCandidate) []batchCandidate { + out := make([]batchCandidate, 0, len(rows)) + for _, r := range rows { + out = append(out, batchCandidate{ + PlacementBatchCandidate: r, + family: string(mailhost.ForMailbox(r.MailHost, r.Provider, r.Email)), + domain: domainOf(r.SendFrom()), + }) + } + return out +} + +func uniqueUUIDs(ids []uuid.UUID) []uuid.UUID { + seen := make(map[uuid.UUID]bool, len(ids)) + out := make([]uuid.UUID, 0, len(ids)) + for _, id := range ids { + if id != uuid.Nil && !seen[id] { + seen[id] = true + out = append(out, id) + } + } + return out +} + +// validateSample checks a sampling request. +func validateSample(sp *models.PlacementSample) *errx.Error { + if sp.Mode == "" { + sp.Mode = models.PlacementSampleAll + } + switch sp.Mode { + case models.PlacementSampleAll: + case models.PlacementSampleRandom, models.PlacementSamplePerDomain, models.PlacementSamplePerProvider: + if sp.Count < 1 || sp.Count > config.PlacementBatchSendersMaxCeiling { + return errx.New(errx.BadRequest, "sample.count must be at least 1") + } + case models.PlacementSamplePercent: + if sp.Percent < 1 || sp.Percent > 100 { + return errx.New(errx.BadRequest, "sample.percent must be between 1 and 100") + } + default: + return errx.New(errx.BadRequest, "sample.mode must be all, random, percent, per_domain or per_provider") + } + switch sp.Stratify { + case "", "provider", "domain": + default: + return errx.New(errx.BadRequest, "sample.stratify must be provider or domain") + } + if sp.Stratify != "" && sp.Mode != models.PlacementSampleRandom && sp.Mode != models.PlacementSamplePercent { + return errx.New(errx.BadRequest, "sample.stratify applies to a random or percent sample") + } + return nil +} + +// sampleSenders keeps part of the candidates. A stratified sample splits its +// size across providers or domains in proportion to each one's share, largest +// remainder first, so a small group is not rounded away. +func sampleSenders(cands []batchCandidate, sp models.PlacementSample, rng *rand.Rand) []batchCandidate { + shuffled := slices.Clone(cands) + rng.Shuffle(len(shuffled), func(i, j int) { shuffled[i], shuffled[j] = shuffled[j], shuffled[i] }) + groupOf := func(c batchCandidate, by string) string { + if by == "domain" { + return c.domain + } + return c.family + } + switch sp.Mode { + case models.PlacementSampleRandom, models.PlacementSamplePercent: + n := min(sp.Count, len(shuffled)) + if sp.Mode == models.PlacementSamplePercent { + n = int(math.Ceil(float64(len(shuffled)) * float64(sp.Percent) / 100)) + } + if sp.Stratify == "" { + return shuffled[:n] + } + groups, keys := groupCandidates(shuffled, func(c batchCandidate) string { return groupOf(c, sp.Stratify) }) + sizes := make([]int, len(keys)) + for i, k := range keys { + sizes[i] = len(groups[k]) + } + quota := allocate(n, sizes) + var out []batchCandidate + for i, k := range keys { + out = append(out, groups[k][:quota[i]]...) + } + return out + case models.PlacementSamplePerDomain, models.PlacementSamplePerProvider: + by := "provider" + if sp.Mode == models.PlacementSamplePerDomain { + by = "domain" + } + groups, keys := groupCandidates(shuffled, func(c batchCandidate) string { return groupOf(c, by) }) + var out []batchCandidate + for _, k := range keys { + g := groups[k] + out = append(out, g[:min(sp.Count, len(g))]...) + } + return out + } + return shuffled +} + +// groupCandidates buckets candidates by key, keeping their order, and returns +// the keys sorted. +func groupCandidates(cands []batchCandidate, key func(batchCandidate) string) (map[string][]batchCandidate, []string) { + groups := map[string][]batchCandidate{} + var keys []string + for _, c := range cands { + k := key(c) + if _, ok := groups[k]; !ok { + keys = append(keys, k) + } + groups[k] = append(groups[k], c) + } + sort.Strings(keys) + return groups, keys +} + +// allocate splits n across groups of the given sizes in proportion, by the +// largest remainder, never giving a group more than it has. +func allocate(n int, sizes []int) []int { + total := 0 + for _, s := range sizes { + total += s + } + out := make([]int, len(sizes)) + if total == 0 || n <= 0 { + return out + } + n = min(n, total) + type rem struct { + i int + frac float64 + } + rems := make([]rem, len(sizes)) + given := 0 + for i, size := range sizes { + exact := float64(n) * float64(size) / float64(total) + out[i] = int(math.Floor(exact)) + given += out[i] + rems[i] = rem{i, exact - float64(out[i])} + } + sort.SliceStable(rems, func(a, b int) bool { + if rems[a].frac != rems[b].frac { + return rems[a].frac > rems[b].frac + } + return sizes[rems[a].i] > sizes[rems[b].i] + }) + for k := 0; given < n; k = (k + 1) % len(rems) { + i := rems[k].i + if out[i] < sizes[i] { + out[i]++ + given++ + } + } + return out +} + +// staggerSenders orders a batch so consecutive starts rotate across sending +// providers, and within a provider across domains, rather than running one +// provider's mailboxes back to back. +func staggerSenders(cands []batchCandidate, rng *rand.Rand) []batchCandidate { + shuffled := slices.Clone(cands) + rng.Shuffle(len(shuffled), func(i, j int) { shuffled[i], shuffled[j] = shuffled[j], shuffled[i] }) + byFamily, families := groupCandidates(shuffled, func(c batchCandidate) string { return c.family }) + queues := make([][]batchCandidate, len(families)) + for i, f := range families { + queues[i] = roundRobin(groupCandidates(byFamily[f], func(c batchCandidate) string { return c.domain })) + } + out := make([]batchCandidate, 0, len(cands)) + for len(out) < len(cands) { + for i := range queues { + if len(queues[i]) > 0 { + out = append(out, queues[i][0]) + queues[i] = queues[i][1:] + } + } + } + return out +} + +func roundRobin(groups map[string][]batchCandidate, keys []string) []batchCandidate { + var out []batchCandidate + for { + added := false + for _, k := range keys { + if len(groups[k]) > 0 { + out = append(out, groups[k][0]) + groups[k] = groups[k][1:] + added = true + } + } + if !added { + return out + } + } +} + +// batchCost is what a batch of tests costs against the monthly allowance. +type batchCost struct { + metered bool + usage models.PlacementUsage + free int + paid int + credits int +} + +func (s *service) batchCost(ctx context.Context, orgID uuid.UUID, panel string, tests int) (batchCost, *errx.Error) { + var c batchCost + switch panel { + case models.PlacementPanelInstance: + usage, xerr := s.usage(ctx, orgID) + if xerr != nil { + return c, xerr + } + c.usage = usage + case models.PlacementPanelCloud: + if s.Cloud != nil { + if panel, xerr := s.Cloud.PlacementPanel(ctx); xerr == nil && panel != nil { + c.usage = panel.Usage + } + } + default: + return c, nil + } + if c.usage.Limit == nil { + return c, nil + } + c.metered = true + c.free = min(tests, c.usage.Remaining()) + c.paid = tests - c.free + c.credits = c.paid * c.usage.CreditsPerTest + return c, nil +} + +// planBatch validates a request and resolves the senders it would run. +func (s *service) planBatch(ctx context.Context, in *BatchInput, requireCopy bool) ([]batchCandidate, int, []variant, *errx.Error) { + if xerr := s.batchesReady(); xerr != nil { + return nil, 0, nil, xerr + } + if xerr := validateSample(&in.Sample); xerr != nil { + return nil, 0, nil, xerr + } + variants, xerr := s.validateBatch(ctx, in, requireCopy) + if xerr != nil { + return nil, 0, nil, xerr + } + cands, xerr := s.resolveSenders(ctx, *in) + if xerr != nil { + return nil, 0, nil, xerr + } + matched := len(cands) + return sampleSenders(cands, in.Sample, rand.New(rand.NewPCG(rand.Uint64(), rand.Uint64()))), matched, variants, nil +} + +// PreviewBatch reports how many senders, tests, sends and credits a batch +// request comes to, without starting anything. The copy may be left out. +func (s *service) PreviewBatch(ctx context.Context, in BatchInput) (*BatchPreview, *errx.Error) { + selected, matched, variants, xerr := s.planBatch(ctx, &in, false) + if xerr != nil { + return nil, xerr + } + pol := s.policy(ctx) + p := &BatchPreview{ + Matched: matched, + Selected: len(selected), + Variants: len(variants), + Tests: len(selected) * len(variants), + SeedsPerTest: pol.SeedsPerTest, + SendersMax: pol.BatchSendersMax, + Concurrency: pol.BatchSenderConcurrency, + Providers: []BatchGroupCount{}, + } + perTest := pol.SeedsPerTest + if len(in.SeedIDs) > 0 { + perTest = len(in.SeedIDs) + } + p.SeedsPerTest = perTest + p.MaxSends = p.Tests * perTest + families := map[string]int{} + domains := map[string]bool{} + for _, c := range selected { + families[c.family]++ + domains[c.domain] = true + if c.Status != "active" || c.WorkerID == nil { + p.Inactive++ + } + } + p.Domains = len(domains) + for f, n := range families { + p.Providers = append(p.Providers, BatchGroupCount{Key: f, Label: familyLabel(f), Senders: n}) + } + sort.Slice(p.Providers, func(i, j int) bool { return p.Providers[i].Senders > p.Providers[j].Senders }) + cost, xerr := s.batchCost(ctx, in.OrgID, in.Panel, p.Tests) + if xerr != nil { + return nil, xerr + } + p.Metered, p.Usage, p.FreeTests, p.PaidTests, p.Credits = cost.metered, cost.usage, cost.free, cost.paid, cost.credits + return p, nil +} + +// CreateBatch snapshots the senders and queues the batch. Nothing is sent +// here; the runner starts senders a few at a time. +func (s *service) CreateBatch(ctx context.Context, in BatchInput) (*BatchView, *errx.Error) { + selected, matched, variants, xerr := s.planBatch(ctx, &in, true) + if xerr != nil { + return nil, xerr + } + pol := s.policy(ctx) + if len(selected) == 0 { + return nil, placementErr(errx.BadRequest, "placement_batch_empty", "No sending mailbox matches this selection.") + } + if len(selected) > pol.BatchSendersMax { + return nil, placementErr(errx.BadRequest, "placement_batch_too_large", + "This batch has "+strconv.Itoa(len(selected))+" senders and this instance allows up to "+ + strconv.Itoa(pol.BatchSendersMax)+" in one batch. Narrow the selection or take a sample.") + } + open, err := s.Batches.CountOpenBatches(ctx, in.OrgID) + if err != nil { + errs.CaptureException(err) + return nil, errx.InternalError() + } + if open >= config.PlacementBatchOpenPerOrgMax { + return nil, placementErr(errx.TooManyRequests, "placement_too_many_batches", + "This workspace already has "+strconv.Itoa(open)+" placement batches running. Wait for one to finish, or cancel one.") + } + + tests := len(selected) * len(variants) + cost, xerr := s.batchCost(ctx, in.OrgID, in.Panel, tests) + if xerr != nil { + return nil, xerr + } + if cost.paid > 0 { + if cost.usage.CreditsPerTest == 0 { + return nil, placementErr(errx.PaymentRequired, "placement_quota_exceeded", + "This batch needs "+strconv.Itoa(tests)+" tests and the workspace has "+strconv.Itoa(cost.free)+ + " free this month. Take a smaller sample or test on your own seed inboxes.") + } + if in.MaxCredits < cost.credits { + return nil, placementErr(errx.PaymentRequired, "placement_quota_exceeded", + "This batch can cost up to "+strconv.Itoa(cost.credits)+" credits past the free tests; start it again agreeing to pay that many.") + } + if bal := cost.usage.CreditBalance; bal != nil && *bal < cost.credits { + return nil, placementErr(errx.PaymentRequired, "insufficient_credits", + "This batch can cost up to "+strconv.Itoa(cost.credits)+" credits and the workspace has "+strconv.Itoa(*bal)+".") + } + } else { + // Nothing agreed is ever charged when the batch fits the free tests. + in.MaxCredits = 0 + } + + now := s.now() + ordered := staggerSenders(selected, rand.New(rand.NewPCG(rand.Uint64(), rand.Uint64()))) + b := models.PlacementBatch{ + ID: uuid.New(), + OrganizationID: in.OrgID, + CreatedBy: in.UserID, + CampaignID: in.CampaignID, + SequenceID: in.SequenceID, + ContactID: in.ContactID, + Subject: in.Subject, + BodyHTML: in.BodyHTML, + BodyPlain: in.BodyPlain, + Tracking: in.Tracking, + Panel: in.Panel, + Pace: in.Pace, + Families: in.Families, + SeedIDs: in.SeedIDs, + OnUnavailable: in.OnUnavailable, + Selection: models.PlacementBatchSelection{ + SenderAccountIDs: len(in.SenderAccountIDs), + Scope: in.Scope, + Sample: in.Sample, + Matched: matched, + }, + SenderCount: len(ordered), + MaxCredits: in.MaxCredits, + Status: models.PlacementBatchQueued, + RetryUntil: now.AddDate(0, 0, config.PlacementBatchRetryDays), + } + senders := make([]models.PlacementBatchSender, len(ordered)) + for i, c := range ordered { + id := c.ID + senders[i] = models.PlacementBatchSender{ + ID: uuid.New(), + EmailAccountID: &id, + SenderEmail: c.SendFrom(), + SenderDomain: c.domain, + SenderFamily: c.family, + Position: i, + Status: models.PlacementSenderQueued, + } + } + if err := s.Batches.CreateBatch(ctx, &b, senders); err != nil { + errs.CaptureException(err) + return nil, errx.InternalError() + } + s.publishBatch(ctx, &b) + v := BatchView{PlacementBatch: b, Summary: models.PlacementCounts{}} + v.Progress.Add(models.PlacementSenderQueued, len(senders)) + v.Summary.Finish() + stripBody(&v.PlacementBatch) + return &v, nil +} + +func stripBody(b *models.PlacementBatch) { + b.BodyHTML, b.BodyPlain = "", "" +} + +func (s *service) publishBatch(ctx context.Context, b *models.PlacementBatch) { + if s.Publisher != nil { + s.Publisher.PublishPlacementBatch(ctx, b.OrganizationID, b.ID, b.Status) + } +} + +// ListBatches lists a workspace's batches, newest first. +func (s *service) ListBatches(ctx context.Context, orgID uuid.UUID, limit, offset int) ([]BatchView, int, *errx.Error) { + if xerr := s.batchesReady(); xerr != nil { + return nil, 0, xerr + } + batches, total, err := s.Batches.ListBatches(ctx, orgID, limit, offset) + if err != nil { + errs.CaptureException(err) + return nil, 0, errx.InternalError() + } + views, xerr := s.batchViews(ctx, batches) + if xerr != nil { + return nil, 0, xerr + } + return views, total, nil +} + +func (s *service) batchViews(ctx context.Context, batches []models.PlacementBatch) ([]BatchView, *errx.Error) { + ids := make([]uuid.UUID, len(batches)) + for i, b := range batches { + ids[i] = b.ID + } + progress, err := s.Batches.BatchProgress(ctx, ids) + if err != nil { + errs.CaptureException(err) + return nil, errx.InternalError() + } + summaries, err := s.Batches.BatchSummaries(ctx, ids) + if err != nil { + errs.CaptureException(err) + return nil, errx.InternalError() + } + out := make([]BatchView, 0, len(batches)) + for _, b := range batches { + stripBody(&b) + sum := summaries[b.ID] + sum.Finish() + out = append(out, BatchView{PlacementBatch: b, Progress: progress[b.ID], Summary: sum}) + } + return out, nil +} + +// GetBatch is one batch with its placement overall, by sending domain and +// provider, by recipient provider, and as a domain by recipient matrix. +func (s *service) GetBatch(ctx context.Context, orgID, id uuid.UUID) (*BatchDetail, *errx.Error) { + if xerr := s.batchesReady(); xerr != nil { + return nil, xerr + } + b, err := s.Batches.GetBatch(ctx, orgID, id) + if err != nil { + errs.CaptureException(err) + return nil, errx.InternalError() + } + if b == nil { + return nil, errx.New(errx.NotFound, "placement batch not found") + } + content := contentCheck(models.PlacementTest{Subject: b.Subject, BodyHTML: b.BodyHTML, BodyPlain: b.BodyPlain}) + views, xerr := s.batchViews(ctx, []models.PlacementBatch{*b}) + if xerr != nil { + return nil, xerr + } + rows, err := s.Batches.BatchBreakdown(ctx, id) + if err != nil { + errs.CaptureException(err) + return nil, errx.InternalError() + } + sizes, err := s.Batches.BatchGroupSizes(ctx, id) + if err != nil { + errs.CaptureException(err) + return nil, errx.InternalError() + } + d := buildBatchDetail(views[0], rows, sizes) + d.Content = content + return d, nil +} + +// buildBatchDetail folds the breakdown rows into the detail's groups. +func buildBatchDetail(v BatchView, rows []repository.PlacementBreakdownRow, sizes []repository.PlacementBatchGroupSize) *BatchDetail { + d := &BatchDetail{BatchView: v, Domains: []BatchGroup{}, Providers: []BatchGroup{}, Recipients: []models.PlacementFamilyCounts{}, Matrix: []BatchMatrixRow{}} + compare := v.Tracking == models.PlacementTrackingCompare + var headline, untracked models.PlacementCounts + domains := map[string]*BatchGroup{} + providers := map[string]*BatchGroup{} + recipients := map[string]*models.PlacementCounts{} + matrix := map[string]map[string]*models.PlacementCounts{} + group := func(m map[string]*BatchGroup, key, label string) *BatchGroup { + g, ok := m[key] + if !ok { + g = &BatchGroup{Key: key, Label: label} + m[key] = g + } + return g + } + for _, r := range rows { + switch r.Set { + case "overall": + if compare && !r.Tracked { + untracked.AddN(r.Folder, r.Count) + } else { + headline.AddN(r.Folder, r.Count) + } + case "domain": + group(domains, r.SenderDomain, r.SenderDomain).Counts.AddN(r.Folder, r.Count) + case "provider": + group(providers, r.SenderFamily, familyLabel(r.SenderFamily)).Counts.AddN(r.Folder, r.Count) + case "recipient": + c, ok := recipients[r.RecipientFamily] + if !ok { + c = &models.PlacementCounts{} + recipients[r.RecipientFamily] = c + } + c.AddN(r.Folder, r.Count) + case "matrix": + row, ok := matrix[r.SenderDomain] + if !ok { + row = map[string]*models.PlacementCounts{} + matrix[r.SenderDomain] = row + } + c, ok := row[r.RecipientFamily] + if !ok { + c = &models.PlacementCounts{} + row[r.RecipientFamily] = c + } + c.AddN(r.Folder, r.Count) + } + } + for _, sz := range sizes { + if sz.ByDomain { + g := group(domains, sz.Domain, sz.Domain) + g.Senders, g.Tested = sz.Senders, sz.Completed + } else { + g := group(providers, sz.Family, familyLabel(sz.Family)) + g.Senders, g.Tested = sz.Senders, sz.Completed + } + } + headline.Finish() + d.Summary = headline + if compare { + untracked.Finish() + d.Untracked = &untracked + } + flatten := func(m map[string]*BatchGroup) []BatchGroup { + out := make([]BatchGroup, 0, len(m)) + for _, g := range m { + g.Counts.Finish() + out = append(out, *g) + } + sort.Slice(out, func(i, j int) bool { return worseGroup(out[i], out[j]) }) + return out + } + d.Domains = flatten(domains) + d.Providers = flatten(providers) + for fam, c := range recipients { + c.Finish() + d.Recipients = append(d.Recipients, models.PlacementFamilyCounts{Family: fam, Label: familyLabel(fam), Counts: *c}) + } + sort.Slice(d.Recipients, func(i, j int) bool { return d.Recipients[i].Label < d.Recipients[j].Label }) + for _, g := range d.Domains { + row, ok := matrix[g.Key] + if !ok { + continue + } + mr := BatchMatrixRow{Domain: g.Key, Recipients: make([]models.PlacementFamilyCounts, 0, len(d.Recipients))} + for _, rc := range d.Recipients { + c := models.PlacementCounts{} + if got, ok := row[rc.Family]; ok { + c = *got + } + c.Finish() + mr.Recipients = append(mr.Recipients, models.PlacementFamilyCounts{Family: rc.Family, Label: rc.Label, Counts: c}) + } + d.Matrix = append(d.Matrix, mr) + } + return d +} + +// worseGroup orders groups lowest inbox rate first, untested last, then the +// larger group first. +func worseGroup(a, b BatchGroup) bool { + ra, rb := a.Counts.InboxRate, b.Counts.InboxRate + switch { + case ra != nil && rb == nil: + return true + case ra == nil && rb != nil: + return false + case ra != nil && rb != nil && *ra != *rb: + return *ra < *rb + } + if a.Senders != b.Senders { + return a.Senders > b.Senders + } + return a.Key < b.Key +} + +// ListBatchSenders lists a batch's senders with where each one's copies +// landed, worst inbox rate first by default. +func (s *service) ListBatchSenders(ctx context.Context, orgID, id uuid.UUID, f repository.PlacementBatchSenderFilter) ([]BatchSenderView, int, *errx.Error) { + if xerr := s.batchesReady(); xerr != nil { + return nil, 0, xerr + } + switch f.Sort { + case "", "worst", "best", "email", "status": + default: + return nil, 0, errx.New(errx.BadRequest, "sort must be worst, best, email or status") + } + switch f.Status { + case "", models.PlacementSenderQueued, models.PlacementSenderDeferred, models.PlacementSenderRunning, + models.PlacementSenderCompleted, models.PlacementSenderSkipped, models.PlacementSenderFailed, models.PlacementSenderCancelled: + default: + return nil, 0, errx.New(errx.BadRequest, "invalid status") + } + if len(f.Search) > 200 { + return nil, 0, errx.New(errx.BadRequest, "search is too long") + } + b, err := s.Batches.GetBatch(ctx, orgID, id) + if err != nil { + errs.CaptureException(err) + return nil, 0, errx.InternalError() + } + if b == nil { + return nil, 0, errx.New(errx.NotFound, "placement batch not found") + } + rows, total, err := s.Batches.ListBatchSenders(ctx, orgID, id, f) + if err != nil { + errs.CaptureException(err) + return nil, 0, errx.InternalError() + } + out := make([]BatchSenderView, 0, len(rows)) + for _, r := range rows { + out = append(out, BatchSenderView{ + PlacementBatchSender: r.PlacementBatchSender, + SenderFamilyLabel: familyLabel(r.SenderFamily), + Summary: r.Counts, + TestIDs: r.TestIDs, + }) + } + return out, total, nil +} + +// CancelBatch stops a batch: no sender starts again, copies not sent yet are +// cancelled, and copies already sent keep being classified. +func (s *service) CancelBatch(ctx context.Context, orgID, id uuid.UUID) (*BatchView, *errx.Error) { + if xerr := s.batchesReady(); xerr != nil { + return nil, xerr + } + ok, running, err := s.Batches.CancelBatch(ctx, orgID, id) + if err != nil { + errs.CaptureException(err) + return nil, errx.InternalError() + } + for _, testID := range running { + if _, err := s.Repo.CancelTest(ctx, orgID, testID); err != nil { + errs.CaptureException(err) + } + } + b, err := s.Batches.GetBatch(ctx, orgID, id) + if err != nil { + errs.CaptureException(err) + return nil, errx.InternalError() + } + if b == nil { + return nil, errx.New(errx.NotFound, "placement batch not found") + } + if !ok { + return nil, placementErr(errx.Conflict, "placement_batch_not_running", "This placement batch is no longer running.") + } + s.publishBatch(ctx, b) + views, xerr := s.batchViews(ctx, []models.PlacementBatch{*b}) + if xerr != nil { + return nil, xerr + } + return &views[0], nil +} + +// Coverage is how much of the workspace's connected fleet delivered a +// placement test recently. +func (s *service) Coverage(ctx context.Context, orgID uuid.UUID) (*repository.PlacementCoverage, *errx.Error) { + if xerr := s.batchesReady(); xerr != nil { + return nil, xerr + } + c, err := s.Batches.Coverage(ctx, orgID, s.now()) + if err != nil { + errs.CaptureException(err) + return nil, errx.InternalError() + } + return &c, nil +} diff --git a/internal/app/placement/batch_runner.go b/internal/app/placement/batch_runner.go new file mode 100644 index 000000000..92552c85d --- /dev/null +++ b/internal/app/placement/batch_runner.go @@ -0,0 +1,322 @@ +package placement + +import ( + "context" + "fmt" + "math/rand/v2" + "time" + + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/config" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/observability/errs" +) + +// senderOutcome is what one attempt to start a batch sender came to. +type senderOutcome int + +const ( + outcomeStarted senderOutcome = iota + // outcomeNotStarted: skipped or deferred, the batch goes on. + outcomeNotStarted + // outcomeStopBatch: no other sender could start either (the allowance or + // the credits ran out, the campaign went away). + outcomeStopBatch +) + +// runBatches advances every active batch: resolve senders whose tests +// finished, start the next due ones within the workspace's concurrency and the +// start rate, and close a batch with nothing left to do. +func (s *service) runBatches(ctx context.Context) { + if s.Batches == nil { + return + } + if err := s.Batches.CloseInactiveBatches(ctx); err != nil { + errs.CaptureException(err) + } + s.settleCancelledBatches(ctx) + now := s.now() + batches, err := s.Batches.ClaimBatches(ctx, now, 2*time.Minute, config.PlacementBatchRunnerBatchesPerTick) + if err != nil { + errs.CaptureException(err) + return + } + pol := s.policy(ctx) + instance, err := s.Batches.CountSendingBatchSenders(ctx, nil) + if err != nil { + errs.CaptureException(err) + return + } + pace := batchPace{ + orgLimit: pol.BatchSenderConcurrency, instanceLimit: pol.BatchInstanceConcurrency, + perMinute: pol.BatchStartsPerMinute, instance: instance, sending: map[uuid.UUID]int{}, + } + for i := range batches { + b := &batches[i] + s.advanceBatch(ctx, b, &pace, now) + if err := s.Batches.ReleaseBatch(ctx, b.ID); err != nil { + errs.CaptureException(err) + } + } +} + +// settleCancelledBatches finishes what a cancel leaves behind: a test a sender +// started while the batch was being cancelled is cancelled too, and senders +// whose tests have since finished are resolved. +func (s *service) settleCancelledBatches(ctx context.Context) { + late, err := s.Batches.RunningTestsOfCancelledBatches(ctx, 100) + if err != nil { + errs.CaptureException(err) + } + for _, t := range late { + if _, err := s.Repo.CancelTest(ctx, t.OrganizationID, t.TestID); err != nil { + errs.CaptureException(err) + } + } + if err := s.Batches.SyncClosedBatchSenders(ctx); err != nil { + errs.CaptureException(err) + } +} + +// batchPace is one runner pass's view of how many batch senders are sending, +// per workspace and across the instance, against their limits. +type batchPace struct { + orgLimit, instanceLimit, perMinute int + instance int + sending map[uuid.UUID]int +} + +func (s *service) advanceBatch(ctx context.Context, b *models.PlacementBatch, pace *batchPace, now time.Time) { + stale := now.Add(-time.Duration(config.PlacementBatchSenderStaleMinutes) * time.Minute) + if err := s.Batches.SyncBatchSenders(ctx, b.ID, stale); err != nil { + errs.CaptureException(err) + return + } + if now.After(b.RetryUntil) { + if err := s.Batches.CloseOpenBatchSenders(ctx, b.ID, []string{models.PlacementSenderDeferred}, models.PlacementSenderSkipped, + "placement_batch_retry_expired", "Still could not run when the batch's retry window closed."); err != nil { + errs.CaptureException(err) + } + } + + if _, ok := pace.sending[b.OrganizationID]; !ok { + orgID := b.OrganizationID + n, err := s.Batches.CountSendingBatchSenders(ctx, &orgID) + if err != nil { + errs.CaptureException(err) + return + } + pace.sending[b.OrganizationID] = n + } + slots := min(pace.orgLimit-pace.sending[b.OrganizationID], pace.instanceLimit-pace.instance, + startAllowance(pace.perMinute, b.LastTickAt, now)) + moved := false + if slots > 0 { + due, err := s.Batches.DueBatchSenders(ctx, b.ID, now, slots*4+10) + if err != nil { + errs.CaptureException(err) + return + } + started := 0 + for _, snd := range due { + if started >= slots { + break + } + outcome, reason, detail := s.startBatchSender(ctx, b, snd, now) + moved = true + if outcome == outcomeStarted { + started++ + pace.sending[b.OrganizationID]++ + pace.instance++ + continue + } + if outcome == outcomeStopBatch { + if err := s.Batches.CloseOpenBatchSenders(ctx, b.ID, + []string{models.PlacementSenderQueued, models.PlacementSenderDeferred}, + models.PlacementSenderSkipped, reason, detail); err != nil { + errs.CaptureException(err) + } + break + } + } + if started > 0 && b.Status == models.PlacementBatchQueued { + if err := s.Batches.MarkBatchStarted(ctx, b.ID); err != nil { + errs.CaptureException(err) + } + b.Status = models.PlacementBatchRunning + } + } + + progress, err := s.Batches.BatchProgress(ctx, []uuid.UUID{b.ID}) + if err != nil { + errs.CaptureException(err) + return + } + p := progress[b.ID] + if p.Open() > 0 { + if moved { + s.publishBatch(ctx, b) + } + return + } + s.finishBatch(ctx, b, p) +} + +// startAllowance is how many senders a batch may start this pass: its rate +// over the time since it was last advanced, at most a minute's worth, and at +// least one so a slow rate still moves. +func startAllowance(perMinute int, last *time.Time, now time.Time) int { + if perMinute <= 0 { + return 0 + } + elapsed := time.Minute + if last != nil { + elapsed = min(max(now.Sub(*last), 0), time.Minute) + } + return max(1, int(float64(perMinute)*elapsed.Seconds()/60)) +} + +// startBatchSender starts one sender's test, or records why it did not start. +// The sender is claimed first, so a restart between the claim and the test +// can never start it twice; SyncBatchSenders returns a claim with no test. +func (s *service) startBatchSender(ctx context.Context, b *models.PlacementBatch, snd models.PlacementBatchSender, now time.Time) (senderOutcome, string, string) { + record := func(status, reason, detail string, next time.Time) (senderOutcome, string, string) { + if err := s.Batches.SetBatchSenderOutcome(ctx, snd.ID, status, reason, detail, next); err != nil { + errs.CaptureException(err) + } + return outcomeNotStarted, reason, detail + } + if snd.EmailAccountID == nil { + return record(models.PlacementSenderSkipped, "placement_sender_deleted", "The mailbox was removed from the workspace.", now) + } + // The mailbox is checked here so a not-found from the test below means + // the batch's campaign or step went away, which stops every sender. + acct, xerr := s.Emails.GetByID(ctx, *snd.EmailAccountID) + if xerr != nil || acct == nil || acct.OrganizationID == nil || *acct.OrganizationID != b.OrganizationID { + return record(models.PlacementSenderSkipped, "placement_sender_deleted", "The mailbox was removed from the workspace.", now) + } + ok, err := s.Batches.ClaimBatchSender(ctx, snd.ID) + if err != nil { + errs.CaptureException(err) + return outcomeNotStarted, "", "" + } + if !ok { + return outcomeNotStarted, "", "" + } + batchID, senderID := b.ID, snd.ID + views, xerr := s.CreateTests(ctx, CreateInput{ + OrgID: b.OrganizationID, + UserID: b.CreatedBy, + SenderAccountID: *snd.EmailAccountID, + CampaignID: b.CampaignID, + SequenceID: b.SequenceID, + ContactID: b.ContactID, + Subject: b.Subject, + BodyHTML: b.BodyHTML, + BodyPlain: b.BodyPlain, + Tracking: b.Tracking, + Panel: b.Panel, + SeedIDs: b.SeedIDs, + Families: b.Families, + Pace: models.PlacementPaceSpaced, + MaxCredits: max(0, b.MaxCredits-b.CreditsSpent), + Origin: models.PlacementOriginBatch, + BatchID: &batchID, + BatchSenderID: &senderID, + }) + if xerr == nil { + charged := 0 + for _, v := range views { + charged += v.CreditsCharged + } + if charged > 0 { + b.CreditsSpent += charged + if err := s.Batches.AddBatchCredits(ctx, b.ID, charged); err != nil { + errs.CaptureException(err) + } + } + return outcomeStarted, "", "" + } + + retry := b.OnUnavailable == models.PlacementUnavailableDefer + unavailable := func(next time.Time) (senderOutcome, string, string) { + if retry { + return record(models.PlacementSenderDeferred, xerr.Identifier, xerr.Message, next) + } + return record(models.PlacementSenderSkipped, xerr.Identifier, xerr.Message, now) + } + switch xerr.Identifier { + case "placement_sender_busy": + // Another test is still sending from it; that is minutes, not a day. + return record(models.PlacementSenderDeferred, xerr.Identifier, xerr.Message, now.Add(15*time.Minute)) + case "placement_daily_budget": + return unavailable(nextSendingDay(now)) + case "placement_sender_unavailable", "placement_invalid_seeds": + return unavailable(now.Add(time.Hour)) + case "placement_no_seeds": + // Every seed is on this sender's own domain; no retry changes that. + return record(models.PlacementSenderSkipped, xerr.Identifier, xerr.Message, now) + case "placement_quota_exceeded", "insufficient_credits", "usage_cap_exceeded", "placement_not_entitled", + "placement_panel_unavailable", "placement_invalid_tracking": + record(models.PlacementSenderSkipped, xerr.Identifier, xerr.Message, now) + return outcomeStopBatch, xerr.Identifier, xerr.Message + } + switch xerr.Code { + case errx.NotFound, errx.BadRequest: + record(models.PlacementSenderSkipped, "placement_batch_copy_unavailable", xerr.Message, now) + return outcomeStopBatch, "placement_batch_copy_unavailable", "The batch's campaign or email is no longer available: " + xerr.Message + } + // Anything else is unexpected: try again later, then give up on the sender. + if snd.Attempts+1 >= config.PlacementBatchSenderErrorAttemptsMax { + if err := s.Batches.SetBatchSenderOutcome(ctx, snd.ID, models.PlacementSenderFailed, "placement_batch_start_failed", + "The test could not be started.", now); err != nil { + errs.CaptureException(err) + } + return outcomeNotStarted, "", "" + } + return record(models.PlacementSenderDeferred, "placement_batch_start_failed", "The test could not be started; it is retried shortly.", + now.Add(time.Duration(snd.Attempts+1)*10*time.Minute)) +} + +// nextSendingDay is a moment early in the next UTC day, when a mailbox's daily +// count starts over, spread over two hours so a deferred fleet does not all +// start at midnight. +func nextSendingDay(now time.Time) time.Time { + day := now.UTC().Truncate(24 * time.Hour).Add(24 * time.Hour) + return day.Add(time.Duration(rand.Int64N(int64(2 * time.Hour)))) +} + +// finishBatch closes a batch with nothing left to run and tells its creator. +func (s *service) finishBatch(ctx context.Context, b *models.PlacementBatch, p models.PlacementBatchProgress) { + status, msg := models.PlacementBatchCompleted, "" + switch { + case p.Completed == 0: + status, msg = models.PlacementBatchFailed, "No sender finished a test." + case p.Skipped+p.Failed+p.Cancelled > 0: + status = models.PlacementBatchCompletedWithWarnings + } + ok, err := s.Batches.FinishBatch(ctx, b.ID, status, msg) + if err != nil { + errs.CaptureException(err) + return + } + if !ok { + return + } + b.Status = status + s.publishBatch(ctx, b) + if b.CreatedBy == nil || s.Notifier == nil { + return + } + body := fmt.Sprintf("%d of %d mailboxes finished a test.", p.Completed, p.Total) + if sums, err := s.Batches.BatchSummaries(ctx, []uuid.UUID{b.ID}); err == nil { + c := sums[b.ID] + c.Finish() + body += " " + summaryLine(c) + } + orgID := b.OrganizationID + s.Notifier.Notify(ctx, *b.CreatedBy, &orgID, models.NotifPlacementFinished, "Placement batch finished", body, + "/app/placement/batches/"+b.ID.String(), map[string]any{"placement_batch_id": b.ID.String()}) +} diff --git a/internal/app/placement/batch_test.go b/internal/app/placement/batch_test.go new file mode 100644 index 000000000..18c848edb --- /dev/null +++ b/internal/app/placement/batch_test.go @@ -0,0 +1,548 @@ +package placement + +import ( + "context" + "fmt" + "math/rand/v2" + "slices" + "sort" + "testing" + "time" + + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +type fakeBatches struct { + repository.PlacementBatchRepository + cands []repository.PlacementBatchCandidate + candOrg map[uuid.UUID]uuid.UUID + batches map[uuid.UUID]*models.PlacementBatch + senders []*models.PlacementBatchSender + open int + credits int + finished []string +} + +func newFakeBatches() *fakeBatches { + return &fakeBatches{candOrg: map[uuid.UUID]uuid.UUID{}, batches: map[uuid.UUID]*models.PlacementBatch{}} +} + +func (f *fakeBatches) ListBatchCandidates(_ context.Context, orgID uuid.UUID, fl repository.PlacementCandidateFilter) ([]repository.PlacementBatchCandidate, error) { + var out []repository.PlacementBatchCandidate + for _, c := range f.cands { + if f.candOrg[c.ID] != orgID { + continue + } + if fl.IDs != nil && !slices.Contains(fl.IDs, c.ID) { + continue + } + if !fl.IncludeInactive && c.Status != "active" { + continue + } + out = append(out, c) + } + return out, nil +} +func (f *fakeBatches) CountOpenBatches(context.Context, uuid.UUID) (int, error) { return f.open, nil } +func (f *fakeBatches) CreateBatch(_ context.Context, b *models.PlacementBatch, senders []models.PlacementBatchSender) error { + b.Active = true + cp := *b + f.batches[b.ID] = &cp + for i := range senders { + s := senders[i] + s.BatchID = b.ID + f.senders = append(f.senders, &s) + } + return nil +} +func (f *fakeBatches) CloseInactiveBatches(context.Context) error { return nil } +func (f *fakeBatches) SyncClosedBatchSenders(context.Context) error { return nil } +func (f *fakeBatches) RunningTestsOfCancelledBatches(context.Context, int) ([]repository.PlacementOrgTest, error) { + return nil, nil +} +func (f *fakeBatches) ClaimBatches(context.Context, time.Time, time.Duration, int) ([]models.PlacementBatch, error) { + var out []models.PlacementBatch + for _, b := range f.batches { + if b.Active { + out = append(out, *b) + } + } + return out, nil +} +func (f *fakeBatches) ReleaseBatch(context.Context, uuid.UUID) error { return nil } +func (f *fakeBatches) SyncBatchSenders(context.Context, uuid.UUID, time.Time) error { return nil } +func (f *fakeBatches) CountSendingBatchSenders(context.Context, *uuid.UUID) (int, error) { + n := 0 + for _, s := range f.senders { + if s.Status == models.PlacementSenderRunning { + n++ + } + } + return n, nil +} +func (f *fakeBatches) DueBatchSenders(_ context.Context, batchID uuid.UUID, now time.Time, limit int) ([]models.PlacementBatchSender, error) { + var out []models.PlacementBatchSender + for _, s := range f.senders { + if s.BatchID == batchID && (s.Status == models.PlacementSenderQueued || s.Status == models.PlacementSenderDeferred) && !s.NextAttemptAt.After(now) { + out = append(out, *s) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Position < out[j].Position }) + return out[:min(limit, len(out))], nil +} +func (f *fakeBatches) sender(id uuid.UUID) *models.PlacementBatchSender { + for _, s := range f.senders { + if s.ID == id { + return s + } + } + return nil +} +func (f *fakeBatches) ClaimBatchSender(_ context.Context, id uuid.UUID) (bool, error) { + s := f.sender(id) + if s == nil || (s.Status != models.PlacementSenderQueued && s.Status != models.PlacementSenderDeferred) { + return false, nil + } + s.Status, s.Attempts = models.PlacementSenderRunning, s.Attempts+1 + return true, nil +} +func (f *fakeBatches) SetBatchSenderOutcome(_ context.Context, id uuid.UUID, status, reason, detail string, next time.Time) error { + s := f.sender(id) + s.Status, s.Reason, s.Detail, s.NextAttemptAt = status, reason, detail, next + return nil +} +func (f *fakeBatches) CloseOpenBatchSenders(_ context.Context, batchID uuid.UUID, from []string, status, reason, detail string) error { + for _, s := range f.senders { + if s.BatchID == batchID && slices.Contains(from, s.Status) { + s.Status, s.Reason, s.Detail = status, reason, detail + } + } + return nil +} +func (f *fakeBatches) AddBatchCredits(_ context.Context, _ uuid.UUID, n int) error { + f.credits += n + return nil +} +func (f *fakeBatches) MarkBatchStarted(_ context.Context, id uuid.UUID) error { + f.batches[id].Status = models.PlacementBatchRunning + return nil +} +func (f *fakeBatches) BatchProgress(_ context.Context, ids []uuid.UUID) (map[uuid.UUID]models.PlacementBatchProgress, error) { + out := map[uuid.UUID]models.PlacementBatchProgress{} + for _, s := range f.senders { + if slices.Contains(ids, s.BatchID) { + p := out[s.BatchID] + p.Add(s.Status, 1) + out[s.BatchID] = p + } + } + return out, nil +} +func (f *fakeBatches) BatchSummaries(context.Context, []uuid.UUID) (map[uuid.UUID]models.PlacementCounts, error) { + return map[uuid.UUID]models.PlacementCounts{}, nil +} +func (f *fakeBatches) FinishBatch(_ context.Context, id uuid.UUID, status, _ string) (bool, error) { + b := f.batches[id] + if !b.Active { + return false, nil + } + b.Active, b.Status = false, status + f.finished = append(f.finished, status) + return true, nil +} + +// fleetHarness is a harness whose workspace has n extra mailboxes spread +// over domains and providers, all on the batch store as candidates. +func fleetHarness(t *testing.T, n int) (*harness, *fakeBatches) { + t.Helper() + h := newHarness(t) + fb := newFakeBatches() + h.svc.Batches = fb + emails := h.svc.Emails.(*fakeEmails) + worker := uuid.New() + hosts := []string{"google_workspace", "google_workspace", "microsoft365", "other"} + for i := range n { + id := uuid.New() + host := hosts[i%len(hosts)] + addr := fmt.Sprintf("rep%d@domain%d.test", i, i%7) + emails.accounts[id] = &models.Email{ID: id, OrganizationID: &h.org, Email: addr, Status: "active", WorkerID: &worker, CampaignLimit: 50} + fb.cands = append(fb.cands, repository.PlacementBatchCandidate{ID: id, Email: addr, Provider: "smtp_imap", MailHost: host, Status: "active", WorkerID: &worker}) + fb.candOrg[id] = h.org + } + return h, fb +} + +func (h *harness) batchInput() BatchInput { + return BatchInput{OrgID: h.org, Scope: &models.PlacementSenderScope{Type: models.PlacementScopeWorkspace}, Subject: "Quick question", BodyPlain: "Hi there"} +} + +func TestAllocateSplitsBySizeByLargestRemainder(t *testing.T) { + got := allocate(135, []int{800, 400, 147}) + if !slices.Equal(got, []int{80, 40, 15}) { + t.Fatalf("allocate(135, 800/400/147) = %v; want 80/40/15", got) + } + got = allocate(3, []int{100, 1, 1}) + if got[0]+got[1]+got[2] != 3 || got[0] > 100 || got[1] > 1 || got[2] > 1 { + t.Fatalf("allocate(3, 100/1/1) = %v; want 3 in total within each size", got) + } + if got := allocate(10, []int{2, 3}); !slices.Equal(got, []int{2, 3}) { + t.Fatalf("allocate beyond the total = %v; want every member", got) + } +} + +func TestSampleSendersPercentIsStratifiedByProvider(t *testing.T) { + var cands []batchCandidate + add := func(n int, family string) { + for i := range n { + cands = append(cands, batchCandidate{family: family, domain: fmt.Sprintf("%s%d.test", family, i%9)}) + } + } + add(800, "google_workspace") + add(400, "other") + add(147, "microsoft365") + rng := rand.New(rand.NewPCG(1, 2)) + got := sampleSenders(cands, models.PlacementSample{Mode: models.PlacementSamplePercent, Percent: 10, Stratify: "provider"}, rng) + per := map[string]int{} + for _, c := range got { + per[c.family]++ + } + if len(got) != 135 || per["google_workspace"] != 80 || per["other"] != 40 || per["microsoft365"] != 15 { + t.Fatalf("10%% stratified sample = %d (%v); want 135 split 80/40/15", len(got), per) + } + if got := sampleSenders(cands, models.PlacementSample{Mode: models.PlacementSampleRandom, Count: 100}, rng); len(got) != 100 { + t.Fatalf("random 100 kept %d", len(got)) + } + if got := sampleSenders(cands, models.PlacementSample{Mode: models.PlacementSampleAll}, rng); len(got) != len(cands) { + t.Fatalf("all kept %d of %d", len(got), len(cands)) + } +} + +func TestSampleSendersPerDomainCapsEveryDomainAlike(t *testing.T) { + var cands []batchCandidate + for i := range 500 { + cands = append(cands, batchCandidate{family: "gmail", domain: "big.test", PlacementBatchCandidate: repository.PlacementBatchCandidate{Email: fmt.Sprint(i)}}) + } + for i := range 3 { + cands = append(cands, batchCandidate{family: "gmail", domain: "small.test", PlacementBatchCandidate: repository.PlacementBatchCandidate{Email: fmt.Sprint(i)}}) + } + got := sampleSenders(cands, models.PlacementSample{Mode: models.PlacementSamplePerDomain, Count: 5}, rand.New(rand.NewPCG(3, 4))) + per := map[string]int{} + for _, c := range got { + per[c.domain]++ + } + if per["big.test"] != 5 || per["small.test"] != 3 { + t.Fatalf("5 per domain = %v; want 5 from the large domain and all 3 of the small one", per) + } +} + +func TestStaggerSendersRotatesProviders(t *testing.T) { + var cands []batchCandidate + for i := range 30 { + cands = append(cands, batchCandidate{family: "google_workspace", domain: fmt.Sprintf("g%d.test", i%3)}) + } + for i := range 3 { + cands = append(cands, batchCandidate{family: "microsoft365", domain: fmt.Sprintf("m%d.test", i)}) + } + got := staggerSenders(cands, rand.New(rand.NewPCG(5, 6))) + if len(got) != len(cands) { + t.Fatalf("stagger kept %d of %d", len(got), len(cands)) + } + for i := range 6 { + want := "google_workspace" + if i%2 == 1 { + want = "microsoft365" + } + if got[i].family != want { + t.Fatalf("position %d is %s; want providers to alternate while both have senders", i, got[i].family) + } + } + if got[0].domain == got[2].domain { + t.Fatalf("consecutive google senders share %s; want domains to rotate", got[0].domain) + } +} + +func TestStartAllowance(t *testing.T) { + now := time.Now() + half := now.Add(-30 * time.Second) + if got := startAllowance(10, &half, now); got != 5 { + t.Fatalf("10/min over 30s = %d; want 5", got) + } + if got := startAllowance(1, &half, now); got != 1 { + t.Fatalf("a slow rate gave %d; want at least one", got) + } + long := now.Add(-time.Hour) + if got := startAllowance(10, &long, now); got != 10 { + t.Fatalf("an idle hour gave %d; want at most a minute's worth", got) + } +} + +func TestCreateBatchHoldsMoreSendersThanRunAtOnce(t *testing.T) { + h, fb := fleetHarness(t, 1300) + view, xerr := h.svc.CreateBatch(context.Background(), h.batchInput()) + if xerr != nil { + t.Fatalf("CreateBatch: %v", xerr) + } + if view.SenderCount != 1300 || len(fb.senders) != 1300 || view.Status != models.PlacementBatchQueued { + t.Fatalf("batch = %d senders (%d rows), %s; want 1300 queued", view.SenderCount, len(fb.senders), view.Status) + } + if len(h.repo.created) != 0 { + t.Fatalf("creating the batch started %d tests; want none until the runner", len(h.repo.created)) + } + pol := h.svc.policy(context.Background()) + pol.BatchSenderConcurrency, pol.BatchStartsPerMinute = 20, 600 + h.svc.Policy = fakePolicy{pol} + + h.svc.runBatches(context.Background()) + if len(h.repo.created) != 20 { + t.Fatalf("one pass started %d tests; want the concurrency of 20", len(h.repo.created)) + } + seen := map[uuid.UUID]bool{} + for _, test := range h.repo.created { + if test.BatchID == nil || *test.BatchID != view.ID || test.BatchSenderID == nil || test.Origin != models.PlacementOriginBatch { + t.Fatalf("test %+v is not tied to the batch", test) + } + if seen[*test.SenderAccountID] { + t.Fatalf("sender %s started twice", test.SenderAccountID) + } + seen[*test.SenderAccountID] = true + } + // The same 20 are still sending: nothing more starts. + h.svc.runBatches(context.Background()) + if len(h.repo.created) != 20 { + t.Fatalf("a second pass started %d tests while 20 were sending", len(h.repo.created)-20) + } +} + +func TestCreateBatchRefusals(t *testing.T) { + cases := []struct { + name string + setup func(*harness, *fakeBatches, *BatchInput) + wantID string + }{ + {"too many senders for the instance", func(h *harness, _ *fakeBatches, in *BatchInput) { + pol := h.svc.policy(context.Background()) + pol.BatchSendersMax = 10 + h.svc.Policy = fakePolicy{pol} + }, "placement_batch_too_large"}, + {"too many open batches", func(_ *harness, fb *fakeBatches, _ *BatchInput) { fb.open = 5 }, "placement_too_many_batches"}, + {"nothing matches", func(_ *harness, _ *fakeBatches, in *BatchInput) { in.Scope.Domains = []string{"nowhere.test"} }, "placement_batch_empty"}, + {"both ids and a scope", func(h *harness, _ *fakeBatches, in *BatchInput) { in.SenderAccountIDs = []uuid.UUID{h.sender} }, ""}, + {"another workspace's mailbox", func(_ *harness, _ *fakeBatches, in *BatchInput) { + in.Scope, in.SenderAccountIDs = nil, []uuid.UUID{uuid.New()} + }, ""}, + {"a stratified per-domain sample", func(_ *harness, _ *fakeBatches, in *BatchInput) { + in.Sample = models.PlacementSample{Mode: models.PlacementSamplePerDomain, Count: 2, Stratify: "provider"} + }, ""}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + h, fb := fleetHarness(t, 40) + in := h.batchInput() + c.setup(h, fb, &in) + _, xerr := h.svc.CreateBatch(context.Background(), in) + if xerr == nil { + t.Fatalf("CreateBatch succeeded; want a refusal") + } + if c.wantID != "" && xerr.Identifier != c.wantID { + t.Fatalf("refused with %q (%s); want %q", xerr.Identifier, xerr.Message, c.wantID) + } + if len(fb.batches) != 0 { + t.Fatalf("a refused batch was written") + } + }) + } +} + +func TestCreateBatchDedupesChosenSendersAndHonoursAKeysMailboxes(t *testing.T) { + h, fb := fleetHarness(t, 5) + ids := []uuid.UUID{fb.cands[0].ID, fb.cands[1].ID, fb.cands[0].ID} + in := h.batchInput() + in.Scope, in.SenderAccountIDs = nil, ids + view, xerr := h.svc.CreateBatch(context.Background(), in) + if xerr != nil { + t.Fatalf("CreateBatch: %v", xerr) + } + if view.SenderCount != 2 { + t.Fatalf("batch has %d senders; want the duplicate dropped", view.SenderCount) + } + + in = h.batchInput() + in.AllowedSenders = []uuid.UUID{fb.cands[2].ID} + view, xerr = h.svc.CreateBatch(context.Background(), in) + if xerr != nil { + t.Fatalf("CreateBatch: %v", xerr) + } + if view.SenderCount != 1 { + t.Fatalf("a key limited to one mailbox resolved %d senders", view.SenderCount) + } + in = h.batchInput() + in.Scope, in.SenderAccountIDs, in.AllowedSenders = nil, []uuid.UUID{fb.cands[3].ID}, []uuid.UUID{fb.cands[2].ID} + if _, xerr := h.svc.CreateBatch(context.Background(), in); xerr == nil { + t.Fatalf("a key named a mailbox it may not send from and the batch was created") + } +} + +func TestBatchSenderWithoutDailyHeadroomIsDeferredOrSkipped(t *testing.T) { + for _, policy := range []string{models.PlacementUnavailableDefer, models.PlacementUnavailableSkip} { + t.Run(policy, func(t *testing.T) { + h, fb := fleetHarness(t, 3) + h.tasks.sentToday = 49 + in := h.batchInput() + in.OnUnavailable = policy + view, xerr := h.svc.CreateBatch(context.Background(), in) + if xerr != nil { + t.Fatalf("CreateBatch: %v", xerr) + } + h.svc.runBatches(context.Background()) + for _, s := range fb.senders { + if s.Reason != "placement_daily_budget" { + t.Fatalf("sender %s: reason %q; want the daily budget", s.SenderEmail, s.Reason) + } + switch policy { + case models.PlacementUnavailableDefer: + if s.Status != models.PlacementSenderDeferred || !s.NextAttemptAt.After(time.Now().Add(time.Minute)) { + t.Fatalf("sender %s is %s until %v; want deferred to the next sending day", s.SenderEmail, s.Status, s.NextAttemptAt) + } + default: + if s.Status != models.PlacementSenderSkipped { + t.Fatalf("sender %s is %s; want skipped", s.SenderEmail, s.Status) + } + } + } + b := fb.batches[view.ID] + if policy == models.PlacementUnavailableSkip && (b.Active || b.Status != models.PlacementBatchFailed) { + t.Fatalf("an all-skipped batch is %s (active %v); want failed", b.Status, b.Active) + } + if policy == models.PlacementUnavailableDefer && !b.Active { + t.Fatalf("a deferred batch closed; want it to wait for tomorrow") + } + }) + } +} + +func TestBatchStopsWhenTheAllowanceRunsOut(t *testing.T) { + h, fb := fleetHarness(t, 4) + in := h.batchInput() + if _, xerr := h.svc.CreateBatch(context.Background(), in); xerr != nil { + t.Fatalf("CreateBatch: %v", xerr) + } + // Between creation and the run, another test used the allowance up. + t.Setenv("DEPLOYMENT_MODE", "cloud") + h.repo.metered = 1000 + h.svc.runBatches(context.Background()) + for _, s := range fb.senders { + if s.Status != models.PlacementSenderSkipped || s.Reason != "placement_quota_exceeded" { + t.Fatalf("sender %s is %s (%s); want every sender skipped for the allowance", s.SenderEmail, s.Status, s.Reason) + } + } + if len(h.repo.created) != 0 { + t.Fatalf("%d tests started past the allowance", len(h.repo.created)) + } +} + +func TestFinishedBatchStatus(t *testing.T) { + for _, c := range []struct { + name string + statuses []string + want string + }{ + {"all completed", []string{"completed", "completed"}, models.PlacementBatchCompleted}, + {"some skipped", []string{"completed", "skipped"}, models.PlacementBatchCompletedWithWarnings}, + {"none completed", []string{"failed", "skipped"}, models.PlacementBatchFailed}, + } { + t.Run(c.name, func(t *testing.T) { + h, fb := fleetHarness(t, len(c.statuses)) + view, xerr := h.svc.CreateBatch(context.Background(), h.batchInput()) + if xerr != nil { + t.Fatalf("CreateBatch: %v", xerr) + } + for i, s := range fb.senders { + s.Status = c.statuses[i] + } + h.svc.runBatches(context.Background()) + if got := fb.batches[view.ID].Status; got != c.want { + t.Fatalf("batch finished %s; want %s", got, c.want) + } + }) + } +} + +func TestBuildBatchDetailGroupsWorstFirst(t *testing.T) { + rows := []repository.PlacementBreakdownRow{ + {Set: "overall", Tracked: true, Folder: "inbox", Count: 6}, + {Set: "overall", Tracked: true, Folder: "spam", Count: 4}, + {Set: "overall", Tracked: false, Folder: "inbox", Count: 9}, + {Set: "domain", SenderDomain: "good.test", Folder: "inbox", Count: 5}, + {Set: "domain", SenderDomain: "bad.test", Folder: "inbox", Count: 1}, + {Set: "domain", SenderDomain: "bad.test", Folder: "spam", Count: 4}, + {Set: "provider", SenderFamily: "google_workspace", Folder: "inbox", Count: 6}, + {Set: "recipient", RecipientFamily: "gmail", Folder: "inbox", Count: 6}, + {Set: "matrix", SenderDomain: "bad.test", RecipientFamily: "gmail", Folder: "spam", Count: 4}, + } + sizes := []repository.PlacementBatchGroupSize{ + {ByDomain: true, Domain: "good.test", Senders: 2, Completed: 2}, + {ByDomain: true, Domain: "bad.test", Senders: 3, Completed: 3}, + {Family: "google_workspace", Senders: 5, Completed: 5}, + } + v := BatchView{PlacementBatch: models.PlacementBatch{Tracking: models.PlacementTrackingCompare}} + d := buildBatchDetail(v, rows, sizes) + if d.Summary.Inbox != 6 || d.Summary.Spam != 4 || d.Untracked == nil || d.Untracked.Inbox != 9 { + t.Fatalf("summary %+v untracked %+v; want the tracked half as headline", d.Summary, d.Untracked) + } + if len(d.Domains) != 2 || d.Domains[0].Key != "bad.test" || d.Domains[0].Senders != 3 { + t.Fatalf("domains = %+v; want bad.test first with its 3 senders", d.Domains) + } + if len(d.Matrix) != 1 || d.Matrix[0].Domain != "bad.test" || d.Matrix[0].Recipients[0].Counts.Spam != 4 { + t.Fatalf("matrix = %+v", d.Matrix) + } + if len(d.Providers) != 1 || d.Providers[0].Senders != 5 { + t.Fatalf("providers = %+v", d.Providers) + } +} + +func TestBatchRespectsTheInstanceWideLimit(t *testing.T) { + h, _ := fleetHarness(t, 50) + if _, xerr := h.svc.CreateBatch(context.Background(), h.batchInput()); xerr != nil { + t.Fatalf("CreateBatch: %v", xerr) + } + pol := h.svc.policy(context.Background()) + pol.BatchSenderConcurrency, pol.BatchInstanceConcurrency, pol.BatchStartsPerMinute = 20, 5, 600 + h.svc.Policy = fakePolicy{pol} + h.svc.runBatches(context.Background()) + if len(h.repo.created) != 5 { + t.Fatalf("started %d tests; want the instance-wide limit of 5 below the workspace's 20", len(h.repo.created)) + } + for _, test := range h.repo.created { + if test.Pace != models.PlacementPaceSpaced { + t.Fatalf("a batch test runs %s; want spaced", test.Pace) + } + } +} + +func TestBatchRefusesQuickPace(t *testing.T) { + h, _ := fleetHarness(t, 3) + in := h.batchInput() + in.Pace = models.PlacementPaceQuick + if _, xerr := h.svc.CreateBatch(context.Background(), in); xerr == nil { + t.Fatalf("a quick batch was created") + } +} + +func TestPreviewCountsSendersBeforeTheCopyIsChosen(t *testing.T) { + h, _ := fleetHarness(t, 12) + in := h.batchInput() + in.Subject, in.BodyPlain, in.Tracking = "", "", models.PlacementTrackingCompare + p, xerr := h.svc.PreviewBatch(context.Background(), in) + if xerr != nil { + t.Fatalf("PreviewBatch without copy: %v", xerr) + } + if p.Selected != 12 || p.Tests != 24 { + t.Fatalf("preview = %d senders, %d tests; want 12 and 24 for a comparison", p.Selected, p.Tests) + } + if _, xerr := h.svc.CreateBatch(context.Background(), in); xerr == nil { + t.Fatalf("a batch without copy was created") + } +} diff --git a/internal/app/placement/service.go b/internal/app/placement/service.go index 3466c45ba..7f869a55e 100644 --- a/internal/app/placement/service.go +++ b/internal/app/placement/service.go @@ -51,6 +51,7 @@ type Entitlements interface { // satisfies it. type Publisher interface { PublishPlacementTest(ctx context.Context, orgID, testID uuid.UUID, campaignID *uuid.UUID, status string) + PublishPlacementBatch(ctx context.Context, orgID, batchID uuid.UUID, status string) } // Notifier reaches people about a finished test or a monitor alert. @@ -99,6 +100,8 @@ type Deps struct { // Credits pays for tests past the monthly free allowance; nil turns // paid tests off. Credits Credits + // Batches stores placement batches; nil turns batches off. + Batches repository.PlacementBatchRepository } // Credits is the slice of the credit ledger a paid test uses. @@ -133,6 +136,14 @@ type Service interface { RemoteSends(ctx context.Context, inst *models.PoolLinkInstance, testID uuid.UUID, sends []models.PlacementCloudSend) *errx.Error RemoteGet(ctx context.Context, inst *models.PoolLinkInstance, testID uuid.UUID) (*models.PlacementCloudTest, *errx.Error) + PreviewBatch(ctx context.Context, in BatchInput) (*BatchPreview, *errx.Error) + CreateBatch(ctx context.Context, in BatchInput) (*BatchView, *errx.Error) + ListBatches(ctx context.Context, orgID uuid.UUID, limit, offset int) ([]BatchView, int, *errx.Error) + GetBatch(ctx context.Context, orgID, id uuid.UUID) (*BatchDetail, *errx.Error) + ListBatchSenders(ctx context.Context, orgID, id uuid.UUID, f repository.PlacementBatchSenderFilter) ([]BatchSenderView, int, *errx.Error) + CancelBatch(ctx context.Context, orgID, id uuid.UUID) (*BatchView, *errx.Error) + Coverage(ctx context.Context, orgID uuid.UUID) (*repository.PlacementCoverage, *errx.Error) + // Tick classifies delivered probes, closes finished tests, syncs cloud // tests and runs due monitors. The poller calls it. Tick(ctx context.Context) error @@ -172,6 +183,9 @@ type CreateInput struct { MaxCredits int Origin string MonitorID *uuid.UUID + // BatchID and BatchSenderID tie a batch's test to its sender row. + BatchID *uuid.UUID + BatchSenderID *uuid.UUID } func (s *service) policy(ctx context.Context) instancesettings.Placement { @@ -245,86 +259,17 @@ func (s *service) CreateTests(ctx context.Context, in CreateInput) ([]TestView, return nil, placementErr(errx.Conflict, "placement_sender_unavailable", "A seed mailbox receives tests; it cannot send one.") } - // The copy: a campaign step (snapshotted now, so an edit mid-test does not - // change what the later seeds get) or an ad-hoc template. - var campaign *models.Campaign - if in.CampaignID != nil { - c, err := s.Campaigns.GetByID(ctx, *in.CampaignID) - if err != nil || c == nil || c.OrganizationID == nil || *c.OrganizationID != in.OrgID { - return nil, errx.New(errx.NotFound, "campaign not found") - } - campaign = c - if in.SequenceID != nil { - seq := s.campaignStep(ctx, c.ID, *in.SequenceID) - if seq == nil { - return nil, errx.New(errx.NotFound, "campaign step not found") - } - if strings.TrimSpace(in.Subject) == "" && in.BodyHTML == "" && in.BodyPlain == "" { - in.Subject, in.BodyHTML, in.BodyPlain = seq.Subject, seq.BodyHTML, seq.BodyPlain - } - if strings.TrimSpace(in.Subject) == "" { - in.Subject = s.threadSubject(ctx, c.ID, seq) - } - } - } else if in.SequenceID != nil { - return nil, errx.New(errx.BadRequest, "sequence_id needs campaign_id") + spec := copySpec{ + CampaignID: in.CampaignID, SequenceID: in.SequenceID, ContactID: in.ContactID, + Subject: in.Subject, BodyHTML: in.BodyHTML, BodyPlain: in.BodyPlain, Panel: in.Panel, } - // A campaign test renders for the campaign's first lead unless told - // otherwise, so merge fields and AI blocks read as a lead would get them. - // Not on the cloud panel: those copies land in another operator's inboxes, - // so a real lead's details go there only when someone chose that lead. - if in.ContactID == nil && campaign != nil && in.Panel != models.PlacementPanelCloud { - if lead, err := s.Repo.SampleLead(ctx, campaign.ID); err == nil { - in.ContactID = lead - } + if xerr := s.resolveCopy(ctx, in.OrgID, &spec); xerr != nil { + return nil, xerr } - if in.ContactID != nil { - found, xerr := s.contactsInOrg(ctx, in.OrgID, *in.ContactID) - if xerr != nil { - return nil, xerr - } - if !found { - return nil, errx.New(errx.NotFound, "contact not found") - } - } - in.Subject = strings.TrimSpace(in.Subject) - if in.Subject == "" { - return nil, errx.New(errx.BadRequest, "subject is required") - } - if !mailhtml.HasContent(in.BodyHTML) && strings.TrimSpace(in.BodyPlain) == "" { - return nil, errx.New(errx.BadRequest, "a plain-text or HTML body is required") - } - if len(in.Subject) > config.SequenceSubjectLimit*4 || len(in.BodyHTML)+len(in.BodyPlain) > config.SequenceBodyLimit*4 { - return nil, errx.New(errx.BadRequest, "the template is too long") - } - - // Tracking, resolved to what each test's copies carry. - textOnly := campaign != nil && campaign.TextOnly - campOpen, campLink := false, false - if campaign != nil { - campOpen, campLink = campaign.OpenTracking, campaign.LinkTracking - } - type variant struct{ open, link bool } - var variants []variant - switch in.Tracking { - case models.PlacementTrackingCampaign: - variants = []variant{{campOpen, campLink}} - case models.PlacementTrackingOn: - variants = []variant{{true, true}} - case models.PlacementTrackingOff: - variants = []variant{{false, false}} - case models.PlacementTrackingCompare: - tracked := variant{campOpen, campLink} - if !tracked.open && !tracked.link { - tracked = variant{true, true} - } - variants = []variant{{false, false}, tracked} - } - if textOnly && (in.Tracking == models.PlacementTrackingOn || in.Tracking == models.PlacementTrackingCompare) { - return nil, placementErr(errx.BadRequest, "placement_invalid_tracking", "This campaign sends plain text, which carries no tracking to compare.") - } - if textOnly { - variants = []variant{{false, false}} + in.ContactID, in.Subject, in.BodyHTML, in.BodyPlain = spec.ContactID, spec.Subject, spec.BodyHTML, spec.BodyPlain + variants, xerr := trackingVariants(in.Tracking, spec.campaign) + if xerr != nil { + return nil, xerr } // Limits: tests in flight, the sender's own queue, the monthly allowance. @@ -357,7 +302,7 @@ func (s *service) CreateTests(ctx context.Context, in CreateInput) ([]TestView, } if free := usage.Remaining(); free >= 0 && free < len(variants) { paid, price = len(variants)-free, usage.CreditsPerTest - if price == 0 || in.Origin != models.PlacementOriginManual { + if price == 0 || (in.Origin != models.PlacementOriginManual && in.Origin != models.PlacementOriginBatch) { return nil, placementErr(errx.PaymentRequired, "placement_quota_exceeded", "This workspace has used its free placement tests for the month.") } @@ -505,6 +450,8 @@ func (s *service) CreateTests(ctx context.Context, in CreateInput) ([]TestView, SequenceID: in.SequenceID, ContactID: in.ContactID, MonitorID: in.MonitorID, + BatchID: in.BatchID, + BatchSenderID: in.BatchSenderID, Subject: in.Subject, BodyHTML: in.BodyHTML, BodyPlain: in.BodyPlain, @@ -586,6 +533,106 @@ func (s *service) CreateTests(ctx context.Context, in CreateInput) ([]TestView, return views, nil } +// copySpec is the email a test sends and where it comes from. +type copySpec struct { + CampaignID, SequenceID, ContactID *uuid.UUID + Subject, BodyHTML, BodyPlain string + Panel string + campaign *models.Campaign +} + +// resolveCopy checks that a test's copy belongs to the workspace and fills it +// in from the campaign step. +func (s *service) resolveCopy(ctx context.Context, orgID uuid.UUID, c *copySpec) *errx.Error { + // The copy: a campaign step (snapshotted now, so an edit mid-test does not + // change what the later seeds get) or an ad-hoc template. + if c.CampaignID != nil { + campaign, err := s.Campaigns.GetByID(ctx, *c.CampaignID) + if err != nil || campaign == nil || campaign.OrganizationID == nil || *campaign.OrganizationID != orgID { + return errx.New(errx.NotFound, "campaign not found") + } + c.campaign = campaign + if c.SequenceID != nil { + seq := s.campaignStep(ctx, campaign.ID, *c.SequenceID) + if seq == nil { + return errx.New(errx.NotFound, "campaign step not found") + } + if strings.TrimSpace(c.Subject) == "" && c.BodyHTML == "" && c.BodyPlain == "" { + c.Subject, c.BodyHTML, c.BodyPlain = seq.Subject, seq.BodyHTML, seq.BodyPlain + } + if strings.TrimSpace(c.Subject) == "" { + c.Subject = s.threadSubject(ctx, campaign.ID, seq) + } + } + } else if c.SequenceID != nil { + return errx.New(errx.BadRequest, "sequence_id needs campaign_id") + } + // A campaign test renders for the campaign's first lead unless told + // otherwise, so merge fields and AI blocks read as a lead would get them. + // Not on the cloud panel: those copies land in another operator's inboxes, + // so a real lead's details go there only when someone chose that lead. + if c.ContactID == nil && c.campaign != nil && c.Panel != models.PlacementPanelCloud { + if lead, err := s.Repo.SampleLead(ctx, c.campaign.ID); err == nil { + c.ContactID = lead + } + } + if c.ContactID != nil { + found, xerr := s.contactsInOrg(ctx, orgID, *c.ContactID) + if xerr != nil { + return xerr + } + if !found { + return errx.New(errx.NotFound, "contact not found") + } + } + c.Subject = strings.TrimSpace(c.Subject) + if c.Subject == "" { + return errx.New(errx.BadRequest, "subject is required") + } + if !mailhtml.HasContent(c.BodyHTML) && strings.TrimSpace(c.BodyPlain) == "" { + return errx.New(errx.BadRequest, "a plain-text or HTML body is required") + } + if len(c.Subject) > config.SequenceSubjectLimit*4 || len(c.BodyHTML)+len(c.BodyPlain) > config.SequenceBodyLimit*4 { + return errx.New(errx.BadRequest, "the template is too long") + } + return nil +} + +// variant is the tracking one test's copies carry. +type variant struct{ open, link bool } + +// trackingVariants resolves a tracking choice to the tests it runs: one, or +// two for a comparison. +func trackingVariants(tracking string, campaign *models.Campaign) ([]variant, *errx.Error) { + textOnly := campaign != nil && campaign.TextOnly + campOpen, campLink := false, false + if campaign != nil { + campOpen, campLink = campaign.OpenTracking, campaign.LinkTracking + } + var variants []variant + switch tracking { + case models.PlacementTrackingCampaign: + variants = []variant{{campOpen, campLink}} + case models.PlacementTrackingOn: + variants = []variant{{true, true}} + case models.PlacementTrackingOff: + variants = []variant{{false, false}} + case models.PlacementTrackingCompare: + tracked := variant{campOpen, campLink} + if !tracked.open && !tracked.link { + tracked = variant{true, true} + } + variants = []variant{{false, false}, tracked} + } + if textOnly && (tracking == models.PlacementTrackingOn || tracking == models.PlacementTrackingCompare) { + return nil, placementErr(errx.BadRequest, "placement_invalid_tracking", "This campaign sends plain text, which carries no tracking to compare.") + } + if textOnly { + variants = []variant{{false, false}} + } + return variants, nil +} + // remainingSends is what is left of the sender's daily campaign limit, // counting campaign sends and probes already queued today. func (s *service) remainingSends(ctx context.Context, sender *models.Email) (int, *errx.Error) { @@ -908,8 +955,10 @@ func (s *service) CancelTest(ctx context.Context, orgID, id uuid.UUID) (*TestVie // ListTests lists tests newest first; a nil orgID lists every workspace's. func (s *service) ListTests(ctx context.Context, orgID *uuid.UUID, campaignID *uuid.UUID, limit, offset int) ([]TestView, int, *errx.Error) { + // The operator's list shows every test; a workspace's leaves batch tests + // to their batch. tests, total, err := s.Repo.ListTests(ctx, repository.PlacementTestFilter{ - OrganizationID: orgID, CampaignID: campaignID, Limit: limit, Offset: offset, + OrganizationID: orgID, CampaignID: campaignID, IncludeBatch: orgID == nil, Limit: limit, Offset: offset, }) if err != nil { errs.CaptureException(err) diff --git a/internal/app/placement/tick.go b/internal/app/placement/tick.go index f64413cce..c9783d590 100644 --- a/internal/app/placement/tick.go +++ b/internal/app/placement/tick.go @@ -16,7 +16,8 @@ import ( ) // Tick is one pass of the poller: read verdicts, expire what never arrived, -// sync the cloud's panel, close finished tests and start due monitors. +// sync the cloud's panel, close finished tests, start due monitors and +// advance batches. func (s *service) Tick(ctx context.Context) error { now := s.now() touched := map[uuid.UUID]bool{} @@ -75,6 +76,7 @@ func (s *service) Tick(ctx context.Context) error { } s.runMonitors(ctx) + s.runBatches(ctx) return nil } diff --git a/internal/app/releases/service.go b/internal/app/releases/service.go index c6e81a5dd..c79c30de7 100644 --- a/internal/app/releases/service.go +++ b/internal/app/releases/service.go @@ -33,6 +33,8 @@ type Config struct { WorkerImageRepo string // "ghcr.io/warmbly/warmbly/worker" GithubToken string // optional, raises API rate limit HTTPClient *http.Client + // SchemaGate refuses a tag whose bus schemas are not registered; nil allows any. + SchemaGate func(ctx context.Context, tag string) error } type Service struct { @@ -139,6 +141,14 @@ func (s *Service) CheckGitHub(ctx context.Context) (*models.FleetReleaseState, e return current, nil } + if s.cfg.SchemaGate != nil { + if err := s.cfg.SchemaGate(ctx, head.TagName); err != nil { + s.recordError(err.Error()) + log.Printf("releases: fleet held at %s: %v", current.Tag, err) + return current, nil + } + } + next := &models.FleetReleaseState{ Channel: current.Channel, Tag: head.TagName, diff --git a/internal/app/releases/service_test.go b/internal/app/releases/service_test.go new file mode 100644 index 000000000..541c0fc2b --- /dev/null +++ b/internal/app/releases/service_test.go @@ -0,0 +1,60 @@ +package releases + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "testing" + + "github.com/warmbly/warmbly/internal/models" +) + +type memSettings struct{ release *models.FleetReleaseState } + +func (m *memSettings) GetRelease(context.Context) (*models.FleetReleaseState, error) { + return m.release, nil +} + +func (m *memSettings) SetRelease(_ context.Context, s *models.FleetReleaseState) error { + m.release = s + return nil +} + +func (m *memSettings) GetJoinTokenHash(context.Context) (string, error) { return "", nil } +func (m *memSettings) SetJoinTokenHash(context.Context, string) error { return nil } + +type githubStub struct{} + +func (githubStub) RoundTrip(*http.Request) (*http.Response, error) { + body := `[{"tag_name":"v2.0.0","published_at":"2026-09-29T12:00:00Z"}]` + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(body)), Header: http.Header{}}, nil +} + +func TestCheckGitHubHoldsTheFleetWhenTheSchemaGateRefuses(t *testing.T) { + for _, tc := range []struct { + name string + gate func(context.Context, string) error + want string + }{ + {"refused", func(context.Context, string) error { return errors.New("incompatible") }, "v1.0.0"}, + {"accepted", func(context.Context, string) error { return nil }, "v2.0.0"}, + } { + t.Run(tc.name, func(t *testing.T) { + settings := &memSettings{release: &models.FleetReleaseState{Channel: models.FleetChannelStable, Tag: "v1.0.0"}} + svc := New(Config{ + Enabled: true, + GithubRepo: "warmbly/warmbly", + HTTPClient: &http.Client{Transport: githubStub{}}, + SchemaGate: tc.gate, + }, settings) + if _, err := svc.CheckGitHub(context.Background()); err != nil { + t.Fatal(err) + } + if settings.release.Tag != tc.want { + t.Fatalf("fleet target %s, want %s", settings.release.Tag, tc.want) + } + }) + } +} diff --git a/internal/app/segment/service.go b/internal/app/segment/service.go index 9c665860e..a64e28911 100644 --- a/internal/app/segment/service.go +++ b/internal/app/segment/service.go @@ -283,12 +283,13 @@ func (s *service) CountAudience(ctx context.Context, orgID uuid.UUID, segmentIDs return s.repo.CountAudience(ctx, orgID, ids, campaignID) } -func parseContactIDs(raw []string) ([]uuid.UUID, *errx.Error) { +func parseContactIDs(raw []string, max int) ([]uuid.UUID, *errx.Error) { if len(raw) == 0 { return nil, errx.New(errx.BadRequest, "no contacts provided") } - if len(raw) > 1000 { - return nil, errx.New(errx.BadRequest, "at most 1000 contacts per request") + if len(raw) > max { + return nil, errx.NewWithIdentifier(errx.BadRequest, "too_many_contacts", + fmt.Sprintf("too many contacts, maximum is %d per request", max)) } out := make([]uuid.UUID, 0, len(raw)) for _, r := range raw { @@ -307,7 +308,9 @@ func (s *service) SetMembers(ctx context.Context, orgID, id uuid.UUID, in *model default: return 0, errx.New(errx.BadRequest, "mode must be include, exclude or auto") } - ids, xerr := parseContactIDs(in.Contacts) + // The handler already bounded the selection by its shape; this is the + // ceiling for callers that pass ids straight in. + ids, xerr := parseContactIDs(in.Contacts, models.MaxContactBulkSelection) if xerr != nil { return 0, xerr } @@ -326,7 +329,7 @@ func (s *service) SetMembers(ctx context.Context, orgID, id uuid.UUID, in *model } func (s *service) MemberModes(ctx context.Context, orgID, id uuid.UUID, contactIDs []string) (map[uuid.UUID]models.SegmentMemberMode, *errx.Error) { - ids, xerr := parseContactIDs(contactIDs) + ids, xerr := parseContactIDs(contactIDs, models.MaxContactBatchIDs) if xerr != nil { return nil, xerr } diff --git a/internal/app/warmup/service.go b/internal/app/warmup/service.go index 46fa0987c..71611579d 100644 --- a/internal/app/warmup/service.go +++ b/internal/app/warmup/service.go @@ -59,10 +59,10 @@ const ( minComplaintSample = 100 - // Tampering: harm done to warmup mail the mailbox received, as weighted - // strikes over the seven-day window (a deletion is one, a spam flag two). - // One deletion is housekeeping until proven otherwise, so it only warns; - // the ladder climbs from there and every step lapses on its own. + // Tampering: harm done to warmup mail the mailbox received, one strike per + // deletion or spam move over the seven-day window. One is housekeeping or a + // provider's filter until proven otherwise, so it only warns; the ladder + // climbs from there and every step lapses on its own. tamperingWatchStrikes = 1 tamperingQuarantineStrikes = 2 tamperingBlockStrikes = 4 @@ -446,7 +446,7 @@ func tamperingVerb(kind string) string { case "deletion": return "deleted" case "spam_flag": - return "marked as spam" + return "moved to spam" default: return "tampered with" } @@ -464,7 +464,7 @@ func tamperingKind(m *models.WarmupHealthMetrics) string { func tamperingSummary(m *models.WarmupHealthMetrics) string { parts := []string{} if m.SpamFlagsLast7d > 0 { - parts = append(parts, fmt.Sprintf("%d warmup %s marked as spam", m.SpamFlagsLast7d, plural(m.SpamFlagsLast7d, "email", "emails"))) + parts = append(parts, fmt.Sprintf("%d warmup %s moved to spam", m.SpamFlagsLast7d, plural(m.SpamFlagsLast7d, "email", "emails"))) } if m.DeletionsLast7d > 0 { parts = append(parts, fmt.Sprintf("%d warmup %s deleted", m.DeletionsLast7d, plural(m.DeletionsLast7d, "email", "emails"))) @@ -769,9 +769,9 @@ func moreSevere(a, b evaluationDecision) evaluationDecision { return a } -// evaluateTampering needs no sample: each strike is one deliberate act on mail -// the mailbox verifiably received. A single deletion only warns, because the -// most likely cause is someone tidying the folder by hand. A deletion is only +// evaluateTampering needs no sample: each strike is one act on mail the mailbox +// verifiably received. A single one only warns, because the likeliest cause is +// someone tidying the folder or a provider filing it as spam. A deletion is only // recorded inside config.WarmupDeletionStrikeHours of arrival, and only once a // search of the mailbox found the message in the trash or gone. func evaluateTampering(metrics *models.WarmupHealthMetrics, now time.Time) evaluationDecision { diff --git a/internal/app/warmup/service_test.go b/internal/app/warmup/service_test.go index fe09dc446..91d17059c 100644 --- a/internal/app/warmup/service_test.go +++ b/internal/app/warmup/service_test.go @@ -187,10 +187,11 @@ func TestEvaluateMetricsTamperingLadder(t *testing.T) { {"nothing", 0, 0, models.WarmupHealthHealthy, 0}, {"one deletion warns", 1, 0, models.WarmupHealthWatch, 0}, {"two deletions pause", 2, 0, models.WarmupHealthQuarantined, warmupQuarantineDuration}, - {"one spam flag pauses", 0, 1, models.WarmupHealthQuarantined, warmupQuarantineDuration}, + {"one spam flag warns", 0, 1, models.WarmupHealthWatch, 0}, + {"two spam flags pause", 0, 2, models.WarmupHealthQuarantined, warmupQuarantineDuration}, {"four deletions block", 4, 0, models.WarmupHealthBlocked, warmupBlockDuration}, - {"two spam flags block", 0, 2, models.WarmupHealthBlocked, warmupBlockDuration}, - {"a flag and two deletions block", 2, 1, models.WarmupHealthBlocked, warmupBlockDuration}, + {"four spam flags block", 0, 4, models.WarmupHealthBlocked, warmupBlockDuration}, + {"a flag and three deletions block", 3, 1, models.WarmupHealthBlocked, warmupBlockDuration}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -238,7 +239,7 @@ func TestEvaluateMetricsTamperingCombinesWithRates(t *testing.T) { {"one deletion does not mask a placement throttle", models.WarmupHealthMetrics{DeletionsLast7d: 1, PlacementSample: 20, SpamPlacementRate: 50}, models.WarmupHealthThrottled, func() *time.Time { u := now.Add(warmupThrottleDuration); return &u }()}, {"a placement watch does not mask a tampering quarantine", models.WarmupHealthMetrics{DeletionsLast7d: 2, PlacementSample: 20, SpamPlacementRate: 10}, models.WarmupHealthQuarantined, &quarantine}, {"a complaint-rate quarantine does not mask a tampering block", models.WarmupHealthMetrics{DeletionsLast7d: 4, DeliveredLast30d: 100, ComplaintRate: complaintRateQuarantinePct}, models.WarmupHealthBlocked, &block}, - {"a bounce-rate quarantine does not mask a tampering block", models.WarmupHealthMetrics{SpamFlagsLast7d: 2, DeliveredLast30d: 100, BounceRate: bounceRateQuarantinePct}, models.WarmupHealthBlocked, &block}, + {"a bounce-rate quarantine does not mask a tampering block", models.WarmupHealthMetrics{SpamFlagsLast7d: 4, DeliveredLast30d: 100, BounceRate: bounceRateQuarantinePct}, models.WarmupHealthBlocked, &block}, {"a warmup-complaint quarantine does not mask a tampering block", models.WarmupHealthMetrics{DeletionsLast7d: 4, SentLast7d: 20, WarmupComplaintRate: warmupComplaintQuarantinePct}, models.WarmupHealthBlocked, &block}, {"a placement throttle does not mask a tampering quarantine", models.WarmupHealthMetrics{DeletionsLast7d: 2, PlacementSample: 20, SpamPlacementRate: spamPlacementThrottlePct}, models.WarmupHealthQuarantined, &quarantine}, {"heavy placement does not soften a tampering block", models.WarmupHealthMetrics{DeletionsLast7d: 4, PlacementSample: 20, SpamPlacementRate: 90}, models.WarmupHealthBlocked, &block}, diff --git a/internal/app/worker/wmail/event_google.go b/internal/app/worker/wmail/event_google.go index 5fdac97d0..5165c21c4 100644 --- a/internal/app/worker/wmail/event_google.go +++ b/internal/app/worker/wmail/event_google.go @@ -2,8 +2,13 @@ package wmail import ( "context" + "errors" + "slices" "github.com/google/uuid" + "github.com/rs/zerolog/log" + "github.com/warmbly/warmbly/internal/client/goog" + "github.com/warmbly/warmbly/internal/errx" "github.com/warmbly/warmbly/internal/models" ) @@ -62,15 +67,15 @@ func translateGmailLabels(labelIDs []string, added bool) (addFlags, removeFlags return addFlags, removeFlags } -func (w *WMail) onGoogleMessageLabelsAdded(ctx context.Context, messageID string, labelIDs []string) error { - return w.emitGoogleFlagEvents(ctx, messageID, labelIDs, true) +func (w *WMail) onGoogleMessageLabelsAdded(ctx context.Context, messageID string, changed, current []string) error { + return w.emitGoogleLabelEvents(ctx, messageID, changed, current, true) } -func (w *WMail) onGoogleMessageLabelsRemoved(ctx context.Context, messageID string, labelIDs []string) error { - return w.emitGoogleFlagEvents(ctx, messageID, labelIDs, false) +func (w *WMail) onGoogleMessageLabelsRemoved(ctx context.Context, messageID string, changed, current []string) error { + return w.emitGoogleLabelEvents(ctx, messageID, changed, current, false) } -func (w *WMail) emitGoogleFlagEvents(ctx context.Context, messageID string, labelIDs []string, added bool) error { +func (w *WMail) emitGoogleLabelEvents(ctx context.Context, messageID string, changed, current []string, added bool) error { internalMessage, err := w.EmailMessageMapRepository.Get(ctx, w.UserID, w.ID, messageID) if err != nil { return err @@ -85,7 +90,7 @@ func (w *WMail) emitGoogleFlagEvents(ctx context.Context, messageID string, labe return err } - addFlags, removeFlags := translateGmailLabels(labelIDs, added) + addFlags, removeFlags := translateGmailLabels(changed, added) if len(addFlags) > 0 { if err := w.onEvent(models.JobEventTypeFlagsAdd, &models.JobEventFlags{ @@ -108,5 +113,65 @@ func (w *WMail) emitGoogleFlagEvents(ctx context.Context, messageID string, labe } } + if !slices.ContainsFunc(changed, goog.IsFolderLabel) { + return nil + } + // Archive, Delete, Report spam and Move to inbox are label changes in + // Gmail, so the folder is recomputed from the labels the message now has. + // Labels that contradict the change are not trusted and are looked up. + labels := current + if labels == nil || !labelsReflect(changed, labels, added) { + found := false + labels, found, err = w.GoogleData.Client.MessageLabels(ctx, messageID) + if err != nil { + // Only a failure the whole walk should retry holds the checkpoint; + // anything else is left to the folder reconciliation. + if gmailRetryable(err) { + return err + } + log.Debug().Err(err).Str("email_id", w.ID.String()).Msg("gmail labels lookup failed; folder left to reconciliation") + return nil + } + if !found { + return nil + } + } + folder := goog.Folder(labels) + if w.googleFolders[messageID] == folder { + return nil + } + if err := w.onEvent(models.JobEventTypeFolderUpdate, &models.JobEventFolderUpdate{ + UserID: w.UserID, + EmailID: w.ID, + ID: internalID, + Folder: folder, + }); err != nil { + return err + } + if w.googleFolders == nil { + w.googleFolders = make(map[string]string) + } + w.googleFolders[messageID] = folder return nil } + +// gmailRetryable reports a Gmail failure that is about the mailbox or the +// connection (auth, throttling, transport) rather than one message. +func gmailRetryable(err error) bool { + var merr *errx.MailError + if !errors.As(err, &merr) { + return true + } + return merr.Type == errx.MailErrorCritical || merr.Code == errx.MailErrorCodeSendingTooFast || isTransportError(merr) +} + +// labelsReflect reports whether labels already show the folder labels in +// changed as added (or removed). +func labelsReflect(changed, labels []string, added bool) bool { + for _, l := range changed { + if goog.IsFolderLabel(l) && slices.Contains(labels, l) != added { + return false + } + } + return true +} diff --git a/internal/app/worker/wmail/gmail_folder_test.go b/internal/app/worker/wmail/gmail_folder_test.go new file mode 100644 index 000000000..9b36eafe0 --- /dev/null +++ b/internal/app/worker/wmail/gmail_folder_test.go @@ -0,0 +1,283 @@ +package wmail + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "slices" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +// labelGmail is the Gmail API as the folder paths see it: an inbox listing +// and a minimal messages.get that answers with labels, or 404 when a message +// is not in labels at all. +type labelGmail struct { + inbox []string + labels map[string][]string + refuse map[string]int // messages.get answers this status instead + queries []url.Values + gets []string +} + +func (g *labelGmail) serve(t *testing.T) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if id, isGet := strings.CutPrefix(r.URL.Path, "/gmail/v1/users/me/messages/"); isGet { + g.gets = append(g.gets, id) + if code, ok := g.refuse[id]; ok { + w.WriteHeader(code) + _, _ = fmt.Fprintf(w, `{"error":{"code":%d,"message":"refused"}}`, code) + return + } + labels, ok := g.labels[id] + if !ok { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"error":{"code":404,"message":"Requested entity was not found."}}`)) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{"id": id, "labelIds": labels}) + return + } + g.queries = append(g.queries, r.URL.Query()) + msgs := make([]map[string]string, 0, len(g.inbox)) + for _, id := range g.inbox { + msgs = append(msgs, map[string]string{"id": id, "threadId": "t-" + id}) + } + _ = json.NewEncoder(w).Encode(map[string]any{"messages": msgs}) + })) + t.Cleanup(srv.Close) + return srv +} + +func folderUpdates(events []captured) map[uuid.UUID]string { + out := map[uuid.UUID]string{} + for _, e := range events { + if e.eventType != models.JobEventTypeFolderUpdate { + continue + } + u := e.body.(*models.JobEventFolderUpdate) + out[u.ID] = u.Folder + } + return out +} + +// Archiving in Gmail only removes the INBOX label, and that has to reach the +// platform as a move to archive, not only as a flag nobody files by. +func TestGmailLabelChangeReportsTheFolder(t *testing.T) { + g := &labelGmail{ + labels: map[string][]string{ + "gone-to-trash": {"TRASH", "UNREAD"}, + "stale-record": {"CATEGORY_UPDATES"}, + }, + refuse: map[string]int{"refused": http.StatusBadRequest}, + } + var events []captured + w := newGoogleTestMail(t, g.serve(t), &events) + rowID := uuid.New() + w.EmailMessageMapRepository = knownMessageMap{id: rowID.String()} + + cases := []struct { + name string + id string + changed []string + current []string + added bool + want string + }{ + {"archive", "m1", []string{"INBOX"}, []string{"CATEGORY_UPDATES"}, false, models.FolderArchive}, + {"move to inbox", "m2", []string{"INBOX"}, []string{"INBOX", "UNREAD"}, true, models.FolderInbox}, + {"report spam", "m3", []string{"SPAM"}, []string{"SPAM"}, true, models.FolderSpam}, + {"labels looked up when the record has none", "gone-to-trash", []string{"TRASH"}, nil, true, models.FolderTrash}, + {"labels that contradict the change are looked up", "stale-record", []string{"INBOX"}, []string{"INBOX"}, false, models.FolderArchive}, + {"a star moves nothing", "m4", []string{"STARRED"}, []string{"INBOX", "STARRED"}, true, ""}, + {"a lookup Gmail refuses leaves the folder to reconciliation", "refused", []string{"INBOX"}, nil, false, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + events = nil + if err := w.emitGoogleLabelEvents(t.Context(), tc.id, tc.changed, tc.current, tc.added); err != nil { + t.Fatalf("emit: %v", err) + } + got, moved := folderUpdates(events)[rowID] + if tc.want == "" { + if moved { + t.Fatalf("reported a move to %q for a label that is not a folder", got) + } + return + } + if got != tc.want { + t.Fatalf("folder = %q, want %q", got, tc.want) + } + }) + } + if strings.Join(g.gets, ",") != "gone-to-trash,stale-record,refused" { + t.Errorf("looked up %v, want only the records whose labels could not be trusted", g.gets) + } + + // Delete in Gmail adds TRASH and removes INBOX: one move, one report. + events = nil + current := []string{"TRASH"} + if err := w.emitGoogleLabelEvents(t.Context(), "m5", []string{"TRASH"}, current, true); err != nil { + t.Fatal(err) + } + if err := w.emitGoogleLabelEvents(t.Context(), "m5", []string{"INBOX"}, current, false); err != nil { + t.Fatal(err) + } + n := 0 + for _, e := range events { + if e.eventType == models.JobEventTypeFolderUpdate { + n++ + } + } + if n != 1 { + t.Errorf("reported the move %d times, want once", n) + } +} + +// providerRows stands in for the control plane's provider-folder listing. +type providerRows struct { + fakeSyncContext + rows []repository.ProviderFolderMessage + calls int +} + +func (p *providerRows) ListProviderFolderMessages(_ context.Context, _, _ uuid.UUID, folders []string, _ int) ([]repository.ProviderFolderMessage, error) { + p.calls++ + var out []repository.ProviderFolderMessage + for _, r := range p.rows { + if slices.Contains(folders, r.ProviderFolder) { + out = append(out, r) + } + } + return out, nil +} + +func removedIDs(events []captured) []uuid.UUID { + var out []uuid.UUID + for _, e := range events { + if e.eventType == models.JobEventTypeRemoveEmail { + out = append(out, e.body.(*models.JobEventRemoveEmail).ID) + } + } + return out +} + +// Mail archived in Gmail before labels were followed stays in the unibox +// inbox until the reconciliation finds it. +func TestGmailReconcileMovesWhatGmailMoved(t *testing.T) { + now := time.Now() + row := func(providerID, folder string, age time.Duration) repository.ProviderFolderMessage { + return repository.ProviderFolderMessage{ID: uuid.New(), ProviderID: providerID, ProviderFolder: folder, InternalDate: now.Add(-age)} + } + stillInbox := row("still-inbox", models.FolderInbox, time.Hour) + archived := row("archived", models.FolderInbox, 2*time.Hour) + trashed := row("trashed", models.FolderInbox, 3*time.Hour) + refused := row("refused", models.FolderInbox, 3*time.Hour+30*time.Minute) + deleted := row("deleted", models.FolderInbox, 4*time.Hour) + backToInbox := row("back", models.FolderArchive, 5*time.Hour) + stillArchived := row("old-archive", models.FolderArchive, 6*time.Hour) + + g := &labelGmail{ + inbox: []string{"still-inbox", "back"}, + labels: map[string][]string{ + "archived": {"CATEGORY_PERSONAL"}, + "trashed": {"TRASH"}, + }, + refuse: map[string]int{"refused": http.StatusBadRequest}, + } + var events []captured + w := newGoogleTestMail(t, g.serve(t), &events) + sc := &providerRows{rows: []repository.ProviderFolderMessage{stillInbox, archived, trashed, refused, deleted, backToInbox, stillArchived}} + w.SyncContext = sc + + if merr := w.googleReconcileFolders(t.Context(), now, &tickStats{}); merr != nil { + t.Fatalf("reconcile: %v", merr.Message) + } + + want := map[uuid.UUID]string{ + archived.ID: models.FolderArchive, + trashed.ID: models.FolderTrash, + backToInbox.ID: models.FolderInbox, + } + got := folderUpdates(events) + if len(got) != len(want) { + t.Fatalf("reported %d moves, want %d: %v", len(got), len(want), got) + } + for id, folder := range want { + if got[id] != folder { + t.Errorf("row %s moved to %q, want %q", id, got[id], folder) + } + } + // Gone from Gmail is removed, as the live feed's delete would. + if got := removedIDs(events); len(got) != 1 || got[0] != deleted.ID { + t.Errorf("removed %v, want only the deleted row", got) + } + // Only inbox rows missing from the listing cost a lookup, and one Gmail + // refuses does not stop the rows after it. + if strings.Join(g.gets, ",") != "archived,trashed,refused,deleted" { + t.Errorf("looked up %v, want archived, trashed, refused, deleted", g.gets) + } + // The bound reaches the oldest row of either set, here an archived one + // older than every inbox row, or its move back to the inbox is never seen. + wantQ := fmt.Sprintf("after:%d", stillArchived.InternalDate.Add(-24*time.Hour).Unix()) + if len(g.queries) != 1 || g.queries[0].Get("labelIds") != "INBOX" || g.queries[0].Get("q") != wantQ { + t.Errorf("inbox listing queries = %v, want labelIds=INBOX and q=%s", g.queries, wantQ) + } + + // Inside the interval nothing runs; after it, a message already found + // gone is not looked up again. + events, g.gets = nil, nil + if merr := w.googleReconcileFolders(t.Context(), now.Add(time.Minute), &tickStats{}); merr != nil { + t.Fatalf("second pass: %v", merr.Message) + } + if sc.calls != 2 { + t.Errorf("listed stored rows %d times inside the interval, want the first pass's 2", sc.calls) + } + sc.rows = []repository.ProviderFolderMessage{stillInbox, deleted} + if merr := w.googleReconcileFolders(t.Context(), now.Add(7*time.Hour), &tickStats{}); merr != nil { + t.Fatalf("third pass: %v", merr.Message) + } + if len(g.gets) != 0 { + t.Errorf("looked up %v again, want nothing", g.gets) + } + if len(events) != 0 { + t.Errorf("reported %d events for rows already where Gmail has them", len(events)) + } + + // A day on, the same message is worth one more look. + if merr := w.googleReconcileFolders(t.Context(), now.Add(25*time.Hour), &tickStats{}); merr != nil { + t.Fatalf("fourth pass: %v", merr.Message) + } + if strings.Join(g.gets, ",") != "deleted" { + t.Errorf("looked up %v a day later, want deleted", g.gets) + } +} + +// A pass Gmail throttles is tried again soon, not after the full interval. +func TestGmailReconcileRetriesAFailedPassSoon(t *testing.T) { + now := time.Now() + g := &labelGmail{refuse: map[string]int{"throttled": http.StatusTooManyRequests}} + var events []captured + w := newGoogleTestMail(t, g.serve(t), &events) + w.SyncContext = &providerRows{rows: []repository.ProviderFolderMessage{ + {ID: uuid.New(), ProviderID: "throttled", ProviderFolder: models.FolderInbox, InternalDate: now}, + }} + + if merr := w.googleReconcileFolders(t.Context(), now, &tickStats{}); merr == nil { + t.Fatal("a throttled lookup was swallowed") + } + g.gets = nil + if merr := w.googleReconcileFolders(t.Context(), now.Add(20*time.Minute), &tickStats{}); merr == nil || len(g.gets) != 1 { + t.Errorf("20 minutes on the pass did not run again (looked up %v)", g.gets) + } +} diff --git a/internal/app/worker/wmail/send.go b/internal/app/worker/wmail/send.go index 0b2b0b1ad..e74552912 100644 --- a/internal/app/worker/wmail/send.go +++ b/internal/app/worker/wmail/send.go @@ -563,5 +563,6 @@ func MailErrorToSendError(err *errx.MailError) *models.EmailSendError { UserTitle: userInfo.Title, UserMessage: userInfo.Message, ActionRequired: userInfo.ActionRequired, + Recipient: err.Recipient, } } diff --git a/internal/app/worker/wmail/sync_google.go b/internal/app/worker/wmail/sync_google.go index 08f3ef7d8..54e601d6e 100644 --- a/internal/app/worker/wmail/sync_google.go +++ b/internal/app/worker/wmail/sync_google.go @@ -6,9 +6,13 @@ import ( "fmt" "time" + "github.com/google/uuid" + "github.com/rs/zerolog/log" "github.com/warmbly/warmbly/internal/client/goog" + "github.com/warmbly/warmbly/internal/config" "github.com/warmbly/warmbly/internal/errx" "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" ) // googleBackfillPage is how many ids one messages.list call returns. Small @@ -22,6 +26,7 @@ func (w *WMail) SyncGoogle(ctx context.Context) *errx.MailError { w.beginTick() stats := &tickStats{} w.googleTick = stats + w.googleFolders = nil if !w.retryUnmap(ctx) { return nil } @@ -50,6 +55,13 @@ func (w *WMail) SyncGoogle(ctx context.Context) *errx.MailError { return merr } } + if !stats.aborted { + // The pass's own work is done; only a refusal the owner has to act + // on is worth failing it for. + if merr := w.googleReconcileFolders(ctx, time.Now(), stats); merr != nil && merr.Type == errx.MailErrorCritical { + return merr + } + } w.endTick(stats) return nil } @@ -227,6 +239,150 @@ func (w *WMail) googleBackfill(ctx context.Context, stats *tickStats) *errx.Mail return nil } +// reconcileElsewhere is every folder a stored Gmail message can be moved back +// to the inbox from. +var reconcileElsewhere = []string{models.FolderArchive, models.FolderSpam, models.FolderTrash} + +// googleReconcileFolders repairs stored mail the history feed did not move: +// changes from before the sync followed labels, and any a checkpoint Gmail +// expired skipped over. Inbox rows missing from Gmail's inbox are looked up; +// every other row only moves when Gmail lists it in the inbox. A pass that +// fails is tried again after GmailFolderReconcileRetry. +func (w *WMail) googleReconcileFolders(ctx context.Context, now time.Time, stats *tickStats) *errx.MailError { + if w.SyncContext == nil || now.Sub(w.googleReconciledAt) < config.GmailFolderReconcileInterval { + return nil + } + w.googleReconciledAt = now.Add(config.GmailFolderReconcileRetry - config.GmailFolderReconcileInterval) + + inboxRows, err := w.SyncContext.ListProviderFolderMessages(ctx, w.UserID, w.ID, []string{models.FolderInbox}, config.GmailFolderReconcileMessages) + if err != nil { + return w.controlPlaneError(err, stats) + } + otherRows, err := w.SyncContext.ListProviderFolderMessages(ctx, w.UserID, w.ID, reconcileElsewhere, config.GmailFolderReconcileMessages) + if err != nil { + return w.controlPlaneError(err, stats) + } + if len(inboxRows) == 0 && len(otherRows) == 0 { + w.googleReconciledAt = now + return nil + } + + // Each set comes newest first, and the listing has to reach the oldest row + // of either; a day of slack covers Gmail reading after: against its own + // calendar. + var oldest time.Time + for _, rows := range [][]repository.ProviderFolderMessage{inboxRows, otherRows} { + if len(rows) > 0 && (oldest.IsZero() || rows[len(rows)-1].InternalDate.Before(oldest)) { + oldest = rows[len(rows)-1].InternalDate + } + } + q := "" + if after := oldest.Add(-24 * time.Hour); after.Unix() > 0 { + q = fmt.Sprintf("after:%d", after.Unix()) + } + inInbox := make(map[string]struct{}) + token := "" + for page := 0; page < config.GmailFolderReconcilePages; page++ { + ids, next, err := w.GoogleData.Client.ListLabelMessages(ctx, goog.Inbox, q, token, 500) + if err != nil { + return w.googleReconcileError(err) + } + for _, id := range ids { + inInbox[id] = struct{}{} + } + if next == "" { + break + } + token = next + } + + for _, m := range otherRows { + if _, ok := inInbox[m.ProviderID]; !ok { + continue + } + if err := w.emitFolder(m.ID, models.FolderInbox); err != nil { + return w.controlPlaneError(err, stats) + } + } + + if w.googleInboxChecked == nil { + w.googleInboxChecked = make(map[string]time.Time) + } + for id, at := range w.googleInboxChecked { + if now.Sub(at) >= config.GmailFolderReconcileRecheck { + delete(w.googleInboxChecked, id) + } + } + lookups := 0 + for _, m := range inboxRows { + if _, ok := inInbox[m.ProviderID]; ok { + continue + } + if _, checked := w.googleInboxChecked[m.ProviderID]; checked { + continue + } + if lookups >= config.GmailFolderReconcileLookups { + break + } + lookups++ + labels, found, err := w.GoogleData.Client.MessageLabels(ctx, m.ProviderID) + if err != nil { + if gmailRetryable(err) { + return w.googleReconcileError(err) + } + // One message Gmail refuses must not hold up every row after it. + log.Debug().Err(err).Str("email_id", w.ID.String()).Str("gmail_id", m.ProviderID).Msg("gmail folder reconciliation: lookup refused") + w.googleInboxChecked[m.ProviderID] = now + continue + } + if !found { + // Gone from Gmail, which is what the live feed's delete reports. + if err := w.onEvent(models.JobEventTypeRemoveEmail, &models.JobEventRemoveEmail{ + UserID: w.UserID, + EmailID: w.ID, + ID: m.ID, + }); err != nil { + return w.controlPlaneError(err, stats) + } + w.googleInboxChecked[m.ProviderID] = now + continue + } + folder := goog.Folder(labels) + if folder == models.FolderInbox { + w.googleInboxChecked[m.ProviderID] = now + continue + } + if err := w.emitFolder(m.ID, folder); err != nil { + return w.controlPlaneError(err, stats) + } + } + w.googleReconciledAt = now + return nil +} + +func (w *WMail) emitFolder(id uuid.UUID, folder string) error { + return w.onEvent(models.JobEventTypeFolderUpdate, &models.JobEventFolderUpdate{ + UserID: w.UserID, + EmailID: w.ID, + ID: id, + Folder: folder, + }) +} + +// googleReconcileError returns a Gmail failure to the caller, which fails the +// tick only when it is critical; the rest are captured unless transient. +func (w *WMail) googleReconcileError(err error) *errx.MailError { + var errMail *errx.MailError + if !errors.As(err, &errMail) { + w.CaptureError(err) + return nil + } + if errMail.Type != errx.MailErrorCritical && !gmailRetryable(err) { + w.CaptureError(err) + } + return errMail +} + // NewHistoryID persists the mailbox's Gmail history checkpoint. UserID and // EmailID address the row: email_history_ids is keyed (user_id, email_id) with // a foreign key to users, so omitting them sends a zero UUID and the write is diff --git a/internal/app/worker/wmail/sync_imap_test.go b/internal/app/worker/wmail/sync_imap_test.go index 5e660b626..c4d3cb95f 100644 --- a/internal/app/worker/wmail/sync_imap_test.go +++ b/internal/app/worker/wmail/sync_imap_test.go @@ -649,6 +649,10 @@ func (f *fakeSyncContext) ListFolderMessages(_ context.Context, _, _ uuid.UUID, return f.stored[folderPath], nil } +func (fakeSyncContext) ListProviderFolderMessages(context.Context, uuid.UUID, uuid.UUID, []string, int) ([]repository.ProviderFolderMessage, error) { + return nil, nil +} + func removeIDs(events []captured) []uuid.UUID { var out []uuid.UUID for _, e := range events { diff --git a/internal/app/worker/wmail/wmail.go b/internal/app/worker/wmail/wmail.go index d909b8a6c..188376dcf 100644 --- a/internal/app/worker/wmail/wmail.go +++ b/internal/app/worker/wmail/wmail.go @@ -2,6 +2,7 @@ package wmail import ( "context" + "time" "github.com/google/uuid" "github.com/warmbly/warmbly/internal/app/cipher" @@ -95,6 +96,14 @@ type WMail struct { laneCache laneCache googleTick *tickStats graphTick *tickStats + // googleReconciledAt is when the Gmail folder reconciliation last ran; + // googleInboxChecked is when it last looked up an id and found it still + // in the inbox (or gone), so the next passes spend lookups elsewhere. + googleReconciledAt time.Time + googleInboxChecked map[string]time.Time + // googleFolders is the folder last reported per Gmail id this tick, so a + // move that both adds and removes labels is reported once. + googleFolders map[string]string // flagScan is the previous flag snapshot per folder name, used only on // IMAP servers without CONDSTORE, which cannot say what changed. flagScan map[string]*folderFlagScan diff --git a/internal/client/goog/goog.go b/internal/client/goog/goog.go index b6f2559a0..3b6058d5d 100644 --- a/internal/client/goog/goog.go +++ b/internal/client/goog/goog.go @@ -37,9 +37,11 @@ type Client struct { // (fair use), which pins the history checkpoint before that record. OnMessageAdded func(ctx context.Context, id, threadID string) (added bool, err error) OnMessageRemove func(ctx context.Context, messageID string) error - OnLabelAdd func(ctx context.Context, messageID string, labelIds []string) error - OnLabelRemove func(ctx context.Context, messageID string, labelIds []string) error - OnTokenRefresh func(ctx context.Context, token *oauth2.Token) error + // OnLabelAdd and OnLabelRemove get the labels that changed and the + // message's labels as the history record reports them, which may be nil. + OnLabelAdd func(ctx context.Context, messageID string, changed, current []string) error + OnLabelRemove func(ctx context.Context, messageID string, changed, current []string) error + OnTokenRefresh func(ctx context.Context, token *oauth2.Token) error } func (c *Client) Init(ctx context.Context, token *oauth2.Token, cfg oauth2.Config) *errx.MailError { diff --git a/internal/client/goog/history.go b/internal/client/goog/history.go index d86e8e5e5..1eece73c8 100644 --- a/internal/client/goog/history.go +++ b/internal/client/goog/history.go @@ -71,12 +71,12 @@ func (c *Client) FetchHistory(ctx context.Context, lastHistoryID uint64) (uint64 } } for _, m := range h.LabelsAdded { - if err := c.OnLabelAdd(ctx, m.Message.Id, m.LabelIds); err != nil { + if err := c.OnLabelAdd(ctx, m.Message.Id, m.LabelIds, m.Message.LabelIds); err != nil { return checkpoint, err } } for _, m := range h.LabelsRemoved { - if err := c.OnLabelRemove(ctx, m.Message.Id, m.LabelIds); err != nil { + if err := c.OnLabelRemove(ctx, m.Message.Id, m.LabelIds, m.Message.LabelIds); err != nil { return checkpoint, err } } @@ -113,6 +113,43 @@ func (c *Client) GetMessage(ctx context.Context, id string) (*models.EmailMessag return GmailMessageToEmailData(full), nil } +// MessageLabels returns a message's current label ids. found is false when +// Gmail no longer has the message. +func (c *Client) MessageLabels(ctx context.Context, id string) (labels []string, found bool, err error) { + msg, err := c.srv.Users.Messages.Get("me", id).Format("minimal").Context(ctx).Do() + if err != nil { + var gerr *googleapi.Error + if errors.As(err, &gerr) && gerr.Code == 404 { + return nil, false, nil + } + return nil, false, HandleError(err) + } + return msg.LabelIds, true, nil +} + +// ListLabelMessages is one page of the ids of messages carrying labelID, +// matched per message rather than per thread, narrowed by q. +func (c *Client) ListLabelMessages(ctx context.Context, labelID, q, pageToken string, max int64) ([]string, string, error) { + call := c.srv.Users.Messages.List("me").LabelIds(labelID).MaxResults(max).Context(ctx) + if q != "" { + call = call.Q(q) + } + if pageToken != "" { + call = call.PageToken(pageToken) + } + resp, err := call.Do() + if err != nil { + return nil, "", HandleError(err) + } + ids := make([]string, 0, len(resp.Messages)) + for _, m := range resp.Messages { + if m != nil && m.Id != "" { + ids = append(ids, m.Id) + } + } + return ids, resp.NextPageToken, nil +} + // ListMessages is one page of the backfill: message ids matching q, newest // first, and the token for the next page ("" when the query is exhausted). func (c *Client) ListMessages(ctx context.Context, q, pageToken string, max int64) ([]string, string, error) { diff --git a/internal/client/goog/message.go b/internal/client/goog/message.go index d3b5e8198..6b4ee5c06 100644 --- a/internal/client/goog/message.go +++ b/internal/client/goog/message.go @@ -109,11 +109,11 @@ func parseGmailDate(dateText string) time.Time { return date } -// gmailFolder maps Gmail labels to the canonical unibox folder. Precedence +// Folder maps Gmail labels to the canonical unibox folder. Precedence // mirrors Gmail's own UI: trash and spam are exclusive, a draft is a draft, // inbox wins over sent for self-addressed mail, and mail carrying none of // these labels is archived. -func gmailFolder(labels []string) string { +func Folder(labels []string) string { var inbox, sent bool for _, l := range labels { switch l { @@ -138,6 +138,15 @@ func gmailFolder(labels []string) string { return models.FolderArchive } +// IsFolderLabel reports whether gaining or losing label can change Folder. +func IsFolderLabel(label string) bool { + switch label { + case Inbox, Sent, Draft, Spam, Trash: + return true + } + return false +} + func GmailMessageToEmailData(msg *gmail.Message) *models.EmailMessageData { var headers []*gmail.MessagePartHeader if msg.Payload != nil { @@ -153,7 +162,7 @@ func GmailMessageToEmailData(msg *gmail.Message) *models.EmailMessageData { GmailID: msg.Id, UID: 0, // Gmail has no IMAP UID ThreadID: msg.ThreadId, - Folder: gmailFolder(msg.LabelIds), + Folder: Folder(msg.LabelIds), Flags: func() []string { flags := []string{} // Gmail models read state inversely: the UNREAD label is present on diff --git a/internal/client/smtpimap/smtp/client.go b/internal/client/smtpimap/smtp/client.go index 1feca07fd..035093b4b 100644 --- a/internal/client/smtpimap/smtp/client.go +++ b/internal/client/smtpimap/smtp/client.go @@ -475,7 +475,9 @@ func (c *Client) sendRaw(ctx context.Context, from string, to []string, data []b if !permanentReply(err) { return errx.ErrMailServerUnreachableAt("rcpt to", err) } - return errx.ErrMailRecipientRejected(err.Error()) + refused := errx.ErrMailRecipientRejected(err.Error()) + refused.Recipient = r + return refused } } w, err := client.Data() diff --git a/internal/client/smtpimap/smtp/server_test.go b/internal/client/smtpimap/smtp/server_test.go index 8ec23ba3c..29b51f4b8 100644 --- a/internal/client/smtpimap/smtp/server_test.go +++ b/internal/client/smtpimap/smtp/server_test.go @@ -412,4 +412,8 @@ func TestRecipientRejectionCarriesTheServersReason(t *testing.T) { if !strings.Contains(err.Message, "no such user here") { t.Errorf("message = %q, want the server's own reason in it", err.Message) } + // Named apart from the message, so a refused copy is not read as the lead. + if err.Recipient != "to@example.test" { + t.Errorf("recipient = %q, want the refused address", err.Recipient) + } } diff --git a/internal/config/constants.go b/internal/config/constants.go index 5dcd02d55..edbb21db1 100644 --- a/internal/config/constants.go +++ b/internal/config/constants.go @@ -98,6 +98,18 @@ const ( SyncSkipFoldersMax = 50 // folders one mailbox may exclude from sync SyncSkipFolderNameMax = 255 // characters in one excluded folder name + // Gmail folder reconciliation: how often stored Gmail mail is checked + // against where Gmail has it now, how many rows one pass checks, how many + // inbox listing pages it reads, how many messages it looks up, how long a + // message found still in the inbox is not looked up again, and how soon a + // pass that failed is tried again. + GmailFolderReconcileInterval = 6 * time.Hour + GmailFolderReconcileRetry = 15 * time.Minute + GmailFolderReconcileMessages = 1_000 + GmailFolderReconcilePages = 10 + GmailFolderReconcileLookups = 100 + GmailFolderReconcileRecheck = 24 * time.Hour + // Forms. Funnel events feed analytics ranges up to 90 days, so the default // window keeps double coverage. Operator-editable under Instance settings. FormEventsRetentionDaysDefault = 180 @@ -127,6 +139,22 @@ const ( PlacementMonitorIntervalDaysDef = 7 PlacementMonitorAlertBelowDefault = 70 // inbox rate, percent + // Placement batches run one test from many senders. Membership is bounded + // only by an operator ceiling; execution by concurrency and a start rate. + PlacementBatchSendersMaxDefault = 10_000 + PlacementBatchSendersMaxCeiling = 100_000 + PlacementBatchSenderConcurrencyDefault = 20 // senders of one workspace sending probes at once + PlacementBatchSenderConcurrencyMax = 500 + PlacementBatchInstanceConcurrencyDefault = 200 // batch senders sending at once across every workspace, so no shared seed floods + PlacementBatchInstanceConcurrencyMax = 5_000 + PlacementBatchStartsPerMinuteDefault = 10 + PlacementBatchStartsPerMinuteMax = 600 + PlacementBatchRetryDays = 7 // a deferred sender is retried this long, then skipped + PlacementBatchOpenPerOrgMax = 5 // batches one workspace may have open at once + PlacementBatchSenderErrorAttemptsMax = 5 // unexpected errors before a sender fails + PlacementBatchRunnerBatchesPerTick = 20 + PlacementBatchSenderStaleMinutes = 10 // a claimed sender with no test after this goes back to the queue + // Sequences. Empty by default so the editor shows a smart, position-based // label (e.g. "Email 1") until the user names the step themselves. SequenceDefaultName = "" @@ -143,6 +171,10 @@ const ( // tick retries it; this bounds that loop for a mailbox that can never send. CampaignSendMaxAttempts = 5 + // CampaignLeadMaxCC caps the contacts copied on one lead's emails. Every + // copy is one more recipient who did not ask for the email. + CampaignLeadMaxCC = 2 + // CampaignNotDueGraceSeconds is how far in the future a step's hard // constraints (wait_after, start date, sending window, mailbox min-gap) // may sit while a firing task still sends it. Beyond this the scheduler @@ -314,6 +346,22 @@ const ( // strikes behind a live hold are always there to re-decide it. WarmupTamperingKeepDays = 37 + // A warmup email moved to spam names no actor on any provider, so the move + // is held this long before it is attributed, to see the activity around it. + WarmupSpamMoveSettleMinutes = 30 + // Owner activity this close to a move, either side, attributes it to the owner. + WarmupSpamMoveActivityMinutes = 30 + // A move this soon after arrival, with nobody active, is the filter catching up. + WarmupSpamMoveQuickMinutes = 15 + // The same sender's mail moved to spam in another workspace this close is the provider re-judging it. + WarmupSpamMoveCorrelationHours = 24 + // Unexplained moves from this many distinct senders in seven days, in a mailbox someone uses, are its owner's. + WarmupSpamMovePatternSenders = 3 + // A mailbox with no owner activity this long has nobody to have moved anything. + WarmupOwnerDormantDays = 14 + // Owner activity is kept this long; it only answers the two windows above. + WarmupOwnerActivityKeepDays = 30 + // CampaignSendStampAttempts is how many times the control plane retries the // sent_at stamp after a send is already on the bus. The reservation is what // keeps the step from being re-sent, so a lost stamp is a pacing problem, diff --git a/internal/errx/email.go b/internal/errx/email.go index ed067ae34..bdbcf43af 100644 --- a/internal/errx/email.go +++ b/internal/errx/email.go @@ -109,6 +109,8 @@ type MailError struct { ResolvedAt *time.Time `json:"resolved_at"` Message string `json:"message"` + // Recipient is the one address a per-recipient refusal was about. + Recipient string `json:"recipient,omitempty"` // RetryAfter is provider guidance for transient throttles. It stays local // to the worker; persisted error records should not depend on a stale delay. diff --git a/internal/infrastructure/codec/avro.go b/internal/infrastructure/codec/avro.go index b2f77bf7b..4a23f41d0 100644 --- a/internal/infrastructure/codec/avro.go +++ b/internal/infrastructure/codec/avro.go @@ -11,10 +11,13 @@ import ( "encoding/binary" "errors" "fmt" + "strings" "sync" "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rest" "github.com/hamba/avro/v2" + "github.com/rs/zerolog/log" "github.com/warmbly/warmbly/internal/models" ) @@ -92,6 +95,37 @@ func (c *AvroCodec) Serialize(_ context.Context, topic string, value any) ([]byt return append(out, body...), nil } +// RegisterSchemas satisfies SchemaRegistrar. +func (c *AvroCodec) RegisterSchemas(_ context.Context, schemas map[string]avro.Schema) error { + var all []string + var errs []error + for topic, schema := range schemas { + topics := []string{topic} + if prefix, ok := strings.CutSuffix(topic, "*"); ok { + if all == nil { + var err error + if all, err = c.client.GetAllSubjects(); err != nil { + return fmt.Errorf("codec: list subjects: %w", err) + } + } + topics = topics[:0] + for _, subject := range all { + if t, ok := strings.CutSuffix(subject, "-value"); ok && strings.HasPrefix(t, prefix) { + topics = append(topics, t) + } + } + } + for _, t := range topics { + if _, err := c.register(subjectFor(t), schema); err != nil { + errs = append(errs, err) + } + } + } + return errors.Join(errs...) +} + +var _ SchemaRegistrar = (*AvroCodec)(nil) + // Deserialize decodes with the schema the payload names, which is the writer's // rather than whatever this process happens to hold. That is the whole point of // carrying the id: a consumer reads what was actually written. @@ -132,6 +166,9 @@ func (c *AvroCodec) register(subject string, schema avro.Schema) (int, error) { return id, nil } id, err = c.client.Register(subject, schemaregistry.SchemaInfo{Schema: string(doc)}, true) + if isIncompatible(err) && c.pinBackward(subject) { + id, err = c.client.Register(subject, schemaregistry.SchemaInfo{Schema: string(doc)}, true) + } if err != nil { return 0, fmt.Errorf("codec: register %s: %w", subject, err) } @@ -141,6 +178,36 @@ func (c *AvroCodec) register(subject string, schema avro.Schema) (int, error) { return id, nil } +// isIncompatible is the registry refusing a schema against an earlier version. +func isIncompatible(err error) bool { + var rerr *rest.Error + return errors.As(err, &rerr) && rerr.Code == 409 +} + +// pinBackward sets the subject to BACKWARD when its effective level refuses a +// new union branch, since every new event type adds one to the envelope. A +// level that already admits it is left alone, and so is the refusal. +func (c *AvroCodec) pinBackward(subject string) bool { + level, err := c.client.GetCompatibility(subject) + if err != nil { + if level, err = c.client.GetDefaultCompatibility(); err != nil { + return false + } + } + switch level { + case schemaregistry.Forward, schemaregistry.ForwardTransitive, + schemaregistry.Full, schemaregistry.FullTransitive: + default: + return false + } + if _, err := c.client.UpdateCompatibility(subject, schemaregistry.Backward); err != nil { + return false + } + log.Warn().Str("subject", subject).Str("was", level.String()). + Msg("schema registry: subject set to BACKWARD so a new event type can be registered") + return true +} + func (c *AvroCodec) schemaByID(subject string, id int) (avro.Schema, error) { c.mu.RLock() schema, ok := c.schemas[id] diff --git a/internal/infrastructure/codec/avro_test.go b/internal/infrastructure/codec/avro_test.go index dcecbc087..1a3407edf 100644 --- a/internal/infrastructure/codec/avro_test.go +++ b/internal/infrastructure/codec/avro_test.go @@ -4,7 +4,14 @@ package codec import ( "context" + "errors" + "slices" "testing" + + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry" + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/rest" + "github.com/hamba/avro/v2" + "github.com/warmbly/warmbly/internal/models" ) // Full round-trip coverage for AvroCodec requires a live Confluent Schema @@ -46,3 +53,91 @@ func TestAvroCodec_NilReceiverIsSafe(t *testing.T) { t.Fatal("expected error on nil receiver") } } + +// fakeRegistry refuses a new schema while the subject's effective level is +// in the forward family, the way a registry does for a new union branch. +type fakeRegistry struct { + schemaregistry.Client + global schemaregistry.Compatibility + subject schemaregistry.Compatibility + subjects []string + registered []string +} + +func (f *fakeRegistry) GetAllSubjects() ([]string, error) { return f.subjects, nil } + +func (f *fakeRegistry) effective() schemaregistry.Compatibility { + if f.subject != 0 { + return f.subject + } + return f.global +} + +func (f *fakeRegistry) Register(subject string, _ schemaregistry.SchemaInfo, _ bool) (int, error) { + f.registered = append(f.registered, subject) + switch f.effective() { + case schemaregistry.Forward, schemaregistry.Full: + return 0, &rest.Error{Code: 409, Message: "incompatible"} + } + return 7, nil +} + +func (f *fakeRegistry) GetCompatibility(string) (schemaregistry.Compatibility, error) { + if f.subject == 0 { + return 0, &rest.Error{Code: 40408, Message: "no subject-level compatibility"} + } + return f.subject, nil +} + +func (f *fakeRegistry) GetDefaultCompatibility() (schemaregistry.Compatibility, error) { + return f.global, nil +} + +func (f *fakeRegistry) UpdateCompatibility(_ string, c schemaregistry.Compatibility) (schemaregistry.Compatibility, error) { + f.subject = c + return c, nil +} + +func TestAvroCodec_RegisterPinsForwardSubjectBackward(t *testing.T) { + for _, global := range []schemaregistry.Compatibility{schemaregistry.Forward, schemaregistry.Full} { + reg := &fakeRegistry{global: global} + c := &AvroCodec{client: reg, ids: map[string]int{}, schemas: map[int]avro.Schema{}} + ev := models.JobEvent{Type: models.JobEventTypeEmailSent, Body: models.SendEmailResult{}} + if _, err := c.Serialize(context.Background(), "jobs.worker-events", ev); err != nil { + t.Fatalf("global %s: %v", global.String(), err) + } + if reg.subject != schemaregistry.Backward { + t.Fatalf("global %s: subject left at %v", global.String(), reg.subject) + } + } +} + +func TestAvroCodec_RegisterLeavesOtherRefusalsAlone(t *testing.T) { + reg := &fakeRegistry{global: schemaregistry.Forward, subject: schemaregistry.BackwardTransitive} + c := &AvroCodec{client: reg, ids: map[string]int{}, schemas: map[int]avro.Schema{}} + if !isIncompatible(&rest.Error{Code: 409}) || isIncompatible(errors.New("x")) { + t.Fatal("isIncompatible misreads the refusal") + } + if c.pinBackward("s") || reg.subject != schemaregistry.BackwardTransitive { + t.Fatal("a level that admits a new branch was changed") + } +} + +func TestAvroCodec_RegisterSchemasExpandsPerNodeTopics(t *testing.T) { + reg := &fakeRegistry{global: schemaregistry.Backward, subjects: []string{ + "w.a-value", "w.b-value", "warmup-events-value", "jobs.worker-events-value", + }} + c := &AvroCodec{client: reg, ids: map[string]int{}, schemas: map[int]avro.Schema{}} + err := c.RegisterSchemas(context.Background(), map[string]avro.Schema{ + "w.*": models.WorkerEvent{}.Schema(), + "jobs.worker-events": models.JobEvent{}.Schema(), + }) + if err != nil { + t.Fatal(err) + } + slices.Sort(reg.registered) + want := []string{"jobs.worker-events-value", "w.a-value", "w.b-value"} + if !slices.Equal(reg.registered, want) { + t.Fatalf("registered %v, want %v", reg.registered, want) + } +} diff --git a/internal/infrastructure/codec/codec.go b/internal/infrastructure/codec/codec.go index f892b0106..77d3d7d60 100644 --- a/internal/infrastructure/codec/codec.go +++ b/internal/infrastructure/codec/codec.go @@ -15,7 +15,11 @@ // in-band marker, this is an operator-level configuration choice. package codec -import "context" +import ( + "context" + + "github.com/hamba/avro/v2" +) // Codec serializes and deserializes event payloads. Implementations may // require external services (Schema Registry for Avro) or be standalone @@ -37,6 +41,14 @@ type Codec interface { Name() string } +// SchemaRegistrar is a codec that resolves schemas against a registry, so a +// release can register what it publishes before any node publishes it. +type SchemaRegistrar interface { + // RegisterSchemas registers each schema under its topic's subject. A topic + // ending in ".*" names every topic already registered under that prefix. + RegisterSchemas(ctx context.Context, schemas map[string]avro.Schema) error +} + // Compile-time interface check. The AvroCodec check lives in avro.go (behind // the `kafka` build tag) since that type isn't compiled into the default build. var _ Codec = (*JSONCodec)(nil) diff --git a/internal/infrastructure/db/migrations/000230_campaign_lead_cc.down.sql b/internal/infrastructure/db/migrations/000230_campaign_lead_cc.down.sql new file mode 100644 index 000000000..619770f83 --- /dev/null +++ b/internal/infrastructure/db/migrations/000230_campaign_lead_cc.down.sql @@ -0,0 +1,16 @@ +DROP TRIGGER IF EXISTS campaign_lead_enrol_cc_hold ON campaign_leads; +DROP FUNCTION IF EXISTS campaign_lead_enrol_cc_hold(); +DROP TRIGGER IF EXISTS campaign_lead_cc_hold ON campaign_lead_cc; +DROP FUNCTION IF EXISTS campaign_lead_cc_hold(); + +DROP TABLE IF EXISTS campaign_lead_cc; + +-- The source is gone, so its holds go with it rather than failing the check. +UPDATE campaign_leads +SET paused_at = NULL, paused_until = NULL, pause_reason = NULL, pause_source = NULL +WHERE pause_source = 'cc'; + +ALTER TABLE public.campaign_leads DROP CONSTRAINT IF EXISTS campaign_leads_pause_source_check; +ALTER TABLE public.campaign_leads + ADD CONSTRAINT campaign_leads_pause_source_check + CHECK (pause_source IS NULL OR pause_source IN ('manual', 'out_of_office', 'inbox_tagging')) NOT VALID; diff --git a/internal/infrastructure/db/migrations/000230_campaign_lead_cc.up.sql b/internal/infrastructure/db/migrations/000230_campaign_lead_cc.up.sql new file mode 100644 index 000000000..6bb5aceb5 --- /dev/null +++ b/internal/infrastructure/db/migrations/000230_campaign_lead_cc.up.sql @@ -0,0 +1,93 @@ +-- Extra recipients copied on every email one campaign sends one lead, so two +-- people at the same company can be reached in one thread instead of two +-- parallel sequences (issue #731). The copies are contacts, not free text, so +-- suppression, bounces and verification apply to them exactly as to a lead. +CREATE TABLE campaign_lead_cc ( + campaign_id uuid NOT NULL, + contact_id uuid NOT NULL, + cc_contact_id uuid NOT NULL REFERENCES contacts (id) ON DELETE CASCADE, + position smallint NOT NULL DEFAULT 0, + -- A bounce attributed to this copy on this lead's thread. It is dropped + -- from later emails whatever the workspace's auto-suppress setting says. + bounced_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (campaign_id, contact_id, cc_contact_id), + FOREIGN KEY (campaign_id, contact_id) REFERENCES campaign_leads (campaign_id, contact_id) ON DELETE CASCADE, + CONSTRAINT campaign_lead_cc_not_self CHECK (cc_contact_id <> contact_id) +); + +-- Serves the contact FK's cascade and "is this contact copied on a lead here". +CREATE INDEX idx_campaign_lead_cc_cc ON campaign_lead_cc (cc_contact_id, campaign_id); + +-- A contact copied on another lead's thread is reached there, so their own +-- lead in the same campaign is held with source 'cc' instead of starting a +-- second sequence. +ALTER TABLE public.campaign_leads DROP CONSTRAINT IF EXISTS campaign_leads_pause_source_check; +-- NOT VALID: 000231 validates it in its own transaction. +ALTER TABLE public.campaign_leads + ADD CONSTRAINT campaign_leads_pause_source_check + CHECK (pause_source IS NULL OR pause_source IN ('manual', 'out_of_office', 'inbox_tagging', 'cc')) NOT VALID; + +-- Triggers rather than callers, so every path that enrols a lead (segments, +-- imports, contact edits, org transfer) holds a copied contact the same way. +CREATE FUNCTION campaign_lead_cc_hold() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + IF TG_OP = 'INSERT' THEN + -- A member's own live pause outranks this; anything else is replaced. + UPDATE campaign_leads cl + SET paused_at = CASE + WHEN cl.paused_at IS NOT NULL AND (cl.paused_until IS NULL OR cl.paused_until > NOW()) + THEN cl.paused_at ELSE NOW() END, + paused_until = NULL, + pause_reason = (SELECT c.email FROM contacts c WHERE c.id = NEW.contact_id), + pause_source = 'cc' + WHERE cl.campaign_id = NEW.campaign_id + AND cl.contact_id = NEW.cc_contact_id + AND NOT (cl.pause_source = 'manual' AND cl.paused_at IS NOT NULL + AND (cl.paused_until IS NULL OR cl.paused_until > NOW())); + RETURN NEW; + END IF; + -- Released once no lead in the campaign copies them any more. + UPDATE campaign_leads cl + SET paused_at = NULL, paused_until = NULL, pause_reason = NULL, pause_source = NULL + WHERE cl.campaign_id = OLD.campaign_id + AND cl.contact_id = OLD.cc_contact_id + AND cl.pause_source = 'cc' + AND NOT EXISTS ( + SELECT 1 FROM campaign_lead_cc x + WHERE x.campaign_id = OLD.campaign_id AND x.cc_contact_id = OLD.cc_contact_id + ); + RETURN OLD; +END; +$$; + +CREATE TRIGGER campaign_lead_cc_hold + AFTER INSERT OR DELETE ON campaign_lead_cc + FOR EACH ROW EXECUTE FUNCTION campaign_lead_cc_hold(); + +-- The other order: a contact already copied on a lead is enrolled later. +CREATE FUNCTION campaign_lead_enrol_cc_hold() RETURNS trigger +LANGUAGE plpgsql AS $$ +DECLARE + lead_email text; +BEGIN + SELECT c.email INTO lead_email + FROM campaign_lead_cc x + JOIN contacts c ON c.id = x.contact_id + WHERE x.campaign_id = NEW.campaign_id AND x.cc_contact_id = NEW.contact_id + ORDER BY x.created_at + LIMIT 1; + IF FOUND AND NEW.paused_at IS NULL THEN + NEW.paused_at := NOW(); + NEW.paused_until := NULL; + NEW.pause_reason := lead_email; + NEW.pause_source := 'cc'; + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER campaign_lead_enrol_cc_hold + BEFORE INSERT ON campaign_leads + FOR EACH ROW EXECUTE FUNCTION campaign_lead_enrol_cc_hold(); diff --git a/internal/infrastructure/db/migrations/000231_validate_lead_cc_hold_source.down.sql b/internal/infrastructure/db/migrations/000231_validate_lead_cc_hold_source.down.sql new file mode 100644 index 000000000..cbdfb637a --- /dev/null +++ b/internal/infrastructure/db/migrations/000231_validate_lead_cc_hold_source.down.sql @@ -0,0 +1,3 @@ +-- A validated constraint has no unvalidated form to return to; 000230's down +-- migration replaces it. +SELECT 1; diff --git a/internal/infrastructure/db/migrations/000231_validate_lead_cc_hold_source.up.sql b/internal/infrastructure/db/migrations/000231_validate_lead_cc_hold_source.up.sql new file mode 100644 index 000000000..4662b98e1 --- /dev/null +++ b/internal/infrastructure/db/migrations/000231_validate_lead_cc_hold_source.up.sql @@ -0,0 +1,3 @@ +-- Validates the pause_source CHECK 000230 re-added NOT VALID. A VALIDATE only +-- takes a SHARE UPDATE EXCLUSIVE lock, so writes continue while it scans. +ALTER TABLE public.campaign_leads VALIDATE CONSTRAINT campaign_leads_pause_source_check; diff --git a/internal/infrastructure/db/migrations/000232_placement_batches.down.sql b/internal/infrastructure/db/migrations/000232_placement_batches.down.sql new file mode 100644 index 000000000..c2210e302 --- /dev/null +++ b/internal/infrastructure/db/migrations/000232_placement_batches.down.sql @@ -0,0 +1,12 @@ +DROP INDEX IF EXISTS idx_placement_tests_sender_created; +DROP INDEX IF EXISTS idx_placement_tests_batch_sender; +DROP INDEX IF EXISTS idx_placement_tests_batch; +UPDATE placement_tests SET origin = 'manual' WHERE origin = 'batch'; +ALTER TABLE placement_tests + DROP CONSTRAINT placement_tests_origin_check, + ADD CONSTRAINT placement_tests_origin_check + CHECK (origin IN ('manual', 'monitor', 'admin', 'remote')), + DROP COLUMN batch_sender_id, + DROP COLUMN batch_id; +DROP TABLE IF EXISTS placement_batch_senders; +DROP TABLE IF EXISTS placement_batches; diff --git a/internal/infrastructure/db/migrations/000232_placement_batches.up.sql b/internal/infrastructure/db/migrations/000232_placement_batches.up.sql new file mode 100644 index 000000000..ac4cb400b --- /dev/null +++ b/internal/infrastructure/db/migrations/000232_placement_batches.up.sql @@ -0,0 +1,83 @@ +-- A placement batch runs the same test from many sending mailboxes. It is an +-- orchestration layer over placement_tests: each sender still gets its own +-- test, started by the backend a few at a time so a batch of thousands never +-- sends as a burst. +CREATE TABLE placement_batches ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + organization_id uuid NOT NULL REFERENCES organizations (id) ON DELETE CASCADE, + created_by uuid REFERENCES users (id) ON DELETE SET NULL, + campaign_id uuid REFERENCES campaigns (id) ON DELETE SET NULL, + sequence_id uuid REFERENCES sequences (id) ON DELETE SET NULL, + contact_id uuid REFERENCES contacts (id) ON DELETE SET NULL, + -- The copy, snapshotted at creation so every sender tests the same email. + subject text NOT NULL DEFAULT '', + body_plain text NOT NULL DEFAULT '', + body_html text NOT NULL DEFAULT '', + tracking text NOT NULL CHECK (tracking IN ('campaign', 'on', 'off', 'compare')), + panel text NOT NULL CHECK (panel IN ('instance', 'workspace', 'cloud')), + pace text NOT NULL CHECK (pace IN ('spaced', 'quick')), + families text[] NOT NULL DEFAULT '{}', + seed_ids uuid[] NOT NULL DEFAULT '{}', + on_unavailable text NOT NULL DEFAULT 'defer' CHECK (on_unavailable IN ('skip', 'defer')), + -- How the senders were chosen, for display; the snapshot is the sender rows. + selection jsonb NOT NULL DEFAULT '{}', + sender_count integer NOT NULL CHECK (sender_count >= 0), + max_credits integer NOT NULL DEFAULT 0 CHECK (max_credits >= 0), + credits_spent integer NOT NULL DEFAULT 0 CHECK (credits_spent >= 0), + status text NOT NULL CHECK (status IN ('queued', 'running', 'completed', 'completed_with_warnings', 'cancelled', 'failed')), + error text NOT NULL DEFAULT '', + -- Only an active batch is advanced by the runner. An imported batch lands + -- inactive, so it never starts sending on the instance it moved to. + active boolean NOT NULL DEFAULT false, + lease_until timestamptz, + last_tick_at timestamptz, + -- A deferred sender is retried until this moment, then skipped. + retry_until timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT NOW(), + started_at timestamptz, + finished_at timestamptz, + updated_at timestamptz NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_placement_batches_org ON placement_batches (organization_id, created_at DESC); +CREATE INDEX idx_placement_batches_active ON placement_batches (lease_until) WHERE active; + +-- One row per sender, written when the batch is created: the snapshot a later +-- tag or campaign edit cannot change. +CREATE TABLE placement_batch_senders ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + batch_id uuid NOT NULL REFERENCES placement_batches (id) ON DELETE CASCADE, + email_account_id uuid REFERENCES email_accounts (id) ON DELETE SET NULL, + sender_email text NOT NULL, + sender_domain text NOT NULL DEFAULT '', + -- Who hosts the sending mailbox, a mailhost value. + sender_family text NOT NULL DEFAULT '', + position integer NOT NULL, + status text NOT NULL DEFAULT 'queued' + CHECK (status IN ('queued', 'deferred', 'running', 'completed', 'skipped', 'failed', 'cancelled')), + -- A refusal identifier (placement_daily_budget, ...) and its sentence. + reason text NOT NULL DEFAULT '', + detail text NOT NULL DEFAULT '', + attempts integer NOT NULL DEFAULT 0, + next_attempt_at timestamptz NOT NULL DEFAULT NOW(), + started_at timestamptz, + finished_at timestamptz +); + +CREATE UNIQUE INDEX idx_placement_batch_senders_account ON placement_batch_senders (batch_id, email_account_id); +CREATE INDEX idx_placement_batch_senders_due ON placement_batch_senders (batch_id, position) + WHERE status IN ('queued', 'deferred'); +CREATE INDEX idx_placement_batch_senders_running ON placement_batch_senders (batch_id) WHERE status = 'running'; + +ALTER TABLE placement_tests + ADD COLUMN batch_id uuid REFERENCES placement_batches (id) ON DELETE SET NULL, + ADD COLUMN batch_sender_id uuid REFERENCES placement_batch_senders (id) ON DELETE SET NULL, + DROP CONSTRAINT placement_tests_origin_check, + ADD CONSTRAINT placement_tests_origin_check + CHECK (origin IN ('manual', 'monitor', 'admin', 'remote', 'batch')); + +CREATE INDEX idx_placement_tests_batch ON placement_tests (batch_id) WHERE batch_id IS NOT NULL; +CREATE INDEX idx_placement_tests_batch_sender ON placement_tests (batch_sender_id) WHERE batch_sender_id IS NOT NULL; +-- Fleet coverage reads each sender's latest test. +CREATE INDEX idx_placement_tests_sender_created ON placement_tests (sender_account_id, created_at DESC) + WHERE sender_account_id IS NOT NULL; diff --git a/internal/infrastructure/db/migrations/000233_warmup_spam_move_attribution.down.sql b/internal/infrastructure/db/migrations/000233_warmup_spam_move_attribution.down.sql new file mode 100644 index 000000000..85058ed4d --- /dev/null +++ b/internal/infrastructure/db/migrations/000233_warmup_spam_move_attribution.down.sql @@ -0,0 +1,3 @@ +DROP TABLE IF EXISTS mailbox_owner_activity; +DROP TABLE IF EXISTS warmup_spam_moves; +ALTER TABLE warmup_received DROP COLUMN IF EXISTS landed_spam; diff --git a/internal/infrastructure/db/migrations/000233_warmup_spam_move_attribution.up.sql b/internal/infrastructure/db/migrations/000233_warmup_spam_move_attribution.up.sql new file mode 100644 index 000000000..01d30531c --- /dev/null +++ b/internal/infrastructure/db/migrations/000233_warmup_spam_move_attribution.up.sql @@ -0,0 +1,47 @@ +-- landed_spam: the warmup email was in the recipient's spam folder when it arrived, so a spam label on it is the filter's, not the owner's. +ALTER TABLE warmup_received ADD COLUMN IF NOT EXISTS landed_spam boolean NOT NULL DEFAULT false; + +UPDATE warmup_received wr +SET landed_spam = true +WHERE wr.message_id <> '' + AND EXISTS ( + SELECT 1 FROM warmup_spam_reports sr + WHERE sr.reporter_account_id = wr.email_account_id + AND sr.message_id = wr.message_id + AND sr.report_type = 'spam_placement' + ); + +-- A spam strike on mail that arrived in spam was never the owner's act. +DELETE FROM warmup_tampering_events t +USING warmup_received wr +WHERE t.kind = 'spam_flag' + AND wr.email_account_id = t.email_account_id + AND wr.message_id = t.message_id + AND wr.landed_spam; + +-- A received warmup email moved to spam after arrival, held until the activity around it attributes it. +CREATE TABLE IF NOT EXISTS warmup_spam_moves ( + email_account_id uuid NOT NULL REFERENCES email_accounts(id) ON DELETE CASCADE, + message_id text NOT NULL, + sender_account_id uuid NOT NULL REFERENCES email_accounts(id) ON DELETE CASCADE, + received_at timestamptz NOT NULL, + observed_at timestamptz NOT NULL DEFAULT now(), + verdict text NOT NULL DEFAULT 'pending' + CHECK (verdict IN ('pending', 'owner', 'provider', 'unattributed')), + signals text[] NOT NULL DEFAULT '{}', + -- claimed_until: one consumer holds the move while it applies the verdict; decided_at: the verdict's effects are applied. + claimed_until timestamptz, + decided_at timestamptz, + PRIMARY KEY (email_account_id, message_id) +); +CREATE INDEX IF NOT EXISTS idx_warmup_spam_moves_undecided ON warmup_spam_moves (observed_at) WHERE decided_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_warmup_spam_moves_sender ON warmup_spam_moves (sender_account_id, observed_at); + +-- Five-minute buckets in which the owner acted on their own mail at the provider (read, unread, star), never Warmbly's echo. +CREATE TABLE IF NOT EXISTS mailbox_owner_activity ( + email_account_id uuid NOT NULL REFERENCES email_accounts(id) ON DELETE CASCADE, + bucket timestamptz NOT NULL, + events integer NOT NULL DEFAULT 1, + PRIMARY KEY (email_account_id, bucket) +); +CREATE INDEX IF NOT EXISTS idx_mailbox_owner_activity_bucket ON mailbox_owner_activity (bucket); diff --git a/internal/infrastructure/pubsub/events.go b/internal/infrastructure/pubsub/events.go index c5e34b94e..7a8566163 100644 --- a/internal/infrastructure/pubsub/events.go +++ b/internal/infrastructure/pubsub/events.go @@ -626,7 +626,8 @@ func (p *StreamingPublisher) PublishFormSubmission(ctx context.Context, orgID, f type PlacementTestEvent struct { BaseEvent OrgID string `json:"org_id"` - TestID string `json:"test_id"` + TestID string `json:"test_id,omitempty"` + BatchID string `json:"batch_id,omitempty"` CampaignID string `json:"campaign_id,omitempty"` Status string `json:"status"` } @@ -652,6 +653,25 @@ func (p *StreamingPublisher) PublishPlacementTest(ctx context.Context, orgID, te _ = p.client.Publish(ctx, TopicUserEvents, event, attrs) } +// PublishPlacementBatch emits the placement signal for a batch that moved +// without one of its tests moving (a sender skipped, the batch finished). +func (p *StreamingPublisher) PublishPlacementBatch(ctx context.Context, orgID, batchID uuid.UUID, status string) { + if p == nil || p.client == nil || orgID == uuid.Nil { + return + } + event := &PlacementTestEvent{ + BaseEvent: BaseEvent{EventType: EventPlacementTest, Timestamp: time.Now()}, + OrgID: orgID.String(), + BatchID: batchID.String(), + Status: status, + } + attrs := map[string]string{ + "org_id": orgID.String(), + "event_type": string(EventPlacementTest), + } + _ = p.client.Publish(ctx, TopicUserEvents, event, attrs) +} + // AutomationEvent is an org-scoped automation lifecycle/run signal. The web // client invalidates the ['automations'] queries on any "AUTOMATION" event. type AutomationEvent struct { diff --git a/internal/models/audit.go b/internal/models/audit.go index 9af262f07..af75393fc 100644 --- a/internal/models/audit.go +++ b/internal/models/audit.go @@ -53,7 +53,7 @@ const ( AuditEntityCampaign AuditEntityType = "campaign" // AuditEntityCampaignLead is ONE contact inside ONE campaign: the entity id // is the contact and metadata carries the campaign. Written when a member - // pauses or resumes that lead's flow. + // pauses or resumes that lead's flow, or changes who it copies. AuditEntityCampaignLead AuditEntityType = "campaign_lead" AuditEntityContact AuditEntityType = "contact" AuditEntityEmailAccount AuditEntityType = "email_account" @@ -147,10 +147,11 @@ const ( AuditEntityPoolLink AuditEntityType = "pool_link" AuditEntityCloudLink AuditEntityType = "cloud_link" - // Inbox placement: a test started or cancelled, and a campaign's + // Inbox placement: a test or batch started or cancelled, and a campaign's // scheduled test set up, changed or removed. AuditEntityPlacementTest AuditEntityType = "placement_test" AuditEntityPlacementMonitor AuditEntityType = "placement_monitor" + AuditEntityPlacementBatch AuditEntityType = "placement_batch" ) // AuditActor is the minimal identity of the member who performed an action, diff --git a/internal/models/campaign_lead_cc.go b/internal/models/campaign_lead_cc.go new file mode 100644 index 000000000..ebdc76219 --- /dev/null +++ b/internal/models/campaign_lead_cc.go @@ -0,0 +1,60 @@ +package models + +import ( + "time" + + "github.com/google/uuid" +) + +// CampaignLeadCC is a contact copied on every email one campaign sends one +// lead, so several people at one company share a single thread (issue #731). +type CampaignLeadCC struct { + ContactID uuid.UUID `json:"contact_id"` + Email string `json:"email"` + FirstName string `json:"first_name"` + LastName string `json:"last_name"` + Company string `json:"company,omitempty"` + // Status says whether the next email copies them: one of the + // LeadCCStatus constants. Anything but "active" is left off. + Status string `json:"status"` + BouncedAt *time.Time `json:"bounced_at,omitempty"` +} + +// Copied reports whether the next email to the lead carries this address. +func (c CampaignLeadCC) Copied() bool { return c.Status == LeadCCStatusActive } + +// Why a copy is or is not on the next email, in the order they are decided. +const ( + LeadCCStatusActive = "active" + // LeadCCStatusUnsubscribed is an opted-out or suppressed address. + LeadCCStatusUnsubscribed = "unsubscribed" + // LeadCCStatusBounced is an address that bounced on this thread or on any + // campaign email of its own. + LeadCCStatusBounced = "bounced" + // LeadCCStatusUndeliverable is an address verification refused, under the + // same rule the campaign applies to its leads. + LeadCCStatusUndeliverable = "undeliverable" +) + +// LeadHoldSourceCC holds a contact's own lead while they are copied on another +// lead's thread in the same campaign, so they never get two sequences. Written +// only by the campaign_lead_cc triggers (migration 000230). +const LeadHoldSourceCC = "cc" + +// SetCampaignLeadCC replaces the contacts copied on one lead. An empty list +// removes them all. +type SetCampaignLeadCC struct { + ContactIDs []string `json:"contact_ids"` +} + +// CampaignLeadCCSuggestion is a contact who looks like a colleague of the +// lead, offered first when picking who to copy. Reason is "company" when the +// company names match and "domain" when only the email domain does. +type CampaignLeadCCSuggestion struct { + ContactID uuid.UUID `json:"contact_id"` + Email string `json:"email"` + FirstName string `json:"first_name"` + LastName string `json:"last_name"` + Company string `json:"company,omitempty"` + Reason string `json:"reason"` +} diff --git a/internal/models/contact.go b/internal/models/contact.go index 802d85cd7..a96530f48 100644 --- a/internal/models/contact.go +++ b/internal/models/contact.go @@ -111,6 +111,8 @@ type ContactCampaignProgress struct { // Hold is the per-lead pause, set only while it is live. Present on any // status: a held lead that has also replied still reads "replied". Hold *LeadHold `json:"hold,omitempty"` + // CC is the contacts copied on every email to this lead in this campaign. + CC []CampaignLeadCC `json:"cc,omitempty"` } // LeadHold is one contact's flow parked inside one campaign. Source is @@ -328,7 +330,10 @@ const ( // MaxContactBulkSelection bounds how many contacts one "select all matching" // bulk action may resolve to. Past it the action is refused and the user // narrows the filters, so a stray click can never walk a whole workspace. -const MaxContactBulkSelection = 50000 +const MaxContactBulkSelection = 250000 + +// MaxContactBatchIDs bounds an explicit contact id list in one request body. +const MaxContactBatchIDs = 10000 // ContactSelection names the contacts a bulk action applies to. Either an // explicit id list (Contacts), or every contact matching a search (All + diff --git a/internal/models/contact_campaign_state.go b/internal/models/contact_campaign_state.go index b9c2a2aa5..da025d76d 100644 --- a/internal/models/contact_campaign_state.go +++ b/internal/models/contact_campaign_state.go @@ -37,6 +37,9 @@ type ContactCampaignState struct { // hand. The drawer renders it with "resume now" and "stop" next to it. Hold *LeadHold `json:"hold,omitempty"` + // CC is the contacts copied on every email to this lead in this campaign. + CC []CampaignLeadCC `json:"cc"` + // Next is nil once the flow has ended for the contact; EndedReason says why. Next *ContactNextAction `json:"next,omitempty"` EndedReason string `json:"ended_reason,omitempty"` diff --git a/internal/models/event.go b/internal/models/event.go index 3e1288600..f60c877a9 100644 --- a/internal/models/event.go +++ b/internal/models/event.go @@ -32,8 +32,11 @@ const ( JobEventTypeFlagsAdd JobEventType = "FLAGS_ADD" JobEventTypeFlagsRemove JobEventType = "FLAGS_REMOVE" JobEventTypeEmailUpdate JobEventType = "UPDATE_EMAIL" - JobEventTypeMailboxUpdate JobEventType = "UPDATE_MAILBOX" - JobEventTypeMailboxDelete JobEventType = "DELETE_MAILBOX" + // JobEventTypeFolderUpdate is a provider moving a message between folders + // without a full rescan of it (a Gmail label change). + JobEventTypeFolderUpdate JobEventType = "UPDATE_FOLDER" + JobEventTypeMailboxUpdate JobEventType = "UPDATE_MAILBOX" + JobEventTypeMailboxDelete JobEventType = "DELETE_MAILBOX" // JobEventTypeMailboxRename is a folder that kept its UIDVALIDITY under a // new name. Distinct from a delete plus an insert because the folder's // stored mail has to move with it rather than be orphaned. diff --git a/internal/models/event_variants.go b/internal/models/event_variants.go index 003274069..ea676a86a 100644 --- a/internal/models/event_variants.go +++ b/internal/models/event_variants.go @@ -43,6 +43,7 @@ var JobEventBodies = map[JobEventType]any{ JobEventTypeRemoveEmail: (*JobEventRemoveEmail)(nil), JobEventTypeFlagsAdd: (*JobEventFlags)(nil), JobEventTypeFlagsRemove: (*JobEventFlags)(nil), + JobEventTypeFolderUpdate: (*JobEventFolderUpdate)(nil), JobEventTypeMailboxUpdate: (*JobEventMailboxUpdate)(nil), JobEventTypeMailboxDelete: (*JobEventMailboxDelete)(nil), JobEventTypeMailboxRename: (*JobEventMailboxRename)(nil), diff --git a/internal/models/event_w_emails.go b/internal/models/event_w_emails.go index 6cd2ad0e3..570db3386 100644 --- a/internal/models/event_w_emails.go +++ b/internal/models/event_w_emails.go @@ -43,6 +43,16 @@ type JobEventFlags struct { Flags []string `json:"flags" avro:"flags"` } +// JobEventFolderUpdate reports the canonical folder the provider now has a +// message in. The consumer resolves it against provider_folder, so local filing +// survives unless the provider itself moved the message. +type JobEventFolderUpdate struct { + UserID uuid.UUID `json:"user_id" avro:"user_id"` + EmailID uuid.UUID `json:"email_id" avro:"email_id"` + ID uuid.UUID `json:"id" avro:"id"` + Folder string `json:"folder" avro:"folder"` +} + type JobEventEmailUpdate struct { UserID uuid.UUID `json:"user_id" avro:"user_id"` EmailID uuid.UUID `json:"email_id" avro:"email_id"` diff --git a/internal/models/placement.go b/internal/models/placement.go index 961c1b5bb..720c0867b 100644 --- a/internal/models/placement.go +++ b/internal/models/placement.go @@ -132,16 +132,19 @@ type PlacementTest struct { SequenceID *uuid.UUID `json:"sequence_id"` ContactID *uuid.UUID `json:"contact_id"` MonitorID *uuid.UUID `json:"monitor_id"` - Subject string `json:"subject"` - BodyHTML string `json:"body_html,omitempty"` - BodyPlain string `json:"body_plain,omitempty"` - OpenTracking bool `json:"open_tracking"` - LinkTracking bool `json:"link_tracking"` - CompareGroupID *uuid.UUID `json:"compare_group_id"` - Origin string `json:"origin"` - Panel string `json:"panel"` - Status string `json:"status"` - Error string `json:"error,omitempty"` + // BatchID and BatchSenderID name the batch that started the test. + BatchID *uuid.UUID `json:"batch_id"` + BatchSenderID *uuid.UUID `json:"-"` + Subject string `json:"subject"` + BodyHTML string `json:"body_html,omitempty"` + BodyPlain string `json:"body_plain,omitempty"` + OpenTracking bool `json:"open_tracking"` + LinkTracking bool `json:"link_tracking"` + CompareGroupID *uuid.UUID `json:"compare_group_id"` + Origin string `json:"origin"` + Panel string `json:"panel"` + Status string `json:"status"` + Error string `json:"error,omitempty"` // RemoteInstanceID is set on the cloud's copy of a test it runs for a // linked instance; RemoteTestID on the instance's copy, naming the cloud's. RemoteInstanceID *uuid.UUID `json:"-"` @@ -205,25 +208,28 @@ type PlacementCounts struct { } // Add counts one probe. -func (c *PlacementCounts) Add(folder string) { - c.Total++ +func (c *PlacementCounts) Add(folder string) { c.AddN(folder, 1) } + +// AddN counts n probes in one folder. +func (c *PlacementCounts) AddN(folder string, n int) { + c.Total += n switch folder { case PlacementFolderInbox: - c.Inbox++ + c.Inbox += n case PlacementFolderPromotions: - c.Promotions++ + c.Promotions += n case PlacementFolderOther: - c.Other++ + c.Other += n case PlacementFolderSpam: - c.Spam++ + c.Spam += n case PlacementFolderMissing: - c.Missing++ + c.Missing += n case PlacementFolderFailed: - c.Failed++ + c.Failed += n case PlacementFolderCancelled: - c.Cancelled++ + c.Cancelled += n default: - c.Pending++ + c.Pending += n } } diff --git a/internal/models/placement_batch.go b/internal/models/placement_batch.go new file mode 100644 index 000000000..0c3ecb040 --- /dev/null +++ b/internal/models/placement_batch.go @@ -0,0 +1,185 @@ +package models + +import ( + "time" + + "github.com/google/uuid" +) + +// PlacementOriginBatch is a test a placement batch started for one sender. +const PlacementOriginBatch = "batch" + +// Placement batch statuses. +const ( + PlacementBatchQueued = "queued" + PlacementBatchRunning = "running" + PlacementBatchCompleted = "completed" + PlacementBatchCompletedWithWarnings = "completed_with_warnings" + PlacementBatchCancelled = "cancelled" + PlacementBatchFailed = "failed" +) + +// PlacementBatchFinished reports whether a batch status is final. +func PlacementBatchFinished(status string) bool { + switch status { + case PlacementBatchCompleted, PlacementBatchCompletedWithWarnings, PlacementBatchCancelled, PlacementBatchFailed: + return true + } + return false +} + +// Statuses of one sender in a batch. +const ( + PlacementSenderQueued = "queued" + // PlacementSenderDeferred could not run yet (no daily headroom, busy, + // offline) and is retried later. + PlacementSenderDeferred = "deferred" + PlacementSenderRunning = "running" + PlacementSenderCompleted = "completed" + PlacementSenderSkipped = "skipped" + PlacementSenderFailed = "failed" + PlacementSenderCancelled = "cancelled" +) + +// What a batch does with a sender that cannot run when its turn comes. +const ( + PlacementUnavailableSkip = "skip" + PlacementUnavailableDefer = "defer" +) + +// Sender scopes a batch resolves on the server. +const ( + PlacementScopeCampaign = "campaign" + PlacementScopeWorkspace = "workspace" +) + +// Sampling modes. +const ( + PlacementSampleAll = "all" + PlacementSampleRandom = "random" + PlacementSamplePercent = "percent" + PlacementSamplePerDomain = "per_domain" + PlacementSamplePerProvider = "per_provider" +) + +// PlacementSenderScope selects a batch's senders on the server, so a fleet of +// thousands needs no id list. Filters narrow whichever scope is chosen. +type PlacementSenderScope struct { + Type string `json:"type"` + CampaignID *uuid.UUID `json:"campaign_id,omitempty"` + // Providers keeps mailboxes hosted by these mailhost families. + Providers []string `json:"providers,omitempty"` + Domains []string `json:"domains,omitempty"` + TagIDs []uuid.UUID `json:"tag_ids,omitempty"` + // IncludeInactive keeps disconnected mailboxes; they are skipped or + // deferred when their turn comes. + IncludeInactive bool `json:"include_inactive,omitempty"` + // UntestedDays keeps mailboxes with no delivered placement test in the + // last that many days. + UntestedDays int `json:"untested_days,omitempty"` +} + +// PlacementSample picks part of the resolved senders. +type PlacementSample struct { + Mode string `json:"mode"` + Count int `json:"count,omitempty"` + Percent int `json:"percent,omitempty"` + // Stratify spreads a random or percent sample across "provider" or + // "domain" in proportion to each group's size. + Stratify string `json:"stratify,omitempty"` +} + +// PlacementBatchSelection is how a batch's senders were chosen, stored for +// display. The sender rows are the snapshot. +type PlacementBatchSelection struct { + SenderAccountIDs int `json:"sender_account_ids,omitempty"` + Scope *PlacementSenderScope `json:"sender_scope,omitempty"` + Sample PlacementSample `json:"sample"` + // Matched is how many senders the scope resolved to before sampling. + Matched int `json:"matched"` +} + +// PlacementBatch runs one placement test from many senders. +type PlacementBatch struct { + ID uuid.UUID `json:"id"` + OrganizationID uuid.UUID `json:"-"` + CreatedBy *uuid.UUID `json:"created_by"` + CampaignID *uuid.UUID `json:"campaign_id"` + SequenceID *uuid.UUID `json:"sequence_id"` + ContactID *uuid.UUID `json:"contact_id"` + Subject string `json:"subject"` + BodyHTML string `json:"body_html,omitempty"` + BodyPlain string `json:"body_plain,omitempty"` + Tracking string `json:"tracking"` + Panel string `json:"panel"` + Pace string `json:"pace"` + Families []string `json:"families"` + SeedIDs []uuid.UUID `json:"seed_ids"` + OnUnavailable string `json:"on_unavailable"` + Selection PlacementBatchSelection `json:"selection"` + SenderCount int `json:"sender_count"` + MaxCredits int `json:"max_credits"` + CreditsSpent int `json:"credits_spent"` + Status string `json:"status"` + Error string `json:"error,omitempty"` + Active bool `json:"-"` + LastTickAt *time.Time `json:"-"` + RetryUntil time.Time `json:"retry_until"` + CreatedAt time.Time `json:"created_at"` + StartedAt *time.Time `json:"started_at"` + FinishedAt *time.Time `json:"finished_at"` +} + +// PlacementBatchSender is one sender of a batch. +type PlacementBatchSender struct { + ID uuid.UUID `json:"id"` + BatchID uuid.UUID `json:"batch_id"` + EmailAccountID *uuid.UUID `json:"email_account_id"` + SenderEmail string `json:"sender_email"` + SenderDomain string `json:"sender_domain"` + SenderFamily string `json:"sender_family"` + Position int `json:"-"` + Status string `json:"status"` + Reason string `json:"reason,omitempty"` + Detail string `json:"detail,omitempty"` + Attempts int `json:"attempts"` + NextAttemptAt time.Time `json:"next_attempt_at"` + StartedAt *time.Time `json:"started_at"` + FinishedAt *time.Time `json:"finished_at"` +} + +// PlacementBatchProgress counts a batch's senders by status. +type PlacementBatchProgress struct { + Total int `json:"total"` + Queued int `json:"queued"` + Deferred int `json:"deferred"` + Running int `json:"running"` + Completed int `json:"completed"` + Skipped int `json:"skipped"` + Failed int `json:"failed"` + Cancelled int `json:"cancelled"` +} + +// Add counts one sender. +func (p *PlacementBatchProgress) Add(status string, n int) { + p.Total += n + switch status { + case PlacementSenderQueued: + p.Queued += n + case PlacementSenderDeferred: + p.Deferred += n + case PlacementSenderRunning: + p.Running += n + case PlacementSenderCompleted: + p.Completed += n + case PlacementSenderSkipped: + p.Skipped += n + case PlacementSenderFailed: + p.Failed += n + case PlacementSenderCancelled: + p.Cancelled += n + } +} + +// Open is how many senders still have something to do. +func (p PlacementBatchProgress) Open() int { return p.Queued + p.Deferred + p.Running } diff --git a/internal/models/segment.go b/internal/models/segment.go index 99173cf10..e8078a179 100644 --- a/internal/models/segment.go +++ b/internal/models/segment.go @@ -148,7 +148,7 @@ var SegmentFieldCatalog = []SegmentFieldSpec{ {Field: "esp_provider", Label: "Email provider family", Group: "Contact", Kind: SegmentFieldEnum, Options: []string{"gmail", "outlook", "other"}, OptionLabels: map[string]string{"gmail": "Google", "outlook": "Microsoft", "other": "Other"}}, {Field: "created_at", Label: "Created", Group: "Contact", Kind: SegmentFieldDate}, {Field: "updated_at", Label: "Updated", Group: "Contact", Kind: SegmentFieldDate}, - {Field: "category", Label: "Category", Group: "Contact", Kind: SegmentFieldCategory}, + {Field: "category", Label: "Label", Group: "Contact", Kind: SegmentFieldCategory}, {Field: "company", Label: "Company name", Group: "Company", Kind: SegmentFieldText}, diff --git a/internal/models/unibox.go b/internal/models/unibox.go index 06445c031..28bab199e 100644 --- a/internal/models/unibox.go +++ b/internal/models/unibox.go @@ -2,6 +2,7 @@ package models import ( "slices" + "strings" "time" "github.com/google/uuid" @@ -160,6 +161,24 @@ type EmailMessageStoreData struct { CreatedAt time.Time `json:"created_at" avro:"created_at"` } +// ValidText makes every text field storable: Postgres refuses invalid UTF-8 +// and NUL, and an unlabelled 8-bit header or body carries both. +func (e *EmailMessageStoreData) ValidText() { + for _, s := range []*string{&e.FolderPath, &e.Folder, &e.ProviderFolder, &e.ThreadID, &e.MessageID, + &e.GmailID, &e.ParentID, &e.Subject, &e.Snippet, &e.BodyText} { + *s = validText(*s) + } + for _, list := range [][]string{e.Flags, e.BCC, e.CC, e.FromAddr, e.InReplyTo, e.ReplyTo, e.ToAddr} { + for i := range list { + list[i] = validText(list[i]) + } + } +} + +func validText(s string) string { + return strings.ToValidUTF8(strings.ReplaceAll(s, "\x00", ""), "\uFFFD") +} + type EmailMessageStoreDataPreview struct { ID uuid.UUID `json:"id"` EmailID uuid.UUID `json:"email_id"` @@ -576,6 +595,9 @@ type SeenRelayTarget struct { // isRead. const FlagSeen = `\Seen` +// FlagFlagged is the star: Gmail STARRED, IMAP \Flagged. +const FlagFlagged = `\Flagged` + // SeenFromFlags reads a message's read state out of its flags. The stored // `seen` column has to follow the provider: mail the customer already read in // their own client is read in Warmbly, and a copy the worker files in Sent diff --git a/internal/models/unibox_validtext_test.go b/internal/models/unibox_validtext_test.go new file mode 100644 index 000000000..ce7979194 --- /dev/null +++ b/internal/models/unibox_validtext_test.go @@ -0,0 +1,28 @@ +package models + +import ( + "testing" + "unicode/utf8" +) + +func TestEmailMessageStoreDataValidText(t *testing.T) { + m := &EmailMessageStoreData{ + Subject: "caf\xe9 \xe2\xa2\x77", + Snippet: "a\x00b", + BodyText: "ok ✓", + FromAddr: []string{"Ren\xe9 "}, + } + m.ValidText() + + for _, s := range append([]string{m.Subject, m.Snippet, m.BodyText}, m.FromAddr...) { + if !utf8.ValidString(s) { + t.Fatalf("still invalid: %q", s) + } + } + if m.Snippet != "ab" { + t.Fatalf("NUL kept: %q", m.Snippet) + } + if m.BodyText != "ok ✓" { + t.Fatalf("valid text changed: %q", m.BodyText) + } +} diff --git a/internal/models/warmup.go b/internal/models/warmup.go index a2db2a9fc..2b5c7c0f6 100644 --- a/internal/models/warmup.go +++ b/internal/models/warmup.go @@ -339,13 +339,13 @@ type WarmupHealthMetrics struct { BounceRate float64 `json:"bounce_rate"` // DeletionsLast7d and SpamFlagsLast7d are warmup messages this mailbox - // received and then deleted or flagged as spam. TamperingStrikes weighs - // them: a spam flag counts double, because nobody flags mail by accident. + // received and then deleted or moved to spam. TamperingStrikes weighs them + // equally: no provider says who moved a message into spam. DeletionsLast7d int `json:"deletions_last_7d"` SpamFlagsLast7d int `json:"spam_flags_last_7d"` } // TamperingStrikes is the weighted harm count the tampering band reads. func (m *WarmupHealthMetrics) TamperingStrikes() int { - return m.DeletionsLast7d + 2*m.SpamFlagsLast7d + return m.DeletionsLast7d + m.SpamFlagsLast7d } diff --git a/internal/models/worker.go b/internal/models/worker.go index a28aad697..c16e51810 100644 --- a/internal/models/worker.go +++ b/internal/models/worker.go @@ -111,6 +111,9 @@ type EmailSendError struct { UserTitle string `json:"user_title,omitempty" avro:"user_title"` UserMessage string `json:"user_message,omitempty" avro:"user_message"` ActionRequired string `json:"action_required,omitempty" avro:"action_required"` + // Recipient is the address a refusal named, when the server refused one + // recipient rather than the message. + Recipient string `json:"recipient,omitempty" avro:"recipient"` } // SendEmailResult is the result from worker after sending email diff --git a/internal/repository/campaign_lead_cc_live_test.go b/internal/repository/campaign_lead_cc_live_test.go new file mode 100644 index 000000000..b184bf60b --- /dev/null +++ b/internal/repository/campaign_lead_cc_live_test.go @@ -0,0 +1,264 @@ +package repository + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/models" +) + +// Contacts copied on one lead's emails (issue #731). The hold that keeps a +// copied contact from getting a second sequence lives in triggers, so it is +// asserted against the real routing query. +// +// WARMBLY_TEST_DB=postgres://warmbly:warmbly@localhost:15432/?sslmode=disable \ +// go test ./internal/repository/ -run LiveLeadCC -v + +func routedIDs(pairs []ContactSequencePair) map[uuid.UUID]bool { + out := map[uuid.UUID]bool{} + for _, p := range pairs { + out[p.ContactID] = true + } + return out +} + +func TestLiveLeadCCHoldsTheCopiedLeadUntilReleased(t *testing.T) { + _, pool := liveContactDB(t) + f := newRoutedPairsFixture(t, pool, 3) + repo := NewCampaignProgressRepository(pool) + ctx := context.Background() + a, b, c := f.leads[0], f.leads[1], f.leads[2] + + if err := repo.SetLeadCC(ctx, f.org, f.campaign, a, []uuid.UUID{b}); err != nil { + t.Fatalf("SetLeadCC: %v", err) + } + pairs, _, _ := f.find(t, nil, 25) + got := routedIDs(pairs) + if !got[a] || got[b] || !got[c] { + t.Fatalf("routed %v; want the lead and the uncopied lead, not the copied one", got) + } + hold, err := repo.GetLeadHold(ctx, f.campaign, b) + if err != nil || hold == nil || hold.Source != models.LeadHoldSourceCC { + t.Fatalf("copied lead's hold = %+v, %v; want a cc hold", hold, err) + } + // Nothing is left to wait for, so a campaign of only copied leads can finish. + if n, err := repo.CountHeldLeads(ctx, f.campaign); err != nil || n != 0 { + t.Fatalf("CountHeldLeads = %d, %v; want 0 for a cc hold", n, err) + } + + cc, err := repo.ListLeadCC(ctx, f.campaign, a) + if err != nil || len(cc) != 1 || cc[0].ContactID != b || !cc[0].Copied() { + t.Fatalf("ListLeadCC = %+v, %v; want the copy, active", cc, err) + } + + if err := repo.SetLeadCC(ctx, f.org, f.campaign, a, nil); err != nil { + t.Fatalf("SetLeadCC to nobody: %v", err) + } + if hold, err := repo.GetLeadHold(ctx, f.campaign, b); err != nil || hold != nil { + t.Fatalf("hold after release = %+v, %v; want none", hold, err) + } + pairs, _, _ = f.find(t, nil, 25) + if !routedIDs(pairs)[b] { + t.Fatal("a released copy was not routed again") + } +} + +// A contact copied first and enrolled afterwards is held on arrival, whichever +// path enrols them. +func TestLiveLeadCCHoldsAContactEnrolledAfterBeingCopied(t *testing.T) { + _, pool := liveContactDB(t) + f := newRoutedPairsFixture(t, pool, 2) + repo := NewCampaignProgressRepository(pool) + ctx := context.Background() + a, b := f.leads[0], f.leads[1] + + if _, err := pool.Exec(ctx, `DELETE FROM campaign_leads WHERE campaign_id = $1 AND contact_id = $2`, f.campaign, b); err != nil { + t.Fatalf("drop lead: %v", err) + } + if err := repo.SetLeadCC(ctx, f.org, f.campaign, a, []uuid.UUID{b}); err != nil { + t.Fatalf("SetLeadCC: %v", err) + } + if _, err := pool.Exec(ctx, `INSERT INTO campaign_leads (campaign_id, contact_id) VALUES ($1, $2)`, f.campaign, b); err != nil { + t.Fatalf("enrol: %v", err) + } + if hold, err := repo.GetLeadHold(ctx, f.campaign, b); err != nil || hold == nil || hold.Source != models.LeadHoldSourceCC { + t.Fatalf("hold on enrol = %+v, %v; want a cc hold", hold, err) + } + // Removing the lead that copies them releases them too. + if _, err := pool.Exec(ctx, `DELETE FROM campaign_leads WHERE campaign_id = $1 AND contact_id = $2`, f.campaign, a); err != nil { + t.Fatalf("drop copying lead: %v", err) + } + if hold, err := repo.GetLeadHold(ctx, f.campaign, b); err != nil || hold != nil { + t.Fatalf("hold after the copying lead left = %+v, %v; want none", hold, err) + } +} + +func TestLiveLeadCCRefusals(t *testing.T) { + _, pool := liveContactDB(t) + f := newRoutedPairsFixture(t, pool, 3) + other := newRoutedPairsFixture(t, pool, 1) + repo := NewCampaignProgressRepository(pool) + ctx := context.Background() + a, b, c := f.leads[0], f.leads[1], f.leads[2] + + cases := []struct { + name string + lead uuid.UUID + cc []uuid.UUID + want error + }{ + {"self", a, []uuid.UUID{a}, ErrLeadCCSelf}, + {"another workspace's contact", a, []uuid.UUID{other.leads[0]}, ErrLeadCCContactNotFound}, + {"not a lead", other.leads[0], nil, ErrLeadNotInCampaign}, + } + for _, tc := range cases { + if err := repo.SetLeadCC(ctx, f.org, f.campaign, tc.lead, tc.cc); !errors.Is(err, tc.want) { + t.Fatalf("%s: err = %v, want %v", tc.name, err, tc.want) + } + } + + if err := repo.SetLeadCC(ctx, f.org, f.campaign, a, []uuid.UUID{b}); err != nil { + t.Fatalf("SetLeadCC: %v", err) + } + if err := repo.SetLeadCC(ctx, f.org, f.campaign, b, []uuid.UUID{c}); !errors.Is(err, ErrLeadCCLeadIsCopied) { + t.Fatalf("copies on a copied lead: err = %v, want ErrLeadCCLeadIsCopied", err) + } + if err := repo.SetLeadCC(ctx, f.org, f.campaign, c, []uuid.UUID{a}); !errors.Is(err, ErrLeadCCHasCopies) { + t.Fatalf("copying a lead with copies: err = %v, want ErrLeadCCHasCopies", err) + } + // A refused write leaves the list as it was. + if cc, _ := repo.ListLeadCC(ctx, f.campaign, a); len(cc) != 1 || cc[0].ContactID != b { + t.Fatalf("list after refusals = %+v; want it unchanged", cc) + } +} + +func TestLiveLeadCCStatusAndBounces(t *testing.T) { + _, pool := liveContactDB(t) + f := newRoutedPairsFixture(t, pool, 3) + repo := NewCampaignProgressRepository(pool) + ctx := context.Background() + a, b, c := f.leads[0], f.leads[1], f.leads[2] + + if err := repo.SetLeadCC(ctx, f.org, f.campaign, a, []uuid.UUID{b, c}); err != nil { + t.Fatalf("SetLeadCC: %v", err) + } + if _, err := pool.Exec(ctx, `UPDATE contacts SET subscribed = false WHERE id = $1`, b); err != nil { + t.Fatalf("unsubscribe: %v", err) + } + var cEmail string + if err := pool.QueryRow(ctx, `SELECT email FROM contacts WHERE id = $1`, c).Scan(&cEmail); err != nil { + t.Fatalf("read email: %v", err) + } + id, err := repo.MarkLeadCCBounced(ctx, f.campaign, a, strings.ToUpper(cEmail)) + if err != nil || id == nil || *id != c { + t.Fatalf("MarkLeadCCBounced = %v, %v; want the copy", id, err) + } + if id, err := repo.MarkLeadCCBounced(ctx, f.campaign, a, "nobody@test.local"); err != nil || id != nil { + t.Fatalf("MarkLeadCCBounced on a stranger = %v, %v; want nil", id, err) + } + + cc, err := repo.ListLeadCC(ctx, f.campaign, a) + if err != nil || len(cc) != 2 { + t.Fatalf("ListLeadCC = %+v, %v", cc, err) + } + want := map[uuid.UUID]string{b: models.LeadCCStatusUnsubscribed, c: models.LeadCCStatusBounced} + for _, x := range cc { + if x.Status != want[x.ContactID] || x.Copied() { + t.Fatalf("copy %s status %q; want %q and not copied", x.ContactID, x.Status, want[x.ContactID]) + } + } +} + +func TestLiveLeadForCopiedReplyFindsTheLeadsThread(t *testing.T) { + _, pool := liveContactDB(t) + f := newRoutedPairsFixture(t, pool, 2) + repo := NewCampaignProgressRepository(pool) + ctx := context.Background() + a, b := f.leads[0], f.leads[1] + + if err := repo.SetLeadCC(ctx, f.org, f.campaign, a, []uuid.UUID{b}); err != nil { + t.Fatalf("SetLeadCC: %v", err) + } + if _, err := pool.Exec(ctx, `UPDATE campaign_leads SET email_account_id = $3 WHERE campaign_id = $1 AND contact_id = $2`, + f.campaign, a, f.mailbox); err != nil { + t.Fatalf("bind sender: %v", err) + } + if _, err := pool.Exec(ctx, `INSERT INTO campaign_contact_progress (campaign_id, contact_id, sequence_id, sent_at) + VALUES ($1, $2, $3, NOW())`, f.campaign, a, f.step); err != nil { + t.Fatalf("progress: %v", err) + } + + ref, err := repo.LeadForCopiedReply(ctx, b, f.mailbox) + if err != nil || ref == nil || ref.CampaignID != f.campaign || ref.ContactID != a || ref.SequenceID != f.step { + t.Fatalf("LeadForCopiedReply = %+v, %v; want the lead's step", ref, err) + } + // Another mailbox never wrote to the lead, so it is no evidence. + if ref, err := repo.LeadForCopiedReply(ctx, b, uuid.New()); err != nil || ref != nil { + t.Fatalf("LeadForCopiedReply from another mailbox = %+v, %v; want nil", ref, err) + } +} + +// The Leads list carries each lead's copies, and a copied lead reads paused. +func TestLiveLeadCCShowsInTheLeadsList(t *testing.T) { + handle, pool := liveContactDB(t) + f := newRoutedPairsFixture(t, pool, 2) + repo := NewCampaignProgressRepository(pool) + contacts := NewContactRepostory(handle) + ctx := context.Background() + a, b := f.leads[0], f.leads[1] + + if err := repo.SetLeadCC(ctx, f.org, f.campaign, a, []uuid.UUID{b}); err != nil { + t.Fatalf("SetLeadCC: %v", err) + } + res, xerr := contacts.Search(ctx, f.org.String(), nil, nil, models.SearchContacts{ + CampaignIDs: []string{f.campaign.String()}, + }, 25) + if xerr != nil { + t.Fatalf("search: %v", xerr) + } + byID := map[uuid.UUID]*models.ContactCampaignProgress{} + for i := range res.Data { + byID[res.Data[i].ID] = res.Data[i].CampaignLead + } + if lead := byID[a]; lead == nil || len(lead.CC) != 1 || lead.CC[0].ContactID != b || lead.CC[0].Status != models.LeadCCStatusActive { + t.Fatalf("the copying lead reads %+v; want its one active copy", lead) + } + if lead := byID[b]; lead == nil || lead.Status != models.LeadStatusPaused || lead.Hold == nil || lead.Hold.Source != models.LeadHoldSourceCC { + t.Fatalf("the copied lead reads %+v; want paused with a cc hold", lead) + } +} + +// A refused copy walks the step back without spending the lead's attempt, and +// a campaign-wide copy that bounced here is reported so the send leaves it off. +func TestLiveLeadCCRefusedCopyCostsNoAttempt(t *testing.T) { + _, pool := liveContactDB(t) + f := newRoutedPairsFixture(t, pool, 1) + repo := NewCampaignProgressRepository(pool) + ctx := context.Background() + lead := f.leads[0] + + if _, err := pool.Exec(ctx, `INSERT INTO campaign_contact_progress (campaign_id, contact_id, sequence_id, sent_at, dispatched_at) + VALUES ($1, $2, $3, NOW(), NOW())`, f.campaign, lead, f.step); err != nil { + t.Fatalf("progress: %v", err) + } + attempts, _, rolled, err := repo.WalkBackSend(ctx, f.campaign, lead, f.step, "copy refused", false) + if err != nil || !rolled || attempts != 0 { + t.Fatalf("WalkBackSend = %d, %v, %v; want rolled back with no attempt", attempts, rolled, err) + } + + if _, err := pool.Exec(ctx, `INSERT INTO deliverability_events (organization_id, campaign_id, event_type, recipient_email, idempotency_key) + VALUES ($1, $2, 'bounce', 'Crm@Acme.test', $3)`, f.org, f.campaign, "test:"+uuid.NewString()); err != nil { + t.Fatalf("event: %v", err) + } + t.Cleanup(func() { + _, _ = pool.Exec(context.Background(), `DELETE FROM deliverability_events WHERE organization_id = $1`, f.org) + }) + got, err := repo.BouncedCopyAddresses(ctx, f.campaign, []string{"crm@acme.test", "boss@acme.test"}) + if err != nil || !got["crm@acme.test"] || got["boss@acme.test"] { + t.Fatalf("BouncedCopyAddresses = %v, %v; want only the bounced address", got, err) + } +} diff --git a/internal/repository/http_sync_context.go b/internal/repository/http_sync_context.go index 6eec007bb..3f137c347 100644 --- a/internal/repository/http_sync_context.go +++ b/internal/repository/http_sync_context.go @@ -25,6 +25,17 @@ type StoredFolderMessage struct { MessageID string `json:"message_id"` } +// ProviderFolderMessage is one message the platform holds for a mailbox whose +// provider keys messages by id (Gmail): the row, that id, the folder the +// provider last had it in, and its date. The Gmail folder reconciliation +// checks these against where Gmail has each message now. +type ProviderFolderMessage struct { + ID uuid.UUID `json:"id"` + ProviderID string `json:"provider_id"` + ProviderFolder string `json:"provider_folder"` + InternalDate time.Time `json:"internal_date"` +} + // SyncContextRepository is the worker's view of the control plane's answer to // "is this a conversation the mailbox owns?" and "what does the platform still // hold for this folder?". Workers cannot reach Postgres, so the only @@ -36,6 +47,9 @@ type SyncContextRepository interface { // UIDs a UIDVALIDITY change voided are not compared, and therefore not // deleted. ListFolderMessages(ctx context.Context, userID, emailID uuid.UUID, folderPath string, uidValidity uint32) ([]StoredFolderMessage, error) + // ListProviderFolderMessages returns the newest rows the provider last + // placed in one of folders, at most limit of them. + ListProviderFolderMessages(ctx context.Context, userID, emailID uuid.UUID, folders []string, limit int) ([]ProviderFolderMessage, error) } type httpSyncContextRepository struct { @@ -45,8 +59,9 @@ type httpSyncContextRepository struct { } // NewHTTPSyncContextRepository returns the worker-side proxy for -// GET {BaseURL}/api/v1/internal/sync/own-conversation and -// GET {BaseURL}/api/v1/internal/sync/folder-messages. +// GET {BaseURL}/api/v1/internal/sync/own-conversation, +// GET {BaseURL}/api/v1/internal/sync/folder-messages and +// GET {BaseURL}/api/v1/internal/sync/provider-folder-messages. func NewHTTPSyncContextRepository(baseURL, token string) (SyncContextRepository, error) { if baseURL == "" { return nil, errors.New("sync_context.http: baseURL is required") @@ -121,3 +136,35 @@ func (r *httpSyncContextRepository) ListFolderMessages(ctx context.Context, user } return out.Messages, nil } + +// ListProviderFolderMessages asks the control plane which rows the provider +// last placed in folders. +func (r *httpSyncContextRepository) ListProviderFolderMessages(ctx context.Context, userID, emailID uuid.UUID, folders []string, limit int) ([]ProviderFolderMessage, error) { + q := url.Values{} + q.Set("user_id", userID.String()) + q.Set("email_id", emailID.String()) + q.Set("folders", strings.Join(folders, ",")) + q.Set("limit", strconv.Itoa(limit)) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, r.baseURL+"/api/v1/internal/sync/provider-folder-messages?"+q.Encode(), nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+r.token) + req.Header.Set("User-Agent", "warmbly-worker/sync-context-http") + resp, err := r.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + return nil, fmt.Errorf("sync_context.http: provider folder messages: %d %s", resp.StatusCode, strings.TrimSpace(string(b))) + } + var out struct { + Messages []ProviderFolderMessage `json:"messages"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return nil, err + } + return out.Messages, nil +} diff --git a/internal/repository/pg_admin.go b/internal/repository/pg_admin.go index 37c382173..09b7ecac5 100644 --- a/internal/repository/pg_admin.go +++ b/internal/repository/pg_admin.go @@ -1146,7 +1146,13 @@ func (r *adminRepository) BlockAccount(ctx context.Context, accountID uuid.UUID, // UnblockAccount unblocks an account from warmup pools func (r *adminRepository) UnblockAccount(ctx context.Context, accountID uuid.UUID) error { - _, err := r.db.Exec(ctx, ` + tx, err := r.db.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + + if _, err := tx.Exec(ctx, ` UPDATE warmup_pool_participants SET blocked_at = NULL, blocked_reason = NULL, @@ -1156,7 +1162,19 @@ func (r *adminRepository) UnblockAccount(ctx context.Context, accountID uuid.UUI last_health_evaluated_at = NOW(), last_health_score = 0 WHERE email_account_id = $1 - `, accountID) + `, accountID); err != nil { + return err + } + if err := forgiveWarmupStrikes(ctx, tx, accountID); err != nil { + return err + } + return tx.Commit(ctx) +} + +// forgiveWarmupStrikes clears the tampering strikes behind a hold an admin +// lifted, or the next health evaluation reimposes it on the same strikes. +func forgiveWarmupStrikes(ctx context.Context, tx pgx.Tx, accountID uuid.UUID) error { + _, err := tx.Exec(ctx, `DELETE FROM warmup_tampering_events WHERE email_account_id = $1`, accountID) return err } @@ -1297,6 +1315,9 @@ func (r *adminRepository) ReviewAppeal(ctx context.Context, appealID uuid.UUID, if err != nil { return err } + if err := forgiveWarmupStrikes(ctx, tx, accountID); err != nil { + return err + } _, _ = tx.Exec(ctx, ` INSERT INTO warmup_admin_actions (admin_user_id, email_account_id, action, reason) diff --git a/internal/repository/pg_campaign_lead_cc.go b/internal/repository/pg_campaign_lead_cc.go new file mode 100644 index 000000000..b40f33dfd --- /dev/null +++ b/internal/repository/pg_campaign_lead_cc.go @@ -0,0 +1,285 @@ +package repository + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/warmbly/warmbly/internal/models" +) + +// What SetLeadCC refuses. The service turns each into its own error code. +var ( + // ErrLeadCCContactNotFound is a copy that is not a contact of the workspace. + ErrLeadCCContactNotFound = errors.New("a contact to copy was not found") + // ErrLeadCCSelf is the lead copied on their own emails. + ErrLeadCCSelf = errors.New("a lead cannot be copied on their own emails") + // ErrLeadCCLeadIsCopied is a lead already copied on another lead's thread + // in the campaign; their own emails are held, so copies would reach nobody. + ErrLeadCCLeadIsCopied = errors.New("the lead is copied on another lead in this campaign") + // ErrLeadCCHasCopies is a contact whose own lead copies others: holding it + // would silently strand the people it copies. + ErrLeadCCHasCopies = errors.New("a contact to copy has copies of their own in this campaign") +) + +// CopiedLeadRef names the lead, and the step, a copied contact's reply answers. +type CopiedLeadRef struct { + CampaignID uuid.UUID + ContactID uuid.UUID + SequenceID uuid.UUID +} + +// personalMailDomainsSQL never identify a company, so a shared one is not a +// reason to suggest two contacts are colleagues. +const personalMailDomainsSQL = `'gmail.com','googlemail.com','yahoo.com','yahoo.de','hotmail.com','hotmail.de','outlook.com','outlook.de','live.com','live.de','msn.com','aol.com','icloud.com','me.com','gmx.com','gmx.de','gmx.net','web.de','t-online.de','freenet.de','posteo.de','mailbox.org','proton.me','protonmail.com','mail.com','yandex.com'` + +// leadCCStatusSQL derives models.LeadCCStatus* for a copied contact aliased c +// on the campaign_lead_cc row aliased x. cp is the bound campaign id. +func leadCCStatusSQL(cp string) string { + return `CASE + WHEN c.subscribed IS FALSE + OR recipient_suppressed((SELECT organization_id FROM campaigns WHERE id = ` + cp + `), c.email) + THEN '` + models.LeadCCStatusUnsubscribed + `' + WHEN x.bounced_at IS NOT NULL + OR EXISTS (SELECT 1 FROM campaign_contact_progress b WHERE b.contact_id = c.id AND b.bounced_at IS NOT NULL) + THEN '` + models.LeadCCStatusBounced + `' + WHEN ` + undeliverableClause(cp) + ` THEN '` + models.LeadCCStatusUndeliverable + `' + ELSE '` + models.LeadCCStatusActive + `' + END` +} + +// leadCCSelectSQL lists one lead's copies in the order they were chosen. $1 is +// the campaign, $2 the lead. +func leadCCSelectSQL() string { + return ` + SELECT c.id, c.email, c.first_name, c.last_name, c.company, ` + leadCCStatusSQL("$1") + `, x.bounced_at + FROM campaign_lead_cc x + JOIN contacts c ON c.id = x.cc_contact_id + WHERE x.campaign_id = $1 AND x.contact_id = $2 + ORDER BY x.position, x.created_at` +} + +// listLeadCC is shared by the campaign and contact repositories, so the send +// path and the drawer read one definition. +func listLeadCC(ctx context.Context, pool *pgxpool.Pool, campaignID, contactID uuid.UUID) ([]models.CampaignLeadCC, error) { + rows, err := pool.Query(ctx, leadCCSelectSQL(), campaignID, contactID) + if err != nil { + return nil, err + } + defer rows.Close() + out := []models.CampaignLeadCC{} + for rows.Next() { + var cc models.CampaignLeadCC + if err := rows.Scan(&cc.ContactID, &cc.Email, &cc.FirstName, &cc.LastName, &cc.Company, &cc.Status, &cc.BouncedAt); err != nil { + return nil, err + } + out = append(out, cc) + } + return out, rows.Err() +} + +func (r *campaignProgressRepository) ListLeadCC(ctx context.Context, campaignID, contactID uuid.UUID) ([]models.CampaignLeadCC, error) { + return listLeadCC(ctx, r.db, campaignID, contactID) +} + +func (r *campaignProgressRepository) SetLeadCC(ctx context.Context, orgID, campaignID, contactID uuid.UUID, ccIDs []uuid.UUID) error { + tx, err := r.db.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + + // One writer per campaign, so two edits cannot build a chain of copies + // that each checked against the other's snapshot. + if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended('campaign_lead_cc:' || $1::text, 0))`, campaignID); err != nil { + return err + } + + var isLead bool + if err := tx.QueryRow(ctx, ` + SELECT EXISTS ( + SELECT 1 FROM campaign_leads cl + JOIN campaigns cam ON cam.id = cl.campaign_id AND cam.organization_id = $1 + WHERE cl.campaign_id = $2 AND cl.contact_id = $3 + )`, orgID, campaignID, contactID).Scan(&isLead); err != nil { + return err + } + if !isLead { + return ErrLeadNotInCampaign + } + + if len(ccIDs) > 0 { + for _, id := range ccIDs { + if id == contactID { + return ErrLeadCCSelf + } + } + var found int + if err := tx.QueryRow(ctx, + `SELECT COUNT(*) FROM contacts WHERE organization_id = $1 AND id = ANY($2::uuid[])`, + orgID, ccIDs).Scan(&found); err != nil { + return err + } + if found != len(ccIDs) { + return ErrLeadCCContactNotFound + } + var leadIsCopied, ccHasCopies bool + if err := tx.QueryRow(ctx, ` + SELECT + EXISTS (SELECT 1 FROM campaign_lead_cc WHERE campaign_id = $1 AND cc_contact_id = $2), + EXISTS (SELECT 1 FROM campaign_lead_cc WHERE campaign_id = $1 AND contact_id = ANY($3::uuid[]))`, + campaignID, contactID, ccIDs).Scan(&leadIsCopied, &ccHasCopies); err != nil { + return err + } + if leadIsCopied { + return ErrLeadCCLeadIsCopied + } + if ccHasCopies { + return ErrLeadCCHasCopies + } + } + + if _, err := tx.Exec(ctx, ` + DELETE FROM campaign_lead_cc + WHERE campaign_id = $1 AND contact_id = $2 + AND NOT (cc_contact_id = ANY(COALESCE($3::uuid[], '{}')))`, + campaignID, contactID, ccIDs); err != nil { + return err + } + if len(ccIDs) > 0 { + if _, err := tx.Exec(ctx, ` + INSERT INTO campaign_lead_cc (campaign_id, contact_id, cc_contact_id, position) + SELECT $1, $2, u.id, u.ord - 1 + FROM unnest($3::uuid[]) WITH ORDINALITY AS u(id, ord) + ON CONFLICT (campaign_id, contact_id, cc_contact_id) DO UPDATE SET position = EXCLUDED.position`, + campaignID, contactID, ccIDs); err != nil { + return err + } + } + return tx.Commit(ctx) +} + +func (r *campaignProgressRepository) MarkLeadCCBounced(ctx context.Context, campaignID, contactID uuid.UUID, address string) (*uuid.UUID, error) { + address = strings.TrimSpace(address) + if address == "" { + return nil, nil + } + var id uuid.UUID + err := r.db.QueryRow(ctx, ` + UPDATE campaign_lead_cc x + SET bounced_at = COALESCE(x.bounced_at, NOW()) + FROM contacts c + WHERE c.id = x.cc_contact_id + AND x.campaign_id = $1 AND x.contact_id = $2 + AND lower(c.email) = lower($3) + RETURNING x.cc_contact_id`, campaignID, contactID, address).Scan(&id) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + return &id, nil +} + +func (r *campaignProgressRepository) LeadForCopiedReply(ctx context.Context, ccContactID, emailAccountID uuid.UUID) (*CopiedLeadRef, error) { + var ref CopiedLeadRef + err := r.db.QueryRow(ctx, ` + SELECT p.campaign_id, p.contact_id, p.sequence_id + FROM campaign_lead_cc x + JOIN campaign_leads cl ON cl.campaign_id = x.campaign_id AND cl.contact_id = x.contact_id + JOIN campaign_contact_progress p ON p.campaign_id = x.campaign_id AND p.contact_id = x.contact_id + WHERE x.cc_contact_id = $1 + AND cl.email_account_id = $2 + AND p.sent_at IS NOT NULL + AND `+progressIsEmailStep("p")+` + ORDER BY p.sent_at DESC + LIMIT 1`, ccContactID, emailAccountID).Scan(&ref.CampaignID, &ref.ContactID, &ref.SequenceID) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + return &ref, nil +} + +func (r *campaignProgressRepository) SuggestLeadCC(ctx context.Context, orgID, campaignID, contactID uuid.UUID, limit int) ([]models.CampaignLeadCCSuggestion, error) { + rows, err := r.db.Query(ctx, fmt.Sprintf(` + WITH lead AS ( + SELECT lower(btrim(company)) AS co, lower(split_part(email, '@', 2)) AS dom + FROM contacts WHERE id = $2 AND organization_id = $1 + ) + SELECT c.id, c.email, c.first_name, c.last_name, c.company, + CASE WHEN lead.co <> '' AND lower(btrim(c.company)) = lead.co THEN 'company' ELSE 'domain' END AS reason + FROM contacts c, lead + WHERE c.organization_id = $1 + AND c.id <> $2 + AND c.subscribed IS NOT FALSE + AND ( + (lead.co <> '' AND lower(btrim(c.company)) = lead.co) + OR (lead.dom <> '' AND lead.dom NOT IN (%s) AND lower(split_part(c.email, '@', 2)) = lead.dom) + ) + AND NOT EXISTS ( + SELECT 1 FROM campaign_lead_cc x + WHERE x.campaign_id = $3 AND x.contact_id = $2 AND x.cc_contact_id = c.id + ) + ORDER BY (lead.co <> '' AND lower(btrim(c.company)) = lead.co) DESC, c.first_name, c.last_name, c.email + LIMIT $4`, personalMailDomainsSQL), orgID, contactID, campaignID, limit) + if err != nil { + return nil, err + } + defer rows.Close() + out := []models.CampaignLeadCCSuggestion{} + for rows.Next() { + var s models.CampaignLeadCCSuggestion + if err := rows.Scan(&s.ContactID, &s.Email, &s.FirstName, &s.LastName, &s.Company, &s.Reason); err != nil { + return nil, err + } + out = append(out, s) + } + return out, rows.Err() +} + +func (r *campaignProgressRepository) BouncedCopyAddresses(ctx context.Context, campaignID uuid.UUID, addresses []string) (map[string]bool, error) { + out := map[string]bool{} + if len(addresses) == 0 { + return out, nil + } + rows, err := r.db.Query(ctx, ` + SELECT DISTINCT lower(recipient_email) + FROM deliverability_events + WHERE campaign_id = $1 AND event_type = 'bounce' + AND lower(recipient_email) = ANY($2::text[])`, campaignID, addresses) + if err != nil { + return nil, err + } + defer rows.Close() + for rows.Next() { + var a string + if err := rows.Scan(&a); err != nil { + return nil, err + } + out[a] = true + } + return out, rows.Err() +} + +// leadCCJSONSQL is one lead's copies as a JSON array for the Leads list, with +// the lead row aliased hl and the campaign bound at cp. +func leadCCJSONSQL(cp string) string { + return `( + SELECT COALESCE(json_agg(json_build_object( + 'contact_id', c.id, 'email', c.email, 'first_name', c.first_name, + 'last_name', c.last_name, 'company', c.company, + 'status', ` + leadCCStatusSQL(cp) + `, 'bounced_at', x.bounced_at + ) ORDER BY x.position, x.created_at), '[]'::json) + FROM campaign_lead_cc x + JOIN contacts c ON c.id = x.cc_contact_id + WHERE x.campaign_id = hl.campaign_id AND x.contact_id = hl.contact_id + )` +} diff --git a/internal/repository/pg_campaign_progress.go b/internal/repository/pg_campaign_progress.go index a35d822ab..235abc6ab 100644 --- a/internal/repository/pg_campaign_progress.go +++ b/internal/repository/pg_campaign_progress.go @@ -165,6 +165,12 @@ type CampaignProgressRepository interface { // A lead with nothing else delivered is unbound from the mailbox that // failed, so rotation can offer it a working one. RecordSendFailure(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID, reason string) (attempts int, exhausted bool, rolledBack bool, err error) + // WalkBackSend is RecordSendFailure with the attempt optionally left + // uncounted, for a failure the retry is known not to repeat. + WalkBackSend(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID, reason string, countAttempt bool) (attempts int, exhausted bool, rolledBack bool, err error) + // BouncedCopyAddresses reports which of the campaign's own CC/BCC + // addresses have bounced on a send of this campaign. + BouncedCopyAddresses(ctx context.Context, campaignID uuid.UUID, addresses []string) (map[string]bool, error) // LastSenderForLead is the mailbox a lead was LAST actually sent from, // read from the campaign tasks that dispatched its steps. It answers the // case campaign_leads.email_account_id cannot: a lead removed from the @@ -328,6 +334,24 @@ type CampaignProgressRepository interface { // including a dated hold that has since expired). Returns // ErrLeadNotInCampaign when the contact is not a lead of the campaign. GetLeadHold(ctx context.Context, campaignID, contactID uuid.UUID) (*models.LeadHold, error) + + // ListLeadCC reads the contacts copied on one lead, each with whether the + // next email carries them. + ListLeadCC(ctx context.Context, campaignID, contactID uuid.UUID) ([]models.CampaignLeadCC, error) + // SetLeadCC replaces the contacts copied on one lead. See the ErrLeadCC + // errors for what it refuses. + SetLeadCC(ctx context.Context, orgID, campaignID, contactID uuid.UUID, ccIDs []uuid.UUID) error + // MarkLeadCCBounced records a bounce on the copy of one lead's thread sent + // to address, and returns that copy's contact, or nil when address is not + // one of the lead's copies. + MarkLeadCCBounced(ctx context.Context, campaignID, contactID uuid.UUID, address string) (*uuid.UUID, error) + // LeadForCopiedReply finds the lead whose thread a copied contact is + // answering in: the latest email step sent from emailAccountID to a lead + // that copies them. Nil when there is none. + LeadForCopiedReply(ctx context.Context, ccContactID, emailAccountID uuid.UUID) (*CopiedLeadRef, error) + // SuggestLeadCC offers the lead's likely colleagues: same company name, or + // the same email domain when that domain is not a personal mail service. + SuggestLeadCC(ctx context.Context, orgID, campaignID, contactID uuid.UUID, limit int) ([]models.CampaignLeadCCSuggestion, error) } // ErrLeadNotInCampaign is returned when a hold is asked for on a contact that @@ -564,6 +588,14 @@ func (r *campaignProgressRepository) ListStuckDispatches(ctx context.Context, ol // so a duplicate worker result after the step was already walked back (or // re-sent) is a no-op. func (r *campaignProgressRepository) RecordSendFailure(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID, reason string) (int, bool, bool, error) { + return r.WalkBackSend(ctx, campaignID, contactID, sequenceID, reason, true) +} + +func (r *campaignProgressRepository) WalkBackSend(ctx context.Context, campaignID, contactID, sequenceID uuid.UUID, reason string, countAttempt bool) (int, bool, bool, error) { + inc := 0 + if countAttempt { + inc = 1 + } if len(reason) > 500 { reason = reason[:500] } @@ -578,7 +610,7 @@ func (r *campaignProgressRepository) RecordSendFailure(ctx context.Context, camp SET sent_at = NULL, dispatched_at = NULL, dispatch_task_id = NULL, - send_attempts = send_attempts + 1, + send_attempts = send_attempts + $5, failed_at = NOW(), failure_reason = $4 WHERE campaign_id = $1 AND contact_id = $2 AND sequence_id = $3 @@ -599,7 +631,7 @@ func (r *campaignProgressRepository) RecordSendFailure(ctx context.Context, camp SELECT send_attempts FROM walked ` var attempts int - err := r.db.QueryRow(ctx, query, campaignID, contactID, sequenceID, reason).Scan(&attempts) + err := r.db.QueryRow(ctx, query, campaignID, contactID, sequenceID, reason, inc).Scan(&attempts) if err != nil { if errors.Is(err, pgx.ErrNoRows) { return 0, false, false, nil @@ -2295,7 +2327,9 @@ func (r *campaignProgressRepository) CountHeldLeads(ctx context.Context, campaig var n int err := r.db.QueryRow(ctx, ` SELECT COUNT(*) FROM campaign_leads cl - WHERE cl.campaign_id = $1 AND `+liveHold("cl"), + WHERE cl.campaign_id = $1 AND `+liveHold("cl")+` + -- Reached in another lead's thread: nothing is left to wait for. + AND cl.pause_source IS DISTINCT FROM 'cc'`, campaignID).Scan(&n) return n, err } diff --git a/internal/repository/pg_contact.go b/internal/repository/pg_contact.go index 187289c6b..a18cfa26f 100644 --- a/internal/repository/pg_contact.go +++ b/internal/repository/pg_contact.go @@ -1513,6 +1513,14 @@ func (r *contactRepository) buildContactFilter(ctx context.Context, orgID string }, nil } +// leadRowJSON is the campaign_leads half of a Leads-list row: the fields read +// together because they come from one lead row. +type leadRowJSON struct { + Sender *string `json:"sender"` + CC []models.CampaignLeadCC `json:"cc"` + Hold *models.LeadHold `json:"hold"` +} + func (r *contactRepository) Search( ctx context.Context, orgID string, @@ -1650,6 +1658,8 @@ func (r *contactRepository) Search( 'lead', ( SELECT json_build_object( 'sender', (SELECT ea.email FROM email_accounts ea WHERE ea.id = hl.email_account_id), + -- Contacts copied on every email to this lead. + 'cc', %[5]s, 'hold', CASE WHEN %[4]s THEN json_build_object( 'since', hl.paused_at, 'until', hl.paused_until, 'reason', COALESCE(hl.pause_reason, ''), 'source', COALESCE(hl.pause_source, '') @@ -1689,7 +1699,7 @@ func (r *contactRepository) Search( ) FROM campaign_contact_progress p WHERE p.campaign_id = %[1]s AND p.contact_id = c.id - )`, singleCampaignPlaceholder, config.CampaignSendMaxAttempts, undeliverableClause(singleCampaignPlaceholder), liveHold("hl")) + )`, singleCampaignPlaceholder, config.CampaignSendMaxAttempts, undeliverableClause(singleCampaignPlaceholder), liveHold("hl"), leadCCJSONSQL(singleCampaignPlaceholder)) } // campaign_count is only ever read by the min/max filters and the @@ -1828,10 +1838,7 @@ func (r *contactRepository) Search( Step *string `json:"step"` // The lead row's own fields, read together because they come // from one campaign_leads row. - Lead *struct { - Sender *string `json:"sender"` - Hold *models.LeadHold `json:"hold"` - } `json:"lead"` + Lead *leadRowJSON `json:"lead"` Undeliverable bool `json:"undeliverable"` } @@ -1843,10 +1850,7 @@ func (r *contactRepository) Search( // which the outer query already excludes. lead := lp.Lead if lead == nil { - lead = &struct { - Sender *string `json:"sender"` - Hold *models.LeadHold `json:"hold"` - }{} + lead = &leadRowJSON{} } status := models.LeadStatusPending switch { @@ -1888,6 +1892,7 @@ func (r *contactRepository) Search( c.CampaignLead = &models.ContactCampaignProgress{ Status: status, Hold: lead.Hold, + CC: lead.CC, Sender: sender, Sent: lp.Sent, Opened: lp.Opened, @@ -3174,7 +3179,7 @@ func importCategoryNames(names []string) ([]string, map[string]string, *errx.Err } if len(title) > 50 { return nil, nil, errx.New(errx.BadRequest, - "category name "+strconv.Quote(title)+" is longer than 50 characters") + "label name "+strconv.Quote(title)+" is longer than 50 characters") } lower := strings.ToLower(title) if _, dup := seen[lower]; dup { diff --git a/internal/repository/pg_contact_campaign_state.go b/internal/repository/pg_contact_campaign_state.go index 20a2e5c32..112dbd941 100644 --- a/internal/repository/pg_contact_campaign_state.go +++ b/internal/repository/pg_contact_campaign_state.go @@ -76,6 +76,12 @@ func (r *contactRepository) ListCampaignStates(ctx context.Context, orgID, conta } st.Steps = steps st.TotalSteps = len(steps) + cc, err := listLeadCC(ctx, r.DB.Pool, st.CampaignID, contactID) + if err != nil { + db.CaptureError(err, "", nil, "ListCampaignStates cc") + return nil, errx.InternalError() + } + st.CC = cc var sent, replied, bounced, failed bool var emailSteps, emailSent int @@ -227,9 +233,9 @@ func stepLabel(name, kind string, action []byte, emailOrdinal int) string { } switch cfg.Type { case "add_tag": - return "Add tag" + return "Add label" case "remove_tag": - return "Remove tag" + return "Remove label" case "add_to_segment": return "Add to segment" case "remove_from_segment": diff --git a/internal/repository/pg_email_sync_state.go b/internal/repository/pg_email_sync_state.go index 76e4ce2f4..52fdfc6b1 100644 --- a/internal/repository/pg_email_sync_state.go +++ b/internal/repository/pg_email_sync_state.go @@ -31,6 +31,9 @@ type EmailSyncStateRepository interface { // folder in the current UIDVALIDITY generation, keyed by UID. The IMAP // drafts reconciliation uses it to find the rows the server expunged. ListFolderMessages(ctx context.Context, userID, emailID uuid.UUID, folderPath string, uidValidity uint32) ([]StoredFolderMessage, error) + // ListProviderFolderMessages returns the newest rows the provider last + // placed in one of folders, for providers that key messages by id. + ListProviderFolderMessages(ctx context.Context, userID, emailID uuid.UUID, folders []string, limit int) ([]ProviderFolderMessage, error) } type pgEmailSyncStateRepository struct { @@ -187,3 +190,31 @@ func (r *pgEmailSyncStateRepository) ListFolderMessages(ctx context.Context, use } return out, nil } + +func (r *pgEmailSyncStateRepository) ListProviderFolderMessages(ctx context.Context, userID, emailID uuid.UUID, folders []string, limit int) ([]ProviderFolderMessage, error) { + const q = ` + SELECT id, gmail_id, provider_folder, internal_date + FROM unibox_emails + WHERE user_id = $1 AND email_id = $2 AND provider_folder = ANY($3) AND gmail_id <> '' + ORDER BY internal_date DESC + LIMIT $4 + ` + rows, err := r.db.Query(ctx, q, userID, emailID, folders, limit) + if err != nil { + return nil, fmt.Errorf("email_sync_state: list provider folder messages: %w", err) + } + defer rows.Close() + + out := make([]ProviderFolderMessage, 0) + for rows.Next() { + var m ProviderFolderMessage + if err := rows.Scan(&m.ID, &m.ProviderID, &m.ProviderFolder, &m.InternalDate); err != nil { + return nil, fmt.Errorf("email_sync_state: scan provider folder message: %w", err) + } + out = append(out, m) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("email_sync_state: list provider folder messages: %w", err) + } + return out, nil +} diff --git a/internal/repository/pg_form.go b/internal/repository/pg_form.go index 79a0cacb4..838054dda 100644 --- a/internal/repository/pg_form.go +++ b/internal/repository/pg_form.go @@ -238,7 +238,7 @@ func setFormCategories(ctx context.Context, tx pgx.Tx, orgID, formID uuid.UUID, return nil } if len(categoryIDs) > models.FormMaxCategories { - return errx.New(errx.BadRequest, fmt.Sprintf("at most %d categories per form", models.FormMaxCategories)) + return errx.New(errx.BadRequest, fmt.Sprintf("at most %d labels per form", models.FormMaxCategories)) } _, err := tx.Exec(ctx, ` INSERT INTO form_categories (form_id, category_id) diff --git a/internal/repository/pg_placement.go b/internal/repository/pg_placement.go index 69931d453..cbb5a4f27 100644 --- a/internal/repository/pg_placement.go +++ b/internal/repository/pg_placement.go @@ -69,8 +69,12 @@ type PlacementFinished struct { type PlacementTestFilter struct { OrganizationID *uuid.UUID CampaignID *uuid.UUID - Limit int - Offset int + // BatchID lists one batch's tests. Without it a workspace listing leaves + // batch tests out, so a batch of thousands does not bury the rest. + BatchID *uuid.UUID + IncludeBatch bool + Limit int + Offset int } // PlacementBundle is one test with its probes and their tasks. @@ -217,14 +221,14 @@ func NewPlacementRepository(db *db.DB) PlacementRepository { const placementTestCols = `id, organization_id, sender_account_id, sender_email, created_by, campaign_id, sequence_id, contact_id, monitor_id, subject, body_plain, body_html, open_tracking, link_tracking, compare_group_id, origin, panel, status, error, remote_instance_id, remote_test_id, pace, credits_charged, - credits_refunded, credits_settled_at, created_at, finished_at` + credits_refunded, credits_settled_at, created_at, finished_at, batch_id, batch_sender_id` func scanPlacementTest(row pgx.Row) (*models.PlacementTest, error) { var t models.PlacementTest err := row.Scan(&t.ID, &t.OrganizationID, &t.SenderAccountID, &t.SenderEmail, &t.CreatedBy, &t.CampaignID, &t.SequenceID, &t.ContactID, &t.MonitorID, &t.Subject, &t.BodyPlain, &t.BodyHTML, &t.OpenTracking, &t.LinkTracking, &t.CompareGroupID, &t.Origin, &t.Panel, &t.Status, &t.Error, &t.RemoteInstanceID, &t.RemoteTestID, &t.Pace, &t.CreditsCharged, - &t.CreditsRefunded, &t.CreditsSettledAt, &t.CreatedAt, &t.FinishedAt) + &t.CreditsRefunded, &t.CreditsSettledAt, &t.CreatedAt, &t.FinishedAt, &t.BatchID, &t.BatchSenderID) if errors.Is(err, pgx.ErrNoRows) { return nil, nil } @@ -264,12 +268,14 @@ func (r *placementRepository) CreateTests(ctx context.Context, bundles []Placeme err = tx.QueryRow(ctx, ` INSERT INTO placement_tests (id, organization_id, sender_account_id, sender_email, created_by, campaign_id, sequence_id, contact_id, monitor_id, subject, body_plain, body_html, open_tracking, link_tracking, - compare_group_id, origin, panel, status, error, remote_instance_id, remote_test_id, pace, credits_charged, created_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, NOW()) + compare_group_id, origin, panel, status, error, remote_instance_id, remote_test_id, pace, credits_charged, + batch_id, batch_sender_id, created_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, NOW()) RETURNING created_at `, t.ID, t.OrganizationID, t.SenderAccountID, t.SenderEmail, t.CreatedBy, t.CampaignID, t.SequenceID, t.ContactID, t.MonitorID, t.Subject, t.BodyPlain, t.BodyHTML, t.OpenTracking, t.LinkTracking, - t.CompareGroupID, t.Origin, t.Panel, t.Status, t.Error, t.RemoteInstanceID, t.RemoteTestID, pace, t.CreditsCharged).Scan(&t.CreatedAt) + t.CompareGroupID, t.Origin, t.Panel, t.Status, t.Error, t.RemoteInstanceID, t.RemoteTestID, pace, t.CreditsCharged, + t.BatchID, t.BatchSenderID).Scan(&t.CreatedAt) if err != nil { return err } @@ -326,10 +332,12 @@ func (r *placementRepository) ListTests(ctx context.Context, f PlacementTestFilt if f.Limit <= 0 { f.Limit = 25 } - where := `($1::uuid IS NULL OR organization_id = $1) AND ($2::uuid IS NULL OR campaign_id = $2)` + where := `($1::uuid IS NULL OR organization_id = $1) AND ($2::uuid IS NULL OR campaign_id = $2) + AND (CASE WHEN $3::uuid IS NOT NULL THEN batch_id = $3 ELSE $4 OR batch_id IS NULL END)` var total int - if err := r.db.QueryRow(ctx, `SELECT COUNT(*) FROM placement_tests WHERE `+where, f.OrganizationID, f.CampaignID).Scan(&total); err != nil { + if err := r.db.QueryRow(ctx, `SELECT COUNT(*) FROM placement_tests WHERE `+where, + f.OrganizationID, f.CampaignID, f.BatchID, f.IncludeBatch).Scan(&total); err != nil { return nil, 0, err } rows, err := r.db.Query(ctx, ` @@ -337,8 +345,8 @@ func (r *placementRepository) ListTests(ctx context.Context, f PlacementTestFilt FROM placement_tests WHERE `+where+` ORDER BY created_at DESC, id DESC - LIMIT $3 OFFSET $4 - `, f.OrganizationID, f.CampaignID, f.Limit, f.Offset) + LIMIT $5 OFFSET $6 + `, f.OrganizationID, f.CampaignID, f.BatchID, f.IncludeBatch, f.Limit, f.Offset) if err != nil { return nil, 0, err } @@ -749,7 +757,7 @@ func (r *placementRepository) CountMeteredTests(ctx context.Context, orgID uuid. SELECT COUNT(*) FROM placement_tests pt WHERE pt.organization_id = $1 AND pt.panel IN ('instance', 'cloud') - AND pt.origin IN ('manual', 'monitor', 'remote') + AND pt.origin IN ('manual', 'monitor', 'remote', 'batch') AND pt.credits_charged = 0 AND pt.created_at >= $2 AND (pt.finished_at IS NULL OR EXISTS ( diff --git a/internal/repository/pg_placement_batch.go b/internal/repository/pg_placement_batch.go new file mode 100644 index 000000000..0d2ec4322 --- /dev/null +++ b/internal/repository/pg_placement_batch.go @@ -0,0 +1,853 @@ +package repository + +import ( + "context" + "encoding/json" + "errors" + "strings" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "github.com/warmbly/warmbly/internal/infrastructure/db" + "github.com/warmbly/warmbly/internal/models" +) + +// PlacementBatchCandidate is a mailbox a batch could test from. +type PlacementBatchCandidate struct { + ID uuid.UUID + Email string + SendAsEmail string + Provider string + MailHost string + Status string + WorkerID *uuid.UUID +} + +// SendFrom is the address the mailbox sends as. +func (c PlacementBatchCandidate) SendFrom() string { + if c.SendAsEmail != "" { + return c.SendAsEmail + } + return c.Email +} + +// PlacementCandidateFilter narrows a workspace's mailboxes in SQL. Provider and +// domain filters are applied by the caller, which resolves the host family. +type PlacementCandidateFilter struct { + // IDs keeps only these mailboxes; nil keeps every one. + IDs []uuid.UUID + TagIDs []uuid.UUID + IncludeInactive bool + // UntestedSince keeps mailboxes with no delivered test since then. + UntestedSince *time.Time +} + +// PlacementBatchSenderRow is a batch sender with where its copies landed. +type PlacementBatchSenderRow struct { + models.PlacementBatchSender + Counts models.PlacementCounts + TestIDs []uuid.UUID +} + +// PlacementBatchSenderFilter narrows and orders a batch's sender listing. +type PlacementBatchSenderFilter struct { + Status string + Search string + // Sort is worst (lowest inbox rate first), best, email or status. + Sort string + Limit int + Offset int +} + +// PlacementBreakdownRow is one cell of a batch's placement grouped by a +// dimension. Empty dimension fields are the ones the row is not grouped by. +type PlacementBreakdownRow struct { + Set string + Tracked bool + SenderDomain string + SenderFamily string + RecipientFamily string + Folder string + Count int +} + +// PlacementBatchGroupSize is how many senders a domain or provider has in a +// batch, and how many of them finished a test. +type PlacementBatchGroupSize struct { + // ByDomain is a domain's row; otherwise it is a provider's. + ByDomain bool + Domain string + Family string + Senders int + Completed int +} + +// PlacementOrgTest names a test with its workspace. +type PlacementOrgTest struct { + OrganizationID uuid.UUID + TestID uuid.UUID +} + +// PlacementCoverage is how much of a workspace's fleet was tested recently. +type PlacementCoverage struct { + Mailboxes int `json:"mailboxes"` + Tested7d int `json:"tested_7d"` + Tested30d int `json:"tested_30d"` + NeverTested int `json:"never_tested"` +} + +// PlacementBatchRepository stores placement batches and their senders. +type PlacementBatchRepository interface { + // ListBatchCandidates lists a workspace's mailboxes that can send a test: + // never a seed. + ListBatchCandidates(ctx context.Context, orgID uuid.UUID, f PlacementCandidateFilter) ([]PlacementBatchCandidate, error) + // CreateBatch writes a batch and its sender snapshot in one transaction. + CreateBatch(ctx context.Context, b *models.PlacementBatch, senders []models.PlacementBatchSender) error + GetBatch(ctx context.Context, orgID, id uuid.UUID) (*models.PlacementBatch, error) + // CountOpenBatches counts a workspace's batches still running. + CountOpenBatches(ctx context.Context, orgID uuid.UUID) (int, error) + ListBatches(ctx context.Context, orgID uuid.UUID, limit, offset int) ([]models.PlacementBatch, int, error) + BatchProgress(ctx context.Context, ids []uuid.UUID) (map[uuid.UUID]models.PlacementBatchProgress, error) + // BatchSummaries counts where every batch's copies landed, headline copy + // only (the tracked half of a tracking comparison). + BatchSummaries(ctx context.Context, ids []uuid.UUID) (map[uuid.UUID]models.PlacementCounts, error) + + // ClaimBatches leases the active batches no other replica holds and + // returns each with the moment it was last advanced. + ClaimBatches(ctx context.Context, now time.Time, lease time.Duration, limit int) ([]models.PlacementBatch, error) + ReleaseBatch(ctx context.Context, id uuid.UUID) error + // CloseInactiveBatches ends a batch that is not active yet still open: + // one imported from another instance. + CloseInactiveBatches(ctx context.Context) error + // SyncBatchSenders resolves running senders whose tests all finished, and + // returns a sender claimed but never started (a crash in between) to the + // queue after staleAfter. + SyncBatchSenders(ctx context.Context, batchID uuid.UUID, staleBefore time.Time) error + // SyncClosedBatchSenders resolves the running senders of batches that are + // no longer active (cancelled), once their tests finish. + SyncClosedBatchSenders(ctx context.Context) error + // RunningTestsOfCancelledBatches lists tests still running in a cancelled + // batch: one a sender started while the batch was being cancelled. + RunningTestsOfCancelledBatches(ctx context.Context, limit int) ([]PlacementOrgTest, error) + // CountSendingBatchSenders counts batch senders still sending probes, + // across one workspace's batches, or every workspace's when orgID is nil. + CountSendingBatchSenders(ctx context.Context, orgID *uuid.UUID) (int, error) + DueBatchSenders(ctx context.Context, batchID uuid.UUID, now time.Time, limit int) ([]models.PlacementBatchSender, error) + // ClaimBatchSender marks a due sender running before its test is created, + // so a restart can never start it twice. False when it was not due. + ClaimBatchSender(ctx context.Context, id uuid.UUID) (bool, error) + // SetBatchSenderOutcome records a sender that did not start. + SetBatchSenderOutcome(ctx context.Context, id uuid.UUID, status, reason, detail string, next time.Time) error + // CloseOpenBatchSenders gives every sender of a batch in one of the from + // statuses (queued, deferred) one final status. + CloseOpenBatchSenders(ctx context.Context, batchID uuid.UUID, from []string, status, reason, detail string) error + AddBatchCredits(ctx context.Context, id uuid.UUID, credits int) error + MarkBatchStarted(ctx context.Context, id uuid.UUID) error + // FinishBatch closes an active batch; false when it was already closed. + FinishBatch(ctx context.Context, id uuid.UUID, status, errMsg string) (bool, error) + // CancelBatch closes a batch and its unstarted senders, returning the + // running tests to cancel. False when it was not open. + CancelBatch(ctx context.Context, orgID, id uuid.UUID) (bool, []uuid.UUID, error) + + ListBatchSenders(ctx context.Context, orgID, batchID uuid.UUID, f PlacementBatchSenderFilter) ([]PlacementBatchSenderRow, int, error) + BatchBreakdown(ctx context.Context, batchID uuid.UUID) ([]PlacementBreakdownRow, error) + BatchGroupSizes(ctx context.Context, batchID uuid.UUID) ([]PlacementBatchGroupSize, error) + Coverage(ctx context.Context, orgID uuid.UUID, now time.Time) (PlacementCoverage, error) +} + +type placementBatchRepository struct { + db *db.DB +} + +// NewPlacementBatchRepository wires the batch store. +func NewPlacementBatchRepository(db *db.DB) PlacementBatchRepository { + return &placementBatchRepository{db: db} +} + +// deliveredFolders are the folders a copy that left can land in. +const deliveredFolders = `('inbox', 'promotions', 'other', 'spam', 'missing')` + +// headlineTest keeps the copy a campaign really sends: the tracked half of a +// tracking comparison, every test otherwise. b and t are the batch and test. +const headlineTest = `(b.tracking <> 'compare' OR t.open_tracking OR t.link_tracking)` + +func (r *placementBatchRepository) ListBatchCandidates(ctx context.Context, orgID uuid.UUID, f PlacementCandidateFilter) ([]PlacementBatchCandidate, error) { + rows, err := r.db.Query(ctx, ` + SELECT ea.id, ea.email, ea.send_as_email, ea.provider::text, ea.mail_host, ea.status::text, ea.worker_id + FROM email_accounts ea + WHERE ea.organization_id = $1 + AND ea.seed_scope IS NULL + AND ($2::uuid[] IS NULL OR ea.id = ANY($2)) + AND (cardinality($3::uuid[]) = 0 OR EXISTS ( + SELECT 1 FROM email_tags et WHERE et.email_id = ea.id AND et.tag_id = ANY($3) + )) + AND ($4 OR ea.status = 'active') + AND ($5::timestamptz IS NULL OR NOT EXISTS ( + SELECT 1 FROM placement_tests pt + WHERE pt.sender_account_id = ea.id AND pt.organization_id = $1 + AND pt.status = 'completed' AND pt.created_at >= $5 + )) + ORDER BY ea.email, ea.id + `, orgID, f.IDs, nonNilUUIDs(f.TagIDs), f.IncludeInactive, f.UntestedSince) + if err != nil { + return nil, err + } + defer rows.Close() + var out []PlacementBatchCandidate + for rows.Next() { + var c PlacementBatchCandidate + if err := rows.Scan(&c.ID, &c.Email, &c.SendAsEmail, &c.Provider, &c.MailHost, &c.Status, &c.WorkerID); err != nil { + return nil, err + } + out = append(out, c) + } + return out, rows.Err() +} + +func nonNilUUIDs(ids []uuid.UUID) []uuid.UUID { + if ids == nil { + return []uuid.UUID{} + } + return ids +} + +const placementBatchCols = `id, organization_id, created_by, campaign_id, sequence_id, contact_id, subject, + body_plain, body_html, tracking, panel, pace, families, seed_ids, on_unavailable, selection, sender_count, + max_credits, credits_spent, status, error, active, last_tick_at, retry_until, created_at, started_at, finished_at` + +func scanPlacementBatch(row pgx.Row) (*models.PlacementBatch, error) { + var b models.PlacementBatch + var selection []byte + err := row.Scan(&b.ID, &b.OrganizationID, &b.CreatedBy, &b.CampaignID, &b.SequenceID, &b.ContactID, &b.Subject, + &b.BodyPlain, &b.BodyHTML, &b.Tracking, &b.Panel, &b.Pace, &b.Families, &b.SeedIDs, &b.OnUnavailable, &selection, &b.SenderCount, + &b.MaxCredits, &b.CreditsSpent, &b.Status, &b.Error, &b.Active, &b.LastTickAt, &b.RetryUntil, &b.CreatedAt, &b.StartedAt, &b.FinishedAt) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + if len(selection) > 0 { + // A selection that no longer parses only costs the display. + _ = json.Unmarshal(selection, &b.Selection) + } + if b.Families == nil { + b.Families = []string{} + } + if b.SeedIDs == nil { + b.SeedIDs = []uuid.UUID{} + } + return &b, nil +} + +func (r *placementBatchRepository) CreateBatch(ctx context.Context, b *models.PlacementBatch, senders []models.PlacementBatchSender) error { + selection, err := json.Marshal(b.Selection) + if err != nil { + return err + } + tx, err := r.db.Begin(ctx) + if err != nil { + return err + } + defer func() { _ = tx.Rollback(ctx) }() + + families := b.Families + if families == nil { + families = []string{} + } + err = tx.QueryRow(ctx, ` + INSERT INTO placement_batches (id, organization_id, created_by, campaign_id, sequence_id, contact_id, subject, + body_plain, body_html, tracking, panel, pace, families, seed_ids, on_unavailable, selection, sender_count, + max_credits, status, active, retry_until) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, true, $20) + RETURNING created_at + `, b.ID, b.OrganizationID, b.CreatedBy, b.CampaignID, b.SequenceID, b.ContactID, b.Subject, + b.BodyPlain, b.BodyHTML, b.Tracking, b.Panel, b.Pace, families, nonNilUUIDs(b.SeedIDs), b.OnUnavailable, selection, b.SenderCount, + b.MaxCredits, b.Status, b.RetryUntil).Scan(&b.CreatedAt) + if err != nil { + return err + } + b.Active = true + + rows := make([][]any, len(senders)) + for i, s := range senders { + rows[i] = []any{s.ID, b.ID, s.EmailAccountID, s.SenderEmail, s.SenderDomain, s.SenderFamily, s.Position, models.PlacementSenderQueued} + } + if _, err := tx.CopyFrom(ctx, pgx.Identifier{"placement_batch_senders"}, + []string{"id", "batch_id", "email_account_id", "sender_email", "sender_domain", "sender_family", "position", "status"}, + pgx.CopyFromRows(rows)); err != nil { + return err + } + return tx.Commit(ctx) +} + +func (r *placementBatchRepository) GetBatch(ctx context.Context, orgID, id uuid.UUID) (*models.PlacementBatch, error) { + return scanPlacementBatch(r.db.QueryRow(ctx, + `SELECT `+placementBatchCols+` FROM placement_batches WHERE id = $2 AND organization_id = $1`, orgID, id)) +} + +func (r *placementBatchRepository) CountOpenBatches(ctx context.Context, orgID uuid.UUID) (int, error) { + var n int + err := r.db.QueryRow(ctx, `SELECT COUNT(*) FROM placement_batches WHERE organization_id = $1 AND active`, orgID).Scan(&n) + return n, err +} + +func (r *placementBatchRepository) ListBatches(ctx context.Context, orgID uuid.UUID, limit, offset int) ([]models.PlacementBatch, int, error) { + if limit <= 0 { + limit = 25 + } + var total int + if err := r.db.QueryRow(ctx, `SELECT COUNT(*) FROM placement_batches WHERE organization_id = $1`, orgID).Scan(&total); err != nil { + return nil, 0, err + } + rows, err := r.db.Query(ctx, ` + SELECT `+placementBatchCols+` FROM placement_batches + WHERE organization_id = $1 + ORDER BY created_at DESC, id DESC + LIMIT $2 OFFSET $3 + `, orgID, limit, offset) + if err != nil { + return nil, 0, err + } + defer rows.Close() + out := make([]models.PlacementBatch, 0, limit) + for rows.Next() { + b, err := scanPlacementBatch(rows) + if err != nil { + return nil, 0, err + } + out = append(out, *b) + } + return out, total, rows.Err() +} + +func (r *placementBatchRepository) BatchProgress(ctx context.Context, ids []uuid.UUID) (map[uuid.UUID]models.PlacementBatchProgress, error) { + out := make(map[uuid.UUID]models.PlacementBatchProgress, len(ids)) + if len(ids) == 0 { + return out, nil + } + rows, err := r.db.Query(ctx, ` + SELECT batch_id, status, COUNT(*) FROM placement_batch_senders + WHERE batch_id = ANY($1) + GROUP BY batch_id, status + `, ids) + if err != nil { + return nil, err + } + defer rows.Close() + for rows.Next() { + var id uuid.UUID + var status string + var n int + if err := rows.Scan(&id, &status, &n); err != nil { + return nil, err + } + p := out[id] + p.Add(status, n) + out[id] = p + } + return out, rows.Err() +} + +func (r *placementBatchRepository) BatchSummaries(ctx context.Context, ids []uuid.UUID) (map[uuid.UUID]models.PlacementCounts, error) { + out := make(map[uuid.UUID]models.PlacementCounts, len(ids)) + if len(ids) == 0 { + return out, nil + } + rows, err := r.db.Query(ctx, ` + SELECT b.id, r.folder, COUNT(*) + FROM placement_batches b + JOIN placement_tests t ON t.batch_id = b.id + JOIN placement_results r ON r.test_id = t.id + WHERE b.id = ANY($1) AND `+headlineTest+` + GROUP BY b.id, r.folder + `, ids) + if err != nil { + return nil, err + } + defer rows.Close() + for rows.Next() { + var id uuid.UUID + var folder string + var n int + if err := rows.Scan(&id, &folder, &n); err != nil { + return nil, err + } + c := out[id] + c.AddN(folder, n) + out[id] = c + } + for id, c := range out { + c.Finish() + out[id] = c + } + return out, rows.Err() +} + +func (r *placementBatchRepository) ClaimBatches(ctx context.Context, now time.Time, lease time.Duration, limit int) ([]models.PlacementBatch, error) { + if limit <= 0 { + limit = 20 + } + rows, err := r.db.Query(ctx, ` + WITH due AS ( + SELECT id, last_tick_at AS prev FROM placement_batches + WHERE active AND (lease_until IS NULL OR lease_until < $1) + ORDER BY created_at + LIMIT $3 + FOR UPDATE SKIP LOCKED + ) + UPDATE placement_batches b + SET lease_until = $1 + make_interval(secs => $2), last_tick_at = $1 + FROM due + WHERE b.id = due.id + RETURNING `+strings.Replace(prefixCols("b.", placementBatchCols), "b.last_tick_at", "due.prev", 1), + now, lease.Seconds(), limit) + if err != nil { + return nil, err + } + defer rows.Close() + var out []models.PlacementBatch + for rows.Next() { + b, err := scanPlacementBatch(rows) + if err != nil { + return nil, err + } + out = append(out, *b) + } + return out, rows.Err() +} + +func (r *placementBatchRepository) ReleaseBatch(ctx context.Context, id uuid.UUID) error { + _, err := r.db.Exec(ctx, `UPDATE placement_batches SET lease_until = NULL WHERE id = $1`, id) + return err +} + +func (r *placementBatchRepository) CloseInactiveBatches(ctx context.Context) error { + _, err := r.db.Exec(ctx, ` + WITH closed AS ( + UPDATE placement_batches + SET status = 'cancelled', finished_at = COALESCE(finished_at, NOW()), updated_at = NOW(), + error = 'The batch was moved from another instance before it finished.' + WHERE NOT active AND status IN ('queued', 'running') + RETURNING id + ) + UPDATE placement_batch_senders SET status = 'cancelled', finished_at = NOW() + WHERE batch_id IN (SELECT id FROM closed) AND status IN ('queued', 'deferred') + `) + return err +} + +// resolveFinishedSenders sets a running sender whose tests have all finished +// to what they came to. The caller adds the WHERE condition choosing senders. +const resolveFinishedSenders = ` + UPDATE placement_batch_senders s + SET status = CASE + WHEN EXISTS (SELECT 1 FROM placement_tests t WHERE t.batch_sender_id = s.id AND t.status = 'completed') THEN 'completed' + WHEN EXISTS (SELECT 1 FROM placement_tests t WHERE t.batch_sender_id = s.id AND t.status = 'cancelled') THEN 'cancelled' + ELSE 'failed' + END, + detail = COALESCE(( + SELECT t.error FROM placement_tests t + WHERE t.batch_sender_id = s.id AND t.status = 'failed' AND t.error <> '' + LIMIT 1 + ), ''), + finished_at = NOW() + WHERE s.status = 'running' + AND EXISTS (SELECT 1 FROM placement_tests t WHERE t.batch_sender_id = s.id) + AND NOT EXISTS (SELECT 1 FROM placement_tests t WHERE t.batch_sender_id = s.id AND t.finished_at IS NULL) + AND ` + +func (r *placementBatchRepository) SyncBatchSenders(ctx context.Context, batchID uuid.UUID, staleBefore time.Time) error { + if _, err := r.db.Exec(ctx, resolveFinishedSenders+`s.batch_id = $1`, batchID); err != nil { + return err + } + _, err := r.db.Exec(ctx, ` + UPDATE placement_batch_senders s + SET status = 'deferred', next_attempt_at = NOW() + WHERE s.batch_id = $1 AND s.status = 'running' AND s.started_at < $2 + AND NOT EXISTS (SELECT 1 FROM placement_tests t WHERE t.batch_sender_id = s.id) + `, batchID, staleBefore) + return err +} + +func (r *placementBatchRepository) SyncClosedBatchSenders(ctx context.Context) error { + if _, err := r.db.Exec(ctx, resolveFinishedSenders+`s.batch_id IN ( + SELECT id FROM placement_batches WHERE NOT active AND status = 'cancelled' + )`); err != nil { + return err + } + // Claimed but never started before the cancel: nothing will start it now. + _, err := r.db.Exec(ctx, ` + UPDATE placement_batch_senders s + SET status = 'cancelled', detail = 'The batch was cancelled before this mailbox started.', finished_at = NOW() + WHERE s.status = 'running' + AND s.batch_id IN (SELECT id FROM placement_batches WHERE NOT active AND status = 'cancelled') + AND NOT EXISTS (SELECT 1 FROM placement_tests t WHERE t.batch_sender_id = s.id) + AND s.started_at < NOW() - interval '10 minutes' + `) + return err +} + +func (r *placementBatchRepository) RunningTestsOfCancelledBatches(ctx context.Context, limit int) ([]PlacementOrgTest, error) { + rows, err := r.db.Query(ctx, ` + SELECT t.organization_id, t.id FROM placement_tests t + JOIN placement_batches b ON b.id = t.batch_id + WHERE b.status = 'cancelled' AND NOT b.active AND t.status = 'running' AND t.organization_id = b.organization_id + ORDER BY t.created_at + LIMIT $1 + `, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var out []PlacementOrgTest + for rows.Next() { + var o PlacementOrgTest + if err := rows.Scan(&o.OrganizationID, &o.TestID); err != nil { + return nil, err + } + out = append(out, o) + } + return out, rows.Err() +} + +func (r *placementBatchRepository) CountSendingBatchSenders(ctx context.Context, orgID *uuid.UUID) (int, error) { + var n int + err := r.db.QueryRow(ctx, ` + SELECT COUNT(*) FROM placement_batch_senders s + JOIN placement_batches b ON b.id = s.batch_id + WHERE ($1::uuid IS NULL OR b.organization_id = $1) AND b.active AND s.status = 'running' + AND ( + NOT EXISTS (SELECT 1 FROM placement_tests t WHERE t.batch_sender_id = s.id) + OR EXISTS ( + SELECT 1 FROM placement_tests t + JOIN placement_results pr ON pr.test_id = t.id + WHERE t.batch_sender_id = s.id AND t.status = 'running' + AND pr.folder = 'pending' AND pr.sent_at IS NULL + ) + ) + `, orgID).Scan(&n) + return n, err +} + +const placementBatchSenderCols = `id, batch_id, email_account_id, sender_email, sender_domain, sender_family, position, + status, reason, detail, attempts, next_attempt_at, started_at, finished_at` + +func scanBatchSender(row pgx.Row, extra ...any) (models.PlacementBatchSender, error) { + var s models.PlacementBatchSender + dest := append([]any{&s.ID, &s.BatchID, &s.EmailAccountID, &s.SenderEmail, &s.SenderDomain, &s.SenderFamily, &s.Position, + &s.Status, &s.Reason, &s.Detail, &s.Attempts, &s.NextAttemptAt, &s.StartedAt, &s.FinishedAt}, extra...) + err := row.Scan(dest...) + return s, err +} + +func (r *placementBatchRepository) DueBatchSenders(ctx context.Context, batchID uuid.UUID, now time.Time, limit int) ([]models.PlacementBatchSender, error) { + rows, err := r.db.Query(ctx, ` + SELECT `+placementBatchSenderCols+` FROM placement_batch_senders + WHERE batch_id = $1 AND status IN ('queued', 'deferred') AND next_attempt_at <= $2 + ORDER BY position + LIMIT $3 + `, batchID, now, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var out []models.PlacementBatchSender + for rows.Next() { + s, err := scanBatchSender(rows) + if err != nil { + return nil, err + } + out = append(out, s) + } + return out, rows.Err() +} + +func (r *placementBatchRepository) ClaimBatchSender(ctx context.Context, id uuid.UUID) (bool, error) { + tag, err := r.db.Exec(ctx, ` + UPDATE placement_batch_senders + SET status = 'running', attempts = attempts + 1, started_at = NOW(), reason = '', detail = '' + WHERE id = $1 AND status IN ('queued', 'deferred') + `, id) + if err != nil { + return false, err + } + return tag.RowsAffected() == 1, nil +} + +func (r *placementBatchRepository) SetBatchSenderOutcome(ctx context.Context, id uuid.UUID, status, reason, detail string, next time.Time) error { + _, err := r.db.Exec(ctx, ` + UPDATE placement_batch_senders + SET status = $2, reason = $3, detail = $4, next_attempt_at = $5, + finished_at = CASE WHEN $2 IN ('skipped', 'failed', 'cancelled') THEN NOW() ELSE NULL END + WHERE id = $1 + `, id, status, reason, truncateRunes(detail, 500), next) + return err +} + +func (r *placementBatchRepository) CloseOpenBatchSenders(ctx context.Context, batchID uuid.UUID, from []string, status, reason, detail string) error { + _, err := r.db.Exec(ctx, ` + UPDATE placement_batch_senders + SET status = $3, reason = $4, detail = $5, finished_at = NOW() + WHERE batch_id = $1 AND status = ANY($2) AND status IN ('queued', 'deferred') + `, batchID, from, status, reason, truncateRunes(detail, 500)) + return err +} + +func (r *placementBatchRepository) AddBatchCredits(ctx context.Context, id uuid.UUID, credits int) error { + _, err := r.db.Exec(ctx, ` + UPDATE placement_batches SET credits_spent = credits_spent + $2, updated_at = NOW() WHERE id = $1 + `, id, credits) + return err +} + +func (r *placementBatchRepository) MarkBatchStarted(ctx context.Context, id uuid.UUID) error { + _, err := r.db.Exec(ctx, ` + UPDATE placement_batches SET status = 'running', started_at = COALESCE(started_at, NOW()), updated_at = NOW() + WHERE id = $1 AND status = 'queued' + `, id) + return err +} + +func (r *placementBatchRepository) FinishBatch(ctx context.Context, id uuid.UUID, status, errMsg string) (bool, error) { + tag, err := r.db.Exec(ctx, ` + UPDATE placement_batches + SET status = $2, error = $3, active = false, lease_until = NULL, finished_at = NOW(), updated_at = NOW() + WHERE id = $1 AND active + `, id, status, truncateRunes(errMsg, 500)) + if err != nil { + return false, err + } + return tag.RowsAffected() == 1, nil +} + +func (r *placementBatchRepository) CancelBatch(ctx context.Context, orgID, id uuid.UUID) (bool, []uuid.UUID, error) { + tx, err := r.db.Begin(ctx) + if err != nil { + return false, nil, err + } + defer func() { _ = tx.Rollback(ctx) }() + tag, err := tx.Exec(ctx, ` + UPDATE placement_batches + SET status = 'cancelled', active = false, lease_until = NULL, finished_at = NOW(), updated_at = NOW() + WHERE id = $2 AND organization_id = $1 AND active + `, orgID, id) + if err != nil { + return false, nil, err + } + if tag.RowsAffected() == 0 { + return false, nil, nil + } + if _, err := tx.Exec(ctx, ` + UPDATE placement_batch_senders + SET status = 'cancelled', reason = '', detail = 'The batch was cancelled before this mailbox started.', finished_at = NOW() + WHERE batch_id = $1 AND status IN ('queued', 'deferred') + `, id); err != nil { + return false, nil, err + } + rows, err := tx.Query(ctx, ` + SELECT id FROM placement_tests WHERE batch_id = $1 AND organization_id = $2 AND status = 'running' + `, id, orgID) + if err != nil { + return false, nil, err + } + var running []uuid.UUID + for rows.Next() { + var tid uuid.UUID + if err := rows.Scan(&tid); err != nil { + rows.Close() + return false, nil, err + } + running = append(running, tid) + } + rows.Close() + if err := rows.Err(); err != nil { + return false, nil, err + } + return true, running, tx.Commit(ctx) +} + +func (r *placementBatchRepository) ListBatchSenders(ctx context.Context, orgID, batchID uuid.UUID, f PlacementBatchSenderFilter) ([]PlacementBatchSenderRow, int, error) { + if f.Limit <= 0 { + f.Limit = 50 + } + order := `(counts.inbox::float / NULLIF(counts.delivered, 0)) ASC NULLS LAST, counts.delivered DESC, s.sender_email` + switch f.Sort { + case "best": + order = `(counts.inbox::float / NULLIF(counts.delivered, 0)) DESC NULLS LAST, counts.delivered DESC, s.sender_email` + case "email": + order = `s.sender_email, s.id` + case "status": + order = `s.status, s.sender_email` + } + where := `s.batch_id = $1 AND EXISTS (SELECT 1 FROM placement_batches ob WHERE ob.id = s.batch_id AND ob.organization_id = $2) + AND ($3 = '' OR s.status = $3) + AND ($4 = '' OR s.sender_email ILIKE '%' || $4 || '%')` + search := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(strings.TrimSpace(f.Search)) + + var total int + if err := r.db.QueryRow(ctx, `SELECT COUNT(*) FROM placement_batch_senders s WHERE `+where, + batchID, orgID, f.Status, search).Scan(&total); err != nil { + return nil, 0, err + } + rows, err := r.db.Query(ctx, ` + SELECT `+prefixCols("s.", placementBatchSenderCols)+`, + counts.inbox, counts.promotions, counts.other, counts.spam, counts.missing, counts.pending, counts.failed, counts.cancelled, + ARRAY(SELECT t.id FROM placement_tests t WHERE t.batch_sender_id = s.id ORDER BY t.open_tracking OR t.link_tracking, t.created_at) + FROM placement_batch_senders s + JOIN placement_batches b ON b.id = s.batch_id + CROSS JOIN LATERAL ( + SELECT + COUNT(*) FILTER (WHERE r.folder = 'inbox') AS inbox, + COUNT(*) FILTER (WHERE r.folder = 'promotions') AS promotions, + COUNT(*) FILTER (WHERE r.folder = 'other') AS other, + COUNT(*) FILTER (WHERE r.folder = 'spam') AS spam, + COUNT(*) FILTER (WHERE r.folder = 'missing') AS missing, + COUNT(*) FILTER (WHERE r.folder = 'pending') AS pending, + COUNT(*) FILTER (WHERE r.folder = 'failed') AS failed, + COUNT(*) FILTER (WHERE r.folder = 'cancelled') AS cancelled, + COUNT(*) FILTER (WHERE r.folder IN `+deliveredFolders+`) AS delivered + FROM placement_tests t + JOIN placement_results r ON r.test_id = t.id + WHERE t.batch_sender_id = s.id AND `+headlineTest+` + ) counts + WHERE `+where+` + ORDER BY `+order+` + LIMIT $5 OFFSET $6 + `, batchID, orgID, f.Status, search, f.Limit, f.Offset) + if err != nil { + return nil, 0, err + } + defer rows.Close() + out := make([]PlacementBatchSenderRow, 0, f.Limit) + for rows.Next() { + var row PlacementBatchSenderRow + var inbox, promotions, other, spam, missing, pending, failed, cancelled int + s, err := scanBatchSender(rows, &inbox, &promotions, &other, &spam, &missing, &pending, &failed, &cancelled, &row.TestIDs) + if err != nil { + return nil, 0, err + } + row.PlacementBatchSender = s + for folder, n := range map[string]int{ + models.PlacementFolderInbox: inbox, models.PlacementFolderPromotions: promotions, models.PlacementFolderOther: other, + models.PlacementFolderSpam: spam, models.PlacementFolderMissing: missing, models.PlacementFolderPending: pending, + models.PlacementFolderFailed: failed, models.PlacementFolderCancelled: cancelled, + } { + row.Counts.AddN(folder, n) + } + row.Counts.Finish() + if row.TestIDs == nil { + row.TestIDs = []uuid.UUID{} + } + out = append(out, row) + } + return out, total, rows.Err() +} + +func (r *placementBatchRepository) BatchBreakdown(ctx context.Context, batchID uuid.UUID) ([]PlacementBreakdownRow, error) { + var out []PlacementBreakdownRow + // Overall, per tracking variant, so a comparison reads both halves. + rows, err := r.db.Query(ctx, ` + SELECT t.open_tracking OR t.link_tracking, r.folder, COUNT(*) + FROM placement_tests t + JOIN placement_results r ON r.test_id = t.id + WHERE t.batch_id = $1 + GROUP BY 1, 2 + `, batchID) + if err != nil { + return nil, err + } + for rows.Next() { + b := PlacementBreakdownRow{Set: "overall"} + if err := rows.Scan(&b.Tracked, &b.Folder, &b.Count); err != nil { + rows.Close() + return nil, err + } + out = append(out, b) + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, err + } + + // The headline copy by sending domain, sending provider, recipient + // provider, and sending domain by recipient provider, in one pass. + rows, err = r.db.Query(ctx, ` + SELECT + CASE GROUPING(s.sender_domain, s.sender_family, r.provider) + WHEN 3 THEN 'domain' + WHEN 5 THEN 'provider' + WHEN 6 THEN 'recipient' + ELSE 'matrix' + END, + COALESCE(s.sender_domain, ''), COALESCE(s.sender_family, ''), COALESCE(r.provider, ''), + r.folder, COUNT(*) + FROM placement_tests t + JOIN placement_batches b ON b.id = t.batch_id + JOIN placement_batch_senders s ON s.id = t.batch_sender_id + JOIN placement_results r ON r.test_id = t.id + WHERE t.batch_id = $1 AND `+headlineTest+` + GROUP BY GROUPING SETS ( + (s.sender_domain, r.folder), + (s.sender_family, r.folder), + (r.provider, r.folder), + (s.sender_domain, r.provider, r.folder) + ) + `, batchID) + if err != nil { + return nil, err + } + defer rows.Close() + for rows.Next() { + var b PlacementBreakdownRow + if err := rows.Scan(&b.Set, &b.SenderDomain, &b.SenderFamily, &b.RecipientFamily, &b.Folder, &b.Count); err != nil { + return nil, err + } + out = append(out, b) + } + return out, rows.Err() +} + +func (r *placementBatchRepository) BatchGroupSizes(ctx context.Context, batchID uuid.UUID) ([]PlacementBatchGroupSize, error) { + rows, err := r.db.Query(ctx, ` + SELECT GROUPING(sender_domain) = 0, COALESCE(sender_domain, ''), COALESCE(sender_family, ''), COUNT(*), + COUNT(*) FILTER (WHERE status = 'completed') + FROM placement_batch_senders + WHERE batch_id = $1 + GROUP BY GROUPING SETS ((sender_domain), (sender_family)) + `, batchID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []PlacementBatchGroupSize + for rows.Next() { + var g PlacementBatchGroupSize + if err := rows.Scan(&g.ByDomain, &g.Domain, &g.Family, &g.Senders, &g.Completed); err != nil { + return nil, err + } + out = append(out, g) + } + return out, rows.Err() +} + +func (r *placementBatchRepository) Coverage(ctx context.Context, orgID uuid.UUID, now time.Time) (PlacementCoverage, error) { + var c PlacementCoverage + err := r.db.QueryRow(ctx, ` + SELECT COUNT(*), + COUNT(*) FILTER (WHERE last >= $2), + COUNT(*) FILTER (WHERE last >= $3), + COUNT(*) FILTER (WHERE last IS NULL) + FROM ( + SELECT ( + SELECT MAX(pt.created_at) FROM placement_tests pt + WHERE pt.sender_account_id = ea.id AND pt.organization_id = $1 AND pt.status = 'completed' + ) AS last + FROM email_accounts ea + WHERE ea.organization_id = $1 AND ea.seed_scope IS NULL AND ea.status = 'active' + ) fleet + `, orgID, now.AddDate(0, 0, -7), now.AddDate(0, 0, -30)).Scan(&c.Mailboxes, &c.Tested7d, &c.Tested30d, &c.NeverTested) + return c, err +} diff --git a/internal/repository/pg_warmup.go b/internal/repository/pg_warmup.go index 17bfafad3..ca5c9c3ad 100644 --- a/internal/repository/pg_warmup.go +++ b/internal/repository/pg_warmup.go @@ -92,6 +92,9 @@ type WarmupReceived struct { // RetiredAt is when the retention sweep sent the deletion for this // message. A removal observed after that is the platform's own. RetiredAt *time.Time + // LandedSpam is set when the message arrived in the spam folder, so a + // spam label on it is the provider's filter, not the owner. + LandedSpam bool } // WarmupMailToRetire is one warmup message whose retention window has passed, @@ -236,8 +239,18 @@ type WarmupRepository interface { // Tampering protection: track delivered warmup mail so a later deletion or // spam-flag can be attributed, and count "harm" events per mailbox. - RecordWarmupReceived(ctx context.Context, accountID, internalID uuid.UUID, messageID string, senderAccountID uuid.UUID) error + RecordWarmupReceived(ctx context.Context, accountID, internalID uuid.UUID, messageID string, senderAccountID uuid.UUID, landedSpam bool) error GetWarmupReceived(ctx context.Context, accountID, internalID uuid.UUID) (*WarmupReceived, error) + // Spam moves after arrival are held and attributed on the activity around them. + RecordWarmupSpamMove(ctx context.Context, m WarmupSpamMove) (bool, error) + ListSettledWarmupSpamMoves(ctx context.Context, settledBefore time.Time, limit int) ([]WarmupSpamMove, error) + WarmupSpamMoveEvidence(ctx context.Context, m WarmupSpamMove) (WarmupSpamMoveEvidence, error) + ClaimWarmupSpamMove(ctx context.Context, accountID uuid.UUID, messageID string, lease time.Duration) (bool, error) + FixWarmupSpamMoveVerdict(ctx context.Context, accountID uuid.UUID, messageID, verdict string, signals []string) (bool, error) + CompleteWarmupSpamMove(ctx context.Context, accountID uuid.UUID, messageID string) error + CorrelatedOwnerSpamMoves(ctx context.Context, senderID, exceptAccountID uuid.UUID, at time.Time) ([]WarmupSpamMove, error) + ReattributeOwnerSpamMoves(ctx context.Context, senderID, exceptAccountID uuid.UUID, at time.Time) error + RecordOwnerActivity(ctx context.Context, accountID uuid.UUID, at time.Time) error RecordWarmupTampering(ctx context.Context, accountID uuid.UUID, messageID, kind string) (bool, error) CountWarmupTamperingSince(ctx context.Context, accountID uuid.UUID, since time.Time) (int, error) // WithdrawWarmupTampering deletes a strike the mailbox did not earn, @@ -1749,13 +1762,13 @@ func (r *warmupRepository) GetRecentPartnerCounts(ctx context.Context, accountID // RecordWarmupReceived stores a delivered warmup email keyed by recipient + // internal message id. Idempotent on re-delivery of the same message. -func (r *warmupRepository) RecordWarmupReceived(ctx context.Context, accountID, internalID uuid.UUID, messageID string, senderAccountID uuid.UUID) error { +func (r *warmupRepository) RecordWarmupReceived(ctx context.Context, accountID, internalID uuid.UUID, messageID string, senderAccountID uuid.UUID, landedSpam bool) error { query := ` - INSERT INTO warmup_received (email_account_id, internal_id, message_id, sender_account_id) - VALUES ($1, $2, $3, $4) + INSERT INTO warmup_received (email_account_id, internal_id, message_id, sender_account_id, landed_spam) + VALUES ($1, $2, $3, $4, $5) ON CONFLICT (email_account_id, internal_id) DO NOTHING ` - _, err := r.db.Exec(ctx, query, accountID, internalID, messageID, senderAccountID) + _, err := r.db.Exec(ctx, query, accountID, internalID, messageID, senderAccountID, landedSpam) return err } @@ -1763,13 +1776,13 @@ func (r *warmupRepository) RecordWarmupReceived(ctx context.Context, accountID, // message id. Returns nil when the message was not a warmup email. func (r *warmupRepository) GetWarmupReceived(ctx context.Context, accountID, internalID uuid.UUID) (*WarmupReceived, error) { query := ` - SELECT email_account_id, internal_id, message_id, sender_account_id, created_at, retired_at + SELECT email_account_id, internal_id, message_id, sender_account_id, created_at, retired_at, landed_spam FROM warmup_received WHERE email_account_id = $1 AND internal_id = $2 ` var w WarmupReceived err := r.db.QueryRow(ctx, query, accountID, internalID).Scan( - &w.EmailAccountID, &w.InternalID, &w.MessageID, &w.SenderAccountID, &w.CreatedAt, &w.RetiredAt, + &w.EmailAccountID, &w.InternalID, &w.MessageID, &w.SenderAccountID, &w.CreatedAt, &w.RetiredAt, &w.LandedSpam, ) if errors.Is(err, sql.ErrNoRows) { return nil, nil @@ -1883,6 +1896,11 @@ func (r *warmupRepository) PruneWarmupEventsBefore(ctx context.Context, before t `DELETE FROM warmup_tampering_events WHERE created_at < LEAST($1, NOW() - make_interval(days => ` + strconv.Itoa(config.WarmupTamperingKeepDays) + `))`, `DELETE FROM warmup_spam_reports WHERE created_at < $1`, + `DELETE FROM warmup_spam_moves + WHERE decided_at IS NOT NULL + AND observed_at < LEAST($1, NOW() - make_interval(days => ` + strconv.Itoa(config.WarmupTamperingKeepDays) + `))`, + `DELETE FROM mailbox_owner_activity + WHERE bucket < NOW() - make_interval(days => ` + strconv.Itoa(config.WarmupOwnerActivityKeepDays) + `)`, `DELETE FROM warmup_received WHERE created_at < $1 AND retired_at IS NOT NULL`, `DELETE FROM warmup_tokens WHERE created_at < $1 diff --git a/internal/repository/pg_warmup_spam_moves.go b/internal/repository/pg_warmup_spam_moves.go new file mode 100644 index 000000000..9b4aa5298 --- /dev/null +++ b/internal/repository/pg_warmup_spam_moves.go @@ -0,0 +1,217 @@ +package repository + +import ( + "context" + "time" + + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/config" +) + +// Verdicts a warmup spam move is attributed to. +const ( + SpamMovePending = "pending" + SpamMoveOwner = "owner" + SpamMoveProvider = "provider" + SpamMoveUnattributed = "unattributed" +) + +// WarmupSpamMove is a received warmup email seen moving into spam after it +// arrived, which no provider attributes to anyone. +type WarmupSpamMove struct { + EmailAccountID uuid.UUID + MessageID string + SenderAccountID uuid.UUID + ReceivedAt time.Time + ObservedAt time.Time + // Verdict is pending until fixed; a fixed verdict whose effects were not + // all applied is re-applied as it stands, never decided again. + Verdict string + Signals []string +} + +// WarmupSpamMoveEvidence is what the pool and the mailbox show around one move. +type WarmupSpamMoveEvidence struct { + // OwnerActiveNear: the owner acted on their own mail within + // config.WarmupSpamMoveActivityMinutes of the move. + OwnerActiveNear bool + // OwnerActiveRecently: any owner activity in config.WarmupOwnerDormantDays. + OwnerActiveRecently bool + // CorrelatedElsewhere counts the same sender's mail moved to spam in other + // workspaces within config.WarmupSpamMoveCorrelationHours. + CorrelatedElsewhere int + // PatternSenders is distinct senders across this mailbox's unexplained + // moves in the last seven days, this one included. + PatternSenders int +} + +// RecordWarmupSpamMove holds a move for attribution; the first sighting wins. +func (r *warmupRepository) RecordWarmupSpamMove(ctx context.Context, m WarmupSpamMove) (bool, error) { + cmd, err := r.db.Exec(ctx, ` + INSERT INTO warmup_spam_moves (email_account_id, message_id, sender_account_id, received_at) + VALUES ($1, $2, $3, $4) + ON CONFLICT (email_account_id, message_id) DO NOTHING`, + m.EmailAccountID, m.MessageID, m.SenderAccountID, m.ReceivedAt) + if err != nil { + return false, err + } + return cmd.RowsAffected() > 0, nil +} + +// ListSettledWarmupSpamMoves is unclaimed moves observed before settledBefore +// whose verdict is not yet applied, oldest first. +func (r *warmupRepository) ListSettledWarmupSpamMoves(ctx context.Context, settledBefore time.Time, limit int) ([]WarmupSpamMove, error) { + rows, err := r.db.Query(ctx, ` + SELECT email_account_id, message_id, sender_account_id, received_at, observed_at, verdict, signals + FROM warmup_spam_moves + WHERE decided_at IS NULL AND observed_at < $1 + AND (claimed_until IS NULL OR claimed_until < NOW()) + ORDER BY observed_at + LIMIT $2`, settledBefore, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var out []WarmupSpamMove + for rows.Next() { + var m WarmupSpamMove + if err := rows.Scan(&m.EmailAccountID, &m.MessageID, &m.SenderAccountID, &m.ReceivedAt, &m.ObservedAt, &m.Verdict, &m.Signals); err != nil { + return nil, err + } + out = append(out, m) + } + return out, rows.Err() +} + +// ClaimWarmupSpamMove gives one consumer the move for lease; false when another holds it or it is done. +func (r *warmupRepository) ClaimWarmupSpamMove(ctx context.Context, accountID uuid.UUID, messageID string, lease time.Duration) (bool, error) { + cmd, err := r.db.Exec(ctx, ` + UPDATE warmup_spam_moves + SET claimed_until = NOW() + make_interval(secs => $3) + WHERE email_account_id = $1 AND message_id = $2 AND decided_at IS NULL + AND (claimed_until IS NULL OR claimed_until < NOW())`, + accountID, messageID, lease.Seconds()) + if err != nil { + return false, err + } + return cmd.RowsAffected() > 0, nil +} + +// WarmupSpamMoveEvidence gathers the evidence for one move in one round trip. +func (r *warmupRepository) WarmupSpamMoveEvidence(ctx context.Context, m WarmupSpamMove) (WarmupSpamMoveEvidence, error) { + var ev WarmupSpamMoveEvidence + err := r.db.QueryRow(ctx, ` + SELECT + EXISTS (SELECT 1 FROM mailbox_owner_activity a + WHERE a.email_account_id = $1 + AND a.bucket >= $2::timestamptz - make_interval(mins => $4 + 5) + AND a.bucket <= $2::timestamptz + make_interval(mins => $4)), + EXISTS (SELECT 1 FROM mailbox_owner_activity a + WHERE a.email_account_id = $1 + AND a.bucket >= $2::timestamptz - make_interval(days => $5)), + (SELECT COUNT(*) FROM warmup_spam_moves o + JOIN email_accounts oa ON oa.id = o.email_account_id + WHERE o.sender_account_id = $3 + AND o.email_account_id <> $1 + AND oa.organization_id IS DISTINCT FROM (SELECT organization_id FROM email_accounts WHERE id = $1) + AND o.observed_at BETWEEN $2::timestamptz - make_interval(hours => $6) AND $2::timestamptz + make_interval(hours => $6)), + (SELECT COUNT(DISTINCT s) FROM ( + SELECT sender_account_id AS s FROM warmup_spam_moves + WHERE email_account_id = $1 + AND verdict IN ('owner', 'unattributed') + AND observed_at >= $2::timestamptz - INTERVAL '7 days' AND observed_at <= $2::timestamptz + UNION SELECT $3::uuid) p)`, + m.EmailAccountID, m.ObservedAt, m.SenderAccountID, + config.WarmupSpamMoveActivityMinutes, config.WarmupOwnerDormantDays, config.WarmupSpamMoveCorrelationHours, + ).Scan(&ev.OwnerActiveNear, &ev.OwnerActiveRecently, &ev.CorrelatedElsewhere, &ev.PatternSenders) + return ev, err +} + +// FixWarmupSpamMoveVerdict records the verdict once, before its effects are applied. +func (r *warmupRepository) FixWarmupSpamMoveVerdict(ctx context.Context, accountID uuid.UUID, messageID, verdict string, signals []string) (bool, error) { + cmd, err := r.db.Exec(ctx, ` + UPDATE warmup_spam_moves + SET verdict = $3, signals = $4 + WHERE email_account_id = $1 AND message_id = $2 AND verdict = 'pending' AND decided_at IS NULL`, + accountID, messageID, verdict, signals) + if err != nil { + return false, err + } + return cmd.RowsAffected() > 0, nil +} + +// CompleteWarmupSpamMove marks the verdict's effects applied and releases the claim. +func (r *warmupRepository) CompleteWarmupSpamMove(ctx context.Context, accountID uuid.UUID, messageID string) error { + _, err := r.db.Exec(ctx, ` + UPDATE warmup_spam_moves + SET decided_at = NOW(), claimed_until = NULL + WHERE email_account_id = $1 AND message_id = $2 AND decided_at IS NULL`, + accountID, messageID) + return err +} + +// correlatedOwnerMovesWhere selects the sender's applied owner verdicts in +// other workspaces near $3, for sender $1 and the correlating mailbox $2. One +// still being applied keeps its verdict, so a strike always matches its row. +const correlatedOwnerMovesWhere = ` + o.sender_account_id = $1 + AND o.email_account_id <> $2 + AND o.verdict = 'owner' + AND o.decided_at IS NOT NULL + AND o.email_account_id IN ( + SELECT oa.id FROM email_accounts oa + WHERE oa.organization_id IS DISTINCT FROM (SELECT organization_id FROM email_accounts WHERE id = $2)) + AND o.observed_at BETWEEN $3::timestamptz - make_interval(hours => $4) AND $3::timestamptz + make_interval(hours => $4)` + +// CorrelatedOwnerSpamMoves lists the owner verdicts a correlation now explains. +func (r *warmupRepository) CorrelatedOwnerSpamMoves(ctx context.Context, senderID, exceptAccountID uuid.UUID, at time.Time) ([]WarmupSpamMove, error) { + rows, err := r.db.Query(ctx, ` + SELECT o.email_account_id, o.message_id, o.sender_account_id, o.received_at, o.observed_at + FROM warmup_spam_moves o + WHERE `+correlatedOwnerMovesWhere, + senderID, exceptAccountID, at, config.WarmupSpamMoveCorrelationHours) + if err != nil { + return nil, err + } + defer rows.Close() + var out []WarmupSpamMove + for rows.Next() { + var m WarmupSpamMove + if err := rows.Scan(&m.EmailAccountID, &m.MessageID, &m.SenderAccountID, &m.ReceivedAt, &m.ObservedAt); err != nil { + return nil, err + } + out = append(out, m) + } + return out, rows.Err() +} + +// ReattributeOwnerSpamMoves turns those moves into provider moves, and the +// complaint each filed against the sender into placement. +func (r *warmupRepository) ReattributeOwnerSpamMoves(ctx context.Context, senderID, exceptAccountID uuid.UUID, at time.Time) error { + _, err := r.db.Exec(ctx, ` + WITH moved AS ( + UPDATE warmup_spam_moves o + SET verdict = 'provider', signals = array_append(o.signals, 'correlated_later'), decided_at = NOW() + WHERE `+correlatedOwnerMovesWhere+` + RETURNING o.email_account_id, o.message_id + ) + UPDATE warmup_spam_reports sr + SET report_type = 'spam_placement' + FROM moved + WHERE sr.reporter_account_id = moved.email_account_id + AND sr.message_id = moved.message_id + AND sr.report_type = 'user_complaint'`, + senderID, exceptAccountID, at, config.WarmupSpamMoveCorrelationHours) + return err +} + +// RecordOwnerActivity marks the five-minute bucket the owner acted in. +func (r *warmupRepository) RecordOwnerActivity(ctx context.Context, accountID uuid.UUID, at time.Time) error { + _, err := r.db.Exec(ctx, ` + INSERT INTO mailbox_owner_activity (email_account_id, bucket) + VALUES ($1, to_timestamp(floor(extract(epoch FROM $2::timestamptz) / 300) * 300)) + ON CONFLICT (email_account_id, bucket) DO UPDATE SET events = mailbox_owner_activity.events + 1`, + accountID, at) + return err +} diff --git a/internal/repository/placement_batch_live_test.go b/internal/repository/placement_batch_live_test.go new file mode 100644 index 000000000..583c492a5 --- /dev/null +++ b/internal/repository/placement_batch_live_test.go @@ -0,0 +1,373 @@ +package repository + +import ( + "context" + "testing" + "time" + + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/models" +) + +// Run against a migrated scratch database: +// +// WARMBLY_TEST_DB=postgres://warmbly:warmbly@localhost:15432/?sslmode=disable \ +// go test ./internal/repository/ -run LivePlacementBatch -v + +func TestLivePlacementBatchLifecycle(t *testing.T) { + f := newPlacementFixture(t) + handle, pool := liveContactDB(t) + requireSchemaVersion(t, pool, 232) + batches := NewPlacementBatchRepository(handle) + ctx := context.Background() + now := time.Now() + + // A second sender on another domain, and a tag on the first. + other := uuid.New() + f.exec(`INSERT INTO email_accounts (id, user_id, organization_id, email, name, signature_plain, signature_html, provider, campaign_limit) + VALUES ($1, $2, $3, $4, 'Placement', '', '', 'outlook', 50)`, other, f.owner, f.org, "rep-"+other.String()[:8]+"@beta.test") + tagID := uuid.New() + f.exec(`INSERT INTO tags (id, organization_id, user_id, title, color, position) VALUES ($1, $2, $3, 'Client A', '#0ea5e9', 0)`, tagID, f.org, f.owner) + f.exec(`INSERT INTO email_tags (email_id, tag_id) VALUES ($1, $2)`, f.sender, tagID) + f.exec(`UPDATE email_accounts SET status = 'active' WHERE organization_id = $1`, f.org) + + all, err := batches.ListBatchCandidates(ctx, f.org, PlacementCandidateFilter{}) + if err != nil || len(all) != 2 { + t.Fatalf("ListBatchCandidates = %d, %v; want both senders", len(all), err) + } + tagged, err := batches.ListBatchCandidates(ctx, f.org, PlacementCandidateFilter{TagIDs: []uuid.UUID{tagID}}) + if err != nil || len(tagged) != 1 || tagged[0].ID != f.sender { + t.Fatalf("tag filter = %+v, %v; want the tagged sender", tagged, err) + } + // Another workspace's mailboxes (the operator's seeds) never appear. + if foreign, err := batches.ListBatchCandidates(ctx, f.org, PlacementCandidateFilter{IDs: f.seeds}); err != nil || len(foreign) != 0 { + t.Fatalf("foreign ids resolved %d, %v", len(foreign), err) + } + + b := &models.PlacementBatch{ + ID: uuid.New(), OrganizationID: f.org, Subject: "Quick question", BodyPlain: "Hi there", + Tracking: models.PlacementTrackingCompare, Panel: models.PlacementPanelInstance, Pace: models.PlacementPaceSpaced, + OnUnavailable: models.PlacementUnavailableDefer, SenderCount: 2, Status: models.PlacementBatchQueued, + Selection: models.PlacementBatchSelection{Scope: &models.PlacementSenderScope{Type: models.PlacementScopeWorkspace}, Matched: 2}, + RetryUntil: now.Add(7 * 24 * time.Hour), + } + senderRow, otherRow := uuid.New(), uuid.New() + sender, otherID := f.sender, other + if err := batches.CreateBatch(ctx, b, []models.PlacementBatchSender{ + {ID: senderRow, EmailAccountID: &sender, SenderEmail: "a@acme.test", SenderDomain: "acme.test", SenderFamily: "gmail", Position: 0}, + {ID: otherRow, EmailAccountID: &otherID, SenderEmail: "b@beta.test", SenderDomain: "beta.test", SenderFamily: "outlook", Position: 1}, + }); err != nil { + t.Fatalf("CreateBatch: %v", err) + } + if n, err := batches.CountOpenBatches(ctx, f.org); err != nil || n != 1 { + t.Fatalf("CountOpenBatches = %d, %v", n, err) + } + + claimed, err := batches.ClaimBatches(ctx, now, 2*time.Minute, 100) + if err != nil { + t.Fatalf("ClaimBatches: %v", err) + } + var mine *models.PlacementBatch + for i := range claimed { + if claimed[i].ID == b.ID { + mine = &claimed[i] + } + } + if mine == nil || mine.LastTickAt != nil || mine.Selection.Matched != 2 { + t.Fatalf("claimed batch = %+v; want it with no previous tick and its selection", mine) + } + again, err := batches.ClaimBatches(ctx, now, 2*time.Minute, 100) + if err != nil { + t.Fatalf("ClaimBatches: %v", err) + } + for _, c := range again { + if c.ID == b.ID { + t.Fatalf("a leased batch was claimed twice") + } + } + if err := batches.ReleaseBatch(ctx, b.ID); err != nil { + t.Fatalf("ReleaseBatch: %v", err) + } + again, err = batches.ClaimBatches(ctx, now.Add(time.Second), time.Minute, 100) + if err != nil || !containsBatch(again, b.ID) { + t.Fatalf("a released batch was not claimable (%v)", err) + } + for _, c := range again { + if c.ID == b.ID && (c.LastTickAt == nil || c.LastTickAt.Sub(now).Abs() > time.Millisecond) { + t.Fatalf("second claim reports last tick %v; want the first claim's %v", c.LastTickAt, now) + } + } + _ = batches.ReleaseBatch(ctx, b.ID) + + due, err := batches.DueBatchSenders(ctx, b.ID, now.Add(time.Minute), 10) + if err != nil || len(due) != 2 || due[0].ID != senderRow { + t.Fatalf("DueBatchSenders = %+v, %v; want both in position order", due, err) + } + if ok, err := batches.ClaimBatchSender(ctx, senderRow); err != nil || !ok { + t.Fatalf("ClaimBatchSender = %v, %v", ok, err) + } + if ok, _ := batches.ClaimBatchSender(ctx, senderRow); ok { + t.Fatalf("a running sender was claimed twice") + } + // Claimed with no test yet: it counts as sending. + if n, err := batches.CountSendingBatchSenders(ctx, &f.org); err != nil || n != 1 { + t.Fatalf("CountSendingBatchSenders = %d, %v; want the claimed sender", n, err) + } + + // The claimed sender gets its two tests (a comparison); one copy lands in + // the inbox and one in spam for the tracked half, one inbox untracked. + batchID, rowID := b.ID, senderRow + mk := func(tracked bool, folders ...string) models.PlacementTest { + test := models.PlacementTest{ + ID: uuid.New(), OrganizationID: &f.org, SenderAccountID: &sender, SenderEmail: "a@acme.test", + Subject: "Quick question", BodyPlain: "Hi there", Origin: models.PlacementOriginBatch, + Panel: models.PlacementPanelInstance, Status: models.PlacementStatusRunning, + OpenTracking: tracked, BatchID: &batchID, BatchSenderID: &rowID, + } + var results []models.PlacementResult + for i, folder := range folders { + seed := f.seeds[i] + results = append(results, models.PlacementResult{ + ID: uuid.New(), SeedAccountID: &seed, SeedAddress: seed.String() + "@gmail.com", Family: "gmail", Folder: folder, + }) + } + if err := f.repo.CreateTest(ctx, &test, results, nil); err != nil { + t.Fatalf("CreateTest: %v", err) + } + return test + } + tracked := mk(true, "inbox", "spam") + untracked := mk(false, "inbox", "inbox") + + // Workspace listings leave batch tests to their batch. + listed, _, err := f.repo.ListTests(ctx, PlacementTestFilter{OrganizationID: &f.org}) + if err != nil || len(listed) != 0 { + t.Fatalf("ListTests = %d, %v; want batch tests left out", len(listed), err) + } + listed, _, err = f.repo.ListTests(ctx, PlacementTestFilter{OrganizationID: &f.org, BatchID: &batchID}) + if err != nil || len(listed) != 2 { + t.Fatalf("ListTests by batch = %d, %v; want its two tests", len(listed), err) + } + if got, _ := f.repo.GetTest(ctx, tracked.ID); got == nil || got.BatchID == nil || *got.BatchID != batchID { + t.Fatalf("GetTest lost the batch id: %+v", got) + } + + // Nothing unsent any more: the sender stops counting as sending, and once + // both tests finish it resolves to completed. + if n, _ := batches.CountSendingBatchSenders(ctx, &f.org); n != 0 { + t.Fatalf("CountSendingBatchSenders = %d with every copy resolved", n) + } + if err := batches.SyncBatchSenders(ctx, b.ID, now.Add(-time.Hour)); err != nil { + t.Fatalf("SyncBatchSenders: %v", err) + } + if p := progressOf(t, batches, b.ID); p.Running != 1 { + t.Fatalf("progress %+v; want the sender still running while its tests are open", p) + } + f.exec(`UPDATE placement_tests SET status = 'completed', finished_at = NOW() WHERE batch_id = $1`, b.ID) + if err := batches.SyncBatchSenders(ctx, b.ID, now.Add(-time.Hour)); err != nil { + t.Fatalf("SyncBatchSenders: %v", err) + } + if p := progressOf(t, batches, b.ID); p.Completed != 1 || p.Queued != 1 { + t.Fatalf("progress %+v; want one completed and one queued", p) + } + + // Headline is the tracked half of the comparison. + sums, err := batches.BatchSummaries(ctx, []uuid.UUID{b.ID}) + if err != nil || sums[b.ID].Inbox != 1 || sums[b.ID].Spam != 1 { + t.Fatalf("BatchSummaries = %+v, %v; want the tracked half only", sums[b.ID], err) + } + rows, err := batches.BatchBreakdown(ctx, b.ID) + if err != nil { + t.Fatalf("BatchBreakdown: %v", err) + } + sets := map[string]int{} + var untrackedInbox int + for _, r := range rows { + sets[r.Set] += r.Count + if r.Set == "overall" && !r.Tracked && r.Folder == "inbox" { + untrackedInbox = r.Count + } + if r.Set == "domain" && r.SenderDomain != "acme.test" { + t.Fatalf("domain row for %q", r.SenderDomain) + } + if r.Set == "matrix" && (r.SenderDomain != "acme.test" || r.RecipientFamily != "gmail") { + t.Fatalf("matrix row %+v", r) + } + } + if sets["overall"] != 4 || sets["domain"] != 2 || sets["provider"] != 2 || sets["recipient"] != 2 || sets["matrix"] != 2 || untrackedInbox != 2 { + t.Fatalf("breakdown totals %v (untracked inbox %d); want 4 overall and the 2 tracked copies in every group", sets, untrackedInbox) + } + sizes, err := batches.BatchGroupSizes(ctx, b.ID) + if err != nil { + t.Fatalf("BatchGroupSizes: %v", err) + } + var domainRows, familyRows int + for _, s := range sizes { + if s.ByDomain { + domainRows++ + } else { + familyRows++ + } + if s.ByDomain && s.Domain == "acme.test" && (s.Senders != 1 || s.Completed != 1) { + t.Fatalf("acme.test size %+v", s) + } + } + if domainRows != 2 || familyRows != 2 { + t.Fatalf("group sizes %+v; want two domains and two providers", sizes) + } + + senders, total, err := batches.ListBatchSenders(ctx, f.org, b.ID, PlacementBatchSenderFilter{Limit: 10}) + if err != nil || total != 2 || len(senders) != 2 { + t.Fatalf("ListBatchSenders = %d/%d, %v", len(senders), total, err) + } + if senders[0].ID != senderRow || senders[0].Counts.Inbox != 1 || senders[0].Counts.Spam != 1 || len(senders[0].TestIDs) != 2 { + t.Fatalf("first sender %+v; want the tested one with its tracked counts and both tests", senders[0]) + } + if senders[0].TestIDs[0] != untracked.ID { + t.Fatalf("test ids %v; want the untracked half first", senders[0].TestIDs) + } + found, total, err := batches.ListBatchSenders(ctx, f.org, b.ID, PlacementBatchSenderFilter{Search: "beta", Limit: 10}) + if err != nil || total != 1 || found[0].ID != otherRow { + t.Fatalf("search = %+v, %d, %v", found, total, err) + } + if _, total, _ := batches.ListBatchSenders(ctx, uuid.New(), b.ID, PlacementBatchSenderFilter{Limit: 10}); total != 0 { + t.Fatalf("another workspace listed %d of this batch's senders", total) + } + + cov, err := batches.Coverage(ctx, f.org, time.Now()) + if err != nil || cov.Mailboxes != 2 || cov.Tested7d != 1 || cov.NeverTested != 1 { + t.Fatalf("Coverage = %+v, %v; want one of two tested this week", cov, err) + } + if untested, err := batches.ListBatchCandidates(ctx, f.org, PlacementCandidateFilter{UntestedSince: ptrTime(now.Add(-24 * time.Hour))}); err != nil || len(untested) != 1 || untested[0].ID != other { + t.Fatalf("untested filter = %+v, %v", untested, err) + } + + // Cancelling closes the queued sender; a second cancel is refused. + ok, running, err := batches.CancelBatch(ctx, f.org, b.ID) + if err != nil || !ok || len(running) != 0 { + t.Fatalf("CancelBatch = %v, %v, %v", ok, running, err) + } + if p := progressOf(t, batches, b.ID); p.Cancelled != 1 || p.Open() != 0 { + t.Fatalf("after cancel %+v", p) + } + if ok, _, _ := batches.CancelBatch(ctx, f.org, b.ID); ok { + t.Fatalf("a cancelled batch cancelled again") + } + if ok, err := batches.FinishBatch(ctx, b.ID, models.PlacementBatchCompleted, ""); err != nil || ok { + t.Fatalf("FinishBatch on a cancelled batch = %v, %v", ok, err) + } +} + +func TestLivePlacementBatchImportedLandsClosed(t *testing.T) { + f := newPlacementFixture(t) + handle, pool := liveContactDB(t) + requireSchemaVersion(t, pool, 232) + batches := NewPlacementBatchRepository(handle) + ctx := context.Background() + + // An imported batch is written without its active flag. + id := uuid.New() + f.exec(`INSERT INTO placement_batches (id, organization_id, tracking, panel, pace, sender_count, status, retry_until) + VALUES ($1, $2, 'campaign', 'instance', 'spaced', 1, 'running', NOW())`, id, f.org) + f.exec(`INSERT INTO placement_batch_senders (batch_id, sender_email, position) VALUES ($1, 'a@acme.test', 0)`, id) + if claimed, err := batches.ClaimBatches(ctx, time.Now(), time.Minute, 100); err != nil || containsBatch(claimed, id) { + t.Fatalf("an inactive batch was claimed (%v)", err) + } + if err := batches.CloseInactiveBatches(ctx); err != nil { + t.Fatalf("CloseInactiveBatches: %v", err) + } + got, err := batches.GetBatch(ctx, f.org, id) + if err != nil || got == nil || got.Status != models.PlacementBatchCancelled { + t.Fatalf("imported batch = %+v, %v; want it closed", got, err) + } + if p := progressOf(t, batches, id); p.Cancelled != 1 { + t.Fatalf("imported batch senders %+v; want them cancelled", p) + } +} + +func progressOf(t *testing.T, r PlacementBatchRepository, id uuid.UUID) models.PlacementBatchProgress { + t.Helper() + p, err := r.BatchProgress(context.Background(), []uuid.UUID{id}) + if err != nil { + t.Fatalf("BatchProgress: %v", err) + } + return p[id] +} + +func containsBatch(bs []models.PlacementBatch, id uuid.UUID) bool { + for _, b := range bs { + if b.ID == id { + return true + } + } + return false +} + +func ptrTime(t time.Time) *time.Time { return &t } + +func TestLivePlacementBatchCancelResolvesRunningSenders(t *testing.T) { + f := newPlacementFixture(t) + handle, pool := liveContactDB(t) + requireSchemaVersion(t, pool, 232) + batches := NewPlacementBatchRepository(handle) + ctx := context.Background() + + f.exec(`UPDATE email_accounts SET status = 'active' WHERE organization_id = $1`, f.org) + b := &models.PlacementBatch{ + ID: uuid.New(), OrganizationID: f.org, Subject: "Quick question", BodyPlain: "Hi there", + Tracking: models.PlacementTrackingCampaign, Panel: models.PlacementPanelInstance, Pace: models.PlacementPaceSpaced, + OnUnavailable: models.PlacementUnavailableDefer, SenderCount: 1, Status: models.PlacementBatchQueued, + RetryUntil: time.Now().Add(time.Hour), + } + row, sender := uuid.New(), f.sender + if err := batches.CreateBatch(ctx, b, []models.PlacementBatchSender{ + {ID: row, EmailAccountID: &sender, SenderEmail: "a@acme.test", SenderDomain: "acme.test", SenderFamily: "gmail"}, + }); err != nil { + t.Fatalf("CreateBatch: %v", err) + } + if ok, err := batches.ClaimBatchSender(ctx, row); err != nil || !ok { + t.Fatalf("ClaimBatchSender = %v, %v", ok, err) + } + batchID := b.ID + test := models.PlacementTest{ + ID: uuid.New(), OrganizationID: &f.org, SenderAccountID: &sender, SenderEmail: "a@acme.test", + Subject: "Quick question", BodyPlain: "Hi there", Origin: models.PlacementOriginBatch, + Panel: models.PlacementPanelInstance, Status: models.PlacementStatusRunning, BatchID: &batchID, BatchSenderID: &row, + } + seed := f.seeds[0] + if err := f.repo.CreateTest(ctx, &test, []models.PlacementResult{{ID: uuid.New(), SeedAccountID: &seed, SeedAddress: "s@gmail.com", Family: "gmail", Folder: "pending"}}, nil); err != nil { + t.Fatalf("CreateTest: %v", err) + } + + ok, running, err := batches.CancelBatch(ctx, f.org, b.ID) + if err != nil || !ok || len(running) != 1 || running[0] != test.ID { + t.Fatalf("CancelBatch = %v, %v, %v; want the running test to cancel", ok, running, err) + } + late, err := batches.RunningTestsOfCancelledBatches(ctx, 1000) + if err != nil || !containsOrgTest(late, test.ID) { + t.Fatalf("RunningTestsOfCancelledBatches missed the running test (%v)", err) + } + if err := batches.SyncClosedBatchSenders(ctx); err != nil { + t.Fatalf("SyncClosedBatchSenders: %v", err) + } + if p := progressOf(t, batches, b.ID); p.Running != 1 { + t.Fatalf("progress %+v; want the sender running while its test is open", p) + } + f.exec(`UPDATE placement_tests SET status = 'cancelled', finished_at = NOW() WHERE id = $1`, test.ID) + if err := batches.SyncClosedBatchSenders(ctx); err != nil { + t.Fatalf("SyncClosedBatchSenders: %v", err) + } + if p := progressOf(t, batches, b.ID); p.Cancelled != 1 || p.Open() != 0 { + t.Fatalf("progress %+v; want the sender resolved as cancelled", p) + } +} + +func containsOrgTest(ts []PlacementOrgTest, id uuid.UUID) bool { + for _, t := range ts { + if t.TestID == id { + return true + } + } + return false +} diff --git a/internal/repository/pool_link_live_test.go b/internal/repository/pool_link_live_test.go index 729fabd2d..c7e8e662b 100644 --- a/internal/repository/pool_link_live_test.go +++ b/internal/repository/pool_link_live_test.go @@ -143,7 +143,7 @@ func TestLivePoolLinkWarmupDeliveryMatchesAndRefuses(t *testing.T) { // A delivery already recorded, whose token has since been cleaned up. const receivedID = "" - if err := f.warmup.RecordWarmupReceived(ctx, f.recipient, uuid.New(), receivedID, f.sender); err != nil { + if err := f.warmup.RecordWarmupReceived(ctx, f.recipient, uuid.New(), receivedID, f.sender, false); err != nil { t.Fatalf("RecordWarmupReceived: %v", err) } ok, err = f.warmup.IsWarmupDelivery(ctx, f.recipient, "someone@elsewhere.test", receivedID, "Anything") diff --git a/internal/repository/provider_folder_messages_live_test.go b/internal/repository/provider_folder_messages_live_test.go new file mode 100644 index 000000000..bb7377f2f --- /dev/null +++ b/internal/repository/provider_folder_messages_live_test.go @@ -0,0 +1,63 @@ +package repository + +import ( + "context" + "testing" + "time" + + "github.com/google/uuid" +) + +// The Gmail folder reconciliation reads the rows Gmail last had in a folder, +// newest first, for one mailbox, and only those it can look up by Gmail id. +// +// WARMBLY_TEST_DB=postgres://warmbly:warmbly@localhost:15432/?sslmode=disable \ +// go test ./internal/repository/ -run LiveProviderFolderMessages -v +func TestLiveProviderFolderMessages(t *testing.T) { + d, pool := liveContactDB(t) + f := newThreadParentFixture(t, pool) + t.Cleanup(func() { + if _, err := pool.Exec(context.Background(), + `DELETE FROM unibox_emails WHERE email_id IN (SELECT id FROM email_accounts WHERE organization_id = $1)`, f.org); err != nil { + t.Errorf("cleanup: %v", err) + } + }) + + now := time.Now().UTC().Truncate(time.Second) + row := func(mailbox uuid.UUID, gmailID, folder, providerFolder string, age time.Duration) uuid.UUID { + id := uuid.New() + f.exec(`INSERT INTO unibox_emails (id, user_id, email_id, gmail_id, folder, provider_folder, internal_date) + VALUES ($1, $2, $3, $4, $5, $6, $7)`, + id, f.owner, mailbox, gmailID, folder, providerFolder, now.Add(-age)) + return id + } + newest := row(f.mailbox, "g-new", "inbox", "inbox", time.Hour) + filedHere := row(f.mailbox, "g-filed", "archive", "inbox", 2*time.Hour) + archived := row(f.mailbox, "g-arch", "archive", "archive", 3*time.Hour) + row(f.mailbox, "g-sent", "sent", "sent", 30*time.Minute) + row(f.mailbox, "", "inbox", "inbox", 10*time.Minute) + row(f.other, "g-other", "inbox", "inbox", 5*time.Minute) + + repo := NewEmailSyncStateRepository(d) + got, err := repo.ListProviderFolderMessages(context.Background(), f.owner, f.mailbox, []string{"inbox", "archive"}, 10) + if err != nil { + t.Fatalf("list: %v", err) + } + want := []uuid.UUID{newest, filedHere, archived} + if len(got) != len(want) { + t.Fatalf("got %d rows, want %d: %+v", len(got), len(want), got) + } + for i, id := range want { + if got[i].ID != id { + t.Errorf("row %d = %s, want %s", i, got[i].ID, id) + } + } + if got[1].ProviderID != "g-filed" || got[1].ProviderFolder != "inbox" { + t.Errorf("filed row = %+v, want Gmail id g-filed in provider folder inbox", got[1]) + } + + limited, err := repo.ListProviderFolderMessages(context.Background(), f.owner, f.mailbox, []string{"inbox", "archive"}, 1) + if err != nil || len(limited) != 1 || limited[0].ID != newest { + t.Fatalf("limit 1 = %+v (%v), want only the newest row", limited, err) + } +} diff --git a/internal/tasks/campaign_copies.go b/internal/tasks/campaign_copies.go new file mode 100644 index 000000000..f8a34f743 --- /dev/null +++ b/internal/tasks/campaign_copies.go @@ -0,0 +1,88 @@ +package tasks + +import ( + "context" + "strings" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/pkg/mailhdr" +) + +// campaignCopies resolves who is copied on one send: the campaign's own CC and +// BCC, then the lead's copied contacts. An address that is suppressed, is the +// lead's own, or already appears earlier is left off, so a copy can never +// reach someone the lead's email would not have been allowed to. +func (s *tasksService) campaignCopies(ctx context.Context, orgID uuid.UUID, campaign *models.Campaign, contact *models.Contact) (cc, bcc []string, err error) { + seen := map[string]bool{strings.ToLower(strings.TrimSpace(contact.Email)): true} + keep := func(addr string) (bool, error) { + bare := strings.ToLower(mailhdr.Bare(addr)) + if bare == "" || seen[bare] { + return false, nil + } + if s.advanced != nil { + suppressed, _, xerr := s.advanced.ShouldSuppressRecipient(ctx, orgID, bare) + if xerr != nil { + return false, xerr + } + if suppressed { + return false, nil + } + } + seen[bare] = true + return true, nil + } + + // A campaign-wide copy that bounced on this campaign is dropped, so one bad + // address cannot keep failing every lead's send. + var wide []string + for _, a := range append(append([]string{}, campaign.CC...), campaign.BCC...) { + if bare := strings.ToLower(mailhdr.Bare(a)); bare != "" { + wide = append(wide, bare) + } + } + bounced, berr := s.campaignProgressRepo.BouncedCopyAddresses(ctx, campaign.ID, wide) + if berr != nil { + return nil, nil, berr + } + for a := range bounced { + seen[a] = true + } + + for _, a := range campaign.CC { + ok, kerr := keep(a) + if kerr != nil { + return nil, nil, kerr + } + if ok { + cc = append(cc, a) + } + } + for _, a := range campaign.BCC { + ok, kerr := keep(a) + if kerr != nil { + return nil, nil, kerr + } + if ok { + bcc = append(bcc, a) + } + } + + copies, lerr := s.campaignProgressRepo.ListLeadCC(ctx, campaign.ID, contact.ID) + if lerr != nil { + return nil, nil, lerr + } + for _, c := range copies { + // The status already applied suppression, bounces and verification. + if !c.Copied() { + continue + } + bare := strings.ToLower(strings.TrimSpace(c.Email)) + if bare == "" || seen[bare] { + continue + } + seen[bare] = true + cc = append(cc, c.Email) + } + return cc, bcc, nil +} diff --git a/internal/tasks/campaign_copies_test.go b/internal/tasks/campaign_copies_test.go new file mode 100644 index 000000000..69a6fed21 --- /dev/null +++ b/internal/tasks/campaign_copies_test.go @@ -0,0 +1,68 @@ +package tasks + +import ( + "context" + "reflect" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/app/advanced" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +type copiesAdvanced struct { + advanced.Service + suppressed map[string]bool +} + +func (f copiesAdvanced) ShouldSuppressRecipient(_ context.Context, _ uuid.UUID, recipient string) (bool, string, *errx.Error) { + return f.suppressed[strings.ToLower(recipient)], "", nil +} + +type copiesProgress struct { + repository.CampaignProgressRepository + cc []models.CampaignLeadCC +} + +func (f copiesProgress) ListLeadCC(context.Context, uuid.UUID, uuid.UUID) ([]models.CampaignLeadCC, error) { + return f.cc, nil +} + +func (f copiesProgress) BouncedCopyAddresses(context.Context, uuid.UUID, []string) (map[string]bool, error) { + return map[string]bool{"refused@acme.test": true}, nil +} + +// A copy never reaches someone the lead's own email could not: suppressed +// campaign copies, the lead's own address, repeats and lead copies the status +// already refused are all left off. +func TestCampaignCopiesFiltersEveryCopy(t *testing.T) { + s := &tasksService{ + advanced: copiesAdvanced{suppressed: map[string]bool{"gone@acme.test": true}}, + campaignProgressRepo: copiesProgress{cc: []models.CampaignLeadCC{ + {Email: "jonas@acme.test", Status: models.LeadCCStatusActive}, + {Email: "bounced@acme.test", Status: models.LeadCCStatusBounced}, + {Email: "Boss@acme.test", Status: models.LeadCCStatusActive}, + {Email: "ana@acme.test", Status: models.LeadCCStatusActive}, + }}, + } + campaign := &models.Campaign{ + ID: uuid.New(), + CC: []string{"Boss ", "gone@acme.test", "ANA@acme.test"}, + BCC: []string{"crm@acme.test", "boss@acme.test", "Refused@acme.test"}, + } + contact := &models.Contact{ID: uuid.New(), Email: "ana@acme.test"} + + cc, bcc, err := s.campaignCopies(context.Background(), uuid.New(), campaign, contact) + if err != nil { + t.Fatalf("campaignCopies: %v", err) + } + if want := []string{"Boss ", "jonas@acme.test"}; !reflect.DeepEqual(cc, want) { + t.Fatalf("cc = %v, want %v", cc, want) + } + if want := []string{"crm@acme.test"}; !reflect.DeepEqual(bcc, want) { + t.Fatalf("bcc = %v, want %v", bcc, want) + } +} diff --git a/internal/tasks/campaign_task.go b/internal/tasks/campaign_task.go index a061af98c..3f63dc3df 100644 --- a/internal/tasks/campaign_task.go +++ b/internal/tasks/campaign_task.go @@ -540,6 +540,18 @@ func (s *tasksService) HandleCampaignTask(task *proto.ProcessTask) (result *errx taskRecord.EmailAccountID = account.ID } + // Who else the email copies, read for an email step before the send is + // reserved. Fail closed: copies that cannot be checked against suppression + // are not sent, and neither is the email without the copies chosen. + copyCC, copyBCC, cerr := s.campaignCopies(ctx, orgID, campaign, contact) + if cerr != nil { + errs.CaptureException(cerr) + s.taskRepo.RecordTaskFailure(ctx, taskID, "Could not read who the email copies", cerr.Error()) + s.retryCampaignTickLater(ctx, taskRecord) + executionStatus = "failed" + return errx.InternalError() + } + // STEP 9.4: The conversation this step joins. A follow-up is a nudge on the // email the contact already has, not a second cold email, so every step // after their first is threaded onto the last one they received: the @@ -815,8 +827,8 @@ func (s *tasksService) HandleCampaignTask(task *proto.ProcessTask) (result *errx emailMsg := EmailMessage{ From: account.Email, To: []string{contact.Email}, - CC: campaign.CC, - BCC: campaign.BCC, + CC: copyCC, + BCC: copyBCC, Subject: subject, BodyHTML: bodyHTML, BodyPlain: bodyPlain, diff --git a/site/src/layouts/Layout.astro b/site/src/layouts/Layout.astro index 2eb0c59c2..0a08ddd46 100644 --- a/site/src/layouts/Layout.astro +++ b/site/src/layouts/Layout.astro @@ -217,21 +217,27 @@ const websiteJsonLd = { 'ResizeObserver loop completed with undelivered notifications.', 'ResizeObserver loop limit exceeded', ]; + // A link scanner's embedded browser (Outlook Safe Links and the like) + // rejects with this while it walks the page; it is not our code. + var scanner = 'Object Not Found Matching Id:'; + var isNoise = function (v) { + return typeof v === 'string' && (noise.indexOf(v.trim()) !== -1 || v.indexOf(scanner) !== -1); + }; var exceptions = event.properties && event.properties.$exception_list; if (Array.isArray(exceptions)) { for (var i = 0; i < exceptions.length; i++) { var value = exceptions[i] && (exceptions[i].value || exceptions[i].$exception_value); - if (typeof value === 'string' && noise.indexOf(value.trim()) !== -1) return null; + if (isNoise(value)) return null; } } var message = event.properties && event.properties.$exception_message; - if (typeof message === 'string' && noise.indexOf(message.trim()) !== -1) return null; + if (isNoise(message)) return null; // Keep accepting flattened payloads while cached SDK chunks are // still in browsers during a rolling release. var values = event.properties && event.properties.$exception_values; if (!Array.isArray(values)) return event; for (var i = 0; i < values.length; i++) { - if (typeof values[i] === 'string' && noise.indexOf(values[i].trim()) !== -1) return null; + if (isNoise(values[i])) return null; } return event; }, diff --git a/skills/warmbly-api/SKILL.md b/skills/warmbly-api/SKILL.md index b55c0122b..e332010f7 100644 --- a/skills/warmbly-api/SKILL.md +++ b/skills/warmbly-api/SKILL.md @@ -36,7 +36,7 @@ Run `warmblyctl --help` for subcommands and `warmblyctl | Family | Covers | |---|---| | `me` | Identity and granted scopes | -| `campaign` | list, get, create, update, delete, steps, senders, preflight, start, stop, test-email, logs, plan, pause-lead / resume-lead | +| `campaign` | list, get, create, update, delete, steps, senders, preflight, start, stop, test-email, logs, plan, pause-lead / resume-lead, lead-cc / set-lead-cc / lead-cc-suggestions | | `contact` | list (search), get, lookup, create, update, delete, notes, timeline, import, imports, import-status, import-start, import-cancel, export | | `mailbox` | list, get, update, delete, auth-check, sync, skip-folders, identity, refresh-identity, behavior, verify, send, warmup-start/pause/resume/stop/status | | `inbox` | list, count, thread, seen, reply, compose, agent drafts, scheduled sends | @@ -99,6 +99,12 @@ These commands put real mail on the wire: `campaign start`, `campaign resume-lead` lifts it. An out-of-office auto-reply already writes the same hold by itself, across every campaign that contact is a lead of, so a lead reading `paused` for that reason needs nothing from you. +- To reach two people at one company in ONE thread, copy the second on the + first lead's emails: `campaign set-lead-cc --id --contact + --data '{"contact_ids":[""]}'` (at most two; + `campaign lead-cc-suggestions` lists likely colleagues). Do not enrol both as + leads of the same campaign for this: a copied contact's own lead is held + anyway, and resuming that hold sends them a second thread. - If deliverability analytics show rising bounces or complaints, stop the campaign first and report; do not push volume into a degrading mailbox. diff --git a/skills/warmbly-cli/SKILL.md b/skills/warmbly-cli/SKILL.md index bb8cbbb5a..47c9661c7 100644 --- a/skills/warmbly-cli/SKILL.md +++ b/skills/warmbly-cli/SKILL.md @@ -72,7 +72,7 @@ gives the arguments and flags. Ids are positional, not flags. | Command | Covers | |---|---| | `status` | one call for "what is happening": mailboxes needing attention, what is sending, what is unread | -| `campaign` | list, view, create, edit, delete, steps, senders, segments, preflight, test, start, stop, logs, plan, pause-lead / resume-lead | +| `campaign` | list, view, create, edit, delete, steps, senders, segments, preflight, test, start, stop, logs, plan, pause-lead / resume-lead, lead-cc / set-lead-cc / lead-cc-suggestions | | `contact` | list, view, create, edit, delete, lookup, timeline, emails, notes, import, imports, import-status, import-start, import-cancel, export, verify | | `mailbox` | list, view, edit, check, sync, skip-folders, identity, refresh-identity, behavior, warmup, hold, release, send | | `inbox` | list, view, thread, read, reply, compose, drafts, scheduled, snooze | @@ -80,7 +80,7 @@ gives the arguments and flags. Ids are positional, not flags. | `segment`, `template`, `automation`, `form` | audiences, reply templates, automations, lead capture | | `deal`, `pipeline`, `task` | the CRM | | `analytics`, `audit`, `advisor` | numbers, the audit trail, recommendations | -| `placement` | inbox placement tests: overview, test, list, view, cancel, seed inboxes, a campaign's scheduled test | +| `placement` | inbox placement tests: overview, test, list, view, cancel, seed inboxes, a campaign's scheduled test, batches across many mailboxes, fleet coverage | | `webhook`, `key`, `oauth-app`, `integration` | the developer surface | | `org`, `team`, `settings`, `warmup-routing` | the workspace surface a key can reach | | `tool` | the AI tool registry, listed and called | @@ -103,9 +103,9 @@ warmbly api /contacts/search -X POST --input filter.json ## Sending safety, read before anything that sends These put real mail on the wire and prompt before doing so: -`campaign start`, `campaign test`, `placement test`, `mailbox send`, -`inbox reply`, `inbox compose`, `inbox approve-draft`. Everything else is safe -to run freely. +`campaign start`, `campaign test`, `placement test`, `placement batch-start`, +`mailbox send`, `inbox reply`, `inbox compose`, `inbox approve-draft`. +Everything else is safe to run freely. - With no terminal they refuse rather than send. `--yes` is what proceeds, so **only pass `--yes` when the user asked for that specific send.** Never add @@ -130,6 +130,13 @@ to run freely. recipient answers with an out-of-office auto-reply, in every campaign that contact is a lead of, so do not also pause a lead that reads `paused` for that reason. +- To reach two people at one company in ONE thread, copy the second on the + first lead's emails: `warmbly campaign set-lead-cc CAMPAIGN_ID CONTACT_ID + --cc COLLEAGUE_ID` (at most two; `lead-cc-suggestions` lists likely + colleagues). Do not enrol both as leads of the same campaign for this: a + copied contact's own lead is held anyway, and `resume-lead` on that hold + sends them a second thread. Every copy is a recipient who did not ask for + the email, so keep it to small, personal campaigns. - If deliverability shows rising bounces or complaints, stop the campaign and report. Do not push volume into a degrading mailbox. - To check where copy lands before a launch, `warmbly placement test --mailbox @@ -138,6 +145,15 @@ to run freely. TEST_ID` reads the result. One test is a signal, not a verdict: compare a few before rewriting copy, and `--tracking compare` isolates open and click tracking. +- To check a whole fleet, `warmbly placement batch-preview` with the same flags + as `batch-start` states the mailboxes, tests, copies and credits first; show + that to the user before starting. `batch-start --scope campaign + --scope-campaign CAMPAIGN_ID` (or `--scope workspace`, narrowed with + `--only-provider`, `--only-domain`, `--untested-days`, and sampled with + `--sample`) runs one test per mailbox, a few at a time, over hours. + `placement batch BATCH_ID` groups the result by sending domain and provider, + and `batch-senders BATCH_ID` lists mailboxes worst first: a whole domain low + is that domain, one mailbox low on a healthy domain is that mailbox. ## Errors diff --git a/web/src/app/app/contacts/categories/page.tsx b/web/src/app/app/contacts/labels/page.tsx similarity index 91% rename from web/src/app/app/contacts/categories/page.tsx rename to web/src/app/app/contacts/labels/page.tsx index e86052092..516d6361d 100644 --- a/web/src/app/app/contacts/categories/page.tsx +++ b/web/src/app/app/contacts/labels/page.tsx @@ -1,4 +1,4 @@ -// Categories tab: the workspace's contact labels with a live contact count, +// Labels tab: the workspace's contact labels with a live contact count, // inline rename, color, create and delete. Clicking a row opens the contact // list filtered to that category. @@ -31,7 +31,7 @@ import { cn } from "@/lib/utils"; const COLORS = ["#0284c7", "#7c3aed", "#db2777", "#dc2626", "#ea580c", "#ca8a04", "#16a34a", "#0d9488", "#475569"]; -export default function CategoriesPage() { +export default function LabelsPage() { const { user } = useUserProfile(); const write = useWriteGuard("MANAGE_CONTACTS"); const guarded = (fn: () => void) => () => write.guard(fn)({}); @@ -76,14 +76,14 @@ export default function CategoriesPage() { return ( - + } onClick={guarded(() => setCreating(true))}> - New category + New label - - + + @@ -95,7 +95,7 @@ export default function CategoriesPage() { }} className="h-11 px-5 flex items-center gap-2 border-b border-slate-200/60 bg-sky-50/40" > - + + ); + })} + + + {tab === "mailboxes" && } + {tab === "domains" && } + {tab === "providers" && } + {tab === "recipients" && } + + + + + ); +} + +// Overall rates for one copy (or one half of a comparison). +function Overall({ title, counts, className }: { title?: string; counts: PlacementCounts; className?: string }) { + const tabs = counts.promotions + counts.other; + const cells: { label: string; rate: number | null; n: number; tone: string }[] = [ + { label: "Inbox", rate: counts.inbox_rate, n: counts.inbox, tone: FOLDER.inbox.text }, + { label: "Gmail tabs", rate: counts.tabs_rate, n: tabs, tone: FOLDER.promotions.text }, + { label: "Spam", rate: counts.spam_rate, n: counts.spam, tone: FOLDER.spam.text }, + { label: "Never arrived", rate: counts.missing_rate, n: counts.missing, tone: FOLDER.missing.text }, + ]; + return ( +
+ {title &&
{title}
} +
+ {cells.map((c) => ( +
+
{c.label}
+
+ {fmtRate(c.rate)} +
+
+ {c.n.toLocaleString()} cop{c.n === 1 ? "y" : "ies"} +
+
+ ))} +
+ +

+ {counts.total === 0 + ? "No copy has been sent yet." + : `${counts.delivered.toLocaleString()} of ${counts.total.toLocaleString()} copies sent and classified${counts.pending > 0 ? `, ${counts.pending.toLocaleString()} waiting` : ""}. Rates are shares of the copies that were sent.`} +

+
+ ); +} + +const SORTS: { value: PlacementBatchSenderSort; label: string }[] = [ + { value: "worst", label: "Worst inbox rate first" }, + { value: "best", label: "Best inbox rate first" }, + { value: "email", label: "Address" }, + { value: "status", label: "Status" }, +]; + +const STATUS_ORDER: PlacementBatchSenderStatus[] = ["running", "queued", "deferred", "completed", "skipped", "failed", "cancelled"]; + +function SendersTab({ batch }: { batch: PlacementBatchDetail }) { + const navigate = useNavigate(); + const [sort, setSort] = React.useState("worst"); + const [status, setStatus] = React.useState(""); + const [q, setQ] = React.useState(""); + const debounced = useDebouncedValue(q.trim(), 300); + const list = usePlacementBatchSenders(batch.id, { sort, status, q: debounced }); + const p = batch.progress; + + const statusOptions = [ + { value: "", label: `Every status (${p.total.toLocaleString()})` }, + ...STATUS_ORDER.filter((s) => p[s] > 0 || s === status).map((s) => ({ + value: s, + label: `${SENDER_STATUS[s].label} (${p[s].toLocaleString()})`, + })), + ]; + const showResults = list.senders.some((s) => s.summary.total > 0); + + return ( +
+
+
+ +
+
+ setStatus(v as PlacementBatchSenderStatus | "")} + options={statusOptions} + aria-label="Status" + align="end" + /> + setSort(v as PlacementBatchSenderSort)} + options={SORTS} + aria-label="Sort" + align="end" + /> +
+
+ + {list.isLoading ? ( +
+ {Array.from({ length: 5 }).map((_, i) => ( +
+
+
+
+ ))} +
+ ) : list.isError ? ( + + ) : list.senders.length === 0 ? ( +

{debounced || status ? "No mailbox matches that." : "No senders in this batch."}

+ ) : ( +
+ + + + + + {showResults && ( + <> + + + + + )} + + + + {list.senders.map((s) => ( + 0 ? () => navigate(`/app/placement/${s.test_ids[0]}`) : undefined} + /> + ))} + +
MailboxStatusWhere it landedInboxSpam
+ {list.hasNextPage && ( +
+ +
+ )} +
+ )} +
+ ); +} + +function SenderRow({ sender: s, showResults, onOpen }: { sender: PlacementBatchSender; showResults: boolean; onOpen?: () => void }) { + const explained = s.status === "skipped" || s.status === "deferred" || s.status === "failed"; + const reason = s.reason ? (SENDER_REASON[s.reason] ?? s.detail) : s.detail; + const sub = + s.status === "deferred" + ? `${reason ? `${reason}. ` : ""}Retries ${fmtDate(s.next_attempt_at)}` + : explained + ? reason + : undefined; + return ( + { + if (e.key === "Enter" && onOpen) onOpen(); + }} + tabIndex={onOpen ? 0 : undefined} + className={cn("h-12", onOpen && "cursor-pointer hover:bg-slate-50/80 transition-colors outline-none focus-visible:bg-slate-50")} + > + +
+ {s.sender_email || "Deleted mailbox"} + {s.sender_family_label && ( + + {s.sender_family_label} + + )} +
+ + + + {sub && ( +
+ {sub} +
+ )} + + {showResults && ( + <> + {s.summary.total > 0 && } + + {fmtRate(s.summary.inbox_rate)} + + + {s.summary.spam_rate == null ? — : fmtRate(s.summary.spam_rate)} + + + )} + + ); +} + +function GroupTable({ groups, label, empty }: { groups: PlacementBatchGroup[]; label: string; empty: string }) { + if (groups.length === 0) return

{empty}

; + return ( +
+ + + + + + + + + + + + {groups.map((g) => ( + + + + + + + + + ))} + +
{label}Tested + InboxSpamMissing
{g.label || g.key} + {g.tested.toLocaleString()}/{g.senders.toLocaleString()} + + + + {fmtRate(g.counts.inbox_rate)} + {fmtRate(g.counts.spam_rate)} + {fmtRate(g.counts.missing_rate)} +
+
+ ); +} + +// Sending domain by recipient provider: the inbox rate in each cell. +function Matrix({ batch }: { batch: PlacementBatchDetail }) { + const [all, setAll] = React.useState(false); + const cols = batch.recipients; + if (cols.length === 0 || batch.matrix.length === 0) { + return

No copy has a verdict yet.

; + } + const rows = all ? batch.matrix : batch.matrix.slice(0, MATRIX_ROWS); + const cell = (c: PlacementCounts | undefined) => + !c || c.inbox_rate == null ? ( + — + ) : ( + + {fmtRate(c.inbox_rate)} + + ); + return ( +
+

+ Inbox rate for each sending domain at each recipient provider, worst domains first. +

+
+ + + + + {cols.map((c) => ( + + ))} + + + + + + {cols.map((c) => ( + + ))} + + {rows.map((r) => ( + + + {cols.map((c) => ( + + ))} + + ))} + +
Sending domain + {c.label || c.family} +
Every domain + {cell(c.counts)} +
{r.domain} + {cell(r.recipients.find((x) => x.family === c.family)?.counts)} +
+
+ {batch.matrix.length > MATRIX_ROWS && ( +
+ +
+ )} +
+ ); +} + +function ContentCheck({ batch }: { batch: PlacementBatchDetail }) { + const { score } = batch.content; + const issues = batch.content.issues ?? []; + const tone = score >= 80 ? "text-emerald-600" : score >= 50 ? "text-amber-600" : "text-rose-600"; + const label = score >= 80 ? "Looks good" : score >= 50 ? "Could improve" : "Needs work"; + return ( +
+ +
+
+ {score} + out of 100 + {label} +
+ {issues.length === 0 ? ( +

Nothing in the copy stands out to a spam filter.

+ ) : ( +
    + {issues.map((issue, i) => ( + + ))} +
+ )} +

+ The copy is the same for every sender, so a sender or domain that lands worse than the rest points at + reputation, not content. +

+
+
+ ); +} diff --git a/web/src/app/app/placement/page.tsx b/web/src/app/app/placement/page.tsx index 5fecfd0f0..5c6dbf12f 100644 --- a/web/src/app/app/placement/page.tsx +++ b/web/src/app/app/placement/page.tsx @@ -1,20 +1,28 @@ -// Inbox placement tests: the workspace's tests, the seed panels it can test -// against, and its own seed inboxes. Live through PLACEMENT_TEST_UPDATED and +// Inbox placement tests: the workspace's tests and batches, the seed panels it +// can test against, and its own seed inboxes. Live through PLACEMENT_TEST_UPDATED and // the audit spine; nothing here polls. import React from "react"; import { Link, useNavigate, useSearchParams } from "react-router-dom"; import { motion } from "framer-motion"; -import { InboxIcon, Loader2Icon, ListIcon, PlusIcon, XIcon } from "lucide-react"; +import { InboxIcon, Layers3Icon, Loader2Icon, ListIcon, PlusIcon, XIcon } from "lucide-react"; import { EmptyBlock, Page, PageTopbar, SectionBar, TopbarAction } from "@/components/layout/Page"; import ScrollStrip from "@/components/ui/scroll-strip"; import { usePermission, showPermissionDenied } from "@/hooks/usePermission"; import useCampaign from "@/lib/api/hooks/app/campaigns/useCampaign"; -import { usePlacementOverview, usePlacementTests } from "@/lib/api/hooks/app/placement/usePlacement"; -import { PANEL_LABEL, type PlacementTest } from "@/lib/api/models/app/placement/Placement"; +import { + usePlacementBatches, + usePlacementCoverage, + usePlacementOverview, + usePlacementTests, +} from "@/lib/api/hooks/app/placement/usePlacement"; +import { PANEL_LABEL, type PlacementBatch, type PlacementTest } from "@/lib/api/models/app/placement/Placement"; import type { AppError } from "@/lib/api/client/normalizeError"; import buildError from "@/lib/helper/buildError"; import NewPlacementTestDialog from "@/components/app/placement/tests/NewPlacementTestDialog"; +import NewPlacementBatchDialog, { type NewPlacementBatchPrefill } from "@/components/app/placement/batches/NewPlacementBatchDialog"; +import { BatchProgressBar, BatchStatusChip } from "@/components/app/placement/batches/BatchParts"; +import { scopeSummary } from "@/components/app/placement/batches/placementBatches"; import SeedInboxes from "@/components/app/placement/tests/SeedInboxes"; import { PanelStrip, @@ -27,20 +35,25 @@ import { import { ORIGIN_LABEL, fmtDate, fmtRate, rateTone, usageLabel } from "@/components/app/placement/tests/placementTests"; import { cn } from "@/lib/utils"; -type Tab = "tests" | "seeds"; +type Tab = "tests" | "batches" | "seeds"; const TABS: { key: Tab; label: string; icon: typeof ListIcon }[] = [ { key: "tests", label: "Tests", icon: ListIcon }, + { key: "batches", label: "Batches", icon: Layers3Icon }, { key: "seeds", label: "Seed inboxes", icon: InboxIcon }, ]; export default function PlacementPage() { const [params, setParams] = useSearchParams(); - const tab: Tab = params.get("tab") === "seeds" ? "seeds" : "tests"; + const tabParam = params.get("tab"); + const tab: Tab = tabParam === "seeds" || tabParam === "batches" ? tabParam : "tests"; const campaignId = params.get("campaign_id"); const canStart = usePermission("SEND_CAMPAIGNS"); const overview = usePlacementOverview(); + const batches = usePlacementBatches(); const [dialogOpen, setDialogOpen] = React.useState(false); + // Open with this prefill; null is closed. + const [batchDialog, setBatchDialog] = React.useState(null); const setTab = (t: Tab) => { const next = new URLSearchParams(params); @@ -57,6 +70,14 @@ export default function PlacementPage() { setDialogOpen(true); }; + const openBatch = (prefill: NewPlacementBatchPrefill = {}) => { + if (!canStart) { + showPermissionDenied("SEND_CAMPAIGNS"); + return; + } + setBatchDialog(campaignId && !prefill.scope ? { campaignId, ...prefill } : prefill); + }; + const usage = overview.data?.usage; const exhausted = usage?.limit != null && usage.used >= usage.limit; @@ -73,7 +94,18 @@ export default function PlacementPage() { : "Your own seed inboxes are never counted."} )} - } onClick={openNew}> + } + onClick={() => openBatch()} + variant={tab === "batches" ? "primary" : "ghost"} + > + New batch + + } + onClick={openNew} + variant={tab === "batches" ? "ghost" : "primary"} + > New test @@ -97,6 +129,9 @@ export default function PlacementPage() { {t.key === "seeds" && overview.data && ( {overview.data.workspace_seeds} )} + {t.key === "batches" && batches.total != null && batches.total > 0 && ( + {batches.total.toLocaleString()} + )} {active && ( + ) : tab === "batches" ? ( + <> + openBatch({ scope: "workspace", untestedDays: 30 })} /> + openBatch()} /> + ) : ( <> {overview.data && } + openBatch({ scope: "workspace", untestedDays: 30 })} /> { @@ -131,6 +172,7 @@ export default function PlacementPage() { onClose={() => setDialogOpen(false)} prefill={campaignId ? { campaignId } : undefined} /> + setBatchDialog(null)} prefill={batchDialog ?? undefined} /> ); } @@ -277,3 +319,151 @@ function TestRow({ test, onOpen }: { test: PlacementTest; onOpen: () => void }) ); } + + +// How much of the sending fleet has a recent placement result, in one line. +function CoverageLine({ onTest }: { onTest: () => void }) { + const coverage = usePlacementCoverage(); + const c = coverage.data; + if (!c || c.mailboxes === 0) return null; + const untested30 = Math.max(0, c.mailboxes - c.tested_30d); + return ( +
+ + {untested30 === 0 ? ( + `Every one of your ${c.mailboxes.toLocaleString()} mailboxes was tested in the last 30 days.` + ) : ( + <> + Tested in the last 30 days:{" "} + + {c.tested_30d.toLocaleString()} of {c.mailboxes.toLocaleString()} + {" "} + mailboxes + {c.tested_7d > 0 && c.tested_7d < c.tested_30d && ` (${c.tested_7d.toLocaleString()} in the last 7)`} + {c.never_tested > 0 && · {c.never_tested.toLocaleString()} never tested} + + )} + + {untested30 > 0 && ( + + )} +
+ ); +} + +function BatchesTable({ onNew }: { onNew: () => void }) { + const navigate = useNavigate(); + const list = usePlacementBatches(); + + return ( +
+ + + + + {list.isLoading ? ( +
+ {Array.from({ length: 4 }).map((_, i) => ( +
+
+
+
+ ))} +
+ ) : list.isError ? ( + + ) : list.batches.length === 0 ? ( + + + New batch + + } + /> + ) : ( +
+ + + + + + + + + + + + {list.batches.map((b) => ( + navigate(`/app/placement/batches/${b.id}`)} /> + ))} + +
Subject and sendersStatusWhere it landedInboxStarted
+ {list.hasNextPage && ( +
+ +
+ )} +
+ )} + + {list.batches.length > 0 && } +
+ ); +} + +function BatchRow({ batch, onOpen }: { batch: PlacementBatch; onOpen: () => void }) { + const s = batch.summary; + const scope = scopeSummary(batch); + const running = batch.status === "running" || batch.status === "queued"; + return ( + { + if (e.key === "Enter") onOpen(); + }} + tabIndex={0} + className="h-12 cursor-pointer hover:bg-slate-50/80 transition-colors outline-none focus-visible:bg-slate-50" + > + +
{batch.subject || "(no subject)"}
+
+ {batch.sender_count.toLocaleString()} sender{batch.sender_count === 1 ? "" : "s"} + {scope && ` · ${scope}`} +
+ + + + {running && batch.progress.total > 0 && } + + {s.total > 0 && } + {fmtRate(s.inbox_rate)} + + e.stopPropagation()} className="hover:text-slate-700"> + {fmtDate(batch.started_at ?? batch.created_at)} + + + + ); +} diff --git a/web/src/app/app/unibox/page.tsx b/web/src/app/app/unibox/page.tsx index 372047c68..f8f743970 100644 --- a/web/src/app/app/unibox/page.tsx +++ b/web/src/app/app/unibox/page.tsx @@ -213,34 +213,45 @@ export default function UniboxPage() { } }, [urlScope, urlScopeRef]); + // Switching scope closes the open conversation: it may not be in the new + // list, and a reader showing a thread the list does not have reads as a + // misfiled message. const setScope = React.useCallback( (s: UniboxScope) => { switch (s.kind) { case "folder": - goTo({ scope: s.folder, ref: null }); + goTo({ scope: s.folder, ref: null, threadId: null }); return; case "mailbox": - goTo({ scope: "mailbox", ref: s.mailboxId }); + goTo({ scope: "mailbox", ref: s.mailboxId, threadId: null }); return; case "tag": - goTo({ scope: "tag", ref: s.tagId }); + goTo({ scope: "tag", ref: s.tagId, threadId: null }); return; case "category": - goTo({ scope: "category", ref: s.categoryId }); + goTo({ scope: "category", ref: s.categoryId, threadId: null }); return; case "view": - goTo({ scope: "view", ref: s.view }); + goTo({ scope: "view", ref: s.view, threadId: null }); return; case "all": - goTo({ scope: "all", ref: null }); + goTo({ scope: "all", ref: null, threadId: null }); return; default: - goTo({ scope: s.kind, ref: null }); + goTo({ scope: s.kind, ref: null, threadId: null }); } }, [goTo], ); + // The same as Escape on the list: the store moves, the reconciler above + // clears the URL. + const setSelectedAccountId = useAppStore((s) => s.setSelectedAccountId); + const closeThread = React.useCallback(() => { + setSelectedThreadId(null); + setSelectedAccountId(null); + }, [setSelectedThreadId, setSelectedAccountId]); + // ── Scope → server search params ─────────────────────────────── // Derived synchronously (initial state + render-phase reset), NOT in // an effect: an effect runs after paint, so on a reload of a scoped @@ -473,9 +484,10 @@ export default function UniboxPage() { ? undefined : () => { // Widening keeps the query; the reset below reads - // this flag on the scope change it causes. + // this flag on the scope change it causes. All mail + // holds the open conversation, so it stays open. keepSearch.current = true; - setScope({ kind: "all" }); + goTo({ scope: "all", ref: null }); } } onOpenScopeSheet={() => setScopeSheetOpen(true)} @@ -504,18 +516,18 @@ export default function UniboxPage() { <>
{/* Keyed: the list is what has to survive a thread change, the reader is what has to start clean, so a half-typed reply never follows you to the next conversation. */} - +
) : ( diff --git a/web/src/app/app/unibox/uniboxScopeChange.test.tsx b/web/src/app/app/unibox/uniboxScopeChange.test.tsx new file mode 100644 index 000000000..6a4140a9a --- /dev/null +++ b/web/src/app/app/unibox/uniboxScopeChange.test.tsx @@ -0,0 +1,81 @@ +// Issue #741: switching scope carried the open conversation into a list that +// did not have it, with no way to close it on desktop but Escape. + +import React from "react"; +import { describe, it, expect, vi, beforeAll } from "vitest"; +import { screen, act, fireEvent } from "@testing-library/react"; +import { installLayoutShims, mount, settle, SUITE } from "./uniboxHarness"; + +beforeAll(installLayoutShims); + +vi.mock("@/lib/api/client/Request", () => ({ + default: async (cfg: { url?: string }) => { + const { route } = await import("./uniboxHarness"); + return route(String(cfg?.url ?? "")); + }, +})); +vi.mock("@/lib/helper/getToken", () => ({ + default: () => ({ + access_token: "a", + refresh_token: "r", + access_token_expires_at: new Date(Date.now() + 3600e3).toISOString(), + refresh_token_expires_at: new Date(Date.now() + 3600e3).toISOString(), + }), +})); +vi.mock("@/hooks/SocketProvider", () => ({ + default: ({ children }: { children: React.ReactNode }) => <>{children}, +})); +vi.mock("@/hooks/context/socket", async (orig) => { + const actual = (await orig()) as Record; + return { + ...actual, + useSocket: () => ({ + isConnected: false, + subscribeToChannel: () => () => {}, + pushToChannel: () => {}, + socket: null, + status: "closed", + }), + useChannel: () => ({ state: "closed", push: () => {}, channel: null }), + useChannelEvent: () => {}, + useChannelSubscription: () => {}, + }; +}); + +async function openThread(subject: string) { + await act(async () => { + fireEvent.click(screen.getByText(subject).closest('[role="button"]')!); + }); + await settle(); +} + +describe("unibox scope change", SUITE, () => { + it("closes the open conversation when another scope is picked", async () => { + const router = await mount("/app/unibox/all"); + await settle(); + await openThread("Subject 4"); + expect(router.state.location.pathname).toBe("/app/unibox/all/thread-4"); + + await act(async () => { + fireEvent.click(screen.getAllByTitle("Unread")[0]); + }); + await settle(); + + expect(router.state.location.pathname).toBe("/app/unibox/unread"); + expect(screen.queryByText("No conversation open")).toBeTruthy(); + }); + + it("closes the conversation from the reader's own close button", async () => { + const router = await mount("/app/unibox/all"); + await settle(); + await openThread("Subject 2"); + expect(router.state.location.pathname).toBe("/app/unibox/all/thread-2"); + + await act(async () => { + fireEvent.click(screen.getByRole("button", { name: "Close conversation" })); + }); + await settle(); + + expect(router.state.location.pathname).toBe("/app/unibox/all"); + }); +}); diff --git a/web/src/app/app/unibox/uniboxScroll.test.tsx b/web/src/app/app/unibox/uniboxScroll.test.tsx index 2887e3d56..ff051c4c3 100644 --- a/web/src/app/app/unibox/uniboxScroll.test.tsx +++ b/web/src/app/app/unibox/uniboxScroll.test.tsx @@ -131,9 +131,10 @@ describe("unibox scroll position", SUITE, () => { await settle(); scroller().scrollTop = 0; - // The thread pane's back link, the mobile way back to the list. + // The thread pane's back link, the mobile way back to the list. It + // names the scope the list shows. const back = screen - .getAllByRole("button", { name: "Inbox" }) + .getAllByRole("button", { name: "Today" }) .find((b) => b.className.includes("md:hidden"))!; await act(async () => { fireEvent.click(back); diff --git a/web/src/app/auth/login/page.tsx b/web/src/app/auth/login/page.tsx index a0cc6f48e..9cb19c42c 100644 --- a/web/src/app/auth/login/page.tsx +++ b/web/src/app/auth/login/page.tsx @@ -414,6 +414,9 @@ export default function LoginPage() { } else if (e.reason === "timeout") { setPasskeyStatus("timeout"); toast.error("Safari didn't show a passkey prompt. Try again, or use password sign-in."); + } else { + // Aborted: nothing to explain, but the button has to be usable again. + setPasskeyStatus("ready"); } } else { setPasskeyStatus("error"); diff --git a/web/src/components/app/agent/AgentPanel.tsx b/web/src/components/app/agent/AgentPanel.tsx index c01920de4..e747f006d 100644 --- a/web/src/components/app/agent/AgentPanel.tsx +++ b/web/src/components/app/agent/AgentPanel.tsx @@ -1724,8 +1724,8 @@ function toolLabel(tool: string): string { search_contacts: "Searched contacts", get_contact: "Read contact", update_contact_fields: "Update contact", - add_tag: "Add tag", - remove_tag: "Remove tag", + add_tag: "Add label", + remove_tag: "Remove label", list_campaigns: "Listed campaigns", get_campaign_stats: "Campaign stats", create_campaign_draft: "Create campaign draft", diff --git a/web/src/components/app/automations/AutomationFlow.tsx b/web/src/components/app/automations/AutomationFlow.tsx index 5642bdd29..06147c066 100644 --- a/web/src/components/app/automations/AutomationFlow.tsx +++ b/web/src/components/app/automations/AutomationFlow.tsx @@ -1078,7 +1078,7 @@ export default function AutomationFlow({ if (isNativeAction(d.action)) { const need = nativeActionNeeds(d.action); if (need === "tag" && !String(d.config?.category_id ?? "").trim()) { - toast.error("A tag action needs a tag"); + toast.error("A label action needs a label"); setSelectedId(n.id); return false; } @@ -1088,7 +1088,7 @@ export default function AutomationFlow({ return false; } if (need === "label" && !triggerCarriesThread(trigger)) { - toast.error("Label email only runs on a “Reply received” automation"); + toast.error("Label the conversation only runs on a “Reply received” automation"); setSelectedId(n.id); return false; } @@ -2286,8 +2286,8 @@ function ConditionEditor({ // the editor header. Drives both the action dropdown glyphs and the editor // header, so the picker reads like the campaign step picker. const ACTION_VISUAL: Record = { - "warmbly.add_tag": { Icon: TagIcon, tint: "text-emerald-600", bg: "bg-emerald-50", desc: "Add a tag to the contact." }, - "warmbly.remove_tag": { Icon: TagIcon, tint: "text-amber-600", bg: "bg-amber-50", desc: "Remove a tag from the contact." }, + "warmbly.add_tag": { Icon: TagIcon, tint: "text-emerald-600", bg: "bg-emerald-50", desc: "Add a label to the contact." }, + "warmbly.remove_tag": { Icon: TagIcon, tint: "text-amber-600", bg: "bg-amber-50", desc: "Remove a label from the contact." }, "warmbly.create_task": { Icon: CheckSquareIcon, tint: "text-violet-600", bg: "bg-violet-50", desc: "Open a CRM task for the contact." }, "warmbly.create_deal": { Icon: BriefcaseIcon, tint: "text-sky-600", bg: "bg-sky-50", desc: "Create a CRM deal for the contact." }, "warmbly.move_deal_stage": { Icon: BriefcaseIcon, tint: "text-sky-600", bg: "bg-sky-50", desc: "Move the contact's open deal to another stage." }, @@ -2628,11 +2628,11 @@ function NativeActionConfig({
{need === "tag" && (
- + patchConfig({ category_id: ids.length ? ids[ids.length - 1] : "" })} - placeholder="Pick a tag…" + placeholder="Pick a label…" />
)} @@ -2813,7 +2813,7 @@ function NativeActionConfig({ type SetVarRow = { key: string; value: string }; const IF_EXISTS_OPTIONS: SelectOption[] = [ - { value: "update", label: "Update it (fill blanks, add tags and campaign)" }, + { value: "update", label: "Update it (fill blanks, add labels and campaign)" }, { value: "skip", label: "Leave it alone" }, ]; @@ -2915,11 +2915,11 @@ function UpsertContactFields({
- + patchConfig({ category_ids: ids })} - placeholder="Pick tags…" + placeholder="Pick labels…" />
@@ -2942,7 +2942,7 @@ function UpsertContactFields({

A blank value never erases what the contact already has. The written contact becomes this event's contact, so the - steps after it (tag, task, deal) act on it. A campaign picked here keeps running for new leads instead of finishing between runs. + steps after it (label, task, deal) act on it. A campaign picked here keeps running for new leads instead of finishing between runs.

); @@ -3432,7 +3432,7 @@ function AITagPoolField({

{value.length ? "The agent chooses among these for each event." - : "Empty, so the agent may use any of your tags for each event."} + : "Empty, so the agent may use any of your labels for each event."}

); @@ -3465,7 +3465,7 @@ function AIAgentFields({ patchConfig({ instruction: v })} - placeholder="Read the reply. If they ask about pricing, tag them 'pricing' and create a follow-up task." + placeholder="Read the reply. If they ask about pricing, label them 'pricing' and create a follow-up task." />
@@ -3495,10 +3495,10 @@ function AIAgentFields({ patchConfig({ [poolKey]: refs })} @@ -3523,7 +3523,7 @@ function AIAgentFields({ > {!!config.ai_allow_create_tags && } - Let the agent create a new tag/label when none fits + Let the agent create a new label when none fits )}

diff --git a/web/src/components/app/campaigns/new/steps.tsx b/web/src/components/app/campaigns/new/steps.tsx index 313e6dc9c..ad2e78908 100644 --- a/web/src/components/app/campaigns/new/steps.tsx +++ b/web/src/components/app/campaigns/new/steps.tsx @@ -21,7 +21,7 @@ import type Sequence from "@/lib/api/models/app/campaigns/sequences/Sequence"; import type { DraftMeta } from "./serverDraft"; import EmailContentEditor from "@/components/app/campaigns/sequences/EmailContentEditor"; import { useSegments } from "@/lib/api/hooks/app/segments"; -import { CheckSquare } from "@/components/ui/check-square"; +import { Checkbox } from "@/components/ui/checkbox"; import TagSelector from "@/components/app/popup/select/TagSelector"; import ScrollStrip from "@/components/ui/scroll-strip"; import { DateTimePicker } from "@/components/ui/DateTimePicker"; @@ -148,20 +148,14 @@ export function LeadsStep({ {shown.map((l) => { const on = picked.has(l.id); return ( - + ); })} {!q && ( diff --git a/web/src/components/app/campaigns/preferences/CampaignPlacementMonitor.tsx b/web/src/components/app/campaigns/preferences/CampaignPlacementMonitor.tsx index 814c2c554..cba496ca4 100644 --- a/web/src/components/app/campaigns/preferences/CampaignPlacementMonitor.tsx +++ b/web/src/components/app/campaigns/preferences/CampaignPlacementMonitor.tsx @@ -5,7 +5,7 @@ import React from "react"; import { Link } from "react-router-dom"; -import { AlertTriangleIcon, ArrowUpRightIcon, Loader2Icon } from "lucide-react"; +import { AlertTriangleIcon, ArrowUpRightIcon, Layers3Icon, Loader2Icon } from "lucide-react"; import toast from "react-hot-toast"; import { Label, NumberInput } from "@/components/ui/field"; import { SelectMenu } from "@/components/ui/select-menu"; @@ -27,6 +27,7 @@ import { import type { AppError } from "@/lib/api/client/normalizeError"; import buildError from "@/lib/helper/buildError"; import { fmtDate } from "@/components/app/placement/tests/placementTests"; +import NewPlacementBatchDialog from "@/components/app/placement/batches/NewPlacementBatchDialog"; import { SettingRow, Toggle } from "./components/CampaignPreferenceBoolBox"; // Defaults a new monitor starts from, matching the backend's. @@ -40,6 +41,7 @@ export function PlacementMonitorSection({ campaignId }: { campaignId: string }) const remove = useDeletePlacementMonitor(campaignId); const confirm = useConfirm(); const canEdit = usePermission("SEND_CAMPAIGNS"); + const [batchOpen, setBatchOpen] = React.useState(false); const m = monitor.data ?? null; const [intervalDays, setIntervalDays] = React.useState(m?.interval_days ?? DEFAULT_INTERVAL); @@ -233,6 +235,17 @@ export function PlacementMonitorSection({ campaignId }: { campaignId: string })

+ {canEdit && ( + + )} )}
+ + setBatchOpen(false)} prefill={{ campaignId, scope: "campaign" }} /> ); } diff --git a/web/src/components/app/campaigns/sequences/CampaignFlow.tsx b/web/src/components/app/campaigns/sequences/CampaignFlow.tsx index 37500659f..a9d5252b4 100644 --- a/web/src/components/app/campaigns/sequences/CampaignFlow.tsx +++ b/web/src/components/app/campaigns/sequences/CampaignFlow.tsx @@ -549,11 +549,11 @@ function StopNode() { // Per-type chrome for action nodes (icon + label + accent). const ACTION_META: Record = { - add_tag: { label: "Add tag", Icon: TagIcon, tint: "text-emerald-600" }, - remove_tag: { label: "Remove tag", Icon: TagIcon, tint: "text-amber-600" }, + add_tag: { label: "Add label", Icon: TagIcon, tint: "text-emerald-600" }, + remove_tag: { label: "Remove label", Icon: TagIcon, tint: "text-amber-600" }, add_to_segment: { label: "Add to segment", Icon: LayersIcon, tint: "text-emerald-600" }, remove_from_segment: { label: "Remove from segment", Icon: LayersIcon, tint: "text-amber-600" }, - label_email: { label: "Label email", Icon: TagsIcon, tint: "text-fuchsia-600" }, + label_email: { label: "Label conversation", Icon: TagsIcon, tint: "text-fuchsia-600" }, create_task: { label: "Create task", Icon: CheckSquareIcon, tint: "text-violet-600" }, create_deal: { label: "Create deal", Icon: HandshakeIcon, tint: "text-emerald-600" }, move_deal_stage: { label: "Move deal stage", Icon: ArrowRightLeftIcon, tint: "text-sky-600" }, @@ -569,9 +569,9 @@ function actionSummary(a?: SequenceAction | null): string { if (!a) return "Not configured"; switch (a.type) { case "add_tag": - return a.category_id ? "Add a tag" : "Pick a tag…"; + return a.category_id ? "Add a label" : "Pick a label…"; case "remove_tag": - return a.category_id ? "Remove a tag" : "Pick a tag…"; + return a.category_id ? "Remove a label" : "Pick a label…"; case "add_to_segment": return a.segment_id ? "Pin into a segment" : "Pick a segment…"; case "remove_from_segment": @@ -2612,11 +2612,11 @@ function ConnectionEditor({ // bottom dot unconnected (shows "Ends here") or routing a branch to Stop. That // keeps the cleaner Stop/"Ends here" visual instead of a configurable end node. const ADD_ACTION_OPTIONS: { type: SequenceActionType; label: string }[] = [ - { type: "add_tag", label: "Add tag" }, - { type: "remove_tag", label: "Remove tag" }, + { type: "add_tag", label: "Add label" }, + { type: "remove_tag", label: "Remove label" }, { type: "add_to_segment", label: "Add to segment" }, { type: "remove_from_segment", label: "Remove from segment" }, - { type: "label_email", label: "Label email" }, + { type: "label_email", label: "Label conversation" }, { type: "create_task", label: "Create task" }, { type: "create_deal", label: "Create deal" }, { type: "move_deal_stage", label: "Move deal stage" }, @@ -3052,15 +3052,14 @@ function ActionConfigFields({ <> {(action.type === "add_tag" || action.type === "remove_tag") && (
- + setAction((a) => ({ ...a, category_id: ids.length ? ids[ids.length - 1] : null })) } - placeholder="Pick a tag…" + placeholder="Pick a label…" /> -

Tags are your contact categories.

)} @@ -3475,7 +3474,7 @@ function TagPoolField({

{value.length ? "The agent chooses among these for each contact." - : "Empty, so the agent may use any of your tags for each contact."} + : "Empty, so the agent may use any of your labels for each contact."}

); @@ -3512,7 +3511,7 @@ function AIStepFields({ value={action.ai_instruction ?? ""} onChange={(e) => setAction((a) => ({ ...a, ai_instruction: e.target.value }))} rows={3} - placeholder="Read the reply. If they ask about pricing, tag them 'pricing' and create a follow-up task." + placeholder="Read the reply. If they ask about pricing, label them 'pricing' and create a follow-up task." className="w-full resize-y rounded-md border border-slate-200 px-2.5 py-1.5 text-[12.5px] text-slate-700 focus:border-sky-400 focus:outline-none focus:ring-2 focus:ring-sky-100" /> @@ -3549,10 +3548,10 @@ function AIStepFields({ @@ -3578,7 +3577,7 @@ function AIStepFields({ > {action.ai_allow_create_tags && } - Let the agent create a new tag/label when none fits + Let the agent create a new label when none fits )}

@@ -3779,7 +3778,7 @@ function SwitchStepFields({

Every case gets its own dot on the node — drag each dot to the step that path leads to, and the bottom - dot is the “otherwise” fallback for contacts no case matched. Put normal action steps (tag, deal, task…) + dot is the “otherwise” fallback for contacts no case matched. Put normal action steps (label, deal, task…) on a path to make things happen for the contacts routed down it.

diff --git a/web/src/components/app/contacts/AddFromContactsDialog.tsx b/web/src/components/app/contacts/AddFromContactsDialog.tsx index 4864f5c01..be1b06392 100644 --- a/web/src/components/app/contacts/AddFromContactsDialog.tsx +++ b/web/src/components/app/contacts/AddFromContactsDialog.tsx @@ -2,10 +2,10 @@ // // The Leads tab could import a file, sync a sheet, or type a new contact, but // had no way to pull in people already in the workspace. This dialog searches -// the contact list (query + categories), shows who is already a lead, and +// the contact list (query + labels), shows who is already a lead, and // attaches the selection through the bulk contact update (add_campaigns), the // same path the import wizard uses. "Select all matching" hands the server the -// search itself, so a whole category is one request with no cap. +// search itself, so a whole category is one request. import React from "react"; import { AnimatePresence, motion } from "framer-motion"; @@ -32,11 +32,11 @@ import type ContactSelection from "@/lib/api/models/app/contacts/ContactSelectio import * as rowSelection from "./selection"; import type { RowSelection } from "./selection"; -// Backend caps: 100 rows per search page, 1000 contacts per explicit batch. +// Backend caps: 100 rows per search page, 10,000 contacts per explicit batch. // "Select all matching" sends the filter instead, so it is not bound by the // second one. const PAGE = 100; -const MAX_SELECTION = 1000; +const MAX_SELECTION = 10_000; // The target is either a campaign (contacts become leads) or a segment // (contacts are pinned in as manual includes). @@ -234,7 +234,7 @@ export default function AddFromContactsDialog({ open, onClose, campaign: campaig @@ -290,7 +290,7 @@ export default function AddFromContactsDialog({ open, onClose, campaign: campaig

{debounced || categoryIds.length > 0 - ? "Try a different search or category." + ? "Try a different search or label." : "Import a file or add contacts first."}

diff --git a/web/src/components/app/contacts/CategoryPicker.tsx b/web/src/components/app/contacts/CategoryPicker.tsx index b88f95cc7..315587f36 100644 --- a/web/src/components/app/contacts/CategoryPicker.tsx +++ b/web/src/components/app/contacts/CategoryPicker.tsx @@ -44,7 +44,7 @@ interface Props { export default function CategoryPicker({ value, onChange, - placeholder = "Click to add categories…", + placeholder = "Click to add labels…", className, allowCreate = true, }: Props) { @@ -101,7 +101,7 @@ export default function CategoryPicker({ onChange([...value, c.id]); setQuery(""); } catch (err) { - toast.error(errorMessage(err, "Failed to create category")); + toast.error(errorMessage(err, "Failed to create label")); } } @@ -160,7 +160,7 @@ export default function CategoryPicker({
{filtered.length === 0 && !allowCreate && (
- No categories. + No labels.
)} {filtered.map((c) => { diff --git a/web/src/components/app/contacts/ContactsEditBulk.tsx b/web/src/components/app/contacts/ContactsEditBulk.tsx index 4d20313ab..7ec5aeaab 100644 --- a/web/src/components/app/contacts/ContactsEditBulk.tsx +++ b/web/src/components/app/contacts/ContactsEditBulk.tsx @@ -232,7 +232,7 @@ export default function ContactsEditBulk({ -
+
@@ -241,7 +241,7 @@ export default function ContactsEditBulk({ value={categoriesRemove} onChange={setCategoriesRemove} allowCreate={false} - placeholder="Pick categories to strip…" + placeholder="Pick labels to strip…" />
diff --git a/web/src/components/app/contacts/ContactsTable.tsx b/web/src/components/app/contacts/ContactsTable.tsx index cea0d5c7c..425006d02 100644 --- a/web/src/components/app/contacts/ContactsTable.tsx +++ b/web/src/components/app/contacts/ContactsTable.tsx @@ -69,6 +69,7 @@ import type { CampaignLeadCounts } from "@/lib/api/models/app/contacts/SearchCon import ContactsEditBulk from "./ContactsEditBulk"; import PauseLeadDialog from "./PauseLeadDialog"; import { useResumeLead } from "@/lib/api/hooks/app/campaigns/useLeadHold"; +import { CC_RESUME_CONFIRM } from "@/lib/leadHold"; import { selectionOf } from "@/lib/api/models/app/contacts/ContactSelection"; import type ContactSelection from "@/lib/api/models/app/contacts/ContactSelection"; import * as rowSelection from "./selection"; @@ -124,6 +125,8 @@ type SubFilter = "all" | "subscribed" | "unsubscribed"; // Mirrors maxIntegrationPushSize on the backend: one synchronous push is a // live call per contact against the CRM's API. const MAX_CRM_PUSH = 500; +// Mirrors research.MaxBatch: every contact is a metered AI run. +const MAX_RESEARCH_BATCH = 500; export default function ContactsTable({ current_campaign, @@ -498,6 +501,10 @@ export default function ContactsTable({ const metered = useAiMetered(); function bulkResearch() { if (selectionCount === 0) return; + if (selectionCount > MAX_RESEARCH_BATCH) { + toast.error(`Research takes up to ${MAX_RESEARCH_BATCH.toLocaleString()} contacts at a time. Narrow the selection and try again.`); + return; + } confirm?.show( `Research ${selectionCount.toLocaleString()} ${selectionCount === 1 ? "contact" : "contacts"}? ${ metered @@ -549,22 +556,26 @@ export default function ContactsTable({ const [pauseTarget, setPauseTarget] = React.useState<{ id: string; name: string } | null>(null); const resumeLead = useResumeLead(); const resumeOne = React.useCallback( - async (contactId: string) => { + (contactId: string, copied?: boolean) => { if (!current_campaign) return; - try { - await toast.promise( - resumeLead.mutateAsync({ campaignId: current_campaign.id, contactId }), - { - loading: "Resuming lead…", - success: "Lead resumed", - error: (err: AppError) => buildError(err), - }, - ); - } catch { - /* toast.promise already surfaced it */ - } + const run = async () => { + try { + await toast.promise( + resumeLead.mutateAsync({ campaignId: current_campaign.id, contactId }), + { + loading: "Resuming lead…", + success: "Lead resumed", + error: (err: AppError) => buildError(err), + }, + ); + } catch { + /* toast.promise already surfaced it */ + } + }; + if (copied) confirm.show(CC_RESUME_CONFIRM, run); + else void run(); }, - [current_campaign, resumeLead], + [current_campaign, resumeLead, confirm], ); // Leads-view scope chips write straight into the search request, so the @@ -1185,7 +1196,7 @@ function ContactsTableBody({ // member without campaign write access, which takes the control off the // row rather than offering one that fails. onPauseLead?: (id: string, name: string) => void; - onResumeLead?: (id: string) => void; + onResumeLead?: (id: string, copied?: boolean) => void; emptyTitle: string; emptyBody: string; emptyCta: React.ReactNode; @@ -1405,7 +1416,7 @@ function ContactsTableBody({ type="button" aria-label="Resume lead" title={`${holdSummary(lead.hold)}. Resume now`} - onClick={() => onResumeLead(c.id)} + onClick={() => onResumeLead(c.id, lead.hold?.source === "cc")} className="size-6 rounded text-violet-500 hover:text-violet-700 hover:bg-violet-50 flex items-center justify-center transition-colors" > diff --git a/web/src/components/app/contacts/ExportDialog.tsx b/web/src/components/app/contacts/ExportDialog.tsx index 572a0a980..79a69353d 100644 --- a/web/src/components/app/contacts/ExportDialog.tsx +++ b/web/src/components/app/contacts/ExportDialog.tsx @@ -71,7 +71,7 @@ const STANDARD_FIELDS: { id: string; label: string; preset: "basic" | "full" | " { id: "company", label: "Company", preset: "basic" }, { id: "phone", label: "Phone", preset: "basic" }, { id: "subscribed", label: "Subscribed", preset: "basic" }, - { id: "categories", label: "Categories", preset: "full" }, + { id: "categories", label: "Labels", preset: "full" }, { id: "campaigns", label: "Campaigns", preset: "full" }, { id: "created_at", label: "Created at", preset: "full" }, { id: "updated_at", label: "Updated at", preset: "full" }, @@ -89,7 +89,7 @@ const CAMPAIGN_FIELDS: { id: string; label: string }[] = [ const PRESETS: { id: "basic" | "full" | "campaign-ready" | "custom"; label: string; hint: string }[] = [ { id: "basic", label: "Basic", hint: "Core contact details — what most CRMs expect." }, - { id: "full", label: "Full", hint: "Every standard column including categories + campaigns." }, + { id: "full", label: "Full", hint: "Every standard column including labels + campaigns." }, { id: "campaign-ready", label: "Campaign-ready", hint: "Email, names and company, plus lead status and engagement inside a campaign." }, { id: "custom", label: "Custom", hint: "Pick exactly what you need." }, ]; diff --git a/web/src/components/app/contacts/LeadHoldButtons.tsx b/web/src/components/app/contacts/LeadHoldButtons.tsx index 50602601f..96706df95 100644 --- a/web/src/components/app/contacts/LeadHoldButtons.tsx +++ b/web/src/components/app/contacts/LeadHoldButtons.tsx @@ -5,6 +5,7 @@ import { Loader2Icon, PauseIcon, PlayIcon } from "lucide-react"; import toast from "react-hot-toast"; import PauseLeadDialog from "./PauseLeadDialog"; import { useResumeLead } from "@/lib/api/hooks/app/campaigns/useLeadHold"; +import { useConfirm } from "@/hooks/context/confirm"; import type { AppError } from "@/lib/api/client/normalizeError"; import buildError from "@/lib/helper/buildError"; @@ -40,14 +41,18 @@ export function ResumeLeadButton({ label = "Resume", disabled = false, onBusyChange, + confirmText, }: { campaignId: string; contactId: string; label?: string; disabled?: boolean; onBusyChange?: (busy: boolean) => void; + // Asked first when resuming has a consequence worth a second look. + confirmText?: string; }) { const resume = useResumeLead(); + const confirm = useConfirm(); async function run() { onBusyChange?.(true); try { @@ -65,7 +70,7 @@ export function ResumeLeadButton({ return (
- +
diff --git a/web/src/components/app/contacts/SheetSyncWizard.tsx b/web/src/components/app/contacts/SheetSyncWizard.tsx index c16901f53..309147d45 100644 --- a/web/src/components/app/contacts/SheetSyncWizard.tsx +++ b/web/src/components/app/contacts/SheetSyncWizard.tsx @@ -689,10 +689,10 @@ function OptionsStep({

- Apply categories + Apply labels

- Every synced contact gets these categories. Skip to leave them untagged. + Every synced contact gets these labels. Skip to leave them unlabeled.

diff --git a/web/src/components/app/contacts/SyncSourceEditDrawer.tsx b/web/src/components/app/contacts/SyncSourceEditDrawer.tsx index b48f00caa..5a6f77857 100644 --- a/web/src/components/app/contacts/SyncSourceEditDrawer.tsx +++ b/web/src/components/app/contacts/SyncSourceEditDrawer.tsx @@ -223,7 +223,7 @@ export default function SyncSourceEditDrawer({

- Apply categories + Apply labels

diff --git a/web/src/components/app/contacts/columns.tsx b/web/src/components/app/contacts/columns.tsx index 499418ca5..870c4a97c 100644 --- a/web/src/components/app/contacts/columns.tsx +++ b/web/src/components/app/contacts/columns.tsx @@ -156,6 +156,16 @@ const nameColumn: ContactColumn = { {c.email} + {!!c.campaign_lead?.cc?.length && ( + (x.status === "active" ? x.email : `${x.email} (${x.status}, left off)`)) + .join(", ")}`} + > + CC {c.campaign_lead.cc.length} + + )}
diff --git a/web/src/components/app/contacts/contact-edit/ActivityTab.tsx b/web/src/components/app/contacts/contact-edit/ActivityTab.tsx index 47c9059d9..09b15ca96 100644 --- a/web/src/components/app/contacts/contact-edit/ActivityTab.tsx +++ b/web/src/components/app/contacts/contact-edit/ActivityTab.tsx @@ -60,8 +60,9 @@ import { holdSummary } from "@/lib/api/models/app/contacts/Contact"; import type { LeadHold } from "@/lib/api/models/app/contacts/Contact"; import LeadStatusPill from "@/components/app/contacts/LeadStatusPill"; import { usePauseLead } from "@/lib/api/hooks/app/campaigns/useLeadHold"; -import { leadCanBePaused } from "@/lib/leadHold"; +import { CC_RESUME_CONFIRM, leadCanBePaused } from "@/lib/leadHold"; import { PauseLeadButton, ResumeLeadButton } from "@/components/app/contacts/LeadHoldButtons"; +import LeadCCBar from "./LeadCCBar"; import toast from "react-hot-toast"; import type { AppError } from "@/lib/api/client/normalizeError"; import buildError from "@/lib/helper/buildError"; @@ -385,6 +386,17 @@ function CampaignCard({ leadCanBePaused(state) && )} + {/* A lead reached in someone else's thread sends nothing to copy anyone on. */} + {state.hold?.source !== "cc" && ( + + )} + {open && ( @@ -1149,7 +1162,7 @@ function detailsFor(e: ContactTimelineEvent): [string, React.ReactNode][] { : e.email_account_email, ); } - add("Category", e.category_title); + add("Label", e.category_title); add("Intent", e.intent); if (e.type === "deliverability" || e.type === "suppressed") { add("Type", e.source); @@ -1595,9 +1608,9 @@ function visualFor(e: ContactTimelineEvent): { case "campaign_removed": return { Icon: MegaphoneIcon, label: "Removed from campaign" }; case "category_added": - return { Icon: TagIcon, label: "Added to category" }; + return { Icon: TagIcon, label: "Label added" }; case "category_removed": - return { Icon: TagIcon, label: "Removed from category" }; + return { Icon: TagIcon, label: "Label removed" }; case "form_submitted": return { Icon: ClipboardListIcon, label: "Submitted a form" }; case "page_hit": diff --git a/web/src/components/app/contacts/contact-edit/DetailsTab.tsx b/web/src/components/app/contacts/contact-edit/DetailsTab.tsx index fa2622648..28cc3dadb 100644 --- a/web/src/components/app/contacts/contact-edit/DetailsTab.tsx +++ b/web/src/components/app/contacts/contact-edit/DetailsTab.tsx @@ -106,7 +106,7 @@ export default function DetailsTab({
{categoryIds.length} diff --git a/web/src/components/app/contacts/contact-edit/LeadCCBar.tsx b/web/src/components/app/contacts/contact-edit/LeadCCBar.tsx new file mode 100644 index 000000000..f9b4ef2da --- /dev/null +++ b/web/src/components/app/contacts/contact-edit/LeadCCBar.tsx @@ -0,0 +1,280 @@ +// LeadCCBar: the contacts copied on every email one campaign sends this lead, +// so colleagues at one company share a single thread (issue #731). Sits under +// the campaign card's header, next to the hold and pause strips. + +import React from "react"; +import { AnimatePresence, motion } from "framer-motion"; +import { Loader2Icon, PlusIcon, SearchIcon, UsersIcon, XIcon } from "lucide-react"; +import toast from "react-hot-toast"; +import useClickOutside from "@/hooks/useClickOutside"; +import useFlipPlacement from "@/hooks/useFlipPlacement"; +import useDebouncedValue from "@/hooks/useDebouncedValue"; +import { useWriteGuard } from "@/hooks/usePermission"; +import useSearchContacts from "@/lib/api/hooks/app/contacts/useSearchContacts"; +import { useLeadCCSuggestions, useSetLeadCC } from "@/lib/api/hooks/app/campaigns/useLeadCC"; +import { LEAD_CC_MAX, leadCCName, type LeadCC, type LeadCCStatus } from "@/lib/api/models/app/contacts/Contact"; +import type { AppError } from "@/lib/api/client/normalizeError"; +import buildError from "@/lib/helper/buildError"; +import { cn } from "@/lib/utils"; + +const STATUS_NOTE: Record, string> = { + unsubscribed: "Unsubscribed, so left off the next email", + bounced: "Bounced, so left off the next email", + undeliverable: "Failed verification, so left off the next email", +}; + +interface Candidate { + id: string; + email: string; + name: string; + company?: string; + tag?: string; +} + +export default function LeadCCBar({ + campaignId, + contactId, + contactName, + cc, + sending, +}: { + campaignId: string; + contactId: string; + contactName: string; + cc: LeadCC[]; + // The lead still has emails to send; copying anyone on a finished flow does nothing. + sending: boolean; +}) { + const write = useWriteGuard("MANAGE_CAMPAIGNS"); + const setCC = useSetLeadCC(); + const [open, setOpen] = React.useState(false); + + if (cc.length === 0 && (!write.allowed || !sending)) return null; + + async function save(ids: string[], success: string) { + try { + await toast.promise(setCC.mutateAsync({ campaignId, contactId, contactIds: ids }), { + loading: "Saving…", + success, + error: (err: AppError) => buildError(err), + }); + } catch { + /* toast.promise already surfaced it */ + } + } + + const ids = cc.map((c) => c.contact_id); + const canAdd = write.allowed && sending && cc.length < LEAD_CC_MAX; + + return ( +
+ + CC + + {cc.map((c) => { + const note = c.status === "active" ? undefined : STATUS_NOTE[c.status]; + return ( + + {leadCCName(c)} + {write.allowed && ( + + )} + + ); + })} + {canAdd && ( + { + setOpen(false); + void save([...ids, id], "Copied on every email to this lead"); + }} + /> + )} +
+ ); +} + +function CCPicker({ + open, + setOpen, + campaignId, + contactId, + contactName, + exclude, + busy, + empty, + onPick, +}: { + open: boolean; + setOpen: (v: boolean) => void; + campaignId: string; + contactId: string; + contactName: string; + exclude: string[]; + busy: boolean; + empty: boolean; + onPick: (id: string) => void; +}) { + const ref = React.useRef(null); + const triggerRef = React.useRef(null); + const [query, setQuery] = React.useState(""); + useClickOutside(ref, () => setOpen(false)); + const placement = useFlipPlacement(triggerRef, open, 300); + + const q = useDebouncedValue(query.trim(), 250); + const suggestions = useLeadCCSuggestions(campaignId, contactId, open); + const search = useSearchContacts({ + options: { + query: q, + custom_field_filters: [], + campaign_ids: [], + sort_by: "updated_at", + reverse: false, + }, + limit: 8, + enabled: open && q.length > 0, + keepPrevious: true, + }); + + const skip = React.useMemo(() => new Set([contactId, ...exclude]), [contactId, exclude]); + const candidates: Candidate[] = React.useMemo(() => { + if (q.length > 0) { + return (search.contacts ?? []) + .filter((c) => !skip.has(c.id)) + .map((c) => ({ + id: c.id, + email: c.email, + name: `${c.first_name ?? ""} ${c.last_name ?? ""}`.trim() || c.email, + company: c.company, + })); + } + return (suggestions.data?.data ?? []) + .filter((s) => !skip.has(s.contact_id)) + .map((s) => ({ + id: s.contact_id, + email: s.email, + name: leadCCName(s), + company: s.company, + tag: s.reason === "company" ? "Same company" : "Same domain", + })); + }, [q, search.contacts, suggestions.data, skip]); + + const loading = q.length > 0 ? search.isFetching && !search.contacts : suggestions.isLoading; + + return ( +
{ + if (e.key === "Escape" && open) { + e.stopPropagation(); + setOpen(false); + } + }} + > + + + {open && ( + +
+ + setQuery(e.target.value)} + placeholder="Search contacts…" + autoFocus + className="w-full h-5 bg-transparent text-[12px] text-slate-900 placeholder:text-slate-400 outline-none" + /> +
+
+ {loading ? ( +
+ +
+ ) : candidates.length === 0 ? ( +
+ {q.length > 0 ? "No matching contacts." : "No colleagues found. Search for a contact."} +
+ ) : ( + candidates.map((c) => ( + + )) + )} +
+
+ Copied on every email to {contactName} in this campaign, up to {LEAD_CC_MAX}. A reply from + anyone counts as {contactName}'s reply. +
+ + )} + +
+ ); +} diff --git a/web/src/components/app/contacts/contact-edit/OverviewTab.tsx b/web/src/components/app/contacts/contact-edit/OverviewTab.tsx index 94dc25930..6c8c003aa 100644 --- a/web/src/components/app/contacts/contact-edit/OverviewTab.tsx +++ b/web/src/components/app/contacts/contact-edit/OverviewTab.tsx @@ -211,7 +211,7 @@ export default function OverviewTab({ /> 0 ? ( diff --git a/web/src/components/app/contacts/filters/FilterBar.tsx b/web/src/components/app/contacts/filters/FilterBar.tsx index 427d326a5..c987e5fe1 100644 --- a/web/src/components/app/contacts/filters/FilterBar.tsx +++ b/web/src/components/app/contacts/filters/FilterBar.tsx @@ -221,14 +221,14 @@ export default function FilterBar({
setFilters((s) => ({ ...s, category_ids: v.length ? v : undefined }))} options={categoryOptions} - empty="No categories yet." - hint="Contacts must have every selected category." + empty="No labels yet." + hint="Contacts must have every selected label." /> {!hideSegments && (

{dedup === "skip" - ? "Their details stay as they are. They still get the segments, categories and campaigns below." + ? "Their details stay as they are. They still get the segments, labels and campaigns below." : "Empty details are filled in from the file. Nothing they already have is erased."}

@@ -131,7 +131,7 @@ export default function ReviewStep({ placeholder={lockedSegment ? "Add another segment…" : "Pick or create a segment…"} /> - + diff --git a/web/src/components/app/contacts/importShared.ts b/web/src/components/app/contacts/importShared.ts index 8c2992d0a..a397951ca 100644 --- a/web/src/components/app/contacts/importShared.ts +++ b/web/src/components/app/contacts/importShared.ts @@ -20,7 +20,7 @@ export const STANDARD_TARGETS: { id: string; label: string }[] = [ { id: "company", label: "Company" }, { id: "phone", label: "Phone" }, { id: "subscribed", label: "Subscribed" }, - { id: "categories", label: "Categories" }, + { id: "categories", label: "Labels" }, { id: "verification_status", label: "Verification status" }, ]; @@ -44,7 +44,7 @@ export const DEDUP_OPTIONS: { id: ImportDedupStrategy; label: string; hint: stri { id: "skip", label: "Skip existing", - hint: "Leave their details alone. They still join the campaigns, categories and segments you pick.", + hint: "Leave their details alone. They still join the campaigns, labels and segments you pick.", }, { id: "update", @@ -247,7 +247,7 @@ export function fillRate(preview: ImportPreview, idx: number): number | null { // from scratch. export function sampleCSV(): string { return [ - "Email,First name,Last name,Company,Phone,Categories,Job title", + "Email,First name,Last name,Company,Phone,Labels,Job title", "dana@acme.com,Dana,Reyes,Acme,+1 555 0100,Prospects;Q3,Head of Growth", "sam@northwind.io,Sam,Okafor,Northwind,,Prospects,Founder", ].join("\n") + "\n"; diff --git a/web/src/components/app/forms/SettingsPanel.tsx b/web/src/components/app/forms/SettingsPanel.tsx index cc0a197d8..35b68e6a4 100644 --- a/web/src/components/app/forms/SettingsPanel.tsx +++ b/web/src/components/app/forms/SettingsPanel.tsx @@ -86,11 +86,11 @@ export default function SettingsPanel({ description="Where a submitted contact lands in your workspace." >
- + onChange({ category_ids: next })} - placeholder="Pick categories, e.g. Website leads" + placeholder="Pick labels, e.g. Website leads" />

Every submitted contact is filed under these.

diff --git a/web/src/components/app/placement/batches/BatchParts.tsx b/web/src/components/app/placement/batches/BatchParts.tsx new file mode 100644 index 000000000..2ea7e5174 --- /dev/null +++ b/web/src/components/app/placement/batches/BatchParts.tsx @@ -0,0 +1,40 @@ +// Small presentational pieces shared by the batch list and the batch page. +import { DitherStack } from "@/components/ui/dither"; +import { cn } from "@/lib/utils"; +import type { + PlacementBatchProgress, + PlacementBatchSenderStatus, + PlacementBatchStatus, +} from "@/lib/api/models/app/placement/Placement"; +import { BATCH_STATUS, SENDER_STATUS, doneCount, progressLine, progressSegments } from "./placementBatches"; + +export function BatchStatusChip({ status, progress }: { status: PlacementBatchStatus; progress?: PlacementBatchProgress }) { + const s = BATCH_STATUS[status] ?? BATCH_STATUS.failed; + const counter = status === "running" && progress ? ` ${doneCount(progress).toLocaleString()}/${progress.total.toLocaleString()}` : ""; + return ( + + + {s.label} + {counter && {counter}} + + ); +} + +export function SenderStatusChip({ status }: { status: PlacementBatchSenderStatus }) { + const s = SENDER_STATUS[status] ?? SENDER_STATUS.failed; + return ( + + + {s.label} + + ); +} + +// Senders done or sending as shares of the whole batch; the rest is the empty track. +export function BatchProgressBar({ progress, height = 6, className }: { progress: PlacementBatchProgress; height?: number; className?: string }) { + return ( +
+ +
+ ); +} diff --git a/web/src/components/app/placement/batches/NewPlacementBatchDialog.tsx b/web/src/components/app/placement/batches/NewPlacementBatchDialog.tsx new file mode 100644 index 000000000..ce5f4fd84 --- /dev/null +++ b/web/src/components/app/placement/batches/NewPlacementBatchDialog.tsx @@ -0,0 +1,1366 @@ +// Starts a placement batch: the same placement test from many sending +// mailboxes, a few at a time, so the results can be read by domain and +// provider. Three steps (Senders, Email, Review); every count comes from the +// server's preview of the same request, so nothing here guesses. + +import React from "react"; +import { createPortal } from "react-dom"; +import { AnimatePresence, motion } from "framer-motion"; +import { Link, useNavigate } from "react-router-dom"; +import { + AlertCircleIcon, + AlertTriangleIcon, + CheckIcon, + ChevronLeftIcon, + ChevronRightIcon, + Layers3Icon, + Loader2Icon, + PlayIcon, + XIcon, +} from "lucide-react"; +import toast from "react-hot-toast"; +import { Label, NumberInput, SearchInput } from "@/components/ui/field"; +import { Checkbox } from "@/components/ui/checkbox"; +import { OptionSelect, Toggle } from "@/components/app/campaigns/preferences/components/CampaignPreferenceBoolBox"; +import TagSelector from "@/components/app/popup/select/TagSelector"; +import { useConfirm } from "@/hooks/context/confirm"; +import { usePermission } from "@/hooks/usePermission"; +import useDebouncedValue from "@/hooks/useDebouncedValue"; +import useCampaign from "@/lib/api/hooks/app/campaigns/useCampaign"; +import { + useCreatePlacementBatch, + usePlacementBatchPreview, + usePlacementOverview, + usePlacementSeeds, +} from "@/lib/api/hooks/app/placement/usePlacement"; +import { + PANEL_LABEL, + type PlacementBatchPreview, + type PlacementBatchRequest, + type PlacementPanel, + type PlacementSample, + type PlacementSampleMode, + type PlacementSenderScope, + type PlacementTracking, + type PlacementUnavailable, + type PlacementWorkspaceSeed, +} from "@/lib/api/models/app/placement/Placement"; +import { domainOf } from "@/lib/api/models/app/emails/MailboxSources"; +import type { AppError } from "@/lib/api/client/normalizeError"; +import { cn } from "@/lib/utils"; +import { + CampaignPicker, + CopySourceFields, + FamilyChips, + InlineError, + PanelChoice, + SectionLabel, + TrackingChoice, +} from "../tests/PlacementFormParts"; +import { + copyBody, + copyIssue, + newIdempotencyKey, + useCampaignEmailSteps, + type CopyDraft, + type CopySource, +} from "../tests/placementCopy"; +import SeedChooser from "../tests/SeedChooser"; +import { BATCH_RETRY_DAYS, batchErrorMessage, fmtDuration, type BatchErrorField } from "./placementBatches"; + +type Scope = "mailboxes" | "campaign" | "workspace"; +type StepKey = "senders" | "email" | "review"; + +const STEPS: { key: StepKey; label: string }[] = [ + { key: "senders", label: "Senders" }, + { key: "email", label: "Email" }, + { key: "review", label: "Review" }, +]; + +// Rows drawn in the mailbox list at once; search narrows the rest. +const MAILBOX_ROWS_SHOWN = 300; + +interface Draft extends CopyDraft { + scope: Scope; + senderIds: string[]; + scopeCampaignId: string; + providers: string[]; + domains: string[]; + tagIds: string[]; + untested: boolean; + untestedDays: number; + includeInactive: boolean; + sampleMode: PlacementSampleMode; + sampleCount: number; + samplePercent: number; + spread: boolean; + tracking: PlacementTracking; + panel: PlacementPanel; + seedIds: string[]; + families: string[]; + onUnavailable: PlacementUnavailable; + // The credit total the user agreed to; a different total needs asking again. + payConsent: number | null; +} + +export interface NewPlacementBatchPrefill { + campaignId?: string; + scope?: Scope; + untestedDays?: number; + senderIds?: string[]; +} + +function emptyDraft(prefill?: NewPlacementBatchPrefill): Draft { + const scope: Scope = + prefill?.scope ?? (prefill?.senderIds?.length ? "mailboxes" : prefill?.campaignId ? "campaign" : "workspace"); + const fromStep = !!prefill?.campaignId; + return { + source: fromStep ? "step" : "custom", + campaignId: prefill?.campaignId ?? "", + stepId: "", + subject: "", + bodyHtml: "", + bodyPlain: "", + bodyCode: false, + contact: null, + scope, + senderIds: prefill?.senderIds ?? [], + scopeCampaignId: prefill?.campaignId ?? "", + providers: [], + domains: [], + tagIds: [], + untested: !!prefill?.untestedDays, + untestedDays: prefill?.untestedDays ?? 30, + includeInactive: false, + sampleMode: "all", + sampleCount: 50, + samplePercent: 10, + spread: true, + tracking: fromStep ? "campaign" : "off", + panel: "instance", + seedIds: [], + families: [], + onUnavailable: "defer", + payConsent: null, + }; +} + +// What the user typed or picked, for the discard prompt. Defaults the dialog +// fills in itself (the panel, the first step) do not count. +function draftKey(d: Draft): string { + return JSON.stringify({ ...d, panel: "", stepId: "", payConsent: null, contact: d.contact?.id ?? "" }); +} + +function sampleOf(d: Draft): PlacementSample { + const stratify = d.spread ? { stratify: "provider" as const } : {}; + switch (d.sampleMode) { + case "random": + return { mode: "random", count: d.sampleCount, ...stratify }; + case "percent": + return { mode: "percent", percent: d.samplePercent, ...stratify }; + case "per_domain": + case "per_provider": + return { mode: d.sampleMode, count: d.sampleCount }; + default: + return { mode: "all" }; + } +} + +// The sender half of a request, or null while it names nobody. +function selectionBody(d: Draft, opts: { base?: boolean } = {}): PlacementBatchRequest | null { + if (d.scope === "mailboxes") return d.senderIds.length > 0 ? { sender_account_ids: d.senderIds, sample: { mode: "all" } } : null; + if (d.scope === "campaign" && !d.scopeCampaignId) return null; + const scope: PlacementSenderScope = { + type: d.scope, + ...(d.scope === "campaign" ? { campaign_id: d.scopeCampaignId } : {}), + ...(!opts.base && d.providers.length > 0 ? { providers: d.providers } : {}), + ...(d.domains.length > 0 ? { domains: d.domains } : {}), + ...(d.tagIds.length > 0 ? { tag_ids: d.tagIds } : {}), + ...(d.includeInactive ? { include_inactive: true } : {}), + ...(d.untested && d.untestedDays > 0 ? { untested_days: d.untestedDays } : {}), + }; + return { sender_scope: scope, sample: opts.base ? { mode: "all" } : sampleOf(d) }; +} + +// Holds a request body until it has been stable for a moment. +function useSettledBody(body: PlacementBatchRequest | null) { + const key = body ? JSON.stringify(body) : ""; + const debounced = useDebouncedValue(key, 400); + const settledBody = React.useMemo(() => (debounced ? (JSON.parse(debounced) as PlacementBatchRequest) : null), [debounced]); + return { body: settledBody, settled: debounced === key }; +} + +function usePreview(body: PlacementBatchRequest | null) { + const settled = useSettledBody(body); + const q = usePlacementBatchPreview(settled.body); + const error = q.isError ? batchErrorMessage(q.error as unknown as AppError) : null; + const ready = !!body && settled.settled && !!q.data && !q.isPlaceholderData && !q.isError; + return { data: ready ? q.data : undefined, stale: q.data, error, loading: !!body && !ready && !error }; +} + +const n = (v: number) => v.toLocaleString(); +const plural = (v: number, one: string, many = `${one}s`) => `${n(v)} ${v === 1 ? one : many}`; + +export default function NewPlacementBatchDialog({ + open, + onClose, + prefill, +}: { + open: boolean; + onClose: () => void; + prefill?: NewPlacementBatchPrefill; +}) { + if (typeof document === "undefined") return null; + return createPortal( + {open && }, + document.body, + ); +} + +function DialogBody({ onClose, prefill }: { onClose: () => void; prefill?: NewPlacementBatchPrefill }) { + const navigate = useNavigate(); + const confirm = useConfirm(); + const overview = usePlacementOverview(); + const seeds = usePlacementSeeds(); + const create = useCreatePlacementBatch(); + const canBuy = usePermission("MANAGE_BILLING"); + + const [draft, setDraft] = React.useState(() => emptyDraft(prefill)); + const initialKey = React.useRef(draftKey(emptyDraft(prefill))); + const [step, setStep] = React.useState(0); + const [direction, setDirection] = React.useState<1 | -1>(1); + const [nudged, setNudged] = React.useState(false); + const [error, setError] = React.useState<{ field: BatchErrorField; message: string } | null>(null); + + // A retried submit of the same draft lands on the batch the first one + // queued; any edit makes it a new request and clears the last refusal. + const idemKey = React.useRef(newIdempotencyKey()); + const patch = React.useCallback((p: Partial) => { + idemKey.current = newIdempotencyKey(); + setError(null); + setDraft((d) => ({ ...d, ...p })); + }, []); + + // A campaign picked as the senders is also the copy, until the copy is chosen by hand. + const setScopeCampaign = (id: string) => { + idemKey.current = newIdempotencyKey(); + setError(null); + setDraft((d) => { + const copyUntouched = + d.source === "custom" + ? !d.subject.trim() && !d.bodyPlain.trim() && !d.bodyHtml.trim() + : !d.campaignId || d.campaignId === d.scopeCampaignId; + if (!copyUntouched) return { ...d, scopeCampaignId: id }; + return { + ...d, + scopeCampaignId: id, + source: "step", + campaignId: id, + stepId: "", + contact: null, + tracking: d.source === "custom" ? "campaign" : d.tracking, + }; + }); + }; + + const scopeCampaign = useCampaign(draft.scope === "campaign" ? draft.scopeCampaignId : ""); + const campaign = useCampaign(draft.source === "step" ? draft.campaignId : ""); + const steps = useCampaignEmailSteps(draft.campaignId, draft.source === "step"); + + // Default the step to the campaign's first email step. + React.useEffect(() => { + if (draft.source !== "step" || !draft.campaignId || steps.emailSteps.length === 0) return; + if (steps.emailSteps.some((s) => s.id === draft.stepId)) return; + setDraft((d) => ({ ...d, stepId: steps.emailSteps[0].id })); + }, [draft.source, draft.campaignId, draft.stepId, steps.emailSteps]); + + // Default the panel to the first one that can run a test. + const panels = React.useMemo(() => overview.data?.panels ?? [], [overview.data]); + const panel = panels.find((p) => p.panel === draft.panel); + React.useEffect(() => { + if (panels.length === 0 || panel?.available) return; + const first = panels.find((p) => p.available); + if (first) setDraft((d) => ({ ...d, panel: first.panel })); + }, [panels, panel?.available]); + + const textOnly = draft.source === "step" && !!campaign.data?.text_only; + React.useEffect(() => { + if (textOnly && (draft.tracking === "on" || draft.tracking === "compare")) { + setDraft((d) => ({ ...d, tracking: "campaign" })); + } + }, [textOnly, draft.tracking]); + + const mailboxes = React.useMemo(() => (seeds.data ?? []).filter((m) => !m.seed), [seeds.data]); + const ownSeeds = React.useMemo(() => (seeds.data ?? []).filter((m) => m.seed), [seeds.data]); + const chosenSeeds = draft.panel === "workspace" ? ownSeeds.filter((m) => draft.seedIds.includes(m.email_account_id) && m.status === "active") : []; + const panelFamilies = React.useMemo(() => panel?.families ?? [], [panel]); + const chosenFamilies = draft.panel === "workspace" ? [] : draft.families.filter((f) => panelFamilies.some((p) => p.family === f)); + const familySeeds = chosenFamilies.length + ? panelFamilies.filter((p) => chosenFamilies.includes(p.family)).reduce((acc, p) => acc + p.seeds, 0) + : (panel?.seeds ?? 0); + + // Previews: the selection alone, the scope before the provider filter and + // sample (for the provider chips), and the whole request for the review. + const selection = selectionBody(draft); + const senders = usePreview(selection); + const baseSelection = draft.scope === "mailboxes" ? null : selectionBody(draft, { base: true }); + const base = usePreview(baseSelection); + + const emailIssue: string | null = + copyIssue(draft, steps) ?? + (!panel || !panel.available + ? "Pick a panel that can run a test." + : panel.seeds === 0 + ? panel.panel === "workspace" + ? "You have no seed inboxes yet. Mark one on the Seed inboxes tab of Placement tests." + : "This panel has no seed inboxes yet." + : draft.panel === "workspace" && draft.seedIds.length > 0 && chosenSeeds.length === 0 + ? "None of the seed inboxes you chose are connected. Choose others, or clear the choice." + : chosenFamilies.length > 0 && familySeeds === 0 + ? "This panel has no seeds at the providers you chose." + : null); + + const fullBody: PlacementBatchRequest | null = + selection && !emailIssue + ? { + ...selection, + ...copyBody(draft), + tracking: draft.tracking, + panel: draft.panel, + ...(chosenFamilies.length > 0 ? { families: chosenFamilies } : {}), + ...(chosenSeeds.length > 0 ? { seed_ids: chosenSeeds.map((m) => m.email_account_id) } : {}), + on_unavailable: draft.onUnavailable, + } + : null; + const full = usePreview(fullBody); + + const tooLarge = (p: PlacementBatchPreview) => + p.selected > p.senders_max + ? `This selection has ${n(p.selected)} senders and a batch can hold ${n(p.senders_max)}. Narrow the filters or take a sample.` + : null; + const emptyIssue = (p: PlacementBatchPreview) => + p.selected === 0 + ? p.matched === 0 + ? "No sending mailbox matches this selection." + : "This sample comes to no mailboxes. Raise the number." + : null; + + const sendersIssue: string | null = + draft.scope === "mailboxes" && draft.senderIds.length === 0 + ? "Choose at least one mailbox." + : draft.scope === "campaign" && !draft.scopeCampaignId + ? "Pick a campaign." + : senders.error + ? senders.error.message + : !senders.data + ? "Counting the mailboxes…" + : (emptyIssue(senders.data) ?? tooLarge(senders.data)); + + const fp = full.data; + const consented = !!fp && fp.paid_tests > 0 && draft.payConsent === fp.credits; + const canPay = !fp || fp.paid_tests === 0 || (fp.usage.credits_per_test > 0 && (fp.usage.credit_balance == null || fp.usage.credit_balance >= fp.credits)); + const reviewIssue: string | null = full.error + ? full.error.message + : !fp + ? "Working out the cost…" + : (emptyIssue(fp) ?? + tooLarge(fp) ?? + (fp.paid_tests > 0 && fp.usage.credits_per_test === 0 + ? `This month's free tests cover ${n(fp.free_tests)} of the ${n(fp.tests)} tests, and tests past them cannot be paid for. Take a smaller sample, or use your own seed inboxes.` + : fp.paid_tests > 0 && !canPay + ? `The paid tests cost up to ${n(fp.credits)} credits and the workspace has ${n(fp.usage.credit_balance ?? 0)}. Top up under Settings > Billing, or take a smaller sample.` + : fp.paid_tests > 0 && !consented + ? `Tick the box to pay up to ${n(fp.credits)} credits for this batch.` + : null)); + + const issueOf = React.useCallback( + (key: StepKey) => (key === "senders" ? sendersIssue : key === "email" ? emailIssue : reviewIssue), + [sendersIssue, emailIssue, reviewIssue], + ); + const current = STEPS[step]; + const issue = issueOf(current.key); + React.useEffect(() => { + if (!issue) setNudged(false); + }, [issue]); + + // A step is reachable when every step before it is complete. + const canReach = React.useCallback( + (target: number) => { + for (let i = 0; i < target; i++) if (issueOf(STEPS[i].key)) return false; + return true; + }, + [issueOf], + ); + const goTo = React.useCallback( + (target: number, force = false) => { + if (target === step) return; + if (!force && target > step && !canReach(target)) { + setNudged(true); + return; + } + setDirection(target > step ? 1 : -1); + setNudged(false); + setStep(target); + }, + [step, canReach], + ); + + const dirty = draftKey(draft) !== initialKey.current; + const pending = create.isPending; + + const requestClose = React.useCallback(() => { + if (pending) return; + if (dirty) { + confirm.show("Discard this placement batch?", async () => onClose()); + return; + } + onClose(); + }, [pending, dirty, confirm, onClose]); + + React.useEffect(() => { + const onKey = (e: KeyboardEvent) => { + if (e.key !== "Escape") return; + // An open picker or the discard confirm owns this Escape. + if (document.querySelector("[data-floating], [role='alertdialog']")) return; + e.preventDefault(); + requestClose(); + }; + document.addEventListener("keydown", onKey); + return () => document.removeEventListener("keydown", onKey); + }, [requestClose]); + + const next = () => { + if (issue) { + setNudged(true); + return; + } + goTo(step + 1); + }; + + async function submit() { + if (pending) return; + if (issue || !fullBody || !fp) { + setNudged(true); + return; + } + const body: PlacementBatchRequest = { ...fullBody, ...(fp.paid_tests > 0 ? { max_credits: fp.credits } : {}) }; + try { + const batch = await create.mutateAsync({ body, idempotencyKey: idemKey.current }); + toast.success("Placement batch queued."); + onClose(); + navigate(`/app/placement/batches/${batch.id}`); + } catch (err) { + const e = batchErrorMessage(err as AppError, { resetsOn: fp.usage.period_end, panel: draft.panel }); + setError(e); + const at = STEPS.findIndex((s) => s.key === e.field); + if (at >= 0 && at < step) goTo(at, true); + } + } + + const fieldError = (f: BatchErrorField) => (error?.field === f ? : null); + const lastStep = STEPS.length - 1; + // "Counting…" is a wait, not a mistake, so it never reads as a warning. + const waiting = issue === "Counting the mailboxes…" || issue === "Working out the cost…"; + + return ( + + e.stopPropagation()} + className="w-full max-w-[720px] rounded-lg bg-white border border-slate-200 shadow-[0_24px_48px_-12px_rgba(15,23,42,0.18),0_8px_16px_-8px_rgba(15,23,42,0.1)] overflow-hidden flex flex-col h-[min(88dvh,820px)]" + > +
+
+ +
+ New +
+ Placement batch + +
+ + + +
+ + + {current.key === "senders" && ( + + )} + {current.key === "email" && ( + <> + + patch({ + source: v, + ...(v === "step" && !draft.campaignId && draft.scopeCampaignId ? { campaignId: draft.scopeCampaignId } : {}), + tracking: v === "step" ? "campaign" : draft.tracking === "campaign" ? "off" : draft.tracking, + }) + } + campaignName={campaign.data?.name} + steps={steps} + /> + patch({ tracking: v })} + source={draft.source} + textOnly={textOnly} + compareHint="Every sender runs two tests to the same seeds, so tests, copies and credits double." + /> +
+ Seed panel + patch({ panel: p })} + usage={overview.data?.usage} + /> + {draft.panel !== "workspace" && panel?.available && panelFamilies.length > 1 && ( + patch({ families })} /> + )} + {draft.panel === "workspace" && panel?.available && ownSeeds.length > 0 && ( + <> + patch({ seedIds })} + /> +

+ A seed on a sender's own domain is skipped for that sender. +

+ + )} + {fieldError("email")} +
+ + )} + {current.key === "review" && ( + goTo(STEPS.findIndex((s) => s.key === k))} + error={fieldError("review")} + /> + )} + + +
+ +
+ {step > 0 ? ( + + ) : ( + + )} + +
+ {error?.field === "general" ? ( + + + + ) : ( + issue && ( + + {waiting ? : } + + {issue} + + + ) + )} + {step < lastStep ? ( + + ) : ( + + )} +
+
+ + + ); +} + +const paneVariants = { + enter: (dir: 1 | -1) => ({ x: dir * 28, opacity: 0 }), + center: { x: 0, opacity: 1 }, + exit: (dir: 1 | -1) => ({ x: dir * -28, opacity: 0 }), +}; + +function stepName(steps: { id: string; name?: string; subject?: string }[], id: string): string { + const i = steps.findIndex((s) => s.id === id); + if (i < 0) return "step"; + return steps[i].name || `step ${i + 1}`; +} + +function scopeLine(d: Draft, campaignName?: string): string { + const parts: string[] = []; + if (d.scope === "mailboxes") return plural(d.senderIds.length, "chosen mailbox", "chosen mailboxes"); + parts.push(d.scope === "campaign" ? `The senders of ${campaignName ?? "the campaign"}` : "Every sending mailbox"); + if (d.providers.length > 0) parts.push(plural(d.providers.length, "provider")); + if (d.domains.length > 0) parts.push(d.domains.length === 1 ? d.domains[0] : plural(d.domains.length, "domain")); + if (d.tagIds.length > 0) parts.push(plural(d.tagIds.length, "tag")); + if (d.untested && d.untestedDays > 0) parts.push(`not tested in ${d.untestedDays} days`); + if (d.includeInactive) parts.push("disconnected included"); + switch (d.sampleMode) { + case "random": + parts.push(`random ${n(d.sampleCount)}`); + break; + case "percent": + parts.push(`${d.samplePercent}% sample`); + break; + case "per_domain": + parts.push(`${d.sampleCount} per domain`); + break; + case "per_provider": + parts.push(`${d.sampleCount} per provider`); + break; + } + return parts.join(" · "); +} + +function Stepper({ + step, + canReach, + goTo, + issueOf, +}: { + step: number; + canReach: (s: number) => boolean; + goTo: (s: number) => void; + issueOf: (k: StepKey) => string | null; +}) { + return ( +
+ {STEPS.map((s, i) => { + const active = i === step; + const done = i < step && !issueOf(s.key); + const reachable = i <= step || canReach(i); + return ( + + + {i < STEPS.length - 1 && ( + + + + )} + + ); + })} +
+ ); +} + +// ---- Senders + +const SAMPLE_MODES: { value: PlacementSampleMode; label: string }[] = [ + { value: "all", label: "All" }, + { value: "random", label: "Random" }, + { value: "percent", label: "Percent" }, + { value: "per_domain", label: "Per domain" }, + { value: "per_provider", label: "Per provider" }, +]; + +function chipClass(active: boolean) { + return cn( + "h-6 px-2 rounded-md border text-[11px] font-medium inline-flex items-center gap-1 transition-colors", + active ? "border-sky-200 bg-sky-50 text-sky-700" : "border-slate-200 bg-white text-slate-600 hover:border-slate-300", + ); +} + +function SendersStep({ + draft, + patch, + setScopeCampaign, + scopeCampaignName, + mailboxes, + mailboxesLoading, + preview, + base, + error, +}: { + draft: Draft; + patch: (p: Partial) => void; + setScopeCampaign: (id: string) => void; + scopeCampaignName?: string; + mailboxes: PlacementWorkspaceSeed[]; + mailboxesLoading: boolean; + preview: ReturnType; + base?: PlacementBatchPreview; + error: React.ReactNode; +}) { + const domainSuggestions = React.useMemo(() => { + const counts = new Map(); + for (const m of mailboxes) { + const d = domainOf(m.email); + if (d) counts.set(d, (counts.get(d) ?? 0) + 1); + } + return [...counts.entries()].sort((a, b) => b[1] - a[1]).map(([d]) => d); + }, [mailboxes]); + + // The providers the scope has, plus any chosen that dropped out of it. + const providerChips = React.useMemo(() => { + const list = (base?.providers ?? []).map((p) => ({ key: p.key, label: p.label, senders: p.senders as number | null })); + for (const key of draft.providers) { + if (!list.some((p) => p.key === key)) list.push({ key, label: key, senders: null }); + } + return list; + }, [base?.providers, draft.providers]); + + return ( + <> +
+ Send from + + value={draft.scope} + onChange={(v) => patch({ scope: v })} + cols={3} + aria-label="Senders" + options={[ + { value: "mailboxes", label: "Choose mailboxes", hint: "Pick them from a list." }, + { value: "campaign", label: "A campaign's senders", hint: "The mailboxes a campaign sends from." }, + { value: "workspace", label: "All mailboxes", hint: "Every sending mailbox in the workspace." }, + ]} + /> +
+ + {draft.scope === "mailboxes" && ( + patch({ senderIds })} + /> + )} + + {draft.scope === "campaign" && ( +
+ + +
+ )} + + {draft.scope !== "mailboxes" && ( +
+ Filters +
+ Providers +
+ + {providerChips.map((p) => { + const active = draft.providers.includes(p.key); + return ( + + ); + })} +
+
+
+ Sending domains + patch({ domains })} suggestions={domainSuggestions} /> +
+
+ Mailboxes with any of these tags + patch({ tagIds: [...draft.tagIds, id] })} + onRemove={(id) => patch({ tagIds: draft.tagIds.filter((t) => t !== id) })} + /> +
+
+ + patch({ untestedDays: v, untested: true })} + suffix="days" + className="w-28" + /> +
+
+
+
Include disconnected mailboxes
+
+ They wait for a reconnect when retrying, or are skipped. +
+
+ patch({ includeInactive: v })} + ariaLabel="Include disconnected mailboxes" + /> +
+
+ )} + + {draft.scope !== "mailboxes" && ( +
+ Sample +
+ {SAMPLE_MODES.map((m) => ( + + ))} +
+ {draft.sampleMode !== "all" && ( +
+ {draft.sampleMode === "percent" ? ( + patch({ samplePercent: v })} + suffix="% of the mailboxes" + className="w-48" + /> + ) : ( + patch({ sampleCount: v })} + suffix={ + draft.sampleMode === "per_domain" + ? "per domain" + : draft.sampleMode === "per_provider" + ? "per provider" + : "mailboxes" + } + className="w-44" + /> + )} + {(draft.sampleMode === "random" || draft.sampleMode === "percent") && ( + + )} +
+ )} +
+ )} + + + {error} + + ); +} + +function PreviewSummary({ preview }: { preview: ReturnType }) { + const p = preview.data ?? preview.stale; + if (preview.error) return ; + if (!p) { + if (!preview.loading) return null; + return ( +
+ + Counting the mailboxes… +
+ ); + } + const spread: string[] = []; + if (p.providers.length > 1) spread.push(plural(p.providers.length, "provider")); + if (p.domains > 1) spread.push(plural(p.domains, "domain")); + const across = spread.length > 0 ? ` across ${spread.join(" and ")}` : ""; + const head = + p.matched === p.selected + ? `${plural(p.selected, "mailbox", "mailboxes")}${across}` + : `${n(p.matched)} match · ${n(p.selected)} selected${across}`; + return ( +
+
+ {preview.loading && } + {head} + {p.inactive > 0 && · {n(p.inactive)} not connected} +
+ {p.providers.length > 0 && ( +
+ {p.providers.map((pr) => ( + + {pr.label} + {n(pr.senders)} + + ))} +
+ )} +
+ ); +} + +function MailboxChooser({ + mailboxes, + loading, + value, + onChange, +}: { + mailboxes: PlacementWorkspaceSeed[]; + loading: boolean; + value: string[]; + onChange: (ids: string[]) => void; +}) { + const [q, setQ] = React.useState(""); + const needle = q.trim().toLowerCase(); + const shown = React.useMemo( + () => (needle ? mailboxes.filter((m) => m.email.toLowerCase().includes(needle) || m.label.toLowerCase().includes(needle)) : mailboxes), + [mailboxes, needle], + ); + const chosen = React.useMemo(() => new Set(value), [value]); + const allShown = shown.length > 0 && shown.every((m) => chosen.has(m.email_account_id)); + const toggle = (id: string) => onChange(chosen.has(id) ? value.filter((v) => v !== id) : [...value, id]); + const toggleShown = () => { + const ids = new Set(shown.map((m) => m.email_account_id)); + onChange(allShown ? value.filter((v) => !ids.has(v)) : [...value, ...shown.map((m) => m.email_account_id).filter((id) => !chosen.has(id))]); + }; + + return ( +
+
+
+ +
+ {n(value.length)} selected + {shown.length > 0 && ( + + )} + {value.length > 0 && !allShown && ( + + )} +
+
+ {loading ? ( +
+ Loading mailboxes… +
+ ) : shown.length === 0 ? ( +
+ {mailboxes.length === 0 ? "No mailbox can send a test. Seed inboxes are left out." : "No mailbox matches that."} +
+ ) : ( + shown.slice(0, MAILBOX_ROWS_SHOWN).map((m) => ( + + )) + )} +
+ {shown.length > MAILBOX_ROWS_SHOWN && ( +

+ Showing {n(MAILBOX_ROWS_SHOWN)} of {n(shown.length)}. Search to narrow, or select all shown. +

+ )} +
+ ); +} + +// Sending domains as chips; typing suggests the workspace's own domains. +function DomainInput({ value, onChange, suggestions }: { value: string[]; onChange: (v: string[]) => void; suggestions: string[] }) { + const [text, setText] = React.useState(""); + const add = (raw: string) => { + const d = raw.trim().toLowerCase().replace(/^@/, ""); + if (d && !value.includes(d)) onChange([...value, d]); + setText(""); + }; + const needle = text.trim().toLowerCase(); + const matches = needle ? suggestions.filter((s) => s.includes(needle) && !value.includes(s)).slice(0, 6) : []; + return ( +
+
+ {value.map((d) => ( + + {d} + + + ))} + setText(e.target.value)} + onKeyDown={(e) => { + if (e.key === "Enter" || e.key === "," || e.key === " " || e.key === "Tab") { + if (!text.trim()) return; + e.preventDefault(); + add(text); + } else if (e.key === "Backspace" && !text && value.length > 0) { + onChange(value.slice(0, -1)); + } + }} + onBlur={() => text.trim() && add(text)} + placeholder={value.length === 0 ? "Every domain. Type one to narrow, e.g. acme.com" : ""} + className="flex-1 min-w-[140px] h-5 bg-transparent outline-none text-[16px] md:text-[12px] text-slate-900 placeholder:text-slate-400" + /> +
+ {matches.length > 0 && ( +
+ {matches.map((s) => ( + + ))} +
+ )} +
+ ); +} + +// ---- Review + +function ReviewStep({ + draft, + patch, + preview: p, + loading, + consented, + canPay, + canBuy, + spacingSeconds, + scopeLine: senders, + copyLine, + goTo, + error, +}: { + draft: Draft; + patch: (p: Partial) => void; + preview?: PlacementBatchPreview; + loading: boolean; + consented: boolean; + canPay: boolean; + canBuy: boolean; + spacingSeconds: number; + scopeLine: string; + copyLine: string; + goTo: (k: StepKey) => void; + error: React.ReactNode; +}) { + const summary: { label: string; value: string; step: StepKey }[] = [ + { label: "Senders", value: senders, step: "senders" }, + { label: "Email", value: copyLine || "(no subject)", step: "email" }, + { + label: "Panel", + value: `${PANEL_LABEL[draft.panel]}${draft.tracking === "compare" ? ", with and without tracking" : draft.tracking === "on" ? ", tracked" : draft.tracking === "off" ? ", untracked" : ""}`, + step: "email", + }, + ]; + + return ( + <> +
+ {summary.map((row) => ( +
+
{row.label}
+
+ {row.value} +
+ +
+ ))} +
+ + {!p ? ( +
+ ) : ( +
+ + {p.metered && p.paid_tests > 0 && ( +
+

+ {p.free_tests > 0 + ? `This month's free tests cover ${n(p.free_tests)} of the ${n(p.tests)} tests. The other ${n(p.paid_tests)}` + : `This month's free tests are used up, so all ${n(p.paid_tests)} tests`}{" "} + cost up to {n(p.credits)} credits + {p.usage.credits_per_test > 0 && ` (${p.usage.credits_per_test} each)`}. + {p.usage.credit_balance != null && ` The workspace has ${n(p.usage.credit_balance)}.`} A test that delivers no copy gets its credits back. +

+ {p.usage.credits_per_test === 0 ? null : canPay ? ( + + ) : canBuy ? ( + + Top up credits in a new tab + + ) : ( +

Ask someone with the Manage billing permission to top up.

+ )} +
+ )} + {p.inactive > 0 && ( +

+ + + {plural(p.inactive, "selected mailbox is", "selected mailboxes are")} not connected right now.{" "} + {draft.onUnavailable === "defer" + ? `They are tried again as they reconnect, for up to ${BATCH_RETRY_DAYS} days.` + : "They will be skipped."} + +

+ )} +
+ )} + {error} + +
+ When a sender cannot send + + value={draft.onUnavailable} + onChange={(v) => patch({ onUnavailable: v })} + aria-label="When a sender cannot send" + options={[ + { + value: "defer", + label: "Retry until every sender has been tested", + hint: `A mailbox that is out of today's sending, busy or disconnected when its turn comes is tried again later, for up to ${BATCH_RETRY_DAYS} days.`, + }, + { + value: "skip", + label: "Skip senders that can't send right away", + hint: "The batch finishes sooner. Skipped mailboxes are listed with the reason.", + }, + ]} + /> +
+ + ); +} + +// A batch always sends at the spaced pace. +function Workload({ preview: p, spacingSeconds }: { preview: PlacementBatchPreview; spacingSeconds: number }) { + const perSender = p.seeds_per_test * p.variants * spacingSeconds; + const atOnce = Math.max(1, Math.min(p.concurrency || 1, p.selected)); + const total = Math.ceil(p.selected / atOnce) * perSender; + const freeLeft = p.usage.limit != null ? Math.max(0, p.usage.limit - p.usage.used) : null; + return ( +
+

+ {plural(p.selected, "mailbox", "mailboxes")} + {p.variants > 1 ? ", two tests each (with and without tracking)" : ""}, {plural(p.seeds_per_test, "seed")} per test: up to{" "} + {plural(p.max_sends, "copy", "copies")} + {p.variants > 1 ? ` in ${plural(p.tests, "test")}` : ""}. +

+

+ {p.selected <= atOnce + ? `They all send at once, each sending its copies about ${spacingSeconds} seconds apart for ${fmtDuration(perSender)}.` + : `About ${n(atOnce)} send at a time, each sending its copies about ${spacingSeconds} seconds apart for ${fmtDuration(perSender)}, so sending takes ${fmtDuration(total)} or longer.`}{" "} + Every copy counts against its mailbox's daily limit, and a mailbox with less left today sends fewer. A copy + not seen within 2 hours counts as never arrived. +

+ {!p.metered ? ( +

Not counted toward your monthly tests.

+ ) : p.paid_tests === 0 && freeLeft != null ? ( +

+ Uses {n(p.tests)} of the {plural(freeLeft, "free test")} left this month. +

+ ) : null} +
+ ); +} diff --git a/web/src/components/app/placement/batches/placementBatches.ts b/web/src/components/app/placement/batches/placementBatches.ts new file mode 100644 index 000000000..c2673d830 --- /dev/null +++ b/web/src/components/app/placement/batches/placementBatches.ts @@ -0,0 +1,182 @@ +// Shared vocabulary for placement batches: status labels and tones, the +// summary lines, and the messages for every refusal the batch endpoints return. +import type { AppError } from "@/lib/api/client/normalizeError"; +import type { DitherTone } from "@/components/ui/dither"; +import type { + PlacementBatch, + PlacementBatchProgress, + PlacementBatchSenderStatus, + PlacementBatchStatus, + PlacementPanel, +} from "@/lib/api/models/app/placement/Placement"; +import { placementErrorMessage } from "../tests/placementTests"; + +// Mirrors config.PlacementBatchRetryDays. +export const BATCH_RETRY_DAYS = 7; +// Mirrors config.PlacementBatchOpenPerOrgMax. +export const BATCH_OPEN_MAX = 5; + +export const BATCH_STATUS: Record = { + queued: { label: "Queued", chip: "bg-white text-slate-600 border-slate-200", dot: "bg-slate-400" }, + running: { label: "Running", chip: "bg-sky-50 text-sky-700 border-sky-200", dot: "bg-sky-500 animate-pulse" }, + completed: { label: "Completed", chip: "bg-emerald-50 text-emerald-700 border-emerald-200", dot: "bg-emerald-500" }, + completed_with_warnings: { + label: "Completed with warnings", + chip: "bg-amber-50 text-amber-700 border-amber-200", + dot: "bg-amber-500", + }, + cancelled: { label: "Cancelled", chip: "bg-slate-50 text-slate-600 border-slate-200", dot: "bg-slate-400" }, + failed: { label: "Failed", chip: "bg-rose-50 text-rose-700 border-rose-200", dot: "bg-rose-500" }, +}; + +export const SENDER_STATUS: Record = { + queued: { label: "Queued", chip: "bg-white text-slate-500 border-slate-200", dot: "bg-slate-300" }, + deferred: { label: "Deferred", chip: "bg-amber-50 text-amber-700 border-amber-200", dot: "bg-amber-400" }, + running: { label: "Sending", chip: "bg-sky-50 text-sky-700 border-sky-200", dot: "bg-sky-500 animate-pulse" }, + completed: { label: "Tested", chip: "bg-emerald-50 text-emerald-700 border-emerald-200", dot: "bg-emerald-500" }, + skipped: { label: "Skipped", chip: "bg-amber-50 text-amber-700 border-amber-200", dot: "bg-amber-500" }, + failed: { label: "Failed", chip: "bg-rose-50 text-rose-700 border-rose-200", dot: "bg-rose-500" }, + cancelled: { label: "Cancelled", chip: "bg-slate-50 text-slate-500 border-slate-200", dot: "bg-slate-300" }, +}; + +// Short labels for why a sender was skipped or deferred; `detail` has the sentence. +export const SENDER_REASON: Record = { + placement_daily_budget: "Daily limit reached", + placement_sender_busy: "Sending another test", + placement_sender_unavailable: "Not connected", + placement_invalid_seeds: "Seed choice no longer valid", + placement_no_seeds: "No seed it can reach", + placement_sender_deleted: "Mailbox deleted", + placement_batch_retry_expired: "Retry window ended", + placement_batch_start_failed: "Could not start", + placement_batch_copy_unavailable: "Copy no longer available", + placement_quota_exceeded: "Monthly tests used up", + insufficient_credits: "Out of credits", + usage_cap_exceeded: "Credit spend limit reached", +}; + +export function batchOpen(status: PlacementBatchStatus): boolean { + return status === "queued" || status === "running"; +} + +// Progress parts in display order, with the tone each takes in the bar. +const PROGRESS_PARTS: { key: keyof Omit; label: string; tone: DitherTone }[] = [ + { key: "completed", label: "tested", tone: "emerald" }, + { key: "running", label: "sending", tone: "sky" }, + { key: "queued", label: "queued", tone: "slate" }, + { key: "deferred", label: "deferred", tone: "amber" }, + { key: "skipped", label: "skipped", tone: "amber" }, + { key: "failed", label: "failed", tone: "rose" }, + { key: "cancelled", label: "cancelled", tone: "slate" }, +]; + +/** "120 tested · 20 sending · 5 skipped", only the parts that are not zero. */ +export function progressLine(p: PlacementBatchProgress): string { + return PROGRESS_PARTS.filter((part) => p[part.key] > 0) + .map((part) => `${p[part.key].toLocaleString()} ${part.label}`) + .join(" · "); +} + +/** Bar segments for senders that are done one way or another, or sending. */ +export function progressSegments(p: PlacementBatchProgress): { frac: number; tone: DitherTone }[] { + const denom = Math.max(1, p.total); + return PROGRESS_PARTS.filter((part) => part.key !== "queued" && part.key !== "deferred" && p[part.key] > 0).map((part) => ({ + frac: p[part.key] / denom, + tone: part.tone, + })); +} + +/** Senders that have finished one way or another. */ +export function doneCount(p: PlacementBatchProgress): number { + return p.completed + p.skipped + p.failed + p.cancelled; +} + +/** "Campaign senders · 10% sample". */ +export function scopeSummary(b: Pick): string { + const { selection } = b; + const parts: string[] = []; + const scope = selection.sender_scope; + if (selection.sender_account_ids) { + parts.push(`${selection.sender_account_ids.toLocaleString()} chosen mailbox${selection.sender_account_ids === 1 ? "" : "es"}`); + } else if (scope?.type === "campaign") { + parts.push("Campaign senders"); + } else if (scope) { + parts.push("All mailboxes"); + } + if (scope?.untested_days) parts.push(`untested for ${scope.untested_days} days`); + const s = selection.sample; + switch (s?.mode) { + case "random": + parts.push(`random ${(s.count ?? 0).toLocaleString()}`); + break; + case "percent": + parts.push(`${s.percent ?? 0}% sample`); + break; + case "per_domain": + parts.push(`${s.count ?? 0} per domain`); + break; + case "per_provider": + parts.push(`${s.count ?? 0} per provider`); + break; + } + return parts.join(" · "); +} + +/** "about 2 hours", "about 25 minutes", "under 2 minutes". */ +export function fmtDuration(seconds: number): string { + if (seconds < 90) return "under 2 minutes"; + const minutes = Math.round(seconds / 60); + if (minutes < 90) return `about ${minutes} minutes`; + const hours = Math.round(minutes / 60); + if (hours < 36) return `about ${hours} hours`; + const days = Math.round(hours / 24); + return `about ${days} days`; +} + +// Which step of the new-batch dialog a refusal is about, so it shows there. +export type BatchErrorField = "senders" | "email" | "review" | "general"; + +export function batchErrorMessage( + err: AppError, + ctx: { resetsOn?: Date | null; panel?: PlacementPanel } = {}, +): { field: BatchErrorField; message: string } { + switch (err?.code) { + case "placement_batch_empty": + return { field: "senders", message: "No sending mailbox matches this selection. Widen the filters or choose other mailboxes." }; + case "placement_batch_too_large": + return { field: "senders", message: err.message || "This batch has more senders than this instance allows. Narrow the selection or take a sample." }; + case "placement_too_many_batches": + return { + field: "general", + message: `${BATCH_OPEN_MAX} batches are already running in this workspace. Wait for one to finish, or cancel one.`, + }; + case "placement_quota_exceeded": + return { + field: "review", + message: "This month's free placement tests do not cover this batch, and tests past them cannot be paid for. Take a smaller sample, or use your own seed inboxes, which are never counted.", + }; + case "insufficient_credits": + return { field: "review", message: "The workspace does not have enough credits for this batch. Top up under Settings > Billing, or take a smaller sample." }; + case "usage_cap_exceeded": + return { field: "review", message: "This batch would go past the workspace's credit spend limit. An admin can raise it under Settings > Billing." }; + case "placement_no_seeds": + return { + field: "email", + message: + ctx.panel === "workspace" + ? "None of your seed inboxes can take this batch. Add a seed inbox, or clear the seed choice." + : "This panel has no seed inbox these senders can reach.", + }; + } + const single = placementErrorMessage(err, { resetsOn: ctx.resetsOn, panel: ctx.panel }); + switch (single.field) { + case "sender": + return { field: "senders", message: single.message }; + case "source": + case "tracking": + case "panel": + return { field: "email", message: single.message }; + default: + return { field: "general", message: single.message }; + } +} diff --git a/web/src/components/app/placement/tests/NewPlacementTestDialog.tsx b/web/src/components/app/placement/tests/NewPlacementTestDialog.tsx index 74951643b..321dd229a 100644 --- a/web/src/components/app/placement/tests/NewPlacementTestDialog.tsx +++ b/web/src/components/app/placement/tests/NewPlacementTestDialog.tsx @@ -7,19 +7,9 @@ import React from "react"; import { createPortal } from "react-dom"; import { AnimatePresence, motion } from "framer-motion"; import { Link, useNavigate } from "react-router-dom"; -import { useQuery } from "@tanstack/react-query"; -import { - AlertCircleIcon, - Loader2Icon, - MailCheckIcon, - MailIcon, - MegaphoneIcon, - PlayIcon, - UserRoundIcon, - XIcon, -} from "lucide-react"; +import { Loader2Icon, MailCheckIcon, MailIcon, PlayIcon, XIcon } from "lucide-react"; import toast from "react-hot-toast"; -import { Label, SearchInput, TextInput } from "@/components/ui/field"; +import { Label, SearchInput } from "@/components/ui/field"; import { Checkbox } from "@/components/ui/checkbox"; import { PopoverMenu, @@ -30,40 +20,31 @@ import { PopoverMenuTrigger, SelectButton, } from "@/components/ui/popover-menu"; -import { SelectMenu } from "@/components/ui/select-menu"; -import { OptionSelect, Segmented } from "@/components/app/campaigns/preferences/components/CampaignPreferenceBoolBox"; -import RichTextEditor from "@/components/app/campaigns/sequences/RichTextEditor"; -import { VARIABLES, htmlToPlain } from "@/components/app/campaigns/sequences/emailPreview"; -import { contactLabel } from "@/components/app/campaigns/sequences/previewContext"; -import { LINK_VARIABLES } from "@/lib/templateVars"; import { useConfirm } from "@/hooks/context/confirm"; import { usePermission } from "@/hooks/usePermission"; -import useDebouncedValue from "@/hooks/useDebouncedValue"; -import useCampaigns from "@/lib/api/hooks/app/campaigns/useCampaigns"; import useCampaign from "@/lib/api/hooks/app/campaigns/useCampaign"; import useCampaignSenders from "@/lib/api/hooks/app/campaigns/useCampaignSenders"; -import useSearchContacts from "@/lib/api/hooks/app/contacts/useSearchContacts"; -import getSequences from "@/lib/api/client/app/campaigns/sequences/getSequences"; +import { htmlToPlain } from "@/components/app/campaigns/sequences/emailPreview"; import { useCreatePlacementTest, usePlacementOverview, usePlacementSeeds } from "@/lib/api/hooks/app/placement/usePlacement"; -import { - PANEL_LABEL, - type CreatePlacementTestRequest, - type PlacementPace, - type PlacementPanel, - type PlacementPanelFamily, - type PlacementTracking, +import type { + CreatePlacementTestRequest, + PlacementPace, + PlacementPanel, + PlacementTracking, } from "@/lib/api/models/app/placement/Placement"; import type Contact from "@/lib/api/models/app/contacts/Contact"; import type { AppError } from "@/lib/api/client/normalizeError"; import { cn } from "@/lib/utils"; import { placementErrorMessage, seedBlocker, testCost, type PlacementErrorField } from "./placementTests"; +import { CopySourceFields, FamilyChips, InlineError, PaceChoice, PanelChoice, TrackingChoice } from "./PlacementFormParts"; +import { + QUICK_SPACING_SECONDS, + newIdempotencyKey as newKey, + useCampaignEmailSteps, + type CopySource as Source, +} from "./placementCopy"; import SeedChooser from "./SeedChooser"; -type Source = "step" | "custom"; - -// Mirrors config.PlacementQuickSpacingSeconds. -const QUICK_SPACING_SECONDS = 8; - interface Draft { senderId: string; source: Source; @@ -117,12 +98,6 @@ function draftKey(d: Draft): string { return JSON.stringify([d.source, d.campaignId, d.stepId, d.subject, d.bodyHtml, d.contact?.id ?? "", d.tracking, d.seedIds, d.families, d.pace]); } -function newKey(): string { - return typeof crypto !== "undefined" && "randomUUID" in crypto - ? crypto.randomUUID() - : `${Date.now()}-${Math.random().toString(36).slice(2)}`; -} - export default function NewPlacementTestDialog({ open, onClose, @@ -163,15 +138,8 @@ function DialogBody({ onClose, prefill }: { onClose: () => void; prefill?: NewPl const campaign = useCampaign(draft.source === "step" ? draft.campaignId : ""); const campaignSenders = useCampaignSenders(draft.campaignId, draft.source === "step" && !!draft.campaignId); - const steps = useQuery({ - queryKey: ["campaigns", draft.campaignId, "sequences"], - queryFn: () => getSequences(draft.campaignId), - enabled: draft.source === "step" && !!draft.campaignId, - }); - const emailSteps = React.useMemo( - () => (steps.data ?? []).filter((s) => (s.kind ?? "email") === "email"), - [steps.data], - ); + const steps = useCampaignEmailSteps(draft.campaignId, draft.source === "step"); + const emailSteps = steps.emailSteps; // Senders: connected mailboxes that are not seeds, the campaign's own first. const inCampaign = React.useMemo( @@ -395,212 +363,42 @@ function DialogBody({ onClose, prefill }: { onClose: () => void; prefill?: NewPl
{/* What to test */} -
-
- What to test - - value={draft.source} - onChange={(v) => - patch({ - source: v, - tracking: v === "step" ? "campaign" : draft.tracking === "campaign" ? "off" : draft.tracking, - }) - } - options={[ - { value: "step", label: "Campaign step" }, - { value: "custom", label: "Custom copy" }, - ]} - /> -
- - {draft.source === "step" ? ( -
-
- - patch({ campaignId: id, stepId: "", contact: null })} - /> -
-
- - patch({ stepId: v })} - disabled={!draft.campaignId || steps.isLoading} - fullWidth - placeholder={ - !draft.campaignId - ? "Pick a campaign first" - : steps.isLoading - ? "Loading steps…" - : emailSteps.length === 0 - ? "No email steps" - : "Pick a step" - } - options={emailSteps.map((s, i) => ({ - value: s.id, - label: `${s.name || `Step ${i + 1}`}${s.subject ? `: ${s.subject}` : ""}`, - }))} - aria-label="Step" - /> -
-

- The saved step is rendered exactly as the campaign sends it: merge fields, spintax, - signature, opt-out footer and unsubscribe header. -

-
- ) : ( -
-
- - patch({ subject: v })} - placeholder="Quick question, {{.FirstName}}" - /> -
-
- - - patch({ bodyHtml: html, bodyPlain: draft.bodyCode ? "" : htmlToPlain(html) }) - } - code={draft.bodyCode} - onCodeChange={(c) => patch({ bodyCode: c })} - variables={VARIABLES} - links={LINK_VARIABLES} - placeholder="Hi {{.FirstName}}, …" - /> -
-
- )} - {fieldError("source")} - -
- - patch({ contact: c })} - /> -

- Fills the merge fields. Nobody but the seed inboxes receives the copies. -

-
-
+ + patch({ + source: v, + tracking: v === "step" ? "campaign" : draft.tracking === "campaign" ? "off" : draft.tracking, + }) + } + campaignName={campaign.data?.name} + steps={steps} + error={fieldError("source")} + /> {/* Tracking */} -
- Tracking - - value={draft.tracking} - onChange={(v) => patch({ tracking: v })} - cols={2} - aria-label="Tracking" - options={[ - ...(draft.source === "step" - ? [{ value: "campaign" as const, label: "As the campaign", hint: "Uses the campaign's open and click tracking." }] - : []), - ...(textOnly - ? [] - : [{ value: "on" as const, label: "On", hint: "Open pixel and tracked links." }]), - { value: "off" as const, label: "Off", hint: "No pixel, links left as written." }, - ...(textOnly - ? [] - : [ - { - value: "compare" as const, - label: "Compare with and without", - hint: "Two tests to the same seeds. Counts as 2 tests.", - }, - ]), - ]} - /> - {textOnly && ( -

This campaign sends plain text, which carries no tracking.

- )} - {fieldError("tracking")} -
+ patch({ tracking: v })} + source={draft.source} + textOnly={textOnly} + error={fieldError("tracking")} + /> {/* Pace */} -
- Pace - - value={draft.pace} - onChange={(v) => patch({ pace: v })} - cols={2} - aria-label="Pace" - options={[ - { value: "spaced", label: "Spaced", hint: "About a minute between copies, the way a campaign sends." }, - { value: "quick", label: "Quick", hint: "A few seconds apart, so results come in within minutes." }, - ]} - /> -
+ patch({ pace: v })} /> {/* Panel */}
Seed panel - {overview.isLoading ? ( -
- ) : ( -
- {panels.map((p) => { - const active = p.panel === draft.panel; - return ( - - ); - })} -
- )} + patch({ panel: p })} + usage={usage} + /> {draft.panel !== "workspace" && panel?.available && panelFamilies.length > 1 && ( void; prefill?: NewPl ); } -// The shared panels' provider families; nothing picked tests every provider. -function FamilyChips({ - families, - value, - onChange, -}: { - families: PlacementPanelFamily[]; - value: string[]; - onChange: (families: string[]) => void; -}) { - const chip = (active: boolean) => - cn( - "h-6 px-2 rounded-md border text-[11px] font-medium inline-flex items-center gap-1 transition-colors", - active ? "border-sky-200 bg-sky-50 text-sky-700" : "border-slate-200 bg-white text-slate-600 hover:border-slate-300", - ); - return ( -
- Providers -
- - {families.map((f) => { - const active = value.includes(f.family); - return ( - - ); - })} -
-
- ); -} - -function InlineError({ message, compact = false }: { message: string; compact?: boolean }) { - return ( -

- - {message} -

- ); -} - function SenderPicker({ senders, inCampaign, @@ -822,109 +569,3 @@ function SenderPicker({ ); } - -function CampaignPicker({ value, name, onChange }: { value: string; name?: string; onChange: (id: string) => void }) { - const [open, setOpen] = React.useState(false); - const [q, setQ] = React.useState(""); - const debounced = useDebouncedValue(q.trim(), 250); - const list = useCampaigns({ query: debounced, folder: "", limit: 20, enabled: open, all: false }); - return ( - - - } - label={value ? (name ?? "Loading…") : "Pick a campaign"} - className="w-full [&>span:nth-child(2)]:max-w-none [&>span:nth-child(2)]:flex-1 [&>span:nth-child(2)]:text-left" - /> - - -
- -
- {list.isLoading && list.campaigns.length === 0 ? ( -
- Loading… -
- ) : list.campaigns.length === 0 ? ( -
No campaign matches that.
- ) : ( - list.campaigns.map((c) => ( - onChange(c.id)}> - {c.name} - - )) - )} - - - ); -} - -// Whose merge fields fill the copy. Empty = the campaign's first lead, or the -// built-in sample contact for custom copy. -function ContactPicker({ - campaignId, - value, - onChange, -}: { - campaignId: string; - value: Contact | null; - onChange: (c: Contact | null) => void; -}) { - const [open, setOpen] = React.useState(false); - const [q, setQ] = React.useState(""); - const debounced = useDebouncedValue(q.trim(), 250); - const searching = debounced.length > 0; - const search = useSearchContacts({ - options: { - query: debounced, - custom_field_filters: [], - campaign_ids: searching || !campaignId ? [] : [campaignId], - sort_by: "updated_at", - reverse: false, - }, - limit: 8, - enabled: open, - keepPrevious: true, - }); - const contacts = search.contacts ?? []; - const fallback = campaignId ? "The campaign's first lead" : "A sample contact"; - return ( - - - } - label={value ? contactLabel(value) : fallback} - className="w-full [&>span:nth-child(2)]:max-w-none [&>span:nth-child(2)]:flex-1 [&>span:nth-child(2)]:text-left" - /> - - -
- -
- onChange(null)} icon={}> - {fallback} - - - {searching || !campaignId ? "Contacts" : "Leads in this campaign"} -
- {search.isLoading && contacts.length === 0 ? ( -
- Loading… -
- ) : contacts.length === 0 ? ( -
- {searching ? "No contact matches that." : "No contacts yet. Type to search."} -
- ) : ( - contacts.map((c) => ( - onChange(c)}> - {contactLabel(c)} - {c.email} - - )) - )} -
- - - ); -} diff --git a/web/src/components/app/placement/tests/PlacementFormParts.tsx b/web/src/components/app/placement/tests/PlacementFormParts.tsx new file mode 100644 index 000000000..3b0b3c961 --- /dev/null +++ b/web/src/components/app/placement/tests/PlacementFormParts.tsx @@ -0,0 +1,435 @@ +// Form pieces shared by the new placement test and new placement batch +// dialogs: the copy to test, tracking, pace, seed panel and the pickers they use. + +import React from "react"; +import { AlertCircleIcon, Loader2Icon, MegaphoneIcon, UserRoundIcon } from "lucide-react"; +import { Label, SearchInput, TextInput } from "@/components/ui/field"; +import { + PopoverMenu, + PopoverMenuContent, + PopoverMenuItem, + PopoverMenuLabel, + PopoverMenuSeparator, + PopoverMenuTrigger, + SelectButton, +} from "@/components/ui/popover-menu"; +import { SelectMenu } from "@/components/ui/select-menu"; +import { OptionSelect, Segmented } from "@/components/app/campaigns/preferences/components/CampaignPreferenceBoolBox"; +import RichTextEditor from "@/components/app/campaigns/sequences/RichTextEditor"; +import { VARIABLES, htmlToPlain } from "@/components/app/campaigns/sequences/emailPreview"; +import { contactLabel } from "@/components/app/campaigns/sequences/previewContext"; +import { LINK_VARIABLES } from "@/lib/templateVars"; +import useDebouncedValue from "@/hooks/useDebouncedValue"; +import useCampaigns from "@/lib/api/hooks/app/campaigns/useCampaigns"; +import useSearchContacts from "@/lib/api/hooks/app/contacts/useSearchContacts"; +import { + PANEL_LABEL, + type PlacementPace, + type PlacementPanel, + type PlacementPanelFamily, + type PlacementPanelInfo, + type PlacementTracking, + type PlacementUsage, +} from "@/lib/api/models/app/placement/Placement"; +import type Contact from "@/lib/api/models/app/contacts/Contact"; +import { cn } from "@/lib/utils"; +import type { CopyDraft, CopySource } from "./placementCopy"; + +export function SectionLabel({ children }: { children: React.ReactNode }) { + return {children}; +} + +export function CopySourceFields({ + value, + patch, + onSource, + campaignName, + steps, + error, +}: { + value: CopyDraft; + patch: (p: Partial) => void; + onSource: (s: CopySource) => void; + campaignName?: string; + steps: { emailSteps: { id: string; name?: string; subject?: string }[]; isLoading: boolean }; + error?: React.ReactNode; +}) { + const { emailSteps } = steps; + return ( +
+
+ What to test + + value={value.source} + onChange={onSource} + options={[ + { value: "step", label: "Campaign step" }, + { value: "custom", label: "Custom copy" }, + ]} + /> +
+ + {value.source === "step" ? ( +
+
+ + patch({ campaignId: id, stepId: "", contact: null })} + /> +
+
+ + patch({ stepId: v })} + disabled={!value.campaignId || steps.isLoading} + fullWidth + placeholder={ + !value.campaignId + ? "Pick a campaign first" + : steps.isLoading + ? "Loading steps…" + : emailSteps.length === 0 + ? "No email steps" + : "Pick a step" + } + options={emailSteps.map((s, i) => ({ + value: s.id, + label: `${s.name || `Step ${i + 1}`}${s.subject ? `: ${s.subject}` : ""}`, + }))} + aria-label="Step" + /> +
+

+ The saved step is rendered exactly as the campaign sends it: merge fields, spintax, + signature, opt-out footer and unsubscribe header. +

+
+ ) : ( +
+
+ + patch({ subject: v })} + placeholder="Quick question, {{.FirstName}}" + /> +
+
+ + patch({ bodyHtml: html, bodyPlain: value.bodyCode ? "" : htmlToPlain(html) })} + code={value.bodyCode} + onCodeChange={(c) => patch({ bodyCode: c })} + variables={VARIABLES} + links={LINK_VARIABLES} + placeholder="Hi {{.FirstName}}, …" + /> +
+
+ )} + {error} + +
+ + patch({ contact: c })} + /> +

+ Fills the merge fields. Nobody but the seed inboxes receives the copies. +

+
+
+ ); +} + +export function TrackingChoice({ + value, + onChange, + source, + textOnly, + compareHint = "Two tests to the same seeds. Counts as 2 tests.", + error, +}: { + value: PlacementTracking; + onChange: (v: PlacementTracking) => void; + source: CopySource; + textOnly: boolean; + compareHint?: string; + error?: React.ReactNode; +}) { + return ( +
+ Tracking + + value={value} + onChange={onChange} + cols={2} + aria-label="Tracking" + options={[ + ...(source === "step" + ? [{ value: "campaign" as const, label: "As the campaign", hint: "Uses the campaign's open and click tracking." }] + : []), + ...(textOnly ? [] : [{ value: "on" as const, label: "On", hint: "Open pixel and tracked links." }]), + { value: "off" as const, label: "Off", hint: "No pixel, links left as written." }, + ...(textOnly ? [] : [{ value: "compare" as const, label: "Compare with and without", hint: compareHint }]), + ]} + /> + {textOnly &&

This campaign sends plain text, which carries no tracking.

} + {error} +
+ ); +} + +export function PaceChoice({ value, onChange }: { value: PlacementPace; onChange: (v: PlacementPace) => void }) { + return ( +
+ Pace + + value={value} + onChange={onChange} + cols={2} + aria-label="Pace" + options={[ + { value: "spaced", label: "Spaced", hint: "About a minute between copies, the way a campaign sends." }, + { value: "quick", label: "Quick", hint: "A few seconds apart, so results come in within minutes." }, + ]} + /> +
+ ); +} + +// The seed panels as radio cards; an unavailable one says why. +export function PanelChoice({ + panels, + loading, + value, + onChange, + usage, +}: { + panels: PlacementPanelInfo[]; + loading: boolean; + value: PlacementPanel; + onChange: (p: PlacementPanel) => void; + usage?: PlacementUsage; +}) { + if (loading) return
; + return ( +
+ {panels.map((p) => { + const active = p.panel === value; + return ( + + ); + })} +
+ ); +} + +// The shared panels' provider families; nothing picked tests every provider. +export function FamilyChips({ + families, + value, + onChange, +}: { + families: PlacementPanelFamily[]; + value: string[]; + onChange: (families: string[]) => void; +}) { + const chip = (active: boolean) => + cn( + "h-6 px-2 rounded-md border text-[11px] font-medium inline-flex items-center gap-1 transition-colors", + active ? "border-sky-200 bg-sky-50 text-sky-700" : "border-slate-200 bg-white text-slate-600 hover:border-slate-300", + ); + return ( +
+ Providers +
+ + {families.map((f) => { + const active = value.includes(f.family); + return ( + + ); + })} +
+
+ ); +} + +export function InlineError({ message, compact = false }: { message: string; compact?: boolean }) { + return ( +

+ + {message} +

+ ); +} + +export function CampaignPicker({ value, name, onChange }: { value: string; name?: string; onChange: (id: string) => void }) { + const [open, setOpen] = React.useState(false); + const [q, setQ] = React.useState(""); + const debounced = useDebouncedValue(q.trim(), 250); + const list = useCampaigns({ query: debounced, folder: "", limit: 20, enabled: open, all: false }); + return ( + + + } + label={value ? (name ?? "Loading…") : "Pick a campaign"} + className="w-full [&>span:nth-child(2)]:max-w-none [&>span:nth-child(2)]:flex-1 [&>span:nth-child(2)]:text-left" + /> + + +
+ +
+ {list.isLoading && list.campaigns.length === 0 ? ( +
+ Loading… +
+ ) : list.campaigns.length === 0 ? ( +
No campaign matches that.
+ ) : ( + list.campaigns.map((c) => ( + onChange(c.id)}> + {c.name} + + )) + )} + + + ); +} + +// Whose merge fields fill the copy. Empty = the campaign's first lead, or the +// built-in sample contact for custom copy. +export function ContactPicker({ + campaignId, + value, + onChange, +}: { + campaignId: string; + value: Contact | null; + onChange: (c: Contact | null) => void; +}) { + const [open, setOpen] = React.useState(false); + const [q, setQ] = React.useState(""); + const debounced = useDebouncedValue(q.trim(), 250); + const searching = debounced.length > 0; + const search = useSearchContacts({ + options: { + query: debounced, + custom_field_filters: [], + campaign_ids: searching || !campaignId ? [] : [campaignId], + sort_by: "updated_at", + reverse: false, + }, + limit: 8, + enabled: open, + keepPrevious: true, + }); + const contacts = search.contacts ?? []; + const fallback = campaignId ? "The campaign's first lead" : "A sample contact"; + return ( + + + } + label={value ? contactLabel(value) : fallback} + className="w-full [&>span:nth-child(2)]:max-w-none [&>span:nth-child(2)]:flex-1 [&>span:nth-child(2)]:text-left" + /> + + +
+ +
+ onChange(null)} icon={}> + {fallback} + + + {searching || !campaignId ? "Contacts" : "Leads in this campaign"} +
+ {search.isLoading && contacts.length === 0 ? ( +
+ Loading… +
+ ) : contacts.length === 0 ? ( +
+ {searching ? "No contact matches that." : "No contacts yet. Type to search."} +
+ ) : ( + contacts.map((c) => ( + onChange(c)}> + {contactLabel(c)} + {c.email} + + )) + )} +
+ + + ); +} diff --git a/web/src/components/app/placement/tests/SeedChooser.tsx b/web/src/components/app/placement/tests/SeedChooser.tsx index b8c63c617..194388d84 100644 --- a/web/src/components/app/placement/tests/SeedChooser.tsx +++ b/web/src/components/app/placement/tests/SeedChooser.tsx @@ -103,7 +103,6 @@ export default function SeedChooser({ )} > toggle(s.email_account_id)} diff --git a/web/src/components/app/placement/tests/placementCopy.ts b/web/src/components/app/placement/tests/placementCopy.ts new file mode 100644 index 000000000..61540aef3 --- /dev/null +++ b/web/src/components/app/placement/tests/placementCopy.ts @@ -0,0 +1,74 @@ +// The copy a placement test or batch sends, as a draft and as request fields, +// shared by both new-test dialogs. +import React from "react"; +import { useQuery } from "@tanstack/react-query"; +import { htmlToPlain } from "@/components/app/campaigns/sequences/emailPreview"; +import getSequences from "@/lib/api/client/app/campaigns/sequences/getSequences"; +import type Contact from "@/lib/api/models/app/contacts/Contact"; + +export type CopySource = "step" | "custom"; + +// Mirrors config.PlacementQuickSpacingSeconds. +export const QUICK_SPACING_SECONDS = 8; + +export function newIdempotencyKey(): string { + return typeof crypto !== "undefined" && "randomUUID" in crypto + ? crypto.randomUUID() + : `${Date.now()}-${Math.random().toString(36).slice(2)}`; +} + +// The copy a test sends: a saved campaign step or a template written here. +export interface CopyDraft { + source: CopySource; + campaignId: string; + stepId: string; + subject: string; + bodyHtml: string; + bodyPlain: string; + bodyCode: boolean; + contact: Contact | null; +} + +// Whether the copy is complete enough to send, and what is missing when not. +export function copyIssue(d: CopyDraft, steps: { emailSteps: { id: string }[]; isLoading: boolean }): string | null { + if (d.source === "step") { + if (!d.campaignId) return "Pick a campaign."; + if (!d.stepId) return steps.emailSteps.length === 0 && !steps.isLoading ? "This campaign has no email step to test." : "Pick a step."; + return null; + } + if (!d.subject.trim()) return "Write a subject."; + if (!(d.bodyPlain.trim() || (d.bodyCode && d.bodyHtml.trim()))) return "Write the email body."; + return null; +} + +// The copy fields of a request body. +export function copyBody(d: CopyDraft): { + campaign_id?: string; + sequence_id?: string; + contact_id?: string; + subject?: string; + body_html?: string; + body_plain?: string; +} { + const contact = d.contact ? { contact_id: d.contact.id } : {}; + if (d.source === "step") return { campaign_id: d.campaignId, sequence_id: d.stepId, ...contact }; + return { + subject: d.subject.trim(), + body_html: d.bodyHtml, + body_plain: d.bodyCode ? htmlToPlain(d.bodyHtml) : d.bodyPlain, + ...contact, + }; +} + +export function useCampaignEmailSteps(campaignId: string, enabled: boolean) { + const steps = useQuery({ + queryKey: ["campaigns", campaignId, "sequences"], + queryFn: () => getSequences(campaignId), + enabled: enabled && !!campaignId, + }); + const emailSteps = React.useMemo( + () => (steps.data ?? []).filter((s) => (s.kind ?? "email") === "email"), + [steps.data], + ); + return { emailSteps, isLoading: steps.isLoading }; +} diff --git a/web/src/components/app/placement/tests/placementTests.ts b/web/src/components/app/placement/tests/placementTests.ts index 8efdf4a75..f3fe6c4da 100644 --- a/web/src/components/app/placement/tests/placementTests.ts +++ b/web/src/components/app/placement/tests/placementTests.ts @@ -48,6 +48,7 @@ export const ORIGIN_LABEL: Record = { monitor: "Monitor", admin: "Operator", remote: "Linked instance", + batch: "Batch", }; /** A 0..1 fraction as a whole percentage, or a dash while there is none. */ diff --git a/web/src/components/app/segments/SegmentEditor.tsx b/web/src/components/app/segments/SegmentEditor.tsx index 6f2803417..027341026 100644 --- a/web/src/components/app/segments/SegmentEditor.tsx +++ b/web/src/components/app/segments/SegmentEditor.tsx @@ -490,7 +490,7 @@ function ValueInput({ case "enum": return ; case "category": - return ; + return ; case "campaign": return ; case "segment": diff --git a/web/src/components/app/unibox/ContactContextPanel.tsx b/web/src/components/app/unibox/ContactContextPanel.tsx index 073a046b5..50ddc4166 100644 --- a/web/src/components/app/unibox/ContactContextPanel.tsx +++ b/web/src/components/app/unibox/ContactContextPanel.tsx @@ -46,7 +46,7 @@ import useContactCampaignStates from "@/lib/api/hooks/app/contacts/useContactCam import type ContactCampaignState from "@/lib/api/models/app/contacts/ContactCampaignState"; import type MiniCampaign from "@/lib/api/models/app/campaigns/MiniCampaign"; import { holdSummary } from "@/lib/api/models/app/contacts/Contact"; -import { leadCanBePaused } from "@/lib/leadHold"; +import { CC_RESUME_CONFIRM, leadCanBePaused } from "@/lib/leadHold"; import { usePermission } from "@/hooks/usePermission"; import { PauseLeadButton, ResumeLeadButton } from "@/components/app/contacts/LeadHoldButtons"; import LeadStatusPill from "@/components/app/contacts/LeadStatusPill"; @@ -372,7 +372,11 @@ function CampaignsSection({ {line.text} {canWrite && s.hold ? ( - + ) : canWrite && leadCanBePaused(s) ? (
@@ -752,12 +785,14 @@ function IconAction({ icon, danger, disabled, + className, onClick, }: { label: string; icon: React.ReactNode; danger?: boolean; disabled?: boolean; + className?: string; onClick?: () => void; }) { return ( @@ -768,12 +803,13 @@ function IconAction({ onClick={onClick} disabled={disabled} aria-label={label} - className={ - "size-7 rounded-md inline-flex items-center justify-center transition-colors disabled:opacity-40 disabled:pointer-events-none " + - (danger + className={cn( + "size-7 rounded-md inline-flex items-center justify-center transition-colors disabled:opacity-40 disabled:pointer-events-none", + danger ? "text-slate-500 hover:text-red-600 hover:bg-red-50" - : "text-slate-500 hover:text-slate-900 hover:bg-slate-100") - } + : "text-slate-500 hover:text-slate-900 hover:bg-slate-100", + className, + )} > {icon} diff --git a/web/src/components/app/unibox/compose/ContactRecipientField.tsx b/web/src/components/app/unibox/compose/ContactRecipientField.tsx index 19acd303e..6a06ea485 100644 --- a/web/src/components/app/unibox/compose/ContactRecipientField.tsx +++ b/web/src/components/app/unibox/compose/ContactRecipientField.tsx @@ -396,7 +396,7 @@ export default function ContactRecipientField({ {c.title} - filter by category + filter by label ))} @@ -502,7 +502,7 @@ export default function ContactRecipientField({ {allCategories.length > 0 && ( ({ id: c.id, label: c.title, diff --git a/web/src/components/app/unibox/scopeRail.test.tsx b/web/src/components/app/unibox/scopeRail.test.tsx new file mode 100644 index 000000000..393461c50 --- /dev/null +++ b/web/src/components/app/unibox/scopeRail.test.tsx @@ -0,0 +1,635 @@ +// The scope rail's sections fold and their rows can be hidden. +// +// Both are per-browser preferences that live in the persisted store, so what is +// pinned here is the part that is easy to get wrong: the fold survives, a +// folded section still shows the scope you are on, hiding a row only takes it +// off the rail (the active scope never disappears), and a stored value that is +// not what the setters would have written cannot reach the screen. + +import React from "react"; +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { render, screen, fireEvent, cleanup } from "@testing-library/react"; +import { useAppStore } from "@/stores"; +import { applyRailOrder, sanitizeUniboxRailHidden, sanitizeUniboxRailOrder } from "@/stores/slices/uiSlice"; +import { ScopeRail, type UniboxScope } from "./ScopeRail"; + +const overview = vi.hoisted(() => ({ + data: { + total: 12, + unread: 3, + today: 0, + week: 0, + snoozed: 0, + awaiting_reply: 0, + automated: 0, + automated_unread: 0, + awaiting_agent_draft: 0, + scheduled_pending: 0, + scheduled_pending_max: 100, + folders: [ + { folder: "inbox", unread: 3, total: 9 }, + { folder: "spam", unread: 0, total: 1 }, + ], + mailboxes: [{ id: "m1", email: "me@example.com", name: "Me", unread: 2, total: 5 }], + tags: [], + categories: [{ id: "c1", title: "Interested", color: "#0ea5e9", unread: 1, total: 4 }], + }, +})); + +vi.mock("@/lib/api/hooks/app/unibox/useUniboxOverview", () => ({ + default: () => ({ data: overview.data, isPending: false }), +})); +vi.mock("@/lib/api/hooks/app/unibox/useMarkSeen", () => ({ + default: () => ({ mutate: () => {} }), +})); +vi.mock("@/components/app/unibox/compose/ComposeDraftsItem", () => ({ + default: () => null, +})); +// Exit animations never finish in jsdom; a closed menu unmounts at once instead. +vi.mock("framer-motion", async (importOriginal) => ({ + ...(await importOriginal>()), + AnimatePresence: ({ children }: { children: React.ReactNode }) => <>{children}, +})); +// The count-up tween needs a real animation frame; a plain number is enough here. +vi.mock("@/components/ui/AnimatedNumber", () => ({ + default: ({ value }: { value: number }) => <>{value}, +})); + +// The toggle's name grows by the dot's screen-reader text when a highlighted +// count is folded away, so match on how it starts. +const MAIL_TOGGLE = /^Mail(?!boxes| section)/; +const DOT = /highlighted count folded away/; + +// The header's always-visible pencil. +function startEditing(section: "Mail" | "Views") { + fireEvent.click(screen.getByRole("button", { name: `Edit ${section} rows` })); +} + +function mountRail(scope: UniboxScope = { kind: "all" }) { + return render( {}} />); +} + +beforeEach(() => { + useAppStore.setState({ + uniboxRailFolded: {}, + uniboxRailHidden: [], + uniboxRailOrder: {}, + uniboxRailSectionOrder: [], + }); +}); + +afterEach(() => { + cleanup(); +}); + +describe("ScopeRail sections", () => { + it("starts expanded with nothing hidden and a Mail header over the mail rows", () => { + mountRail(); + expect(screen.getByRole("button", { name: "Mail", expanded: true })).toBeTruthy(); + expect(screen.getByRole("button", { name: "Views", expanded: true })).toBeTruthy(); + for (const label of ["All mail", "Inbox", "Spam", "Trash", "Scheduled"]) { + expect(screen.getByText(label)).toBeTruthy(); + } + }); + + it("folds a section, hides its rows and remembers it in the persisted store", () => { + mountRail(); + fireEvent.click(screen.getByRole("button", { name: MAIL_TOGGLE })); + + expect(useAppStore.getState().uniboxRailFolded.mail).toBe(true); + expect(screen.getByRole("button", { name: MAIL_TOGGLE, expanded: false })).toBeTruthy(); + expect(screen.queryByText("Spam")).toBeNull(); + expect(screen.queryByText("Inbox")).toBeNull(); + + // Part of what the store writes to storage, not just in-memory state. + const persisted = useAppStore.persist.getOptions().partialize?.(useAppStore.getState()) as + | { uniboxRailFolded?: Record } + | undefined; + expect(persisted?.uniboxRailFolded).toEqual({ mail: true }); + + fireEvent.click(screen.getByRole("button", { name: MAIL_TOGGLE })); + expect(screen.getByText("Spam")).toBeTruthy(); + }); + + it("keeps the row you are on when the section is folded", () => { + useAppStore.setState({ uniboxRailFolded: { mail: true } }); + mountRail({ kind: "folder", folder: "spam" }); + expect(screen.getByText("Spam")).toBeTruthy(); + expect(screen.queryByText("Inbox")).toBeNull(); + }); + + it("keeps the active mailbox visible in a folded Mailboxes section", () => { + useAppStore.setState({ uniboxRailFolded: { mailboxes: true } }); + mountRail({ kind: "mailbox", mailboxId: "m1" }); + expect(screen.getByText("me@example.com")).toBeTruthy(); + + cleanup(); + mountRail(); + expect(screen.queryByText("me@example.com")).toBeNull(); + }); + + it("flags a highlighted count folded out of sight with a dot, and drops it when the section opens", () => { + useAppStore.setState({ uniboxRailFolded: { mail: true } }); + mountRail({ kind: "folder", folder: "spam" }); + expect(screen.getByRole("button", { name: "Mail, highlighted count folded away" })).toBeTruthy(); + + fireEvent.click(screen.getByRole("button", { name: MAIL_TOGGLE })); + expect(screen.queryByText(DOT)).toBeNull(); + }); + + it("says highlighted count, not unread, because Scheduled raises the dot too", () => { + const saved = overview.data; + overview.data = { + ...saved, + unread: 0, + scheduled_pending: 5, + folders: [ + { folder: "inbox", unread: 0, total: 9 }, + { folder: "spam", unread: 0, total: 1 }, + ], + }; + try { + useAppStore.setState({ uniboxRailFolded: { mail: true } }); + mountRail({ kind: "folder", folder: "spam" }); + expect(screen.getByText(DOT)).toBeTruthy(); + expect(screen.queryByText(/unread/i)).toBeNull(); + } finally { + overview.data = saved; + } + }); + + it("does not raise the dot for rows the user hid", () => { + useAppStore.setState({ + uniboxRailFolded: { mail: true }, + uniboxRailHidden: ["folder:inbox", "unread"], + }); + mountRail({ kind: "folder", folder: "spam" }); + expect(screen.queryByText(DOT)).toBeNull(); + }); + + it("marks the row you are on with aria-current", () => { + mountRail({ kind: "unread" }); + const current = document.querySelectorAll("[aria-current]"); + expect(current).toHaveLength(1); + expect(current[0].textContent).toContain("Unread"); + }); + + it("folds Views on its own", () => { + mountRail(); + expect(screen.getByText("Hot leads")).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { name: "Views" })); + expect(useAppStore.getState().uniboxRailFolded.views).toBe(true); + expect(screen.queryByText("Hot leads")).toBeNull(); + // Another section is untouched. + expect(screen.getByText("Inbox")).toBeTruthy(); + }); +}); + +describe("ScopeRail edit mode", () => { + it("hides a row through Edit, uncheck, Done", () => { + mountRail(); + startEditing("Mail"); + + // Every row is a checkbox, all checked to start with. + const spam = screen.getByRole("checkbox", { name: "Spam" }); + expect(spam.getAttribute("aria-checked")).toBe("true"); + fireEvent.click(spam); + expect(screen.getByRole("checkbox", { name: "Spam" }).getAttribute("aria-checked")).toBe("false"); + // Still listed while editing, so it can be turned back on. + expect(screen.getByText("Spam")).toBeTruthy(); + + fireEvent.click(screen.getByRole("button", { name: "Done editing Mail" })); + expect(useAppStore.getState().uniboxRailHidden).toEqual(["folder:spam"]); + expect(screen.queryByText("Spam")).toBeNull(); + expect(screen.getByText("Trash")).toBeTruthy(); + }); + + it("the pencil does not fold the section", () => { + mountRail(); + fireEvent.click(screen.getByRole("button", { name: "Edit Mail rows" })); + expect(useAppStore.getState().uniboxRailFolded.mail).toBeUndefined(); + expect(screen.getByText("Spam")).toBeTruthy(); + // One click enters edit mode: no menu in between. + expect(screen.queryByRole("menuitem")).toBeNull(); + expect(screen.getByRole("button", { name: "Done editing Mail" })).toBeTruthy(); + }); + + it("counts hidden rows beside the pencil, open or folded, and drops the count when they are shown", () => { + mountRail(); + expect(screen.queryByText(/\d+ hidden/)).toBeNull(); + + startEditing("Mail"); + fireEvent.click(screen.getByRole("checkbox", { name: "Spam" })); + fireEvent.click(screen.getByRole("checkbox", { name: "Trash" })); + // Not shown while editing: every row is on screen then. + expect(screen.queryByText("2 hidden")).toBeNull(); + fireEvent.click(screen.getByRole("button", { name: "Done editing Mail" })); + expect(screen.getByText("2 hidden")).toBeTruthy(); + // Tabbing to the pencil hears the number too. + expect( + screen.getByRole("button", { name: "Edit Mail rows", description: "2 hidden" }), + ).toBeTruthy(); + // Views has none of its own. + expect(screen.queryByText("1 hidden")).toBeNull(); + + fireEvent.click(screen.getByRole("button", { name: MAIL_TOGGLE })); + expect(screen.getByText("2 hidden")).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { name: MAIL_TOGGLE })); + + startEditing("Mail"); + fireEvent.click(screen.getByRole("checkbox", { name: "Spam" })); + fireEvent.click(screen.getByRole("checkbox", { name: "Trash" })); + fireEvent.click(screen.getByRole("button", { name: "Done editing Mail" })); + expect(screen.queryByText(/\d+ hidden/)).toBeNull(); + }); + + it("does not navigate while editing and drops the folder menu", () => { + const onChange = vi.fn(); + render(); + expect(screen.getByLabelText("Inbox folder actions")).toBeTruthy(); + + startEditing("Mail"); + fireEvent.click(screen.getByRole("checkbox", { name: "Inbox" })); + expect(onChange).not.toHaveBeenCalled(); + expect(screen.queryByLabelText("Inbox folder actions")).toBeNull(); + }); + + it("shows hidden rows and unfolds the section while editing", () => { + useAppStore.setState({ uniboxRailFolded: { mail: true }, uniboxRailHidden: ["folder:trash"] }); + mountRail(); + expect(screen.queryByText("Trash")).toBeNull(); + + startEditing("Mail"); + expect(screen.getByRole("checkbox", { name: "Trash" }).getAttribute("aria-checked")).toBe("false"); + expect(screen.getByRole("checkbox", { name: "Inbox" })).toBeTruthy(); + }); + + it("ends edit mode on Escape", () => { + mountRail(); + startEditing("Mail"); + expect(screen.getAllByRole("checkbox").length).toBeGreaterThan(0); + + fireEvent.keyDown(screen.getByRole("checkbox", { name: "Inbox" }), { key: "Escape" }); + expect(screen.queryAllByRole("checkbox")).toHaveLength(0); + expect(screen.getByRole("button", { name: "Edit Mail rows" })).toBeTruthy(); + }); + + it("holds the fold while editing, so Done never folds by surprise", () => { + mountRail(); + startEditing("Mail"); + const toggle = screen.getByRole("button", { name: MAIL_TOGGLE }); + expect((toggle as HTMLButtonElement).disabled).toBe(true); + fireEvent.click(toggle); + expect(useAppStore.getState().uniboxRailFolded.mail).toBeUndefined(); + + fireEvent.click(screen.getByRole("button", { name: "Done editing Mail" })); + expect(screen.getByText("Spam")).toBeTruthy(); + }); + + it("moves focus into the checkboxes and back to the options button", () => { + mountRail(); + startEditing("Mail"); + expect(document.activeElement).toBe(screen.getByRole("checkbox", { name: "All mail" })); + + fireEvent.keyDown(document.activeElement as Element, { key: "Escape" }); + expect(document.activeElement).toBe(screen.getByRole("button", { name: "Edit Mail rows" })); + + startEditing("Mail"); + fireEvent.click(screen.getByRole("button", { name: "Done editing Mail" })); + expect(document.activeElement).toBe(screen.getByRole("button", { name: "Edit Mail rows" })); + }); + + it("edits Views separately from Mail", () => { + mountRail(); + startEditing("Views"); + fireEvent.click(screen.getByRole("checkbox", { name: "Hot leads" })); + fireEvent.click(screen.getByRole("button", { name: "Done editing Views" })); + expect(useAppStore.getState().uniboxRailHidden).toEqual(["view:hot"]); + expect(screen.queryByText("Hot leads")).toBeNull(); + expect(screen.getByText("Follow up")).toBeTruthy(); + }); + + it("keeps the header, and so Edit, when every row is hidden", () => { + useAppStore.setState({ + uniboxRailHidden: [ + "all", "folder:inbox", "unread", "awaiting", "agent_drafts", "snoozed", + "folder:drafts", "folder:sent", "scheduled", "folder:archive", "folder:spam", "folder:trash", + ], + }); + mountRail({ kind: "mailbox", mailboxId: "m1" }); + expect(screen.queryByText("Inbox")).toBeNull(); + expect(screen.getByRole("button", { name: "Edit Mail rows" })).toBeTruthy(); + }); +}); + +describe("hidden rows", () => { + it("never hide the active scope", () => { + useAppStore.setState({ uniboxRailHidden: ["folder:spam"] }); + mountRail({ kind: "folder", folder: "spam" }); + expect(screen.getByText("Spam")).toBeTruthy(); + + cleanup(); + mountRail({ kind: "all" }); + expect(screen.queryByText("Spam")).toBeNull(); + }); +}); + +describe("sanitizeUniboxRailHidden", () => { + it("drops anything that is not a string and removes duplicates", () => { + expect(sanitizeUniboxRailHidden(["folder:spam", 4, null, {}, "folder:spam", "view:hot"])).toEqual([ + "folder:spam", + "view:hot", + ]); + expect(sanitizeUniboxRailHidden("folder:spam")).toEqual([]); + expect(sanitizeUniboxRailHidden(undefined)).toEqual([]); + }); + + it("runs on rehydration, where the setters are bypassed", async () => { + const original = useAppStore.persist.getOptions().storage; + useAppStore.persist.setOptions({ + storage: { + getItem: () => + ({ + state: { + uniboxRailHidden: ["folder:spam", 7, "folder:spam"], + uniboxRailFolded: { mail: true, views: "yes" }, + }, + }) as never, + setItem: () => {}, + removeItem: () => {}, + }, + }); + try { + await useAppStore.persist.rehydrate(); + expect(useAppStore.getState().uniboxRailHidden).toEqual(["folder:spam"]); + expect(useAppStore.getState().uniboxRailFolded).toEqual({ mail: true }); + } finally { + useAppStore.persist.setOptions({ storage: original }); + } + }); +}); + +const MAIL_DEFAULT = [ + "all", "folder:inbox", "unread", "awaiting", "agent_drafts", "snoozed", + "folder:drafts", "folder:sent", "scheduled", "folder:archive", "folder:spam", "folder:trash", +]; + +// The rail row element around a label, as the keyboard sees it. +const rowOf = (label: string) => screen.getByText(label).closest("[data-rail-row]") as HTMLElement; + +// Headers in the order they are on screen. +const headerOrder = () => + screen.getAllByRole("button", { expanded: true }).map((b) => b.textContent?.replace(/\d+$/, "")); + +describe("row order", () => { + it("moves a row with the grip's arrow keys while editing, and says where it went", () => { + mountRail(); + startEditing("Mail"); + fireEvent.keyDown(screen.getByRole("button", { name: /^Move Spam/ }), { key: "ArrowUp" }); + + const order = useAppStore.getState().uniboxRailOrder.mail; + expect(order.indexOf("folder:spam")).toBe(order.indexOf("folder:archive") - 1); + expect(screen.getByText("Spam moved to position 10 of 12")).toBeTruthy(); + + // Checkboxes follow the new order. + const names = screen.getAllByRole("checkbox").map((c) => c.textContent); + expect(names.indexOf("Spam")).toBeLessThan(names.indexOf("Archive")); + }); + + it("moves a row with Alt+arrow outside edit mode, stepping over a hidden neighbour", () => { + useAppStore.setState({ uniboxRailHidden: ["unread"] }); + mountRail(); + fireEvent.keyDown(rowOf("Inbox"), { key: "ArrowDown", altKey: true }); + expect(useAppStore.getState().uniboxRailOrder.mail.slice(0, 4)).toEqual([ + "all", "unread", "awaiting", "folder:inbox", + ]); + }); + + it("stores nothing once a row is moved back to where it started", () => { + mountRail(); + fireEvent.keyDown(rowOf("Inbox"), { key: "ArrowDown", altKey: true }); + expect(useAppStore.getState().uniboxRailOrder.mail).toBeDefined(); + fireEvent.keyDown(rowOf("Inbox"), { key: "ArrowUp", altKey: true }); + expect(useAppStore.getState().uniboxRailOrder.mail).toBeUndefined(); + }); + + it("renders a stored order", () => { + useAppStore.setState({ uniboxRailOrder: { mail: ["folder:trash", ...MAIL_DEFAULT.slice(0, -1)] } }); + mountRail(); + const rows = Array.from(document.querySelectorAll("[data-rail-row]")).map((r) => r.textContent); + expect(rows[0]).toContain("Trash"); + }); + + it("resets order and hidden rows from the edit footer", () => { + useAppStore.setState({ + uniboxRailHidden: ["folder:spam", "view:hot"], + uniboxRailOrder: { mail: ["folder:trash", ...MAIL_DEFAULT.slice(0, -1)] }, + }); + mountRail(); + startEditing("Mail"); + fireEvent.click(screen.getByRole("button", { name: "Reset" })); + expect(useAppStore.getState().uniboxRailOrder.mail).toBeUndefined(); + // Only this section's rows come back. + expect(useAppStore.getState().uniboxRailHidden).toEqual(["view:hot"]); + expect((screen.getByRole("button", { name: "Reset" }) as HTMLButtonElement).disabled).toBe(true); + }); +}); + +describe("row menu", () => { + it("hides a row from its menu", () => { + mountRail(); + fireEvent.click(screen.getByLabelText("Spam folder actions")); + fireEvent.click(screen.getByRole("menuitem", { name: "Hide from rail" })); + expect(useAppStore.getState().uniboxRailHidden).toEqual(["folder:spam"]); + expect(screen.queryByText("Spam")).toBeNull(); + }); + + it("opens the same menu on right-click, without opening the scope", () => { + const onChange = vi.fn(); + render(); + fireEvent.contextMenu(rowOf("Unread"), { clientX: 40, clientY: 80 }); + expect(screen.getByRole("menuitem", { name: /Move up/ })).toBeTruthy(); + expect(onChange).not.toHaveBeenCalled(); + }); + + it("disables moves the row cannot make", () => { + mountRail(); + fireEvent.click(screen.getByLabelText("All mail actions")); + expect((screen.getByRole("menuitem", { name: /Move up/ }) as HTMLButtonElement).disabled).toBe(true); + expect((screen.getByRole("menuitem", { name: /Move down/ }) as HTMLButtonElement).disabled).toBe(false); + }); + + it("walks its items with the arrow keys", async () => { + mountRail(); + fireEvent.click(screen.getByLabelText("Inbox folder actions")); + const menu = screen.getByRole("menu"); + fireEvent.keyDown(menu, { key: "ArrowDown" }); + expect(document.activeElement?.textContent).toBe("Mark all as read"); + fireEvent.keyDown(document.activeElement as Element, { key: "ArrowDown" }); + expect(document.activeElement?.textContent).toContain("Move up"); + fireEvent.keyDown(document.activeElement as Element, { key: "End" }); + expect(document.activeElement?.textContent).toContain("Edit Mail rows"); + }); +}); + +describe("section menu", () => { + it("moves a section and remembers it", () => { + mountRail(); + expect(headerOrder().slice(0, 2)).toEqual(["Mail", "Views"]); + fireEvent.click(screen.getByLabelText("Mail section options")); + fireEvent.click(screen.getByRole("menuitem", { name: "Move section down" })); + expect(useAppStore.getState().uniboxRailSectionOrder.slice(0, 2)).toEqual(["views", "mail"]); + expect(headerOrder().slice(0, 2)).toEqual(["Views", "Mail"]); + }); + + it("folds every other section", () => { + mountRail(); + fireEvent.click(screen.getByLabelText("Views section options")); + fireEvent.click(screen.getByRole("menuitem", { name: "Fold other sections" })); + const folded = useAppStore.getState().uniboxRailFolded; + expect(folded).toMatchObject({ mail: true, views: false, mailboxes: true, labels: true }); + expect(screen.getByText("Hot leads")).toBeTruthy(); + expect(screen.queryByText("Inbox")).toBeNull(); + }); + + it("offers to show the hidden rows", () => { + useAppStore.setState({ uniboxRailHidden: ["folder:spam", "folder:trash"] }); + mountRail(); + fireEvent.click(screen.getByLabelText("Mail section options")); + fireEvent.click(screen.getByRole("menuitem", { name: "Show 2 hidden rows" })); + expect(useAppStore.getState().uniboxRailHidden).toEqual([]); + expect(screen.getByText("Spam")).toBeTruthy(); + }); + + it("starts editing from the menu", () => { + mountRail(); + fireEvent.click(screen.getByLabelText("Views section options")); + fireEvent.click(screen.getByRole("menuitem", { name: "Edit rows…" })); + expect(screen.getByRole("checkbox", { name: "Hot leads" })).toBeTruthy(); + }); +}); + +describe("edit mode ends", () => { + it("on a click outside the section", () => { + mountRail(); + startEditing("Mail"); + fireEvent.mouseDown(screen.getByText("me@example.com")); + expect(screen.queryAllByRole("checkbox")).toHaveLength(0); + }); + + it("not on a click inside it", () => { + mountRail(); + startEditing("Mail"); + fireEvent.mouseDown(screen.getByRole("checkbox", { name: "Spam" })); + expect(screen.getAllByRole("checkbox").length).toBeGreaterThan(0); + }); +}); + +describe("arrow keys between rows", () => { + it("move focus down the rail and across sections", () => { + mountRail(); + rowOf("All mail").focus(); + fireEvent.keyDown(rowOf("All mail"), { key: "ArrowDown" }); + expect(document.activeElement).toBe(rowOf("Inbox")); + fireEvent.keyDown(document.activeElement as Element, { key: "End" }); + expect(document.activeElement?.textContent).toContain("Interested"); + }); +}); + +describe("applyRailOrder", () => { + it("keeps the stored order, drops keys that are gone, and slots new ones after their default neighbour", () => { + expect(applyRailOrder(["a", "b", "c"], ["c", "a", "b"])).toEqual(["c", "a", "b"]); + expect(applyRailOrder(["a", "b"], ["b", "gone", "a"])).toEqual(["b", "a"]); + expect(applyRailOrder(["a", "b", "new", "c"], ["c", "b", "a"])).toEqual(["c", "b", "new", "a"]); + expect(applyRailOrder(["first", "a"], ["a"])).toEqual(["first", "a"]); + expect(applyRailOrder(["a"], undefined)).toEqual(["a"]); + }); +}); + +describe("sanitizeUniboxRailOrder", () => { + it("keeps string lists only, deduplicated", () => { + expect(sanitizeUniboxRailOrder({ mail: ["a", "a", 3, "b"], views: "x", tags: [] })).toEqual({ + mail: ["a", "b"], + tags: [], + }); + expect(sanitizeUniboxRailOrder(["mail"])).toEqual({}); + }); + + it("is persisted with the section order", () => { + useAppStore.setState({ uniboxRailOrder: { mail: ["unread"] }, uniboxRailSectionOrder: ["views"] }); + const persisted = useAppStore.persist.getOptions().partialize?.(useAppStore.getState()) as { + uniboxRailOrder?: unknown; + uniboxRailSectionOrder?: unknown; + }; + expect(persisted.uniboxRailOrder).toEqual({ mail: ["unread"] }); + expect(persisted.uniboxRailSectionOrder).toEqual(["views"]); + }); +}); + +describe("keyboard moves keep their place", () => { + // Browsers blur a focused node that is moved in the DOM (React restores it); jsdom does not, so mimic it. + const insertBefore = Node.prototype.insertBefore; + beforeEach(() => { + Node.prototype.insertBefore = function (this: Node, node: T, child: Node | null): T { + const focused = document.activeElement; + if (node.isConnected && focused && node.contains(focused)) (focused as HTMLElement).blur(); + return insertBefore.call(this, node, child) as T; + }; + }); + afterEach(() => { + Node.prototype.insertBefore = insertBefore; + }); + + it("keeps focus on the row it moved", () => { + mountRail(); + rowOf("Inbox").focus(); + fireEvent.keyDown(rowOf("Inbox"), { key: "ArrowDown", altKey: true }); + expect(document.activeElement).toBe(rowOf("Inbox")); + fireEvent.keyDown(rowOf("Inbox"), { key: "ArrowDown", altKey: true }); + expect(useAppStore.getState().uniboxRailOrder.mail.slice(0, 4)).toEqual([ + "all", "unread", "awaiting", "folder:inbox", + ]); + }); + + it("keeps focus on the grip it moved", () => { + mountRail(); + startEditing("Mail"); + const grip = () => screen.getByRole("button", { name: /^Move Inbox/ }); + grip().focus(); + fireEvent.keyDown(grip(), { key: "ArrowDown" }); + expect(document.activeElement).toBe(grip()); + }); + + it("offers no moves in a folded section", () => { + useAppStore.setState({ uniboxRailFolded: { mail: true } }); + mountRail({ kind: "unread" }); + fireEvent.click(screen.getByLabelText("Unread actions")); + expect((screen.getByRole("menuitem", { name: /Move up/ }) as HTMLButtonElement).disabled).toBe(true); + expect((screen.getByRole("menuitem", { name: /Move down/ }) as HTMLButtonElement).disabled).toBe(true); + }); +}); + +describe("dropdown keys", () => { + it("jump to the first matching item from the panel, and never reach global shortcuts", () => { + const globalKeys = vi.fn(); + window.addEventListener("keydown", globalKeys); + try { + mountRail(); + fireEvent.click(screen.getByLabelText("Mail section options")); + fireEvent.keyDown(screen.getByRole("menu"), { key: "f" }); + expect(document.activeElement?.textContent).toBe("Fold section"); + fireEvent.keyDown(document.activeElement as Element, { key: "e" }); + expect(document.activeElement?.textContent).toBe("Edit rows…"); + expect(globalKeys).not.toHaveBeenCalled(); + } finally { + window.removeEventListener("keydown", globalKeys); + } + }); + + it("leave Tab alone", () => { + mountRail(); + fireEvent.click(screen.getByLabelText("Mail section options")); + fireEvent.keyDown(screen.getByRole("menu"), { key: "Tab" }); + expect(screen.getByRole("menu")).toBeTruthy(); + }); +}); diff --git a/web/src/components/layout/AppHeader.tsx b/web/src/components/layout/AppHeader.tsx index 26d734227..aefda2937 100644 --- a/web/src/components/layout/AppHeader.tsx +++ b/web/src/components/layout/AppHeader.tsx @@ -39,7 +39,7 @@ const labelMap: Record = { unibox: "Inbox", contacts: "Contacts", segments: "Segments", - categories: "Categories", + labels: "Labels", campaigns: "Campaigns", analytics: "Analytics", crm: "CRM", diff --git a/web/src/components/layout/AppNav.tsx b/web/src/components/layout/AppNav.tsx index 0d0ea550e..9a63c9330 100644 --- a/web/src/components/layout/AppNav.tsx +++ b/web/src/components/layout/AppNav.tsx @@ -213,11 +213,11 @@ function NavTip({ } // The two row shapes share one element and transition between each other in -// step with the sidebar's width: the icon holds its place (it drifts 3px into -// the rail's centre) while the label column fades and is clipped. -const ROW_BASE = "group relative flex items-center rounded-md text-[12.5px] transition-[margin,width,height,padding,gap,background-color,color] duration-200 ease-out motion-reduce:transition-none"; +// step with the sidebar's width. Margin and padding match, so the icon sits at +// the rail's centre in both and never moves while the label fades and clips. +const ROW_BASE = "group relative flex items-center rounded-md text-[12.5px] transition-[width,height,gap,background-color,color] duration-200 ease-out motion-reduce:transition-none"; const ICON_ROW = `${ROW_BASE} mx-3 w-8 h-8 px-[9px] gap-0`; -const LABEL_ROW = `${ROW_BASE} mx-2 w-[calc(100%-1rem)] h-7 px-2.5 gap-2.5`; +const LABEL_ROW = `${ROW_BASE} mx-3 w-[calc(100%-1.5rem)] h-7 px-[9px] gap-2.5`; const rowClass = (collapsed: boolean) => (collapsed ? ICON_ROW : LABEL_ROW); // Fades out fast on collapse, and back in once the column has room again. @@ -769,7 +769,7 @@ function Section({ onClick={() => toggleNavSection(section.id)} inert={collapsed} className={cn( - "group/section mx-2 flex w-[calc(100%-1rem)] items-center gap-1.5 overflow-hidden whitespace-nowrap rounded-md px-2 text-[10px] font-medium uppercase tracking-[0.14em] text-slate-400 transition-[height,margin,opacity,color] duration-200 ease-out hover:text-slate-700 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-sky-400 motion-reduce:transition-none", + "group/section mx-3 flex w-[calc(100%-1.5rem)] items-center gap-1.5 overflow-hidden whitespace-nowrap rounded-md px-[9px] text-[10px] font-medium uppercase tracking-[0.14em] text-slate-400 transition-[height,margin,opacity,color] duration-200 ease-out hover:text-slate-700 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-sky-400 motion-reduce:transition-none", collapsed ? "mb-0 h-0 opacity-0" : "mb-1 h-6 opacity-100", )} > diff --git a/web/src/components/layout/DynamicBreadcrumb.tsx b/web/src/components/layout/DynamicBreadcrumb.tsx index ead2f1540..6c2f078b2 100644 --- a/web/src/components/layout/DynamicBreadcrumb.tsx +++ b/web/src/components/layout/DynamicBreadcrumb.tsx @@ -6,7 +6,7 @@ const labelMap: Record = { emails: 'Accounts', contacts: 'Contacts', segments: 'Segments', - categories: 'Categories', + labels: 'Labels', campaigns: 'Campaigns', unibox: 'Inbox', analytics: 'Analytics', diff --git a/web/src/components/layout/UserNav.tsx b/web/src/components/layout/UserNav.tsx index 989befaa2..462e0b576 100644 --- a/web/src/components/layout/UserNav.tsx +++ b/web/src/components/layout/UserNav.tsx @@ -47,12 +47,12 @@ export function UserNav({ collapsed = false }: { collapsed?: boolean }) {