diff --git a/cmd/cli/specs.go b/cmd/cli/specs.go index 7b0f6e69..3e6147e5 100644 --- a/cmd/cli/specs.go +++ b/cmd/cli/specs.go @@ -984,6 +984,10 @@ func templateSpec() resource { Name: "score", Short: "Score a draft for deliverability and tone", Method: http.MethodPost, Path: "/templates/score", Body: bodyRequired, }, + { + Name: "analyze", Short: "AI spam analysis of a draft, with the wording to fix", + Method: http.MethodPost, Path: "/templates/analyze", Body: bodyRequired, + }, { Name: "reorder", Short: "Change the order templates appear in", Method: http.MethodPatch, Path: "/templates/reorder", Body: bodyRequired, diff --git a/docs/content/docs/api/endpoints.mdx b/docs/content/docs/api/endpoints.mdx index 79ba6fe8..4e6a7949 100644 --- a/docs/content/docs/api/endpoints.mdx +++ b/docs/content/docs/api/endpoints.mdx @@ -226,6 +226,7 @@ The `agent-drafts` endpoints back the [inbox agent](/guides/inbox-agent/): the a | Method | Path | API Permission | |--------|------|----------------| | GET | `/templates`, `/templates/:id` | `READ_TEMPLATES` | +| POST | `/templates/score`, `/templates/analyze` | `READ_TEMPLATES` | | POST/PATCH/DELETE | `/templates[/:id]` | `WRITE_TEMPLATES` | | GET | `/crm/pipelines`, `/crm/pipelines/:id` | `READ_CRM` | | POST/PATCH/DELETE | `/crm/pipelines[/:id][/stages[/:stageId]]` | `WRITE_CRM` | diff --git a/docs/content/docs/api/error-codes.mdx b/docs/content/docs/api/error-codes.mdx index 57220045..7159b92f 100644 --- a/docs/content/docs/api/error-codes.mdx +++ b/docs/content/docs/api/error-codes.mdx @@ -320,6 +320,23 @@ A `503` whose `code` is `mailbox_provider_not_configured` is not transient and r - Or connect the mailbox over SMTP and IMAP instead, which needs no configuration - Full walkthrough: [connect mailboxes](/development/deployment-guide/#connect-mailboxes) +#### `ai_not_configured` + +A `503` whose `code` is `ai_not_configured` is not transient and retrying will not help. It comes from `POST /templates/analyze` and means the deployment has no AI provider set up at all. It is how a client tells "there is no AI here" apart from "the provider is having a bad minute", which returns the generic `service_unavailable` and is worth retrying. + +```json +{ + "error": "Service Unavailable", + "message": "AI analysis is not configured on this deployment.", + "code": "ai_not_configured", + "request_id": "4bbbd1b2-8f86-47dd-8a7f-9476501ad20e" +} +``` + +**How to fix:** +- Set `AI_PROVIDER` and `AI_API_KEY` in the `.env` at your install root, then restart. See the [configuration reference](/development/configuration/) +- Hide the AI affordance in your client rather than retrying: nothing about the request will make it succeed + #### `mailbox_allowance_reached` A `403` whose `code` is `mailbox_allowance_reached` comes from every path that connects a mailbox: `POST /emails/onboarding/oauth/start`, `POST /emails/onboarding/oauth/finish`, `POST /emails/onboarding/smtp-imap`, and per row inside `POST /emails/onboarding/smtp-imap/bulk`. It is not a permission problem: the workspace holds its whole [mailbox allowance](/guides/mailboxes/#mailbox-allowance), which on a paid plan is one mailbox for every send a day the plan includes, and `10` on a free workspace. Nothing was connected. diff --git a/docs/content/docs/api/reference/deliverability-ops.mdx b/docs/content/docs/api/reference/deliverability-ops.mdx index eb24d0a9..664160f2 100644 --- a/docs/content/docs/api/reference/deliverability-ops.mdx +++ b/docs/content/docs/api/reference/deliverability-ops.mdx @@ -714,15 +714,119 @@ Auth: **Scope** `READ_TEMPLATES` · **Org permission** `view_campaigns` `200 OK` with the score and any advisory issues. +Each issue is located. `field` is `subject` or `body` when the issue lives in exactly one of them, and absent when it straddles both or describes the send as a whole (an attachment count). `spans` quote the exact fragments that caused it, with the 1-based line within that field and the whole line for context. `suggestion` is the fix in one sentence. + ```json { - "score": 92, + "score": 84, "issues": [ { "severity": "warn", - "code": "too_many_links", - "message": "4 links found; keep cold-email link count low." + "code": "spam_trigger_terms", + "message": "2 spam-trigger term(s) found in subject/body: free, limited time.", + "field": "body", + "spans": [ + { + "field": "body", + "text": "free", + "line": 3, + "excerpt": "Here is a free look at what we do." + }, + { + "field": "body", + "text": "limited time", + "line": 5, + "excerpt": "This is a limited time offer." + } + ], + "suggestion": "Rewrite those words in plain language, or cut the sentence they sit in." } ] } ``` + +A `spans` list is capped at eight entries. The count that drives the deduction is taken before that cap, so a message naming fourteen links is consistent with a score that deducted for fourteen. + +## Analyze template content with AI + +`POST /templates/analyze` + +Runs the deployment's configured AI provider over the same subject and body and returns what to change: the specific word or sentence, quoted from the copy, whether it is in the subject or the body, why it hurts, and what to write instead. The rules-based pass runs in the same request and comes back under `rules`, so both halves are scored from one reading of the copy. + +Advisory only, exactly like `/templates/score`: nothing here blocks or delays a send. + +Auth: **Scope** `READ_TEMPLATES` · **Org permission** `view_campaigns` and `use_ai` + +This endpoint spends AI credits (see [AI credits](/guides/ai-credits/)). Send an `Idempotency-Key` header to make a retry safe; the same key is never charged twice. + +### Request body + +Identical to `/templates/score`. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `subject` | string | No | Subject line to analyze. | +| `body_html` | string | No | HTML body (used when `body_plain` is empty). | +| `body_plain` | string | No | Plain-text body, preferred over HTML when present. | + +At least one of the three must carry something, and the three together are capped at 60,000 bytes. + +### Response + +`200 OK` with the analysis. + +```json +{ + "score": 62, + "verdict": "This reads like a promotion rather than a note from a person.", + "findings": [ + { + "severity": "high", + "field": "subject", + "text": "FREE bonus inside", + "line": 1, + "excerpt": "Your FREE bonus inside", + "issue": "Capitalised offer wording in the subject is one of the strongest promotional signals a filter reads.", + "suggestion": "Name the thing you noticed about them instead, in lower case.", + "category": "trigger_word" + } + ], + "suggested_subject": "quick question about your onboarding", + "improvements": [ + "Cut the closing paragraph. One ask lands better than three." + ], + "rules": { + "score": 84, + "issues": [] + }, + "model": "gpt-4o-mini", + "tokens_used": 812, + "credits_remaining": 248, + "credits_charged": 2 +} +``` + +| Field | Description | +|-------|-------------| +| `score` | Overall deliverability score, 0 to 100, higher is safer. Grounded on `rules`, never argued against it. | +| `verdict` | One sentence on how the email will land. | +| `findings` | Located problems, most severe first. `severity` is `high`, `warn` or `info`; `field` is `subject` or `body`; `category` is one of `trigger_word`, `tone`, `formatting`, `links`, `structure`, `authenticity`. | +| `findings[].text` | The fragment quoted from your copy. Absent when the finding is about the email as a whole. | +| `suggested_subject` | A rewritten subject, or absent when the current one is fine. | +| `improvements` | Copy-level advice with no single fragment to quote. | +| `rules` | The `/templates/score` result for the same content. | + + +Every `text` is checked against the submitted subject and body before it is returned. A fragment that is not in the copy is dropped and the finding keeps only its `issue` and `suggestion`, so a response never quotes a sentence you did not send. A finding whose quote was found in the other half has its `field` corrected. + + +The analysis runs at a fixed temperature, so re-analyzing unchanged copy returns the same score. A change in the score means a change in the copy, which is what makes re-checking after an edit worth doing. + +### Errors + +| Status | When | +|--------|------| +| `400` | Nothing written to analyze, or the template is over the size cap. | +| `402` | The workspace is out of AI credits. Nothing is charged and no provider call is made. | +| `403` | The plan or trial does not cover AI, or the member lacks `use_ai`. | +| `503` | No AI provider is configured on this deployment (`code` is `ai_not_configured`, which is permanent for that deployment), or the provider failed (`code` is `service_unavailable`, and the reserved credits are refunded). | diff --git a/docs/content/docs/guides/ai-credits.mdx b/docs/content/docs/guides/ai-credits.mdx index b42ae8f8..b05b0169 100644 --- a/docs/content/docs/guides/ai-credits.mdx +++ b/docs/content/docs/guides/ai-credits.mdx @@ -27,6 +27,7 @@ Each action reserves a flat **minimum** up front (which also gates an empty bala | Writing assistant draft, selection edit ("Edit with AI") | 1 | | AI assistant step | 1 per turn | | Reply draft, compose draft, contact research run | 2 | +| AI spam analysis of a campaign step, and each Re-check | 2 | | Single-shot automation AI step (classify, extract, generate) | 1 | | Automation AI agent step | 1 per step | | AI switch or campaign switch step, AI decider (value decider is free) | 1 | diff --git a/docs/content/docs/guides/campaigns.mdx b/docs/content/docs/guides/campaigns.mdx index c320af56..3c47ebd4 100644 --- a/docs/content/docs/guides/campaigns.mdx +++ b/docs/content/docs/guides/campaigns.mdx @@ -225,11 +225,13 @@ If scheduling stalls on a transient infrastructure hiccup, Warmbly re-seeds with Warmbly scores each step's copy out of 100 for the signals spam filters weight: trigger wording, stacked punctuation, ALL-CAPS subjects, link and image counts, image-only bodies, empty or oversized bodies, and attachments. +Every issue says **where** it is and **what** caused it: the badge names the subject or the body (with the line number for a body issue), the offending words are quoted back exactly as you typed them, the line they sit in is shown for context, and each one carries the fix in a sentence. You never have to hunt through your own copy for a word the checker already found. + You see it in three places: - **In the editor**, re-scored as you write a step. -- **In the launch dialog**, alongside the other pre-send checks, with the lowest-scoring step named. Only email steps are scored; wait and action steps carry no copy. -- **In the campaign activity feed**, if the copy that actually goes out scores below your floor. +- **In the launch dialog**, alongside the other pre-send checks, with the lowest-scoring step named and its worst issue quoted as `Subject:` or `Body:`. Only email steps are scored; wait and action steps carry no copy. +- **In the campaign activity feed**, if the copy that actually goes out scores below your floor, again naming the half of the message at fault. That last one catches what the first two cannot. The editor scores the template; the send path scores the message after merge fields, spintax, A/B selection and AI blocks have resolved, which is where a clean template becomes "Hi ," or picks the one spammy spintax branch. It logs once per step per day, not once per recipient. @@ -239,6 +241,29 @@ That last one catches what the first two cannot. The editor scores the template; A low score never blocks or delays a send. It is a signal to rewrite, not a verdict: a legitimate email can score badly, and a well-scoring one sent to a bad list will still fail. +### Analyze with AI + +The rules above catch what rules can catch. **Analyze with AI**, under the score in the step editor, reads the copy itself and reports what a filter and a human reader will object to: + +- the specific word, phrase or sentence, quoted from your draft, and whether it is in the subject or the body +- why that fragment hurts deliverability or replies +- what to write instead +- a rewritten subject line, which you can apply in one click +- copy-level advice that has no single fragment to point at +- an overall deliverability score out of 100 + +It runs on the AI provider configured for the deployment (see [AI credits](/guides/ai-credits/) for the cost, and [Configuration](/development/configuration/) for self-host setup). It needs the **Use AI** permission. A deployment with no provider configured says so in place of the button rather than offering something that cannot run. + +Every quote is checked against your draft before it is shown. A fragment the model could not have copied from your copy is dropped rather than displayed, so the panel never points at a sentence you did not write. + +### Re-check + +**Re-check** re-scores the copy on the spot instead of waiting for the editor's debounce, and once you have run an AI analysis it re-runs that too. Edit what the findings told you to edit, press it, and the panel shows the new score against the previous one: `+14 since your last check`. + +The AI score is deliberately not sampled, so re-checking copy you did not touch returns the same number. A change in the score is a change in the copy. + +When the draft has moved on from the analysis on screen, the panel says so and highlights the button, rather than presenting stale findings as current. + ## Easing out of warmup A mailbox that has been warming does not jump straight to its full cold cap. Going from 40 warmup emails a day to 50 cold emails the next morning is the volume jump mailbox providers penalise, so Warmbly ramps into it. diff --git a/docs/public/openapi.json b/docs/public/openapi.json index 7a0ca06e..aba8dcc4 100644 --- a/docs/public/openapi.json +++ b/docs/public/openapi.json @@ -17511,6 +17511,114 @@ } } }, + "/templates/analyze": { + "post": { + "operationId": "deliverability-ops_analyze_template", + "summary": "Analyze template content with AI", + "description": "Runs the deployment's configured AI provider over a subject and body and returns the specific words and sentences that hurt deliverability, quoted from the copy, with whether each one is in the subject or the body and what to write instead. The rules-based score runs in the same request and comes back under `rules`. Advisory only and never blocks sending. Spends AI credits; honors `Idempotency-Key`. Requires READ_TEMPLATES scope, and the view_campaigns and use_ai permissions.", + "tags": [ + "deliverability-ops" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "Idempotency-Key", + "in": "header", + "required": false, + "description": "Makes a retry safe: the same key is never charged twice.", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ScoreTemplateRequest" + } + } + } + }, + "responses": { + "200": { + "description": "The analysis, with the rules-based pass alongside it.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TemplateAnalysis" + } + } + } + }, + "400": { + "description": "Nothing written to analyze, or the template is over the size cap", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "402": { + "description": "Out of AI credits. Nothing is charged and no provider call is made.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "403": { + "description": "Forbidden", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "429": { + "description": "Rate limited", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + }, + "503": { + "description": "No AI provider configured (code `ai_not_configured`, permanent for that deployment), or the provider failed (code `service_unavailable`, and the reserved credits are refunded).", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } + }, "/templates/score": { "post": { "operationId": "deliverability-ops_score_template", @@ -28904,6 +29012,35 @@ } } }, + "TemplateScoreSpan": { + "type": "object", + "description": "One exact fragment of the copy that triggered an issue.", + "required": [ + "field", + "text" + ], + "properties": { + "field": { + "type": "string", + "enum": [ + "subject", + "body" + ] + }, + "text": { + "type": "string", + "description": "The fragment as it is written in the copy." + }, + "line": { + "type": "integer", + "description": "1-based line within that field." + }, + "excerpt": { + "type": "string", + "description": "The whole line, for context around the fragment." + } + } + }, "TemplateScoreIssue": { "type": "object", "required": [ @@ -28925,6 +29062,136 @@ }, "message": { "type": "string" + }, + "field": { + "type": "string", + "description": "Set when the issue lives in exactly one half of the template. Absent when it straddles both or describes the send as a whole.", + "enum": [ + "subject", + "body" + ] + }, + "spans": { + "type": "array", + "description": "The exact fragments that caused the issue, in reading order. Capped at 8; the count that drives the deduction is taken before the cap.", + "items": { + "$ref": "#/components/schemas/TemplateScoreSpan" + } + }, + "suggestion": { + "type": "string", + "description": "The fix, in one sentence." + } + } + }, + "SpamFinding": { + "type": "object", + "description": "One located problem the AI analysis found. `text` is verified against the submitted copy before it is returned, so it is never a fragment the caller did not send.", + "required": [ + "severity", + "field", + "issue" + ], + "properties": { + "severity": { + "type": "string", + "enum": [ + "high", + "warn", + "info" + ] + }, + "field": { + "type": "string", + "enum": [ + "subject", + "body" + ] + }, + "text": { + "type": "string", + "description": "The fragment quoted from the copy. Absent when the finding is about the email as a whole." + }, + "line": { + "type": "integer" + }, + "excerpt": { + "type": "string" + }, + "issue": { + "type": "string", + "description": "Why this fragment hurts deliverability or replies." + }, + "suggestion": { + "type": "string", + "description": "What to write instead." + }, + "category": { + "type": "string", + "enum": [ + "trigger_word", + "tone", + "formatting", + "links", + "structure", + "authenticity" + ] + } + } + }, + "TemplateAnalysis": { + "type": "object", + "description": "AI spam analysis of a campaign template, with the rules-based pass alongside it.", + "required": [ + "score", + "verdict", + "findings", + "rules", + "model" + ], + "properties": { + "score": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "description": "Overall deliverability score (higher is safer), grounded on the rules pass." + }, + "verdict": { + "type": "string", + "description": "One sentence on how the email will land." + }, + "findings": { + "type": "array", + "description": "Located problems, most severe first.", + "items": { + "$ref": "#/components/schemas/SpamFinding" + } + }, + "suggested_subject": { + "type": "string", + "description": "A rewritten subject, absent when the current one is fine." + }, + "improvements": { + "type": "array", + "description": "Copy-level advice with no single fragment to quote.", + "items": { + "type": "string" + } + }, + "rules": { + "$ref": "#/components/schemas/TemplateScoreResult" + }, + "model": { + "type": "string" + }, + "tokens_used": { + "type": "integer" + }, + "credits_remaining": { + "type": "integer" + }, + "credits_charged": { + "type": "integer" } } }, diff --git a/internal/api/handler/template_analyze.go b/internal/api/handler/template_analyze.go new file mode 100644 index 00000000..ea63d73e --- /dev/null +++ b/internal/api/handler/template_analyze.go @@ -0,0 +1,145 @@ +// AI spam analysis for a campaign template. The rules-based score +// (POST /templates/score) says how safe the copy is; this says which word, +// which sentence, and whether it is in the subject or the body, and what to +// write instead. Flow matches every other metered AI endpoint: gate the org, +// reserve credits before the provider call, refund on a provider failure, +// settle real token usage after. +// +// The rules pass runs in the same request and comes back under "rules", so the +// two halves of the editor's panel are scored from one reading of the copy +// rather than from two drafts a keystroke apart. +package handler + +import ( + "net/http" + "strings" + + "github.com/gin-gonic/gin" + + "github.com/warmbly/warmbly/internal/api/middleware" + "github.com/warmbly/warmbly/internal/app/credits" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/pkg/spamcheck" + "github.com/warmbly/warmbly/internal/pkg/warmlint" +) + +// analyzeMaxLen bounds the template a single request may analyze. Well past any +// cold email; a body beyond it is a pasted newsletter, not outreach. +const analyzeMaxLen = 60000 + +type analyzeTemplateRequest struct { + Subject string `json:"subject"` + BodyHTML string `json:"body_html"` + BodyPlain string `json:"body_plain"` +} + +// AnalyzeTemplateContent — POST /templates/analyze +func (h *Handler) AnalyzeTemplateContent(c *gin.Context) { + orgID := middleware.GetOrganizationID(c) + if orgID == nil { + errx.JSON(c, errx.New(errx.BadRequest, "no organization selected")) + return + } + + var req analyzeTemplateRequest + if err := c.ShouldBindJSON(&req); err != nil { + errx.JSON(c, errx.New(errx.BadRequest, "invalid request body")) + return + } + if len(req.Subject)+len(req.BodyHTML)+len(req.BodyPlain) > analyzeMaxLen { + errx.JSON(c, errx.New(errx.BadRequest, "this template is too long to analyze")) + return + } + if strings.TrimSpace(req.Subject) == "" && strings.TrimSpace(req.BodyPlain) == "" && strings.TrimSpace(req.BodyHTML) == "" { + errx.JSON(c, errx.New(errx.BadRequest, "there is nothing written to analyze yet")) + return + } + + if h.AIProvider == nil { + // Identified, because "no provider here" is permanent for this + // deployment while a provider outage is not, and the editor hides the + // button only for the first. + errx.JSON(c, errx.NewWithIdentifier(errx.ServiceUnavailable, "ai_not_configured", + "AI analysis is not configured on this deployment.")) + return + } + allowed, xerr := h.FeatureGateService.CanUseWritingAssistant(c.Request.Context(), *orgID) + if xerr != nil { + errx.JSON(c, xerr) + return + } + if !allowed { + errx.JSON(c, errx.New(errx.Forbidden, "AI spam analysis requires an active plan or trial.")) + return + } + + rules := warmlint.Score(req.Subject, req.BodyHTML, req.BodyPlain) + + paid, _ := h.FeatureGateService.IsPaidOrganization(c.Request.Context(), *orgID) + model := h.AIProvider.ModelForTier(paid) + + idemKey := strings.TrimSpace(c.GetHeader("Idempotency-Key")) + local := h.AIProvider.IsLocal() + reqCtx := c.Request.Context() + { + meta := models.CreditMeta{Context: models.CreditContext{Detail: "spam analysis"}} + if actor, aerr := middleware.GetUserUUID(c); aerr == nil { + meta.ActorID = actor + } + reqCtx = models.WithCreditMeta(reqCtx, meta) + } + + var remaining int + if local { + if bal, berr := h.CreditService.GetBalance(reqCtx, *orgID); berr == nil { + remaining = bal + } + } else { + var cerr error + remaining, cerr = h.CreditService.Consume(reqCtx, *orgID, credits.CostSpamAnalysis, "spam_analysis", model, 0, idemKey) + if cerr != nil { + mapCreditError(c, cerr) + return + } + } + + analysis, aerr := spamcheck.Analyze(c.Request.Context(), h.AIProvider, model, spamcheck.Input{ + Subject: req.Subject, + BodyHTML: req.BodyHTML, + BodyPlain: req.BodyPlain, + Rules: rules, + Voice: h.orgVoice(c.Request.Context(), *orgID, ""), + }) + if aerr != nil { + if !local { + if bal, rerr := h.CreditService.Grant(reqCtx, *orgID, credits.CostSpamAnalysis, "spam_analysis_refund"); rerr == nil { + remaining = bal + } + } + errx.JSON(c, errx.New(errx.ServiceUnavailable, "The spam analyzer is temporarily unavailable. Your credits were not charged.")) + return + } + + charged := 0 + if !local { + charged = credits.CostSpamAnalysis + if extra, serr := h.CreditService.SettleUsage(reqCtx, *orgID, credits.CostSpamAnalysis, analysis.Model, analysis.TokensUsed, "spam_analysis", settleKey(idemKey)); serr == nil && extra > 0 { + remaining -= extra + charged += extra + } + } + + c.JSON(http.StatusOK, gin.H{ + "score": analysis.Score, + "verdict": analysis.Verdict, + "findings": analysis.Findings, + "suggested_subject": analysis.SuggestedSubject, + "improvements": analysis.Improvements, + "rules": rules, + "model": analysis.Model, + "tokens_used": analysis.TokensUsed, + "credits_remaining": remaining, + "credits_charged": charged, + }) +} diff --git a/internal/api/routes.go b/internal/api/routes.go index 067d1bff..14e454d8 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -1051,6 +1051,10 @@ func Run( templates.POST("/:id/duplicate", m.RequireAccess(models.PermManageCampaigns, models.APIPermWriteTemplates), h.DuplicateTemplate) templates.POST("/:id/render", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadTemplates), h.RenderTemplate) templates.POST("/score", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadTemplates), h.ScoreTemplateContent) + // The AI half of the same check. It spends credits, so the + // use-AI member gate layers on top for JWT callers exactly as + // it does on /generation. + templates.POST("/analyze", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadTemplates), m.RequireAccess(models.PermUseAI, models.APIPermReadTemplates), h.AnalyzeTemplateContent) } // Workspace image library for email bodies. The bytes are public diff --git a/internal/app/advanced/service.go b/internal/app/advanced/service.go index 8452e940..744d2ec8 100644 --- a/internal/app/advanced/service.go +++ b/internal/app/advanced/service.go @@ -2042,16 +2042,7 @@ func worstStepContentScore(seqs []models.Sequence, attachmentsFor func(models.Se if r.Score >= worst { continue } - worst, worstStep, issue = r.Score, seq.Position+1, "" - for _, is := range r.Issues { - if is.Severity == "high" { - issue = is.Message - break - } - } - if issue == "" && len(r.Issues) > 0 { - issue = r.Issues[0].Message - } + worst, worstStep, issue = r.Score, seq.Position+1, warmlint.LeadIssue(r) } return worst, worstStep, issue, scored } diff --git a/internal/app/credits/costs.go b/internal/app/credits/costs.go index 53a7753c..e6b921a4 100644 --- a/internal/app/credits/costs.go +++ b/internal/app/credits/costs.go @@ -32,6 +32,10 @@ const ( // CostWebSearch is one web-search lookup made on behalf of an AI step // (charged only when the search returned results). CostWebSearch = 1 + + // CostSpamAnalysis is one AI spam analysis of a campaign template + // (POST /templates/analyze, and every Re-check after an edit). + CostSpamAnalysis = 2 ) // Usage-based metering. The per-feature constants above are the up-front diff --git a/internal/pkg/spamcheck/spamcheck.go b/internal/pkg/spamcheck/spamcheck.go new file mode 100644 index 00000000..66a8331d --- /dev/null +++ b/internal/pkg/spamcheck/spamcheck.go @@ -0,0 +1,442 @@ +// Package spamcheck is the AI half of the campaign content check. warmlint +// scores the rules everyone agrees on (trigger terms, ALL-CAPS, link and image +// counts); this reads the copy with the deployment's configured LLM and says +// which sentence a filter will object to, in the subject or in the body, and +// what to write instead. +// +// Two things make its output usable rather than decorative. Every finding must +// quote the copy verbatim, and a quote that is not actually in the template is +// dropped rather than shown, so the panel can never point at a sentence the +// writer did not write. And the run is deterministic: re-checking unchanged +// copy has to return the same score, or "did my edit help?" cannot be answered. +package spamcheck + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + + "github.com/warmbly/warmbly/internal/pkg/generation" + "github.com/warmbly/warmbly/internal/pkg/mailhtml" + "github.com/warmbly/warmbly/internal/pkg/warmlint" +) + +// Bounds on what goes to the provider and what comes back, so one pasted +// newsletter cannot drive a very large completion or a very large response. +const ( + maxBodyRunes = 8000 + maxSubjectRunes = 400 + maxFindings = 12 + maxImprovements = 6 + maxTextRunes = 200 + excerptMax = 180 + completionMax = 1600 +) + +// Input is one template to analyze. +type Input struct { + Subject string + BodyHTML string + BodyPlain string + // Rules is the heuristic pass. It is given to the model as grounding so the + // two halves of the panel never contradict each other. + Rules warmlint.ScoreResult + // Voice is the workspace's product and tone context, which changes what + // counts as a plausible claim rather than a spam-flavoured one. + Voice generation.VoiceContext +} + +// Finding is one located problem with the copy. +type Finding struct { + // Severity is "high", "warn" or "info". + Severity string `json:"severity"` + // Field is "subject" or "body". + Field string `json:"field"` + // Text is the exact fragment quoted from the copy, empty when the finding + // is about the email as a whole. + Text string `json:"text,omitempty"` + // Line is the 1-based line of that field the fragment sits on. + Line int `json:"line,omitempty"` + // Excerpt is the whole line, for context around the fragment. + Excerpt string `json:"excerpt,omitempty"` + // Issue is why it hurts deliverability. + Issue string `json:"issue"` + // Suggestion is what to write instead. + Suggestion string `json:"suggestion,omitempty"` + // Category is a coarse grouping ("trigger_word", "tone", "formatting", + // "links", "structure", "authenticity"). + Category string `json:"category,omitempty"` +} + +// Result is the analysis returned to the editor. +type Result struct { + // Score is the overall 0-100 deliverability score (higher is safer), + // grounded on the rules pass rather than argued against it. + Score int `json:"score"` + Verdict string `json:"verdict"` + // Findings are located problems, most severe first. + Findings []Finding `json:"findings"` + // SuggestedSubject is a rewritten subject line, empty when the current one + // is fine. + SuggestedSubject string `json:"suggested_subject,omitempty"` + // Improvements are copy-level suggestions with nothing specific to quote. + Improvements []string `json:"improvements,omitempty"` + Model string `json:"model"` + TokensUsed int `json:"tokens_used"` +} + +const systemPrompt = `You are a cold-email deliverability analyst. You are given one campaign email template and you report exactly what in it would push the message to spam, where that thing is, and what to write instead. + +OUTPUT +Return ONLY a JSON object, no prose around it, no markdown fence: +{ + "score": 0-100, + "verdict": "one plain sentence on how this email will land", + "findings": [ + { + "severity": "high" | "warn" | "info", + "field": "subject" | "body", + "text": "the exact words from the template, copied character for character", + "issue": "why this specific fragment hurts deliverability or replies", + "suggestion": "what to write instead, concretely", + "category": "trigger_word" | "tone" | "formatting" | "links" | "structure" | "authenticity" + } + ], + "suggested_subject": "a better subject line, or \"\" when the current one is fine", + "improvements": ["copy-level advice with no single fragment to quote"] +} + +RULES +- "text" MUST be copied verbatim from the subject or body you were given. Never paraphrase it, never fix its spelling, never quote a sentence that is not there. A fragment you cannot copy exactly belongs in "improvements" instead. +- Point at the smallest thing that is wrong: the word, the phrase, or the one sentence. Never quote a whole paragraph. +- "field" must say which half the fragment is in. Getting this wrong sends the writer to the wrong box. +- score is 0-100 where 100 is safest. Take the rule-based findings you are given as already proven and score with them, never against them. A clean, plain, personal email with one ask scores above 85; an email with several trigger phrases, hype and multiple links scores below 40. +- Judge cold-email deliverability and reply odds, not marketing polish. Plain, short and specific is good. Formatting, hype, urgency, unearned claims, link stacking and mail-merge that reads like a mail merge are bad. +- At most 8 findings. Do not invent problems to fill the list: an email with nothing wrong returns an empty findings array and a high score. +- Write every sentence in plain English, no em dashes, no jargon.` + +// Analyze runs the model over the template and returns a verified report. +func Analyze(ctx context.Context, p generation.Provider, model string, in Input) (*Result, error) { + if p == nil { + return nil, generation.ErrProviderNotConfigured + } + subject := truncate(strings.TrimSpace(in.Subject), maxSubjectRunes) + body := truncate(bodyText(in), maxBodyRunes) + + res, err := p.Complete(ctx, generation.CompletionRequest{ + System: systemPrompt, + Prompt: buildPrompt(subject, body, in), + Model: model, + MaxTokens: completionMax, + // Deterministic: the whole point of Re-check is comparing this score + // with the last one, and a sampled score moves on unchanged copy. + Temperature: generation.Deterministic(), + }) + if err != nil { + return nil, err + } + if res == nil { + return nil, fmt.Errorf("spamcheck: the model returned nothing") + } + + out := parse(res.Text) + // A response with no score, no verdict and no findings was not an analysis. + // Failing here refunds the credit; filling the gaps in would render as a + // confident 100/100 on copy nothing actually read. + if out.Score < 0 && out.Verdict == "" && len(out.Findings) == 0 { + return nil, fmt.Errorf("spamcheck: could not read the model's response") + } + out.Model = res.Model + out.TokensUsed = res.TokensUsed + verify(out, subject, body) + return out, nil +} + +// bodyText is the copy a recipient reads: the plain part when the editor stored +// one, otherwise the HTML rendered down to its words. +func bodyText(in Input) string { + if b := strings.TrimSpace(in.BodyPlain); b != "" { + return b + } + return strings.TrimSpace(mailhtml.ToPlainText(in.BodyHTML)) +} + +func buildPrompt(subject, body string, in Input) string { + var b strings.Builder + b.WriteString("SUBJECT:\n") + if subject == "" { + b.WriteString("(empty)\n") + } else { + b.WriteString(subject + "\n") + } + b.WriteString("\nBODY:\n") + if body == "" { + b.WriteString("(empty)\n") + } else { + b.WriteString(body + "\n") + } + + // The body may be HTML the reader never sees as markup, but its shape is + // part of how it lands, so the counts travel even though the tags do not. + if strings.TrimSpace(in.BodyHTML) != "" { + b.WriteString("\nThis body is HTML. Judge the words above, not the markup.\n") + } + + b.WriteString("\nRULE-BASED CHECK (already proven, score with it):\n") + fmt.Fprintf(&b, "score %d/100\n", in.Rules.Score) + if len(in.Rules.Issues) == 0 { + b.WriteString("no rule-based issues\n") + } + for _, issue := range in.Rules.Issues { + where := issue.Field + if where == "" { + where = "template" + } + fmt.Fprintf(&b, "- [%s] %s (%s)", where, issue.Message, issue.Code) + if len(issue.Spans) > 0 { + quoted := make([]string, 0, len(issue.Spans)) + for _, s := range issue.Spans { + if s.Text != "" { + quoted = append(quoted, quoteFragment(s.Text)) + } + } + if len(quoted) > 0 { + fmt.Fprintf(&b, ": %s", strings.Join(quoted, ", ")) + } + } + b.WriteString("\n") + } + + if ctx := voiceContext(in.Voice); ctx != "" { + b.WriteString("\nWHAT THIS WORKSPACE SELLS (context, not something to grade):\n") + b.WriteString(ctx) + } + b.WriteString("\nAnalyze this template now. JSON only.") + return b.String() +} + +// voiceContext renders the workspace's own description, so a specific claim is +// read as a real one rather than as hype. +func voiceContext(v generation.VoiceContext) string { + var b strings.Builder + for _, part := range []struct{ label, value string }{ + {"Product", v.ProductDescription}, + {"Who they sell to", v.ICPNotes}, + {"Voice", v.VoiceProfile}, + } { + if s := strings.TrimSpace(part.value); s != "" { + fmt.Fprintf(&b, "%s: %s\n", part.label, truncate(s, 600)) + } + } + return b.String() +} + +// quoteFragment wraps a fragment in quotes for the prompt, keeping an embedded +// quote from ending the quoted run. +func quoteFragment(s string) string { + return `"` + strings.ReplaceAll(s, `"`, `'`) + `"` +} + +// parse reads the model's JSON, tolerating a ```json fence or a stray sentence +// either side of the object. A response nothing can be read out of returns an +// empty result rather than an error: the panel still has the rules pass. +func parse(text string) *Result { + text = strings.TrimSpace(text) + if i := strings.Index(text, "{"); i >= 0 { + if j := strings.LastIndex(text, "}"); j >= i { + text = text[i : j+1] + } + } + var raw struct { + Score *int `json:"score"` + Verdict string `json:"verdict"` + SuggestedSubject string `json:"suggested_subject"` + Improvements []string `json:"improvements"` + Findings []struct { + Severity string `json:"severity"` + Field string `json:"field"` + Text string `json:"text"` + Issue string `json:"issue"` + Suggestion string `json:"suggestion"` + Category string `json:"category"` + } `json:"findings"` + } + out := &Result{Score: -1, Findings: []Finding{}} + if err := json.Unmarshal([]byte(text), &raw); err != nil { + return out + } + if raw.Score != nil { + out.Score = clamp(*raw.Score) + } + out.Verdict = truncate(strings.TrimSpace(raw.Verdict), 300) + out.SuggestedSubject = truncate(strings.TrimSpace(raw.SuggestedSubject), maxSubjectRunes) + for _, s := range raw.Improvements { + if s = truncate(strings.TrimSpace(s), 300); s != "" { + out.Improvements = append(out.Improvements, s) + } + if len(out.Improvements) == maxImprovements { + break + } + } + for _, f := range raw.Findings { + issue := truncate(strings.TrimSpace(f.Issue), 300) + if issue == "" { + continue + } + out.Findings = append(out.Findings, Finding{ + Severity: severity(f.Severity), + Field: field(f.Field), + // Cut, not truncated: verify looks this up in the copy, and an + // appended ellipsis is never in there, so every long quote would + // fail verification and lose the fragment it named. + Text: cut(strings.TrimSpace(f.Text), maxTextRunes), + Issue: issue, + Suggestion: truncate(strings.TrimSpace(f.Suggestion), 300), + Category: truncate(strings.TrimSpace(strings.ToLower(f.Category)), 40), + }) + if len(out.Findings) == maxFindings { + break + } + } + return out +} + +// verify anchors every finding in the copy the user actually wrote. A quote the +// model got right is given its real line and surrounding line; one it invented +// loses the quote and stays as general advice, because a panel that points at a +// sentence nobody wrote is worse than one that points at nothing. +func verify(res *Result, subject, body string) { + kept := res.Findings[:0] + seen := map[string]struct{}{} + for _, f := range res.Findings { + if f.Text != "" { + if fld, line, excerpt, ok := locate(subject, body, f.Text, f.Field); ok { + f.Field, f.Line, f.Excerpt = fld, line, excerpt + } else { + f.Text, f.Line, f.Excerpt = "", 0, "" + } + } + key := f.Field + "\x00" + strings.ToLower(f.Text) + "\x00" + strings.ToLower(f.Issue) + if _, dup := seen[key]; dup { + continue + } + seen[key] = struct{}{} + kept = append(kept, f) + } + res.Findings = kept + + // Severity order, stable within a band, so the worst thing is read first + // and equal findings keep the model's own reading order through the email. + rank := map[string]int{"high": 0, "warn": 1, "info": 2} + sort.SliceStable(res.Findings, func(i, j int) bool { + return rank[res.Findings[i].Severity] < rank[res.Findings[j].Severity] + }) + + // A model that answered with findings but no score would render as 0/100. + if res.Score < 0 { + res.Score = scoreFromFindings(res.Findings) + } +} + +// locate finds the fragment in the copy, preferring the field the model named +// and falling back to the other one (the analysis is still right, the label was +// wrong). Matching is case-insensitive: a model retyping a quote tends to +// normalize its capitalization. +func locate(subject, body, text, preferred string) (field string, line int, excerpt string, ok bool) { + order := []struct{ name, text string }{ + {warmlint.FieldSubject, subject}, + {warmlint.FieldBody, body}, + } + if preferred == warmlint.FieldBody { + order[0], order[1] = order[1], order[0] + } + needle := strings.ToLower(strings.TrimSpace(text)) + if needle == "" { + return "", 0, "", false + } + for _, o := range order { + lower := strings.ToLower(o.text) + i := strings.Index(lower, needle) + if i < 0 { + continue + } + n := 1 + strings.Count(lower[:i], "\n") + return o.name, n, lineText(o.text, n), true + } + return "", 0, "", false +} + +// lineText returns one line of a field, trimmed and capped for display. +func lineText(s string, line int) string { + lines := strings.Split(s, "\n") + if line < 1 || line > len(lines) { + return "" + } + return truncate(strings.TrimSpace(lines[line-1]), excerptMax) +} + +// scoreFromFindings is the fallback when the model returned findings but no +// usable score. +func scoreFromFindings(findings []Finding) int { + score := 100 + for _, f := range findings { + switch f.Severity { + case "high": + score -= 20 + case "warn": + score -= 10 + default: + score -= 3 + } + } + return clamp(score) +} + +func severity(s string) string { + switch strings.ToLower(strings.TrimSpace(s)) { + case "high", "critical", "severe", "error": + return "high" + case "info", "low", "minor", "note": + return "info" + default: + return "warn" + } +} + +func field(s string) string { + if strings.Contains(strings.ToLower(strings.TrimSpace(s)), "subject") { + return warmlint.FieldSubject + } + return warmlint.FieldBody +} + +func clamp(n int) int { + if n < 0 { + return 0 + } + if n > 100 { + return 100 + } + return n +} + +// truncate bounds prose for display, marking that it was shortened. +func truncate(s string, max int) string { + r := []rune(s) + if len(r) <= max { + return s + } + return strings.TrimSpace(string(r[:max])) + "…" +} + +// cut bounds a fragment that still has to be found in the copy, so it adds +// nothing: a prefix of a substring is still a substring. +func cut(s string, max int) string { + r := []rune(s) + if len(r) <= max { + return s + } + return strings.TrimSpace(string(r[:max])) +} diff --git a/internal/pkg/spamcheck/spamcheck_test.go b/internal/pkg/spamcheck/spamcheck_test.go new file mode 100644 index 00000000..eff4d88c --- /dev/null +++ b/internal/pkg/spamcheck/spamcheck_test.go @@ -0,0 +1,294 @@ +package spamcheck + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/warmbly/warmbly/internal/pkg/generation" + "github.com/warmbly/warmbly/internal/pkg/warmlint" +) + +const testBody = "Hi Ada,\n\nThis is a limited time offer you cannot miss.\n\nWorth ten minutes?" + +// analyzed runs the parse + verify halves the way Analyze does, over a model +// response, so the tests exercise exactly what a real completion goes through. +func analyzed(t *testing.T, response, subject, body string) *Result { + t.Helper() + res := parse(response) + verify(res, subject, body) + return res +} + +func TestVerifyAnchorsAQuoteInTheBody(t *testing.T) { + res := analyzed(t, `{ + "score": 55, + "verdict": "Reads like a promotion.", + "findings": [ + {"severity":"high","field":"body","text":"limited time offer","issue":"Urgency wording filters weight heavily.","suggestion":"Say what changed instead."} + ] + }`, "Quick question", testBody) + + if len(res.Findings) != 1 { + t.Fatalf("got %d findings, want the quoted one kept: %+v", len(res.Findings), res.Findings) + } + f := res.Findings[0] + if f.Line != 3 { + t.Errorf("line = %d, want the third line of the body", f.Line) + } + if f.Excerpt != "This is a limited time offer you cannot miss." { + t.Errorf("excerpt = %q, want the sentence around the quote", f.Excerpt) + } + if res.Score != 55 { + t.Errorf("score = %d, want the model's own", res.Score) + } +} + +// A panel that points at a sentence the writer never wrote is worse than one +// that points at nothing, so an unverifiable quote loses the quote and keeps +// the advice. +func TestVerifyDropsAQuoteThatIsNotInTheCopy(t *testing.T) { + res := analyzed(t, `{ + "score": 60, + "findings": [ + {"severity":"warn","field":"body","text":"ACT NOW while stocks last","issue":"Hype wording hurts.","suggestion":"Cut it."} + ] + }`, "Quick question", testBody) + + if len(res.Findings) != 1 { + t.Fatalf("got %d findings, want the advice kept: %+v", len(res.Findings), res.Findings) + } + f := res.Findings[0] + if f.Text != "" || f.Line != 0 || f.Excerpt != "" { + t.Errorf("invented quote survived verification: %+v", f) + } + if f.Issue == "" { + t.Error("the advice was dropped along with the quote") + } +} + +// The analysis can be right while the label is wrong. Sending the writer to the +// wrong box for a real problem is a worse outcome than correcting the label. +func TestVerifyCorrectsTheFieldWhenTheQuoteIsInTheOtherHalf(t *testing.T) { + res := analyzed(t, `{ + "score": 70, + "findings": [ + {"severity":"warn","field":"subject","text":"limited time offer","issue":"Urgency wording.","suggestion":"Cut it."} + ] + }`, "Quick question", testBody) + + if res.Findings[0].Field != "body" { + t.Errorf("field = %q, want it corrected to body", res.Findings[0].Field) + } +} + +// A model retyping a quote tends to normalize its capitalization. +func TestVerifyMatchesAQuoteCaseInsensitively(t *testing.T) { + res := analyzed(t, `{ + "score": 70, + "findings": [ + {"severity":"warn","field":"subject","text":"free bonus","issue":"Trigger wording.","suggestion":"Cut it."} + ] + }`, "Your FREE BONUS inside", testBody) + + f := res.Findings[0] + if f.Field != "subject" || f.Excerpt != "Your FREE BONUS inside" { + t.Errorf("quote was not matched against the subject: %+v", f) + } +} + +func TestVerifyOrdersFindingsBySeverity(t *testing.T) { + res := analyzed(t, `{ + "score": 40, + "findings": [ + {"severity":"low","field":"body","issue":"Third."}, + {"severity":"medium","field":"body","issue":"Second."}, + {"severity":"critical","field":"body","issue":"First."} + ] + }`, "Quick question", testBody) + + got := []string{} + for _, f := range res.Findings { + got = append(got, f.Issue+"/"+f.Severity) + } + want := "First./high Second./warn Third./info" + if strings.Join(got, " ") != want { + t.Errorf("order = %q, want %q", strings.Join(got, " "), want) + } +} + +func TestParseToleratesAFencedResponse(t *testing.T) { + res := parse("Here you go:\n```json\n{\"score\": 88, \"verdict\": \"Fine.\"}\n```\n") + if res.Score != 88 || res.Verdict != "Fine." { + t.Errorf("fenced JSON was not read: %+v", res) + } +} + +// A response nothing can be read out of must not render as 0/100: the panel +// still has the rules pass, and a fabricated zero would contradict it. +func TestParseRefusesToInventAScore(t *testing.T) { + res := parse("I could not analyze that.") + if res.Score != -1 { + t.Errorf("score = %d, want the missing-score marker", res.Score) + } + if res.Findings == nil { + t.Error("findings must be an empty list, never null in JSON") + } +} + +// A model that returned findings but no usable score still has to produce one, +// or Re-check has nothing to compare. +func TestVerifyDerivesAScoreWhenTheModelOmittedIt(t *testing.T) { + res := analyzed(t, `{ + "findings": [ + {"severity":"high","field":"body","issue":"One."}, + {"severity":"warn","field":"body","issue":"Two."} + ] + }`, "Quick question", testBody) + + if res.Score != 70 { + t.Errorf("score = %d, want one derived from the findings", res.Score) + } +} + +func TestVerifyDropsDuplicateFindings(t *testing.T) { + res := analyzed(t, `{ + "score": 50, + "findings": [ + {"severity":"warn","field":"body","text":"limited time offer","issue":"Urgency wording."}, + {"severity":"warn","field":"body","text":"limited time offer","issue":"Urgency wording."} + ] + }`, "Quick question", testBody) + + if len(res.Findings) != 1 { + t.Errorf("got %d findings, want the repeat dropped: %+v", len(res.Findings), res.Findings) + } +} + +func TestParseBoundsTheResponse(t *testing.T) { + var b strings.Builder + b.WriteString(`{"score": 20, "improvements": [`) + for i := 0; i < 20; i++ { + if i > 0 { + b.WriteString(",") + } + b.WriteString(`"tip"`) + } + b.WriteString(`], "findings": [`) + for i := 0; i < 30; i++ { + if i > 0 { + b.WriteString(",") + } + b.WriteString(`{"severity":"warn","field":"body","issue":"something"}`) + } + b.WriteString(`]}`) + + res := parse(b.String()) + if len(res.Improvements) != maxImprovements { + t.Errorf("kept %d improvements, want %d", len(res.Improvements), maxImprovements) + } + if len(res.Findings) != maxFindings { + t.Errorf("kept %d findings, want %d", len(res.Findings), maxFindings) + } +} + +// stubProvider answers with a canned completion, so the tests can drive Analyze +// end to end without a provider. +type stubProvider struct { + reply string + got generation.CompletionRequest +} + +func (s *stubProvider) RunAgent(context.Context, generation.AgentRequest) (*generation.AgentResult, error) { + return nil, errors.New("not used") +} + +func (s *stubProvider) Complete(_ context.Context, req generation.CompletionRequest) (*generation.WritingResult, error) { + s.got = req + return &generation.WritingResult{Text: s.reply, Model: "stub-model", TokensUsed: 321}, nil +} + +func (s *stubProvider) ModelForTier(bool) string { return "stub-model" } +func (s *stubProvider) Name() string { return "stub" } +func (s *stubProvider) IsLocal() bool { return true } + +// A response nothing could be read out of must fail the call, which refunds the +// credit, rather than render as a confident 100/100 on copy nothing read. +func TestAnalyzeRefusesAnUnreadableResponse(t *testing.T) { + _, err := Analyze(context.Background(), &stubProvider{reply: "I could not analyze that."}, "stub-model", Input{ + Subject: "Quick question", + BodyPlain: testBody, + }) + if err == nil { + t.Fatal("an unreadable response was accepted as an analysis") + } +} + +func TestAnalyzeIsDeterministicAndGroundsOnTheRulesPass(t *testing.T) { + p := &stubProvider{reply: `{"score": 62, "verdict": "Reads promotional."}`} + rules := warmlint.Score("Your FREE bonus", "", testBody) + + res, err := Analyze(context.Background(), p, "stub-model", Input{ + Subject: "Your FREE bonus", + BodyPlain: testBody, + Rules: rules, + }) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + if res.Score != 62 || res.Model != "stub-model" || res.TokensUsed != 321 { + t.Errorf("result did not carry the completion through: %+v", res) + } + // Re-check compares this score with the last one, so a sampled score would + // move on copy the writer never touched. + if p.got.Temperature == nil || *p.got.Temperature != 0 { + t.Errorf("temperature = %v, want pinned to 0", p.got.Temperature) + } + // The model is told what the rules pass already proved, so the two halves of + // the panel cannot contradict each other. + if !strings.Contains(p.got.Prompt, "spam_trigger_terms") || !strings.Contains(p.got.Prompt, `"FREE"`) { + t.Errorf("prompt did not carry the rule-based findings:\n%s", p.got.Prompt) + } +} + +func TestAnalyzeTruncatesAVeryLongBody(t *testing.T) { + p := &stubProvider{reply: `{"score": 80, "verdict": "Fine."}`} + if _, err := Analyze(context.Background(), p, "stub-model", Input{ + Subject: "Quick question", + BodyPlain: strings.Repeat("word ", 40000), + }); err != nil { + t.Fatalf("Analyze: %v", err) + } + if len([]rune(p.got.Prompt)) > maxBodyRunes+2000 { + t.Errorf("prompt is %d runes, want the body bounded at %d", len([]rune(p.got.Prompt)), maxBodyRunes) + } +} + +// A quote longer than the cap is shortened to fit, and the shortened form still +// has to verify: an ellipsis on the end would never be found in the copy, so +// every long quote would silently lose the fragment it named. +func TestVerifyKeepsALongQuoteThatHadToBeShortened(t *testing.T) { + sentence := strings.Repeat("this is a long promotional sentence that keeps going ", 12) + body := "Hi Ada,\n\n" + sentence + "\n\nWorth ten minutes?" + quoted, err := json.Marshal(sentence) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + res := analyzed(t, `{"score": 40, "findings": [ + {"severity":"high","field":"body","text":`+string(quoted)+`,"issue":"Too long and too promotional."} + ]}`, "Quick question", body) + + f := res.Findings[0] + if f.Text == "" { + t.Fatal("a quote that only needed shortening was dropped as unverifiable") + } + if len([]rune(f.Text)) > maxTextRunes { + t.Errorf("quote is %d runes, want it bounded at %d", len([]rune(f.Text)), maxTextRunes) + } + if f.Line != 3 { + t.Errorf("line = %d, want the line the sentence sits on", f.Line) + } +} diff --git a/internal/pkg/warmlint/lint.go b/internal/pkg/warmlint/lint.go index 619fe17f..ac22503c 100644 --- a/internal/pkg/warmlint/lint.go +++ b/internal/pkg/warmlint/lint.go @@ -66,11 +66,40 @@ func Check(subject, body string, isReply bool) error { return nil } +// Field names the half of the template a finding sits in, so the editor can +// say "in the subject" rather than leaving the writer to search for the word. +const ( + FieldSubject = "subject" + FieldBody = "body" +) + +// Span locates one exact fragment that triggered an issue. Without it a writer +// reads "3 spam-trigger term(s) found" and has to guess which words those are. +type Span struct { + // Field is FieldSubject or FieldBody. + Field string `json:"field"` + // Text is the fragment as it is written in the copy. + Text string `json:"text"` + // Line is the 1-based line the fragment sits on within that field. + Line int `json:"line,omitempty"` + // Excerpt is that whole line, so the fragment can be shown in context. + Excerpt string `json:"excerpt,omitempty"` +} + // Issue is a single advisory content problem found by Score. type Issue struct { Severity string `json:"severity"` // "warn" | "high" Code string `json:"code"` Message string `json:"message"` + // Field is FieldSubject or FieldBody when the issue lives in exactly one of + // them, and empty when it spans both or describes the send as a whole. + Field string `json:"field,omitempty"` + // Spans are the exact fragments that triggered the issue, in reading order. + // Empty for an issue with nothing to point at (an empty subject, an + // attachment count). + Spans []Span `json:"spans,omitempty"` + // Suggestion is the concrete fix, in one line. + Suggestion string `json:"suggestion,omitempty"` } // ScoreResult is an advisory content assessment for a campaign template. @@ -84,11 +113,17 @@ type ScoreResult struct { // warmup mail — Score never blocks: it surfaces guidance before the user sends // the mail that actually reaches prospects and drives complaints. It reuses the // same trigger-term and ALL-CAPS heuristics as the warmup lint. +// +// Every issue carries where it is (subject or body) and the exact fragments +// that caused it, because a score with no location is advice nobody can act on. func Score(subject, bodyHTML, bodyPlain string) ScoreResult { res := ScoreResult{Score: 100, Issues: []Issue{}} - deduct := func(n int, severity, code, msg string) { + deduct := func(n int, issue Issue) { res.Score -= n - res.Issues = append(res.Issues, Issue{Severity: severity, Code: code, Message: msg}) + if issue.Field == "" { + issue.Field = commonField(issue.Spans) + } + res.Issues = append(res.Issues, issue) } subj := strings.TrimSpace(subject) @@ -96,17 +131,31 @@ func Score(subject, bodyHTML, bodyPlain string) ScoreResult { if strings.TrimSpace(body) == "" { body = stripTags(bodyHTML) } - combined := subj + "\n" + body if subj == "" { - deduct(20, "high", "empty_subject", "Subject is empty.") + deduct(20, Issue{ + Severity: "high", Code: "empty_subject", Field: FieldSubject, + Message: "Subject is empty.", + Suggestion: "Write a short, specific subject: lowercase, under six words, about them.", + }) } else if isAllCaps(subj) { - deduct(15, "high", "all_caps_subject", "Subject is all caps, a strong spam signal.") + deduct(15, Issue{ + Severity: "high", Code: "all_caps_subject", Field: FieldSubject, + Message: "Subject is all caps, a strong spam signal.", + Spans: []Span{{Field: FieldSubject, Text: subj, Line: 1, Excerpt: subj}}, + Suggestion: "Write the subject in ordinary sentence case.", + }) } - if stackedPunct.MatchString(combined) { - deduct(10, "warn", "stacked_punctuation", "Stacked punctuation (e.g. !!! or ?!) reads as promotional.") + if punct := punctuationSpans(subj, body); len(punct) > 0 { + deduct(10, Issue{ + Severity: "warn", Code: "stacked_punctuation", + Message: "Stacked punctuation (e.g. !!! or ?!) reads as promotional.", + Spans: punct, + Suggestion: "Use a single full stop or question mark.", + }) } - if n := countTriggerTerms(withoutURLs(combined)); n > 0 { + terms, termSpans := triggerSpans(subj, body) + if n := len(terms); n > 0 { d := n * 8 if d > 40 { d = 40 @@ -115,19 +164,38 @@ func Score(subject, bodyHTML, bodyPlain string) ScoreResult { if n >= 3 { severity = "high" } - deduct(d, severity, "spam_trigger_terms", fmt.Sprintf("%d spam-trigger term(s) found in subject/body.", n)) + deduct(d, Issue{ + Severity: severity, Code: "spam_trigger_terms", + Message: fmt.Sprintf("%d spam-trigger term(s) found in subject/body: %s.", n, joinTerms(terms)), + Spans: termSpans, + Suggestion: "Rewrite those words in plain language, or cut the sentence they sit in.", + }) } - if links := countLinks(combined, bodyHTML); links > 3 { + linked := linkSpans(subj, body, bodyHTML) + if links := len(linked); links > 3 { d := (links - 3) * 5 if d > 20 { d = 20 } - deduct(d, "warn", "too_many_links", fmt.Sprintf("%d links. Keep the link count low in cold email.", links)) + deduct(d, Issue{ + Severity: "warn", Code: "too_many_links", + Message: fmt.Sprintf("%d links. Keep the link count low in cold email.", links), + Spans: capSpans(linked), + Suggestion: "Keep one link at most on a first touch, and cut the rest.", + }) } if strings.TrimSpace(body) == "" { - deduct(25, "high", "empty_body", "Body has no text content (image-only or empty body hurts deliverability).") + deduct(25, Issue{ + Severity: "high", Code: "empty_body", Field: FieldBody, + Message: "Body has no text content (image-only or empty body hurts deliverability).", + Suggestion: "Write the message as text. Filters cannot read an image.", + }) } else if len(body) > 15000 { - deduct(10, "warn", "oversized_body", "Body is very large; trim it for deliverability.") + deduct(10, Issue{ + Severity: "warn", Code: "oversized_body", Field: FieldBody, + Message: "Body is very large; trim it for deliverability.", + Suggestion: "Cut it to a few short paragraphs and one ask.", + }) } // Images: cold mail from a real person is usually plain. A wall of images, @@ -135,14 +203,21 @@ func Score(subject, bodyHTML, bodyPlain string) ScoreResult { if images := len(imgTag.FindAllString(bodyHTML, -1)); images > 0 { switch { case len(strings.TrimSpace(body)) < 200 && images >= 1: - deduct(20, "high", "image_heavy", - "Almost all of this email is images. Filters cannot read it and treat that as evasion.") + deduct(20, Issue{ + Severity: "high", Code: "image_heavy", Field: FieldBody, + Message: "Almost all of this email is images. Filters cannot read it and treat that as evasion.", + Suggestion: "Put the message in text and keep images to a signature at most.", + }) case images > 3: d := (images - 3) * 5 if d > 15 { d = 15 } - deduct(d, "warn", "many_images", fmt.Sprintf("%d images. Cold email from a person rarely has many.", images)) + deduct(d, Issue{ + Severity: "warn", Code: "many_images", Field: FieldBody, + Message: fmt.Sprintf("%d images. Cold email from a person rarely has many.", images), + Suggestion: "Drop all but the one image the message actually needs.", + }) } } @@ -152,6 +227,31 @@ func Score(subject, bodyHTML, bodyPlain string) ScoreResult { return res } +// LeadIssue renders the one issue worth naming in a one-line summary: the first +// high-severity one, else the first of any. It says where the problem is, +// because "3 spam-trigger terms" on its own sends the reader looking in the +// wrong box. Empty when there is nothing to report. +func LeadIssue(res ScoreResult) string { + lead := Issue{} + for _, issue := range res.Issues { + if issue.Severity == "high" { + lead = issue + break + } + if lead.Code == "" { + lead = issue + } + } + switch lead.Field { + case FieldSubject: + return "Subject: " + lead.Message + case FieldBody: + return "Body: " + lead.Message + default: + return lead.Message + } +} + // ScoreWithAttachments is Score plus the attachment heuristic, which needs // context the template alone does not carry. func ScoreWithAttachments(subject, bodyHTML, bodyPlain string, attachments int) ScoreResult { @@ -164,9 +264,10 @@ func ScoreWithAttachments(subject, bodyHTML, bodyPlain string, attachments int) res.Score = 0 } res.Issues = append(res.Issues, Issue{ - Severity: "warn", - Code: "has_attachments", - Message: fmt.Sprintf("%d attachment(s) on a cold email. Link to the file instead.", attachments), + Severity: "warn", + Code: "has_attachments", + Message: fmt.Sprintf("%d attachment(s) on a cold email. Link to the file instead.", attachments), + Suggestion: "Remove the file and offer to send it once they reply.", }) } return res @@ -202,25 +303,6 @@ func isAllCaps(s string) bool { return letters >= 4 } -// countLinks counts every anchor plus any bare URL in the text that is not -// already an anchor's destination. Stripping tags throws hrefs away, so the text -// alone reports zero links for an HTML email; matching destinations keeps a URL -// used as its own anchor text from counting twice. -func countLinks(text, bodyHTML string) int { - destinations := map[string]struct{}{} - n := 0 - for _, m := range hrefPattern.FindAllStringSubmatch(bodyHTML, -1) { - destinations[trimURL(m[1])] = struct{}{} - n++ - } - for _, u := range linkPattern.FindAllString(text, -1) { - if _, seen := destinations[trimURL(u)]; !seen { - n++ - } - } - return n -} - // trimURL drops the sentence punctuation a URL picks up in prose, so the same // link matches whether it was written inline or as an anchor's destination. func trimURL(u string) string { diff --git a/internal/pkg/warmlint/locate.go b/internal/pkg/warmlint/locate.go new file mode 100644 index 00000000..e88ec47c --- /dev/null +++ b/internal/pkg/warmlint/locate.go @@ -0,0 +1,214 @@ +package warmlint + +import ( + "fmt" + "sort" + "strings" +) + +// This file turns each heuristic in Score into a location. A score with no +// location ("3 spam-trigger terms found") leaves the writer hunting through +// their own copy for words the checker already knows, so every issue points at +// the fragments that caused it and says which half of the template they are in. + +// maxSpans bounds how many fragments one issue carries. The count that drives +// the deduction is computed before this cap, so trimming the list never moves +// the score. +const maxSpans = 8 + +// excerptMax caps a quoted line, so one pasted wall of text cannot blow up the +// response. +const excerptMax = 180 + +// listedTerms bounds how many trigger terms the message names before it counts +// the rest. +const listedTerms = 5 + +// commonField returns the field every span shares, or "" when they straddle the +// subject and the body (or there are none). +func commonField(spans []Span) string { + field := "" + for _, s := range spans { + if field == "" { + field = s.Field + continue + } + if s.Field != field { + return "" + } + } + return field +} + +// capSpans drops anything that could not be quoted and trims the list to +// maxSpans, in reading order. A span with no text is one spanAt could not slice +// (a lowercased offset that moved under a non-ASCII rune); showing it would put +// an empty highlight in the panel. Dropping it never moves the score, which is +// computed from the counts, not from this list. +func capSpans(spans []Span) []Span { + out := make([]Span, 0, min(len(spans), maxSpans)) + for _, s := range spans { + if s.Text == "" { + continue + } + out = append(out, s) + if len(out) == maxSpans { + break + } + } + return out +} + +// spanAt describes scan[start:end] as a span. scan is what was searched and +// display is what the excerpt is quoted from: the trigger-term pass searches a +// URL-stripped copy of the text, and quoting that would show the writer a line +// with their own links blanked out. Both keep every newline, so a line number +// found in one addresses the same line in the other. +func spanAt(field, scan, display string, start, end int) Span { + text := "" + if start >= 0 && end <= len(scan) && start < end { + text = strings.TrimSpace(scan[start:end]) + } + line := lineNumber(scan, start) + return Span{Field: field, Text: text, Line: line, Excerpt: lineText(display, line)} +} + +// lineNumber is the 1-based line the byte offset sits on. +func lineNumber(s string, at int) int { + if at < 0 || at > len(s) { + return 0 + } + return 1 + strings.Count(s[:at], "\n") +} + +// lineText returns that line, trimmed and capped for display. +func lineText(s string, line int) string { + if line < 1 { + return "" + } + lines := strings.Split(s, "\n") + if line > len(lines) { + return "" + } + out := strings.TrimSpace(lines[line-1]) + if r := []rune(out); len(r) > excerptMax { + out = strings.TrimSpace(string(r[:excerptMax])) + "…" + } + return out +} + +// scanned pairs a field with the text searched for it and the text an excerpt +// is quoted from. +type scanned struct { + field string + scan string + display string +} + +// punctuationSpans locates every run of stacked punctuation, subject first. +func punctuationSpans(subject, body string) []Span { + var spans []Span + for _, f := range []scanned{{FieldSubject, subject, subject}, {FieldBody, body, body}} { + for _, loc := range stackedPunct.FindAllStringIndex(f.scan, -1) { + spans = append(spans, spanAt(f.field, f.scan, f.display, loc[0], loc[1])) + } + } + return capSpans(spans) +} + +// termHit is one trigger term and where it was first seen. +type termHit struct { + term string + at int +} + +// triggerTermsIn returns the distinct trigger terms in the text, in order of +// first appearance. It matches countTriggerTerms exactly on which terms count; +// only the ordering and the offsets are extra. +func triggerTermsIn(text string) []termHit { + lower := strings.ToLower(text) + first := map[string]int{} + for _, loc := range wordToken.FindAllStringIndex(lower, -1) { + w := lower[loc[0]:loc[1]] + if _, ok := triggerWords[w]; !ok { + continue + } + if _, seen := first[w]; !seen { + first[w] = loc[0] + } + } + for _, p := range triggerPhrases { + if i := strings.Index(lower, p); i >= 0 { + if _, seen := first[p]; !seen { + first[p] = i + } + } + } + hits := make([]termHit, 0, len(first)) + for term, at := range first { + hits = append(hits, termHit{term: term, at: at}) + } + sort.Slice(hits, func(i, j int) bool { + if hits[i].at != hits[j].at { + return hits[i].at < hits[j].at + } + return hits[i].term < hits[j].term + }) + return hits +} + +// triggerSpans returns the distinct trigger terms across the template and where +// each one is. A term written in both halves counts once and is shown where it +// appears first, which keeps the count identical to countTriggerTerms over the +// two joined together. +func triggerSpans(subject, body string) (terms []string, spans []Span) { + seen := map[string]struct{}{} + for _, f := range []scanned{ + {FieldSubject, withoutURLs(subject), subject}, + {FieldBody, withoutURLs(body), body}, + } { + for _, hit := range triggerTermsIn(f.scan) { + if _, dup := seen[hit.term]; dup { + continue + } + seen[hit.term] = struct{}{} + terms = append(terms, hit.term) + spans = append(spans, spanAt(f.field, f.scan, f.display, hit.at, hit.at+len(hit.term))) + } + } + return terms, capSpans(spans) +} + +// joinTerms names the trigger terms found, counting the tail once the list gets +// long enough to read as noise. +func joinTerms(terms []string) string { + if len(terms) <= listedTerms { + return strings.Join(terms, ", ") + } + return fmt.Sprintf("%s and %d more", strings.Join(terms[:listedTerms], ", "), len(terms)-listedTerms) +} + +// linkSpans locates every anchor plus any bare URL that is not already an +// anchor's destination. Stripping tags throws hrefs away, so the text alone +// reports zero links for an HTML email; matching destinations keeps a URL used +// as its own anchor text from counting twice. The caller counts the result, so +// this returns one entry per link before any display cap. +func linkSpans(subject, body, bodyHTML string) []Span { + destinations := map[string]struct{}{} + var spans []Span + for _, m := range hrefPattern.FindAllStringSubmatch(bodyHTML, -1) { + destinations[trimURL(m[1])] = struct{}{} + // An href lives in markup, not in a line of copy, so it carries the + // destination and no excerpt. + spans = append(spans, Span{Field: FieldBody, Text: m[1]}) + } + for _, f := range []scanned{{FieldSubject, subject, subject}, {FieldBody, body, body}} { + for _, loc := range linkPattern.FindAllStringIndex(f.scan, -1) { + if _, seen := destinations[trimURL(f.scan[loc[0]:loc[1]])]; seen { + continue + } + spans = append(spans, spanAt(f.field, f.scan, f.display, loc[0], loc[1])) + } + } + return spans +} diff --git a/internal/pkg/warmlint/score_test.go b/internal/pkg/warmlint/score_test.go index 315bf550..920d56c6 100644 --- a/internal/pkg/warmlint/score_test.go +++ b/internal/pkg/warmlint/score_test.go @@ -1,6 +1,7 @@ package warmlint import ( + "fmt" "strings" "testing" ) @@ -180,3 +181,195 @@ func TestScoreIgnoresAStylesheet(t *testing.T) { t.Errorf("a stylesheet changed the content score: %d vs %d (%v)", styled.Score, clean.Score, styled.Issues) } } + +func issueByCode(res ScoreResult, code string) (Issue, bool) { + for _, i := range res.Issues { + if i.Code == code { + return i, true + } + } + return Issue{}, false +} + +// A score with no location is advice nobody can act on: the writer is told +// three trigger terms exist and left to hunt for them. +func TestScoreLocatesTriggerTermsInTheSubject(t *testing.T) { + res := Score("Your FREE bonus inside", "

Hi Ada, worth a chat?

", "Hi Ada, worth a chat?") + issue, ok := issueByCode(res, "spam_trigger_terms") + if !ok { + t.Fatalf("trigger terms not flagged: %+v", res.Issues) + } + if issue.Field != FieldSubject { + t.Errorf("issue field = %q, want subject", issue.Field) + } + if len(issue.Spans) != 2 { + t.Fatalf("got %d spans, want one per term: %+v", len(issue.Spans), issue.Spans) + } + for _, s := range issue.Spans { + if s.Field != FieldSubject { + t.Errorf("span %q sits in %q, want subject", s.Text, s.Field) + } + if s.Excerpt != "Your FREE bonus inside" { + t.Errorf("span %q quoted %q, want the whole subject line", s.Text, s.Excerpt) + } + } + if issue.Spans[0].Text != "FREE" { + t.Errorf("first span = %q, want the term as the writer typed it", issue.Spans[0].Text) + } + if issue.Suggestion == "" { + t.Error("no suggestion on a trigger-term issue") + } +} + +func TestScoreLocatesTriggerTermsInTheBodyByLine(t *testing.T) { + body := "Hi Ada,\n\nThis is a limited time offer.\n\nWorth a chat?" + res := Score("Quick question", "", body) + issue, ok := issueByCode(res, "spam_trigger_terms") + if !ok { + t.Fatalf("trigger phrase not flagged: %+v", res.Issues) + } + if issue.Field != FieldBody { + t.Errorf("issue field = %q, want body", issue.Field) + } + span := issue.Spans[0] + if span.Line != 3 { + t.Errorf("span line = %d, want the third line", span.Line) + } + if span.Excerpt != "This is a limited time offer." { + t.Errorf("excerpt = %q, want the sentence it sits in", span.Excerpt) + } +} + +// A term in both halves counts once, as it always has, and is shown where the +// reader meets it first. +func TestScoreCountsATermInBothHalvesOnce(t *testing.T) { + res := Score("A free look", "", "Here is a free look at it.") + issue, ok := issueByCode(res, "spam_trigger_terms") + if !ok { + t.Fatalf("trigger term not flagged: %+v", res.Issues) + } + if len(issue.Spans) != 1 { + t.Fatalf("got %d spans for one distinct term: %+v", len(issue.Spans), issue.Spans) + } + if issue.Spans[0].Field != FieldSubject { + t.Errorf("span field = %q, want the subject occurrence", issue.Spans[0].Field) + } + if res.Score != 92 { + t.Errorf("score = %d, want one term's deduction only", res.Score) + } +} + +// The trigger pass reads a URL-stripped copy of the text; quoting that copy +// would show the writer a line with their own links blanked out. +func TestScoreExcerptKeepsTheLinkTheTriggerPassStripped(t *testing.T) { + body := "A free look: https://example.com/pricing" + res := Score("Quick question", "", body) + issue, ok := issueByCode(res, "spam_trigger_terms") + if !ok { + t.Fatalf("trigger term not flagged: %+v", res.Issues) + } + if issue.Spans[0].Excerpt != body { + t.Errorf("excerpt = %q, want the line as written", issue.Spans[0].Excerpt) + } +} + +func TestScoreLocatesStackedPunctuationAndImages(t *testing.T) { + res := Score("Hurry!!", "

Ready?!

", "Ready?!") + issue, ok := issueByCode(res, "stacked_punctuation") + if !ok { + t.Fatalf("stacked punctuation not flagged: %+v", res.Issues) + } + if issue.Field != "" { + t.Errorf("field = %q, want empty when the issue straddles both halves", issue.Field) + } + if len(issue.Spans) != 2 { + t.Fatalf("got %d spans, want one per run: %+v", len(issue.Spans), issue.Spans) + } + if issue.Spans[0].Field != FieldSubject || issue.Spans[1].Field != FieldBody { + t.Errorf("spans = %+v, want the subject one first", issue.Spans) + } +} + +func TestScoreLinkIssueNamesTheDestinations(t *testing.T) { + body := strings.Repeat("A real sentence about the recipient's work. ", 10) + html := "

" + body + "

" + + `one two ` + + `three four

` + res := Score("Quick question", html, body) + issue, ok := issueByCode(res, "too_many_links") + if !ok { + t.Fatalf("four links not flagged: %+v", res.Issues) + } + if len(issue.Spans) != 4 { + t.Fatalf("got %d spans, want one per link: %+v", len(issue.Spans), issue.Spans) + } + if issue.Spans[0].Text != "https://a.com/1" { + t.Errorf("first span = %q, want the destination", issue.Spans[0].Text) + } +} + +// Trimming the displayed spans must never move the score, so the count that +// drives the deduction is taken before the cap. +func TestScoreCapsSpansWithoutMovingTheScore(t *testing.T) { + body := strings.Repeat("A real sentence about the recipient's work. ", 10) + links := "" + for i := 0; i < 14; i++ { + links += fmt.Sprintf(`link `, i) + } + res := Score("Quick question", "

"+body+"

"+links+"

", body) + issue, ok := issueByCode(res, "too_many_links") + if !ok { + t.Fatalf("fourteen links not flagged: %+v", res.Issues) + } + if len(issue.Spans) != maxSpans { + t.Errorf("got %d spans, want them capped at %d", len(issue.Spans), maxSpans) + } + if !strings.HasPrefix(issue.Message, "14 links") { + t.Errorf("message = %q, want it to count all fourteen", issue.Message) + } + // 14 links is 11 over the allowance, well past the 20-point cap. + if res.Score != 80 { + t.Errorf("score = %d, want the capped 20-point deduction", res.Score) + } +} + +func TestScoreFieldsTheIssuesWithNothingToQuote(t *testing.T) { + res := Score("", "", "") + for _, code := range []string{"empty_subject", "empty_body"} { + issue, ok := issueByCode(res, code) + if !ok { + t.Fatalf("%s not flagged: %+v", code, res.Issues) + } + if issue.Field == "" { + t.Errorf("%s has no field", code) + } + if issue.Suggestion == "" { + t.Errorf("%s has no suggestion", code) + } + } +} + +// The preflight dialog and the campaign feed both print one line about the +// worst thing in a step's copy, and that line has to say which box to open. +func TestLeadIssueNamesTheField(t *testing.T) { + if got := LeadIssue(Score("FREE CASH PRIZE", "

Hi Ada, worth a chat?

", "Hi Ada, worth a chat?")); !strings.HasPrefix(got, "Subject: ") { + t.Errorf("lead issue = %q, want it to name the subject", got) + } + if got := LeadIssue(Score("Quick question", "", "Here is a limited time offer.")); !strings.HasPrefix(got, "Body: ") { + t.Errorf("lead issue = %q, want it to name the body", got) + } + if got := LeadIssue(Score("Quick question", "

Hi Ada, worth a chat?

", "Hi Ada, worth a chat?")); got != "" { + t.Errorf("lead issue = %q on clean copy, want nothing", got) + } +} + +// A high-severity issue outranks a warning even when the warning came first. +func TestLeadIssuePrefersTheHighSeverityOne(t *testing.T) { + res := ScoreResult{Issues: []Issue{ + {Severity: "warn", Code: "stacked_punctuation", Message: "Stacked punctuation."}, + {Severity: "high", Code: "empty_body", Field: FieldBody, Message: "Body has no text content."}, + }} + if got := LeadIssue(res); got != "Body: Body has no text content." { + t.Errorf("lead issue = %q, want the high-severity one", got) + } +} diff --git a/internal/tasks/content_gate.go b/internal/tasks/content_gate.go index f5215679..69de7784 100644 --- a/internal/tasks/content_gate.go +++ b/internal/tasks/content_gate.go @@ -46,14 +46,8 @@ func (s *tasksService) warnOnWeakContent(ctx context.Context, orgID, campaignID, seq := sequenceID.String() detail := "" - for _, issue := range res.Issues { - if issue.Severity == "high" { - detail = " " + issue.Message - break - } - } - if detail == "" && len(res.Issues) > 0 { - detail = " " + res.Issues[0].Message + if lead := warmlint.LeadIssue(res); lead != "" { + detail = " " + lead } codes := make([]string, 0, len(res.Issues)) diff --git a/web/src/app/app/settings/billing/AIUsageCard.tsx b/web/src/app/app/settings/billing/AIUsageCard.tsx index 0ca46be7..4887e064 100644 --- a/web/src/app/app/settings/billing/AIUsageCard.tsx +++ b/web/src/app/app/settings/billing/AIUsageCard.tsx @@ -29,6 +29,7 @@ const REASON_LABELS: Record = { research_run: "Contact research", automation_ai: "Automation AI", campaign_ai: "Campaign switches", + spam_analysis: "Spam analysis", }; type WindowKey = "day" | "week" | "month"; diff --git a/web/src/app/app/settings/billing/CreditsCard.tsx b/web/src/app/app/settings/billing/CreditsCard.tsx index e5beb659..e36c5383 100644 --- a/web/src/app/app/settings/billing/CreditsCard.tsx +++ b/web/src/app/app/settings/billing/CreditsCard.tsx @@ -275,6 +275,8 @@ function describeReason(reason: string): string { reply_draft_refund: "Reply draft refund", agent_iteration_refund: "AI assistant refund", inbox_agent_draft: "Inbox agent", + spam_analysis: "Spam analysis", + spam_analysis_refund: "Spam analysis refund", credit_topup: "Top-up purchase", credit_auto_topup: "Auto top-up", monthly_reset: "Monthly allowance", diff --git a/web/src/components/app/campaigns/ContentScore.tsx b/web/src/components/app/campaigns/ContentScore.tsx index 5c45eb57..2d123980 100644 --- a/web/src/components/app/campaigns/ContentScore.tsx +++ b/web/src/components/app/campaigns/ContentScore.tsx @@ -1,13 +1,45 @@ -// Advisory campaign-template content check: scores the current subject + body -// against /templates/score and renders a 0-100 score (higher = safer) plus the -// non-blocking issues found, re-scored on the debounce the composer's preview -// uses. It never blocks saving or sending. +// Advisory campaign-template content check. Two passes over the same copy: +// +// - the rules pass (/templates/score), free and re-run on the debounce the +// composer's preview uses. Each issue carries where it is (subject or +// body) and the exact fragments that caused it, so the panel points at the +// words instead of restating the rule; +// - the AI pass (/templates/analyze), on an explicit click because it spends +// credits. It quotes the sentence a filter will object to and says what to +// write instead, and returns the rules pass with it so both halves of the +// panel are scored from one reading of the copy. +// +// Re-check re-runs whichever passes are on screen, and the panel keeps the +// previous AI score so an edit can be measured against it. Neither pass ever +// blocks saving or sending. import * as React from "react"; -import { ShieldCheckIcon, AlertTriangleIcon, AlertCircleIcon } from "lucide-react"; +import { + AlertCircleIcon, + AlertTriangleIcon, + ArrowDownIcon, + ArrowUpIcon, + InfoIcon, + RefreshCwIcon, + ShieldCheckIcon, + SparklesIcon, + WandSparklesIcon, +} from "lucide-react"; +import toast from "react-hot-toast"; import scoreTemplate from "@/lib/api/client/app/campaigns/scoreTemplate"; +import useAnalyzeTemplate from "@/lib/api/hooks/app/campaigns/useAnalyzeTemplate"; import type TemplateScore from "@/lib/api/models/app/campaigns/TemplateScore"; -import type { TemplateScoreIssue } from "@/lib/api/models/app/campaigns/TemplateScore"; +import type { + SpamFinding, + TemplateAnalysis, + TemplateField, + TemplateScoreIssue, + TemplateScoreSpan, +} from "@/lib/api/models/app/campaigns/TemplateScore"; +import type { AppError } from "@/lib/api/client/normalizeError"; +import buildError from "@/lib/helper/buildError"; +import { usePermission } from "@/hooks/usePermission"; +import useAiMetered from "@/hooks/useAiMetered"; import { Loading } from "@/components/loader"; import { cn } from "@/lib/utils"; import { DitherMeter, type DitherTone } from "@/components/ui/dither"; @@ -18,81 +50,269 @@ function scoreTone(score: number) { return { text: "text-rose-600", meter: "rose" as DitherTone, label: "Needs work" }; } +// One value that changes whenever any part of the copy does, so an analysis +// can be told apart from the draft it was run against. +function copyKey(subject: string, bodyHtml: string, bodyPlain: string): string { + return [subject, bodyHtml, bodyPlain].join("\u0000"); +} + +const FIELD_LABEL: Record = { subject: "Subject", body: "Body" }; + +// The first thing a writer needs is which box to open. +function FieldBadge({ field, line }: { field?: TemplateField; line?: number }) { + if (!field) return null; + return ( + + {FIELD_LABEL[field]} + {field === "body" && !!line && line > 1 && line {line}} + + ); +} + +// The offending words, quoted exactly as they are written in the copy. +function Quoted({ text }: { text: string }) { + return ( + + {text} + + ); +} + +function SpanList({ spans }: { spans: TemplateScoreSpan[] }) { + if (spans.length === 0) return null; + return ( +
+ {spans.map((s, i) => ( + + ))} +
+ ); +} + function IssueRow({ issue }: { issue: TemplateScoreIssue }) { const high = issue.severity === "high"; const Icon = high ? AlertCircleIcon : AlertTriangleIcon; + const spans = issue.spans ?? []; + const excerpt = spans[0]?.excerpt; return (
  • -
    - {issue.message} - {issue.code} +
    +
    + + {issue.message} + {issue.code} +
    + + {excerpt && excerpt !== spans[0]?.text && ( +

    {excerpt}

    + )} + {issue.suggestion &&

    {issue.suggestion}

    }
  • ); } +const FINDING_ICON = { + high: { Icon: AlertCircleIcon, className: "text-rose-500" }, + warn: { Icon: AlertTriangleIcon, className: "text-amber-500" }, + info: { Icon: InfoIcon, className: "text-sky-500" }, +} as const; + +function FindingRow({ finding }: { finding: SpamFinding }) { + const { Icon, className } = FINDING_ICON[finding.severity] ?? FINDING_ICON.warn; + return ( +
  • + +
    +
    + + {finding.text && } +
    +

    {finding.issue}

    + {finding.suggestion && ( +

    Try: {finding.suggestion}

    + )} +
    +
  • + ); +} + +// Movement since the previous check, which is what makes Re-check worth +// pressing: it answers whether the edit helped. +function ScoreDelta({ from, to }: { from: number; to: number }) { + const diff = to - from; + if (diff === 0) return No change since your last check.; + const up = diff > 0; + const Icon = up ? ArrowUpIcon : ArrowDownIcon; + return ( + + + {up ? "+" : ""} + {diff} since your last check + + ); +} + export default function ContentScore({ subject, bodyHtml, bodyPlain, + onApplySubject, }: { subject: string; bodyHtml: string; bodyPlain: string; + // Lets the suggested subject be applied in one click. Without it the + // suggestion still shows, it just has to be copied by hand. + onApplySubject?: (subject: string) => void; }) { const [data, setData] = React.useState(null); const [pending, setPending] = React.useState(false); const [failed, setFailed] = React.useState(false); + const [analysis, setAnalysis] = React.useState(null); + // The copy the analysis was run against, so the panel can say when it has + // gone stale rather than presenting old findings as current. + const [analyzedCopy, setAnalyzedCopy] = React.useState(""); + const [previousAiScore, setPreviousAiScore] = React.useState(null); + // A deployment with no provider configured answers 503 once; the button + // then stays out of the way instead of offering something that cannot run. + const [aiUnavailable, setAiUnavailable] = React.useState(false); + + const canAI = usePermission("USE_AI"); + const metered = useAiMetered(); + const analyzeMut = useAnalyzeTemplate(); + + const empty = !subject.trim() && !bodyPlain.trim(); + const currentCopy = copyKey(subject, bodyHtml, bodyPlain); + const stale = !!analysis && analyzedCopy !== currentCopy; + + // Only the newest rules request may write the panel: a manual Re-check and + // a debounced keystroke can be in flight together, and the slower one + // landing last would show a score for copy that is no longer there. + const runID = React.useRef(0); + const runScore = React.useCallback(() => { + const id = ++runID.current; + setPending(true); + scoreTemplate({ subject, body_html: bodyHtml, body_plain: bodyPlain }) + .then((res) => { + if (id !== runID.current) return; + setData(res); + setFailed(false); + }) + .catch(() => { + if (id === runID.current) setFailed(true); + }) + .finally(() => { + if (id === runID.current) setPending(false); + }); + }, [bodyHtml, bodyPlain, subject]); React.useEffect(() => { // A step with nothing written yet is not a content problem, so hold the // panel quiet rather than scoring an empty draft as spam. - if (!subject.trim() && !bodyPlain.trim()) { - // Clears pending too: a cancelled request can no longer do it. + if (empty) { + // Retires any in-flight request too, so its result cannot land. + runID.current++; setData(null); setPending(false); setFailed(false); return; } - let cancelled = false; - // Set inside the timer so the spinner marks a request, not a keystroke. - const t = setTimeout(() => { - setPending(true); - scoreTemplate({ subject, body_html: bodyHtml, body_plain: bodyPlain }) - .then((res) => { - if (cancelled) return; - setData(res); + // Fired inside the timer so the spinner marks a request, not a keystroke. + const t = setTimeout(runScore, 600); + return () => clearTimeout(t); + }, [empty, runScore]); + + const runAnalysis = React.useCallback(() => { + if (empty || analyzeMut.isPending) return; + const copy = copyKey(subject, bodyHtml, bodyPlain); + analyzeMut.mutate( + { subject, body_html: bodyHtml, body_plain: bodyPlain }, + { + onSuccess: (res) => { + // The score being replaced is what the new one is measured + // against, so it is captured before the swap. + setPreviousAiScore(analysis ? analysis.score : null); + setAnalysis(res); + setAnalyzedCopy(copy); + // One request scored both passes; keep them in step, and + // retire any rules request still in flight behind it. + runID.current++; + setData(res.rules); + setPending(false); setFailed(false); - }) - .catch(() => { - if (!cancelled) setFailed(true); - }) - .finally(() => { - if (!cancelled) setPending(false); - }); - }, 600); - return () => { - cancelled = true; - clearTimeout(t); - }; - }, [subject, bodyHtml, bodyPlain]); + }, + onError: (e) => { + const err = e as unknown as AppError; + if (err?.status === 402) { + toast.error("You're out of AI credits. Add more to keep using AI analysis."); + } else if (err?.code === "ai_not_configured") { + // Permanent for this deployment, unlike a provider + // outage, so the button goes away rather than staying + // there to fail again. + setAiUnavailable(true); + } else { + toast.error(buildError(err)); + } + }, + }, + ); + }, [analysis, analyzeMut, bodyHtml, bodyPlain, empty, subject]); + + // Re-check runs everything currently on screen: the rules pass always, and + // the AI pass too once the writer has asked for one. + const aiInPlay = !!analysis && canAI && !aiUnavailable; + const recheck = React.useCallback(() => { + if (empty) return; + if (aiInPlay) { + runAnalysis(); + return; + } + runScore(); + }, [aiInPlay, empty, runAnalysis, runScore]); const tone = data ? scoreTone(data.score) : null; + const aiTone = analysis ? scoreTone(analysis.score) : null; + const busy = pending || analyzeMut.isPending; + const suggestedSubject = analysis?.suggested_subject?.trim(); return (
    Content check
    -

    Advisory deliverability score. It never blocks sending.

    +

    + Advisory deliverability score. It never blocks sending. +

    +
    +
    + {busy && } +
    - {pending && }
    - {failed && ( -
    Couldn't score this template.
    - )} + {failed &&
    Couldn't score this template.
    } {data && tone && (
    @@ -120,6 +340,124 @@ export default function ContentScore({ )}
    )} + + {canAI && !aiUnavailable && !analysis && ( +
    + +

    + Reads the copy and names the words and sentences that hurt deliverability, in the subject and in + the body. +

    +
    + )} + + {aiUnavailable && !analysis && ( +
    + AI analysis is not configured on this deployment. +
    + )} + + {analysis && aiTone && ( +
    +
    + + AI analysis + {analysis.model} +
    + +
    + + {analysis.score} + + / 100 + {aiTone.label} +
    + + {previousAiScore !== null && ( +
    + +
    + )} + + {stale && ( +

    + You've edited the copy since this analysis. Re-check to score the new version. +

    + )} + + {analysis.verdict && ( +

    {analysis.verdict}

    + )} + + {analysis.findings.length > 0 ? ( +
      + {analysis.findings.map((finding, i) => ( + + ))} +
    + ) : ( +

    + Nothing in this copy stood out as spammy. +

    + )} + + {suggestedSubject && suggestedSubject !== subject.trim() && ( +
    +
    + Suggested subject +
    +

    {suggestedSubject}

    + {onApplySubject && ( + + )} +
    + )} + + {(analysis.improvements?.length ?? 0) > 0 && ( +
    +
    + Also worth doing +
    +
      + {analysis.improvements!.map((tip, i) => ( +
    • + + {tip} +
    • + ))} +
    +
    + )} + + {metered && analysis.credits_charged > 0 && ( +

    + {analysis.credits_charged} credit{analysis.credits_charged === 1 ? "" : "s"} spent,{" "} + {analysis.credits_remaining} left +

    + )} +
    + )}
    ); } diff --git a/web/src/components/app/campaigns/sequences/EmailContentEditor.tsx b/web/src/components/app/campaigns/sequences/EmailContentEditor.tsx index f2eddc74..5d14401d 100644 --- a/web/src/components/app/campaigns/sequences/EmailContentEditor.tsx +++ b/web/src/components/app/campaigns/sequences/EmailContentEditor.tsx @@ -405,7 +405,12 @@ export default function EmailContentEditor({ )} - + ); } diff --git a/web/src/lib/api/client/app/campaigns/analyzeTemplate.ts b/web/src/lib/api/client/app/campaigns/analyzeTemplate.ts new file mode 100644 index 00000000..80d64b50 --- /dev/null +++ b/web/src/lib/api/client/app/campaigns/analyzeTemplate.ts @@ -0,0 +1,19 @@ +import type { + ScoreTemplateRequest, + TemplateAnalysis, +} from "@/lib/api/models/app/campaigns/TemplateScore"; +import Request from "../../Request"; + +// AI spam analysis of a campaign template: the rules score plus the specific +// words and sentences a filter objects to, where they are, and what to write +// instead. Spends AI credits, so it only ever runs on an explicit click. +export default async function analyzeTemplate( + body: ScoreTemplateRequest, +): Promise { + return await Request({ + method: "POST", + url: "/templates/analyze", + data: body, + authorization: true, + }); +} diff --git a/web/src/lib/api/hooks/app/campaigns/useAnalyzeTemplate.ts b/web/src/lib/api/hooks/app/campaigns/useAnalyzeTemplate.ts new file mode 100644 index 00000000..8c2bb66f --- /dev/null +++ b/web/src/lib/api/hooks/app/campaigns/useAnalyzeTemplate.ts @@ -0,0 +1,17 @@ +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import analyzeTemplate from "@/lib/api/client/app/campaigns/analyzeTemplate"; +import type { ScoreTemplateRequest } from "@/lib/api/models/app/campaigns/TemplateScore"; + +// On-demand AI spam analysis. A mutation, not a query: it costs credits and +// only ever runs when the writer asks for it (Analyze, then Re-check after an +// edit). Every success refreshes the credits views so the header meter moves +// immediately. +export default function useAnalyzeTemplate() { + const qc = useQueryClient(); + return useMutation({ + mutationFn: (body: ScoreTemplateRequest) => analyzeTemplate(body), + onSuccess: () => { + void qc.invalidateQueries({ queryKey: ["subscription", "credits"] }); + }, + }); +} diff --git a/web/src/lib/api/models/app/campaigns/TemplateScore.ts b/web/src/lib/api/models/app/campaigns/TemplateScore.ts index 027a172f..028e2f55 100644 --- a/web/src/lib/api/models/app/campaigns/TemplateScore.ts +++ b/web/src/lib/api/models/app/campaigns/TemplateScore.ts @@ -1,10 +1,30 @@ // Result of POST /templates/score — an advisory content-quality score for a // campaign template. Higher score = safer; issues are non-blocking hints to // improve deliverability (the score never prevents saving or sending). + +// Which half of the template a finding sits in. +export type TemplateField = "subject" | "body"; + +// One exact fragment that triggered an issue, so the editor can point at the +// words instead of restating the rule. +export interface TemplateScoreSpan { + field: TemplateField; + /** The fragment as it is written in the copy. */ + text: string; + /** 1-based line within that field. */ + line?: number; + /** The whole line, for context around the fragment. */ + excerpt?: string; +} + export interface TemplateScoreIssue { severity: "warn" | "high"; code: string; message: string; + /** Set when the issue lives in exactly one half of the template. */ + field?: TemplateField; + spans?: TemplateScoreSpan[]; + suggestion?: string; } export default interface TemplateScore { @@ -12,9 +32,39 @@ export default interface TemplateScore { issues: TemplateScoreIssue[]; } -// Body for POST /templates/score. +// Body for POST /templates/score and POST /templates/analyze. export interface ScoreTemplateRequest { subject: string; body_html: string; body_plain: string; } + +// One located problem the AI analysis found in the copy. +export interface SpamFinding { + severity: "high" | "warn" | "info"; + field: TemplateField; + /** The exact fragment quoted from the copy, absent when the finding is + * about the email as a whole. Verified server-side against the template, + * so it is never a sentence the writer did not write. */ + text?: string; + line?: number; + excerpt?: string; + issue: string; + suggestion?: string; + category?: string; +} + +// Result of POST /templates/analyze — the AI half of the content check. +export interface TemplateAnalysis { + score: number; + verdict: string; + findings: SpamFinding[]; + suggested_subject?: string; + improvements?: string[]; + /** The rules pass, scored from the same copy in the same request. */ + rules: TemplateScore; + model: string; + tokens_used: number; + credits_remaining: number; + credits_charged: number; +}