diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b9325626..087b1871f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -198,11 +198,11 @@ jobs: go run ./cmd/migrate go test -race ./internal/app/behavior ./internal/repository ./internal/tasks -run 'TestDefaultPlan|TestLiveMailboxPlan|TestLiveCampaignAdmission|TestLiveSharedAdmission|TestLiveDeferredTicksDoNotSpendTheDailyBudget|TestLiveCampaignAtDailyCapWaitsForTomorrow' -count=1 - - name: Warmup dispatch and upgrade regressions (PostgreSQL) + - name: Warmup dispatch, upgrade and admin-send regressions (PostgreSQL) env: WARMBLY_TEST_DB: postgres://warmbly:warmbly@127.0.0.1:5432/warmbly_test?sslmode=disable run: | - go test -race ./internal/repository -run '^TestLive(LastWarmupSendFailure|PersistentMailboxLoadingFailure|DispatchRetry|LineageQueueRevisions|WarmupRecoversOldPreparation|WarmupPreparationRecoveryRetains)' -count=1 + go test -race ./internal/repository -run '^TestLive(LastWarmupSendFailure|PersistentMailboxLoadingFailure|DispatchRetry|LineageQueueRevisions|AdminPlatformOverview|WarmupRecoversOldPreparation|WarmupPreparationRecoveryRetains)' -count=1 go test -race ./internal/tasks -run '^Test(LiveWarmup(SafetyHoldReschedules|Recovery|TaskDoesNotApplyResults)|WarmupSenderDistinguishes)' -count=1 - name: Test the Kafka codec and client lifecycle diff --git a/docs/content/docs/api/endpoints.mdx b/docs/content/docs/api/endpoints.mdx index b776621f0..ff3664ffd 100644 --- a/docs/content/docs/api/endpoints.mdx +++ b/docs/content/docs/api/endpoints.mdx @@ -610,6 +610,7 @@ Alongside them, `api_url` is this API's own public base (for a copyable example - All of `/auth/cli/*` except the two handshake routes below (`GET /auth/cli/codes/:code`, `POST /auth/cli/codes/:code/approve`, `POST /auth/cli/codes/:code/deny`: the browser half of `warmbly auth login`, where a signed-in member reviews the code a CLI is showing and authorizes it. Approving mints an ordinary API key, so it requires the `MANAGE_API_KEYS` organization permission and a fresh authentication, is session-only (an API key must not be able to mint another one this way), and caps the key to the scopes the approver's role allows. `GET /auth/cli/codes/:code?organization_id=` adds `granted_scopes` and `granted_scope_names`: what approving into that workspace would grant) - All of `/pool-link/*` and `/cloud-link/*` (the self-hosted warmup pool link: approving an instance's code, listing and unlinking instances, reading the pool plan's price and opening its checkout, and on a self-hosted instance the connect flow and mailbox enrollment). `/cloud-link/*` exists only on a self-hosted instance. Connections belong to the selected local workspace, but linking and unlinking them (`/connect`, `/connect/poll`, `DELETE /cloud-link`) and the linked cloud workspace's mailboxes (`/workspace-mailboxes`, `.../adopt`) take a platform admin holding `manage_settings` whose session presented a second factor (`admin_mfa_required` otherwise). Reading the link and putting a workspace's own mailboxes on it (`GET /cloud-link`, `/mailboxes`, enroll, pause, resume and the Google and Microsoft sign-in under `/oauth/*`) take the workspace's `MANAGE_EMAILS`. `GET /cloud-link` includes `legacy_connected`; mailbox rows include `legacy` when they still use the old connection. A normal `DELETE /cloud-link` only disconnects the selected workspace's scoped link. `DELETE /cloud-link?legacy=true` explicitly disconnects the legacy link across all workspaces and uses the same platform-admin permission. New connections and enrollments require workspace scope, and a Cloud workspace accepts only one active link (legacy links count). `POST /pool-link/codes` requires `remote_organization_id` and returns `workspace_scoped: true` for supported clients. `POST /pool-link/codes` and `POST /pool-link/poll` are public and per-IP rate limited: they are the device-code handshake an instance uses before it has a token, and `/pool-link/instance/*` accepts only an instance token. `/pool-link/instance/oauth/*`, `/pool-link/instance/mailboxes/:id/token`, `/pool-link/instance/workspace-mailboxes` and `/pool-link/instance/mailboxes/adopt` are the cloud-managed mailbox surface (Google and Microsoft sign-in on Warmbly's OAuth apps, brokered access tokens); their instance-side counterparts are `/cloud-link/oauth/*` and `/cloud-link/workspace-mailboxes/*`. `/pool-link/instance/redirects` (list), and `GET`, `PUT`, `DELETE` and `POST .../verify` on `/pool-link/instance/redirects/:domain`, are the root redirects Warmbly Cloud serves for the calling instance; the instance drives them through `PUT /emails/domains/:domain/redirect` with `served_by: "cloud"`, and `GET /pool-link/instance` carries `redirects` (`available`, `host`, `limit`, `used`) - All of `/admin/*`. `DELETE /admin/instance/invitations/expired` requires platform admin `manage_organizations` and an MFA-verified session. It removes only expired invitation records across all workspaces, leaves active invitations and membership unchanged, records an admin audit entry, and returns `{ "cleaned": true }`. The admin panel confirms before calling it. + - `GET /admin/analytics/overview` requires `view_analytics`. `total_emails_sent` and `emails_sent_today` count confirmed warmup and other outbound sends, not dispatches. Today uses the UTC calendar day. A send-count database error fails the request instead of returning zero counters. See [platform admin send counters](/guides/analytics/#platform-admin-send-counters). - `GET /admin/organizations` and `GET /admin/organizations/:id` require `view_organizations`. Their acquisition fields include the nullable `referrer_host` saved at signup, alongside UTM source, medium, campaign and landing path. The list accepts an exact `referrer_host` filter as well as its existing acquisition filters. An absent source is unknown, not proof of a direct visit. See [signup source attribution](/guides/analytics/#signup-source-attribution-platform-admins). - `GET /admin/warmup-content/overview` includes effective generation controls and requires `view_warmup_pool`; `PUT /admin/warmup-content/settings` writes effective controls with `manage_settings`. These are platform-admin routes, not organization API-key scopes. See [effective generation policy and stopping](/development/warmup-generation/). - `POST /admin/organizations/:id/sample-data`: `manage_testers` and an MFA-verified admin session. Adds the [reviewer sample dataset](/development/accounts-and-access/#adding-sample-data) only to an active dedicated Test workspace. No request body. Returns `200` with `organization_id`, `created` (false when already seeded) and `seeded_at`; retries preserve edits and deletions. An ordinary workspace, revoked or expired tester, or missing/expired managed Test grant returns `400`. Inserts and the admin audit marker commit together. No mailbox connection, email delivery, external request or credit spend occurs. diff --git a/docs/content/docs/guides/analytics.mdx b/docs/content/docs/guides/analytics.mdx index a65bf6ed3..db66ef6b2 100644 --- a/docs/content/docs/guides/analytics.mdx +++ b/docs/content/docs/guides/analytics.mdx @@ -56,6 +56,12 @@ Live, scoped by `7d`, `30d`, or `90d` tabs, and by default covering every campai Sending totals, trends, comparisons, and top-campaign rates count email steps only. Completed wait and action steps move a sequence forward but never inflate the number of messages sent. +### Platform admin send counters + +The admin **Overview** counts confirmed warmup, campaign, direct email, and placement-test sends across this instance. **Emails sent today** uses the UTC calendar day, not each mailbox's local sending timezone. The all-time total uses the available confirmed task history. + +A queued or completed dispatch is not proof of sending. Warmup sends count only after their token records the provider's Message-ID. Other sends count after a successful worker result is applied. Older warmup receipts remain included, but older non-warmup tasks without applied results are not assumed successful: their Message-ID may have been generated before sending. Reserved, unknown, failed, and cancelled dispatches without confirmation do not count. Multiple warmup tokens for one task still count as one send. If the send-count query fails, the overview returns an error instead of reporting zero sends. + ### Filtering by campaign or folder The **All campaigns** menu next to the period tabs narrows the dashboard to the campaigns you pick. Choose one campaign, several, one or more [campaign folders](/guides/campaigns/), or a mix. The figures add the selected campaigns together, and a campaign picked directly and through a folder, or sitting in two picked folders, is counted once. A folder stands for the campaigns in it right now, so moving a campaign in or out of the folder changes what the dashboard counts the next time it loads. The period tabs keep working on top of the filter, and the selection is remembered in your browser tab like the period. diff --git a/internal/repository/admin_schema_live_test.go b/internal/repository/admin_schema_live_test.go index f4823abf8..8604a7a8c 100644 --- a/internal/repository/admin_schema_live_test.go +++ b/internal/repository/admin_schema_live_test.go @@ -28,6 +28,105 @@ type adminFixture struct { tag string } +func TestLiveAdminPlatformOverviewConfirmedSends(t *testing.T) { + _, pool := liveContactDB(t) + ctx := t.Context() + f := newAdminFixture(t, pool) + config := pool.Config() + config.ConnConfig.RuntimeParams["timezone"] = "Pacific/Honolulu" + localPool, err := pgxpool.NewWithConfig(ctx, config) + if err != nil { + t.Fatal(err) + } + defer localPool.Close() + repo := &adminRepository{db: localPool} + before, err := repo.GetPlatformOverview(ctx) + if err != nil { + t.Fatal(err) + } + var ids []uuid.UUID + t.Cleanup(func() { + for _, query := range []string{ + `DELETE FROM warmup_tokens WHERE task_id = ANY($1::uuid[])`, + `DELETE FROM tasks WHERE id = ANY($1::uuid[])`, + } { + if _, err := pool.Exec(context.Background(), query, ids); err != nil { + t.Errorf("cleanup: %v", err) + } + } + }) + today := time.Now().UTC().Truncate(24 * time.Hour) + yesterday := today.Add(-24 * time.Hour) + tomorrow := today.Add(24 * time.Hour) + for _, tt := range []struct { + name, kind, status, messageID, state string + completedAt time.Time + appliedAt *time.Time + tokens int + confirmedToken bool + }{ + {"legacy warmup", "warmup", "completed", "", "", yesterday, nil, 1, true}, + {"delayed warmup result", "warmup", "completed", "planned@test.local", "sent", yesterday, &today, 2, true}, + {"campaign result", "campaign", "completed", "", "sent", yesterday, &today, 0, false}, + {"legacy direct dispatch", "email", "completed", "planned@test.local", "", today, nil, 0, false}, + {"direct result", "email", "completed", "", "sent", today, &today, 0, false}, + {"placement result", "placement", "completed", "", "sent", today, &today, 0, false}, + {"future receipt", "email", "completed", "", "sent", today, &tomorrow, 0, false}, + {"warmup dispatch", "warmup", "completed", "planned@test.local", "unknown", today, nil, 1, false}, + {"warmup without receipt", "warmup", "completed", "planned@test.local", "sent", today, &today, 1, false}, + {"campaign dispatch", "campaign", "completed", "", "", today, nil, 0, false}, + {"unknown with message ID", "email", "completed", "planned@test.local", "unknown", today, nil, 0, false}, + {"unapplied success", "campaign", "completed", "planned@test.local", "sent", today, nil, 0, false}, + {"legacy failure", "email", "failed", "planned@test.local", "", today, nil, 0, false}, + {"failed result", "campaign", "failed", "planned@test.local", "failed", today, &today, 0, false}, + {"cancelled dispatch", "warmup", "cancelled", "planned@test.local", "failed", today, &today, 1, false}, + } { + id := uuid.New() + ids = append(ids, id) + _, err := pool.Exec(ctx, `INSERT INTO tasks + (id, task_type, email_account_id, status, message_id, completed_at, send_result_state, send_result_applied_at) + VALUES ($1, $2, $3, $4, $5, $6, NULLIF($7, ''), $8)`, + id, tt.kind, f.mailbox, tt.status, tt.messageID, tt.completedAt, tt.state, tt.appliedAt) + if err != nil { + t.Fatalf("%s: %v", tt.name, err) + } + for range tt.tokens { + messageID := "" + if tt.confirmedToken { + messageID = "" + } + _, err := pool.Exec(ctx, `INSERT INTO warmup_tokens + (token, task_id, sender_account_id, recipient_account_id, conversation_turn, sent_message_id) + VALUES ($1, $2, $3, $3, 0, $4)`, uuid.New(), id, f.mailbox, messageID) + if err != nil { + t.Fatalf("%s token: %v", tt.name, err) + } + } + } + after, err := repo.GetPlatformOverview(ctx) + if err != nil { + t.Fatal(err) + } + if got := after.TotalEmailsSent - before.TotalEmailsSent; got != 6 { + t.Fatalf("confirmed total delta = %d, want 6", got) + } + if got := after.EmailsSentToday - before.EmailsSentToday; got != 4 { + t.Fatalf("confirmed UTC today delta = %d, want 4", got) + } +} + +func TestAdminPlatformOverviewReturnsSendCountError(t *testing.T) { + pool, err := pgxpool.New(t.Context(), "postgres://test:test@127.0.0.1:1/test?sslmode=disable") + if err != nil { + t.Fatal(err) + } + pool.Close() + got, err := (&adminRepository{db: pool}).GetPlatformOverview(t.Context()) + if err == nil || got != nil { + t.Fatalf("closed database: got %+v, %v; want an error, not zero counters", got, err) + } +} + func newAdminFixture(t *testing.T, pool *pgxpool.Pool) *adminFixture { t.Helper() ctx := context.Background() diff --git a/internal/repository/pg_admin.go b/internal/repository/pg_admin.go index 09b7ecac5..abdb8b66f 100644 --- a/internal/repository/pg_admin.go +++ b/internal/repository/pg_admin.go @@ -1711,6 +1711,25 @@ func (r *adminRepository) SearchAuditLogs(ctx context.Context, search *models.Ad // GetPlatformOverview gets high-level platform statistics func (r *adminRepository) GetPlatformOverview(ctx context.Context) (*models.PlatformOverview, error) { overview := &models.PlatformOverview{} + err := r.db.QueryRow(ctx, ` + WITH confirmed_sends AS ( + SELECT COALESCE(t.send_result_applied_at, t.completed_at) AS sent_at + FROM tasks t + WHERE (t.task_type = 'warmup' AND EXISTS ( + SELECT 1 FROM warmup_tokens wt + WHERE wt.task_id = t.id AND wt.sent_message_id <> '' + )) OR (t.task_type <> 'warmup' AND t.send_result_state = 'sent' + AND t.send_result_applied_at IS NOT NULL) + ) + SELECT COUNT(*), COUNT(*) FILTER ( + WHERE sent_at >= (date_trunc('day', NOW() AT TIME ZONE 'UTC') AT TIME ZONE 'UTC') + AND sent_at <= NOW() + ) + FROM confirmed_sends + `).Scan(&overview.TotalEmailsSent, &overview.EmailsSentToday) + if err != nil { + return nil, err + } // Total users r.db.QueryRow(ctx, `SELECT COUNT(*) FROM users`).Scan(&overview.TotalUsers)