feat: cut database load from issue 993 with placement, sweep and unibox indexes, redundant index drops, identical-result rewrites of contact facets, dashboard stats, campaign progress, advisor bounces and routing pre-filter, bounded delivery evidence, cold-ramp skip, held advisor sections, ANALYZE after restore and hidden-tab realtime deferral

web/src/hooks/useRealtimeEvents.ts

@@ -7,7 +7,12 @@ import { useUserProfile } from './context/user'
 import { markSelfMutation } from '@/lib/realtime/selfActivity'
 import { announceCampaignDeleted } from '@/lib/realtime/campaignDeleted'
 import { MAILBOX_REMOVAL_KEYS } from '@/lib/api/hooks/app/emails/invalidateAfterMailboxRemoval'
-import { createDeliveryDeduper, createRefreshCoalescer } from '@/lib/realtime/refreshCoalescer'
+import {
+  createDeliveryDeduper,
+  createHiddenTabGate,
+  createRefreshCoalescer,
+  type QueryMatch,
+} from '@/lib/realtime/refreshCoalescer'
 
 // What an inbox event moves besides its own thread; refreshed through the coalescer, never per event.
 const INBOX_AGGREGATES: QueryKey[] = [
@@ -33,13 +38,17 @@ export function useRealtimeEvents() {
   const updateDeal = useAppStore((s) => s.updateDeal)
   const setSubscription = useAppStore((s) => s.setSubscription)
 
+  // Every realtime invalidation goes through this, so a hidden tab refetches once on return instead of per event.
+  const [gate] = useState(() => createHiddenTabGate(queryClient))
+  useEffect(() => () => gate.dispose(), [gate])
+
   const invalidate = useCallback(
     (queryKeys: QueryKey[]) => {
       for (const queryKey of queryKeys) {
-        void queryClient.invalidateQueries({ queryKey })
+        gate.invalidate({ queryKey })
       }
     },
-    [queryClient],
+    [gate],
   )
 
   // Warmup deliveries arrive all day across a whole pool, so their refreshes
@@ -67,13 +76,16 @@ export function useRealtimeEvents() {
   }, [])
 
   // One aggregate refresh per quiet interval; per-thread work once per delivery, not per channel.
-  const [inboxRefresh] = useState(() => createRefreshCoalescer(queryClient))
+  const [inboxRefresh] = useState(() => createRefreshCoalescer(queryClient, { invalidate: gate.invalidate }))
   const [isDuplicateDelivery] = useState(() => createDeliveryDeduper())
   useEffect(() => () => inboxRefresh.dispose(), [inboxRefresh])
   // Mailbox events arrive in bursts (a fleet reconnect, a pool re-evaluation),
   // and each one refetches the whole mailbox list and every status chunk.
-  const [accountRefresh] = useState(() => createRefreshCoalescer(queryClient))
+  const [accountRefresh] = useState(() => createRefreshCoalescer(queryClient, { invalidate: gate.invalidate }))
   useEffect(() => () => accountRefresh.dispose(), [accountRefresh])
+  // One send publishes several campaign events within a second or two; they share one refetch.
+  const [campaignRefresh] = useState(() => createRefreshCoalescer(queryClient, { invalidate: gate.invalidate }))
+  useEffect(() => () => campaignRefresh.dispose(), [campaignRefresh])
 
   const handleRealtimeEvent = useCallback(
     (payload: Record<string, unknown>) => {
@@ -304,7 +316,7 @@ export function useRealtimeEvents() {
         if (campaignId) {
           const status = getString('status')
           updateCampaign(campaignId, status ? { status } : {})
-          invalidate([
+          campaignRefresh.add([
             ['campaigns', campaignId],
             ['campaigns', campaignId, 'logs'],
             ['analytics', 'campaigns', campaignId],
@@ -312,15 +324,23 @@ export function useRealtimeEvents() {
             ['analytics', 'campaigns', campaignId, 'hourly'],
           ])
         }
-        invalidate([
+        campaignRefresh.add([
           ['campaigns', 'list'],
           ['analytics'],
         ])
-        // Other contacts' campaign-state (a scheduler preview per campaign) is left alone.
-        void queryClient.invalidateQueries({
-          queryKey: ['contacts'],
-          predicate: (q) => !contactId || q.queryKey[2] !== 'campaign-state' || q.queryKey[1] === contactId,
-        })
+        // A reply can open a CRM task or deal (reply intents, reply actions), which publish no event of their own.
+        if (includes('EMAIL_REPLIED')) campaignRefresh.add([['crm', 'tasks'], ['crm', 'deals']])
+        // Sends and progress record no verification evidence; opens, clicks, replies and bounces do.
+        const movesVerification = !includes('EMAIL_SENT', 'TASK_PROGRESS')
+        const contactsMatch: QueryMatch = (q) => {
+          const [, scope, part] = q.queryKey
+          // Custom field keys are never written by a send, open, click, reply or bounce.
+          if (scope === 'custom-fields') return false
+          if (scope === 'verification') return movesVerification
+          // Other contacts' campaign-state (a scheduler preview per campaign) is left alone.
+          return !contactId || part !== 'campaign-state' || scope === contactId
+        }
+        campaignRefresh.add([['contacts']], contactsMatch)
         return
       }
 
@@ -373,11 +393,17 @@ export function useRealtimeEvents() {
         return
       }
 
-      if (includes('CRM_TASK', 'TASK')) {
+      if (includes('CRM_TASK')) {
         invalidate([['crm', 'tasks'], ['crm', 'deals']])
         return
       }
 
+      // Campaign task status and progress pulse: system-created CRM tasks and deals publish nothing of their own, so refresh coalesced.
+      if (event.startsWith('TASK_')) {
+        campaignRefresh.add([['crm', 'tasks'], ['crm', 'deals']])
+        return
+      }
+
       if (includes('SUBSCRIPTION', 'PLAN', 'BILLING', 'LIMIT')) {
         setSubscription(payload as any)
         invalidate([
@@ -598,7 +624,7 @@ export function useRealtimeEvents() {
           invalidate([['campaigns', entityId], ['segments', 'campaign', entityId]])
           // A campaign write can enrol leads (linking a segment), so only the
           // contact lists scoped to that campaign move, not every list.
-          void queryClient.invalidateQueries({
+          gate.invalidate({
             predicate: (q) => {
               const [root, kind, options] = q.queryKey as [unknown, unknown, { campaign_ids?: string[] } | undefined]
               return root === 'contacts' && kind === 'list' && !!options?.campaign_ids?.includes(entityId)
@@ -626,7 +652,9 @@ export function useRealtimeEvents() {
     [
       accountRefresh,
       addUniboxEmail,
+      campaignRefresh,
       currentOrg?.id,
+      gate,
       inboxRefresh,
       invalidate,
       isDuplicateDelivery,

web/src/lib/realtime/refreshCoalescer.test.ts

@@ -1,6 +1,6 @@
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
-import { QueryClient } from "@tanstack/react-query";
-import { createDeliveryDeduper, createRefreshCoalescer } from "./refreshCoalescer";
+import { QueryClient, QueryObserver } from "@tanstack/react-query";
+import { createDeliveryDeduper, createHiddenTabGate, createRefreshCoalescer } from "./refreshCoalescer";
 
 describe("createRefreshCoalescer", () => {
     let client: QueryClient;
@@ -82,6 +82,114 @@ describe("createRefreshCoalescer", () => {
     });
 });
 
+describe("createRefreshCoalescer with a match", () => {
+    let client: QueryClient;
+
+    beforeEach(() => {
+        vi.useFakeTimers();
+        client = new QueryClient();
+        for (const key of [["contacts", "list"], ["contacts", "a", "campaign-state"], ["contacts", "b", "campaign-state"], ["contacts", "c", "campaign-state"]]) {
+            client.setQueryData(key, 1);
+        }
+    });
+    afterEach(() => {
+        vi.useRealTimers();
+        client.clear();
+    });
+
+    const invalidated = () =>
+        client.getQueryCache().getAll().filter((q) => q.state.isInvalidated).map((q) => q.queryKey.join("/")).sort();
+
+    it("refreshes the union of what each narrowed add asked for", () => {
+        const c = createRefreshCoalescer(client, { delayMs: 10 });
+        const notOtherState = (id: string) => (q: { queryKey: readonly unknown[] }) => q.queryKey[2] !== "campaign-state" || q.queryKey[1] === id;
+        c.add([["contacts"]], notOtherState("a"));
+        c.add([["contacts"]], notOtherState("b"));
+        vi.advanceTimersByTime(10);
+        expect(invalidated()).toEqual(["contacts/a/campaign-state", "contacts/b/campaign-state", "contacts/list"]);
+    });
+
+    it("lets an unnarrowed add widen a narrowed one", () => {
+        const c = createRefreshCoalescer(client, { delayMs: 10 });
+        c.add([["contacts"]], (q) => q.queryKey[1] === "list");
+        c.add([["contacts"]]);
+        vi.advanceTimersByTime(10);
+        expect(invalidated()).toHaveLength(4);
+    });
+
+    it("does not treat a narrowed broader key as covering a narrower one", () => {
+        const c = createRefreshCoalescer(client, { delayMs: 10 });
+        c.add([["contacts"]], (q) => q.queryKey[1] === "list");
+        c.add([["contacts", "c"]]);
+        vi.advanceTimersByTime(10);
+        expect(invalidated()).toEqual(["contacts/c/campaign-state", "contacts/list"]);
+    });
+});
+
+describe("createHiddenTabGate", () => {
+    let client: QueryClient;
+    let visibility: DocumentVisibilityState;
+
+    const setVisibility = (state: DocumentVisibilityState) => {
+        visibility = state;
+        document.dispatchEvent(new Event("visibilitychange"));
+    };
+    const observe = (key: string[]) => {
+        const fn = vi.fn(async () => 1);
+        const observer = new QueryObserver(client, { queryKey: key, queryFn: fn });
+        const unsubscribe = observer.subscribe(() => {});
+        return { fn, unsubscribe };
+    };
+
+    beforeEach(() => {
+        client = new QueryClient();
+        visibility = "visible";
+        vi.spyOn(document, "visibilityState", "get").mockImplementation(() => visibility);
+    });
+    afterEach(() => {
+        vi.restoreAllMocks();
+        client.clear();
+    });
+
+    it("refetches at once on a visible tab", async () => {
+        const gate = createHiddenTabGate(client);
+        const list = observe(["campaigns", "list"]);
+        await vi.waitFor(() => expect(client.getQueryData(["campaigns", "list"])).toBe(1));
+        gate.invalidate({ queryKey: ["campaigns"] });
+        await vi.waitFor(() => expect(list.fn).toHaveBeenCalledTimes(2));
+        list.unsubscribe();
+        gate.dispose();
+    });
+
+    it("defers a hidden tab's refetches to one on return, keeping the unread badge live", async () => {
+        const gate = createHiddenTabGate(client);
+        const list = observe(["campaigns", "list"]);
+        const badge = observe(["unibox", "unseen-count"]);
+        const overview = observe(["unibox", "overview"]);
+        await vi.waitFor(() => expect(client.isFetching()).toBe(0));
+
+        setVisibility("hidden");
+        for (let i = 0; i < 5; i++) {
+            gate.invalidate({ queryKey: ["campaigns", "list"] });
+            gate.invalidate({ queryKey: ["unibox"] });
+        }
+        await vi.waitFor(() => expect(badge.fn).toHaveBeenCalledTimes(6));
+        expect(list.fn).toHaveBeenCalledTimes(1);
+        expect(overview.fn).toHaveBeenCalledTimes(1);
+        expect(client.getQueryState(["campaigns", "list"])?.isInvalidated).toBe(true);
+
+        setVisibility("visible");
+        await vi.waitFor(() => expect(list.fn).toHaveBeenCalledTimes(2));
+        await vi.waitFor(() => expect(overview.fn).toHaveBeenCalledTimes(2));
+        setVisibility("hidden");
+        setVisibility("visible");
+        await Promise.resolve();
+        expect(list.fn).toHaveBeenCalledTimes(2);
+        for (const o of [list, badge, overview]) o.unsubscribe();
+        gate.dispose();
+    });
+});
+
 describe("createDeliveryDeduper", () => {
     beforeEach(() => vi.useFakeTimers());
     afterEach(() => vi.useRealTimers());

web/src/lib/realtime/refreshCoalescer.ts

@@ -1,10 +1,21 @@
 // Bounds realtime refetches: one invalidation per quiet interval (capped), and one handling per delivered event.
 
-import { hashKey, partialMatchKey, type QueryClient, type QueryKey } from "@tanstack/react-query";
+import {
+    hashKey,
+    partialMatchKey,
+    type InvalidateOptions,
+    type InvalidateQueryFilters,
+    type Query,
+    type QueryClient,
+    type QueryKey,
+} from "@tanstack/react-query";
+
+export type QueryMatch = (query: Query) => boolean;
+export type Invalidate = (filters: InvalidateQueryFilters, options?: InvalidateOptions) => void;
 
 export interface RefreshCoalescer {
-    /** Queues keys (prefixes) for the next flush. */
-    add(keys: readonly QueryKey[]): void;
+    /** Queues keys (prefixes) for the next flush, optionally narrowed to the queries `match` accepts. */
+    add(keys: readonly QueryKey[], match?: QueryMatch): void;
     /** Drops anything pending without invalidating it. */
     dispose(): void;
 }
@@ -14,18 +25,29 @@ export interface RefreshCoalescerOptions {
     delayMs?: number;
     /** Longest a key may wait while adds keep arriving. */
     maxWaitMs?: number;
+    /** Runs each flushed invalidation; defaults to the client's own. */
+    invalidate?: Invalidate;
 }
 
 // Kept above the backend's overviewFreshFor (internal/app/unibox/overview_cache.go).
 export const REFRESH_DELAY_MS = 1500;
 export const REFRESH_MAX_WAIT_MS = 5000;
 
+interface Pending {
+    key: QueryKey;
+    match?: QueryMatch;
+}
+
 export function createRefreshCoalescer(
     queryClient: QueryClient,
-    { delayMs = REFRESH_DELAY_MS, maxWaitMs = REFRESH_MAX_WAIT_MS }: RefreshCoalescerOptions = {},
+    {
+        delayMs = REFRESH_DELAY_MS,
+        maxWaitMs = REFRESH_MAX_WAIT_MS,
+        invalidate = (filters, options) => void queryClient.invalidateQueries(filters, options),
+    }: RefreshCoalescerOptions = {},
 ): RefreshCoalescer {
     // Each pending key with the time it was last added.
-    const pending = new Map<string, { key: QueryKey; at: number }>();
+    const pending = new Map<string, Pending & { at: number }>();
     let timer: ReturnType<typeof setTimeout> | null = null;
     let firstAt = 0;
 
@@ -36,36 +58,44 @@ export function createRefreshCoalescer(
         timer = setTimeout(flush, Math.max(0, Math.min(delayMs, firstAt + maxWaitMs - now)));
     };
 
-    const add = (keys: readonly QueryKey[]) => {
-        if (keys.length === 0) return;
+    const queue = (entries: readonly Pending[]) => {
+        if (entries.length === 0) return;
         const now = Date.now();
-        for (const key of keys) pending.set(hashKey(key), { key, at: now });
+        for (const { key, match } of entries) {
+            const hash = hashKey(key);
+            const prev = pending.get(hash);
+            const prevMatch = prev?.match;
+            // Two narrowed adds of one key refresh what either asked for; an unnarrowed one refreshes everything.
+            const merged = !prev ? match : prevMatch && match ? (q: Query) => prevMatch(q) || match(q) : undefined;
+            pending.set(hash, { key, match: merged, at: now });
+        }
         schedule();
     };
 
+    const add = (keys: readonly QueryKey[], match?: QueryMatch) => queue(keys.map((key) => ({ key, match })));
+
     const flush = () => {
         timer = null;
         const now = Date.now();
         const entries = [...pending.values()];
         pending.clear();
-        const keys = entries.map((e) => e.key);
         // A flush forced by the cap may beat the last event's writes, so recent keys get a trailing flush too.
-        const again: QueryKey[] = entries.filter((e) => now - e.at < delayMs).map((e) => e.key);
-        for (const key of keys) {
+        const again: Pending[] = entries.filter((e) => now - e.at < delayMs).map(({ key, match }) => ({ key, match }));
+        for (const { key, match } of entries) {
             // A broader pending key already covers this one.
-            if (keys.some((other) => other !== key && other.length < key.length && partialMatchKey(key, other))) {
+            if (entries.some((o) => o.key !== key && !o.match && o.key.length < key.length && partialMatchKey(key, o.key))) {
                 continue;
             }
             // A running fetch finishes rather than restarts, and is re-checked next round since it may predate the event.
-            for (const query of queryClient.getQueryCache().findAll({ queryKey: key, fetchStatus: "fetching" })) {
-                again.push(query.queryKey);
+            for (const query of queryClient.getQueryCache().findAll({ queryKey: key, fetchStatus: "fetching", predicate: match })) {
+                again.push({ key: query.queryKey });
             }
-            void queryClient.invalidateQueries(
-                { queryKey: key, predicate: (query) => query.state.fetchStatus !== "fetching" },
+            invalidate(
+                { queryKey: key, predicate: (query) => query.state.fetchStatus !== "fetching" && (!match || match(query)) },
                 { cancelRefetch: false },
             );
         }
-        if (again.length > 0) add(again);
+        queue(again);
     };
 
     return {
@@ -78,6 +108,64 @@ export function createRefreshCoalescer(
     };
 }
 
+// Shown on a hidden tab (its title and favicon badge), so these keep refetching there.
+const HIDDEN_TAB_LIVE_KEYS: readonly QueryKey[] = [["unibox", "unseen-count"]];
+
+export interface HiddenTabGate {
+    /** Invalidates now; on a hidden tab only marks stale, and refetches once the tab is visible again. */
+    invalidate: Invalidate;
+    /** Forgets deferred refetches and stops listening. */
+    dispose(): void;
+}
+
+export function createHiddenTabGate(queryClient: QueryClient): HiddenTabGate {
+    // Hashes of the active queries a hidden tab left stale.
+    const deferred = new Set<string>();
+    let listening = false;
+
+    const hidden = () => typeof document !== "undefined" && document.visibilityState === "hidden";
+    const live: QueryMatch = (query) => HIDDEN_TAB_LIVE_KEYS.some((key) => partialMatchKey(query.queryKey, key));
+
+    // On document, so it runs before the focus manager's window listener and a focus refetch joins this one.
+    const onVisibility = () => {
+        if (hidden()) return;
+        stop();
+        if (deferred.size === 0) return;
+        const hashes = new Set(deferred);
+        deferred.clear();
+        void queryClient.refetchQueries({ type: "active", predicate: (query) => hashes.has(query.queryHash) });
+    };
+    const stop = () => {
+        if (!listening) return;
+        document.removeEventListener("visibilitychange", onVisibility);
+        listening = false;
+    };
+
+    const invalidate: Invalidate = (filters, options) => {
+        if (!hidden()) {
+            void queryClient.invalidateQueries(filters, options);
+            return;
+        }
+        const match = filters.predicate;
+        const stale: InvalidateQueryFilters = { ...filters, predicate: (q) => !live(q) && (!match || match(q)) };
+        for (const query of queryClient.getQueryCache().findAll({ ...stale, type: "active" })) deferred.add(query.queryHash);
+        void queryClient.invalidateQueries({ ...stale, refetchType: "none" });
+        void queryClient.invalidateQueries({ ...filters, predicate: (q) => live(q) && (!match || match(q)) }, options);
+        if (!listening) {
+            document.addEventListener("visibilitychange", onVisibility);
+            listening = true;
+        }
+    };
+
+    return {
+        invalidate,
+        dispose: () => {
+            stop();
+            deferred.clear();
+        },
+    };
+}
+
 /**
  * Returns true when the same delivery was already seen within `windowMs`. The
  * key should name the event and its publish (type, timestamp, ids), so a