mirror of
https://github.com/warmbly/warmbly.git
synced 2026-10-22 08:03:26 +00:00
feat: integrate merged monitoring updates with ordered worker command recovery
This commit is contained in:
@@ -202,7 +202,7 @@ jobs:
|
||||
env:
|
||||
WARMBLY_TEST_DB: postgres://warmbly:warmbly@127.0.0.1:5432/warmbly_test?sslmode=disable
|
||||
run: |
|
||||
go test -race ./internal/repository -run '^TestLive(LastWarmupSendFailure|PersistentMailboxLoadingFailure|DispatchRetry|LineageQueueRevisions|AdminPlatformOverview|WarmupRecoversOldPreparation|WarmupPreparationRecoveryRetains|WarmupFilingRequiresAcknowledgement)' -count=1
|
||||
go test -race ./internal/repository -run '^TestLive(Monitoring|LastWarmupSendFailure|PersistentMailboxLoadingFailure|DispatchRetry|LineageQueueRevisions|AdminPlatformOverview|WarmupRecoversOldPreparation|WarmupPreparationRecoveryRetains|WarmupFilingRequiresAcknowledgement)' -count=1
|
||||
go test -race ./internal/tasks -run '^Test(LiveWarmup(SafetyHoldReschedules|Recovery|TaskDoesNotApplyResults)|WarmupSenderDistinguishes)' -count=1
|
||||
|
||||
- name: Test the Kafka codec and client lifecycle
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
import { act, type PropsWithChildren } from "react";
|
||||
import { createRoot, type Root } from "react-dom/client";
|
||||
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
|
||||
import { MemoryRouter } from "react-router-dom";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import LoginPage from "./LoginPage";
|
||||
|
||||
const state = vi.hoisted(() => ({
|
||||
login: vi.fn(), confirm: vi.fn(), verify: vi.fn(), saveToken: vi.fn(),
|
||||
onToken: (_value: string): Promise<void> => Promise.resolve(),
|
||||
onCode: (_value: string): void => {},
|
||||
}));
|
||||
vi.mock("@/lib/api/client/auth", () => ({ getAuthConfig: () => Promise.resolve({ captcha: false }), login: state.login, loginConfirm: state.confirm, verifyTwoFA: state.verify }));
|
||||
vi.mock("@/lib/auth/storage", () => ({ setToken: state.saveToken }));
|
||||
vi.mock("sonner", () => ({ toast: { success: vi.fn(), error: vi.fn() } }));
|
||||
vi.mock("motion/react", () => ({ AnimatePresence: ({ children }: PropsWithChildren) => children, motion: { div: ({ children }: PropsWithChildren) => <div>{children}</div> } }));
|
||||
vi.mock("@/components/captcha/TurnstileModal", () => ({ TurnstileModal: ({ onToken }: { onToken: typeof state.onToken }) => { state.onToken = onToken; return null; } }));
|
||||
vi.mock("@/components/ui/input-otp", () => ({ InputOTP: ({ onChange }: { onChange: typeof state.onCode }) => { state.onCode = onChange; return null; }, InputOTPGroup: () => null, InputOTPSlot: () => null }));
|
||||
|
||||
const token = { access_token: "fixture-access", refresh_token: "fixture-refresh", access_token_expires_at: "2099-01-01T00:00:00Z", refresh_token_expires_at: "2099-01-01T00:00:00Z" };
|
||||
let root: Root;
|
||||
let client: QueryClient;
|
||||
let clear: ReturnType<typeof vi.spyOn>;
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.clearAllMocks();
|
||||
vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true);
|
||||
client = new QueryClient({ defaultOptions: { queries: { retry: false } } });
|
||||
client.setQueryData(["me"], { id: "previous-admin", admin_permissions: 0xffff });
|
||||
client.setQueryData(["system", "mail-status"], { private: "previous-session" });
|
||||
clear = vi.spyOn(client, "clear");
|
||||
root = createRoot(document.createElement("div"));
|
||||
await act(async () => root.render(<QueryClientProvider client={client}><MemoryRouter><LoginPage /></MemoryRouter></QueryClientProvider>));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await act(async () => root.unmount());
|
||||
client.clear();
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
function expectClearedBeforeToken() {
|
||||
expect(client.getQueryCache().getAll()).toHaveLength(0);
|
||||
expect(state.saveToken).toHaveBeenCalledWith(token);
|
||||
expect(clear.mock.invocationCallOrder[0]).toBeLessThan(state.saveToken.mock.invocationCallOrder[0]);
|
||||
}
|
||||
|
||||
describe("admin sign-in session isolation", () => {
|
||||
it("clears cached identity and private data before a direct successful login", async () => {
|
||||
state.login.mockResolvedValue({ code_required: false, token });
|
||||
await act(async () => state.onToken("fixture-captcha"));
|
||||
expectClearedBeforeToken();
|
||||
});
|
||||
|
||||
it("keeps cache during an email-code challenge and clears it on confirmation", async () => {
|
||||
state.login.mockResolvedValue({ code_required: true, session: "fixture-session" });
|
||||
state.confirm.mockResolvedValue(token);
|
||||
await act(async () => state.onToken("fixture-captcha"));
|
||||
expect(clear).not.toHaveBeenCalled();
|
||||
await act(async () => state.onCode("123456"));
|
||||
expectClearedBeforeToken();
|
||||
});
|
||||
|
||||
it("keeps cache during a second-factor challenge and clears it on verification", async () => {
|
||||
state.login.mockResolvedValue({ code_required: false, two_fa_required: true, pending_token: "fixture-pending" });
|
||||
state.verify.mockResolvedValue(token);
|
||||
await act(async () => state.onToken("fixture-captcha"));
|
||||
expect(clear).not.toHaveBeenCalled();
|
||||
await act(async () => state.onCode("123456"));
|
||||
expectClearedBeforeToken();
|
||||
});
|
||||
|
||||
it("does not replace the session or clear caches on a failed login", async () => {
|
||||
state.login.mockRejectedValue(new Error("fixture-login-failure"));
|
||||
await act(async () => state.onToken("fixture-captcha"));
|
||||
expect(clear).not.toHaveBeenCalled();
|
||||
expect(state.saveToken).not.toHaveBeenCalled();
|
||||
expect(client.getQueryCache().getAll()).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
@@ -14,6 +14,7 @@
|
||||
import { useEffect, useRef, useState, type FormEvent } from "react";
|
||||
import { useNavigate, useLocation } from "react-router-dom";
|
||||
import { AnimatePresence, motion, type Transition } from "motion/react";
|
||||
import { useQueryClient } from "@tanstack/react-query";
|
||||
import { toast } from "sonner";
|
||||
import { Eye, EyeOff, Loader2, ArrowLeft, ArrowRight } from "lucide-react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
@@ -40,6 +41,7 @@ function errorMessage(err: unknown): string {
|
||||
|
||||
export default function LoginPage() {
|
||||
const nav = useNavigate();
|
||||
const queryClient = useQueryClient();
|
||||
const loc = useLocation();
|
||||
const [phase, setPhase] = useState<"credentials" | "code" | "twofa">("credentials");
|
||||
const [pendingToken, setPendingToken] = useState("");
|
||||
@@ -118,6 +120,7 @@ export default function LoginPage() {
|
||||
return;
|
||||
}
|
||||
if (res.token) {
|
||||
queryClient.clear();
|
||||
setToken(res.token);
|
||||
const dest = (loc.state as { from?: string } | null)?.from ?? "/";
|
||||
nav(dest, { replace: true });
|
||||
@@ -166,6 +169,7 @@ export default function LoginPage() {
|
||||
return;
|
||||
}
|
||||
if (!res.access_token) throw new Error("The server returned an unexpected sign-in response.");
|
||||
queryClient.clear();
|
||||
setToken(res as LoginResponse);
|
||||
const dest = (loc.state as { from?: string } | null)?.from ?? "/";
|
||||
nav(dest, { replace: true });
|
||||
@@ -188,6 +192,7 @@ export default function LoginPage() {
|
||||
setSubmitting(true);
|
||||
try {
|
||||
const tok = await verifyTwoFA({ pending_token: pendingToken, code: value });
|
||||
queryClient.clear();
|
||||
setToken(tok);
|
||||
const dest = (loc.state as { from?: string } | null)?.from ?? "/";
|
||||
nav(dest, { replace: true });
|
||||
|
||||
@@ -1,7 +1,3 @@
|
||||
// Setup and health: two views of "is this instance ok". Findings are the
|
||||
// backend's own verdicts about configuration and state; services are live
|
||||
// probes of what it runs on. ?tab= deep-links either one.
|
||||
|
||||
import { Link, useSearchParams } from "react-router-dom";
|
||||
import { Activity, ListChecks } from "lucide-react";
|
||||
import { PageHeader } from "@/components/layout/PageHeader";
|
||||
@@ -10,8 +6,11 @@ import { Button } from "@/components/ui/button";
|
||||
import { useInstanceHealth } from "@/hooks/useInstanceHealth";
|
||||
import { FindingsTab } from "./health/FindingsTab";
|
||||
import { ServicesTab } from "./health/ServicesTab";
|
||||
import { InstanceMonitoringPanel } from "./InstanceMonitoringPanel";
|
||||
import { useAdminPerm } from "@/hooks/useAdminPerm";
|
||||
import { AdminPerm } from "@/lib/auth/permissions";
|
||||
|
||||
const TAB_IDS = ["findings", "services"] as const;
|
||||
const TAB_IDS = ["findings", "services", "operations"] as const;
|
||||
type TabId = (typeof TAB_IDS)[number];
|
||||
|
||||
function isTabId(v: string | null): v is TabId {
|
||||
@@ -19,6 +18,7 @@ function isTabId(v: string | null): v is TabId {
|
||||
}
|
||||
|
||||
export default function HealthPage() {
|
||||
const canConfigure = useAdminPerm(AdminPerm.ManageSettings);
|
||||
const [params, setParams] = useSearchParams();
|
||||
const raw = params.get("tab");
|
||||
const tab: TabId = isTabId(raw) ? raw : "findings";
|
||||
@@ -44,11 +44,11 @@ export default function HealthPage() {
|
||||
<div>
|
||||
<PageHeader
|
||||
title="Setup and health"
|
||||
description="What the backend thinks is wrong with this instance, and whether the services it depends on are answering."
|
||||
description="Configuration findings, live service probes and evidence-based operational measurements. Missing coverage is not an all-clear."
|
||||
>
|
||||
<Button size="sm" variant="outline" asChild>
|
||||
{canConfigure && <Button size="sm" variant="outline" asChild>
|
||||
<Link to="/configuration">Configuration</Link>
|
||||
</Button>
|
||||
</Button>}
|
||||
</PageHeader>
|
||||
|
||||
<PageTabs
|
||||
@@ -60,12 +60,13 @@ export default function HealthPage() {
|
||||
badge: findings > 0 ? findings : undefined,
|
||||
},
|
||||
{ id: "services", label: "Services", icon: Activity },
|
||||
{ id: "operations", label: "Operations", icon: Activity },
|
||||
]}
|
||||
value={tab}
|
||||
onChange={setTab}
|
||||
/>
|
||||
|
||||
{tab === "findings" ? <FindingsTab /> : <ServicesTab />}
|
||||
{tab === "findings" ? <FindingsTab /> : tab === "services" ? <ServicesTab /> : <InstanceMonitoringPanel />}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ import { Link } from "react-router-dom";
|
||||
import { useMutation, useQueryClient } from "@tanstack/react-query";
|
||||
import { toast } from "sonner";
|
||||
import { AlertTriangle, ArrowRight, ExternalLink, Info, XCircle, type LucideIcon } from "lucide-react";
|
||||
import { StatusBadge } from "@/components/ui/kit";
|
||||
import { Callout, StatusBadge } from "@/components/ui/kit";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { useConfirm } from "@/components/ConfirmDialog";
|
||||
import { docsUrl } from "@/lib/docs";
|
||||
@@ -140,15 +140,21 @@ function ExpiredInvitationsCleanup() {
|
||||
|
||||
const OVERVIEW_LIMIT = 4;
|
||||
|
||||
// Overview's problems strip. Silent when the instance is clean or when the
|
||||
// caller cannot read the endpoint, so it never becomes noise on the page.
|
||||
export function InstanceProblemsPanel() {
|
||||
const canRead = useAdminPerm(AdminPerm.ViewAnalytics);
|
||||
const healthQ = useInstanceHealth({ enabled: canRead });
|
||||
const checks = healthQ.data?.checks ?? [];
|
||||
const actionable = checks.filter((c) => c.severity === "error" || c.severity === "warning");
|
||||
|
||||
if (healthQ.isError || actionable.length === 0) return null;
|
||||
if (!canRead) return null;
|
||||
if (healthQ.isError || (healthQ.data && actionable.length === 0)) return (
|
||||
<Callout className="mb-4" tone={healthQ.isError ? "warning" : "info"} title={healthQ.isError ? "Configuration findings unavailable" : "No configuration errors or warnings reported"}>
|
||||
{healthQ.isError ? "Could not refresh findings. This is not an all-clear." : "Absent findings do not establish check coverage or operational health."}
|
||||
{healthQ.isError && healthQ.data && ` Previous response listed ${actionable.length} errors or warnings; it is not a fresh verdict.`}
|
||||
{" "}<Link to="/health" className="font-medium hover:underline">Setup and health</Link>
|
||||
</Callout>
|
||||
);
|
||||
if (actionable.length === 0) return null;
|
||||
|
||||
const errors = actionable.filter((c) => c.severity === "error").length;
|
||||
const shown = actionable.slice(0, OVERVIEW_LIMIT);
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
import { renderToStaticMarkup } from "react-dom/server";
|
||||
import { MemoryRouter } from "react-router-dom";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { MonitoringMetric, MonitoringSnapshot, MonitoringSource } from "@/lib/api/client/admin/monitoring";
|
||||
import { AdminPerm } from "@/lib/auth/permissions";
|
||||
import { InstanceMonitoringPanel } from "./InstanceMonitoringPanel";
|
||||
|
||||
const state = vi.hoisted(() => ({
|
||||
mask: 0 as number | undefined,
|
||||
query: { data: undefined as MonitoringSnapshot | undefined, isError: false, isLoading: false, isFetching: false, error: new Error("provider-password-private"), refetch: vi.fn() },
|
||||
}));
|
||||
vi.mock("@/hooks/useMe", () => ({ useMe: () => ({ data: { id: "admin", admin_permissions: state.mask } }) }));
|
||||
vi.mock("@/hooks/useInstanceMonitoring", () => ({ useInstanceMonitoring: () => state.query }));
|
||||
|
||||
const at = new Date().toISOString();
|
||||
const evidence = new Date(Date.now() - 3_600_000).toISOString();
|
||||
function metric(overrides: Partial<MonitoringMetric> = {}): MonitoringMetric {
|
||||
return {
|
||||
id: "failed_send", title: "Stored failure report", unit: "reports", availability: "fresh",
|
||||
condition: "recent_failure", severity: "warning", note: "Only confirmed send evidence closes an incident.",
|
||||
count: 7, affected_mailboxes: 3, affected_organizations: 2, unknown_organization_rows: 1,
|
||||
observed_at: at, evidence_at: evidence, latest_evidence_at: at, evidence_age_seconds: 3600,
|
||||
next_eligible_at: null, window_start: evidence, window_end: at, threshold_seconds: 3600,
|
||||
investigate: "/sends?tab=failures", ...overrides,
|
||||
};
|
||||
}
|
||||
function source(overrides: Partial<MonitoringSource> = {}): MonitoringSource {
|
||||
return { id: "sends", availability: "fresh", coverage: "complete", observed_at: at, checked_at: at,
|
||||
measured_scopes: null, expected_scopes: null, metrics: [metric()], ...overrides };
|
||||
}
|
||||
function snapshot(sources: MonitoringSource[] = [source()]): MonitoringSnapshot {
|
||||
return { version: "1", checked_at: at, refresh_after: at, coverage: "complete", sources };
|
||||
}
|
||||
function render(compact = false) {
|
||||
const html = renderToStaticMarkup(<MemoryRouter><InstanceMonitoringPanel compact={compact} /></MemoryRouter>);
|
||||
return new DOMParser().parseFromString(html, "text/html");
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
state.mask = Object.values(AdminPerm).reduce((mask, bit) => mask | bit, 0);
|
||||
Object.assign(state.query, { data: snapshot(), isError: false, isLoading: false, isFetching: false });
|
||||
});
|
||||
|
||||
describe("operational monitoring evidence", () => {
|
||||
it("keeps metric counts distinct from affected mailboxes, organizations and attention signals", () => {
|
||||
const document = render();
|
||||
const text = document.body.textContent;
|
||||
expect(text).toContain("1 recorded attention signals");
|
||||
expect(text).toContain("7 reports");
|
||||
expect(text).toContain("3 affected mailboxes");
|
||||
expect(text).toContain("2 affected organizations");
|
||||
expect(text).toContain("1 rows with unknown organization");
|
||||
expect(text).toContain("Observed:");
|
||||
expect(text).toContain("Evidence since:");
|
||||
expect(text).toContain("Latest evidence:");
|
||||
expect(document.querySelector(`time[datetime="${evidence}"]`)).not.toBeNull();
|
||||
expect(document.querySelector('a[href="/sends?tab=failures"]')).not.toBeNull();
|
||||
});
|
||||
|
||||
it("renders loading, failed and successful empty requests without a green verdict", () => {
|
||||
state.query.data = undefined;
|
||||
state.query.isLoading = true;
|
||||
expect(render().querySelector('[role="status"]')?.textContent).toContain("Loading operational monitoring");
|
||||
state.query.isLoading = false;
|
||||
state.query.isError = true;
|
||||
expect(render().body.textContent).toContain("Monitoring refresh failed");
|
||||
expect(render().body.textContent).not.toContain("provider-password-private");
|
||||
state.query.isError = false;
|
||||
state.query.data = snapshot([]);
|
||||
expect(render().body.textContent).toContain("No monitoring sources returned");
|
||||
expect(render().body.textContent).not.toContain("recorded attention signals");
|
||||
});
|
||||
|
||||
it.each(["unsupported", "dependency_missing", "schema_absent", "timeout", "query_failed", "permission_denied"])("exposes %s coverage without inventing zero counts", (reason) => {
|
||||
state.query.data = snapshot([source({ availability: "unavailable", coverage: "unavailable", reason, metrics: [] })]);
|
||||
const text = render().body.textContent;
|
||||
expect(text).toContain("1 sources with coverage gaps");
|
||||
expect(text).toContain("No current measurement available");
|
||||
expect(text).not.toContain("0 reports");
|
||||
});
|
||||
|
||||
it("keeps a missing measurement distinct from a measured zero", () => {
|
||||
state.query.data = snapshot([source({ metrics: [metric({ count: null, affected_mailboxes: null, affected_organizations: null }), metric({ id: "sent", title: "Send confirmations", count: 0, condition: "no_recent_evidence" })] })]);
|
||||
const text = render().body.textContent;
|
||||
expect(text).toContain("Not measured");
|
||||
expect(text).toContain("0 reports");
|
||||
expect(text).toContain("No matching recent evidence");
|
||||
expect(text).not.toContain("Confirmed scoped observation");
|
||||
});
|
||||
|
||||
it("retains unsupported metric explanations without presenting their counts as current measurements", () => {
|
||||
state.query.data = snapshot([source({ id: "send_wait_attribution", availability: "unavailable", coverage: "unavailable", reason: "unsupported", metrics: [metric({ title: "Detailed live send waits", condition: "budget", count: 123, note: "Scheduler does not persist per-task budget decisions." })] })]);
|
||||
const text = render().body.textContent;
|
||||
expect(text).toContain("Detailed live send waits");
|
||||
expect(text).toContain("Scheduler does not persist per-task budget decisions");
|
||||
expect(text).toContain("Not measured");
|
||||
expect(text).not.toContain("123 reports");
|
||||
expect(text).not.toContain("3 affected mailboxes");
|
||||
});
|
||||
|
||||
it("shows retained stale observations and partial samples with their original evidence clocks", () => {
|
||||
state.query.data = snapshot([source({ availability: "stale", coverage: "partial", measured_scopes: 20, expected_scopes: 35, metrics: [metric({ availability: "stale" })] })]);
|
||||
const document = render();
|
||||
expect(document.body.textContent).toContain("20 / 35 scopes measured");
|
||||
expect(document.body.textContent).toContain("lower bounds");
|
||||
expect(document.body.textContent).toContain("Stale observation");
|
||||
expect(document.body.textContent).toContain("historical, not proof of a current failure");
|
||||
expect(document.querySelector(`time[datetime="${evidence}"]`)).not.toBeNull();
|
||||
});
|
||||
|
||||
it("a failed refresh cannot make cached healthy data appear fresh", () => {
|
||||
state.query.isError = true;
|
||||
state.query.data = snapshot([source({ metrics: [metric({ condition: "healthy", severity: "info" })] })]);
|
||||
const document = render();
|
||||
expect(document.body.textContent).toContain("previous snapshot with its original timestamps");
|
||||
expect(document.body.textContent).toContain("Stale observation");
|
||||
expect(document.body.textContent).not.toContain("Fresh / complete");
|
||||
expect(document.querySelector(`time[datetime="${at}"]`)).not.toBeNull();
|
||||
});
|
||||
|
||||
it("ages snapshots by observation time, not successful client fetch time", () => {
|
||||
const old = new Date(Date.now() - 300_000).toISOString();
|
||||
state.query.data = snapshot([source({ observed_at: old, metrics: [metric({ observed_at: old })] })]);
|
||||
expect(render().body.textContent).toContain("Stale observation");
|
||||
});
|
||||
|
||||
it("denies metrics and drill-downs when permission bits are absent or unknown", () => {
|
||||
for (const mask of [undefined, 0]) {
|
||||
state.mask = mask;
|
||||
const document = render(true);
|
||||
expect(document.body.textContent).toContain("Monitoring permission required");
|
||||
expect(document.body.textContent).not.toContain("Stored failure report");
|
||||
expect(document.querySelector("a")).toBeNull();
|
||||
}
|
||||
state.mask = AdminPerm.ViewAnalytics;
|
||||
expect(render().body.textContent).toContain("Permission required");
|
||||
expect(render().body.textContent).not.toContain("Stored failure report");
|
||||
expect(render().querySelector('a[href="/sends?tab=failures"]')).toBeNull();
|
||||
});
|
||||
|
||||
it("checks the link destination permission separately from the visible source", () => {
|
||||
state.mask = AdminPerm.ViewAnalytics | AdminPerm.ViewOrganizations;
|
||||
state.query.data = snapshot([source({ id: "webhooks", metrics: [metric({ investigate: "/sends?tab=webhooks" })] })]);
|
||||
expect(render().body.textContent).toContain("7 reports");
|
||||
expect(render().querySelector('a[href="/sends?tab=webhooks"]')).toBeNull();
|
||||
state.mask |= AdminPerm.ViewCampaigns;
|
||||
expect(render().querySelector('a[href="/sends?tab=webhooks"]')).not.toBeNull();
|
||||
});
|
||||
|
||||
it("rejects unrecognized source data, arbitrary links, raw reason and scope payloads", () => {
|
||||
state.query.data = snapshot([source({ reason: "provider-password-private", metrics: [metric({ scope_id: "secret@example.com", reason: "provider-password-private", investigate: "https://evil.invalid/?token=secret" })] }), source({ id: "secret-source", metrics: [metric({ title: "secret-title" })] })]);
|
||||
const text = render().body.innerHTML;
|
||||
for (const secret of ["provider-password-private", "secret@example.com", "evil.invalid", "secret-title", "secret-source"]) expect(text).not.toContain(secret);
|
||||
});
|
||||
|
||||
it.each(["committed_lag", "pending", "ack_pending", "redelivered"])("keeps positive broker %s prominent while preserving unknown/info", (id) => {
|
||||
state.query.data = snapshot([source({ metrics: Array.from({ length: 8 }, (_, i) => metric({ id: `failure-${i}` })) }), source({ id: "backend_event_broker", metrics: [metric({ id, title: `Broker ${id}`, condition: "unknown", severity: "info", count: 45 })] })]);
|
||||
const document = render(true);
|
||||
expect(document.body.textContent).toContain("1 positive broker queue measurements");
|
||||
expect(document.body.textContent).toContain("45 reports");
|
||||
expect(document.body.textContent).toContain("Queue measurement, not outage");
|
||||
expect(document.body.textContent).toContain("Unknown");
|
||||
expect(document.body.textContent).not.toContain("Confirmed scoped observation");
|
||||
});
|
||||
|
||||
it("does not classify waits or a fresh zero queue as a failure", () => {
|
||||
state.query.data = snapshot([source({ metrics: ["waiting_schedule", "waiting_retry", "budget", "daily_limit", "in_flight"].map((condition) => metric({ condition: condition as MonitoringMetric["condition"], severity: "info", next_eligible_at: at })) }), source({ id: "backend_event_broker", metrics: [metric({ id: "pending", condition: "unknown", severity: "info", count: 0 })] })]);
|
||||
const text = render().body.textContent;
|
||||
expect(text).toContain("0 recorded attention signals");
|
||||
expect(text).toContain("Scheduled wait");
|
||||
expect(text).toContain("Retry wait");
|
||||
expect(text).toContain("Budget wait");
|
||||
expect(text).toContain("Daily-limit wait");
|
||||
expect(text).toContain("Next eligible:");
|
||||
expect(text).toContain("not an instance-wide all-clear");
|
||||
});
|
||||
|
||||
it("shows safety holds as restrictions and counts unknown warning observations without outage claims", () => {
|
||||
state.query.data = snapshot([source({ metrics: [metric({ condition: "safety_hold" }), metric({ id: "missing", condition: "unknown", severity: "warning" })] })]);
|
||||
const text = render(true).body.textContent;
|
||||
expect(text).toContain("2 recorded attention signals");
|
||||
expect(text).toContain("Safety hold");
|
||||
expect(text).toContain("Unknown");
|
||||
expect(text).toContain("restrictions, not provider outages");
|
||||
});
|
||||
|
||||
it("does not interpret an unsupported contract version", () => {
|
||||
state.query.data = { ...snapshot(), version: "2" };
|
||||
expect(render().body.textContent).toContain("Unsupported monitoring version");
|
||||
expect(render().body.textContent).not.toContain("7 reports");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,178 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { Link } from "react-router-dom";
|
||||
import { Activity, AlertTriangle, RefreshCw } from "lucide-react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { Callout, EmptyState, StatusBadge } from "@/components/ui/kit";
|
||||
import { useInstanceMonitoring } from "@/hooks/useInstanceMonitoring";
|
||||
import { useMe } from "@/hooks/useMe";
|
||||
import { AdminPerm } from "@/lib/auth/permissions";
|
||||
import type { MonitoringMetric, MonitoringSource } from "@/lib/api/client/admin/monitoring";
|
||||
import {
|
||||
ageLabel, canMonitor, CONDITION_LABELS, investigationPath, isBrokerQueue,
|
||||
MONITORING_SOURCES, monitoringReason, needsMonitoringAttention, observationIsOld,
|
||||
permittedSource, timestamp,
|
||||
} from "@/lib/monitoring";
|
||||
import type { Tone } from "@/lib/tones";
|
||||
|
||||
function ObservationTime({ label, value, now }: { label: string; value: string | null; now: number }) {
|
||||
const at = timestamp(value);
|
||||
return (
|
||||
<span className="break-words">
|
||||
{label}: {at === null ? "unknown" : <time dateTime={value!} title={new Date(at).toISOString()}>{new Date(at).toLocaleString()} ({ageLabel(at, now)})</time>}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
function metricIsStale(metric: MonitoringMetric, source: MonitoringSource, now: number): boolean {
|
||||
return metric.availability === "stale" || source.availability === "stale" ||
|
||||
observationIsOld(metric.observed_at, now) || observationIsOld(source.observed_at, now);
|
||||
}
|
||||
|
||||
function metricTone(metric: MonitoringMetric, stale: boolean): Tone {
|
||||
if (metric.availability === "unavailable") return "neutral";
|
||||
if (stale) return "warning";
|
||||
if (needsMonitoringAttention(metric)) return isBrokerQueue(metric) ? "info" : metric.severity === "error" ? "danger" : "warning";
|
||||
if (metric.condition === "healthy") return "success";
|
||||
return "info";
|
||||
}
|
||||
|
||||
function MetricRow({ metric: m, source, mask, now, compact }: {
|
||||
metric: MonitoringMetric; source: MonitoringSource; mask: number | undefined; now: number; compact?: boolean;
|
||||
}) {
|
||||
const stale = metricIsStale(m, source, now);
|
||||
const unavailable = m.availability === "unavailable";
|
||||
const path = investigationPath(m.investigate, mask);
|
||||
const measured = !unavailable && typeof m.count === "number";
|
||||
return (
|
||||
<li className="min-w-0 space-y-2 px-4 py-3">
|
||||
<div className="flex flex-wrap items-start justify-between gap-2">
|
||||
<h4 className="min-w-0 break-words text-[13px] font-medium">{m.title}</h4>
|
||||
<div className="flex flex-wrap gap-1.5">
|
||||
<StatusBadge tone={!measured ? "neutral" : metricTone(m, stale)}>{unavailable ? "Unavailable" : !measured ? "Unknown" : CONDITION_LABELS[m.condition] ?? "Unknown"}</StatusBadge>
|
||||
{!unavailable && stale && <StatusBadge tone="warning">Stale observation</StatusBadge>}
|
||||
{isBrokerQueue(m) && <StatusBadge tone="info">Queue measurement, not outage</StatusBadge>}
|
||||
</div>
|
||||
</div>
|
||||
{compact && <p className="text-xs text-muted-foreground">{MONITORING_SOURCES[source.id]?.title ?? "Unrecognized source"}</p>}
|
||||
<div className="flex flex-wrap gap-x-4 gap-y-1 text-[13px] tabular-nums">
|
||||
<span className="font-medium">{measured ? `${m.count!.toLocaleString()} ${m.unit}` : "Not measured"}</span>
|
||||
{!unavailable && m.affected_mailboxes !== null && <span>{m.affected_mailboxes.toLocaleString()} affected mailboxes</span>}
|
||||
{!unavailable && m.affected_organizations !== null && <span>{m.affected_organizations.toLocaleString()} affected organizations</span>}
|
||||
{!unavailable && m.unknown_organization_rows !== null && m.unknown_organization_rows > 0 && <span>{m.unknown_organization_rows.toLocaleString()} rows with unknown organization</span>}
|
||||
</div>
|
||||
{m.reason && <p className="text-xs text-muted-foreground">{monitoringReason(m.reason)}</p>}
|
||||
{m.note && <p className="break-words text-xs leading-relaxed text-muted-foreground">{m.note}</p>}
|
||||
<div className="flex flex-wrap gap-x-4 gap-y-1 text-xs text-muted-foreground">
|
||||
<ObservationTime label="Observed" value={m.observed_at} now={now} />
|
||||
<ObservationTime label="Evidence since" value={m.evidence_at} now={now} />
|
||||
{m.latest_evidence_at && <ObservationTime label="Latest evidence" value={m.latest_evidence_at} now={now} />}
|
||||
{m.next_eligible_at && <ObservationTime label="Next eligible" value={m.next_eligible_at} now={now} />}
|
||||
</div>
|
||||
{!compact && <div className="flex flex-wrap gap-x-4 gap-y-1 text-xs text-muted-foreground">
|
||||
{m.evidence_age_seconds !== null && <span>Evidence age reported by snapshot: {m.evidence_age_seconds.toLocaleString()}s</span>}
|
||||
{m.threshold_seconds !== null && <span>Monitoring threshold: {m.threshold_seconds.toLocaleString()}s (not a provider SLA)</span>}
|
||||
{m.window_start && <ObservationTime label="Window start" value={m.window_start} now={now} />}
|
||||
{m.window_end && <ObservationTime label="Window end" value={m.window_end} now={now} />}
|
||||
</div>}
|
||||
{path && <Link to={path} className="inline-flex text-xs font-medium text-[var(--admin-accent-strong)] hover:underline">Investigate {m.title}</Link>}
|
||||
</li>
|
||||
);
|
||||
}
|
||||
|
||||
function SourcePanel({ source, mask, now }: { source: MonitoringSource; mask: number | undefined; now: number }) {
|
||||
const unavailable = source.availability === "unavailable";
|
||||
const stale = !unavailable && (source.availability === "stale" || observationIsOld(source.observed_at, now));
|
||||
return (
|
||||
<section className="min-w-0 overflow-hidden rounded-lg border border-border bg-card" aria-label={MONITORING_SOURCES[source.id]?.title ?? "Unrecognized source"}>
|
||||
<div className="space-y-2 border-b border-border px-4 py-3">
|
||||
<div className="flex flex-wrap items-center justify-between gap-2">
|
||||
<h3 className="text-sm font-medium">{MONITORING_SOURCES[source.id]?.title ?? "Unrecognized source"}</h3>
|
||||
<StatusBadge tone={unavailable ? "neutral" : stale || source.coverage !== "complete" ? "warning" : "info"}>
|
||||
{unavailable ? "Unavailable" : stale ? "Stale" : "Fresh"} / {source.coverage === "complete" ? "complete" : "incomplete"} coverage
|
||||
</StatusBadge>
|
||||
</div>
|
||||
<div className="flex flex-wrap gap-x-4 gap-y-1 text-xs text-muted-foreground">
|
||||
<ObservationTime label="Source observed" value={source.observed_at} now={now} />
|
||||
<ObservationTime label="Collection attempted" value={source.checked_at} now={now} />
|
||||
{source.measured_scopes !== null && <span>{source.measured_scopes.toLocaleString()} / {source.expected_scopes?.toLocaleString() ?? "unknown"} scopes measured</span>}
|
||||
</div>
|
||||
{source.reason && <p className="text-xs text-muted-foreground">{monitoringReason(source.reason)}</p>}
|
||||
{source.coverage !== "complete" && !unavailable && <p className="text-xs text-muted-foreground">Incomplete coverage. Measured counts may be lower bounds; do not sum overlapping metrics.</p>}
|
||||
</div>
|
||||
{(unavailable || source.metrics.length === 0) && (
|
||||
<p className="px-4 py-4 text-[13px] text-muted-foreground">{unavailable ? "No current measurement available. This is not zero backlog or evidence of an outage." : "No metrics returned. Operational health is unknown for this source."}</p>
|
||||
)}
|
||||
{source.metrics.length > 0 && (
|
||||
<ul className="divide-y divide-border">{source.metrics.map((m, i) => <MetricRow key={`${m.id}:${i}`} metric={unavailable ? { ...m, availability: "unavailable", observed_at: null } : m} source={source} mask={mask} now={now} />)}</ul>
|
||||
)}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
export function InstanceMonitoringPanel({ compact = false }: { compact?: boolean }) {
|
||||
const { data: me } = useMe();
|
||||
const query = useInstanceMonitoring();
|
||||
const [now, setNow] = useState(() => Date.now());
|
||||
useEffect(() => {
|
||||
const timer = setInterval(() => setNow(Date.now()), 30_000);
|
||||
return () => clearInterval(timer);
|
||||
}, []);
|
||||
const mask = me?.admin_permissions;
|
||||
const allowed = canMonitor(mask, AdminPerm.ViewAnalytics);
|
||||
const data = allowed ? query.data : undefined;
|
||||
const sources = data?.sources.map((source) => {
|
||||
const permitted = permittedSource(source, mask);
|
||||
return query.isError && permitted.availability === "fresh" ? { ...permitted, availability: "stale" as const } : permitted;
|
||||
}) ?? [];
|
||||
const attention = sources.flatMap((source) => source.availability === "unavailable" ? [] : source.metrics.filter(needsMonitoringAttention).map((metric) => ({ source, metric })));
|
||||
const queues = attention.filter(({ metric }) => isBrokerQueue(metric));
|
||||
const issues = attention.filter(({ metric }) => !isBrokerQueue(metric));
|
||||
const gaps = sources.filter((s) => s.availability === "unavailable" || s.coverage !== "complete" || s.metrics.length === 0 || s.metrics.some((m) => m.availability === "unavailable" || m.count === null));
|
||||
const stale = sources.filter((s) => s.availability === "stale" || s.availability !== "unavailable" && (observationIsOld(s.observed_at, now) || s.metrics.some((m) => metricIsStale(m, s, now))));
|
||||
const unknown = sources.filter((s) => s.metrics.some((m) => ["unknown", "no_recent_evidence"].includes(m.condition)));
|
||||
const missing = data?.version === "1" ? Object.keys(MONITORING_SOURCES).filter((id) => !sources.some((s) => s.id === id)).length : 0;
|
||||
const refreshAt = timestamp(data?.refresh_after);
|
||||
const waiting = !query.isError && refreshAt !== null && refreshAt > now;
|
||||
const unsupported = data && data.version !== "1";
|
||||
return (
|
||||
<section className={compact ? "mb-8 space-y-3" : "space-y-4"} aria-label="Operational monitoring">
|
||||
<div className="flex flex-wrap items-center justify-between gap-3">
|
||||
<h2 className="flex items-center gap-2 text-sm font-semibold"><Activity className="size-4" />Operational monitoring</h2>
|
||||
<div className="flex flex-wrap items-center gap-3">
|
||||
{compact && allowed && <Link to="/health?tab=operations" className="text-xs font-medium text-[var(--admin-accent-strong)] hover:underline">All operational details</Link>}
|
||||
{allowed && <Button size="sm" variant="outline" onClick={() => query.refetch()} disabled={query.isFetching || waiting} title={waiting ? "The backend reuses the snapshot until its refresh time" : undefined}>
|
||||
<RefreshCw className={query.isFetching ? "size-3.5 animate-spin" : "size-3.5"} />{query.isFetching ? "Refreshing..." : "Refresh monitoring"}
|
||||
</Button>}
|
||||
</div>
|
||||
</div>
|
||||
{!allowed && <Callout tone="neutral" title="Monitoring permission required">Analytics access and a known permission mask are required. No operational measurements were requested.</Callout>}
|
||||
{allowed && query.isLoading && <div role="status" aria-label="Loading operational monitoring"><Skeleton className="h-24 w-full" /><span className="sr-only">Loading operational monitoring</span></div>}
|
||||
{allowed && query.isError && <Callout tone="warning" icon={AlertTriangle} title="Monitoring refresh failed">{data ? "Showing the previous snapshot with its original timestamps, not a fresh health verdict." : "Operational measurements are unavailable. An empty or failed request is not an all-clear. Verify access or retry."}</Callout>}
|
||||
{unsupported && <Callout tone="warning" title="Unsupported monitoring version">This admin release cannot interpret this snapshot. Update the admin frontend; health remains unknown.</Callout>}
|
||||
{data && !unsupported && <>
|
||||
<div className="flex flex-wrap gap-x-4 gap-y-1 text-xs text-muted-foreground">
|
||||
<ObservationTime label="Snapshot checked" value={data.checked_at} now={now} />
|
||||
<ObservationTime label="Refresh eligible" value={data.refresh_after} now={now} />
|
||||
</div>
|
||||
{sources.length === 0 ? <EmptyState title="No monitoring sources returned" hint="Coverage is unknown. An empty response does not establish health or zero backlog." /> : <>
|
||||
<Callout tone={gaps.length || stale.length || missing || data.coverage !== "complete" ? "warning" : "info"} title={`${attention.length} recorded attention signals`}>
|
||||
{gaps.length} sources with coverage gaps; {stale.length} stale; {unknown.length} with unknown or no recent evidence{missing > 0 ? `; ${missing} expected sources not returned` : ""}. Snapshot coverage: {data.coverage === "complete" ? "complete for collected sources" : "incomplete"}.
|
||||
{attention.length === 0 && " No attention signals in measured sources. This is not an instance-wide all-clear."}
|
||||
{stale.length > 0 && " Stale observations are historical, not proof of a current failure."}
|
||||
</Callout>
|
||||
<p className="text-xs leading-relaxed text-muted-foreground">Counts are per metric and may overlap. Scheduled, retry, budget and daily-limit waits are informational; safety holds are restrictions, not provider outages. Heartbeats, completed tasks and sync ticks do not prove sends or recovery.</p>
|
||||
{compact ? <>
|
||||
{queues.length > 0 && <section className="overflow-hidden rounded-lg border border-border bg-card" aria-label="Positive broker queue measurements">
|
||||
<h3 className="px-4 pt-3 text-sm font-medium">{queues.length} positive broker queue measurements</h3>
|
||||
<p className="px-4 py-2 text-xs text-muted-foreground">Observed pending or lag counts, not a proven outage or delivery rate. Separate scopes and metrics must not be added together.</p>
|
||||
<ul className="max-h-96 divide-y divide-border overflow-auto">{queues.map(({ source, metric }, i) => <MetricRow key={`${source.id}:${metric.id}:${i}`} metric={metric} source={source} mask={mask} now={now} compact />)}</ul>
|
||||
</section>}
|
||||
{issues.length > 0 && <div className="overflow-hidden rounded-lg border border-border bg-card"><ul className="divide-y divide-border">{issues.slice(0, 6).map(({ source, metric }, i) => <MetricRow key={`${source.id}:${metric.id}:${i}`} metric={metric} source={source} mask={mask} now={now} compact />)}</ul></div>}
|
||||
{issues.length > 6 && <p className="text-xs text-muted-foreground">{issues.length - 6} more signals in operational details.</p>}
|
||||
</> : <div className="grid grid-cols-1 items-start gap-4 xl:grid-cols-2">{sources.map((source) => <SourcePanel key={source.id} source={source} mask={mask} now={now} />)}</div>}
|
||||
</>}
|
||||
</>}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -25,6 +25,7 @@ import { PageHeader } from "@/components/layout/PageHeader";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { EmptyState, Panel, Section, Segmented, Stat, StatGrid } from "@/components/ui/kit";
|
||||
import { InstanceProblemsPanel } from "./InstanceHealthPanel";
|
||||
import { InstanceMonitoringPanel } from "./InstanceMonitoringPanel";
|
||||
import { useAdminPerm } from "@/hooks/useAdminPerm";
|
||||
import { AdminPerm } from "@/lib/auth/permissions";
|
||||
import { TONE_DOT, TONE_TEXT } from "@/lib/tones";
|
||||
@@ -83,6 +84,7 @@ export default function OverviewPage() {
|
||||
</PageHeader>
|
||||
|
||||
<InstanceProblemsPanel />
|
||||
<InstanceMonitoringPanel compact />
|
||||
|
||||
{!canView ? (
|
||||
<div className="rounded-lg border border-dashed border-border-strong">
|
||||
|
||||
@@ -1,7 +1,3 @@
|
||||
// Setup and health, findings: everything this deployment is currently
|
||||
// getting wrong, as decided by the running backend. The endpoint returns
|
||||
// only checks that are not ok, so an empty response is a real all-clear.
|
||||
|
||||
import { useState } from "react";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { useSearchParams } from "react-router-dom";
|
||||
@@ -43,7 +39,7 @@ export function FindingsTab() {
|
||||
<div className="flex items-center gap-2">
|
||||
{healthQ.dataUpdatedAt > 0 && (
|
||||
<span className="text-xs text-subtle-foreground tabular-nums">
|
||||
Last checked {new Date(healthQ.dataUpdatedAt).toLocaleTimeString()}
|
||||
Response received {new Date(healthQ.dataUpdatedAt).toLocaleTimeString()}
|
||||
</span>
|
||||
)}
|
||||
<Button
|
||||
@@ -70,7 +66,7 @@ export function FindingsTab() {
|
||||
|
||||
{healthQ.isError && (
|
||||
<ErrorState
|
||||
error={healthQ.error}
|
||||
error={new Error(healthQ.data ? "The refresh failed. Findings below are from the previous response, not a fresh verdict." : "Findings are unavailable. Verify access or retry; this is not an all-clear.")}
|
||||
title="Could not run the instance checks"
|
||||
onRetry={() => healthQ.refetch()}
|
||||
/>
|
||||
@@ -78,13 +74,12 @@ export function FindingsTab() {
|
||||
|
||||
{healthQ.data && (
|
||||
<>
|
||||
{problemCount === 0 && (
|
||||
<Callout tone="success" icon={CheckCircle2} title="No problems found">
|
||||
No errors or warnings were reported.
|
||||
<Callout tone={healthQ.isError ? "warning" : "info"} icon={Info} title={problemCount === 0 ? "No errors or warnings reported" : "Reported configuration findings"}>
|
||||
{healthQ.data.execution_coverage === "complete" ? "Configuration check coverage was reported complete, not operational send or sync coverage." : "Check execution coverage is not established. Absent findings do not prove every check ran or that the instance is healthy."}
|
||||
{" See Operations for measured activity, backlog and evidence freshness."}
|
||||
{checks.length > 0 &&
|
||||
" The informational notes below do not mean this instance is unhealthy."}
|
||||
</Callout>
|
||||
)}
|
||||
</Callout>
|
||||
{checks.length > 0 && (
|
||||
<div className={problemCount === 0 ? "mt-6" : undefined}>
|
||||
<SummaryStrip summary={summary} />
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
import { renderToStaticMarkup } from "react-dom/server";
|
||||
import { MemoryRouter } from "react-router-dom";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { InstanceHealthResult } from "@/lib/api/client/admin/instance";
|
||||
import type { SystemStatusResult } from "@/lib/api/client/admin/system";
|
||||
import { FindingsTab } from "./FindingsTab";
|
||||
import { ServicesTab } from "./ServicesTab";
|
||||
|
||||
const state = vi.hoisted(() => ({
|
||||
health: { data: undefined as InstanceHealthResult | undefined, isError: false, isLoading: false, isFetching: false, dataUpdatedAt: 0, error: new Error("secret-provider-detail"), refetch: vi.fn() },
|
||||
services: { data: undefined as SystemStatusResult | undefined, isError: false, isLoading: false, isFetching: false, error: new Error("secret-provider-detail"), refetch: vi.fn() },
|
||||
canConfigure: true,
|
||||
}));
|
||||
vi.mock("@/hooks/useInstanceHealth", () => ({ useInstanceHealth: () => state.health, findingCount: () => 0 }));
|
||||
vi.mock("@/hooks/useUpdateState", () => ({ useUpdateState: () => ({ data: undefined }) }));
|
||||
vi.mock("@/components/layout/UpdateDialog", () => ({ UpdateDialog: () => null }));
|
||||
vi.mock("../InstanceHealthPanel", () => ({ InstanceFindings: () => <div>Retained findings</div> }));
|
||||
vi.mock("@tanstack/react-query", () => ({ useQuery: () => state.services }));
|
||||
vi.mock("@/hooks/useAdminPerm", () => ({ useAdminPerm: () => state.canConfigure }));
|
||||
vi.mock("../MailStatusCard", () => ({ MailStatusCard: () => <span>Protected mail probe</span> }));
|
||||
|
||||
function render(page: "findings" | "services") {
|
||||
const html = renderToStaticMarkup(<MemoryRouter>{page === "findings" ? <FindingsTab /> : <ServicesTab />}</MemoryRouter>);
|
||||
return new DOMParser().parseFromString(html, "text/html").body.textContent;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
state.canConfigure = true;
|
||||
Object.assign(state.health, { data: { checks: [], summary: { error: 0, warning: 0, info: 0 }, execution_coverage: "unreported" }, isError: false, dataUpdatedAt: Date.now() });
|
||||
Object.assign(state.services, { data: { data: [], checked_at: new Date().toISOString() }, isError: false });
|
||||
});
|
||||
|
||||
describe("configuration and service coverage", () => {
|
||||
it("does not mount the protected platform mail probe without manage-settings access", () => {
|
||||
state.canConfigure = false;
|
||||
expect(render("services")).toContain("Platform mail permission required");
|
||||
expect(render("services")).not.toContain("Protected mail probe");
|
||||
});
|
||||
it("does not equate absent findings or an older coverage-free response to healthy", () => {
|
||||
for (const coverage of [undefined, "unreported"]) {
|
||||
state.health.data!.execution_coverage = coverage;
|
||||
expect(render("findings")).toContain("Check execution coverage is not established");
|
||||
expect(render("findings")).not.toContain("No problems found");
|
||||
expect(render("findings")).toContain("Response received");
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps failed refresh explicit and redacts raw query errors", () => {
|
||||
state.health.isError = true;
|
||||
state.services.isError = true;
|
||||
for (const page of ["findings", "services"] as const) {
|
||||
expect(render(page)).toContain("refresh");
|
||||
expect(render(page)).not.toContain("secret-provider-detail");
|
||||
expect(render(page)).not.toContain("All systems operational");
|
||||
}
|
||||
});
|
||||
|
||||
it("does not call an empty service list operational", () => {
|
||||
expect(render("services")).toContain("Current service coverage unknown");
|
||||
expect(render("services")).toContain("No components were returned");
|
||||
});
|
||||
|
||||
it("scopes passed probes to reachability and hides connection errors", () => {
|
||||
state.services.data!.data = [{ name: "postgres", ok: true, latency_ms: 5 }, { name: "redis", ok: false, latency_ms: 20, error: "password=private secret-provider-detail" }];
|
||||
expect(render("services")).toContain("1 service probe failed");
|
||||
expect(render("services")).toContain("Probe passed");
|
||||
expect(render("services")).not.toContain("secret-provider-detail");
|
||||
state.services.data!.data = [{ name: "postgres", ok: true, latency_ms: 5 }];
|
||||
expect(render("services")).toContain("Reported service probes passed");
|
||||
expect(render("services")).toContain("does not prove send throughput");
|
||||
});
|
||||
|
||||
it("does not turn stale probes or cached successful probes into a fresh verdict", () => {
|
||||
state.services.data!.data = [{ name: "postgres", ok: true, latency_ms: 5 }];
|
||||
state.services.data!.checked_at = new Date(Date.now() - 300_000).toISOString();
|
||||
expect(render("services")).toContain("Current service coverage unknown");
|
||||
expect(render("services")).toContain("Cached: passed");
|
||||
expect(render("services")).not.toContain("Probe passed");
|
||||
});
|
||||
|
||||
it("retains a cached failure even when the earlier probe supplied no error details", () => {
|
||||
state.services.data!.data = [{ name: "postgres", ok: true, latency_ms: 5 }, { name: "redis", ok: false, latency_ms: 20 }];
|
||||
state.services.isError = true;
|
||||
expect(render("services")).toContain("Cached: passed");
|
||||
expect(render("services")).toContain("Cached: failed");
|
||||
expect(render("services")).not.toContain("Probe passed");
|
||||
});
|
||||
});
|
||||
@@ -11,6 +11,9 @@ import { Callout, StatusBadge, StatusDot } from "@/components/ui/kit";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { MailStatusCard } from "../MailStatusCard";
|
||||
import { getSystemStatus, type SystemComponentStatus } from "@/lib/api/client/admin/system";
|
||||
import { observationIsOld } from "@/lib/monitoring";
|
||||
import { useAdminPerm } from "@/hooks/useAdminPerm";
|
||||
import { AdminPerm } from "@/lib/auth/permissions";
|
||||
|
||||
export const SYSTEM_STATUS_KEY = ["admin", "system", "status"] as const;
|
||||
|
||||
@@ -36,17 +39,20 @@ function titleCase(name: string): string {
|
||||
}
|
||||
|
||||
export function ServicesTab() {
|
||||
const canConfigure = useAdminPerm(AdminPerm.ManageSettings);
|
||||
// Probes have no realtime event, so this is a deliberate poll; the query
|
||||
// only lives while the tab is mounted.
|
||||
const statusQ = useQuery({
|
||||
queryKey: SYSTEM_STATUS_KEY,
|
||||
queryFn: getSystemStatus,
|
||||
refetchInterval: 15_000,
|
||||
refetchIntervalInBackground: false,
|
||||
retry: false,
|
||||
});
|
||||
|
||||
const components = statusQ.data?.data ?? [];
|
||||
const failing = components.filter((c) => !c.ok);
|
||||
const cached = statusQ.isError || observationIsOld(statusQ.data?.checked_at, Date.now());
|
||||
|
||||
return (
|
||||
<div>
|
||||
@@ -85,7 +91,7 @@ export function ServicesTab() {
|
||||
|
||||
{statusQ.isError && (
|
||||
<ErrorState
|
||||
error={statusQ.error}
|
||||
error={new Error(statusQ.data ? "Refresh failed. The previous probe results are retained, not a fresh verdict." : "Service probe results are unavailable. An empty or failed request is not an all-clear.")}
|
||||
title="Could not run health checks"
|
||||
onRetry={() => statusQ.refetch()}
|
||||
/>
|
||||
@@ -93,8 +99,10 @@ export function ServicesTab() {
|
||||
|
||||
{statusQ.data && (
|
||||
<>
|
||||
{failing.length === 0 ? (
|
||||
<Callout tone="success" icon={CheckCircle2} title="All systems operational" className="mb-5" />
|
||||
{cached || components.length === 0 ? (
|
||||
<Callout tone="warning" title="Current service coverage unknown" className="mb-5">{components.length === 0 ? "No components were returned. Reachability has not been established." : "Cached probes cannot establish current reachability."}</Callout>
|
||||
) : failing.length === 0 ? (
|
||||
<Callout tone="info" icon={CheckCircle2} title="Reported service probes passed" className="mb-5">Reachability at the probe time only. This does not prove send throughput, mailbox sync or recovery.</Callout>
|
||||
) : (
|
||||
<Callout
|
||||
tone="danger"
|
||||
@@ -102,8 +110,8 @@ export function ServicesTab() {
|
||||
className="mb-5"
|
||||
title={
|
||||
failing.length === 1
|
||||
? "1 component is down"
|
||||
: `${failing.length} components are down`
|
||||
? "1 service probe failed"
|
||||
: `${failing.length} service probes failed`
|
||||
}
|
||||
>
|
||||
{failing.map((c) => titleCase(c.name)).join(", ")}
|
||||
@@ -123,13 +131,13 @@ export function ServicesTab() {
|
||||
) : (
|
||||
<ul className="divide-y divide-border/70">
|
||||
{components.map((c) => (
|
||||
<ComponentRow key={c.name} component={c} />
|
||||
<ComponentRow key={c.name} component={c} cached={cached} />
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</div>
|
||||
<div className="order-1 lg:order-2">
|
||||
<MailStatusCard />
|
||||
{canConfigure ? <MailStatusCard /> : <Callout tone="neutral" title="Platform mail permission required">Manage settings access is required to inspect or test platform mail.</Callout>}
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
@@ -138,12 +146,12 @@ export function ServicesTab() {
|
||||
);
|
||||
}
|
||||
|
||||
function ComponentRow({ component: c }: { component: SystemComponentStatus }) {
|
||||
function ComponentRow({ component: c, cached }: { component: SystemComponentStatus; cached: boolean }) {
|
||||
return (
|
||||
<li className="px-4 py-3 transition-colors hover:bg-accent/50">
|
||||
<div className="flex items-start justify-between gap-3">
|
||||
<div className="min-w-0">
|
||||
<StatusDot tone={c.ok ? "success" : "danger"} className="font-medium text-foreground">
|
||||
<StatusDot tone={cached ? "neutral" : c.ok ? "success" : "danger"} className="font-medium text-foreground">
|
||||
<span className="truncate">{titleCase(c.name)}</span>
|
||||
</StatusDot>
|
||||
<p className="mt-0.5 pl-3.5 text-xs leading-relaxed text-muted-foreground">
|
||||
@@ -154,14 +162,14 @@ function ComponentRow({ component: c }: { component: SystemComponentStatus }) {
|
||||
<span className="text-xs text-muted-foreground tabular-nums" title="Latency">
|
||||
{c.latency_ms} ms
|
||||
</span>
|
||||
<StatusBadge tone={c.ok ? "success" : "danger"} className="w-[5.75rem] justify-center">
|
||||
{c.ok ? "Operational" : "Down"}
|
||||
<StatusBadge tone={cached ? "neutral" : c.ok ? "success" : "danger"} className="min-w-[5.75rem] justify-center">
|
||||
{cached ? c.ok ? "Cached: passed" : "Cached: failed" : c.ok ? "Probe passed" : "Probe failed"}
|
||||
</StatusBadge>
|
||||
</div>
|
||||
</div>
|
||||
{c.error && (
|
||||
<div className="mt-2 ml-3.5 break-words rounded-md border border-red-500/20 bg-red-500/[0.06] px-2.5 py-1.5 font-mono text-[11.5px] text-red-700 dark:text-red-400">
|
||||
{c.error}
|
||||
Probe failed. Inspect protected service logs for details; raw connection errors are not displayed here.
|
||||
</div>
|
||||
)}
|
||||
</li>
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
import { act } from "react";
|
||||
import { createRoot, type Root } from "react-dom/client";
|
||||
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { AdminPerm } from "@/lib/auth/permissions";
|
||||
import { INSTANCE_MONITORING_KEY, useInstanceMonitoring } from "./useInstanceMonitoring";
|
||||
|
||||
const state = vi.hoisted(() => ({ mask: 0, id: "first-admin", fetch: vi.fn() }));
|
||||
vi.mock("@/hooks/useMe", () => ({ useMe: () => ({ data: { id: state.id, admin_permissions: state.mask } }) }));
|
||||
vi.mock("@/lib/api/client/admin/monitoring", () => ({ getInstanceMonitoring: state.fetch }));
|
||||
|
||||
let root: Root | undefined;
|
||||
let client: QueryClient | undefined;
|
||||
afterEach(async () => {
|
||||
if (root) await act(async () => root!.unmount());
|
||||
root = undefined;
|
||||
client?.clear();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
function Reader() {
|
||||
const query = useInstanceMonitoring();
|
||||
return <span>{query.data?.checked_at ?? "No snapshot"}</span>;
|
||||
}
|
||||
|
||||
describe("shared monitoring query", () => {
|
||||
it("shares one scoped cache between readers, keeps original clocks and bounds polling", async () => {
|
||||
state.mask = AdminPerm.ViewAnalytics;
|
||||
state.id = "first-admin";
|
||||
state.fetch.mockResolvedValue({ version: "1", checked_at: "original-server-observation", sources: [] });
|
||||
client = new QueryClient({ defaultOptions: { queries: { retry: false } } });
|
||||
const container = document.createElement("div");
|
||||
root = createRoot(container);
|
||||
await act(async () => root!.render(<QueryClientProvider client={client!}><Reader /><Reader /></QueryClientProvider>));
|
||||
await vi.waitFor(() => expect(state.fetch).toHaveBeenCalledTimes(1));
|
||||
const queries = client.getQueryCache().getAll();
|
||||
expect(queries).toHaveLength(1);
|
||||
expect(queries[0].queryKey).toEqual([...INSTANCE_MONITORING_KEY, "first-admin", AdminPerm.ViewAnalytics]);
|
||||
expect(queries[0].state.data).toMatchObject({ checked_at: "original-server-observation" });
|
||||
expect(queries[0].options).toMatchObject({ staleTime: 60_000, refetchInterval: 60_000, refetchIntervalInBackground: false, refetchOnWindowFocus: false, retry: false });
|
||||
});
|
||||
|
||||
it("does not fetch without analytics and does not reuse a higher-permission or different-user snapshot", async () => {
|
||||
client = new QueryClient();
|
||||
client.setQueryData([...INSTANCE_MONITORING_KEY, "first-admin", AdminPerm.ViewAnalytics | AdminPerm.ViewUsers], { checked_at: "private-old-observation" });
|
||||
client.setQueryData([...INSTANCE_MONITORING_KEY, "other-admin", AdminPerm.ViewAnalytics], { checked_at: "private-other-admin" });
|
||||
state.mask = 0;
|
||||
state.id = "first-admin";
|
||||
const container = document.createElement("div");
|
||||
root = createRoot(container);
|
||||
await act(async () => root!.render(<QueryClientProvider client={client!}><Reader /></QueryClientProvider>));
|
||||
expect(state.fetch).not.toHaveBeenCalled();
|
||||
expect(container.textContent).toBe("No snapshot");
|
||||
state.mask = AdminPerm.ViewAnalytics;
|
||||
state.fetch.mockResolvedValue({ checked_at: "new-projection" });
|
||||
await act(async () => root!.render(<QueryClientProvider client={client!}><Reader /></QueryClientProvider>));
|
||||
await vi.waitFor(() => expect(state.fetch).toHaveBeenCalledTimes(1));
|
||||
expect(client.getQueryData([...INSTANCE_MONITORING_KEY, "first-admin", AdminPerm.ViewAnalytics])).toMatchObject({ checked_at: "new-projection" });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,21 @@
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import { useMe } from "@/hooks/useMe";
|
||||
import { AdminPerm } from "@/lib/auth/permissions";
|
||||
import { canMonitor } from "@/lib/monitoring";
|
||||
import { getInstanceMonitoring } from "@/lib/api/client/admin/monitoring";
|
||||
|
||||
export const INSTANCE_MONITORING_KEY = ["admin", "instance", "monitoring"] as const;
|
||||
|
||||
export function useInstanceMonitoring() {
|
||||
const { data: me } = useMe();
|
||||
return useQuery({
|
||||
queryKey: [...INSTANCE_MONITORING_KEY, me?.id, me?.admin_permissions],
|
||||
queryFn: getInstanceMonitoring,
|
||||
enabled: canMonitor(me?.admin_permissions, AdminPerm.ViewAnalytics),
|
||||
staleTime: 60_000,
|
||||
refetchInterval: 60_000,
|
||||
refetchIntervalInBackground: false,
|
||||
refetchOnWindowFocus: false,
|
||||
retry: false,
|
||||
});
|
||||
}
|
||||
@@ -58,9 +58,11 @@ export interface InstanceHealthSummary {
|
||||
}
|
||||
|
||||
export interface InstanceHealthResult {
|
||||
// Only non-ok checks are returned, so an empty list means all clear.
|
||||
// Only findings are returned; absent findings do not prove check coverage.
|
||||
checks: InstanceCheck[];
|
||||
summary: InstanceHealthSummary;
|
||||
execution_coverage?: string;
|
||||
operational_monitoring_path?: string;
|
||||
}
|
||||
|
||||
export function getInstanceHealth(): Promise<InstanceHealthResult> {
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import { Request } from "@/lib/api/client";
|
||||
|
||||
export type MonitoringAvailability = "fresh" | "stale" | "unavailable";
|
||||
export type MonitoringCondition = "healthy" | "unknown" | "no_recent_evidence" | "recent_failure" | "persistent_failure" | "recovery_unverified" | "safety_hold" | "backlog" | "in_flight" | "waiting_schedule" | "waiting_retry" | "budget" | "daily_limit" | "stale_telemetry";
|
||||
|
||||
export interface MonitoringMetric {
|
||||
id: string;
|
||||
title: string;
|
||||
unit: string;
|
||||
scope_id?: string;
|
||||
availability: MonitoringAvailability;
|
||||
condition: MonitoringCondition;
|
||||
severity: string;
|
||||
reason?: string;
|
||||
note: string;
|
||||
count: number | null;
|
||||
affected_mailboxes: number | null;
|
||||
affected_organizations: number | null;
|
||||
unknown_organization_rows: number | null;
|
||||
observed_at: string | null;
|
||||
evidence_at: string | null;
|
||||
latest_evidence_at: string | null;
|
||||
evidence_age_seconds: number | null;
|
||||
next_eligible_at: string | null;
|
||||
window_start: string | null;
|
||||
window_end: string | null;
|
||||
threshold_seconds: number | null;
|
||||
investigate?: string;
|
||||
}
|
||||
|
||||
export interface MonitoringSource {
|
||||
id: string;
|
||||
availability: MonitoringAvailability;
|
||||
coverage: string;
|
||||
reason?: string;
|
||||
observed_at: string | null;
|
||||
checked_at: string;
|
||||
measured_scopes: number | null;
|
||||
expected_scopes: number | null;
|
||||
metrics: MonitoringMetric[];
|
||||
}
|
||||
|
||||
export interface MonitoringSnapshot {
|
||||
version: string;
|
||||
checked_at: string;
|
||||
refresh_after: string;
|
||||
coverage: string;
|
||||
sources: MonitoringSource[];
|
||||
}
|
||||
|
||||
export function getInstanceMonitoring(): Promise<MonitoringSnapshot> {
|
||||
return Request({ method: "GET", url: "/admin/instance/monitoring", authorization: true, timeout: 12_000 });
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { AdminPerm } from "@/lib/auth/permissions";
|
||||
import type { MonitoringSource } from "@/lib/api/client/admin/monitoring";
|
||||
import { canMonitor, investigationPath, MONITORING_SOURCES, observationIsOld, permittedSource } from "./monitoring";
|
||||
|
||||
describe("monitoring permission and freshness boundaries", () => {
|
||||
it.each(Object.entries(MONITORING_SOURCES))("requires the native source bit for %s", (id, { permission }) => {
|
||||
const source = { id, availability: "fresh", metrics: [{ count: 42 }] } as MonitoringSource;
|
||||
expect(permittedSource(source, AdminPerm.ViewAnalytics | permission)).toBe(source);
|
||||
if (permission !== AdminPerm.ViewAnalytics) {
|
||||
expect(permittedSource(source, AdminPerm.ViewAnalytics)).toMatchObject({ availability: "unavailable", reason: "permission_denied", metrics: [], measured_scopes: null, expected_scopes: null });
|
||||
}
|
||||
});
|
||||
|
||||
it("fails closed for unknown masks and unsupported destinations", () => {
|
||||
expect(canMonitor(undefined, AdminPerm.ViewAnalytics)).toBe(false);
|
||||
for (const path of ["https://external.invalid/", "//external.invalid/", "/workers/secret@example.com", "/sends?tab=failures&token=secret", "/jobs#payload"]) {
|
||||
expect(investigationPath(path, 0xffffff)).toBeNull();
|
||||
}
|
||||
});
|
||||
|
||||
it("accepts known native destinations only with their separate page permission", () => {
|
||||
for (const [path, permission] of [["/sends?tab=dead-letters", AdminPerm.ViewCampaigns], ["/sync?state=backfilling", AdminPerm.ViewUsers], ["/mailboxes", AdminPerm.ViewUsers], ["/fleet", AdminPerm.ViewWorkers], ["/workers/1a64feee-dee2-41df-a7f5-a85816ba8585", AdminPerm.ViewWorkers], ["/organizations/1a64feee-dee2-41df-a7f5-a85816ba8585", AdminPerm.ViewOrganizations], ["/jobs", AdminPerm.ViewAnalytics], ["/configuration?tab=notifications", AdminPerm.ManageSettings]] as const) {
|
||||
expect(investigationPath(path, permission)).toBe(path);
|
||||
expect(investigationPath(path, 0)).toBeNull();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not call missing, invalid, future or old observation clocks fresh", () => {
|
||||
const now = Date.now();
|
||||
for (const value of [null, "not-a-time", new Date(now - 121_000).toISOString(), new Date(now + 61_000).toISOString()]) expect(observationIsOld(value, now)).toBe(true);
|
||||
expect(observationIsOld(new Date(now - 60_000).toISOString(), now)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,102 @@
|
||||
import { AdminPerm } from "@/lib/auth/permissions";
|
||||
import type { MonitoringCondition, MonitoringMetric, MonitoringSource } from "@/lib/api/client/admin/monitoring";
|
||||
|
||||
export const MONITORING_SOURCES: Record<string, { title: string; permission: number }> = {
|
||||
send_safety: { title: "Send safety", permission: AdminPerm.ViewUsers },
|
||||
sends: { title: "Send confirmations", permission: AdminPerm.ViewCampaigns },
|
||||
campaign_reservations: { title: "Campaign reservations", permission: AdminPerm.ViewCampaigns },
|
||||
dispatch: { title: "Scheduled dispatch", permission: AdminPerm.ViewCampaigns },
|
||||
sync: { title: "Mailbox sync observations", permission: AdminPerm.ViewUsers },
|
||||
mailbox_errors: { title: "Recorded mailbox issues", permission: AdminPerm.ViewUsers },
|
||||
worker_samples: { title: "Worker telemetry", permission: AdminPerm.ViewWorkers },
|
||||
workers: { title: "Worker registration", permission: AdminPerm.ViewWorkers },
|
||||
jobs: { title: "Registered jobs", permission: AdminPerm.ViewAnalytics },
|
||||
dead_letters: { title: "Dead letters", permission: AdminPerm.ViewCampaigns },
|
||||
webhooks: { title: "Customer webhooks", permission: AdminPerm.ViewOrganizations },
|
||||
notifications: { title: "Customer notification rows", permission: AdminPerm.ManageSettings },
|
||||
result_effects: { title: "Send result effects", permission: AdminPerm.ViewCampaigns },
|
||||
arrivals: { title: "Optional arrival outbox", permission: AdminPerm.ViewUsers },
|
||||
domain_auth: { title: "Sending-domain authentication", permission: AdminPerm.ViewUsers },
|
||||
warmup_loading: { title: "Warmup mailbox loading", permission: AdminPerm.ViewCampaigns },
|
||||
backend_event_broker: { title: "Backend result broker", permission: AdminPerm.ViewAnalytics },
|
||||
worker_command_broker: { title: "Worker command broker", permission: AdminPerm.ViewWorkers },
|
||||
send_wait_attribution: { title: "Detailed send wait reasons", permission: AdminPerm.ViewCampaigns },
|
||||
tracking_broker: { title: "External tracking broker", permission: AdminPerm.ViewAnalytics },
|
||||
};
|
||||
|
||||
export const CONDITION_LABELS: Record<MonitoringCondition, string> = {
|
||||
healthy: "Confirmed scoped observation", unknown: "Unknown", no_recent_evidence: "No matching recent evidence",
|
||||
recent_failure: "Recent failure evidence", persistent_failure: "Persistent failure evidence",
|
||||
recovery_unverified: "Recovery unverified", safety_hold: "Safety hold", backlog: "Measured backlog",
|
||||
in_flight: "In flight", waiting_schedule: "Scheduled wait", waiting_retry: "Retry wait",
|
||||
budget: "Budget wait", daily_limit: "Daily-limit wait", stale_telemetry: "Stale telemetry",
|
||||
};
|
||||
|
||||
const REASONS: Record<string, string> = {
|
||||
permission_denied: "Permission required", unsupported: "Not supported by this source",
|
||||
dependency_missing: "Dependency unavailable", schema_absent: "Optional schema not installed",
|
||||
timeout: "Collection timed out", query_failed: "Measurement query failed", collection_failed: "Collection failed",
|
||||
sample_limit: "Bounded sample", scope_limit: "Bounded scope", no_recent_evidence: "No recent measurement evidence",
|
||||
no_registered_scopes: "No registered scopes to measure",
|
||||
};
|
||||
|
||||
export function monitoringReason(reason?: string): string {
|
||||
return REASONS[reason ?? ""] ?? "Measurement unavailable or incomplete";
|
||||
}
|
||||
|
||||
export function canMonitor(mask: number | undefined, permission: number): boolean {
|
||||
return typeof mask === "number" && (mask & permission) === permission;
|
||||
}
|
||||
|
||||
export function permittedSource(source: MonitoringSource, mask: number | undefined): MonitoringSource {
|
||||
const required = MONITORING_SOURCES[source.id]?.permission;
|
||||
if (required !== undefined && canMonitor(mask, required)) return source;
|
||||
return { ...source, availability: "unavailable", coverage: "unavailable", reason: required === undefined ? "unsupported" : "permission_denied", observed_at: null, measured_scopes: null, expected_scopes: null, metrics: [] };
|
||||
}
|
||||
|
||||
const ATTENTION = new Set<MonitoringCondition>(["recent_failure", "persistent_failure", "recovery_unverified", "safety_hold", "backlog", "stale_telemetry"]);
|
||||
|
||||
export function needsMonitoringAttention(metric: MonitoringMetric): boolean {
|
||||
return metric.availability !== "unavailable" && typeof metric.count === "number" && metric.count > 0 &&
|
||||
(ATTENTION.has(metric.condition) || metric.severity === "warning" || metric.severity === "error" || isBrokerQueue(metric));
|
||||
}
|
||||
|
||||
export function isBrokerQueue(metric: MonitoringMetric): boolean {
|
||||
return ["committed_lag", "pending", "ack_pending", "redelivered"].includes(metric.id);
|
||||
}
|
||||
|
||||
export function monitoringAttentionCount(sources: MonitoringSource[]): number {
|
||||
return sources.reduce((count, source) => count + source.metrics.filter(needsMonitoringAttention).length, 0);
|
||||
}
|
||||
|
||||
export function investigationPath(path: string | undefined, mask: number | undefined): string | null {
|
||||
if (!path) return null;
|
||||
let required: number | undefined;
|
||||
if (/^\/sends\?tab=(in-flight|failures|dead-letters)$/.test(path)) required = AdminPerm.ViewCampaigns;
|
||||
else if (path === "/sends?tab=webhooks") required = AdminPerm.ViewCampaigns | AdminPerm.ViewOrganizations;
|
||||
else if (/^\/(sync|mailboxes)(\?state=(throttled|backfilling|stalled|pending|complete))?$/.test(path)) required = AdminPerm.ViewUsers;
|
||||
else if (/^\/(workers|fleet)(\/[0-9a-f-]{36})?$/.test(path)) required = AdminPerm.ViewWorkers;
|
||||
else if (/^\/organizations\/[0-9a-f-]{36}$/.test(path)) required = AdminPerm.ViewOrganizations;
|
||||
else if (/^\/configuration(\?tab=notifications)?$/.test(path)) required = AdminPerm.ManageSettings;
|
||||
else if (/^\/(jobs|health)(\?tab=(findings|services|operations))?$/.test(path)) required = AdminPerm.ViewAnalytics;
|
||||
return required !== undefined && canMonitor(mask, required) ? path : null;
|
||||
}
|
||||
|
||||
export function timestamp(value: string | null | undefined): number | null {
|
||||
const parsed = value ? Date.parse(value) : NaN;
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
|
||||
}
|
||||
|
||||
export function observationIsOld(value: string | null | undefined, now: number): boolean {
|
||||
const at = timestamp(value);
|
||||
return at === null || at > now + 60_000 || now - at > 120_000;
|
||||
}
|
||||
|
||||
export function ageLabel(at: number, now: number): string {
|
||||
const seconds = Math.floor((now - at) / 1000);
|
||||
if (seconds < 0) return "future timestamp, age unknown";
|
||||
if (seconds < 60) return `${seconds}s ago`;
|
||||
if (seconds < 3600) return `${Math.floor(seconds / 60)}m ago`;
|
||||
if (seconds < 86400) return `${Math.floor(seconds / 3600)}h ago`;
|
||||
return `${Math.floor(seconds / 86400)}d ago`;
|
||||
}
|
||||
@@ -77,6 +77,7 @@ import (
|
||||
"github.com/warmbly/warmbly/internal/app/mailboxavatar"
|
||||
"github.com/warmbly/warmbly/internal/app/mailboximport"
|
||||
"github.com/warmbly/warmbly/internal/app/mcp"
|
||||
"github.com/warmbly/warmbly/internal/app/monitoring"
|
||||
"github.com/warmbly/warmbly/internal/app/nativeactions"
|
||||
"github.com/warmbly/warmbly/internal/app/notification"
|
||||
"github.com/warmbly/warmbly/internal/app/oauth"
|
||||
@@ -341,6 +342,7 @@ func main() {
|
||||
// deployment with no channels simply never delivers anything.
|
||||
var opsNotifier opsnotify.Notifier
|
||||
var instanceChecksDB *pgxpool.Pool
|
||||
var monitoringBus eventbus.EventBus
|
||||
var userRepoForHandler repository.UserRepository
|
||||
var organizationRepoForHandler repository.OrganizationRepository
|
||||
var orgRiskService orgrisk.Service
|
||||
@@ -628,6 +630,7 @@ func main() {
|
||||
errs.CaptureFatal(err)
|
||||
log.Fatal(err)
|
||||
}
|
||||
monitoringBus = bus
|
||||
|
||||
// The captcha bypass exists only in dev and only as an explicitly set token; there is no default.
|
||||
turnstileBypassToken := ""
|
||||
@@ -2272,6 +2275,11 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
monitoringRepo := repository.NewMonitoringRepository(instanceChecksDB)
|
||||
monitoringSources := append(monitoringRepo.Sources(), monitoringRepo.BrokerSources(monitoringBus)...)
|
||||
monitoringSources = append(monitoringSources, monitoring.UnobservedSources()...)
|
||||
monitoringService := monitoring.New(monitoringSources)
|
||||
|
||||
// The facts only boot knows, handed to the configuration and health pages so
|
||||
// they report what this process actually resolved rather than re-reading the
|
||||
// environment and drifting from it.
|
||||
@@ -2311,6 +2319,7 @@ func main() {
|
||||
InstanceRuntime: instanceRuntime,
|
||||
InstanceChecks: instanceChecks,
|
||||
InstanceSettings: instanceSettings,
|
||||
Monitoring: monitoringService,
|
||||
OpsNotifier: opsNotifier,
|
||||
|
||||
PoolLinkService: poolLinkService,
|
||||
|
||||
@@ -11,6 +11,8 @@ This page is the source of truth for what an API key can and cannot reach. Every
|
||||
|
||||
When an endpoint says "JWT permission: X / API permission: Y", the dual-auth middleware checks the relevant one based on which credential the caller used.
|
||||
|
||||
A member restricted to [selected resources](/guides/team-roles/#access-scope) is held to a short list of read routes whatever credential they use, and those routes filter to their grants: `GET /campaigns`, `/campaigns-overview`, `/campaigns/:id` with its `/logs`, `/steps` and `/ab-variants`, `GET /analytics/dashboard` (whose `campaign_ids` and `folder_ids` filters must name granted campaigns and folders), `/analytics/campaigns/compare`, `/analytics/campaigns/:id` with `/daily` and `/hourly`, `GET /analytics/accounts` and `/analytics/accounts/:id`, `GET /emails` and `/emails/:id`, and the unibox reads `GET /unibox`, `/unibox/count`, `/unibox/overview`, `/unibox/thread`, `/unibox/thread/labels`, `/unibox/scheduled` and `/unibox/:id` (which, for them, does not mark the message read). Their own account routes under `/auth/*`, `/me/*`, `GET /organization`, `/organization/current`, `/organization/switch/:id`, `/invitations`, `GET /subscription`, `/subscription/features`, `GET /timezones` and `POST /getaway` keep working. Every other route answers `403` `member_access_restricted`, and a campaign or mailbox outside their grants answers `404`. An API key's own mailbox allowlist still applies on top: the key reaches the mailboxes on both lists.
|
||||
|
||||
All paths below are relative to the versioned base URL `https://api.warmbly.com/v1` (for example `/campaigns` is `https://api.warmbly.com/v1/campaigns`). See [versioning](/api/) for details.
|
||||
|
||||
## API key accepted
|
||||
@@ -514,7 +516,7 @@ These changes require the session to have confirmed the account holder within th
|
||||
| Remove a passkey | `DELETE /auth/passkey/credentials/:id` |
|
||||
| Turn on two-factor authentication | `POST /auth/2fa/enroll/start`, `/confirm` (see below for accounts with no password) |
|
||||
| Transfer a workspace | `POST /organization/transfer-ownership` |
|
||||
| Invite a member or change a member's role | `POST /organization/members/invite`, `PATCH /organization/members/:id` |
|
||||
| Invite a member, change a member's role or change their access scope | `POST /organization/members/invite`, `PATCH /organization/members/:id`, `PUT /organization/members/:id/access` |
|
||||
| Export or import a whole workspace | `POST /organization/current/export`, `POST /organization/current/import` |
|
||||
| Schedule a workspace or account for deletion | `POST /organization/current/danger-zone/delete`, `POST /me/danger-zone/delete` |
|
||||
| Approve a self-hosted instance's warmup pool link | `POST /pool-link/codes/:code/approve` |
|
||||
@@ -605,11 +607,17 @@ Alongside them, `api_url` is this API's own public base (for a copyable example
|
||||
- `GET /me/views/:view`, `PUT /me/views/:view`, `DELETE /me/views/:view` (the signed-in member's own column layout and sort for a dashboard list in the current workspace; `view` is `contacts` or `campaign_leads`, or `unibox_rail` for the unibox scope rail, which saves a `layout` of `favorites`, `hidden`, `order` and `section_order` instead of columns. Personal to the session, so no API scope reaches it)
|
||||
- `GET /invitations`, `POST /invitations/accept`
|
||||
- All of `/organization/*` (create, switch, members, invitations, transfer ownership, avatar, danger zone)
|
||||
- `GET /organization/members` includes each member's `access_scope` (`workspace` or `restricted`) and `access` (`scope`, `campaign_folder_ids`, `campaign_ids`, `email_account_ids`); `GET /organization` carries the caller's own `access_scope` per workspace. `GET /organization/invitations` carries each invitation's `access`
|
||||
- `GET /organization/members/:id/access` and `PUT /organization/members/:id/access` (JWT permission `MANAGE_TEAM`; the `PUT` also needs a recent confirmation). `:id` is the member's user id. The body is the whole scope, `{"scope": "restricted", "campaign_folder_ids": [], "campaign_ids": [], "email_account_ids": []}`, and replaces what was there, so a retry lands on the same state and takes no `Idempotency-Key`. `scope: "workspace"` clears the grants. Every id must belong to the workspace (`400` `invalid_access_resource`), at most 500 per list, the owner cannot be restricted (`403` `owner_access_unrestricted`), nobody can change their own access, and the caller must hold every permission the member's roles grant. Answers the member with `access`, and publishes `AUDIT_CREATED` with `entity_type` `organization_member`
|
||||
- `POST /organization/members/invite` accepts the same object as `access`, so an invitee joins already restricted. Ids are checked on invite and again on acceptance, which drops grants on anything deleted in between
|
||||
- `GET /organization/access/suggested-senders?campaign_ids=&campaign_folder_ids=` (JWT permission `MANAGE_TEAM`): the mailboxes those campaigns, and the campaigns in those folders, send from (explicit senders and sending-tag matches), each with the `campaign_ids` it sends for. A suggestion grants nothing
|
||||
- `GET /website-tracking/settings`, `PATCH /website-tracking/settings`, `POST /website-tracking/settings/rotate-key` (the [website tracking](/guides/website-tracking/) snippet's consent mode, location precision, allowed hosts and retention; JWT permission `MANAGE_SETTINGS`. The rotate is bodyless and safe to repeat, each call issues a new key)
|
||||
- All of `/subscription/*` (checkout, portal, cancel, change-plan, preview-change, enterprise-inquiry, discounts, referrals, etc.)
|
||||
- All of `/auth/cli/*` except the two handshake routes below (`GET /auth/cli/codes/:code`, `POST /auth/cli/codes/:code/approve`, `POST /auth/cli/codes/:code/deny`: the browser half of `warmbly auth login`, where a signed-in member reviews the code a CLI is showing and authorizes it. Approving mints an ordinary API key, so it requires the `MANAGE_API_KEYS` organization permission and a fresh authentication, is session-only (an API key must not be able to mint another one this way), and caps the key to the scopes the approver's role allows. `GET /auth/cli/codes/:code?organization_id=` adds `granted_scopes` and `granted_scope_names`: what approving into that workspace would grant)
|
||||
- All of `/pool-link/*` and `/cloud-link/*` (the self-hosted warmup pool link: approving an instance's code, listing and unlinking instances, reading the pool plan's price and opening its checkout, and on a self-hosted instance the connect flow and mailbox enrollment). `/cloud-link/*` exists only on a self-hosted instance. Connections belong to the selected local workspace, but linking and unlinking them (`/connect`, `/connect/poll`, `DELETE /cloud-link`) and the linked cloud workspace's mailboxes (`/workspace-mailboxes`, `.../adopt`) take a platform admin holding `manage_settings` whose session presented a second factor (`admin_mfa_required` otherwise). Reading the link and putting a workspace's own mailboxes on it (`GET /cloud-link`, `/mailboxes`, enroll, pause, resume and the Google and Microsoft sign-in under `/oauth/*`) take the workspace's `MANAGE_EMAILS`. `GET /cloud-link` includes `legacy_connected`; mailbox rows include `legacy` when they still use the old connection. A normal `DELETE /cloud-link` only disconnects the selected workspace's scoped link. `DELETE /cloud-link?legacy=true` explicitly disconnects the legacy link across all workspaces and uses the same platform-admin permission. New connections and enrollments require workspace scope, and a Cloud workspace accepts only one active link (legacy links count). `POST /pool-link/codes` requires `remote_organization_id` and returns `workspace_scoped: true` for supported clients. `POST /pool-link/codes` and `POST /pool-link/poll` are public and per-IP rate limited: they are the device-code handshake an instance uses before it has a token, and `/pool-link/instance/*` accepts only an instance token. `/pool-link/instance/oauth/*`, `/pool-link/instance/mailboxes/:id/token`, `/pool-link/instance/workspace-mailboxes` and `/pool-link/instance/mailboxes/adopt` are the cloud-managed mailbox surface (Google and Microsoft sign-in on Warmbly's OAuth apps, brokered access tokens); their instance-side counterparts are `/cloud-link/oauth/*` and `/cloud-link/workspace-mailboxes/*`. `/pool-link/instance/redirects` (list), and `GET`, `PUT`, `DELETE` and `POST .../verify` on `/pool-link/instance/redirects/:domain`, are the root redirects Warmbly Cloud serves for the calling instance; the instance drives them through `PUT /emails/domains/:domain/redirect` with `served_by: "cloud"`, and `GET /pool-link/instance` carries `redirects` (`available`, `host`, `limit`, `used`)
|
||||
- All of `/admin/*`. `DELETE /admin/instance/invitations/expired` requires platform admin `manage_organizations` and an MFA-verified session. It removes only expired invitation records across all workspaces, leaves active invitations and membership unchanged, records an admin audit entry, and returns `{ "cleaned": true }`. The admin panel confirms before calling it.
|
||||
- `GET /admin/instance/health` requires `view_analytics` and an MFA-verified admin session. Returns configuration `checks`, `summary`, additive `execution_coverage` (currently `unreported`) and `operational_monitoring_path`. Empty findings do not establish execution coverage or operational health.
|
||||
- `GET /admin/instance/monitoring` requires `view_analytics` and an MFA-verified admin session, not an organization API-key scope. Returns the read-only version `"1"` snapshot with `checked_at`, `refresh_after`, `coverage` and permission-shaped `sources`; HTTP responses use `Cache-Control: no-store`, while the backend shares a bounded one-minute collection cache. Each source has availability, coverage, nullable observation/scope measurements and metrics with nullable counts, affected mailboxes/organizations, evidence clocks, conditions and optional investigation paths. Missing permissions return unavailable source placeholders with `permission_denied`, not private metrics. Source bits and interpretation are listed under [operational monitoring](/development/instance-health/#operational-monitoring). Unavailable is not zero or an outage; a positive broker lag/pending observation remains `unknown`/`info`, not proof of failed throughput. This route never subscribes, replays, reconnects, clears holds or changes broker offsets/assignments.
|
||||
- `GET /admin/analytics/overview` requires `view_analytics`. `total_emails_sent` and `emails_sent_today` count confirmed warmup and other outbound sends, not dispatches. Today uses the UTC calendar day. A send-count database error fails the request instead of returning zero counters. See [platform admin send counters](/guides/analytics/#platform-admin-send-counters).
|
||||
- `GET /admin/organizations` and `GET /admin/organizations/:id` require `view_organizations`. Their acquisition fields include the nullable `referrer_host` saved at signup, alongside UTM source, medium, campaign and landing path. The list accepts an exact `referrer_host` filter as well as its existing acquisition filters. An absent source is unknown, not proof of a direct visit. See [signup source attribution](/guides/analytics/#signup-source-attribution-platform-admins).
|
||||
- `GET /admin/warmup-content/overview` includes effective generation controls and requires `view_warmup_pool`; `PUT /admin/warmup-content/settings` writes effective controls with `manage_settings`. These are platform-admin routes, not organization API-key scopes. See [effective generation policy and stopping](/development/warmup-generation/).
|
||||
|
||||
@@ -448,6 +448,7 @@ Returned when authenticated but lacking necessary permissions.
|
||||
- Request IP not in allowlist
|
||||
- Email account not in allowlist
|
||||
- Organization access restricted
|
||||
- The member is restricted to selected resources (`member_access_restricted`)
|
||||
|
||||
**Example:**
|
||||
|
||||
@@ -542,6 +543,31 @@ Turning on two-factor authentication (`POST /auth/2fa/enroll/start` and `/confir
|
||||
}
|
||||
```
|
||||
|
||||
#### Member access scope
|
||||
|
||||
A member restricted to [selected resources](/guides/team-roles/#access-scope), and any API key or OAuth app acting for one, reaches only the read routes that filter to their grants. The scope's own refusals:
|
||||
|
||||
| `code` | Status | Meaning |
|
||||
|--------|--------|---------|
|
||||
| `member_access_restricted` | 403 | The caller is restricted to selected campaigns and mailboxes, and this route is not one their scope covers. Writes are always refused, as are workspace-wide surfaces such as contacts and settings |
|
||||
| `invalid_access_resource` | 400 | `PUT /organization/members/:id/access` or an invitation named a campaign folder, campaign or mailbox that is not in this workspace |
|
||||
| `invalid_access_scope` | 400 | The scope is neither `workspace` nor `restricted` |
|
||||
| `too_many_access_grants` | 400 | More than 500 folders, campaigns or mailboxes in one list |
|
||||
| `owner_access_unrestricted` | 403 | The workspace owner always reaches the entire workspace and cannot be restricted |
|
||||
|
||||
A campaign or mailbox outside a restricted caller's grants answers `404`, the same as one that does not exist, so a scope does not reveal what else the workspace holds.
|
||||
|
||||
```json
|
||||
{
|
||||
"error": "Forbidden",
|
||||
"message": "Your access to this workspace is limited to selected campaigns and mailboxes, and this is outside it.",
|
||||
"code": "member_access_restricted",
|
||||
"request_id": "4bbbd1b2-8f86-47dd-8a7f-9476501ad20e"
|
||||
}
|
||||
```
|
||||
|
||||
**How to fix:** ask a member with **Manage team** to widen the access under **Settings > Members**.
|
||||
|
||||
#### `slack_link_email_mismatch`
|
||||
|
||||
A `403` whose `code` is `slack_link_email_mismatch` comes from `POST /integrations/slack/link` sent without a Sign in with Slack result. Confirming a [Slack link](/guides/slack/#link-your-slack-account) that way needs the email on the Slack account's profile to be the signed-in user's Warmbly email, compared without regard to case. The code stays usable: confirm it again with `slack_code` and `state` from `POST /integrations/slack/link/verify`. `GET /integrations/slack/link/:code` reports the check ahead of time as `email_matches`, and whether Sign in with Slack is available as `verify_available`.
|
||||
|
||||
@@ -84,6 +84,7 @@ Where it applies:
|
||||
- **Creating or editing an API key** from the dashboard refuses a permission outside the member's role with `403` `api_key_permissions_exceed_caller`.
|
||||
- **Creating or editing an API key with an API key** keeps the result inside the calling key: no permission it lacks, and when the calling key is limited to some mailboxes or IP addresses, the new or edited key must be limited to a subset of them. The same `403` `api_key_permissions_exceed_caller` answers otherwise.
|
||||
- **An OAuth grant** carries only the requested scopes the approving member's role covers, re-checked against the member's current role on every request. See [OAuth scopes](/api/oauth/#scopes). `API_KEYS` is never granted to an OAuth app, and OAuth tokens cannot create, edit, revoke or delete API keys (`403` `oauth_token_not_allowed`).
|
||||
- **A member restricted to [selected resources](/guides/team-roles/#access-scope)** holds only `view_campaigns`, `view_analytics` and `access_unibox` from their roles, so a key or grant acting for them is limited to the read permissions those cover, and to the read routes and grants their scope allows ([which routes](/api/endpoints/)). A key's own mailbox allowlist still applies on top. Restricted members cannot create keys.
|
||||
|
||||
## Preset combinations
|
||||
|
||||
|
||||
@@ -59,6 +59,8 @@ Events arrive as channel messages whose event name is the event type, for exampl
|
||||
|
||||
Permission filtering happens per event on the org channel: for example inbox events require `access_unibox`, campaign pulses require `view_campaigns`, placement tests require `view_analytics`, mailbox and mailbox import events require `manage_emails`, member changes require `manage_team`, and billing events require `manage_billing`. A member without the permission simply never receives the event.
|
||||
|
||||
For a member restricted to [selected resources](/guides/team-roles/#access-scope), and any key or app acting for one, the org channel also filters by their grants: inbox events need a granted `email_account_id`, other events need a granted `campaign_id` or `email_account_id`, and `AUDIT_CREATED` arrives only for a granted campaign, folder or mailbox, or for a change to the member themselves. Everything else is dropped, including events that name no resource. Such a socket is never tracked in presence and receives no `presence_state`, `presence_diff` or `live:*` frames. `campaign:<id>` and `account:<id>` refuse to join outside the grants. Grants are re-read when the member's access changes and when a campaign or folder changes, so folder grants follow the folder's contents.
|
||||
|
||||
A change to your own membership, to a role, or to the workspace's owner is re-read on the org channel as it happens, so the filtering above follows it without a reconnect. Removal from a workspace also applies to open connections: every socket the removed member has open is disconnected. On reconnect each channel's join checks membership again, so channels for workspaces they still belong to come back and the removed workspace's are refused with `not_a_member` (code `4010`).
|
||||
|
||||
## Selecting events with intents
|
||||
|
||||
@@ -9,6 +9,8 @@ The admin panel is the operator's view of an instance: every worker, user, organ
|
||||
|
||||
The panel runs on `:5174` in the default stack (`http://localhost:5174` on a stock install, or the host you proxied to it). Sign in with the same account as the dashboard. The first owner holds every admin bit from the moment they claim the instance, and `warmblyctl user create --admin` creates a further account that already holds them. A page the account has no bit for says which bit it needs instead of rendering.
|
||||
|
||||
A completed admin sign-in discards previously cached operator data and permissions before storing the new session. An unsuccessful sign-in or a pending email-code/second-factor step does not replace the session.
|
||||
|
||||
<Callout type="warn" title="Two-factor authentication is required">
|
||||
Every admin route refuses a session that did not present a second factor. Turn on 2FA or add a passkey under **Settings > Security** in the dashboard, then sign in again; the panel says so rather than failing silently.
|
||||
|
||||
@@ -48,6 +50,10 @@ The panel is dark by default. The account menu at the bottom of the sidebar (you
|
||||
|
||||
The landing page. Instance-wide counters and their trends, acquisition by channel, and a strip of open instance problems at the top, which is the same list as the Findings tab of [Setup and health](/development/instance-health/).
|
||||
|
||||
**Operational monitoring** adds recorded failure evidence, unverified recovery, safety restrictions, missing telemetry and measured backlogs. Positive Kafka lag or JetStream pending/ack-pending/redelivered counts stay visible separately, even when their backend condition is unknown. These are observations, not proof of an outage or provider delivery. Counts are per metric and can overlap; the attention count is a count of signals, not customers or failed messages.
|
||||
|
||||
The strip displays coverage gaps, stale sources and unknown/no-recent-evidence sources, rather than hiding a failed request behind an empty panel. **All operational details** opens Setup and health's Operations tab. The snapshot and each investigation link require their own admin permission; hidden sections cannot be recovered by changing tabs. See [monitoring permissions and evidence](/development/instance-health/#operational-monitoring).
|
||||
|
||||
## Operations
|
||||
|
||||
| Page | What it answers |
|
||||
@@ -123,7 +129,7 @@ Reading the tabs needs `view_organizations`; every action needs `manage_organiza
|
||||
|
||||
| Page | What it answers |
|
||||
|---|---|
|
||||
| **Setup and health** | Two tabs. **Findings** is the [instance health](/development/instance-health/) check list, and the sidebar badge counts its open findings. **Services** is the old System Status page: the per-service probes and the platform mail card with its connection check and send test |
|
||||
| **Setup and health** | Three tabs. **Findings** is the [instance health](/development/instance-health/) check list, and the sidebar badge counts its errors and warnings, not operational signals. **Services** holds scoped service reachability probes and the platform mail card. **Operations** holds read-only measurements, coverage, affected counts, evidence age, waits and permission-gated investigation links |
|
||||
| **Configuration** | Four tabs. **Settings** is the database-backed settings document; **Notifications** is the [operator notification](/development/operator-notifications/) channels; **Environment** is the read-only [configuration reference](/development/configuration/) view of what the backend resolved; **Effective limits** lists the compiled pacing constants |
|
||||
| **Transfers** | Workspace export and import for any organization, from the operator side, with the same archives as `warmblyctl org export` and `org import`. A whole-instance backup stays `warmblyctl backup` and `restore` |
|
||||
|
||||
|
||||
@@ -3,13 +3,13 @@ title: Instance health
|
||||
description: The Setup and health page, what every check means, how to clear it, and how to run the same checks from a shell.
|
||||
---
|
||||
|
||||
Warmbly checks its own deployment and tells you what is wrong in the words of the thing that is wrong, with the command that fixes it. A silent instance and a healthy instance look identical from the outside, so the checks exist to make the difference visible.
|
||||
Warmbly reports configuration findings, service reachability and operational measurements separately. A silent instance and a healthy instance can look identical from the outside. The panel shows missing, partial and stale coverage explicitly instead of treating silence as proof of health.
|
||||
|
||||
## Setup and health
|
||||
|
||||
The page lives in the [admin panel](/development/admin-panel/) on `:5174`, under **Instance > Setup and health**, on its **Findings** tab; the **Services** tab next to it holds the per-service probes and the platform mail card. It is operator tooling, so it is not in the customer-facing dashboard, and every route behind it is gated on a platform admin permission.
|
||||
The page lives in the [admin panel](/development/admin-panel/) on `:5174`, under **Instance > Setup and health**. **Findings** lists configuration findings, **Services** holds per-service reachability probes and the platform mail card, and **Operations** holds read-only operational measurements. It is operator tooling, not a customer dashboard, and every route behind it is gated on a platform admin permission. Tabs support direct links such as `/health?tab=operations`.
|
||||
|
||||
Errors and warnings appear in the top panel of the admin overview, and the sidebar entry carries a badge with the number of those problems. Informational notes do not count toward the badge. If there are only informational notes, the Findings tab says **No problems found** and keeps the notes below it.
|
||||
Configuration errors and warnings appear on the admin Overview, and the sidebar badge counts those findings. Informational notes do not count toward the badge. If there are no errors or warnings, Findings says **No errors or warnings reported**, not that all checks ran or the instance is healthy. The separate Overview monitoring panel shows operational signals and positive broker queue measurements.
|
||||
|
||||
| Severity | Means |
|
||||
|---|---|
|
||||
@@ -17,11 +17,11 @@ Errors and warnings appear in the top panel of the admin overview, and the sideb
|
||||
| `warning` | Works today, will hurt you. Misconfiguration, a published secret, a concentration risk |
|
||||
| `info` | No action required, stated so you are not surprised later |
|
||||
|
||||
Only findings are listed. A check that passes returns nothing, so an empty page means the instance is clean.
|
||||
Only findings are listed. A passing check returns nothing, but a skipped or unavailable check may also have no finding. The health response currently reports `execution_coverage: "unreported"`; an empty list is not a confirmed all-clear. Older responses without this additive field are treated the same way. The receipt time shown on Findings is when the browser received the response, not a source-evidence timestamp.
|
||||
|
||||
Each row carries a title, the full message including the fix, and a link to the section of these docs that explains it. Every check that ships today is instance-wide, so no row is scoped to a single mailbox, worker or campaign.
|
||||
|
||||
Every check that runs is listed below. If a check is not on this page, it does not exist.
|
||||
The configuration checks are documented below. Operational evidence has its own coverage and limits.
|
||||
|
||||
## Running the checks from a shell
|
||||
|
||||
@@ -44,12 +44,14 @@ warmblyctl status # bare binary
|
||||
|
||||
## Health endpoints
|
||||
|
||||
There are two, and they answer different questions.
|
||||
These endpoints answer different questions.
|
||||
|
||||
| Endpoint | Auth | Use it for |
|
||||
|---|---|---|
|
||||
| `GET /health` | none | Liveness. Is the process up. This is the one a load balancer or an orchestrator probes |
|
||||
| `GET /admin/instance/health` | platform admin, `view_analytics` | Deep diagnostics: the finding list the panel renders |
|
||||
| `GET /admin/system/status` | platform admin, `view_analytics` | Scoped backing-service probes at `checked_at`, not send or sync throughput |
|
||||
| `GET /admin/instance/monitoring` | platform admin, `view_analytics` | Versioned operational snapshot with per-source permission filtering and explicit measurement coverage |
|
||||
|
||||
<Callout type="warn" title="Do not point a load balancer at the deep endpoint">
|
||||
The diagnostic endpoint reports warnings about configuration, and a warning is not a reason to take a node out of rotation. Use `GET /health` for liveness and leave the diagnostics to the panel and to `make doctor`.
|
||||
@@ -63,6 +65,42 @@ Two neighbouring endpoints round out the surface, both under **Instance** in the
|
||||
| `GET /admin/instance/limits` | `view_analytics` | The effective sending, warmup and rate limits this build compiles in |
|
||||
| `GET /admin/instance/update` | `view_analytics` | The running build, the newest release, and the updater's state. Backs the version pill at the foot of the sidebar; see [Updates](/development/updates/) |
|
||||
|
||||
## Operational monitoring
|
||||
|
||||
**Operations** renders the version `"1"` snapshot from `GET /admin/instance/monitoring`. It covers registered workers, worker sample receipt, confirmed sends, campaign reservations, dispatch, safety holds, sync state and fair-use waits, stored mailbox errors, jobs, dead letters, webhooks, customer notification rows, send-result effects, domain checks, warmup mailbox loading and broker queue observations. Unsupported sources remain visible as unavailable, including detailed send-wait attribution and external tracking-broker diagnostics. Optional arrival-outbox measurements are unavailable when their schema is absent; monitoring needs no arrival migration and does not require a new worker wire version.
|
||||
|
||||
The backend collects with an eight-second overall timeout, three seconds per source and at most four concurrent collectors. It shares a one-minute snapshot across admins, then filters source permissions for each response. HTTP responses are not cacheable. On collection failure it can retain a source's last good measurement for at most five minutes, marked stale and with its original observation/evidence clocks. After that it is unavailable. Source samples are bounded; measured/expected scope counts and incomplete coverage expose truncation. Counts in partial samples may be lower bounds, never extrapolated fleet totals.
|
||||
|
||||
### Reading a snapshot
|
||||
|
||||
- **Snapshot checked** and **Collection attempted** are collection clocks. They do not confirm delivery, sync or recovery. **Source observed** and **Observed** are the measurement clocks; **Evidence since**, **Latest evidence**, their ages and any window are distinct. The UI retains the original clocks after a failed refresh and marks retained measurements stale. A successful browser fetch of old data does not make it fresh.
|
||||
- `fresh`, `stale` and `unavailable` describe measurement availability. The UI also marks observations older than two minutes or implausibly future-dated as stale/unknown. Coverage can be complete only for the measured sources; it does not mean every customer operation succeeded.
|
||||
- A nullable count is **Not measured**, not zero. A successful zero means no matching rows in that metric's scope/window. Zero recent sends means **No matching recent evidence**, not healthy throughput. A completed task, heartbeat, low memory usage or sync tick does not prove delivery or recovery.
|
||||
- Recent/persistent failure evidence, recovery unverified, safety holds, backlog and stale telemetry remain separate conditions. An old failure without later confirmed send evidence is **Recovery unverified**, not proof that a new failure is happening now. Unknown warning observations such as missing assignment remain unknown, not fabricated provider outages.
|
||||
- Schedule waits, retry waits, budget waits, daily-limit waits and in-flight work are informational. **Next eligible** describes eligibility, not a promise of success. A safety hold is a restriction, not an outage; monitoring never clears it.
|
||||
- Positive broker lag/pending/ack-pending/redelivered counts appear prominently on Overview even though the backend deliberately labels them `unknown` with severity `info`. Kafka committed lag excludes uncommitted processing; JetStream pending and acknowledgement-pending are different measurements. Neither is provider-send throughput. Redelivered is not a lifetime retry rate; absent group members can occur during rebalancing.
|
||||
- Counts, affected mailboxes, affected organizations and rows with unknown organization are separate quantities. Metrics and per-worker scopes may overlap. The attention total counts recorded signals, not failed sends or unique customers; do not add metric counts together.
|
||||
- Query errors, permission denials, unsupported capability, missing dependencies/schema and timeouts do not produce an all-clear. No sources or no metrics means unknown coverage. Raw provider/connection errors, credentials, payloads and mailbox addresses are not rendered by the monitoring panel.
|
||||
|
||||
The Overview and Operations tab share the same frontend query cache, scoped to admin identity and permission mask. Monitoring polls at most once per minute while mounted in a visible tab, does not refetch on focus or retry automatically, and offers a refresh control that respects `refresh_after`. Service probes poll every 15 seconds while their tab is mounted; passed probes establish reachability at that time only. Failed or stale probes remain explicit, including whether a cached probe previously passed or failed, and raw component connection errors are withheld. The existing platform mail card is shown only with `manage_settings`, which its inspection and send-test endpoints require.
|
||||
|
||||
This is observation-only tooling. It does not subscribe to brokers, create streams/groups, reconnect mailboxes, move assignments or offsets, reclaim tasks, replay results, clear holds, notify customers or deploy changes. Investigation links open existing admin pages when those pages cover the measured population; actions there retain their existing confirmations and permissions. Aggregate dispatch and send measurements have no direct link because the existing campaign-only detail tabs exclude some counted pending or warmup tasks. Keep the count and its evidence time in view while investigating through authorized backend diagnostics.
|
||||
|
||||
### Monitoring permissions
|
||||
|
||||
Every monitoring request requires an MFA-verified platform admin session and `view_analytics`, not a workspace API key. Source sections additionally require the bit below. Denied sources retain only an unavailable placeholder with `permission_denied`; their counts, scope measurements and private metrics are absent. The frontend also filters sections and checks destination permissions for every investigation link. Monitoring fails closed when an older identity response has no known admin mask.
|
||||
|
||||
| Source IDs | Additional platform admin bit |
|
||||
|---|---|
|
||||
| `send_safety`, `sync`, `mailbox_errors`, `arrivals`, `domain_auth` | `view_users` |
|
||||
| `sends`, `campaign_reservations`, `dispatch`, `dead_letters`, `result_effects`, `warmup_loading`, `send_wait_attribution` | `view_campaigns` |
|
||||
| `workers`, `worker_samples`, `worker_command_broker` | `view_workers` |
|
||||
| `jobs`, `backend_event_broker`, `tracking_broker` | `view_analytics` (already required) |
|
||||
| `webhooks` | `view_organizations` |
|
||||
| `notifications` | `manage_settings` |
|
||||
|
||||
For example, an admin may see webhook measurements with `view_organizations` but not a link into Sends without `view_campaigns`. Worker details, sync/mailboxes, organization details, jobs and configuration links likewise require their native page bit. Arbitrary/external investigation URLs are not accepted.
|
||||
|
||||
## Security and secrets
|
||||
|
||||
### secret_published_default
|
||||
|
||||
@@ -35,6 +35,8 @@ Indicators are workspace-scoped. Teammates see only your name, avatar, current p
|
||||
|
||||
**Events are permission-aware**: a member without inbox access never receives unibox events, and billing events reach only those who can manage billing. See [Team roles](/guides/team-roles/).
|
||||
|
||||
A member restricted to [selected resources](/guides/team-roles/#access-scope) receives only events about what they were granted: new mail in their mailboxes, and activity on their campaigns. They are not part of presence or live cursors, in either direction: they do not appear online, and they do not see who else is or what teammates are viewing.
|
||||
|
||||
## Connection status
|
||||
|
||||
A dot in the header reports the live connection: green **Live** when it is healthy, amber **Slow connection** or red **Poor connection** when heartbeats are taking longer than they should, amber **Reconnecting...** while it retries, and red **Disconnected** when it is down. The reading comes from the measured round trip to the realtime service, so amber means your connection genuinely is slow rather than merely busy.
|
||||
|
||||
@@ -7,6 +7,8 @@ Browsing filters, searches and view choices are [remembered in your browser tab]
|
||||
|
||||
A mailbox is a sender account you connect to Warmbly. Every warmup message and campaign email goes out through one you own.
|
||||
|
||||
The Accounts list checks mailbox health and, on linked self-hosted instances, Cloud warmup standing separately from the mailbox inventory. While those checks are loading or refreshing, it shows **Checking…** rather than treating missing results as Idle, Healthy, or zero sent. A failed or incomplete check shows **Unavailable**, not a measured zero or a claim that the mailbox recovered. If the mailbox list itself cannot be loaded, or only part of it loaded, the summary counts show **Unavailable** too, rather than an empty account list. A previously reported issue may remain marked **Last observed** for up to five minutes during a refresh; this is historical evidence, not a claim that it is still failing. A newly measured healthy result replaces it. A displayed health score is an account-health assessment, not proof that the provider confirmed a send; check send records and the mailbox's error details when investigating a stopped mailbox.
|
||||
|
||||
### Diagnostic participation and stopping
|
||||
|
||||
New mailboxes default to diagnostic participation **off**. The organization-authorized mailbox PATCH exposes `test_mode` (`off`, `diagnostic`, `legacy`), separate `test_send_enabled` and `test_receive_enabled` flags, and optional positive `shared_daily_limit`/`rolling_recipient_limit`. These are API controls, not new dashboard switches. Diagnostic mode sends disclosed tests but does not manufacture reads, stars, importance or spam rescue. Existing upgraded mailboxes with NULL participation retain legacy behavior until explicitly changed. Set `test_mode=off` to stop future diagnostic send/receive admission; current protocol-2 workers also recheck queued sends/actions. Already accepted old-worker commands or a provider send underway cannot be recalled. A local Cloud alias unlink stops local authority, not an independently cloud-owned mailbox or provider grant: pause/remove that mailbox on Cloud separately. See [send safety, repair evidence and rollout](/development/send-safety/).
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
---
|
||||
title: "Team & roles"
|
||||
description: "Invite members and control access with roles and permissions."
|
||||
description: "Invite members and control access with roles, permissions, and resource scopes."
|
||||
---
|
||||
|
||||
Share a workspace with your team: invite by email, give each person a role, and the role decides what they can see and change. The roster and invitations live under **Settings > Members**; roles and the permission matrix under **Settings > Roles & access**.
|
||||
Share a workspace with your team: invite by email, give each person a role, and the role decides what they can see and change. An optional access scope narrows which campaigns and mailboxes the role applies to, so a client or contractor can work inside your workspace without seeing the rest of it. The roster and invitations live under **Settings > Members**; roles and the permission matrix under **Settings > Roles & access**.
|
||||
|
||||
<Callout type="info" title="Who can manage the team">
|
||||
Inviting, re-roling, removing, and managing roles all need the **Manage team** permission (the owner and the seeded Admin role have it; any custom role can carry it). Without it the roster is read-only.
|
||||
@@ -11,13 +11,13 @@ Inviting, re-roling, removing, and managing roles all need the **Manage team** p
|
||||
|
||||
## Inviting members
|
||||
|
||||
In the **Invite teammates** box, paste or type addresses into **Emails**, separated by comma, semicolon, space, or Enter. Each becomes a chip: valid ones grey, malformed ones red so you can remove them first. Pick one or more **Roles** for the batch (a description on the right says what you are granting), then **Send invitations**.
|
||||
In the **Invite teammates** box, paste or type addresses into **Emails**, separated by comma, semicolon, space, or Enter. Each becomes a chip: valid ones grey, malformed ones red so you can remove them first. Pick one or more **Roles** for the batch (a description on the right says what you are granting), optionally narrow the batch's **Access** (see [access scope](#access-scope)), then **Send invitations**. A restricted invitee is restricted from the moment they join; there is no window in which they see the whole workspace.
|
||||
|
||||
Warmbly sends one invitation per address and reports results as "Invited 2 · 1 failed". Invalid addresses are skipped and called out. The chips clear when the batch finishes.
|
||||
|
||||
When several trusted dashboards share one backend, invitation emails link back to the dashboard that sent the invitation. Copying an invitation link also keeps that dashboard's hostname. See [multi-dashboard configuration](/development/configuration/#google-sign-in-per-dashboard-deployment).
|
||||
|
||||
**Pending invitations** shows the email, role, and expiry for each unaccepted invite, with two actions on the right of the row: copy the invite link (to share directly instead of waiting on email) and cancel the invitation. There is no resend button. To give somebody a fresh link, invite the same address again: that replaces the invitation in place with a new token and a reset expiry, and the previously emailed and copied links stop working. Copying the link again also issues a new link, so only the most recently copied link works; the emailed link is unaffected.
|
||||
**Pending invitations** shows the email, role, access, and expiry for each unaccepted invite, with two actions on the right of the row: copy the invite link (to share directly instead of waiting on email) and cancel the invitation. There is no resend button. To give somebody a fresh link, invite the same address again: that replaces the invitation in place with a new token and a reset expiry, and the previously emailed and copied links stop working. Copying the link again also issues a new link, so only the most recently copied link works; the emailed link is unaffected.
|
||||
|
||||
<Callout type="info" title="Roles are not permanent">
|
||||
The invite-time role can be changed inline from the roster later, so it is fine to start people narrow and widen access afterward.
|
||||
@@ -35,12 +35,44 @@ Both are covered in [accounts and access](/development/accounts-and-access/#with
|
||||
|
||||
## The roster
|
||||
|
||||
Each row shows email, role, and join date, with your own row tagged "you". Open the role picker on a row to tick the roles that member holds; a member can hold several, and their effective access is the union. The owner's row is a fixed Owner badge, and your own row is not editable.
|
||||
Each row shows email, role, access, and join date, with your own row tagged "you". Open the role picker on a row to tick the roles that member holds; a member can hold several, and their effective access is the union. The owner's row is a fixed Owner badge, and your own row is not editable.
|
||||
|
||||
To remove someone, hover (or tap) their row and click the remove icon. **You cannot remove the owner or yourself.** To leave a workspace you own, transfer ownership first.
|
||||
|
||||
Removal takes effect at once, including for someone who is signed in right now. Their open dashboard loses the workspace on its next request and its live updates stop, and they are asked to pick another workspace they belong to. Apps they authorized in the workspace are disconnected. Nothing they created is removed with them: mailboxes, contacts, campaigns and API keys belong to the workspace, so revoke their keys under **API keys** if those should go too. A changed role or a new owner reaches open dashboards' live updates without a reload.
|
||||
|
||||
## Access scope
|
||||
|
||||
A role says what a member may do. Their access scope says which resources it applies to:
|
||||
|
||||
- **Entire workspace**, the default and what every member had before scopes existed.
|
||||
- **Selected resources**, which reaches only the campaign folders, campaigns and mailboxes you choose.
|
||||
|
||||
Click the **Access** cell on a member's row to change it. In the dialog, choose **Selected resources**, then pick from three lists:
|
||||
|
||||
| Grant | Gives access to |
|
||||
| --- | --- |
|
||||
| **Campaign folders** | Every campaign the folder holds at the time of each request. Moving a campaign into the folder grants it, moving it out takes it away. |
|
||||
| **Campaigns** | One campaign, for workspaces that do not organize by folder. |
|
||||
| **Mailboxes** | The conversations in that mailbox in the unified inbox. |
|
||||
|
||||
A campaign is reachable when it sits in at least one granted folder or is granted on its own. Campaign access never includes a mailbox, even the ones the campaign sends from: inbox access is always granted mailbox by mailbox. Under the mailbox picker, **Senders of the selected campaigns** lists the mailboxes the chosen campaigns and folders send from, to review and add with one click. A restricted member with nothing selected reaches nothing.
|
||||
|
||||
<Callout type="warn" title="Restricted access is read-only">
|
||||
A restricted member's roles still apply, but only their read permissions do: **View campaigns**, **View analytics** and **Use unified inbox** (reading only). Everything else is set aside while the scope is restricted, so they cannot edit, start, send, reply, label, move, or mark anything read or unread. Opening a conversation does not mark it read for the rest of the team.
|
||||
</Callout>
|
||||
|
||||
What a restricted member sees:
|
||||
|
||||
- **Campaigns**: the list, folder counts, each granted campaign's settings, steps and activity feed, and its analytics. Any other campaign answers as not found, including by direct link.
|
||||
- **Analytics**: the dashboard and campaign reports, computed only over their campaigns and mailboxes.
|
||||
- **Inbox**: the conversations of their mailboxes, with counts and filters to match.
|
||||
- **Live updates** for those resources only. They do not appear in the team's presence and do not see who else is online.
|
||||
|
||||
Everything that cannot yet be narrowed to a scope is unavailable to them, and shows as locked in their sidebar with the reason: contacts, CRM, templates, forms, automations, deliverability, placement tests, settings other than their own profile and security, the AI assistant, and the Slack app. They are not sent workspace notifications. An API key a restricted member already holds, and an app they authorized, are held to the same scope and the same read-only routes; they cannot create new keys.
|
||||
|
||||
Changing a member's access needs **Manage team** and a recent sign-in, takes effect on their next request, and applies to their open dashboard and live updates without a reload. You must hold every permission the member's roles grant, the owner always reaches the entire workspace, and you cannot change your own access. Grants on a deleted folder, campaign or mailbox disappear with it.
|
||||
|
||||
## Roles
|
||||
|
||||
Roles are workspace data. Every workspace starts with three seeded roles that are ordinary roles in every respect: rename, recolor, re-permission, or delete them, and add your own. **Owner is a membership status, not a role**, and there is exactly one per workspace.
|
||||
@@ -87,7 +119,7 @@ Five rules keep this safe:
|
||||
- **Members can hold several roles**, and their effective access is the union of all of them. A member left with no roles keeps membership but has no permissions.
|
||||
- Up to 25 roles per workspace, including the seeded ones.
|
||||
|
||||
Custom roles apply everywhere permissions do: API access checks, dashboard visibility, and which realtime events a member receives.
|
||||
Custom roles apply everywhere permissions do: API access checks, dashboard visibility, and which realtime events a member receives. For a member with a restricted [access scope](#access-scope), only a role's read permissions take effect.
|
||||
|
||||
## Related guides
|
||||
|
||||
|
||||
@@ -19,9 +19,9 @@ The data is split into groups. Every export includes **Workspace**; the rest are
|
||||
|
||||
| Group | Contents |
|
||||
|-------|----------|
|
||||
| Workspace | The organization, members, roles, teams, mailboxes and their profile photos, mailbox tags, the column mappings saved by [mailbox imports](/guides/mailbox-import/), inbox vendor connections, [root redirects](/guides/sending-domains/#root-redirects), API keys, webhooks, and settings, including the website tracking site key. Always included |
|
||||
| Workspace | The organization, members with their [access scope](/guides/team-roles/#access-scope) and mailbox grants, roles, teams, mailboxes and their profile photos, mailbox tags, the column mappings saved by [mailbox imports](/guides/mailbox-import/), inbox vendor connections, [root redirects](/guides/sending-domains/#root-redirects), API keys, webhooks, and settings, including the website tracking site key. Always included |
|
||||
| Contacts | Contacts, labels, the column mappings saved by [contact imports](/guides/contacts-crm/#importing), segments with their manual overrides, forms with their images, submissions, personalized link tickets and funnel events, notes, activities, the suppression list, and each contact's [HubSpot](/guides/hubspot/) record id, owner, lifecycle stage, lead status and company |
|
||||
| Campaigns | Campaigns, folders, sequences, senders, linked segments, attachments, the email image library, per-campaign settings, each lead's step progress with its per-link clicks and per-event opens, the colleagues [copied on each lead](/guides/campaigns/#copying-colleagues-on-one-lead), [placement monitors](/guides/placement-tests/#campaign-placement-monitors), and the [unsubscribe links](/guides/unsubscribe/) already in recipients' inboxes |
|
||||
| Campaigns | Campaigns, folders, restricted members' campaign and folder grants, sequences, senders, linked segments, attachments, the email image library, per-campaign settings, each lead's step progress with its per-link clicks and per-event opens, the colleagues [copied on each lead](/guides/campaigns/#copying-colleagues-on-one-lead), [placement monitors](/guides/placement-tests/#campaign-placement-monitors), and the [unsubscribe links](/guides/unsubscribe/) already in recipients' inboxes |
|
||||
| CRM | Pipelines, deals, tasks, and meeting bookings |
|
||||
| Automations | Automations, connected integrations, lead sync sources, and the workspace's CRM mode: whether it runs on [HubSpot](/guides/hubspot/), its setup choices, the owner matches, and which Warmbly record is which HubSpot record |
|
||||
| Assistant | Remie sessions and messages (including conversations held in [Slack](/guides/slack/)), skills, MCP servers, and AI settings |
|
||||
@@ -39,6 +39,8 @@ The pending warmup-filing queue stays on its source instance. The destination re
|
||||
|
||||
[Test workspace samples](/development/accounts-and-access/#adding-sample-data) move as ordinary contacts, campaigns and CRM records in their corresponding groups; reply templates are in **Assistant**. Their instance-wide admin audit marker does not travel, and import does not grant managed Test access.
|
||||
|
||||
A restricted member stays restricted on the destination. Their campaign and folder grants travel with **Campaigns**, so an import without that group brings them in with no campaign access rather than more.
|
||||
|
||||
<Callout type="info" title="Suppression always travels with contacts">
|
||||
The suppression list is part of the Contacts group and is never optional within it. An import that dropped it would start mailing people who already opted out.
|
||||
</Callout>
|
||||
|
||||
@@ -86,12 +86,11 @@ func (h *Handler) callerMember(c *gin.Context) (*models.OrganizationMember, *err
|
||||
return h.OrganizationService.GetMembership(c.Request.Context(), *orgID, userID)
|
||||
}
|
||||
|
||||
// mailboxAllowed enforces an API key's mailbox allow-list on an account id
|
||||
// taken from a request body, the way RequireAPIKeyEmailAccountParam does for
|
||||
// a route parameter.
|
||||
// mailboxAllowed enforces the caller's mailbox allowlist on an account id from
|
||||
// a request body, the way RequireEmailAccountParam does for a route parameter.
|
||||
func mailboxAllowed(c *gin.Context, accountID uuid.UUID) *errx.Error {
|
||||
if !middleware.APIKeyAllowsEmailAccount(c, accountID) {
|
||||
return errx.New(errx.Forbidden, "email account is not allowed for this API key")
|
||||
if !middleware.EmailAccountAllowed(c, accountID) {
|
||||
return middleware.ErrEmailAccountNotAllowed(c)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -99,7 +98,7 @@ func mailboxAllowed(c *gin.Context, accountID uuid.UUID) *errx.Error {
|
||||
// apiKeyMailboxLimited is the stable code for a mailbox-limited key refused on a workspace-wide surface.
|
||||
const apiKeyMailboxLimited = "api_key_mailbox_limited"
|
||||
|
||||
// keyMailboxLimited reports whether the caller is a key held to an allowlist of mailboxes.
|
||||
// keyMailboxLimited reports whether the caller is held to an allowlist of mailboxes.
|
||||
func keyMailboxLimited(c *gin.Context) bool {
|
||||
return middleware.GetAuthType(c) != "jwt" && len(middleware.GetAPIKeyAllowedEmailAccounts(c)) > 0
|
||||
return middleware.AllowedEmailAccounts(c) != nil
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"github.com/warmbly/warmbly/internal/app/instanceconfig"
|
||||
"github.com/warmbly/warmbly/internal/app/instancesettings"
|
||||
"github.com/warmbly/warmbly/internal/errx"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
"github.com/warmbly/warmbly/internal/observability/errs"
|
||||
)
|
||||
|
||||
@@ -53,7 +54,17 @@ func (h *Handler) AdminInstanceHealth(c *gin.Context) {
|
||||
Forwarded: c.GetHeader("X-Forwarded-For") != "",
|
||||
})
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"checks": checks, "summary": summary})
|
||||
c.JSON(http.StatusOK, gin.H{"checks": checks, "summary": summary, "execution_coverage": "unreported", "operational_monitoring_path": "/admin/instance/monitoring"})
|
||||
}
|
||||
|
||||
func (h *Handler) AdminInstanceMonitoring(c *gin.Context) {
|
||||
permissions := middleware.GetAdminPermissions(c)
|
||||
if !permissions.HasPermission(models.AdminPermViewAnalytics) {
|
||||
errx.JSON(c, errx.ErrForbidden)
|
||||
return
|
||||
}
|
||||
c.Header("Cache-Control", "no-store")
|
||||
c.JSON(http.StatusOK, h.Monitoring.Snapshot(c.Request.Context(), permissions))
|
||||
}
|
||||
|
||||
func (h *Handler) AdminDeleteExpiredInvitations(c *gin.Context) {
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/warmbly/warmbly/internal/api/middleware"
|
||||
"github.com/warmbly/warmbly/internal/app/monitoring"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
)
|
||||
|
||||
func TestMonitoringAdminHandlerPermissionsAndWireCompatibility(t *testing.T) {
|
||||
n := int64(42)
|
||||
h := &Handler{Monitoring: monitoring.New([]monitoring.Source{{ID: "mailboxes", Permission: models.AdminPermViewUsers, Collect: func(context.Context, time.Time) (models.MonitoringSource, error) {
|
||||
return models.MonitoringSource{Metrics: []models.MonitoringMetric{{ID: "private", Count: &n, Availability: models.MonitoringFresh}}}, nil
|
||||
}}})}
|
||||
for _, permissions := range []models.AdminPermission{0, models.AdminPermViewUsers, models.AdminPermViewAnalytics, models.AdminPermViewAnalytics | models.AdminPermViewUsers} {
|
||||
t.Run(fmt.Sprint(permissions), func(t *testing.T) {
|
||||
w := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(w)
|
||||
c.Request = httptest.NewRequest(http.MethodGet, "/admin/instance/monitoring", nil)
|
||||
c.Set(middleware.AdminPermissionsKey, permissions)
|
||||
h.AdminInstanceMonitoring(c)
|
||||
if !permissions.HasPermission(models.AdminPermViewAnalytics) {
|
||||
if w.Code != http.StatusForbidden {
|
||||
t.Fatal(w.Code)
|
||||
}
|
||||
return
|
||||
}
|
||||
var snapshot models.MonitoringSnapshot
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &snapshot); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if w.Code != http.StatusOK || snapshot.Version != "1" || len(snapshot.Sources) != 1 || w.Header().Get("Cache-Control") != "no-store" {
|
||||
t.Fatal(w.Body.String())
|
||||
}
|
||||
source := snapshot.Sources[0]
|
||||
if !permissions.HasPermission(models.AdminPermViewUsers) {
|
||||
if len(source.Metrics) != 0 || source.Reason != "permission_denied" || source.Availability != models.MonitoringUnavailable || source.Coverage != "unavailable" || source.ObservedAt != nil || source.MeasuredScopes != nil || source.ExpectedScopes != nil {
|
||||
t.Fatal("permission leak", w.Body.String())
|
||||
}
|
||||
} else if len(source.Metrics) != 1 || source.Metrics[0].Count == nil || *source.Metrics[0].Count != n {
|
||||
t.Fatal("authorized measurement missing", w.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
h.Monitoring = nil
|
||||
w := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(w)
|
||||
c.Request = httptest.NewRequest(http.MethodGet, "/admin/instance/monitoring", nil)
|
||||
c.Set(middleware.AdminPermissionsKey, models.AdminPermViewAnalytics)
|
||||
h.AdminInstanceMonitoring(c)
|
||||
var snapshot models.MonitoringSnapshot
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &snapshot); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if w.Code != http.StatusOK || snapshot.Version != "1" || snapshot.Coverage != "unavailable" || len(snapshot.Sources) != 0 {
|
||||
t.Fatal(w.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -142,7 +142,7 @@ func (h *Handler) advisorFixInvocation(c *gin.Context, id uuid.UUID) (aitools.In
|
||||
if xerr != nil {
|
||||
return inv, xerr
|
||||
}
|
||||
if len(middleware.GetAPIKeyAllowedEmailAccounts(c)) == 0 {
|
||||
if len(middleware.AllowedEmailAccounts(c)) == 0 {
|
||||
return inv, nil
|
||||
}
|
||||
f, xerr := h.AdvisorService.Get(c.Request.Context(), inv.OrgID, id)
|
||||
|
||||
@@ -43,7 +43,7 @@ func (h *Handler) ListAgentDrafts(c *gin.Context) {
|
||||
if len(restrictedMailboxes(c)) > 0 {
|
||||
kept := drafts[:0]
|
||||
for _, d := range drafts {
|
||||
if middleware.APIKeyAllowsEmailAccount(c, d.EmailAccountID) {
|
||||
if middleware.EmailAccountAllowed(c, d.EmailAccountID) {
|
||||
kept = append(kept, d)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -377,16 +377,35 @@ func (h *Handler) GetDashboardAnalytics(c *gin.Context) {
|
||||
errx.Handle(c, xerr)
|
||||
return
|
||||
}
|
||||
restricted := middleware.IsScopeRestricted(c)
|
||||
if restricted {
|
||||
// The filter's echo names what it asked for, so it may only ask for granted ones.
|
||||
if !idsWithin(filter.CampaignIDs, middleware.AllowedCampaigns(c)) || !idsWithin(filter.FolderIDs, middleware.AllowedFolders(c)) {
|
||||
errx.Handle(c, errx.New(errx.NotFound, "campaign or folder not found"))
|
||||
return
|
||||
}
|
||||
filter.AllowedCampaigns = middleware.AllowedCampaigns(c)
|
||||
filter.AllowedMailboxes = middleware.AllowedEmailAccounts(c)
|
||||
}
|
||||
|
||||
analytics, xerr := h.AnalyticsService.GetDashboardAnalytics(c.Request.Context(), *orgID, period, filter)
|
||||
if xerr != nil {
|
||||
errx.Handle(c, xerr)
|
||||
return
|
||||
}
|
||||
if restricted {
|
||||
// A sender outside the member's mailboxes is not named to them.
|
||||
for i := range analytics.RecentActivity {
|
||||
if a := &analytics.RecentActivity[i]; a.SenderID != nil && !middleware.EmailAccountAllowed(c, *a.SenderID) {
|
||||
a.SenderID, a.SenderEmail = nil, ""
|
||||
}
|
||||
}
|
||||
}
|
||||
// The sidebar meter's denominator: the mailboxes' day under the
|
||||
// scheduler's own clamps, not their caps added up. Best effort; the
|
||||
// dashboard still renders without it.
|
||||
if h.CampaignService != nil {
|
||||
// dashboard still renders without it. It is the workspace's, so a
|
||||
// restricted member is not given it.
|
||||
if h.CampaignService != nil && !restricted {
|
||||
if capacity, cerr := h.CampaignService.WorkspaceCapacity(c.Request.Context(), *orgID); cerr == nil {
|
||||
analytics.CapacityToday = capacity
|
||||
}
|
||||
@@ -461,6 +480,10 @@ func (h *Handler) CompareCampaigns(c *gin.Context) {
|
||||
errx.Handle(c, errx.New(errx.BadRequest, "at least one valid campaign ID is required"))
|
||||
return
|
||||
}
|
||||
if !idsWithin(campaignIDs, middleware.AllowedCampaigns(c)) {
|
||||
errx.Handle(c, errx.New(errx.NotFound, "campaign not found"))
|
||||
return
|
||||
}
|
||||
|
||||
// Limit to 10 campaigns
|
||||
if len(campaignIDs) > 10 {
|
||||
@@ -521,6 +544,26 @@ func uuidListQuery(c *gin.Context, key string, max int) ([]uuid.UUID, *errx.Erro
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
// idsWithin reports whether every id is on allowed; a nil allowed list allows everything.
|
||||
func idsWithin(ids, allowed []uuid.UUID) bool {
|
||||
if allowed == nil {
|
||||
return true
|
||||
}
|
||||
for _, id := range ids {
|
||||
ok := false
|
||||
for _, a := range allowed {
|
||||
if a == id && a != uuid.Nil {
|
||||
ok = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// splitAndTrim splits a comma-separated string and trims whitespace
|
||||
func splitAndTrim(s string) []string {
|
||||
if s == "" {
|
||||
|
||||
@@ -185,11 +185,35 @@ func (h *Handler) GetUser(c *gin.Context) {
|
||||
if cats, cerr := h.CategoryService.List(ctx, *orgID); cerr == nil {
|
||||
u.Categories = cats
|
||||
}
|
||||
// A restricted member sees only the folders and mailbox tags around their grants.
|
||||
if m := middleware.GetAuthMember(c); m.IsRestricted() {
|
||||
scope, xerr := h.OrganizationService.ResolveMemberScope(ctx, *orgID, m.UserID)
|
||||
if xerr != nil {
|
||||
errx.Handle(c, xerr)
|
||||
return
|
||||
}
|
||||
u.Folders = keepGroups(u.Folders, scope.Folders)
|
||||
u.Tags = keepGroups(u.Tags, scope.MailboxTags)
|
||||
}
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, u)
|
||||
}
|
||||
|
||||
// keepGroups is the labels whose ids are in keep.
|
||||
func keepGroups(groups []models.Group, keep []uuid.UUID) []models.Group {
|
||||
out := make([]models.Group, 0, len(keep))
|
||||
for _, g := range groups {
|
||||
for _, id := range keep {
|
||||
if g.ID == id {
|
||||
out = append(out, g)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (h *Handler) ResetPasswordStart(c *gin.Context) {
|
||||
var data auth.ResetPasswordStart
|
||||
|
||||
|
||||
@@ -220,7 +220,7 @@ func (h *Handler) SearchCampaigns(c *gin.Context) {
|
||||
status := c.Query("status")
|
||||
limit := c.Query("limit")
|
||||
|
||||
resp, err := h.CampaignService.Search(c.Request.Context(), orgID.String(), query, cursor, folder, status, limit)
|
||||
resp, err := h.CampaignService.Search(c.Request.Context(), orgID.String(), query, cursor, folder, status, limit, middleware.AllowedCampaigns(c))
|
||||
if err != nil {
|
||||
errx.JSON(c, err)
|
||||
return
|
||||
@@ -264,7 +264,7 @@ func (h *Handler) GetCampaignsOverview(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
resp, err := h.CampaignService.Overview(c.Request.Context(), orgID.String())
|
||||
resp, err := h.CampaignService.Overview(c.Request.Context(), orgID.String(), middleware.AllowedCampaigns(c))
|
||||
if err != nil {
|
||||
errx.JSON(c, err)
|
||||
return
|
||||
|
||||
@@ -68,7 +68,7 @@ func (h *Handler) LookupContactByEmail(c *gin.Context) {
|
||||
errx.Handle(c, errx.ErrUuid)
|
||||
return
|
||||
}
|
||||
if !middleware.APIKeyAllowsEmailAccount(c, id) {
|
||||
if !middleware.EmailAccountAllowed(c, id) {
|
||||
errx.Handle(c, errx.New(errx.Forbidden, "email account is not allowed for this API key"))
|
||||
return
|
||||
}
|
||||
@@ -77,7 +77,7 @@ func (h *Handler) LookupContactByEmail(c *gin.Context) {
|
||||
|
||||
orgID := middleware.GetOrganizationID(c)
|
||||
|
||||
thread := models.ContactLookupThread{ID: threadID, AccountID: accountID, AllowedAccounts: middleware.GetAPIKeyAllowedEmailAccounts(c)}
|
||||
thread := models.ContactLookupThread{ID: threadID, AccountID: accountID, AllowedAccounts: middleware.AllowedEmailAccounts(c)}
|
||||
res, xerr := h.ContactService.LookupSender(c.Request.Context(), orgID, email, thread)
|
||||
if xerr != nil {
|
||||
errx.Handle(c, xerr)
|
||||
|
||||
@@ -23,7 +23,7 @@ func (h *Handler) EmailsSearch(c *gin.Context) {
|
||||
tag := c.Query("tag")
|
||||
limit := c.Query("limit")
|
||||
|
||||
resp, err := h.EmailService.Search(c.Request.Context(), orgID.String(), query, cursor, tag, limit, middleware.GetAPIKeyAllowedEmailAccounts(c))
|
||||
resp, err := h.EmailService.Search(c.Request.Context(), orgID.String(), query, cursor, tag, limit, middleware.AllowedEmailAccounts(c))
|
||||
if err != nil {
|
||||
errx.Handle(c, err)
|
||||
return
|
||||
@@ -130,9 +130,9 @@ func (h *Handler) BulkTagEmails(c *gin.Context) {
|
||||
}
|
||||
// The mailbox scope here is the workspace, so a restricted API key needs
|
||||
// the same allowlist check the per-id routes get from
|
||||
// RequireAPIKeyEmailAccountParam; there is no path param to gate on.
|
||||
// RequireEmailAccountParam; there is no path param to gate on.
|
||||
for _, id := range emailIDs {
|
||||
if !middleware.APIKeyAllowsEmailAccount(c, id) {
|
||||
if !middleware.EmailAccountAllowed(c, id) {
|
||||
errx.Handle(c, errx.New(errx.Forbidden, "email account is not allowed for this API key"))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -41,6 +41,7 @@ import (
|
||||
"github.com/warmbly/warmbly/internal/app/leadsync"
|
||||
"github.com/warmbly/warmbly/internal/app/mailboximport"
|
||||
"github.com/warmbly/warmbly/internal/app/mcp"
|
||||
"github.com/warmbly/warmbly/internal/app/monitoring"
|
||||
"github.com/warmbly/warmbly/internal/app/notification"
|
||||
"github.com/warmbly/warmbly/internal/app/oauth"
|
||||
"github.com/warmbly/warmbly/internal/app/opsnotify"
|
||||
@@ -385,6 +386,7 @@ type Handler struct {
|
||||
// Infrastructure liveness probes for the admin System Status page.
|
||||
// Wired in cmd/backend/main.go where the concrete clients live.
|
||||
SystemChecker *sysstatus.Checker
|
||||
Monitoring *monitoring.Service
|
||||
|
||||
// Admin operations pages: cross-workspace reads of mailbox sync, the send
|
||||
// outcome loop, fleet placement and abuse signals, plus the scheduled job
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"time"
|
||||
|
||||
@@ -205,7 +206,11 @@ func (h *Handler) InviteMember(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
h.auditOrg(c, models.AuditActionInvite, models.AuditEntityOrganizationMember, nil, nil, map[string]string{"email": inv.Email, "role": inv.Role})
|
||||
inviteMeta := map[string]string{"email": inv.Email, "role": inv.Role}
|
||||
if inv.Access.Restricted() {
|
||||
inviteMeta["access_scope"] = string(models.AccessScopeRestricted)
|
||||
}
|
||||
h.auditOrg(c, models.AuditActionInvite, models.AuditEntityOrganizationMember, nil, nil, inviteMeta)
|
||||
|
||||
// Get organization name for email
|
||||
org, _ := h.OrganizationService.Get(c.Request.Context(), *orgID)
|
||||
@@ -562,3 +567,106 @@ func (h *Handler) GetOrganizationLimits(c *gin.Context) {
|
||||
"storage": storage,
|
||||
})
|
||||
}
|
||||
|
||||
// GetMemberAccess is GET /organization/members/:id/access: which resources the member's role applies to.
|
||||
func (h *Handler) GetMemberAccess(c *gin.Context) {
|
||||
orgID := middleware.GetOrganizationID(c)
|
||||
if orgID == nil {
|
||||
errx.JSON(c, errx.ErrNoOrganization)
|
||||
return
|
||||
}
|
||||
memberUserID, err := uuid.Parse(c.Param("id"))
|
||||
if err != nil {
|
||||
errx.JSON(c, errx.New(errx.BadRequest, "invalid member ID"))
|
||||
return
|
||||
}
|
||||
access, xerr := h.OrganizationService.GetMemberAccess(c.Request.Context(), *orgID, memberUserID)
|
||||
if xerr != nil {
|
||||
errx.JSON(c, xerr)
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, access)
|
||||
}
|
||||
|
||||
// SetMemberAccess is PUT /organization/members/:id/access. It replaces the
|
||||
// whole scope, so a retry lands on the same state.
|
||||
func (h *Handler) SetMemberAccess(c *gin.Context) {
|
||||
orgID := middleware.GetOrganizationID(c)
|
||||
if orgID == nil {
|
||||
errx.JSON(c, errx.ErrNoOrganization)
|
||||
return
|
||||
}
|
||||
memberUserID, err := uuid.Parse(c.Param("id"))
|
||||
if err != nil {
|
||||
errx.JSON(c, errx.New(errx.BadRequest, "invalid member ID"))
|
||||
return
|
||||
}
|
||||
actorID, err := middleware.GetUserUUID(c)
|
||||
if err != nil {
|
||||
errx.JSON(c, errx.ErrUnauthorized)
|
||||
return
|
||||
}
|
||||
var req models.SetMemberAccessRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
errx.JSON(c, errx.InvalidBody(err))
|
||||
return
|
||||
}
|
||||
member, xerr := h.OrganizationService.SetMemberAccess(c.Request.Context(), *orgID, actorID, memberUserID, req.Access())
|
||||
if xerr != nil {
|
||||
errx.JSON(c, xerr)
|
||||
return
|
||||
}
|
||||
meta := map[string]string{"access_scope": string(member.AccessScope)}
|
||||
if a := member.Access; a != nil && a.Restricted() {
|
||||
meta["campaign_folders"] = strconv.Itoa(len(a.FolderIDs))
|
||||
meta["campaigns"] = strconv.Itoa(len(a.CampaignIDs))
|
||||
meta["mailboxes"] = strconv.Itoa(len(a.EmailAccountIDs))
|
||||
}
|
||||
h.auditOrg(c, models.AuditActionUpdate, models.AuditEntityOrganizationMember, &memberUserID, nil, meta)
|
||||
c.JSON(http.StatusOK, member)
|
||||
}
|
||||
|
||||
// SuggestAccessSenders is GET /organization/access/suggested-senders: the
|
||||
// mailboxes the given campaigns and folders send from, for an administrator
|
||||
// to review before granting them. Campaign access never grants a mailbox.
|
||||
func (h *Handler) SuggestAccessSenders(c *gin.Context) {
|
||||
orgID := middleware.GetOrganizationID(c)
|
||||
if orgID == nil {
|
||||
errx.JSON(c, errx.ErrNoOrganization)
|
||||
return
|
||||
}
|
||||
campaignIDs, xerr := parseUUIDList(c.Query("campaign_ids"))
|
||||
if xerr != nil {
|
||||
errx.JSON(c, xerr)
|
||||
return
|
||||
}
|
||||
folderIDs, xerr := parseUUIDList(c.Query("campaign_folder_ids"))
|
||||
if xerr != nil {
|
||||
errx.JSON(c, xerr)
|
||||
return
|
||||
}
|
||||
if len(campaignIDs)+len(folderIDs) > 2*models.MaxAccessGrants {
|
||||
errx.JSON(c, errx.New(errx.BadRequest, "too many campaigns or folders"))
|
||||
return
|
||||
}
|
||||
senders, xerr := h.OrganizationService.SuggestCampaignSenders(c.Request.Context(), *orgID, campaignIDs, folderIDs)
|
||||
if xerr != nil {
|
||||
errx.JSON(c, xerr)
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"data": senders})
|
||||
}
|
||||
|
||||
// parseUUIDList reads a comma-separated list of ids; an invalid one is refused.
|
||||
func parseUUIDList(raw string) ([]uuid.UUID, *errx.Error) {
|
||||
parts := splitAndTrim(raw)
|
||||
out := make([]uuid.UUID, 0, len(parts))
|
||||
for _, p := range parts {
|
||||
id, err := uuid.Parse(p)
|
||||
if err != nil {
|
||||
return nil, errx.ErrUuid
|
||||
}
|
||||
out = append(out, id)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -105,7 +105,7 @@ func (h *Handler) CreatePlacementTest(c *gin.Context) {
|
||||
errx.JSON(c, errx.New(errx.BadRequest, "invalid sender_account_id"))
|
||||
return
|
||||
}
|
||||
if !middleware.APIKeyAllowsEmailAccount(c, senderID) {
|
||||
if !middleware.EmailAccountAllowed(c, senderID) {
|
||||
errx.JSON(c, errx.New(errx.Forbidden, "this API key cannot send from that mailbox"))
|
||||
return
|
||||
}
|
||||
@@ -255,7 +255,7 @@ func (h *Handler) ListPlacementSeeds(c *gin.Context) {
|
||||
}
|
||||
allowed := seeds[:0]
|
||||
for _, s := range seeds {
|
||||
if middleware.APIKeyAllowsEmailAccount(c, s.EmailAccountID) {
|
||||
if middleware.EmailAccountAllowed(c, s.EmailAccountID) {
|
||||
allowed = append(allowed, s)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -80,11 +80,9 @@ func (h *Handler) placementBatchInput(c *gin.Context) (placement.BatchInput, boo
|
||||
OnUnavailable: req.OnUnavailable,
|
||||
MaxCredits: req.MaxCredits,
|
||||
}
|
||||
// A key limited to some mailboxes only ever tests from those.
|
||||
if middleware.GetAuthType(c) == middleware.AuthTypeAPIKey {
|
||||
if allowed := middleware.GetAPIKeyAllowedEmailAccounts(c); len(allowed) > 0 {
|
||||
in.AllowedSenders = allowed
|
||||
}
|
||||
// A caller limited to some mailboxes only ever tests from those.
|
||||
if allowed := middleware.AllowedEmailAccounts(c); len(allowed) > 0 {
|
||||
in.AllowedSenders = allowed
|
||||
}
|
||||
if id, err := middleware.GetUserUUID(c); err == nil && id != uuid.Nil {
|
||||
in.UserID = &id
|
||||
|
||||
@@ -262,10 +262,12 @@ func (h *Handler) GetUniboxEmail(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
resp, xerr := h.UniboxService.GetByID(
|
||||
c.Request.Context(),
|
||||
*orgID, mid,
|
||||
)
|
||||
// A restricted member reads the inbox without changing its shared read state.
|
||||
get := h.UniboxService.GetByID
|
||||
if middleware.IsScopeRestricted(c) {
|
||||
get = h.UniboxService.Peek
|
||||
}
|
||||
resp, xerr := get(c.Request.Context(), *orgID, mid)
|
||||
if xerr != nil {
|
||||
errx.Handle(c, xerr)
|
||||
return
|
||||
@@ -328,10 +330,10 @@ func (h *Handler) GetUniboxThread(c *gin.Context) {
|
||||
cursor := c.Query("cursor")
|
||||
limit := c.Query("limit")
|
||||
|
||||
resp, xerr := h.UniboxService.GetByThread(
|
||||
resp, xerr := h.UniboxService.GetByThreadWithin(
|
||||
c.Request.Context(),
|
||||
*orgID, eid,
|
||||
threadID, limit, cursor,
|
||||
threadID, limit, cursor, restrictedMailboxes(c),
|
||||
)
|
||||
if xerr != nil {
|
||||
errx.Handle(c, xerr)
|
||||
@@ -842,7 +844,7 @@ func (h *Handler) ListUniboxScheduled(c *gin.Context) {
|
||||
}
|
||||
|
||||
if threadID := c.Query("thread_id"); threadID != "" {
|
||||
items, xerr := h.UniboxService.ListScheduledByThread(c.Request.Context(), *orgID, threadID, middleware.GetAPIKeyAllowedEmailAccounts(c))
|
||||
items, xerr := h.UniboxService.ListScheduledByThread(c.Request.Context(), *orgID, threadID, middleware.AllowedEmailAccounts(c))
|
||||
if xerr != nil {
|
||||
errx.Handle(c, xerr)
|
||||
return
|
||||
@@ -851,7 +853,7 @@ func (h *Handler) ListUniboxScheduled(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
items, xerr := h.UniboxService.ListScheduled(c.Request.Context(), *orgID, middleware.GetAPIKeyAllowedEmailAccounts(c))
|
||||
items, xerr := h.UniboxService.ListScheduled(c.Request.Context(), *orgID, middleware.AllowedEmailAccounts(c))
|
||||
if xerr != nil {
|
||||
errx.Handle(c, xerr)
|
||||
return
|
||||
@@ -878,7 +880,7 @@ func (h *Handler) CancelUniboxScheduled(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
if xerr := h.UniboxService.CancelScheduled(c.Request.Context(), *orgID, taskID, middleware.GetAPIKeyAllowedEmailAccounts(c)); xerr != nil {
|
||||
if xerr := h.UniboxService.CancelScheduled(c.Request.Context(), *orgID, taskID, middleware.AllowedEmailAccounts(c)); xerr != nil {
|
||||
errx.Handle(c, xerr)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -9,13 +9,10 @@ import (
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
)
|
||||
|
||||
// restrictedMailboxes is an API key's mailbox allowlist; nil when the caller
|
||||
// restrictedMailboxes is the caller's mailbox allowlist; nil when the caller
|
||||
// is not limited to specific mailboxes.
|
||||
func restrictedMailboxes(c *gin.Context) []uuid.UUID {
|
||||
if middleware.GetAuthType(c) != middleware.AuthTypeAPIKey {
|
||||
return nil
|
||||
}
|
||||
return middleware.GetAPIKeyAllowedEmailAccounts(c)
|
||||
return middleware.AllowedEmailAccounts(c)
|
||||
}
|
||||
|
||||
// allowedMailboxFilter narrows a mailbox filter to the caller's allowlist: no
|
||||
@@ -61,7 +58,11 @@ func keepAllowedMessages(c *gin.Context, res *models.MailSearchResult) {
|
||||
}
|
||||
kept := res.Data[:0]
|
||||
for _, m := range res.Data {
|
||||
if middleware.APIKeyAllowsEmailAccount(c, m.EmailID) {
|
||||
if middleware.EmailAccountAllowed(c, m.EmailID) {
|
||||
// Which other mailbox a message answers is not shown outside the allowlist.
|
||||
if m.AnswersMailboxID != nil && !middleware.EmailAccountAllowed(c, *m.AnswersMailboxID) {
|
||||
m.AnswersMailboxID = nil
|
||||
}
|
||||
kept = append(kept, m)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -482,56 +482,6 @@ func (h *Handler) RequireAnyAccess(apiPerm uint64, orgPerms ...models.Organizati
|
||||
}
|
||||
}
|
||||
|
||||
// RequireAPIKeyEmailAccountParam enforces an API key's optional
|
||||
// allowed_email_accounts allowlist against a route parameter. JWT callers and
|
||||
// unrestricted API keys pass through.
|
||||
func RequireAPIKeyEmailAccountParam(param string) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
if c.GetString(AuthTypeKey) != AuthTypeAPIKey {
|
||||
c.Next()
|
||||
return
|
||||
}
|
||||
|
||||
if len(GetAPIKeyAllowedEmailAccounts(c)) == 0 {
|
||||
c.Next()
|
||||
return
|
||||
}
|
||||
|
||||
accountID, err := uuid.Parse(c.Param(param))
|
||||
if err != nil {
|
||||
errx.Handle(c, errx.ErrUuid)
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
|
||||
if APIKeyAllowsEmailAccount(c, accountID) {
|
||||
c.Next()
|
||||
return
|
||||
}
|
||||
|
||||
errx.Handle(c, errx.New(errx.Forbidden, "email account is not allowed for this API key"))
|
||||
c.Abort()
|
||||
}
|
||||
}
|
||||
|
||||
// APIKeyAllowsEmailAccount reports whether the authenticating API key may act
|
||||
// on accountID. JWT callers and unrestricted keys always may.
|
||||
func APIKeyAllowsEmailAccount(c *gin.Context, accountID uuid.UUID) bool {
|
||||
if c.GetString(AuthTypeKey) != AuthTypeAPIKey {
|
||||
return true
|
||||
}
|
||||
allowed := GetAPIKeyAllowedEmailAccounts(c)
|
||||
if len(allowed) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, id := range allowed {
|
||||
if id == accountID {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// GetAuthType returns "jwt" or "api_key" (empty if unauthenticated).
|
||||
func GetAuthType(c *gin.Context) string {
|
||||
return c.GetString(AuthTypeKey)
|
||||
|
||||
@@ -15,7 +15,7 @@ func init() {
|
||||
gin.SetMode(gin.TestMode)
|
||||
}
|
||||
|
||||
func TestRequireAPIKeyEmailAccountParam(t *testing.T) {
|
||||
func TestRequireEmailAccountParam(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
allowedID := uuid.New()
|
||||
deniedID := uuid.New()
|
||||
@@ -43,7 +43,7 @@ func TestRequireAPIKeyEmailAccountParam(t *testing.T) {
|
||||
c.Set(APIKeyAllowedEmailAccountsKey, tt.allowlist)
|
||||
c.Next()
|
||||
},
|
||||
RequireAPIKeyEmailAccountParam("id"),
|
||||
RequireEmailAccountParam("id"),
|
||||
func(c *gin.Context) {
|
||||
c.Status(http.StatusOK)
|
||||
},
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
package middleware
|
||||
|
||||
import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/google/uuid"
|
||||
"github.com/warmbly/warmbly/internal/errx"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
)
|
||||
|
||||
// ResourceScopeKey holds a restricted caller's resolved *models.ResourceScope.
|
||||
const ResourceScopeKey = "resource_scope"
|
||||
|
||||
// MemberAccessRestricted is the code for a restricted member on a surface their scope does not cover.
|
||||
const MemberAccessRestricted = "member_access_restricted"
|
||||
|
||||
var errMemberAccessRestricted = errx.NewWithIdentifier(errx.Forbidden, MemberAccessRestricted,
|
||||
"Your access to this workspace is limited to selected campaigns and mailboxes, and this is outside it.")
|
||||
|
||||
// scopeAwareRoutes is every route a restricted member may call; each one filters to the member's grants.
|
||||
var scopeAwareRoutes = map[string]bool{
|
||||
// The member's own account, workspace list and socket.
|
||||
"GET /v1/me": true,
|
||||
"GET /v1/organization": true,
|
||||
"POST /v1/organization": true,
|
||||
"POST /v1/organization/switch/:id": true,
|
||||
"GET /v1/organization/current": true,
|
||||
"GET /v1/me/views/:view": true,
|
||||
"PUT /v1/me/views/:view": true,
|
||||
"DELETE /v1/me/views/:view": true,
|
||||
"GET /v1/me/danger-zone": true,
|
||||
"POST /v1/me/danger-zone/delete": true,
|
||||
"DELETE /v1/me/danger-zone/delete": true,
|
||||
"GET /v1/invitations": true,
|
||||
"POST /v1/invitations/accept": true,
|
||||
"POST /v1/getaway": true,
|
||||
"GET /v1/subscription": true,
|
||||
"GET /v1/subscription/features": true,
|
||||
"GET /v1/campaigns": true,
|
||||
"GET /v1/campaigns-overview": true,
|
||||
"GET /v1/campaigns/:id": true,
|
||||
"GET /v1/campaigns/:id/logs": true,
|
||||
"GET /v1/campaigns/:id/ab-variants": true,
|
||||
"GET /v1/timezones": true,
|
||||
"GET /v1/campaigns/:id/steps": true,
|
||||
"GET /v1/analytics/dashboard": true,
|
||||
"GET /v1/analytics/campaigns/compare": true,
|
||||
"GET /v1/analytics/campaigns/:id": true,
|
||||
"GET /v1/analytics/campaigns/:id/daily": true,
|
||||
"GET /v1/analytics/campaigns/:id/hourly": true,
|
||||
"GET /v1/analytics/accounts": true,
|
||||
"GET /v1/analytics/accounts/:id": true,
|
||||
"GET /v1/emails": true,
|
||||
"GET /v1/emails/:id": true,
|
||||
"GET /v1/unibox": true,
|
||||
"GET /v1/unibox/count": true,
|
||||
"GET /v1/unibox/overview": true,
|
||||
"GET /v1/unibox/thread": true,
|
||||
"GET /v1/unibox/thread/labels": true,
|
||||
"GET /v1/unibox/scheduled": true,
|
||||
"GET /v1/unibox/:id": true,
|
||||
}
|
||||
|
||||
// ScopeAwareRoutes lists the routes a restricted member may call, for the route-table test.
|
||||
func ScopeAwareRoutes() []string {
|
||||
out := make([]string, 0, len(scopeAwareRoutes))
|
||||
for r := range scopeAwareRoutes {
|
||||
out = append(out, r)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ResourceScopeGate holds a restricted member, and any key or app acting for one, to scope-aware routes and their grants.
|
||||
func (h *Handler) ResourceScopeGate() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
m := GetAuthMember(c)
|
||||
if !m.IsRestricted() {
|
||||
c.Next()
|
||||
return
|
||||
}
|
||||
if !scopeAwareRoutes[c.Request.Method+" "+c.FullPath()] {
|
||||
errx.JSON(c, errMemberAccessRestricted)
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
if h.OrganizationService == nil {
|
||||
errx.JSON(c, errx.InternalError())
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
scope, xerr := h.OrganizationService.ResolveMemberScope(c.Request.Context(), m.OrganizationID, m.UserID)
|
||||
if xerr != nil {
|
||||
errx.JSON(c, xerr)
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
c.Set(ResourceScopeKey, scope)
|
||||
c.Next()
|
||||
}
|
||||
}
|
||||
|
||||
// grantList is a restricted member's grants as an allowlist, which is never empty.
|
||||
func grantList(ids []uuid.UUID) []uuid.UUID {
|
||||
if len(ids) == 0 {
|
||||
return models.NoneMatch()
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
// GetResourceScope is the caller's resolved scope, or nil when they reach the whole workspace.
|
||||
func GetResourceScope(c *gin.Context) *models.ResourceScope {
|
||||
if v, ok := c.Get(ResourceScopeKey); ok {
|
||||
if s, ok := v.(*models.ResourceScope); ok {
|
||||
return s
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// IsScopeRestricted reports whether the caller acts for a restricted member.
|
||||
func IsScopeRestricted(c *gin.Context) bool {
|
||||
return GetResourceScope(c) != nil || GetAuthMember(c).IsRestricted()
|
||||
}
|
||||
|
||||
// AllowedCampaigns is the caller's campaign allowlist: nil is every campaign, and an empty grant matches none.
|
||||
func AllowedCampaigns(c *gin.Context) []uuid.UUID {
|
||||
if s := GetResourceScope(c); s != nil {
|
||||
return grantList(s.Campaigns)
|
||||
}
|
||||
if GetAuthMember(c).IsRestricted() {
|
||||
return models.NoneMatch()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CampaignAllowed reports whether the caller may read campaign id.
|
||||
func CampaignAllowed(c *gin.Context, id uuid.UUID) bool {
|
||||
allowed := AllowedCampaigns(c)
|
||||
if allowed == nil {
|
||||
return true
|
||||
}
|
||||
for _, v := range allowed {
|
||||
if v == id && v != uuid.Nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// AllowedFolders are the campaign folders the caller may see, nil for every folder.
|
||||
func AllowedFolders(c *gin.Context) []uuid.UUID {
|
||||
if s := GetResourceScope(c); s != nil {
|
||||
return grantList(s.Folders)
|
||||
}
|
||||
if GetAuthMember(c).IsRestricted() {
|
||||
return models.NoneMatch()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// AllowedEmailAccounts is the caller's mailbox allowlist (an API key's list and a restricted member's grants together); nil is every mailbox.
|
||||
func AllowedEmailAccounts(c *gin.Context) []uuid.UUID {
|
||||
var key []uuid.UUID
|
||||
if c.GetString(AuthTypeKey) == AuthTypeAPIKey {
|
||||
key = GetAPIKeyAllowedEmailAccounts(c)
|
||||
}
|
||||
var member []uuid.UUID
|
||||
if s := GetResourceScope(c); s != nil {
|
||||
member = grantList(s.Mailboxes)
|
||||
} else if GetAuthMember(c).IsRestricted() {
|
||||
member = models.NoneMatch()
|
||||
}
|
||||
switch {
|
||||
case len(key) == 0:
|
||||
return member
|
||||
case member == nil:
|
||||
return key
|
||||
}
|
||||
both := make([]uuid.UUID, 0, len(key))
|
||||
for _, id := range key {
|
||||
for _, m := range member {
|
||||
if id == m {
|
||||
both = append(both, id)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return grantList(both)
|
||||
}
|
||||
|
||||
// EmailAccountAllowed reports whether the caller may act on mailbox id.
|
||||
func EmailAccountAllowed(c *gin.Context, id uuid.UUID) bool {
|
||||
allowed := AllowedEmailAccounts(c)
|
||||
if allowed == nil {
|
||||
return true
|
||||
}
|
||||
for _, v := range allowed {
|
||||
if v == id && v != uuid.Nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ErrEmailAccountNotAllowed is the refusal for a mailbox outside the caller's allowlist.
|
||||
func ErrEmailAccountNotAllowed(c *gin.Context) *errx.Error {
|
||||
if IsScopeRestricted(c) {
|
||||
return errx.New(errx.NotFound, "email account not found")
|
||||
}
|
||||
return errx.New(errx.Forbidden, "email account is not allowed for this API key")
|
||||
}
|
||||
|
||||
// RequireEmailAccountParam enforces the caller's mailbox allowlist on a route parameter.
|
||||
func RequireEmailAccountParam(param string) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
if AllowedEmailAccounts(c) == nil {
|
||||
c.Next()
|
||||
return
|
||||
}
|
||||
accountID, err := uuid.Parse(c.Param(param))
|
||||
if err != nil {
|
||||
errx.Handle(c, errx.ErrUuid)
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
if EmailAccountAllowed(c, accountID) {
|
||||
c.Next()
|
||||
return
|
||||
}
|
||||
errx.Handle(c, ErrEmailAccountNotAllowed(c))
|
||||
c.Abort()
|
||||
}
|
||||
}
|
||||
|
||||
// RequireCampaignParam answers not found for a campaign outside a restricted caller's scope; routes without the param pass.
|
||||
func RequireCampaignParam(param string) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
if AllowedCampaigns(c) == nil || c.Param(param) == "" {
|
||||
c.Next()
|
||||
return
|
||||
}
|
||||
id, err := uuid.Parse(c.Param(param))
|
||||
if err != nil {
|
||||
errx.Handle(c, errx.ErrUuid)
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
if CampaignAllowed(c, id) {
|
||||
c.Next()
|
||||
return
|
||||
}
|
||||
errx.Handle(c, errx.New(errx.NotFound, "campaign not found"))
|
||||
c.Abort()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
package middleware
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/warmbly/warmbly/internal/app/organization"
|
||||
"github.com/warmbly/warmbly/internal/errx"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
)
|
||||
|
||||
type scopeOrgs struct {
|
||||
organization.OrganizationService
|
||||
scope *models.ResourceScope
|
||||
resolved int
|
||||
}
|
||||
|
||||
func (f *scopeOrgs) ResolveMemberScope(context.Context, uuid.UUID, uuid.UUID) (*models.ResourceScope, *errx.Error) {
|
||||
f.resolved++
|
||||
return f.scope, nil
|
||||
}
|
||||
|
||||
func restrictedMember(orgID uuid.UUID) *models.OrganizationMember {
|
||||
return &models.OrganizationMember{OrganizationID: orgID, UserID: uuid.New(), Role: "Viewer",
|
||||
Permissions: models.RestrictedPermissionMask, AccessScope: models.AccessScopeRestricted}
|
||||
}
|
||||
|
||||
// serveScoped runs one request through the gate as member, with an optional API key allowlist.
|
||||
func serveScoped(t *testing.T, h *Handler, member *models.OrganizationMember, keyList []uuid.UUID, method, route, path string, gates ...gin.HandlerFunc) (*httptest.ResponseRecorder, *gin.Context) {
|
||||
t.Helper()
|
||||
var seen *gin.Context
|
||||
r := gin.New()
|
||||
r.Use(RequestIDMiddleware())
|
||||
chain := []gin.HandlerFunc{func(c *gin.Context) {
|
||||
c.Set(SessionMemberKey, member)
|
||||
c.Set(OrganizationIDKey, member.OrganizationID)
|
||||
c.Set(UserIDKey, member.UserID.String())
|
||||
if keyList != nil {
|
||||
c.Set(AuthTypeKey, AuthTypeAPIKey)
|
||||
c.Set(APIKeyAllowedEmailAccountsKey, keyList)
|
||||
} else {
|
||||
c.Set(AuthTypeKey, AuthTypeJWT)
|
||||
}
|
||||
c.Next()
|
||||
}, h.ResourceScopeGate()}
|
||||
chain = append(chain, gates...)
|
||||
chain = append(chain, func(c *gin.Context) {
|
||||
seen = c.Copy()
|
||||
c.Status(http.StatusOK)
|
||||
})
|
||||
r.Handle(method, route, chain...)
|
||||
rec := httptest.NewRecorder()
|
||||
r.ServeHTTP(rec, httptest.NewRequestWithContext(context.Background(), method, path, nil))
|
||||
return rec, seen
|
||||
}
|
||||
|
||||
func TestResourceScopeGateRefusesRoutesTheScopeCannotFilter(t *testing.T) {
|
||||
orgID := uuid.New()
|
||||
orgs := &scopeOrgs{scope: &models.ResourceScope{}}
|
||||
h := &Handler{OrganizationService: orgs}
|
||||
member := restrictedMember(orgID)
|
||||
|
||||
rec, _ := serveScoped(t, h, member, nil, http.MethodPost, "/v1/contacts/search", "/v1/contacts/search")
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("contacts search: status %d, want 403", rec.Code)
|
||||
}
|
||||
var body struct{ Code string }
|
||||
_ = json.Unmarshal(rec.Body.Bytes(), &body)
|
||||
if body.Code != MemberAccessRestricted {
|
||||
t.Fatalf("code = %q, want %q", body.Code, MemberAccessRestricted)
|
||||
}
|
||||
// A write on a scope-aware path is still refused: the allowlist is per method.
|
||||
if rec, _ := serveScoped(t, h, member, nil, http.MethodPatch, "/v1/campaigns/:id", "/v1/campaigns/"+uuid.NewString()); rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("campaign write: status %d, want 403", rec.Code)
|
||||
}
|
||||
if rec, _ := serveScoped(t, h, member, nil, http.MethodGet, "/v1/campaigns", "/v1/campaigns"); rec.Code != http.StatusOK {
|
||||
t.Fatalf("campaign list: status %d, want 200", rec.Code)
|
||||
}
|
||||
if orgs.resolved != 1 {
|
||||
t.Fatalf("scope resolved %d times, want once for the one allowed request", orgs.resolved)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceScopeGateLeavesWorkspaceMembersAlone(t *testing.T) {
|
||||
orgs := &scopeOrgs{}
|
||||
h := &Handler{OrganizationService: orgs}
|
||||
for _, m := range []*models.OrganizationMember{
|
||||
{OrganizationID: uuid.New(), UserID: uuid.New(), Role: "Viewer", AccessScope: models.AccessScopeWorkspace},
|
||||
// The owner is never restricted, whatever a stale row says.
|
||||
{OrganizationID: uuid.New(), UserID: uuid.New(), Role: string(models.RoleOwner), AccessScope: models.AccessScopeRestricted},
|
||||
} {
|
||||
rec, seen := serveScoped(t, h, m, nil, http.MethodPost, "/v1/contacts/search", "/v1/contacts/search")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: status %d, want 200", m.Role, rec.Code)
|
||||
}
|
||||
if AllowedCampaigns(seen) != nil || AllowedEmailAccounts(seen) != nil {
|
||||
t.Fatalf("%s: workspace member got an allowlist", m.Role)
|
||||
}
|
||||
}
|
||||
if orgs.resolved != 0 {
|
||||
t.Fatalf("scope resolved for a workspace member")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestrictedMemberAllowlists(t *testing.T) {
|
||||
orgID := uuid.New()
|
||||
granted, other, keyOnly := uuid.New(), uuid.New(), uuid.New()
|
||||
campaign := uuid.New()
|
||||
|
||||
t.Run("an empty grant matches nothing", func(t *testing.T) {
|
||||
h := &Handler{OrganizationService: &scopeOrgs{scope: &models.ResourceScope{}}}
|
||||
_, c := serveScoped(t, h, restrictedMember(orgID), nil, http.MethodGet, "/v1/campaigns", "/v1/campaigns")
|
||||
for name, list := range map[string][]uuid.UUID{"campaigns": AllowedCampaigns(c), "mailboxes": AllowedEmailAccounts(c), "folders": AllowedFolders(c)} {
|
||||
if len(list) == 0 {
|
||||
t.Fatalf("%s: empty allowlist reads as unrestricted", name)
|
||||
}
|
||||
}
|
||||
if CampaignAllowed(c, uuid.Nil) || EmailAccountAllowed(c, uuid.Nil) {
|
||||
t.Fatal("the none-match sentinel was treated as a grant")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("grants are the allowlist", func(t *testing.T) {
|
||||
h := &Handler{OrganizationService: &scopeOrgs{scope: &models.ResourceScope{Campaigns: []uuid.UUID{campaign}, Mailboxes: []uuid.UUID{granted}}}}
|
||||
_, c := serveScoped(t, h, restrictedMember(orgID), nil, http.MethodGet, "/v1/campaigns", "/v1/campaigns")
|
||||
if !CampaignAllowed(c, campaign) || CampaignAllowed(c, uuid.New()) {
|
||||
t.Fatal("campaign allowlist wrong")
|
||||
}
|
||||
if !EmailAccountAllowed(c, granted) || EmailAccountAllowed(c, other) {
|
||||
t.Fatal("mailbox allowlist wrong")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an API key is held to its own list and its holder's grants", func(t *testing.T) {
|
||||
h := &Handler{OrganizationService: &scopeOrgs{scope: &models.ResourceScope{Mailboxes: []uuid.UUID{granted, other}}}}
|
||||
_, c := serveScoped(t, h, restrictedMember(orgID), []uuid.UUID{granted, keyOnly}, http.MethodGet, "/v1/emails", "/v1/emails")
|
||||
got := AllowedEmailAccounts(c)
|
||||
if len(got) != 1 || got[0] != granted {
|
||||
t.Fatalf("allowlist = %v, want only %v", got, granted)
|
||||
}
|
||||
_, c = serveScoped(t, h, restrictedMember(orgID), []uuid.UUID{keyOnly}, http.MethodGet, "/v1/emails", "/v1/emails")
|
||||
if got := AllowedEmailAccounts(c); len(got) == 0 || EmailAccountAllowed(c, keyOnly) {
|
||||
t.Fatalf("disjoint key and grants must allow nothing, got %v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRequireCampaignParamHidesOutOfScopeCampaigns(t *testing.T) {
|
||||
orgID := uuid.New()
|
||||
campaign := uuid.New()
|
||||
h := &Handler{OrganizationService: &scopeOrgs{scope: &models.ResourceScope{Campaigns: []uuid.UUID{campaign}}}}
|
||||
for id, want := range map[uuid.UUID]int{campaign: http.StatusOK, uuid.New(): http.StatusNotFound} {
|
||||
rec, _ := serveScoped(t, h, restrictedMember(orgID), nil, http.MethodGet, "/v1/campaigns/:id", "/v1/campaigns/"+id.String(), RequireCampaignParam("id"))
|
||||
if rec.Code != want {
|
||||
t.Fatalf("campaign %s: status %d, want %d", id, rec.Code, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"github.com/warmbly/warmbly/internal/api/handler"
|
||||
"github.com/warmbly/warmbly/internal/api/middleware"
|
||||
)
|
||||
|
||||
// Every route a restricted member may call has to exist, or the allowlist drifts from the router unnoticed.
|
||||
func TestScopeAwareRoutesExist(t *testing.T) {
|
||||
r := Run(&handler.Handler{}, &middleware.Handler{}, &middleware.OidcHandler{}, "", gin.TestMode, nil)
|
||||
registered := map[string]bool{}
|
||||
for _, rt := range r.Routes() {
|
||||
registered[rt.Method+" "+rt.Path] = true
|
||||
}
|
||||
if len(registered) < 100 {
|
||||
t.Fatalf("router registered only %d routes", len(registered))
|
||||
}
|
||||
for _, route := range middleware.ScopeAwareRoutes() {
|
||||
if !registered[route] {
|
||||
t.Errorf("scope-aware route %q is not registered", route)
|
||||
}
|
||||
}
|
||||
}
|
||||
+41
-34
@@ -510,7 +510,7 @@ func Run(
|
||||
// long-lived API key (billing, org governance, websocket bootstrap, and
|
||||
// the email onboarding flow that writes user-encrypted secrets).
|
||||
jwtOnly := base.Group("")
|
||||
jwtOnly.Use(m.AuthMiddleware())
|
||||
jwtOnly.Use(m.AuthMiddleware(), m.ResourceScopeGate())
|
||||
|
||||
// Warmbly MCP server: exposes the shared tool registry over the MCP
|
||||
// streamable-HTTP transport. Accepts an API key (static header) or an OAuth
|
||||
@@ -520,61 +520,63 @@ func Run(
|
||||
// RequiredAPIPerm, send-class tools are never exposed, and per-key rate
|
||||
// limits apply.
|
||||
mcpServer := base.Group("/mcp")
|
||||
mcpServer.Use(m.MCPAuthMiddleware(), m.APIKeyUsageMiddleware(), m.RateLimitMiddleware(models.RateLimitWrite))
|
||||
mcpServer.Use(m.MCPAuthMiddleware(), m.ResourceScopeGate(), m.APIKeyUsageMiddleware(), m.RateLimitMiddleware(models.RateLimitWrite))
|
||||
mcpServer.POST("", h.MCPEndpoint)
|
||||
|
||||
// API-accessible group: routes that accept either a JWT or an API key.
|
||||
// CombinedAuthMiddleware sets the same context keys for both; the usage
|
||||
// middleware records one log row per API-key request (JWT skipped).
|
||||
protected := base.Group("")
|
||||
protected.Use(m.CombinedAuthMiddleware(), m.APIKeyUsageMiddleware(), m.IdempotencyMiddleware(), h.ForgetUniboxOverviewOnWrite)
|
||||
// ResourceScopeGate holds a restricted member, and every key or app acting
|
||||
// for one, to the routes that filter to their grants.
|
||||
protected.Use(m.CombinedAuthMiddleware(), m.ResourceScopeGate(), m.APIKeyUsageMiddleware(), m.IdempotencyMiddleware(), h.ForgetUniboxOverviewOnWrite)
|
||||
{
|
||||
emails := protected.Group("/emails")
|
||||
emails.Use(m.RateLimitMiddleware(models.RateLimitWrite), middleware.UUIDParams("id"))
|
||||
{
|
||||
emails.GET("", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadEmails), h.EmailsSearch)
|
||||
emails.GET("/:id", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.GetEmail)
|
||||
emails.PATCH("/:id", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.UpdateEmail)
|
||||
emails.GET("/:id", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadEmails), middleware.RequireEmailAccountParam("id"), h.GetEmail)
|
||||
emails.PATCH("/:id", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireEmailAccountParam("id"), h.UpdateEmail)
|
||||
// Bulk tag add/remove across many mailboxes (set semantics,
|
||||
// naturally idempotent). Static path beside /:id like /verify.
|
||||
emails.PATCH("/tags", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), h.BulkTagEmails)
|
||||
// How many mailboxes the workspace holds and may hold, and why.
|
||||
emails.GET("/allowance", m.RequireOrganization(), m.RequireAccess(models.PermManageEmails, models.APIPermReadEmails), h.GetMailboxAllowance)
|
||||
emails.GET("/:id/track", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.GetEmailTrackingDomain)
|
||||
emails.PATCH("/:id/track", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.UpdateEmailTrackingDomain)
|
||||
emails.PATCH("/:id/direct-tracking", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.UpdateEmailDirectTracking)
|
||||
emails.GET("/:id/track", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadEmails), middleware.RequireEmailAccountParam("id"), h.GetEmailTrackingDomain)
|
||||
emails.PATCH("/:id/track", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireEmailAccountParam("id"), h.UpdateEmailTrackingDomain)
|
||||
emails.PATCH("/:id/direct-tracking", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireEmailAccountParam("id"), h.UpdateEmailDirectTracking)
|
||||
// Write-scoped like the auth-check refresh: persisting the
|
||||
// verdict is what routes real links through the custom host.
|
||||
emails.POST("/:id/track/verify", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.VerifyEmailTrackingDomain)
|
||||
emails.POST("/:id/warmup/start", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.StartWarmup)
|
||||
emails.POST("/:id/warmup/pause", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.PauseWarmup)
|
||||
emails.POST("/:id/warmup/resume", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.ResumeWarmup)
|
||||
emails.POST("/:id/warmup/stop", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.StopWarmup)
|
||||
emails.POST("/:id/track/verify", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireEmailAccountParam("id"), h.VerifyEmailTrackingDomain)
|
||||
emails.POST("/:id/warmup/start", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireEmailAccountParam("id"), h.StartWarmup)
|
||||
emails.POST("/:id/warmup/pause", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireEmailAccountParam("id"), h.PauseWarmup)
|
||||
emails.POST("/:id/warmup/resume", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireEmailAccountParam("id"), h.ResumeWarmup)
|
||||
emails.POST("/:id/warmup/stop", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireEmailAccountParam("id"), h.StopWarmup)
|
||||
// The owner's hold on campaign sending. Bodyless and idempotent, so no Idempotency-Key.
|
||||
emails.POST("/:id/hold", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.HoldEmail)
|
||||
emails.POST("/:id/release", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.ReleaseEmail)
|
||||
emails.GET("/:id/auth-check", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.GetEmailAuthCheck)
|
||||
emails.POST("/:id/hold", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireEmailAccountParam("id"), h.HoldEmail)
|
||||
emails.POST("/:id/release", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireEmailAccountParam("id"), h.ReleaseEmail)
|
||||
emails.GET("/:id/auth-check", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadEmails), middleware.RequireEmailAccountParam("id"), h.GetEmailAuthCheck)
|
||||
// Write-scoped: recording the verdict is what lifts the cold-send
|
||||
// and warmup gate, so a read-only key must not reach it.
|
||||
emails.POST("/:id/auth-check", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.RefreshEmailAuthCheck)
|
||||
emails.GET("/:id/sync", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.GetEmailSync)
|
||||
emails.PUT("/:id/sync", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.UpdateEmailSync)
|
||||
emails.POST("/:id/auth-check", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireEmailAccountParam("id"), h.RefreshEmailAuthCheck)
|
||||
emails.GET("/:id/sync", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadEmails), middleware.RequireEmailAccountParam("id"), h.GetEmailSync)
|
||||
emails.PUT("/:id/sync", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireEmailAccountParam("id"), h.UpdateEmailSync)
|
||||
// Which addresses the provider will let this mailbox send as,
|
||||
// and where its signature came from. The refresh is the only
|
||||
// half that calls the provider, and storing its answer is what
|
||||
// a send-as choice is validated against, so it is write-scoped.
|
||||
emails.GET("/:id/identity", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.GetEmailSendIdentity)
|
||||
emails.POST("/:id/identity/refresh", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.RefreshEmailSendIdentity)
|
||||
emails.GET("/:id/identity", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadEmails), middleware.RequireEmailAccountParam("id"), h.GetEmailSendIdentity)
|
||||
emails.POST("/:id/identity/refresh", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireEmailAccountParam("id"), h.RefreshEmailSendIdentity)
|
||||
// Human sending behaviour: the ranges the mailbox rolls its
|
||||
// workday from, and the workday it rolled for today.
|
||||
emails.GET("/:id/behavior", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.GetEmailBehavior)
|
||||
emails.PUT("/:id/behavior", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.UpdateEmailBehavior)
|
||||
emails.GET("/:id/behavior/plan", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.GetEmailBehaviorPlan)
|
||||
emails.GET("/:id/behavior", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadEmails), middleware.RequireEmailAccountParam("id"), h.GetEmailBehavior)
|
||||
emails.PUT("/:id/behavior", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireEmailAccountParam("id"), h.UpdateEmailBehavior)
|
||||
emails.GET("/:id/behavior/plan", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadEmails), middleware.RequireEmailAccountParam("id"), h.GetEmailBehaviorPlan)
|
||||
emails.POST("/verify", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadEmails), h.VerifyEmail)
|
||||
emails.GET("/:id/warmup/ban-status", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.GetWarmupBanStatus)
|
||||
emails.POST("/:id/warmup/appeal", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.SubmitWarmupAppeal)
|
||||
emails.DELETE("/:id", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.DeleteEmail)
|
||||
emails.POST("/:id/send", m.RequireOrganization(), m.RequireAccess(models.PermSendCampaigns, models.APIPermSendCampaigns), middleware.RequireAPIKeyEmailAccountParam("id"), h.SendEmailFromAccount)
|
||||
emails.GET("/:id/warmup/ban-status", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadEmails), middleware.RequireEmailAccountParam("id"), h.GetWarmupBanStatus)
|
||||
emails.POST("/:id/warmup/appeal", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireEmailAccountParam("id"), h.SubmitWarmupAppeal)
|
||||
emails.DELETE("/:id", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireEmailAccountParam("id"), h.DeleteEmail)
|
||||
emails.POST("/:id/send", m.RequireOrganization(), m.RequireAccess(models.PermSendCampaigns, models.APIPermSendCampaigns), middleware.RequireEmailAccountParam("id"), h.SendEmailFromAccount)
|
||||
}
|
||||
|
||||
// Email onboarding is JWT-only — it writes user-encrypted refresh
|
||||
@@ -709,7 +711,7 @@ func Run(
|
||||
protected.POST("/campaigns-estimate", m.RateLimitMiddleware(models.RateLimitRead), m.RequireOrganization(), m.RequireAccess(models.PermViewCampaigns, models.APIPermReadCampaigns), h.EstimateCampaign)
|
||||
|
||||
campaigns := protected.Group("/campaigns")
|
||||
campaigns.Use(m.RateLimitMiddleware(models.RateLimitWrite), middleware.UUIDParams("id"))
|
||||
campaigns.Use(m.RateLimitMiddleware(models.RateLimitWrite), middleware.UUIDParams("id"), middleware.RequireCampaignParam("id"))
|
||||
{
|
||||
campaigns.GET("", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadCampaigns), h.SearchCampaigns)
|
||||
campaigns.POST("", m.RequireAccess(models.PermManageCampaigns, models.APIPermWriteCampaigns), h.CreateCampaign)
|
||||
@@ -1070,11 +1072,11 @@ func Run(
|
||||
analytics.GET("/warmup", h.GetWarmupAnalytics)
|
||||
analytics.GET("/warmup/placement", m.RequireOrganization(), h.GetWarmupPlacement)
|
||||
analytics.GET("/campaigns/compare", h.CompareCampaigns)
|
||||
analytics.GET("/campaigns/:id", h.GetCampaignAnalytics)
|
||||
analytics.GET("/campaigns/:id/daily", h.GetCampaignDailyStats)
|
||||
analytics.GET("/campaigns/:id/hourly", h.GetCampaignHourlyStats)
|
||||
analytics.GET("/campaigns/:id", middleware.RequireCampaignParam("id"), h.GetCampaignAnalytics)
|
||||
analytics.GET("/campaigns/:id/daily", middleware.RequireCampaignParam("id"), h.GetCampaignDailyStats)
|
||||
analytics.GET("/campaigns/:id/hourly", middleware.RequireCampaignParam("id"), h.GetCampaignHourlyStats)
|
||||
analytics.GET("/accounts", h.GetAllAccountStatuses)
|
||||
analytics.GET("/accounts/:id", middleware.RequireAPIKeyEmailAccountParam("id"), h.GetAccountStatus)
|
||||
analytics.GET("/accounts/:id", middleware.RequireEmailAccountParam("id"), h.GetAccountStatus)
|
||||
analytics.GET("/usage", h.GetUsageOverview)
|
||||
}
|
||||
|
||||
@@ -1098,7 +1100,7 @@ func Run(
|
||||
placementTests.POST("/batches/:id/cancel", m.RequireAccess(models.PermSendCampaigns, models.APIPermSendCampaigns), h.CancelPlacementBatch)
|
||||
placementTests.GET("/coverage", m.RateLimitMiddleware(models.RateLimitAnalytics), m.RequireAccess(models.PermViewAnalytics, models.APIPermReadAnalytics), h.GetPlacementCoverage)
|
||||
placementTests.GET("/seeds", m.RequireAccess(models.PermViewCampaigns, models.APIPermReadEmails), h.ListPlacementSeeds)
|
||||
placementTests.PUT("/seeds/:id", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireAPIKeyEmailAccountParam("id"), h.SetPlacementSeed)
|
||||
placementTests.PUT("/seeds/:id", m.RequireAccess(models.PermManageEmails, models.APIPermWriteEmails), middleware.RequireEmailAccountParam("id"), h.SetPlacementSeed)
|
||||
}
|
||||
|
||||
// Audit logs
|
||||
@@ -1497,6 +1499,10 @@ func Run(
|
||||
org.POST("/members/invite", m.RequireOrganization(), m.RequirePermission(models.PermManageTeam), middleware.RequireFreshAuth(), h.InviteMember)
|
||||
org.PATCH("/members/:id", m.RequireOrganization(), m.RequirePermission(models.PermManageTeam), middleware.RequireFreshAuth(), h.UpdateMemberRole)
|
||||
org.DELETE("/members/:id", m.RequireOrganization(), m.RequirePermission(models.PermManageTeam), h.RemoveMember)
|
||||
// Resource scope: which campaigns, folders and mailboxes a member's role applies to.
|
||||
org.GET("/members/:id/access", m.RequireOrganization(), m.RequirePermission(models.PermManageTeam), h.GetMemberAccess)
|
||||
org.PUT("/members/:id/access", m.RequireOrganization(), m.RequirePermission(models.PermManageTeam), middleware.RequireFreshAuth(), h.SetMemberAccess)
|
||||
org.GET("/access/suggested-senders", m.RequireOrganization(), m.RequirePermission(models.PermManageTeam), h.SuggestAccessSenders)
|
||||
|
||||
// Custom roles: named permission sets assignable to members.
|
||||
org.GET("/roles", m.RequireOrganization(), h.ListOrganizationRoles)
|
||||
@@ -1929,6 +1935,7 @@ func Run(
|
||||
// only keys no environment variable owns.
|
||||
adminRoutes.GET("/instance/config", middleware.RequireAdminPermission(models.AdminPermManageSettings), h.AdminInstanceConfig)
|
||||
adminRoutes.GET("/instance/health", middleware.RequireAdminPermission(models.AdminPermViewAnalytics), h.AdminInstanceHealth)
|
||||
adminRoutes.GET("/instance/monitoring", middleware.RequireAdminPermission(models.AdminPermViewAnalytics), h.AdminInstanceMonitoring)
|
||||
adminRoutes.DELETE("/instance/invitations/expired", middleware.RequireAdminPermission(models.AdminPermManageOrganizations), h.AdminDeleteExpiredInvitations)
|
||||
adminRoutes.GET("/instance/limits", middleware.RequireAdminPermission(models.AdminPermViewAnalytics), h.AdminInstanceLimits)
|
||||
adminRoutes.GET("/instance/settings", middleware.RequireAdminPermission(models.AdminPermManageSettings), h.AdminGetInstanceSettings)
|
||||
|
||||
@@ -409,7 +409,7 @@ func (d Deps) listCampaigns(ctx context.Context, inv Invocation, args json.RawMe
|
||||
if limit <= 0 || limit > 50 {
|
||||
limit = 20
|
||||
}
|
||||
res, xerr := d.Campaigns.Search(ctx, inv.OrgID.String(), in.Query, "", "", in.Status, fmt.Sprintf("%d", limit))
|
||||
res, xerr := d.Campaigns.Search(ctx, inv.OrgID.String(), in.Query, "", "", in.Status, fmt.Sprintf("%d", limit), nil)
|
||||
if xerr != nil {
|
||||
return "", fromErrx(xerr)
|
||||
}
|
||||
|
||||
@@ -744,9 +744,12 @@ func (s *analyticsService) GetDashboardAnalytics(ctx context.Context, orgID uuid
|
||||
return nil, xerr
|
||||
}
|
||||
}
|
||||
if filter.AllowedCampaigns != nil {
|
||||
scope = withinCampaigns(scope, filter.AllowedCampaigns)
|
||||
}
|
||||
|
||||
// Get overall stats
|
||||
overallStats, xerr := s.analyticsRepo.GetDashboardOverallStats(ctx, orgID, from, to, scope)
|
||||
overallStats, xerr := s.analyticsRepo.GetDashboardOverallStats(ctx, orgID, from, to, scope, filter.AllowedMailboxes)
|
||||
if xerr != nil {
|
||||
return nil, xerr
|
||||
}
|
||||
@@ -764,7 +767,7 @@ func (s *analyticsService) GetDashboardAnalytics(ctx context.Context, orgID uuid
|
||||
}
|
||||
|
||||
// Get account health summary
|
||||
accountHealth, xerr := s.analyticsRepo.GetAccountHealthSummary(ctx, orgID)
|
||||
accountHealth, xerr := s.analyticsRepo.GetAccountHealthSummary(ctx, orgID, filter.AllowedMailboxes)
|
||||
if xerr != nil {
|
||||
accountHealth = &models.AccountHealthSummary{}
|
||||
}
|
||||
@@ -786,6 +789,24 @@ func (s *analyticsService) GetDashboardAnalytics(ctx context.Context, orgID uuid
|
||||
}, nil
|
||||
}
|
||||
|
||||
// withinCampaigns narrows scope to allowed; a nil scope becomes allowed itself.
|
||||
func withinCampaigns(scope *models.CampaignScope, allowed []uuid.UUID) *models.CampaignScope {
|
||||
if scope == nil {
|
||||
return &models.CampaignScope{CampaignIDs: append([]uuid.UUID{}, allowed...)}
|
||||
}
|
||||
keep := make(map[uuid.UUID]bool, len(allowed))
|
||||
for _, id := range allowed {
|
||||
keep[id] = true
|
||||
}
|
||||
out := &models.CampaignScope{CampaignIDs: []uuid.UUID{}}
|
||||
for _, id := range scope.CampaignIDs {
|
||||
if keep[id] {
|
||||
out.CampaignIDs = append(out.CampaignIDs, id)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *analyticsService) GetCampaignHourlyStats(ctx context.Context, orgID, campaignID uuid.UUID, date time.Time) ([]models.CampaignHourlyStats, *errx.Error) {
|
||||
// Verify campaign workspace
|
||||
_, err := s.campaignForOrg(ctx, orgID, campaignID)
|
||||
|
||||
@@ -72,7 +72,7 @@ func (s *dashboardAnalyticsRepoStub) ResolveDashboardScope(_ context.Context, _
|
||||
return &models.DashboardScope{CampaignCount: len(s.resolved.CampaignIDs)}, s.resolved, nil
|
||||
}
|
||||
|
||||
func (s *dashboardAnalyticsRepoStub) GetDashboardOverallStats(_ context.Context, _ uuid.UUID, from, to time.Time, scope *models.CampaignScope) (*models.DashboardOverallStats, *errx.Error) {
|
||||
func (s *dashboardAnalyticsRepoStub) GetDashboardOverallStats(_ context.Context, _ uuid.UUID, from, to time.Time, scope *models.CampaignScope, _ []uuid.UUID) (*models.DashboardOverallStats, *errx.Error) {
|
||||
s.overallFrom, s.overallTo = from, to
|
||||
s.record("overall", scope)
|
||||
return &models.DashboardOverallStats{}, nil
|
||||
@@ -88,7 +88,7 @@ func (s *dashboardAnalyticsRepoStub) GetTopCampaigns(_ context.Context, _ uuid.U
|
||||
return []models.TopCampaignStats{}, nil
|
||||
}
|
||||
|
||||
func (*dashboardAnalyticsRepoStub) GetAccountHealthSummary(context.Context, uuid.UUID) (*models.AccountHealthSummary, *errx.Error) {
|
||||
func (*dashboardAnalyticsRepoStub) GetAccountHealthSummary(context.Context, uuid.UUID, []uuid.UUID) (*models.AccountHealthSummary, *errx.Error) {
|
||||
return &models.AccountHealthSummary{}, nil
|
||||
}
|
||||
|
||||
@@ -229,3 +229,35 @@ func TestDashboardFilterNarrowsEveryCampaignSectionToOneSet(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Issue #861: a restricted member's dashboard reads only their granted campaigns, with or without a filter.
|
||||
func TestDashboardHoldsARestrictedMemberToTheirCampaigns(t *testing.T) {
|
||||
granted, other := uuid.New(), uuid.New()
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
resolved *models.CampaignScope
|
||||
filter models.DashboardFilter
|
||||
want []uuid.UUID
|
||||
}{
|
||||
{"no filter reads the grants", nil, models.DashboardFilter{}, []uuid.UUID{granted}},
|
||||
{"a filter is narrowed to the grants", &models.CampaignScope{CampaignIDs: []uuid.UUID{granted, other}}, models.DashboardFilter{FolderIDs: []uuid.UUID{uuid.New()}}, []uuid.UUID{granted}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
repo := &dashboardAnalyticsRepoStub{resolved: tc.resolved}
|
||||
if repo.resolved == nil {
|
||||
repo.resolved = &models.CampaignScope{}
|
||||
}
|
||||
svc := &analyticsService{analyticsRepo: repo}
|
||||
tc.filter.AllowedCampaigns = []uuid.UUID{granted}
|
||||
if _, xerr := svc.GetDashboardAnalytics(context.Background(), uuid.New(), "7d", tc.filter); xerr != nil {
|
||||
t.Fatalf("GetDashboardAnalytics: %v", xerr)
|
||||
}
|
||||
for _, section := range []string{"overall", "recent", "top", "trend"} {
|
||||
got := repo.scopes[section]
|
||||
if got == nil || len(got.CampaignIDs) != len(tc.want) || got.CampaignIDs[0] != tc.want[0] {
|
||||
t.Errorf("%s read %+v, want only the granted campaign", section, got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,7 +89,7 @@ func (s *campaignService) Get(ctx context.Context, orgID, id string) (*models.Ca
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *campaignService) Search(ctx context.Context, orgID, query, cursor, folder, status, limit string) (*models.CampaignsResult, *errx.Error) {
|
||||
func (s *campaignService) Search(ctx context.Context, orgID, query, cursor, folder, status, limit string, allowed []uuid.UUID) (*models.CampaignsResult, *errx.Error) {
|
||||
cursorId, err := paging.DecodeCursor(cursor)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -108,7 +108,7 @@ func (s *campaignService) Search(ctx context.Context, orgID, query, cursor, fold
|
||||
return nil, errx.New(errx.BadRequest, "invalid status filter: must be draft, active, paused, or completed")
|
||||
}
|
||||
|
||||
resp, xerr := s.campaignRepository.Search(ctx, orgID, query, cursorId, folderId, status, limitN)
|
||||
resp, xerr := s.campaignRepository.Search(ctx, orgID, query, cursorId, folderId, status, limitN, allowed)
|
||||
if xerr != nil {
|
||||
return nil, errx.InternalError()
|
||||
}
|
||||
@@ -116,8 +116,8 @@ func (s *campaignService) Search(ctx context.Context, orgID, query, cursor, fold
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *campaignService) Overview(ctx context.Context, orgID string) (*models.CampaignsOverview, *errx.Error) {
|
||||
resp, err := s.campaignRepository.Overview(ctx, orgID)
|
||||
func (s *campaignService) Overview(ctx context.Context, orgID string, allowed []uuid.UUID) (*models.CampaignsOverview, *errx.Error) {
|
||||
resp, err := s.campaignRepository.Overview(ctx, orgID, allowed)
|
||||
if err != nil {
|
||||
return nil, errx.InternalError()
|
||||
}
|
||||
|
||||
@@ -22,8 +22,9 @@ type CampaignService interface {
|
||||
Create(ctx context.Context, userID string, orgID *uuid.UUID, data *models.CreateCampaign) (*models.Campaign, *errx.Error)
|
||||
// Get loads one of orgID's campaigns; any other id is not found.
|
||||
Get(ctx context.Context, orgID, id string) (*models.Campaign, *errx.Error)
|
||||
Search(ctx context.Context, userID, query, cursor, folder, status, limit string) (*models.CampaignsResult, *errx.Error)
|
||||
Overview(ctx context.Context, orgID string) (*models.CampaignsOverview, *errx.Error)
|
||||
// allowed limits the results to those campaigns; nil is every campaign in the workspace.
|
||||
Search(ctx context.Context, userID, query, cursor, folder, status, limit string, allowed []uuid.UUID) (*models.CampaignsResult, *errx.Error)
|
||||
Overview(ctx context.Context, orgID string, allowed []uuid.UUID) (*models.CampaignsOverview, *errx.Error)
|
||||
// Estimate projects how many contacts a set of segments reaches and how
|
||||
// many sending days a mailbox pool needs under the per-mailbox caps.
|
||||
// Read-only; the new-campaign flow shows it as its launch plan.
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
)
|
||||
|
||||
const (
|
||||
CacheTTL = time.Minute
|
||||
RetainLastGood = 5 * time.Minute
|
||||
CollectionTimeout = 8 * time.Second
|
||||
SourceTimeout = 3 * time.Second
|
||||
MaxConcurrentSources = 4
|
||||
)
|
||||
|
||||
type Source struct {
|
||||
ID string
|
||||
Permission models.AdminPermission
|
||||
Collect func(context.Context, time.Time) (models.MonitoringSource, error)
|
||||
}
|
||||
|
||||
// Service shares sanitized, immutable snapshots and filters permissions after lookup.
|
||||
type Service struct {
|
||||
sources []Source
|
||||
mu sync.Mutex
|
||||
snapshot models.MonitoringSnapshot
|
||||
refreshing chan struct{}
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func New(sources []Source) *Service { return &Service{sources: sources, now: time.Now} }
|
||||
|
||||
func (s *Service) Snapshot(ctx context.Context, permissions models.AdminPermission) models.MonitoringSnapshot {
|
||||
if s == nil {
|
||||
return models.MonitoringSnapshot{Version: "1", Coverage: "unavailable", Sources: []models.MonitoringSource{}}
|
||||
}
|
||||
s.mu.Lock()
|
||||
if !s.snapshot.CheckedAt.IsZero() && s.now().Before(s.snapshot.RefreshAfter) {
|
||||
out := s.project(s.snapshot, permissions)
|
||||
s.mu.Unlock()
|
||||
return out
|
||||
}
|
||||
if s.refreshing == nil {
|
||||
s.refreshing = make(chan struct{})
|
||||
go s.refresh(s.refreshing)
|
||||
}
|
||||
ready := s.refreshing
|
||||
s.mu.Unlock()
|
||||
select {
|
||||
case <-ready:
|
||||
s.mu.Lock()
|
||||
out := s.project(s.snapshot, permissions)
|
||||
s.mu.Unlock()
|
||||
return out
|
||||
case <-ctx.Done():
|
||||
out := models.MonitoringSnapshot{Version: "1", CheckedAt: s.now().UTC(), Coverage: "unavailable", Sources: []models.MonitoringSource{}}
|
||||
for _, source := range s.sources {
|
||||
out.Sources = append(out.Sources, unavailable(source.ID, out.CheckedAt, "timeout"))
|
||||
}
|
||||
return s.project(out, permissions)
|
||||
}
|
||||
}
|
||||
|
||||
func unavailable(id string, at time.Time, reason string) models.MonitoringSource {
|
||||
return models.MonitoringSource{ID: id, CheckedAt: at, Availability: models.MonitoringUnavailable, Coverage: "unavailable", Reason: reason, Metrics: []models.MonitoringMetric{}}
|
||||
}
|
||||
|
||||
func (s *Service) refresh(ready chan struct{}) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), CollectionTimeout)
|
||||
defer cancel()
|
||||
now := s.now().UTC()
|
||||
out := models.MonitoringSnapshot{Version: "1", CheckedAt: now, RefreshAfter: now.Add(CacheTTL), Sources: make([]models.MonitoringSource, len(s.sources))}
|
||||
sem := make(chan struct{}, MaxConcurrentSources)
|
||||
var wg sync.WaitGroup
|
||||
for i, source := range s.sources {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
out.Sources[i] = unavailable(source.ID, now, "dependency_missing")
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
out.Sources[i] = unavailable(source.ID, now, "collection_failed")
|
||||
}
|
||||
}()
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
out.Sources[i] = unavailable(source.ID, now, "timeout")
|
||||
return
|
||||
}
|
||||
defer func() { <-sem }()
|
||||
if source.Collect == nil {
|
||||
return
|
||||
}
|
||||
cctx, stop := context.WithTimeout(ctx, SourceTimeout)
|
||||
defer stop()
|
||||
result, err := source.Collect(cctx, now)
|
||||
if err == nil {
|
||||
err = cctx.Err()
|
||||
}
|
||||
if err != nil {
|
||||
out.Sources[i] = unavailable(source.ID, now, ErrorReason(err))
|
||||
return
|
||||
}
|
||||
result.ID, result.CheckedAt = source.ID, now
|
||||
if result.Availability == "" {
|
||||
result.Availability = models.MonitoringFresh
|
||||
}
|
||||
if result.Coverage == "" {
|
||||
result.Coverage = "complete"
|
||||
}
|
||||
if result.Availability == models.MonitoringFresh {
|
||||
result.ObservedAt = &now
|
||||
}
|
||||
if result.Metrics == nil {
|
||||
result.Metrics = []models.MonitoringMetric{}
|
||||
}
|
||||
for _, metric := range result.Metrics {
|
||||
if metric.Availability != models.MonitoringFresh && result.Coverage == "complete" {
|
||||
result.Coverage = "partial"
|
||||
}
|
||||
}
|
||||
out.Sources[i] = result
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for i := range out.Sources {
|
||||
cur := &out.Sources[i]
|
||||
if cur.Availability != models.MonitoringUnavailable || i >= len(s.snapshot.Sources) {
|
||||
continue
|
||||
}
|
||||
old := s.snapshot.Sources[i]
|
||||
if old.ObservedAt == nil || now.Sub(*old.ObservedAt) > RetainLastGood {
|
||||
continue
|
||||
}
|
||||
old.CheckedAt, old.Availability, old.Reason = now, models.MonitoringStale, cur.Reason
|
||||
old.Metrics = append([]models.MonitoringMetric(nil), old.Metrics...)
|
||||
for j := range old.Metrics {
|
||||
old.Metrics[j].Availability = models.MonitoringStale
|
||||
old.Metrics[j].Reason = cur.Reason
|
||||
age(&old.Metrics[j], now)
|
||||
}
|
||||
*cur = old
|
||||
}
|
||||
s.snapshot = out
|
||||
s.refreshing = nil
|
||||
close(ready)
|
||||
}
|
||||
|
||||
func (s *Service) project(snapshot models.MonitoringSnapshot, permissions models.AdminPermission) models.MonitoringSnapshot {
|
||||
snapshot.Sources = append([]models.MonitoringSource(nil), snapshot.Sources...)
|
||||
known := 0
|
||||
for i, source := range s.sources {
|
||||
if !permissions.HasPermission(source.Permission) {
|
||||
snapshot.Sources[i] = unavailable(source.ID, snapshot.CheckedAt, "permission_denied")
|
||||
continue
|
||||
}
|
||||
v := &snapshot.Sources[i]
|
||||
v.Metrics = append([]models.MonitoringMetric{}, v.Metrics...)
|
||||
for j := range v.Metrics {
|
||||
age(&v.Metrics[j], s.now())
|
||||
}
|
||||
if v.Availability == models.MonitoringFresh && v.Coverage == "complete" {
|
||||
known++
|
||||
}
|
||||
}
|
||||
snapshot.Coverage = "partial"
|
||||
if known == 0 {
|
||||
snapshot.Coverage = "unavailable"
|
||||
} else if known == len(s.sources) {
|
||||
snapshot.Coverage = "complete"
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
func age(m *models.MonitoringMetric, now time.Time) {
|
||||
m.EvidenceAgeSeconds = nil
|
||||
if m.EvidenceAt != nil && !m.EvidenceAt.After(now) {
|
||||
v := int64(now.Sub(*m.EvidenceAt).Seconds())
|
||||
m.EvidenceAgeSeconds = &v
|
||||
}
|
||||
}
|
||||
|
||||
func ErrorReason(err error) string {
|
||||
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
|
||||
return "timeout"
|
||||
}
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) {
|
||||
if pgErr.Code == "42501" {
|
||||
return "permission_denied"
|
||||
}
|
||||
if pgErr.Code == "57014" {
|
||||
return "timeout"
|
||||
}
|
||||
}
|
||||
return "query_failed"
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
)
|
||||
|
||||
func TestMonitoringCoalescesCacheAndPreservesObservationTimes(t *testing.T) {
|
||||
var clock atomic.Int64
|
||||
clock.Store(time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC).Unix())
|
||||
var calls atomic.Int64
|
||||
start, release := make(chan struct{}), make(chan struct{})
|
||||
s := New([]Source{{ID: "test", Permission: models.AdminPermViewAnalytics, Collect: func(_ context.Context, at time.Time) (models.MonitoringSource, error) {
|
||||
if calls.Add(1) == 1 {
|
||||
close(start)
|
||||
<-release
|
||||
}
|
||||
evidence := at.Add(-time.Hour)
|
||||
n := int64(0)
|
||||
return models.MonitoringSource{Metrics: []models.MonitoringMetric{{ID: "zero", Count: &n, ObservedAt: &at, EvidenceAt: &evidence, Availability: models.MonitoringFresh, Condition: models.MonitoringNoEvidence}}}, nil
|
||||
}}})
|
||||
s.now = func() time.Time { return time.Unix(clock.Load(), 0).UTC() }
|
||||
var wg sync.WaitGroup
|
||||
results := make(chan models.MonitoringSnapshot, 20)
|
||||
for i := 0; i < 20; i++ {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); results <- s.Snapshot(t.Context(), models.AdminPermViewAnalytics) }()
|
||||
}
|
||||
<-start
|
||||
close(release)
|
||||
wg.Wait()
|
||||
close(results)
|
||||
if calls.Load() != 1 {
|
||||
t.Fatalf("coalesced calls=%d", calls.Load())
|
||||
}
|
||||
var first models.MonitoringSnapshot
|
||||
for result := range results {
|
||||
first = result
|
||||
if result.Coverage != "complete" || result.Sources[0].Metrics[0].Count == nil || *result.Sources[0].Metrics[0].Count != 0 {
|
||||
t.Fatal(result)
|
||||
}
|
||||
}
|
||||
clock.Add(30)
|
||||
next := s.Snapshot(t.Context(), models.AdminPermViewAnalytics)
|
||||
if !next.CheckedAt.Equal(first.CheckedAt) || !next.Sources[0].ObservedAt.Equal(*first.Sources[0].ObservedAt) || *next.Sources[0].Metrics[0].EvidenceAgeSeconds != 3630 || calls.Load() != 1 {
|
||||
t.Fatal("cache refreshed evidence", next)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitoringFailedRefreshRetainsOriginalEvidenceThenExpires(t *testing.T) {
|
||||
now := time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC)
|
||||
fail := false
|
||||
s := New([]Source{{ID: "test", Permission: models.AdminPermViewAnalytics, Collect: func(_ context.Context, at time.Time) (models.MonitoringSource, error) {
|
||||
if fail {
|
||||
return models.MonitoringSource{}, errors.New("postgres://sensitive-user:secret@private.host/customer@example.test")
|
||||
}
|
||||
n := int64(2)
|
||||
evidence := at.Add(-time.Hour)
|
||||
return models.MonitoringSource{Metrics: []models.MonitoringMetric{{ID: "backlog", Count: &n, EvidenceAt: &evidence, ObservedAt: &at, Availability: models.MonitoringFresh, Condition: models.MonitoringBacklog}}}, nil
|
||||
}}})
|
||||
s.now = func() time.Time { return now }
|
||||
first := s.Snapshot(t.Context(), models.AdminPermViewAnalytics)
|
||||
fail = true
|
||||
now = now.Add(2 * time.Minute)
|
||||
next := s.Snapshot(t.Context(), models.AdminPermViewAnalytics)
|
||||
source := next.Sources[0]
|
||||
if source.Availability != models.MonitoringStale || source.Reason != "query_failed" || !source.ObservedAt.Equal(*first.Sources[0].ObservedAt) || *source.Metrics[0].EvidenceAgeSeconds != 3720 || source.Metrics[0].Availability != models.MonitoringStale {
|
||||
t.Fatal(next)
|
||||
}
|
||||
if first.Sources[0].Metrics[0].Availability != models.MonitoringFresh {
|
||||
t.Fatal("previous response was mutated")
|
||||
}
|
||||
now = now.Add(4 * time.Minute)
|
||||
expired := s.Snapshot(t.Context(), models.AdminPermViewAnalytics)
|
||||
if expired.Sources[0].Availability != models.MonitoringUnavailable || expired.Sources[0].ObservedAt != nil || len(expired.Sources[0].Metrics) != 0 {
|
||||
t.Fatal(expired)
|
||||
}
|
||||
raw, _ := json.Marshal(expired)
|
||||
if strings.Contains(string(raw), "sensitive-user") || strings.Contains(string(raw), "customer@") || strings.Contains(string(raw), "private.host") {
|
||||
t.Fatal("raw dependency error leaked")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitoringPermissionsNilPanicAndErrorReasons(t *testing.T) {
|
||||
n := int64(123)
|
||||
s := New([]Source{
|
||||
{ID: "mailboxes", Permission: models.AdminPermViewUsers, Collect: func(context.Context, time.Time) (models.MonitoringSource, error) {
|
||||
return models.MonitoringSource{Metrics: []models.MonitoringMetric{{ID: "private", ScopeID: "private-worker-id", Count: &n, Availability: models.MonitoringFresh}}}, nil
|
||||
}},
|
||||
{ID: "nil", Permission: models.AdminPermViewAnalytics},
|
||||
{ID: "panic", Permission: models.AdminPermViewAnalytics, Collect: func(context.Context, time.Time) (models.MonitoringSource, error) { panic("secret") }},
|
||||
{ID: "denied", Permission: models.AdminPermViewAnalytics, Collect: func(context.Context, time.Time) (models.MonitoringSource, error) {
|
||||
return models.MonitoringSource{}, &pgconn.PgError{Code: "42501", Message: "secret"}
|
||||
}},
|
||||
})
|
||||
out := s.Snapshot(t.Context(), models.AdminPermViewAnalytics)
|
||||
for i, reason := range []string{"permission_denied", "dependency_missing", "collection_failed", "permission_denied"} {
|
||||
if out.Sources[i].Reason != reason || out.Sources[i].Availability != models.MonitoringUnavailable || len(out.Sources[i].Metrics) != 0 {
|
||||
t.Fatal(out.Sources[i])
|
||||
}
|
||||
}
|
||||
raw, _ := json.Marshal(out)
|
||||
if strings.Contains(string(raw), "private-worker") || strings.Contains(string(raw), "123") || strings.Contains(string(raw), "secret") {
|
||||
t.Fatal("restricted evidence leaked")
|
||||
}
|
||||
admin := s.Snapshot(t.Context(), models.AdminPermViewAnalytics|models.AdminPermViewUsers)
|
||||
if *admin.Sources[0].Metrics[0].Count != 123 {
|
||||
t.Fatal("permission projection contaminated cache")
|
||||
}
|
||||
for _, err := range []error{context.Canceled, context.DeadlineExceeded, &pgconn.PgError{Code: "57014"}} {
|
||||
if ErrorReason(err) != "timeout" {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var absent *Service
|
||||
if absent.Snapshot(t.Context(), models.AdminPermViewAnalytics).Coverage != "unavailable" {
|
||||
t.Fatal("nil service is healthy")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitoringCallerCancellationAndPartialCoverage(t *testing.T) {
|
||||
s := New([]Source{{ID: "partial", Permission: models.AdminPermViewAnalytics, Collect: func(ctx context.Context, _ time.Time) (models.MonitoringSource, error) {
|
||||
<-ctx.Done()
|
||||
return models.MonitoringSource{}, ctx.Err()
|
||||
}}})
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
start := time.Now()
|
||||
out := s.Snapshot(ctx, models.AdminPermViewAnalytics)
|
||||
if time.Since(start) > time.Second || out.Coverage != "unavailable" || out.Sources[0].Reason != "timeout" {
|
||||
t.Fatal(out)
|
||||
}
|
||||
s.mu.Lock()
|
||||
ready := s.refreshing
|
||||
s.mu.Unlock()
|
||||
<-ready
|
||||
out = s.Snapshot(t.Context(), models.AdminPermViewAnalytics)
|
||||
if out.Sources[0].ObservedAt != nil {
|
||||
t.Fatal("timeout observation became fresh")
|
||||
}
|
||||
partial := New([]Source{{ID: "partial", Permission: models.AdminPermViewAnalytics, Collect: func(context.Context, time.Time) (models.MonitoringSource, error) {
|
||||
return models.MonitoringSource{Metrics: []models.MonitoringMetric{{ID: "absent", Availability: models.MonitoringUnavailable, Count: nil, Reason: "no_recent_evidence"}}}, nil
|
||||
}}})
|
||||
if out := partial.Snapshot(t.Context(), models.AdminPermViewAnalytics); out.Coverage == "complete" || out.Sources[0].Coverage != "partial" {
|
||||
t.Fatal(out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitoringMetricJSONKeepsUnknownMeasurementsNull(t *testing.T) {
|
||||
raw, err := json.Marshal(models.MonitoringMetric{ID: "unknown", Availability: models.MonitoringUnavailable})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, field := range []string{"count", "affected_mailboxes", "affected_organizations", "evidence_at", "evidence_age_seconds"} {
|
||||
if !strings.Contains(string(raw), `"`+field+`":null`) {
|
||||
t.Fatal(string(raw))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
)
|
||||
|
||||
func UnobservedSources() []Source {
|
||||
return []Source{
|
||||
{ID: "send_wait_attribution", Permission: models.AdminPermViewCampaigns, Collect: func(_ context.Context, _ time.Time) (models.MonitoringSource, error) {
|
||||
out := models.MonitoringSource{Availability: models.MonitoringUnavailable, Coverage: "unavailable", Reason: "unsupported", Metrics: []models.MonitoringMetric{}}
|
||||
for _, m := range []models.MonitoringMetric{
|
||||
{ID: "send_daily_limit_wait", Title: "Live send daily-limit waits", Condition: models.MonitoringDailyLimit},
|
||||
{ID: "send_budget_wait", Title: "Live send budget waits", Condition: models.MonitoringBudget},
|
||||
{ID: "send_schedule_reason", Title: "Detailed send schedule wait reasons", Condition: models.MonitoringSchedule},
|
||||
} {
|
||||
m.Unit = "tasks"
|
||||
m.Availability = models.MonitoringUnavailable
|
||||
m.Severity = "info"
|
||||
m.Reason = "unsupported"
|
||||
m.Note = "Current scheduler does not persist per-task pacing/daily-cap/budget decisions. Future schedules are observable separately; a specific reason cannot be inferred."
|
||||
out.Metrics = append(out.Metrics, m)
|
||||
}
|
||||
return out, nil
|
||||
}},
|
||||
{ID: "tracking_broker", Permission: models.AdminPermViewAnalytics, Collect: func(_ context.Context, _ time.Time) (models.MonitoringSource, error) {
|
||||
return models.MonitoringSource{Availability: models.MonitoringUnavailable, Coverage: "unavailable", Reason: "unsupported", Metrics: []models.MonitoringMetric{{ID: "tracking_queue", Title: "External tracking consumer backlog", Unit: "messages", Availability: models.MonitoringUnavailable, Condition: models.MonitoringUnknown, Reason: "unsupported", Severity: "info", Note: "This backend does not own the external tracking service's effective consumer scope. Backend worker-events and registered worker commands are measured separately."}}}, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
@@ -211,6 +211,10 @@ func (s *service) notifyMembers(ctx context.Context, orgID uuid.UUID, perm model
|
||||
if m.AcceptedAt == nil || m.UserID == uuid.Nil || m.UserID == exclude {
|
||||
continue
|
||||
}
|
||||
// Workspace notifications name resources a restricted member may not reach.
|
||||
if m.IsRestricted() {
|
||||
continue
|
||||
}
|
||||
if perm != 0 && !m.Permissions.HasPermission(perm) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
package organization
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"github.com/warmbly/warmbly/internal/errx"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
"github.com/warmbly/warmbly/internal/observability/errs"
|
||||
)
|
||||
|
||||
// errInvalidAccessResource refuses a grant naming something outside the workspace.
|
||||
var errInvalidAccessResource = errx.NewWithIdentifier(errx.BadRequest, "invalid_access_resource",
|
||||
"One or more of the selected campaign folders, campaigns or mailboxes are not in this workspace.")
|
||||
|
||||
// validateAccess normalizes a and checks every grant names a resource of the organization.
|
||||
func (s *organizationService) validateAccess(ctx context.Context, orgID uuid.UUID, a *models.MemberAccess) *errx.Error {
|
||||
if !a.Scope.Valid() && a.Scope != "" {
|
||||
return errx.NewWithIdentifier(errx.BadRequest, "invalid_access_scope", `access scope must be "workspace" or "restricted"`)
|
||||
}
|
||||
a.Normalize()
|
||||
if !a.Restricted() {
|
||||
return nil
|
||||
}
|
||||
for _, list := range [][]uuid.UUID{a.FolderIDs, a.CampaignIDs, a.EmailAccountIDs} {
|
||||
if len(list) > models.MaxAccessGrants {
|
||||
return errx.NewWithIdentifier(errx.BadRequest, "too_many_access_grants",
|
||||
fmt.Sprintf("a member can be granted at most %d campaign folders, %d campaigns and %d mailboxes", models.MaxAccessGrants, models.MaxAccessGrants, models.MaxAccessGrants))
|
||||
}
|
||||
}
|
||||
folders, campaigns, mailboxes, err := s.orgRepo.CountOwnedAccessResources(ctx, orgID, a)
|
||||
if err != nil {
|
||||
errs.CaptureException(err)
|
||||
return errx.New(errx.Internal, "failed to check access grants")
|
||||
}
|
||||
if folders != len(a.FolderIDs) || campaigns != len(a.CampaignIDs) || mailboxes != len(a.EmailAccountIDs) {
|
||||
return errInvalidAccessResource
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// memberRolePermissions is the OR of a member's assigned roles, before any scope.
|
||||
func (s *organizationService) memberRolePermissions(ctx context.Context, orgID, userID uuid.UUID) (models.OrganizationPermission, *errx.Error) {
|
||||
roles, err := s.orgRepo.GetMemberRoles(ctx, orgID, userID)
|
||||
if err != nil {
|
||||
errs.CaptureException(err)
|
||||
return 0, errx.New(errx.Internal, "failed to load roles")
|
||||
}
|
||||
var perms models.OrganizationPermission
|
||||
for _, r := range roles {
|
||||
role, err := s.orgRepo.GetRoleByID(ctx, orgID, r.ID)
|
||||
if err != nil {
|
||||
errs.CaptureException(err)
|
||||
return 0, errx.New(errx.Internal, "failed to load role")
|
||||
}
|
||||
if role != nil {
|
||||
perms |= role.Permissions
|
||||
}
|
||||
}
|
||||
return perms, nil
|
||||
}
|
||||
|
||||
// GetMemberAccess is one member's access scope and grants.
|
||||
func (s *organizationService) GetMemberAccess(ctx context.Context, orgID, userID uuid.UUID) (*models.MemberAccess, *errx.Error) {
|
||||
a, err := s.orgRepo.GetMemberAccess(ctx, orgID, userID)
|
||||
if err != nil {
|
||||
errs.CaptureException(err)
|
||||
return nil, errx.New(errx.Internal, "failed to load member access")
|
||||
}
|
||||
if a == nil {
|
||||
return nil, errx.New(errx.NotFound, "member not found")
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// SetMemberAccess replaces a member's access scope and grants.
|
||||
func (s *organizationService) SetMemberAccess(ctx context.Context, orgID, actorID, memberUserID uuid.UUID, a *models.MemberAccess) (*models.OrganizationMember, *errx.Error) {
|
||||
member, err := s.orgRepo.GetMember(ctx, orgID, memberUserID)
|
||||
if err != nil {
|
||||
errs.CaptureException(err)
|
||||
return nil, errx.New(errx.Internal, "failed to get member")
|
||||
}
|
||||
if member == nil {
|
||||
return nil, errx.New(errx.NotFound, "member not found")
|
||||
}
|
||||
if member.IsOwner() {
|
||||
return nil, errx.NewWithIdentifier(errx.Forbidden, "owner_access_unrestricted", "the workspace owner always has access to the entire workspace")
|
||||
}
|
||||
if actorID == memberUserID {
|
||||
return nil, errx.New(errx.Forbidden, "you cannot change your own access")
|
||||
}
|
||||
if a.Scope == "" {
|
||||
return nil, errx.NewWithIdentifier(errx.BadRequest, "invalid_access_scope", `access scope must be "workspace" or "restricted"`)
|
||||
}
|
||||
if xerr := s.validateAccess(ctx, orgID, a); xerr != nil {
|
||||
return nil, xerr
|
||||
}
|
||||
// Either direction decides what the member's roles reach, so the actor must hold all of it.
|
||||
rolePerms, xerr := s.memberRolePermissions(ctx, orgID, memberUserID)
|
||||
if xerr != nil {
|
||||
return nil, xerr
|
||||
}
|
||||
if xerr := s.validateActorHoldsPermissions(ctx, orgID, actorID, rolePerms|member.Permissions); xerr != nil {
|
||||
return nil, xerr
|
||||
}
|
||||
if err := s.orgRepo.SetMemberAccess(ctx, orgID, memberUserID, a, actorID); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, errx.New(errx.NotFound, "member not found")
|
||||
}
|
||||
errs.CaptureException(err)
|
||||
return nil, errx.New(errx.Internal, "failed to update member access")
|
||||
}
|
||||
updated, err := s.orgRepo.GetMember(ctx, orgID, memberUserID)
|
||||
if err != nil || updated == nil {
|
||||
if err != nil {
|
||||
errs.CaptureException(err)
|
||||
}
|
||||
return nil, errx.New(errx.Internal, "failed to load member")
|
||||
}
|
||||
updated.Roles, _ = s.orgRepo.GetMemberRoles(ctx, orgID, memberUserID)
|
||||
if updated.Access, err = s.orgRepo.GetMemberAccess(ctx, orgID, memberUserID); err != nil {
|
||||
errs.CaptureException(err)
|
||||
}
|
||||
return updated, nil
|
||||
}
|
||||
|
||||
// ResolveMemberScope is what a restricted member reaches for one request.
|
||||
func (s *organizationService) ResolveMemberScope(ctx context.Context, orgID, userID uuid.UUID) (*models.ResourceScope, *errx.Error) {
|
||||
scope, err := s.orgRepo.ResolveMemberScope(ctx, orgID, userID)
|
||||
if err != nil {
|
||||
errs.CaptureException(err)
|
||||
return nil, errx.New(errx.Internal, "failed to resolve member access")
|
||||
}
|
||||
return scope, nil
|
||||
}
|
||||
|
||||
// SuggestCampaignSenders lists the mailboxes the selected campaigns and folders send from.
|
||||
func (s *organizationService) SuggestCampaignSenders(ctx context.Context, orgID uuid.UUID, campaignIDs, folderIDs []uuid.UUID) ([]models.SuggestedSender, *errx.Error) {
|
||||
out, err := s.orgRepo.SuggestCampaignSenders(ctx, orgID, campaignIDs, folderIDs)
|
||||
if err != nil {
|
||||
errs.CaptureException(err)
|
||||
return nil, errx.New(errx.Internal, "failed to suggest senders")
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// attachMemberAccess fills each member's grants for the roster.
|
||||
func (s *organizationService) attachMemberAccess(ctx context.Context, orgID uuid.UUID, members []models.OrganizationMember) {
|
||||
byUser, err := s.orgRepo.ListMemberAccess(ctx, orgID)
|
||||
if err != nil {
|
||||
errs.CaptureException(err)
|
||||
return
|
||||
}
|
||||
for i := range members {
|
||||
if a, ok := byUser[members[i].UserID]; ok {
|
||||
members[i].Access = a
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -98,6 +98,11 @@ type OrganizationService interface {
|
||||
UpdateRole(ctx context.Context, orgID, actorID, roleID uuid.UUID, req *models.UpdateOrganizationRoleRequest) (*models.OrganizationRole, *errx.Error)
|
||||
DeleteRole(ctx context.Context, orgID, actorID, roleID uuid.UUID) *errx.Error
|
||||
UpdateMemberRole(ctx context.Context, orgID, actorID, memberUserID uuid.UUID, req *models.UpdateMemberRequest) (*models.OrganizationMember, *errx.Error)
|
||||
// Resource scope: a restricted member's role applies only to the campaigns, folders and mailboxes granted to them.
|
||||
GetMemberAccess(ctx context.Context, orgID, userID uuid.UUID) (*models.MemberAccess, *errx.Error)
|
||||
SetMemberAccess(ctx context.Context, orgID, actorID, memberUserID uuid.UUID, access *models.MemberAccess) (*models.OrganizationMember, *errx.Error)
|
||||
ResolveMemberScope(ctx context.Context, orgID, userID uuid.UUID) (*models.ResourceScope, *errx.Error)
|
||||
SuggestCampaignSenders(ctx context.Context, orgID uuid.UUID, campaignIDs, folderIDs []uuid.UUID) ([]models.SuggestedSender, *errx.Error)
|
||||
RemoveMember(ctx context.Context, orgID, actorID, memberUserID uuid.UUID) *errx.Error
|
||||
|
||||
// Invitations
|
||||
@@ -639,6 +644,7 @@ func (s *organizationService) GetMembers(ctx context.Context, orgID uuid.UUID) (
|
||||
if err := s.orgRepo.HydrateMemberRoles(ctx, orgID, members); err != nil {
|
||||
errs.CaptureException(err)
|
||||
}
|
||||
s.attachMemberAccess(ctx, orgID, members)
|
||||
return members, nil
|
||||
}
|
||||
|
||||
@@ -700,6 +706,13 @@ func (s *organizationService) InviteMember(ctx context.Context, orgID uuid.UUID,
|
||||
if xerr := s.validateActorHoldsPermissions(ctx, orgID, inviterID, permissions); xerr != nil {
|
||||
return nil, xerr
|
||||
}
|
||||
access := req.Access
|
||||
if access == nil {
|
||||
access = models.WorkspaceAccess()
|
||||
}
|
||||
if xerr := s.validateAccess(ctx, orgID, access); xerr != nil {
|
||||
return nil, xerr
|
||||
}
|
||||
|
||||
role := roles[0].Name
|
||||
roleID := &roles[0].ID
|
||||
@@ -722,6 +735,7 @@ func (s *organizationService) InviteMember(ctx context.Context, orgID uuid.UUID,
|
||||
Token: crypt.SHA256(token),
|
||||
ExpiresAt: time.Now().Add(s.invitationTTL(ctx)),
|
||||
CreatedAt: time.Now(),
|
||||
Access: access,
|
||||
}
|
||||
|
||||
if err := s.orgRepo.CreateInvitation(ctx, inv); err != nil {
|
||||
@@ -939,6 +953,8 @@ func (s *organizationService) acceptResolved(ctx context.Context, inv *models.Or
|
||||
InvitedBy: &inv.InvitedBy,
|
||||
InvitedAt: inv.CreatedAt,
|
||||
AcceptedAt: &now,
|
||||
// Grants on resources deleted since the invite are dropped by the insert.
|
||||
Access: inv.Access,
|
||||
}
|
||||
if err := s.orgRepo.AddMemberWithRoles(ctx, member, liveRoleIDs); err != nil {
|
||||
errs.CaptureException(err)
|
||||
|
||||
@@ -230,6 +230,12 @@ var Tables = []Table{
|
||||
},
|
||||
Blobs: []BlobColumn{{Column: "avatar_url", Kind: BlobKindPublicURL}},
|
||||
},
|
||||
{
|
||||
// Below organization_members and email_accounts, which it references.
|
||||
Name: "organization_member_mailbox_access", Group: models.OrgDataGroupCore,
|
||||
Scope: scopeOrg,
|
||||
Note: "The mailboxes a restricted member is granted. The member's scope travels on organization_members.",
|
||||
},
|
||||
{
|
||||
Name: "email_accounts_smtp_imap", Group: models.OrgDataGroupCore,
|
||||
Scope: `email_account_id IN ` + orgMailboxes,
|
||||
@@ -416,6 +422,11 @@ var Tables = []Table{
|
||||
Name: "campaign_folders", Group: models.OrgDataGroupCampaigns,
|
||||
Scope: `campaign_id IN ` + orgCampaigns,
|
||||
},
|
||||
{
|
||||
Name: "organization_member_campaign_access", Group: models.OrgDataGroupCampaigns,
|
||||
Scope: scopeOrg,
|
||||
Note: "The campaign folders and campaigns a restricted member is granted. Left behind with the campaigns, a restricted member arrives with no campaign access.",
|
||||
},
|
||||
{
|
||||
// The forms tables live below campaigns despite belonging to the
|
||||
// Contacts group: each carries a campaign_id foreign key, and Tables is
|
||||
|
||||
@@ -124,14 +124,15 @@ func (s *Service) revokeTeam(ctx context.Context, teamID string, status models.I
|
||||
// actor is a Slack member resolved to a connection and, when linked and still
|
||||
// a member, to the Warmbly member they act as.
|
||||
type actor struct {
|
||||
teamID string
|
||||
userID string
|
||||
conn *models.IntegrationConnection
|
||||
token string
|
||||
link *models.SlackUserLink
|
||||
member *models.OrganizationMember
|
||||
unknown bool // membership could not be read; try again later
|
||||
gone bool // was linked, but is no longer a member
|
||||
teamID string
|
||||
userID string
|
||||
conn *models.IntegrationConnection
|
||||
token string
|
||||
link *models.SlackUserLink
|
||||
member *models.OrganizationMember
|
||||
unknown bool // membership could not be read; try again later
|
||||
gone bool // was linked, but is no longer a member
|
||||
restricted bool // a restricted member, whose grants Slack cannot act within
|
||||
}
|
||||
|
||||
// resolveActor returns nil when the team has no usable connection.
|
||||
@@ -159,6 +160,8 @@ func (s *Service) resolveActor(ctx context.Context, teamID, slackUserID string)
|
||||
a.gone = true
|
||||
case memberUnknown:
|
||||
a.unknown = true
|
||||
case memberRestricted:
|
||||
a.restricted = true
|
||||
}
|
||||
}
|
||||
return a
|
||||
@@ -205,6 +208,10 @@ func (s *Service) promptLink(ctx context.Context, a *actor, q *ask) {
|
||||
s.tell(ctx, a.token, a.userID, q, plainMessage("I couldn't check your Warmbly account just now. Please try again in a moment."))
|
||||
return
|
||||
}
|
||||
if a.restricted {
|
||||
s.tell(ctx, a.token, a.userID, q, plainMessage("Your Warmbly access is limited to selected campaigns and mailboxes, so I can't act for you here. Use the Warmbly dashboard instead."))
|
||||
return
|
||||
}
|
||||
lead := ""
|
||||
if a.gone {
|
||||
lead = "Your Warmbly membership changed, so this Slack account is no longer linked. Link it again to keep using Warmbly here."
|
||||
|
||||
@@ -273,7 +273,7 @@ const autoLinkMissTTL = 10 * time.Minute
|
||||
// Slack email in any workspace connected to the team, so a member whose
|
||||
// emails match never sees a link page. A miss is remembered for a while.
|
||||
func (s *Service) autoLink(ctx context.Context, a *actor) bool {
|
||||
if a == nil || a.link != nil || a.unknown || s.users == nil {
|
||||
if a == nil || a.link != nil || a.unknown || a.restricted || s.users == nil {
|
||||
return false
|
||||
}
|
||||
missKey := "slack:autolink:miss:" + a.teamID + ":" + a.userID
|
||||
|
||||
@@ -270,6 +270,7 @@ const (
|
||||
memberOK memberState = iota
|
||||
memberGone
|
||||
memberUnknown
|
||||
memberRestricted
|
||||
)
|
||||
|
||||
// membership re-reads the linked member's org permissions; a link whose
|
||||
@@ -291,5 +292,9 @@ func (s *Service) membership(ctx context.Context, link *models.SlackUserLink) (*
|
||||
return nil, memberUnknown
|
||||
}
|
||||
}
|
||||
// Slack mirrors the whole workspace's inbox, which a restricted member's grants do not cover.
|
||||
if m.IsRestricted() {
|
||||
return nil, memberRestricted
|
||||
}
|
||||
return m, memberOK
|
||||
}
|
||||
|
||||
@@ -17,6 +17,15 @@ func (s *uniboxService) GetByID(
|
||||
ctx context.Context,
|
||||
orgID, id uuid.UUID,
|
||||
) (*models.EmailMessage, *errx.Error) {
|
||||
return s.getByID(ctx, orgID, id, true)
|
||||
}
|
||||
|
||||
// Peek is GetByID for a reader who must not change the shared read state.
|
||||
func (s *uniboxService) Peek(ctx context.Context, orgID, id uuid.UUID) (*models.EmailMessage, *errx.Error) {
|
||||
return s.getByID(ctx, orgID, id, false)
|
||||
}
|
||||
|
||||
func (s *uniboxService) getByID(ctx context.Context, orgID, id uuid.UUID, markSeen bool) (*models.EmailMessage, *errx.Error) {
|
||||
var resp models.EmailMessage
|
||||
var snippet string
|
||||
var fixtureMessage bool
|
||||
@@ -42,7 +51,7 @@ func (s *uniboxService) GetByID(
|
||||
// member clears the shared unread state. Relayed like any other read
|
||||
// state change, so the mailbox agrees: this is the path a developer
|
||||
// hitting the API reaches, where nothing calls PATCH /unibox/seen.
|
||||
if !msg.Seen {
|
||||
if !msg.Seen && markSeen {
|
||||
if changed, err := s.uniboxRepository.MarkSeenBulk(ctx, orgID, []uuid.UUID{id}, true); err == nil {
|
||||
msg.Seen = true
|
||||
s.relaySeen(ctx, orgID, changed)
|
||||
|
||||
@@ -29,6 +29,8 @@ type UniboxService interface {
|
||||
ctx context.Context,
|
||||
orgID, id uuid.UUID,
|
||||
) (*models.EmailMessage, *errx.Error)
|
||||
// Peek is GetByID without marking the message read.
|
||||
Peek(ctx context.Context, orgID, id uuid.UUID) (*models.EmailMessage, *errx.Error)
|
||||
// ForwardSource is the message a forward carries, read without marking it
|
||||
// seen. NotFound when the id names no message in the organization.
|
||||
ForwardSource(ctx context.Context, orgID, id uuid.UUID) (*models.ForwardedMessage, *errx.Error)
|
||||
@@ -37,6 +39,8 @@ type UniboxService interface {
|
||||
orgID, emailID uuid.UUID,
|
||||
threadID, limit, cursor string,
|
||||
) (*models.MailSearchResult, *errx.Error)
|
||||
// GetByThreadWithin is GetByThread limited to the allowed mailboxes; nil allows every mailbox.
|
||||
GetByThreadWithin(ctx context.Context, orgID, emailID uuid.UUID, threadID, limit, cursor string, allowed []uuid.UUID) (*models.MailSearchResult, *errx.Error)
|
||||
// LatestMessageIDInThread is the RFC Message-ID a reply into this thread
|
||||
// should name in its In-Reply-To header. Returns "" when the thread is
|
||||
// unknown or holds no message id, which callers treat as "no backfill".
|
||||
|
||||
@@ -18,6 +18,10 @@ func (s *uniboxService) GetByThread(
|
||||
orgID, emailID uuid.UUID,
|
||||
threadID, limit, cursor string,
|
||||
) (*models.MailSearchResult, *errx.Error) {
|
||||
return s.GetByThreadWithin(ctx, orgID, emailID, threadID, limit, cursor, nil)
|
||||
}
|
||||
|
||||
func (s *uniboxService) GetByThreadWithin(ctx context.Context, orgID, emailID uuid.UUID, threadID, limit, cursor string, allowed []uuid.UUID) (*models.MailSearchResult, *errx.Error) {
|
||||
l := DefaultThreadLimit
|
||||
if limit != "" {
|
||||
parsed, err := strconv.Atoi(limit)
|
||||
@@ -30,7 +34,7 @@ func (s *uniboxService) GetByThread(
|
||||
l = parsed
|
||||
}
|
||||
|
||||
resp, err := s.uniboxRepository.GetByThread(ctx, orgID, emailID, threadID, l, cursor)
|
||||
resp, err := s.uniboxRepository.GetByThreadWithin(ctx, orgID, emailID, threadID, l, cursor, allowed)
|
||||
if err != nil {
|
||||
errs.CaptureException(err)
|
||||
return nil, errx.InternalError()
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
-- Rolling back would make every restricted member workspace-wide.
|
||||
DO $$ BEGIN
|
||||
IF EXISTS (SELECT 1 FROM organization_members WHERE access_scope = 'restricted')
|
||||
OR EXISTS (SELECT 1 FROM organization_invitations WHERE access_scope = 'restricted') THEN
|
||||
RAISE EXCEPTION 'restricted members or invitations exist; set them to the entire workspace or remove them first';
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
DROP TRIGGER IF EXISTS organization_member_access_mask ON organization_members;
|
||||
DROP FUNCTION IF EXISTS organization_member_access_mask();
|
||||
DROP INDEX IF EXISTS idx_campaign_folders_folder;
|
||||
DROP TABLE IF EXISTS organization_member_campaign_access;
|
||||
DROP TABLE IF EXISTS organization_member_mailbox_access;
|
||||
ALTER TABLE organization_invitations
|
||||
DROP COLUMN access_email_account_ids,
|
||||
DROP COLUMN access_campaign_ids,
|
||||
DROP COLUMN access_folder_ids,
|
||||
DROP COLUMN access_scope;
|
||||
ALTER TABLE organization_members DROP COLUMN access_scope;
|
||||
@@ -0,0 +1,61 @@
|
||||
-- A role says what a member may do; the scope says to which resources ('restricted' is only the grants below).
|
||||
ALTER TABLE organization_members
|
||||
ADD COLUMN access_scope text NOT NULL DEFAULT 'workspace'
|
||||
CONSTRAINT organization_members_access_scope_check CHECK (access_scope IN ('workspace', 'restricted'));
|
||||
|
||||
-- An invitee lands already restricted; the ids are checked against live resources again on accept.
|
||||
ALTER TABLE organization_invitations
|
||||
ADD COLUMN access_scope text NOT NULL DEFAULT 'workspace'
|
||||
CONSTRAINT organization_invitations_access_scope_check CHECK (access_scope IN ('workspace', 'restricted')),
|
||||
ADD COLUMN access_folder_ids uuid[] NOT NULL DEFAULT '{}',
|
||||
ADD COLUMN access_campaign_ids uuid[] NOT NULL DEFAULT '{}',
|
||||
ADD COLUMN access_email_account_ids uuid[] NOT NULL DEFAULT '{}';
|
||||
|
||||
-- Mailbox grants: the conversations of these mailboxes.
|
||||
CREATE TABLE organization_member_mailbox_access (
|
||||
organization_id uuid NOT NULL,
|
||||
user_id uuid NOT NULL,
|
||||
email_account_id uuid NOT NULL REFERENCES email_accounts(id) ON DELETE CASCADE,
|
||||
granted_by uuid REFERENCES users(id) ON DELETE SET NULL,
|
||||
created_at timestamp with time zone NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (organization_id, user_id, email_account_id),
|
||||
FOREIGN KEY (organization_id, user_id) REFERENCES organization_members(organization_id, user_id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE INDEX idx_member_mailbox_access_account ON organization_member_mailbox_access (email_account_id);
|
||||
|
||||
-- Campaign grants: one folder (whatever it holds at read time) or one campaign.
|
||||
CREATE TABLE organization_member_campaign_access (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
organization_id uuid NOT NULL,
|
||||
user_id uuid NOT NULL,
|
||||
folder_id uuid REFERENCES folders(id) ON DELETE CASCADE,
|
||||
campaign_id uuid REFERENCES campaigns(id) ON DELETE CASCADE,
|
||||
granted_by uuid REFERENCES users(id) ON DELETE SET NULL,
|
||||
created_at timestamp with time zone NOT NULL DEFAULT now(),
|
||||
CONSTRAINT organization_member_campaign_access_target CHECK (num_nonnulls(folder_id, campaign_id) = 1),
|
||||
FOREIGN KEY (organization_id, user_id) REFERENCES organization_members(organization_id, user_id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE UNIQUE INDEX uniq_member_campaign_access_folder ON organization_member_campaign_access (organization_id, user_id, folder_id) WHERE folder_id IS NOT NULL;
|
||||
CREATE UNIQUE INDEX uniq_member_campaign_access_campaign ON organization_member_campaign_access (organization_id, user_id, campaign_id) WHERE campaign_id IS NOT NULL;
|
||||
CREATE INDEX idx_member_campaign_access_folder ON organization_member_campaign_access (folder_id) WHERE folder_id IS NOT NULL;
|
||||
CREATE INDEX idx_member_campaign_access_campaign ON organization_member_campaign_access (campaign_id) WHERE campaign_id IS NOT NULL;
|
||||
|
||||
-- Folder grants resolve through campaign_folders by folder.
|
||||
CREATE INDEX IF NOT EXISTS idx_campaign_folders_folder ON campaign_folders (folder_id);
|
||||
|
||||
-- Every write holds a restricted member to view_campaigns (1024), view_analytics (32) and access_unibox (128); the owner is never restricted.
|
||||
CREATE FUNCTION organization_member_access_mask() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
IF NEW.role = 'owner' THEN
|
||||
NEW.access_scope := 'workspace';
|
||||
END IF;
|
||||
IF NEW.access_scope = 'restricted' THEN
|
||||
NEW.permissions := NEW.permissions & 1184;
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE TRIGGER organization_member_access_mask
|
||||
BEFORE INSERT OR UPDATE ON organization_members
|
||||
FOR EACH ROW EXECUTE FUNCTION organization_member_access_mask();
|
||||
@@ -0,0 +1,45 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
)
|
||||
|
||||
type DiagnosticScope struct {
|
||||
ID string
|
||||
Group string
|
||||
Topics []string
|
||||
}
|
||||
|
||||
// Diagnostics is optional and does not extend the worker EventBus envelope.
|
||||
type Diagnostics interface {
|
||||
Diagnose(context.Context, time.Time, []DiagnosticScope) models.MonitoringSource
|
||||
}
|
||||
|
||||
func diagnosticUnavailable(scope DiagnosticScope, at time.Time, reason string) models.MonitoringMetric {
|
||||
return models.MonitoringMetric{ID: "broker_queue", Title: "Broker queue observation", ScopeID: scope.ID, Unit: "messages", Availability: models.MonitoringUnavailable, Condition: models.MonitoringUnknown, Severity: "info", Reason: reason, Note: "Unavailable queue observation is not zero backlog or a broker outage."}
|
||||
}
|
||||
|
||||
func diagnosticMetric(scope DiagnosticScope, at time.Time, id, unit, note string, count int64) models.MonitoringMetric {
|
||||
return models.MonitoringMetric{ID: id, Title: "Broker " + id, ScopeID: scope.ID, Unit: unit, Availability: models.MonitoringFresh, Condition: models.MonitoringUnknown, Severity: "info", Count: &count, ObservedAt: &at, EvidenceAt: &at, LatestEvidenceAt: &at, Note: note}
|
||||
}
|
||||
|
||||
func diagnosticCoverage(out *models.MonitoringSource, measured, expected int64) {
|
||||
out.MeasuredScopes = &measured
|
||||
out.ExpectedScopes = &expected
|
||||
out.Coverage = "complete"
|
||||
if measured < expected {
|
||||
out.Coverage = "partial"
|
||||
}
|
||||
if measured == 0 && expected > 0 {
|
||||
out.Availability = models.MonitoringUnavailable
|
||||
out.Reason = "collection_failed"
|
||||
out.Coverage = "unavailable"
|
||||
}
|
||||
if expected == 0 {
|
||||
out.Coverage = "partial"
|
||||
out.Reason = "no_registered_scopes"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
//go:build kafka
|
||||
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
ckf "github.com/confluentinc/confluent-kafka-go/v2/kafka"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
)
|
||||
|
||||
type kafkaDiagnosticAdmin interface {
|
||||
DescribeTopics(context.Context, ckf.TopicCollection, ...ckf.DescribeTopicsAdminOption) (ckf.DescribeTopicsResult, error)
|
||||
DescribeConsumerGroups(context.Context, []string, ...ckf.DescribeConsumerGroupsAdminOption) (ckf.DescribeConsumerGroupsResult, error)
|
||||
ListConsumerGroupOffsets(context.Context, []ckf.ConsumerGroupTopicPartitions, ...ckf.ListConsumerGroupOffsetsAdminOption) (ckf.ListConsumerGroupOffsetsResult, error)
|
||||
ListOffsets(context.Context, map[ckf.TopicPartition]ckf.OffsetSpec, ...ckf.ListOffsetsAdminOption) (ckf.ListOffsetsResult, error)
|
||||
}
|
||||
|
||||
func (b *KafkaBus) Diagnose(ctx context.Context, at time.Time, scopes []DiagnosticScope) models.MonitoringSource {
|
||||
var admin kafkaDiagnosticAdmin
|
||||
if b != nil {
|
||||
b.topics.mu.Lock()
|
||||
if !b.topics.closed && b.topics.admin != nil {
|
||||
admin = b.topics.admin
|
||||
}
|
||||
b.topics.mu.Unlock()
|
||||
}
|
||||
return diagnoseKafka(ctx, admin, at, scopes)
|
||||
}
|
||||
|
||||
func kafkaDiagnosticReason(err error) string {
|
||||
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
|
||||
return "timeout"
|
||||
}
|
||||
var e ckf.Error
|
||||
if errors.As(err, &e) {
|
||||
switch e.Code() {
|
||||
case ckf.ErrTopicAuthorizationFailed, ckf.ErrGroupAuthorizationFailed, ckf.ErrClusterAuthorizationFailed:
|
||||
return "permission_denied"
|
||||
case ckf.ErrUnknownTopicOrPart, ckf.ErrGroupIDNotFound:
|
||||
return "resource_absent"
|
||||
case ckf.ErrTimedOut, ckf.ErrTimedOutQueue:
|
||||
return "timeout"
|
||||
}
|
||||
}
|
||||
return "collection_failed"
|
||||
}
|
||||
|
||||
func diagnoseKafka(ctx context.Context, admin kafkaDiagnosticAdmin, at time.Time, scopes []DiagnosticScope) models.MonitoringSource {
|
||||
out := models.MonitoringSource{Metrics: []models.MonitoringMetric{}}
|
||||
var measured int64
|
||||
for _, scope := range scopes {
|
||||
if admin == nil {
|
||||
out.Metrics = append(out.Metrics, diagnosticUnavailable(scope, at, "dependency_missing"))
|
||||
continue
|
||||
}
|
||||
lag, members, err := kafkaScopeLag(ctx, admin, scope)
|
||||
if err != nil {
|
||||
out.Metrics = append(out.Metrics, diagnosticUnavailable(scope, at, kafkaDiagnosticReason(err)))
|
||||
continue
|
||||
}
|
||||
measured++
|
||||
out.Metrics = append(out.Metrics, diagnosticMetric(scope, at, "committed_lag", "messages", "All requested topic partitions validated against earliest and latest offsets. Committed lag is not uncommitted processing, throughput or provider delivery.", lag), diagnosticMetric(scope, at, "members", "members", "Group membership can be absent during a rebalance; zero members alone is not an outage.", members))
|
||||
}
|
||||
diagnosticCoverage(&out, measured, int64(len(scopes)))
|
||||
return out
|
||||
}
|
||||
|
||||
type diagnosticPartition struct {
|
||||
topic string
|
||||
partition int32
|
||||
}
|
||||
|
||||
func kafkaScopeLag(ctx context.Context, admin kafkaDiagnosticAdmin, scope DiagnosticScope) (int64, int64, error) {
|
||||
invalid := errors.New("incomplete broker observation")
|
||||
topics, err := admin.DescribeTopics(ctx, ckf.NewTopicCollectionOfTopicNames(scope.Topics))
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
if len(topics.TopicDescriptions) != len(scope.Topics) {
|
||||
return 0, 0, invalid
|
||||
}
|
||||
partitions := []ckf.TopicPartition{}
|
||||
expected := map[diagnosticPartition]bool{}
|
||||
for _, t := range topics.TopicDescriptions {
|
||||
if t.Error.Code() != ckf.ErrNoError {
|
||||
return 0, 0, t.Error
|
||||
}
|
||||
if len(t.Partitions) == 0 {
|
||||
return 0, 0, invalid
|
||||
}
|
||||
found := false
|
||||
for _, requested := range scope.Topics {
|
||||
if t.Name == requested {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return 0, 0, invalid
|
||||
}
|
||||
for _, p := range t.Partitions {
|
||||
name := t.Name
|
||||
key := diagnosticPartition{name, int32(p.Partition)}
|
||||
if expected[key] {
|
||||
return 0, 0, invalid
|
||||
}
|
||||
expected[key] = true
|
||||
partitions = append(partitions, ckf.TopicPartition{Topic: &name, Partition: int32(p.Partition)})
|
||||
}
|
||||
}
|
||||
if len(partitions) == 0 || len(partitions) > 128 {
|
||||
return 0, 0, invalid
|
||||
}
|
||||
groups, err := admin.DescribeConsumerGroups(ctx, []string{scope.Group})
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
if len(groups.ConsumerGroupDescriptions) != 1 || groups.ConsumerGroupDescriptions[0].GroupID != scope.Group {
|
||||
return 0, 0, invalid
|
||||
}
|
||||
g := groups.ConsumerGroupDescriptions[0]
|
||||
if g.Error.Code() != ckf.ErrNoError {
|
||||
return 0, 0, g.Error
|
||||
}
|
||||
committed, err := admin.ListConsumerGroupOffsets(ctx, []ckf.ConsumerGroupTopicPartitions{{Group: scope.Group, Partitions: partitions}})
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
if len(committed.ConsumerGroupsTopicPartitions) != 1 || committed.ConsumerGroupsTopicPartitions[0].Group != scope.Group {
|
||||
return 0, 0, invalid
|
||||
}
|
||||
offsets := map[diagnosticPartition]int64{}
|
||||
for _, p := range committed.ConsumerGroupsTopicPartitions[0].Partitions {
|
||||
if p.Error != nil {
|
||||
return 0, 0, p.Error
|
||||
}
|
||||
if p.Topic == nil || p.Offset < 0 {
|
||||
return 0, 0, invalid
|
||||
}
|
||||
key := diagnosticPartition{*p.Topic, p.Partition}
|
||||
if !expected[key] {
|
||||
return 0, 0, invalid
|
||||
}
|
||||
if _, exists := offsets[key]; exists {
|
||||
return 0, 0, invalid
|
||||
}
|
||||
offsets[key] = int64(p.Offset)
|
||||
}
|
||||
if len(offsets) != len(expected) {
|
||||
return 0, 0, invalid
|
||||
}
|
||||
specs := map[ckf.TopicPartition]ckf.OffsetSpec{}
|
||||
for _, p := range partitions {
|
||||
specs[p] = ckf.LatestOffsetSpec
|
||||
}
|
||||
latest, err := admin.ListOffsets(ctx, specs)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
for _, p := range partitions {
|
||||
specs[p] = ckf.EarliestOffsetSpec
|
||||
}
|
||||
earliest, err := admin.ListOffsets(ctx, specs)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
read := func(result ckf.ListOffsetsResult) (map[diagnosticPartition]int64, error) {
|
||||
values := map[diagnosticPartition]int64{}
|
||||
for p, v := range result.ResultInfos {
|
||||
if v.Error.Code() != ckf.ErrNoError {
|
||||
return nil, v.Error
|
||||
}
|
||||
if p.Topic == nil || v.Offset < 0 {
|
||||
return nil, invalid
|
||||
}
|
||||
key := diagnosticPartition{*p.Topic, p.Partition}
|
||||
if !expected[key] {
|
||||
return nil, invalid
|
||||
}
|
||||
if _, exists := values[key]; exists {
|
||||
return nil, invalid
|
||||
}
|
||||
values[key] = int64(v.Offset)
|
||||
}
|
||||
if len(values) != len(expected) {
|
||||
return nil, invalid
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
ends, err := read(latest)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
starts, err := read(earliest)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
var lag int64
|
||||
for key, offset := range offsets {
|
||||
if starts[key] > offset || offset > ends[key] || starts[key] > ends[key] {
|
||||
return 0, 0, invalid
|
||||
}
|
||||
delta := ends[key] - offset
|
||||
if delta > int64(^uint64(0)>>1)-lag {
|
||||
return 0, 0, invalid
|
||||
}
|
||||
lag += delta
|
||||
}
|
||||
return lag, int64(len(g.Members)), nil
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
//go:build kafka
|
||||
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
ckf "github.com/confluentinc/confluent-kafka-go/v2/kafka"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
)
|
||||
|
||||
type fakeDiagnosticAdmin struct {
|
||||
problem string
|
||||
calls []string
|
||||
}
|
||||
|
||||
func (f *fakeDiagnosticAdmin) DescribeTopics(_ context.Context, _ ckf.TopicCollection, _ ...ckf.DescribeTopicsAdminOption) (ckf.DescribeTopicsResult, error) {
|
||||
f.calls = append(f.calls, "topics")
|
||||
r := ckf.DescribeTopicsResult{TopicDescriptions: []ckf.TopicDescription{{Name: "jobs.worker-events", Partitions: []ckf.TopicPartitionInfo{{Partition: 0}}}}}
|
||||
if f.problem == "topic_denied" {
|
||||
r.TopicDescriptions[0].Error = ckf.NewError(ckf.ErrTopicAuthorizationFailed, "private broker hostname", false)
|
||||
}
|
||||
if f.problem == "no_topic" {
|
||||
r.TopicDescriptions = nil
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
func (f *fakeDiagnosticAdmin) DescribeConsumerGroups(_ context.Context, _ []string, _ ...ckf.DescribeConsumerGroupsAdminOption) (ckf.DescribeConsumerGroupsResult, error) {
|
||||
f.calls = append(f.calls, "groups")
|
||||
r := ckf.DescribeConsumerGroupsResult{ConsumerGroupDescriptions: []ckf.ConsumerGroupDescription{{GroupID: "private-group", Members: []ckf.MemberDescription{{}}}}}
|
||||
if f.problem == "group_denied" {
|
||||
r.ConsumerGroupDescriptions[0].Error = ckf.NewError(ckf.ErrGroupAuthorizationFailed, "private broker hostname", false)
|
||||
}
|
||||
if f.problem == "no_group" {
|
||||
r.ConsumerGroupDescriptions = nil
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
func (f *fakeDiagnosticAdmin) ListConsumerGroupOffsets(_ context.Context, _ []ckf.ConsumerGroupTopicPartitions, _ ...ckf.ListConsumerGroupOffsetsAdminOption) (ckf.ListConsumerGroupOffsetsResult, error) {
|
||||
f.calls = append(f.calls, "committed")
|
||||
topic := "jobs.worker-events"
|
||||
p := ckf.TopicPartition{Topic: &topic, Partition: 0, Offset: 4}
|
||||
r := ckf.ListConsumerGroupOffsetsResult{ConsumerGroupsTopicPartitions: []ckf.ConsumerGroupTopicPartitions{{Group: "private-group", Partitions: []ckf.TopicPartition{p}}}}
|
||||
switch f.problem {
|
||||
case "no_commit":
|
||||
r.ConsumerGroupsTopicPartitions[0].Partitions = nil
|
||||
case "negative":
|
||||
r.ConsumerGroupsTopicPartitions[0].Partitions[0].Offset = -1001
|
||||
case "ahead":
|
||||
r.ConsumerGroupsTopicPartitions[0].Partitions[0].Offset = 12
|
||||
case "behind_retention":
|
||||
r.ConsumerGroupsTopicPartitions[0].Partitions[0].Offset = 0
|
||||
case "partition_denied":
|
||||
r.ConsumerGroupsTopicPartitions[0].Partitions[0].Error = ckf.NewError(ckf.ErrTopicAuthorizationFailed, "private", false)
|
||||
case "commit_top_error":
|
||||
return r, context.DeadlineExceeded
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
func (f *fakeDiagnosticAdmin) ListOffsets(_ context.Context, specs map[ckf.TopicPartition]ckf.OffsetSpec, _ ...ckf.ListOffsetsAdminOption) (ckf.ListOffsetsResult, error) {
|
||||
r := ckf.ListOffsetsResult{ResultInfos: map[ckf.TopicPartition]ckf.ListOffsetsResultInfo{}}
|
||||
for p, spec := range specs {
|
||||
if spec == ckf.LatestOffsetSpec {
|
||||
f.calls = append(f.calls, "latest")
|
||||
r.ResultInfos[p] = ckf.ListOffsetsResultInfo{Offset: 10}
|
||||
} else {
|
||||
f.calls = append(f.calls, "earliest")
|
||||
r.ResultInfos[p] = ckf.ListOffsetsResultInfo{Offset: 2}
|
||||
}
|
||||
}
|
||||
switch f.problem {
|
||||
case "missing_end":
|
||||
r.ResultInfos = map[ckf.TopicPartition]ckf.ListOffsetsResultInfo{}
|
||||
case "end_denied":
|
||||
for p, v := range r.ResultInfos {
|
||||
v.Error = ckf.NewError(ckf.ErrTopicAuthorizationFailed, "private", false)
|
||||
r.ResultInfos[p] = v
|
||||
}
|
||||
case "negative_end":
|
||||
for p, v := range r.ResultInfos {
|
||||
v.Offset = -1
|
||||
r.ResultInfos[p] = v
|
||||
}
|
||||
case "end_top_error":
|
||||
return r, context.DeadlineExceeded
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func TestMonitoringKafkaNeverClampsMissingInvalidOrDeniedOffsets(t *testing.T) {
|
||||
scope := DiagnosticScope{ID: "worker_events", Group: "private-group", Topics: []string{"jobs.worker-events"}}
|
||||
for _, problem := range []string{"", "topic_denied", "group_denied", "no_topic", "no_group", "no_commit", "negative", "ahead", "behind_retention", "partition_denied", "commit_top_error", "missing_end", "end_denied", "negative_end", "end_top_error"} {
|
||||
t.Run(problem, func(t *testing.T) {
|
||||
admin := &fakeDiagnosticAdmin{problem: problem}
|
||||
out := diagnoseKafka(t.Context(), admin, time.Now(), []DiagnosticScope{scope})
|
||||
if problem == "" {
|
||||
if *out.Metrics[0].Count != 6 || *out.Metrics[1].Count != 1 || out.Coverage != "complete" || strings.Join(admin.calls, ",") != "topics,groups,committed,latest,earliest" {
|
||||
t.Fatal(out, admin.calls)
|
||||
}
|
||||
} else if len(out.Metrics) != 1 || out.Metrics[0].Availability != models.MonitoringUnavailable || out.Metrics[0].Count != nil || out.Coverage != "unavailable" {
|
||||
t.Fatal("invalid observation became zero lag", out)
|
||||
}
|
||||
raw, _ := json.Marshal(out)
|
||||
if strings.Contains(string(raw), "private") {
|
||||
t.Fatal("broker credentials/identities leaked", string(raw))
|
||||
}
|
||||
})
|
||||
}
|
||||
var bus *KafkaBus
|
||||
out := bus.Diagnose(t.Context(), time.Now(), []DiagnosticScope{scope})
|
||||
if out.Metrics[0].Count != nil || out.Metrics[0].Reason != "dependency_missing" {
|
||||
t.Fatal(out)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
)
|
||||
|
||||
func natsDiagnosticReason(err error) string {
|
||||
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
|
||||
return "timeout"
|
||||
}
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) || errors.Is(err, jetstream.ErrConsumerNotFound) {
|
||||
return "resource_absent"
|
||||
}
|
||||
if strings.Contains(strings.ToLower(err.Error()), "permission") || strings.Contains(strings.ToLower(err.Error()), "authorization") {
|
||||
return "permission_denied"
|
||||
}
|
||||
return "collection_failed"
|
||||
}
|
||||
|
||||
func (b *NATSBus) Diagnose(ctx context.Context, at time.Time, scopes []DiagnosticScope) models.MonitoringSource {
|
||||
out := models.MonitoringSource{Metrics: []models.MonitoringMetric{}}
|
||||
var measured int64
|
||||
for _, scope := range scopes {
|
||||
if b == nil || b.js == nil || b.nc == nil || !b.nc.IsConnected() {
|
||||
out.Metrics = append(out.Metrics, diagnosticUnavailable(scope, at, "dependency_missing"))
|
||||
continue
|
||||
}
|
||||
consumer, err := b.js.Consumer(ctx, b.stream, b.durable(scope.Group, scope.Topics))
|
||||
if err != nil {
|
||||
out.Metrics = append(out.Metrics, diagnosticUnavailable(scope, at, natsDiagnosticReason(err)))
|
||||
continue
|
||||
}
|
||||
info, err := consumer.Info(ctx)
|
||||
if err != nil {
|
||||
out.Metrics = append(out.Metrics, diagnosticUnavailable(scope, at, natsDiagnosticReason(err)))
|
||||
continue
|
||||
}
|
||||
if info == nil {
|
||||
out.Metrics = append(out.Metrics, diagnosticUnavailable(scope, at, "collection_failed"))
|
||||
continue
|
||||
}
|
||||
if info.NumPending > uint64(^uint64(0)>>1) || info.NumAckPending < 0 || info.NumRedelivered < 0 {
|
||||
out.Metrics = append(out.Metrics, diagnosticUnavailable(scope, at, "collection_failed"))
|
||||
continue
|
||||
}
|
||||
measured++
|
||||
out.Metrics = append(out.Metrics,
|
||||
diagnosticMetric(scope, at, "pending", "messages", "JetStream messages awaiting consumer delivery, not Kafka committed lag or provider sends.", int64(info.NumPending)),
|
||||
diagnosticMetric(scope, at, "ack_pending", "messages", "Delivered messages waiting for acknowledgement; distinct from pending.", int64(info.NumAckPending)),
|
||||
diagnosticMetric(scope, at, "redelivered", "messages", "Current redelivered message count, not a lifetime or recent retry rate.", int64(info.NumRedelivered)),
|
||||
)
|
||||
}
|
||||
diagnosticCoverage(&out, measured, int64(len(scopes)))
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
)
|
||||
|
||||
func TestMonitoringNATSExistingConsumerReadOnlyAndAbsentScopes(t *testing.T) {
|
||||
s, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go s.Start()
|
||||
t.Cleanup(s.Shutdown)
|
||||
if !s.ReadyForConnections(5 * time.Second) {
|
||||
t.Fatal("nats not ready")
|
||||
}
|
||||
nc, err := nats.Connect(s.ClientURL(), nats.NoReconnect())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stream, err := js.CreateStream(t.Context(), jetstream.StreamConfig{Name: "monitoring", Subjects: []string{"warmbly.>"}, Storage: jetstream.MemoryStorage})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := &NATSBus{nc: nc, js: js, stream: "monitoring"}
|
||||
scope := DiagnosticScope{ID: "worker_events", Group: "consumer-group", Topics: []string{"jobs.worker-events"}}
|
||||
consumer, err := stream.CreateConsumer(t.Context(), jetstream.ConsumerConfig{Durable: b.durable(scope.Group, scope.Topics), AckPolicy: jetstream.AckExplicitPolicy, FilterSubject: "warmbly.jobs.worker-events"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 2; i++ {
|
||||
if _, err := js.Publish(t.Context(), "warmbly.jobs.worker-events", []byte("private payload")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
before, err := consumer.Info(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
streamBefore, err := stream.Info(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := b.Diagnose(t.Context(), time.Now(), []DiagnosticScope{scope, {ID: "missing", Group: "absent", Topics: scope.Topics}})
|
||||
if result.Coverage != "partial" || *result.MeasuredScopes != 1 || *result.ExpectedScopes != 2 || len(result.Metrics) != 4 {
|
||||
t.Fatal(result)
|
||||
}
|
||||
for i, count := range []int64{2, 0, 0} {
|
||||
if result.Metrics[i].Count == nil || *result.Metrics[i].Count != count {
|
||||
t.Fatal(result.Metrics)
|
||||
}
|
||||
}
|
||||
if result.Metrics[3].Availability != models.MonitoringUnavailable || result.Metrics[3].Reason != "resource_absent" || result.Metrics[3].Count != nil {
|
||||
t.Fatal(result.Metrics[3])
|
||||
}
|
||||
after, err := consumer.Info(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
streamAfter, err := stream.Info(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(before.Config, after.Config) || !reflect.DeepEqual(before.Delivered, after.Delivered) || !reflect.DeepEqual(before.AckFloor, after.AckFloor) || before.NumPending != after.NumPending || before.NumAckPending != after.NumAckPending || !reflect.DeepEqual(streamBefore.State, streamAfter.State) {
|
||||
t.Fatal("diagnostics altered stream or delivery state")
|
||||
}
|
||||
var absent *NATSBus
|
||||
if result := absent.Diagnose(context.Background(), time.Now(), []DiagnosticScope{scope}); result.Metrics[0].Count != nil || result.Metrics[0].Reason != "dependency_missing" {
|
||||
t.Fatal(result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitoringDiagnosticsContainOnlyReadAPIs(t *testing.T) {
|
||||
for _, path := range []string{"nats_diagnostics.go", "kafka_diagnostics.go"} {
|
||||
file, err := parser.ParseFile(token.NewFileSet(), path, nil, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
forbidden := map[string]bool{"Subscribe": true, "SubscribeTopics": true, "Fetch": true, "Consume": true, "Ack": true, "Nak": true, "Publish": true, "CreateConsumer": true, "CreateOrUpdateConsumer": true, "CreateStream": true, "CreateOrUpdateStream": true, "CreateTopics": true, "AlterConsumerGroupOffsets": true, "NewConsumer": true, "NewAdminClient": true, "Reconnect": true, "ensureStream": true, "adminClient": true}
|
||||
ast.Inspect(file, func(node ast.Node) bool {
|
||||
if call, ok := node.(*ast.CallExpr); ok {
|
||||
if sel, ok := call.Fun.(*ast.SelectorExpr); ok && forbidden[sel.Sel.Name] {
|
||||
t.Errorf("%s calls forbidden %s", path, sel.Sel.Name)
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
type MonitoringAvailability string
|
||||
type MonitoringCondition string
|
||||
|
||||
const (
|
||||
MonitoringFresh MonitoringAvailability = "fresh"
|
||||
MonitoringStale MonitoringAvailability = "stale"
|
||||
MonitoringUnavailable MonitoringAvailability = "unavailable"
|
||||
MonitoringHealthy MonitoringCondition = "healthy"
|
||||
MonitoringUnknown MonitoringCondition = "unknown"
|
||||
MonitoringNoEvidence MonitoringCondition = "no_recent_evidence"
|
||||
MonitoringRecentFailure MonitoringCondition = "recent_failure"
|
||||
MonitoringPersistentFailure MonitoringCondition = "persistent_failure"
|
||||
MonitoringRecoveryUnverified MonitoringCondition = "recovery_unverified"
|
||||
MonitoringSafetyHold MonitoringCondition = "safety_hold"
|
||||
MonitoringBacklog MonitoringCondition = "backlog"
|
||||
MonitoringInFlight MonitoringCondition = "in_flight"
|
||||
MonitoringSchedule MonitoringCondition = "waiting_schedule"
|
||||
MonitoringRetry MonitoringCondition = "waiting_retry"
|
||||
MonitoringBudget MonitoringCondition = "budget"
|
||||
MonitoringDailyLimit MonitoringCondition = "daily_limit"
|
||||
MonitoringTelemetryStale MonitoringCondition = "stale_telemetry"
|
||||
)
|
||||
|
||||
// MonitoringMetric carries aggregates only; source-specific counts may overlap.
|
||||
type MonitoringMetric struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Unit string `json:"unit"`
|
||||
ScopeID string `json:"scope_id,omitempty"`
|
||||
Availability MonitoringAvailability `json:"availability"`
|
||||
Condition MonitoringCondition `json:"condition"`
|
||||
Severity string `json:"severity"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Note string `json:"note"`
|
||||
Count *int64 `json:"count"`
|
||||
AffectedMailboxes *int64 `json:"affected_mailboxes"`
|
||||
AffectedOrganizations *int64 `json:"affected_organizations"`
|
||||
UnknownOrganizationRows *int64 `json:"unknown_organization_rows"`
|
||||
ObservedAt *time.Time `json:"observed_at"`
|
||||
EvidenceAt *time.Time `json:"evidence_at"`
|
||||
LatestEvidenceAt *time.Time `json:"latest_evidence_at"`
|
||||
EvidenceAgeSeconds *int64 `json:"evidence_age_seconds"`
|
||||
NextEligibleAt *time.Time `json:"next_eligible_at"`
|
||||
WindowStart *time.Time `json:"window_start"`
|
||||
WindowEnd *time.Time `json:"window_end"`
|
||||
ThresholdSeconds *int64 `json:"threshold_seconds"`
|
||||
Investigate string `json:"investigate,omitempty"`
|
||||
}
|
||||
|
||||
type MonitoringSource struct {
|
||||
ID string `json:"id"`
|
||||
Availability MonitoringAvailability `json:"availability"`
|
||||
Coverage string `json:"coverage"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
ObservedAt *time.Time `json:"observed_at"`
|
||||
CheckedAt time.Time `json:"checked_at"`
|
||||
MeasuredScopes *int64 `json:"measured_scopes"`
|
||||
ExpectedScopes *int64 `json:"expected_scopes"`
|
||||
Metrics []MonitoringMetric `json:"metrics"`
|
||||
}
|
||||
|
||||
type MonitoringSnapshot struct {
|
||||
Version string `json:"version"`
|
||||
CheckedAt time.Time `json:"checked_at"`
|
||||
RefreshAfter time.Time `json:"refresh_after"`
|
||||
Coverage string `json:"coverage"`
|
||||
Sources []MonitoringSource `json:"sources"`
|
||||
}
|
||||
@@ -422,6 +422,9 @@ type DashboardAnalytics struct {
|
||||
type DashboardFilter struct {
|
||||
CampaignIDs []uuid.UUID
|
||||
FolderIDs []uuid.UUID
|
||||
// AllowedCampaigns and AllowedMailboxes hold a restricted caller to their grants; set by the server, nil allows all.
|
||||
AllowedCampaigns []uuid.UUID
|
||||
AllowedMailboxes []uuid.UUID
|
||||
}
|
||||
|
||||
// Empty reports a filter that leaves the dashboard workspace-wide.
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// AccessScope is which resources a member's role applies to.
|
||||
type AccessScope string
|
||||
|
||||
const (
|
||||
// AccessScopeWorkspace is every resource in the workspace.
|
||||
AccessScopeWorkspace AccessScope = "workspace"
|
||||
// AccessScopeRestricted is only the resources granted to the member.
|
||||
AccessScopeRestricted AccessScope = "restricted"
|
||||
)
|
||||
|
||||
// Valid reports whether s is a known scope.
|
||||
func (s AccessScope) Valid() bool {
|
||||
return s == AccessScopeWorkspace || s == AccessScopeRestricted
|
||||
}
|
||||
|
||||
// RestrictedPermissionMask is the most a restricted member's role can grant; migration 000278's trigger holds the same value.
|
||||
const RestrictedPermissionMask = PermViewCampaigns | PermViewAnalytics | PermAccessUnibox
|
||||
|
||||
// MaxAccessGrants bounds each grant list on one member.
|
||||
const MaxAccessGrants = 500
|
||||
|
||||
// MemberAccess is the resource scope set on a member or an invitation.
|
||||
type MemberAccess struct {
|
||||
Scope AccessScope `json:"scope"`
|
||||
// FolderIDs grant every campaign the folder holds at the time of the request.
|
||||
FolderIDs []uuid.UUID `json:"campaign_folder_ids"`
|
||||
CampaignIDs []uuid.UUID `json:"campaign_ids"`
|
||||
EmailAccountIDs []uuid.UUID `json:"email_account_ids"`
|
||||
}
|
||||
|
||||
// Restricted reports whether a is a restricted scope.
|
||||
func (a *MemberAccess) Restricted() bool {
|
||||
return a != nil && a.Scope == AccessScopeRestricted
|
||||
}
|
||||
|
||||
// Normalize deduplicates the grant lists and clears them for a workspace scope.
|
||||
func (a *MemberAccess) Normalize() {
|
||||
if a.Scope == "" {
|
||||
a.Scope = AccessScopeWorkspace
|
||||
}
|
||||
if a.Scope != AccessScopeRestricted {
|
||||
a.FolderIDs, a.CampaignIDs, a.EmailAccountIDs = []uuid.UUID{}, []uuid.UUID{}, []uuid.UUID{}
|
||||
return
|
||||
}
|
||||
a.FolderIDs = dedupeUUIDs(a.FolderIDs)
|
||||
a.CampaignIDs = dedupeUUIDs(a.CampaignIDs)
|
||||
a.EmailAccountIDs = dedupeUUIDs(a.EmailAccountIDs)
|
||||
}
|
||||
|
||||
// WorkspaceAccess is the scope every member has unless restricted.
|
||||
func WorkspaceAccess() *MemberAccess {
|
||||
return &MemberAccess{Scope: AccessScopeWorkspace, FolderIDs: []uuid.UUID{}, CampaignIDs: []uuid.UUID{}, EmailAccountIDs: []uuid.UUID{}}
|
||||
}
|
||||
|
||||
// ResourceScope is what a restricted member may reach, resolved for one request.
|
||||
type ResourceScope struct {
|
||||
// Campaigns is every campaign granted directly or held by a granted folder.
|
||||
Campaigns []uuid.UUID
|
||||
// Folders are the granted folders and every folder holding a campaign in scope.
|
||||
Folders []uuid.UUID
|
||||
Mailboxes []uuid.UUID
|
||||
// MailboxTags are the tags on mailboxes in scope.
|
||||
MailboxTags []uuid.UUID
|
||||
}
|
||||
|
||||
// HasCampaign reports whether id is in scope.
|
||||
func (s *ResourceScope) HasCampaign(id uuid.UUID) bool { return containsUUID(s.Campaigns, id) }
|
||||
|
||||
// HasMailbox reports whether id is in scope.
|
||||
func (s *ResourceScope) HasMailbox(id uuid.UUID) bool { return containsUUID(s.Mailboxes, id) }
|
||||
|
||||
// HasFolder reports whether id is in scope.
|
||||
func (s *ResourceScope) HasFolder(id uuid.UUID) bool { return containsUUID(s.Folders, id) }
|
||||
|
||||
func containsUUID(ids []uuid.UUID, id uuid.UUID) bool {
|
||||
for _, v := range ids {
|
||||
if v == id {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// NoneMatch stands in for an empty restriction where an empty list would mean unrestricted; uuid.Nil names no row.
|
||||
func NoneMatch() []uuid.UUID {
|
||||
return []uuid.UUID{uuid.Nil}
|
||||
}
|
||||
|
||||
// SetMemberAccessRequest is PUT /organization/members/:id/access.
|
||||
type SetMemberAccessRequest struct {
|
||||
Scope AccessScope `json:"scope" binding:"required"`
|
||||
FolderIDs []uuid.UUID `json:"campaign_folder_ids"`
|
||||
CampaignIDs []uuid.UUID `json:"campaign_ids"`
|
||||
EmailAccountIDs []uuid.UUID `json:"email_account_ids"`
|
||||
}
|
||||
|
||||
// Access is the request as a MemberAccess.
|
||||
func (r *SetMemberAccessRequest) Access() *MemberAccess {
|
||||
return &MemberAccess{Scope: r.Scope, FolderIDs: r.FolderIDs, CampaignIDs: r.CampaignIDs, EmailAccountIDs: r.EmailAccountIDs}
|
||||
}
|
||||
|
||||
// SuggestedSender is a mailbox sending for the campaigns an administrator is about to grant.
|
||||
type SuggestedSender struct {
|
||||
ID uuid.UUID `json:"id"`
|
||||
Email string `json:"email"`
|
||||
Name string `json:"name"`
|
||||
CampaignIDs []uuid.UUID `json:"campaign_ids"`
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// The trigger in migration 000278 masks with a literal; it has to be this mask.
|
||||
func TestRestrictedPermissionMaskMatchesTheTrigger(t *testing.T) {
|
||||
sql, err := os.ReadFile("../infrastructure/db/migrations/000278_member_access_scope.up.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := fmt.Sprintf("NEW.permissions & %d;", uint16(RestrictedPermissionMask))
|
||||
if !strings.Contains(string(sql), want) {
|
||||
t.Fatalf("migration 000278 does not mask with %q", want)
|
||||
}
|
||||
if RestrictedPermissionMask&(PermManageCampaigns|PermManageEmails|PermManageTeam|PermViewContacts|PermUseAI) != 0 {
|
||||
t.Fatal("the restricted mask grants more than reading campaigns, analytics and the inbox")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemberAccessNormalize(t *testing.T) {
|
||||
id := uuid.New()
|
||||
a := &MemberAccess{Scope: AccessScopeWorkspace, CampaignIDs: []uuid.UUID{id}}
|
||||
a.Normalize()
|
||||
if len(a.CampaignIDs) != 0 || a.CampaignIDs == nil {
|
||||
t.Fatalf("a workspace scope kept grants: %+v", a)
|
||||
}
|
||||
r := &MemberAccess{Scope: AccessScopeRestricted, CampaignIDs: []uuid.UUID{id, id, uuid.Nil}}
|
||||
r.Normalize()
|
||||
if len(r.CampaignIDs) != 1 || r.FolderIDs == nil || r.EmailAccountIDs == nil {
|
||||
t.Fatalf("restricted normalize = %+v", r)
|
||||
}
|
||||
owner := &OrganizationMember{Role: string(RoleOwner), AccessScope: AccessScopeRestricted}
|
||||
if owner.IsRestricted() {
|
||||
t.Fatal("the owner read as restricted")
|
||||
}
|
||||
}
|
||||
@@ -86,6 +86,9 @@ type OrganizationMember struct {
|
||||
InvitedBy *uuid.UUID `json:"invited_by,omitempty"`
|
||||
InvitedAt time.Time `json:"invited_at"`
|
||||
AcceptedAt *time.Time `json:"accepted_at,omitempty"`
|
||||
// AccessScope is which resources the role applies to; Access is filled where the grants are listed.
|
||||
AccessScope AccessScope `json:"access_scope"`
|
||||
Access *MemberAccess `json:"access,omitempty"`
|
||||
|
||||
// Joined data
|
||||
User *User `json:"user,omitempty"`
|
||||
@@ -115,6 +118,11 @@ func (m *OrganizationMember) IsOwner() bool {
|
||||
return m.Role == string(RoleOwner)
|
||||
}
|
||||
|
||||
// IsRestricted reports whether the member reaches only the resources granted to them.
|
||||
func (m *OrganizationMember) IsRestricted() bool {
|
||||
return m != nil && !m.IsOwner() && m.AccessScope == AccessScopeRestricted
|
||||
}
|
||||
|
||||
// OrganizationInvitation represents a pending invitation to join an organization
|
||||
type OrganizationInvitation struct {
|
||||
ID uuid.UUID `json:"id"`
|
||||
@@ -128,6 +136,7 @@ type OrganizationInvitation struct {
|
||||
Token string `json:"-"` // Never expose token in JSON
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Access *MemberAccess `json:"access,omitempty"`
|
||||
|
||||
// Joined data
|
||||
Organization *Organization `json:"organization,omitempty"`
|
||||
@@ -200,6 +209,8 @@ type InviteMemberRequest struct {
|
||||
// RoleID stays accepted as a single-role shorthand.
|
||||
RoleIDs []uuid.UUID `json:"role_ids,omitempty"`
|
||||
RoleID *uuid.UUID `json:"role_id,omitempty"`
|
||||
// Access restricts the invitee from the moment they join; absent is the entire workspace.
|
||||
Access *MemberAccess `json:"access,omitempty"`
|
||||
}
|
||||
|
||||
// Resolved returns the requested role ids, merging the single-role shorthand.
|
||||
|
||||
@@ -309,7 +309,7 @@ func TestLiveCloudLinkStandingGatesTheInstance(t *testing.T) {
|
||||
t.Fatalf("GetCloudStanding = %+v", info)
|
||||
}
|
||||
|
||||
summary, xerr := NewAnalyticsRepository(handle).GetAccountHealthSummary(ctx, f.org)
|
||||
summary, xerr := NewAnalyticsRepository(handle).GetAccountHealthSummary(ctx, f.org, nil)
|
||||
if xerr != nil || summary.ErrorAccounts != 1 {
|
||||
t.Fatalf("GetAccountHealthSummary = %+v, %v; want the quarantined mailbox in error", summary, xerr)
|
||||
}
|
||||
|
||||
@@ -98,7 +98,7 @@ func TestLiveDashboardScopeNarrowsCampaignSections(t *testing.T) {
|
||||
t.Fatalf("echoed scope = %+v, want only this workspace's campaign and folder", view)
|
||||
}
|
||||
|
||||
overall, xerr := repo.GetDashboardOverallStats(ctx, orgID, from, to, scope)
|
||||
overall, xerr := repo.GetDashboardOverallStats(ctx, orgID, from, to, scope, nil)
|
||||
if xerr != nil {
|
||||
t.Fatalf("GetDashboardOverallStats: %v", xerr)
|
||||
}
|
||||
@@ -154,14 +154,14 @@ func TestLiveDashboardScopeNarrowsCampaignSections(t *testing.T) {
|
||||
if view.CampaignCount != 0 || len(view.Campaigns) != 0 || len(view.Folders) != 0 {
|
||||
t.Fatalf("foreign ids resolved to %+v, want nothing", view)
|
||||
}
|
||||
overall, xerr = repo.GetDashboardOverallStats(ctx, orgID, from, to, scope)
|
||||
overall, xerr = repo.GetDashboardOverallStats(ctx, orgID, from, to, scope, nil)
|
||||
if xerr != nil {
|
||||
t.Fatalf("GetDashboardOverallStats: %v", xerr)
|
||||
}
|
||||
if overall.TotalEmailsSent != 0 || overall.ActiveCampaigns != 0 {
|
||||
t.Fatalf("an empty scope counted sent %d, active %d; want nothing", overall.TotalEmailsSent, overall.ActiveCampaigns)
|
||||
}
|
||||
whole, xerr := repo.GetDashboardOverallStats(ctx, orgID, from, to, nil)
|
||||
whole, xerr := repo.GetDashboardOverallStats(ctx, orgID, from, to, nil, nil)
|
||||
if xerr != nil {
|
||||
t.Fatalf("GetDashboardOverallStats: %v", xerr)
|
||||
}
|
||||
|
||||
@@ -160,7 +160,7 @@ func TestLiveListCursorOtherListsReturnEveryRowOnce(t *testing.T) {
|
||||
s := cursor.String()
|
||||
c = &s
|
||||
}
|
||||
res, err := campaignRepo.Search(ctx, f.org.String(), "", c, nil, "", int32(limit))
|
||||
res, err := campaignRepo.Search(ctx, f.org.String(), "", c, nil, "", int32(limit), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("campaign Search: %v", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
)
|
||||
|
||||
// Issue #861: a restricted member reaches only their grants, read-only, whatever writes their row.
|
||||
//
|
||||
// WARMBLY_TEST_DB=postgres://warmbly:warmbly@localhost:15432/warmbly_dev?sslmode=disable \
|
||||
// go test ./internal/repository/ -run LiveMemberAccess -v
|
||||
func TestLiveMemberAccessScope(t *testing.T) {
|
||||
handle, pool := liveContactDB(t)
|
||||
requireSchemaVersion(t, pool, 278)
|
||||
ctx := context.Background()
|
||||
repo := NewOrganizationRepository(pool)
|
||||
|
||||
owner, member, invitee := uuid.New(), uuid.New(), uuid.New()
|
||||
org, foreignOrg := uuid.New(), uuid.New()
|
||||
inFolder, direct, outside, foreignCampaign := uuid.New(), uuid.New(), uuid.New(), uuid.New()
|
||||
folder, mailbox, otherMailbox, foreignMailbox := uuid.New(), uuid.New(), uuid.New(), uuid.New()
|
||||
role := uuid.New()
|
||||
exec := func(sql string, args ...any) {
|
||||
t.Helper()
|
||||
if _, err := pool.Exec(ctx, sql, args...); err != nil {
|
||||
t.Fatalf("fixture %q: %v", sql[:min(60, len(sql))], err)
|
||||
}
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
c := context.Background()
|
||||
_, _ = pool.Exec(c, `DELETE FROM campaigns WHERE organization_id = ANY($1)`, []uuid.UUID{org, foreignOrg})
|
||||
_, _ = pool.Exec(c, `DELETE FROM email_accounts WHERE organization_id = ANY($1)`, []uuid.UUID{org, foreignOrg})
|
||||
_, _ = pool.Exec(c, `DELETE FROM organizations WHERE id = ANY($1)`, []uuid.UUID{org, foreignOrg})
|
||||
_, _ = pool.Exec(c, `DELETE FROM users WHERE id = ANY($1)`, []uuid.UUID{owner, member, invitee})
|
||||
})
|
||||
for _, u := range []uuid.UUID{owner, member, invitee} {
|
||||
exec(`INSERT INTO users (id, first_name, last_name, email, password_hash) VALUES ($1, 'Scope', 'Live', $2, 'x')`, u, "i861-"+u.String()[:8]+"@test.local")
|
||||
}
|
||||
for _, o := range []uuid.UUID{org, foreignOrg} {
|
||||
exec(`INSERT INTO organizations (id, name, slug, owner_user_id) VALUES ($1, 'Issue 861', $2, $3)`, o, "i861-"+o.String()[:8], owner)
|
||||
}
|
||||
exec(`INSERT INTO organization_roles (id, organization_id, name, permissions) VALUES ($1, $2, 'Manager861', $3)`,
|
||||
role, org, int64(int16(models.GetRolePermissions(models.RoleManager))))
|
||||
exec(`INSERT INTO organization_members (organization_id, user_id, role, permissions, accepted_at) VALUES ($1, $2, 'owner', -1, NOW())`, org, owner)
|
||||
exec(`INSERT INTO organization_members (organization_id, user_id, role, role_id, permissions, accepted_at) VALUES ($1, $2, 'Manager861', $3, $4, NOW())`,
|
||||
org, member, role, int64(int16(models.GetRolePermissions(models.RoleManager))))
|
||||
exec(`INSERT INTO organization_member_roles (organization_id, user_id, role_id) VALUES ($1, $2, $3)`, org, member, role)
|
||||
for _, c := range []struct{ id, org uuid.UUID }{{inFolder, org}, {direct, org}, {outside, org}, {foreignCampaign, foreignOrg}} {
|
||||
exec(`INSERT INTO campaigns (id, user_id, organization_id, name, description, days, updated_at, created_at) VALUES ($1, $2, $3, 'Scope', '', 62, NOW(), NOW())`, c.id, owner, c.org)
|
||||
}
|
||||
exec(`INSERT INTO folders (id, organization_id, title, color, position) VALUES ($1, $2, 'Client A', '#000000', 0)`, folder, org)
|
||||
exec(`INSERT INTO campaign_folders (campaign_id, folder_id) VALUES ($1, $2)`, inFolder, folder)
|
||||
for _, m := range []struct{ id, org uuid.UUID }{{mailbox, org}, {otherMailbox, org}, {foreignMailbox, foreignOrg}} {
|
||||
exec(`INSERT INTO email_accounts (id, user_id, organization_id, email, name, signature_plain, signature_html, provider) VALUES ($1, $2, $3, $4, 'Scope', '', '', 'smtp_imap')`,
|
||||
m.id, owner, m.org, "i861-"+m.id.String()[:8]+"@test.local")
|
||||
}
|
||||
|
||||
restricted := &models.MemberAccess{Scope: models.AccessScopeRestricted, FolderIDs: []uuid.UUID{folder}, CampaignIDs: []uuid.UUID{direct}, EmailAccountIDs: []uuid.UUID{mailbox}}
|
||||
|
||||
t.Run("foreign ids are counted out", func(t *testing.T) {
|
||||
f, c, m, err := repo.CountOwnedAccessResources(ctx, org, &models.MemberAccess{
|
||||
FolderIDs: []uuid.UUID{folder}, CampaignIDs: []uuid.UUID{direct, foreignCampaign}, EmailAccountIDs: []uuid.UUID{foreignMailbox},
|
||||
})
|
||||
if err != nil || f != 1 || c != 1 || m != 0 {
|
||||
t.Fatalf("owned = %d/%d/%d, %v; want 1/1/0", f, c, m, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("restricting masks the role to read bits and resolves the grants", func(t *testing.T) {
|
||||
if err := repo.SetMemberAccess(ctx, org, member, restricted, owner); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := repo.GetMember(ctx, org, member)
|
||||
if err != nil || m == nil {
|
||||
t.Fatalf("member: %v", err)
|
||||
}
|
||||
if !m.IsRestricted() || m.Permissions&^models.RestrictedPermissionMask != 0 {
|
||||
t.Fatalf("member = scope %q perms %b; want restricted within %b", m.AccessScope, m.Permissions, models.RestrictedPermissionMask)
|
||||
}
|
||||
scope, err := repo.ResolveMemberScope(ctx, org, member)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !scope.HasCampaign(inFolder) || !scope.HasCampaign(direct) || scope.HasCampaign(outside) || len(scope.Campaigns) != 2 {
|
||||
t.Fatalf("campaigns = %v", scope.Campaigns)
|
||||
}
|
||||
if !scope.HasMailbox(mailbox) || len(scope.Mailboxes) != 1 || !scope.HasFolder(folder) {
|
||||
t.Fatalf("scope = %+v", scope)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a role edit cannot widen a restricted member", func(t *testing.T) {
|
||||
exec(`UPDATE organization_members SET permissions = -1 WHERE organization_id = $1 AND user_id = $2`, org, member)
|
||||
m, _ := repo.GetMember(ctx, org, member)
|
||||
if m.Permissions&^models.RestrictedPermissionMask != 0 {
|
||||
t.Fatalf("permissions = %b after a direct write", m.Permissions)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("folder grants follow the folder's contents", func(t *testing.T) {
|
||||
exec(`INSERT INTO campaign_folders (campaign_id, folder_id) VALUES ($1, $2)`, outside, folder)
|
||||
scope, _ := repo.ResolveMemberScope(ctx, org, member)
|
||||
if !scope.HasCampaign(outside) {
|
||||
t.Fatal("a campaign moved into a granted folder is not in scope")
|
||||
}
|
||||
exec(`DELETE FROM campaign_folders WHERE campaign_id = $1 AND folder_id = $2`, outside, folder)
|
||||
scope, _ = repo.ResolveMemberScope(ctx, org, member)
|
||||
if scope.HasCampaign(outside) {
|
||||
t.Fatal("a campaign moved out of a granted folder stayed in scope")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("the scope lists only allowed campaigns", func(t *testing.T) {
|
||||
scope, _ := repo.ResolveMemberScope(ctx, org, member)
|
||||
res, err := NewCampaignRepostory(handle).Search(ctx, org.String(), "", nil, nil, "", 50, scope.Campaigns)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(res.Data) != 2 || res.Pagination.Total == nil || *res.Pagination.Total != 2 {
|
||||
t.Fatalf("search = %d campaigns (total %v), want 2", len(res.Data), res.Pagination.Total)
|
||||
}
|
||||
none, err := NewCampaignRepostory(handle).Search(ctx, org.String(), "", nil, nil, "", 50, models.NoneMatch())
|
||||
if err != nil || len(none.Data) != 0 {
|
||||
t.Fatalf("an empty grant listed %d campaigns, %v", len(none.Data), err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("widening restores the role and drops the grants", func(t *testing.T) {
|
||||
if err := repo.SetMemberAccess(ctx, org, member, models.WorkspaceAccess(), owner); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, _ := repo.GetMember(ctx, org, member)
|
||||
if m.IsRestricted() || m.Permissions != models.GetRolePermissions(models.RoleManager) {
|
||||
t.Fatalf("member = scope %q perms %b, want the role back", m.AccessScope, m.Permissions)
|
||||
}
|
||||
a, _ := repo.GetMemberAccess(ctx, org, member)
|
||||
if a.Restricted() || len(a.CampaignIDs)+len(a.FolderIDs)+len(a.EmailAccountIDs) != 0 {
|
||||
t.Fatalf("access = %+v after widening", a)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("the owner is never restricted", func(t *testing.T) {
|
||||
exec(`UPDATE organization_members SET access_scope = 'restricted' WHERE organization_id = $1 AND user_id = $2`, org, owner)
|
||||
m, _ := repo.GetMember(ctx, org, owner)
|
||||
if m.AccessScope != models.AccessScopeWorkspace {
|
||||
t.Fatalf("owner scope = %q", m.AccessScope)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an invitee joins already restricted, without foreign grants", func(t *testing.T) {
|
||||
inv := &models.OrganizationInvitation{
|
||||
ID: uuid.New(), OrganizationID: org, Email: "i861-" + invitee.String()[:8] + "@test.local", Role: "Manager861", RoleID: &role,
|
||||
Permissions: models.GetRolePermissions(models.RoleManager), InvitedBy: owner, Token: uuid.NewString(),
|
||||
ExpiresAt: time.Now().Add(time.Hour), CreatedAt: time.Now(),
|
||||
Access: &models.MemberAccess{Scope: models.AccessScopeRestricted, EmailAccountIDs: []uuid.UUID{mailbox, foreignMailbox}},
|
||||
}
|
||||
if err := repo.CreateInvitation(ctx, inv); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stored, err := repo.GetInvitationByID(ctx, inv.ID)
|
||||
if err != nil || stored == nil || !stored.Access.Restricted() {
|
||||
t.Fatalf("stored invitation access = %+v, %v", stored, err)
|
||||
}
|
||||
now := time.Now()
|
||||
joined := &models.OrganizationMember{ID: uuid.New(), OrganizationID: org, UserID: invitee, Role: "Manager861", RoleID: &role,
|
||||
Permissions: models.GetRolePermissions(models.RoleManager), InvitedBy: &owner, InvitedAt: now, AcceptedAt: &now, Access: stored.Access}
|
||||
if err := repo.AddMemberWithRoles(ctx, joined, []uuid.UUID{role}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, _ := repo.GetMember(ctx, org, invitee)
|
||||
if !m.IsRestricted() || m.Permissions&^models.RestrictedPermissionMask != 0 {
|
||||
t.Fatalf("invitee = scope %q perms %b", m.AccessScope, m.Permissions)
|
||||
}
|
||||
scope, _ := repo.ResolveMemberScope(ctx, org, invitee)
|
||||
if len(scope.Mailboxes) != 1 || scope.Mailboxes[0] != mailbox {
|
||||
t.Fatalf("invitee mailboxes = %v, want only the workspace's own", scope.Mailboxes)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("deleting a granted resource drops its grant", func(t *testing.T) {
|
||||
exec(`DELETE FROM email_accounts WHERE id = $1`, mailbox)
|
||||
scope, _ := repo.ResolveMemberScope(ctx, org, invitee)
|
||||
if len(scope.Mailboxes) != 0 {
|
||||
t.Fatalf("mailboxes = %v after deletion", scope.Mailboxes)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -48,11 +48,12 @@ type AnalyticsRepository interface {
|
||||
// Dashboard analytics
|
||||
// ResolveDashboardScope resolves a filter to this workspace's campaigns, deduplicated; foreign ids drop out.
|
||||
ResolveDashboardScope(ctx context.Context, orgID uuid.UUID, filter models.DashboardFilter) (*models.DashboardScope, *models.CampaignScope, *errx.Error)
|
||||
GetDashboardOverallStats(ctx context.Context, orgID uuid.UUID, from, to time.Time, scope *models.CampaignScope) (*models.DashboardOverallStats, *errx.Error)
|
||||
// mailboxes limits the mailbox counts; nil is every mailbox.
|
||||
GetDashboardOverallStats(ctx context.Context, orgID uuid.UUID, from, to time.Time, scope *models.CampaignScope, mailboxes []uuid.UUID) (*models.DashboardOverallStats, *errx.Error)
|
||||
GetRecentActivity(ctx context.Context, orgID uuid.UUID, limit int, scope *models.CampaignScope) ([]models.RecentActivityItem, *errx.Error)
|
||||
GetTopCampaigns(ctx context.Context, orgID uuid.UUID, from, to time.Time, limit int, sortBy string, scope *models.CampaignScope) ([]models.TopCampaignStats, *errx.Error)
|
||||
GetDashboardDailyTrend(ctx context.Context, orgID uuid.UUID, from, to time.Time, scope *models.CampaignScope) ([]models.DashboardDailyStats, *errx.Error)
|
||||
GetAccountHealthSummary(ctx context.Context, orgID uuid.UUID) (*models.AccountHealthSummary, *errx.Error)
|
||||
GetAccountHealthSummary(ctx context.Context, orgID uuid.UUID, mailboxes []uuid.UUID) (*models.AccountHealthSummary, *errx.Error)
|
||||
|
||||
// Campaign hourly stats
|
||||
GetCampaignHourlyStats(ctx context.Context, campaignID uuid.UUID, date time.Time) ([]models.CampaignHourlyStats, *errx.Error)
|
||||
@@ -190,6 +191,22 @@ func inCampaignScope(column string, scope *models.CampaignScope, arg int) string
|
||||
return fmt.Sprintf(" AND %s = ANY($%d::uuid[])", column, arg)
|
||||
}
|
||||
|
||||
// inMailboxes narrows column to the allowed mailboxes as $arg; nil adds nothing.
|
||||
func inMailboxes(column string, mailboxes []uuid.UUID, arg int) string {
|
||||
if mailboxes == nil {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf(" AND %s = ANY($%d::uuid[])", column, arg)
|
||||
}
|
||||
|
||||
// mailboxArgs is the parameter inMailboxes refers to, or nothing.
|
||||
func mailboxArgs(mailboxes []uuid.UUID) []any {
|
||||
if mailboxes == nil {
|
||||
return nil
|
||||
}
|
||||
return []any{mailboxes}
|
||||
}
|
||||
|
||||
// scopeArgs is the parameter inCampaignScope refers to, or nothing.
|
||||
func scopeArgs(scope *models.CampaignScope) []any {
|
||||
if scope == nil {
|
||||
@@ -756,7 +773,8 @@ func (r *analyticsRepository) ResolveDashboardScope(ctx context.Context, orgID u
|
||||
return scope, members, nil
|
||||
}
|
||||
|
||||
func (r *analyticsRepository) GetDashboardOverallStats(ctx context.Context, orgID uuid.UUID, from, to time.Time, scope *models.CampaignScope) (*models.DashboardOverallStats, *errx.Error) {
|
||||
func (r *analyticsRepository) GetDashboardOverallStats(ctx context.Context, orgID uuid.UUID, from, to time.Time, scope *models.CampaignScope, mailboxes []uuid.UUID) (*models.DashboardOverallStats, *errx.Error) {
|
||||
mailboxArg := 4 + len(scopeArgs(scope))
|
||||
query := `
|
||||
SELECT
|
||||
COUNT(CASE WHEN ccp.sent_at IS NOT NULL AND ccp.sent_at >= $2 AND ccp.sent_at <= $3 THEN 1 END) as total_sent,
|
||||
@@ -773,14 +791,14 @@ func (r *analyticsRepository) GetDashboardOverallStats(ctx context.Context, orgI
|
||||
COUNT(CASE WHEN ccp.replied_at IS NOT NULL AND ccp.sent_at >= $2 AND ccp.sent_at <= $3 THEN 1 END) as total_replies,
|
||||
COUNT(CASE WHEN ccp.bounced_at IS NOT NULL AND ccp.sent_at >= $2 AND ccp.sent_at <= $3 THEN 1 END) as total_bounces,
|
||||
(SELECT COUNT(*) FROM campaigns ac WHERE ac.organization_id = $1 AND ac.status = 'active'` + inCampaignScope("ac.id", scope, 4) + `) as active_campaigns,
|
||||
(SELECT COUNT(*) FROM email_accounts WHERE organization_id = $1 AND status = 'active') as active_accounts
|
||||
(SELECT COUNT(*) FROM email_accounts WHERE organization_id = $1 AND status = 'active'` + inMailboxes("id", mailboxes, mailboxArg) + `) as active_accounts
|
||||
FROM campaign_contact_progress ccp
|
||||
JOIN campaigns c ON c.id = ccp.campaign_id
|
||||
JOIN sequences s ON s.id = ccp.sequence_id AND s.kind = 'email'
|
||||
WHERE c.organization_id = $1` + inCampaignScope("c.id", scope, 4) + `
|
||||
`
|
||||
|
||||
params := append([]any{orgID, from, to}, scopeArgs(scope)...)
|
||||
params := append(append([]any{orgID, from, to}, scopeArgs(scope)...), mailboxArgs(mailboxes)...)
|
||||
|
||||
var stats models.DashboardOverallStats
|
||||
err := r.DB.QueryRow(ctx, query, params...).Scan(
|
||||
@@ -1027,7 +1045,7 @@ func (r *analyticsRepository) GetDashboardDailyTrend(ctx context.Context, orgID
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func (r *analyticsRepository) GetAccountHealthSummary(ctx context.Context, orgID uuid.UUID) (*models.AccountHealthSummary, *errx.Error) {
|
||||
func (r *analyticsRepository) GetAccountHealthSummary(ctx context.Context, orgID uuid.UUID, mailboxes []uuid.UUID) (*models.AccountHealthSummary, *errx.Error) {
|
||||
query := `
|
||||
WITH account_health AS (
|
||||
SELECT CASE
|
||||
@@ -1049,7 +1067,7 @@ func (r *analyticsRepository) GetAccountHealthSummary(ctx context.Context, orgID
|
||||
FROM email_accounts ea
|
||||
LEFT JOIN LATERAL (` + warmupStandingSQL("ea.id") + `
|
||||
) wh ON true
|
||||
WHERE ea.organization_id = $1
|
||||
WHERE ea.organization_id = $1` + inMailboxes("ea.id", mailboxes, 2) + `
|
||||
)
|
||||
SELECT
|
||||
COUNT(*) as total,
|
||||
@@ -1060,7 +1078,7 @@ func (r *analyticsRepository) GetAccountHealthSummary(ctx context.Context, orgID
|
||||
`
|
||||
|
||||
var summary models.AccountHealthSummary
|
||||
err := r.DB.QueryRow(ctx, query, orgID).Scan(&summary.TotalAccounts, &summary.HealthyAccounts, &summary.WarningAccounts, &summary.ErrorAccounts)
|
||||
err := r.DB.QueryRow(ctx, query, append([]any{orgID}, mailboxArgs(mailboxes)...)...).Scan(&summary.TotalAccounts, &summary.HealthyAccounts, &summary.WarningAccounts, &summary.ErrorAccounts)
|
||||
if err != nil {
|
||||
db.CaptureError(err, query, []any{orgID}, "queryrow")
|
||||
return nil, errx.InternalError()
|
||||
|
||||
@@ -41,10 +41,10 @@ type CampaignRepository interface {
|
||||
// Search filters by name substring, folder id, and status bucket
|
||||
// ("draft" | "active" | "paused" | "completed"; paused matches every
|
||||
// paused_* variant; empty means all).
|
||||
Search(ctx context.Context, userID, query string, cursor, folder *string, status string, limit int32) (*models.CampaignsResult, error)
|
||||
Search(ctx context.Context, userID, query string, cursor, folder *string, status string, limit int32, allowed []uuid.UUID) (*models.CampaignsResult, error)
|
||||
// Overview returns status-bucket counts plus per-folder totals for the
|
||||
// campaigns browser sidebar.
|
||||
Overview(ctx context.Context, orgID string) (*models.CampaignsOverview, error)
|
||||
Overview(ctx context.Context, orgID string, allowed []uuid.UUID) (*models.CampaignsOverview, error)
|
||||
Update(ctx context.Context, orgID, query string, data *models.UpdateCampaign) (*models.Campaign, *errx.Error)
|
||||
UpdateStatus(ctx context.Context, campaignID uuid.UUID, status string) error
|
||||
UpdateStatusWithLock(ctx context.Context, campaignID uuid.UUID, status string) error
|
||||
@@ -770,7 +770,7 @@ func (r *campaignRepository) Get(ctx context.Context, orgID, id string) (*models
|
||||
return &campaign, nil
|
||||
}
|
||||
|
||||
func (r *campaignRepository) Search(ctx context.Context, orgID, query string, cursor, folder *string, status string, limit int32) (*models.CampaignsResult, error) {
|
||||
func (r *campaignRepository) Search(ctx context.Context, orgID, query string, cursor, folder *string, status string, limit int32, allowed []uuid.UUID) (*models.CampaignsResult, error) {
|
||||
tx, err := beginResultTx(ctx, r.DB)
|
||||
if err != nil {
|
||||
db.CaptureError(err, "", nil, "begin")
|
||||
@@ -795,6 +795,7 @@ func (r *campaignRepository) Search(ctx context.Context, orgID, query string, cu
|
||||
SELECT 1 FROM campaign_folders cf WHERE cf.campaign_id = c.id AND cf.folder_id = $4
|
||||
))
|
||||
AND ($5 = '' OR CASE WHEN $5 = 'paused' THEN c.status::text LIKE 'paused%%' ELSE c.status::text = $5 END)
|
||||
AND ($6::uuid[] IS NULL OR c.id = ANY($6))
|
||||
GROUP BY c.id
|
||||
ORDER BY c.created_at DESC, c.id DESC
|
||||
LIMIT %d`,
|
||||
@@ -813,6 +814,7 @@ func (r *campaignRepository) Search(ctx context.Context, orgID, query string, cu
|
||||
SELECT 1 FROM campaign_folders cf WHERE cf.campaign_id = c.id AND cf.folder_id = $3
|
||||
))
|
||||
AND ($4 = '' OR CASE WHEN $4 = 'paused' THEN c.status::text LIKE 'paused%' ELSE c.status::text = $4 END)
|
||||
AND ($5::uuid[] IS NULL OR c.id = ANY($5))
|
||||
`
|
||||
}
|
||||
|
||||
@@ -822,6 +824,7 @@ func (r *campaignRepository) Search(ctx context.Context, orgID, query string, cu
|
||||
query,
|
||||
folder,
|
||||
status,
|
||||
allowed,
|
||||
}
|
||||
|
||||
rows, err := tx.Query(
|
||||
@@ -869,6 +872,7 @@ func (r *campaignRepository) Search(ctx context.Context, orgID, query string, cu
|
||||
query,
|
||||
folder,
|
||||
status,
|
||||
allowed,
|
||||
}
|
||||
var tmp int64
|
||||
err = tx.QueryRow(ctx, countSQL, params...).Scan(&tmp)
|
||||
@@ -889,7 +893,7 @@ func (r *campaignRepository) Search(ctx context.Context, orgID, query string, cu
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *campaignRepository) Overview(ctx context.Context, orgID string) (*models.CampaignsOverview, error) {
|
||||
func (r *campaignRepository) Overview(ctx context.Context, orgID string, allowed []uuid.UUID) (*models.CampaignsOverview, error) {
|
||||
overview := models.CampaignsOverview{Folders: []models.CampaignFolderCount{}}
|
||||
|
||||
countsSQL := `
|
||||
@@ -900,8 +904,8 @@ func (r *campaignRepository) Overview(ctx context.Context, orgID string) (*model
|
||||
COUNT(*) FILTER (WHERE status = 'draft'),
|
||||
COUNT(*) FILTER (WHERE status = 'completed')
|
||||
FROM campaigns
|
||||
WHERE organization_id = $1`
|
||||
err := resultDB(ctx, r.DB).QueryRow(ctx, countsSQL, orgID).Scan(
|
||||
WHERE organization_id = $1 AND ($2::uuid[] IS NULL OR id = ANY($2))`
|
||||
err := resultDB(ctx, r.DB).QueryRow(ctx, countsSQL, orgID, allowed).Scan(
|
||||
&overview.Total,
|
||||
&overview.Active,
|
||||
&overview.Paused,
|
||||
@@ -917,9 +921,9 @@ func (r *campaignRepository) Overview(ctx context.Context, orgID string) (*model
|
||||
SELECT cf.folder_id, COUNT(*)
|
||||
FROM campaign_folders cf
|
||||
JOIN campaigns c ON c.id = cf.campaign_id
|
||||
WHERE c.organization_id = $1
|
||||
WHERE c.organization_id = $1 AND ($2::uuid[] IS NULL OR c.id = ANY($2))
|
||||
GROUP BY cf.folder_id`
|
||||
rows, err := resultDB(ctx, r.DB).Query(ctx, foldersSQL, orgID)
|
||||
rows, err := resultDB(ctx, r.DB).Query(ctx, foldersSQL, orgID, allowed)
|
||||
if err != nil {
|
||||
db.CaptureError(err, foldersSQL, []any{orgID}, "query")
|
||||
return nil, err
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
)
|
||||
|
||||
// memberScope is the scope a new membership row is written with.
|
||||
func memberScope(m *models.OrganizationMember) models.AccessScope {
|
||||
if m.AccessScope.Valid() {
|
||||
return m.AccessScope
|
||||
}
|
||||
return models.AccessScopeWorkspace
|
||||
}
|
||||
|
||||
func textUUIDs(raw []string) []uuid.UUID {
|
||||
out := make([]uuid.UUID, 0, len(raw))
|
||||
for _, v := range raw {
|
||||
if id, err := uuid.Parse(v); err == nil {
|
||||
out = append(out, id)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// invitationAccessCols reads an invitation's scope; the table is aliased i.
|
||||
const invitationAccessCols = `i.access_scope, i.access_folder_ids::text[], i.access_campaign_ids::text[], i.access_email_account_ids::text[]`
|
||||
|
||||
type invitationAccess struct {
|
||||
scope models.AccessScope
|
||||
folders, campaigns, mailboxes []string
|
||||
}
|
||||
|
||||
func (a invitationAccess) access() *models.MemberAccess {
|
||||
if a.scope != models.AccessScopeRestricted {
|
||||
return models.WorkspaceAccess()
|
||||
}
|
||||
return &models.MemberAccess{Scope: a.scope, FolderIDs: textUUIDs(a.folders), CampaignIDs: textUUIDs(a.campaigns), EmailAccountIDs: textUUIDs(a.mailboxes)}
|
||||
}
|
||||
|
||||
// memberAccessSelect lists each member's scope and grants; $1 is the organization.
|
||||
const memberAccessSelect = `
|
||||
SELECT om.user_id, om.access_scope,
|
||||
COALESCE((SELECT array_agg(a.folder_id::text ORDER BY a.created_at) FROM organization_member_campaign_access a
|
||||
WHERE a.organization_id = om.organization_id AND a.user_id = om.user_id AND a.folder_id IS NOT NULL), '{}'),
|
||||
COALESCE((SELECT array_agg(a.campaign_id::text ORDER BY a.created_at) FROM organization_member_campaign_access a
|
||||
WHERE a.organization_id = om.organization_id AND a.user_id = om.user_id AND a.campaign_id IS NOT NULL), '{}'),
|
||||
COALESCE((SELECT array_agg(a.email_account_id::text ORDER BY a.created_at) FROM organization_member_mailbox_access a
|
||||
WHERE a.organization_id = om.organization_id AND a.user_id = om.user_id), '{}')
|
||||
FROM organization_members om
|
||||
WHERE om.organization_id = $1`
|
||||
|
||||
func scanMemberAccess(row pgx.Row) (uuid.UUID, *models.MemberAccess, error) {
|
||||
var userID uuid.UUID
|
||||
var scope models.AccessScope
|
||||
var folders, campaigns, mailboxes []string
|
||||
if err := row.Scan(&userID, &scope, &folders, &campaigns, &mailboxes); err != nil {
|
||||
return uuid.Nil, nil, err
|
||||
}
|
||||
a := &models.MemberAccess{Scope: scope, FolderIDs: textUUIDs(folders), CampaignIDs: textUUIDs(campaigns), EmailAccountIDs: textUUIDs(mailboxes)}
|
||||
if !a.Restricted() {
|
||||
a = models.WorkspaceAccess()
|
||||
}
|
||||
return userID, a, nil
|
||||
}
|
||||
|
||||
// GetMemberAccess is one member's scope and grants, or nil when they are not a member.
|
||||
func (r *organizationRepository) GetMemberAccess(ctx context.Context, orgID, userID uuid.UUID) (*models.MemberAccess, error) {
|
||||
_, a, err := scanMemberAccess(r.db.QueryRow(ctx, memberAccessSelect+` AND om.user_id = $2`, orgID, userID))
|
||||
if err == pgx.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
return a, err
|
||||
}
|
||||
|
||||
// ListMemberAccess is every member's scope and grants, by user id.
|
||||
func (r *organizationRepository) ListMemberAccess(ctx context.Context, orgID uuid.UUID) (map[uuid.UUID]*models.MemberAccess, error) {
|
||||
rows, err := r.db.Query(ctx, memberAccessSelect, orgID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[uuid.UUID]*models.MemberAccess{}
|
||||
for rows.Next() {
|
||||
userID, a, err := scanMemberAccess(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[userID] = a
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ResolveMemberScope is what a restricted member reaches right now: folder grants follow the folder's current contents.
|
||||
func (r *organizationRepository) ResolveMemberScope(ctx context.Context, orgID, userID uuid.UUID) (*models.ResourceScope, error) {
|
||||
var campaigns, folders, mailboxes, tags []string
|
||||
err := r.db.QueryRow(ctx, `
|
||||
WITH granted_folders AS (
|
||||
SELECT f.id FROM folders f
|
||||
JOIN organization_member_campaign_access a ON a.folder_id = f.id
|
||||
WHERE f.organization_id = $1 AND a.organization_id = $1 AND a.user_id = $2
|
||||
), in_scope AS (
|
||||
SELECT c.id FROM campaigns c
|
||||
WHERE c.organization_id = $1 AND (
|
||||
c.id IN (SELECT a.campaign_id FROM organization_member_campaign_access a
|
||||
WHERE a.organization_id = $1 AND a.user_id = $2 AND a.campaign_id IS NOT NULL)
|
||||
OR c.id IN (SELECT cf.campaign_id FROM campaign_folders cf WHERE cf.folder_id IN (SELECT id FROM granted_folders)))
|
||||
), boxes AS (
|
||||
SELECT ea.id FROM email_accounts ea
|
||||
JOIN organization_member_mailbox_access a ON a.email_account_id = ea.id
|
||||
WHERE ea.organization_id = $1 AND a.organization_id = $1 AND a.user_id = $2
|
||||
)
|
||||
SELECT
|
||||
COALESCE((SELECT array_agg(id::text) FROM in_scope), '{}'),
|
||||
COALESCE((SELECT array_agg(DISTINCT id::text) FROM (
|
||||
SELECT id FROM granted_folders
|
||||
UNION SELECT cf.folder_id FROM campaign_folders cf WHERE cf.campaign_id IN (SELECT id FROM in_scope)) v), '{}'),
|
||||
COALESCE((SELECT array_agg(id::text) FROM boxes), '{}'),
|
||||
COALESCE((SELECT array_agg(DISTINCT et.tag_id::text) FROM email_tags et WHERE et.email_id IN (SELECT id FROM boxes)), '{}')
|
||||
`, orgID, userID).Scan(&campaigns, &folders, &mailboxes, &tags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &models.ResourceScope{Campaigns: textUUIDs(campaigns), Folders: textUUIDs(folders), Mailboxes: textUUIDs(mailboxes), MailboxTags: textUUIDs(tags)}, nil
|
||||
}
|
||||
|
||||
// CountOwnedAccessResources counts how many of the ids belong to the organization, per kind.
|
||||
func (r *organizationRepository) CountOwnedAccessResources(ctx context.Context, orgID uuid.UUID, a *models.MemberAccess) (folders, campaigns, mailboxes int, err error) {
|
||||
err = r.db.QueryRow(ctx, `
|
||||
SELECT
|
||||
(SELECT COUNT(*) FROM folders WHERE organization_id = $1 AND id = ANY($2::uuid[])),
|
||||
(SELECT COUNT(*) FROM campaigns WHERE organization_id = $1 AND id = ANY($3::uuid[])),
|
||||
(SELECT COUNT(*) FROM email_accounts WHERE organization_id = $1 AND id = ANY($4::uuid[]))
|
||||
`, orgID, nonNilUUIDs(a.FolderIDs), nonNilUUIDs(a.CampaignIDs), nonNilUUIDs(a.EmailAccountIDs)).Scan(&folders, &campaigns, &mailboxes)
|
||||
return folders, campaigns, mailboxes, err
|
||||
}
|
||||
|
||||
// SetMemberAccess replaces a member's scope and grants in one transaction and recomputes their permissions.
|
||||
func (r *organizationRepository) SetMemberAccess(ctx context.Context, orgID, userID uuid.UUID, a *models.MemberAccess, grantedBy uuid.UUID) error {
|
||||
tx, err := r.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(ctx) //nolint:errcheck
|
||||
tag, err := tx.Exec(ctx, `UPDATE organization_members SET access_scope = $3
|
||||
WHERE organization_id = $1 AND user_id = $2 AND role <> 'owner'`, orgID, userID, a.Scope)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return pgx.ErrNoRows
|
||||
}
|
||||
if err := writeMemberGrants(ctx, tx, orgID, userID, a, grantedBy); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := recomputeMemberPermissions(ctx, tx, orgID, userID); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// writeMemberGrants replaces a member's grants, keeping only resources the organization owns.
|
||||
func writeMemberGrants(ctx context.Context, tx pgx.Tx, orgID, userID uuid.UUID, a *models.MemberAccess, grantedBy uuid.UUID) error {
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM organization_member_campaign_access WHERE organization_id = $1 AND user_id = $2`, orgID, userID); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM organization_member_mailbox_access WHERE organization_id = $1 AND user_id = $2`, orgID, userID); err != nil {
|
||||
return err
|
||||
}
|
||||
if !a.Restricted() {
|
||||
return nil
|
||||
}
|
||||
by := &grantedBy
|
||||
if grantedBy == uuid.Nil {
|
||||
by = nil
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO organization_member_campaign_access (organization_id, user_id, folder_id, granted_by)
|
||||
SELECT $1, $2, f.id, $4 FROM folders f WHERE f.organization_id = $1 AND f.id = ANY($3::uuid[])
|
||||
ON CONFLICT DO NOTHING`, orgID, userID, nonNilUUIDs(a.FolderIDs), by); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO organization_member_campaign_access (organization_id, user_id, campaign_id, granted_by)
|
||||
SELECT $1, $2, c.id, $4 FROM campaigns c WHERE c.organization_id = $1 AND c.id = ANY($3::uuid[])
|
||||
ON CONFLICT DO NOTHING`, orgID, userID, nonNilUUIDs(a.CampaignIDs), by); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := tx.Exec(ctx, `
|
||||
INSERT INTO organization_member_mailbox_access (organization_id, user_id, email_account_id, granted_by)
|
||||
SELECT $1, $2, ea.id, $4 FROM email_accounts ea WHERE ea.organization_id = $1 AND ea.id = ANY($3::uuid[])
|
||||
ON CONFLICT DO NOTHING`, orgID, userID, nonNilUUIDs(a.EmailAccountIDs), by)
|
||||
return err
|
||||
}
|
||||
|
||||
// SuggestCampaignSenders lists the mailboxes the given campaigns, and the campaigns in the given folders, send from.
|
||||
func (r *organizationRepository) SuggestCampaignSenders(ctx context.Context, orgID uuid.UUID, campaignIDs, folderIDs []uuid.UUID) ([]models.SuggestedSender, error) {
|
||||
rows, err := r.db.Query(ctx, `
|
||||
WITH picked AS (
|
||||
SELECT c.id FROM campaigns c
|
||||
WHERE c.organization_id = $1 AND (c.id = ANY($2::uuid[])
|
||||
OR c.id IN (SELECT cf.campaign_id FROM campaign_folders cf
|
||||
JOIN folders f ON f.id = cf.folder_id
|
||||
WHERE f.organization_id = $1 AND f.id = ANY($3::uuid[])))
|
||||
), senders AS (
|
||||
SELECT cs.email_account_id, cs.campaign_id FROM campaign_senders cs
|
||||
WHERE cs.campaign_id IN (SELECT id FROM picked) AND cs.enabled
|
||||
UNION
|
||||
SELECT et.email_id, cet.campaign_id FROM campaign_email_tags cet
|
||||
JOIN email_tags et ON et.tag_id = cet.tag_id
|
||||
WHERE cet.campaign_id IN (SELECT id FROM picked)
|
||||
)
|
||||
SELECT ea.id, ea.email, COALESCE(ea.name, ''), array_agg(DISTINCT s.campaign_id::text)
|
||||
FROM senders s
|
||||
JOIN email_accounts ea ON ea.id = s.email_account_id AND ea.organization_id = $1
|
||||
GROUP BY ea.id, ea.email, ea.name
|
||||
ORDER BY ea.email
|
||||
`, orgID, nonNilUUIDs(campaignIDs), nonNilUUIDs(folderIDs))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []models.SuggestedSender{}
|
||||
for rows.Next() {
|
||||
var s models.SuggestedSender
|
||||
var campaigns []string
|
||||
if err := rows.Scan(&s.ID, &s.Email, &s.Name, &campaigns); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.CampaignIDs = textUUIDs(campaigns)
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
@@ -45,7 +45,8 @@ func recomputeMemberPermissions(ctx context.Context, tx pgx.Tx, orgID, userID uu
|
||||
// AddMemberWithRoles inserts a membership row and its role assignments and
|
||||
// recomputes the effective permission snapshot, all in one transaction.
|
||||
// Used by invite-accept so a partial failure can never strand a member with
|
||||
// no role rows.
|
||||
// no role rows. A restricted member.Access lands with the row, so the member is
|
||||
// never workspace-wide in between.
|
||||
func (r *organizationRepository) AddMemberWithRoles(ctx context.Context, member *models.OrganizationMember, roleIDs []uuid.UUID) error {
|
||||
tx, err := r.db.Begin(ctx)
|
||||
if err != nil {
|
||||
@@ -53,13 +54,25 @@ func (r *organizationRepository) AddMemberWithRoles(ctx context.Context, member
|
||||
}
|
||||
defer tx.Rollback(ctx) //nolint:errcheck
|
||||
|
||||
if member.Access.Restricted() {
|
||||
member.AccessScope = models.AccessScopeRestricted
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO organization_members (id, organization_id, user_id, role, role_id, permissions, invited_by, invited_at, accepted_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
||||
INSERT INTO organization_members (id, organization_id, user_id, role, role_id, permissions, invited_by, invited_at, accepted_at, access_scope)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
|
||||
`, member.ID, member.OrganizationID, member.UserID, member.Role, member.RoleID,
|
||||
member.Permissions, member.InvitedBy, member.InvitedAt, member.AcceptedAt); err != nil {
|
||||
member.Permissions, member.InvitedBy, member.InvitedAt, member.AcceptedAt, memberScope(member)); err != nil {
|
||||
return err
|
||||
}
|
||||
if member.Access.Restricted() {
|
||||
var grantedBy uuid.UUID
|
||||
if member.InvitedBy != nil {
|
||||
grantedBy = *member.InvitedBy
|
||||
}
|
||||
if err := writeMemberGrants(ctx, tx, member.OrganizationID, member.UserID, member.Access, grantedBy); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, roleID := range roleIDs {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO organization_member_roles (organization_id, user_id, role_id)
|
||||
|
||||
@@ -0,0 +1,287 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/warmbly/warmbly/internal/app/monitoring"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type MonitoringRepository struct{ pool *pgxpool.Pool }
|
||||
|
||||
func NewMonitoringRepository(pool *pgxpool.Pool) *MonitoringRepository {
|
||||
return &MonitoringRepository{pool: pool}
|
||||
}
|
||||
|
||||
type monitoringQuery struct {
|
||||
id, title, unit string
|
||||
condition models.MonitoringCondition
|
||||
severity, note, query string
|
||||
window time.Duration
|
||||
}
|
||||
|
||||
type monitoringSection struct {
|
||||
permission models.AdminPermission
|
||||
investigate string
|
||||
queries []monitoringQuery
|
||||
}
|
||||
|
||||
func (r *MonitoringRepository) Sources() []monitoring.Source {
|
||||
out := make([]monitoring.Source, 0, len(monitoringSections)+3)
|
||||
for _, id := range monitoringSectionOrder {
|
||||
section := monitoringSections[id]
|
||||
out = append(out, monitoring.Source{ID: id, Permission: section.permission, Collect: func(ctx context.Context, at time.Time) (models.MonitoringSource, error) {
|
||||
return r.collect(ctx, at, id, section)
|
||||
}})
|
||||
}
|
||||
out = append(out, r.workerSource(), r.domainSource(), r.loadingSource())
|
||||
return out
|
||||
}
|
||||
|
||||
func (r *MonitoringRepository) collect(ctx context.Context, at time.Time, id string, section monitoringSection) (models.MonitoringSource, error) {
|
||||
out := models.MonitoringSource{Metrics: []models.MonitoringMetric{}}
|
||||
if r == nil || r.pool == nil {
|
||||
out.Availability = models.MonitoringUnavailable
|
||||
out.Coverage = "unavailable"
|
||||
out.Reason = "dependency_missing"
|
||||
return out, nil
|
||||
}
|
||||
if id == "arrivals" {
|
||||
var exists bool
|
||||
if err := r.pool.QueryRow(ctx, "SELECT to_regclass('public.sync_arrival_outbox') IS NOT NULL").Scan(&exists); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if !exists {
|
||||
out.Availability = models.MonitoringUnavailable
|
||||
out.Coverage = "unavailable"
|
||||
out.Reason = "schema_absent"
|
||||
for _, q := range section.queries {
|
||||
m := monitoringMetric(q, at, section.investigate)
|
||||
m.Availability = models.MonitoringUnavailable
|
||||
m.Condition = models.MonitoringUnknown
|
||||
m.Reason = "schema_absent"
|
||||
m.ObservedAt = nil
|
||||
out.Metrics = append(out.Metrics, m)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
queries := ""
|
||||
for _, q := range section.queries {
|
||||
if queries != "" {
|
||||
queries += " UNION ALL "
|
||||
}
|
||||
queries += strings.Replace(strings.ReplaceAll(q.query, "$1", "$1::timestamptz"), "SELECT ", "SELECT '"+q.id+"', ", 1) + " AND $1::timestamptz IS NOT NULL"
|
||||
}
|
||||
rows, err := r.pool.Query(ctx, queries, at)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
defer rows.Close()
|
||||
index := 0
|
||||
for rows.Next() {
|
||||
if index >= len(section.queries) {
|
||||
return out, errors.New("unexpected monitoring rows")
|
||||
}
|
||||
var id string
|
||||
m := models.MonitoringMetric{}
|
||||
if err := rows.Scan(&id, &m.Count, &m.AffectedMailboxes, &m.AffectedOrganizations, &m.UnknownOrganizationRows, &m.EvidenceAt, &m.LatestEvidenceAt, &m.NextEligibleAt); err != nil {
|
||||
return out, err
|
||||
}
|
||||
found := false
|
||||
for _, q := range section.queries {
|
||||
if q.id == id {
|
||||
v := monitoringMetric(q, at, section.investigate)
|
||||
v.Count = m.Count
|
||||
v.AffectedMailboxes = m.AffectedMailboxes
|
||||
v.AffectedOrganizations = m.AffectedOrganizations
|
||||
v.UnknownOrganizationRows = m.UnknownOrganizationRows
|
||||
v.EvidenceAt = m.EvidenceAt
|
||||
v.LatestEvidenceAt = m.LatestEvidenceAt
|
||||
v.NextEligibleAt = m.NextEligibleAt
|
||||
m = v
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return out, errors.New("unexpected monitoring metric")
|
||||
}
|
||||
classifyMonitoringMetric(&m)
|
||||
out.Metrics = append(out.Metrics, m)
|
||||
index++
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if index != len(section.queries) {
|
||||
return out, errors.New("incomplete monitoring rows")
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func monitoringMetric(q monitoringQuery, at time.Time, path string) models.MonitoringMetric {
|
||||
m := models.MonitoringMetric{ID: q.id, Title: q.title, Unit: q.unit, Condition: q.condition, Severity: q.severity, Note: q.note, Investigate: path, ObservedAt: &at, Availability: models.MonitoringFresh}
|
||||
if seconds, ok := monitoringThresholds[q.id]; ok {
|
||||
m.ThresholdSeconds = &seconds
|
||||
}
|
||||
if q.window > 0 {
|
||||
start := at.Add(-q.window)
|
||||
m.WindowStart = &start
|
||||
m.WindowEnd = &at
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
var monitoringThresholds = map[string]int64{
|
||||
"reservations_under_5m": 300, "reservations_5_to_30m": 1800, "reservations_over_30m": 1800,
|
||||
"campaign_reserved_aging": 1800, "dispatch_persistent": 600, "dispatch_old_failure": 900,
|
||||
"original_slot_overdue_1h": 3600, "sync_stale": 1200, "sync_held_stale": 1200,
|
||||
"backfill_stale_running": 3600, "worker_samples_missing": 1200,
|
||||
"webhook_stale_claim": 300, "notification_aging": 120, "notification_stale": 600,
|
||||
"arrival_aging": 300,
|
||||
}
|
||||
|
||||
func classifyMonitoringMetric(m *models.MonitoringMetric) {
|
||||
if m.Count == nil {
|
||||
m.Availability = models.MonitoringUnavailable
|
||||
m.Condition = models.MonitoringUnknown
|
||||
m.Reason = "no_recent_evidence"
|
||||
return
|
||||
}
|
||||
if *m.Count == 0 {
|
||||
m.Condition = models.MonitoringNoEvidence
|
||||
m.Severity = "info"
|
||||
if m.ID == "confirmed_warmup_1h" || m.ID == "confirmed_other_1h" || m.ID == "confirmed_warmup_proxy_1h" {
|
||||
m.Note += " Zero recent confirmations establishes neither recovery nor an outage."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var monitoringSectionOrder = []string{"send_safety", "sends", "campaign_reservations", "dispatch", "sync", "mailbox_errors", "worker_samples", "jobs", "dead_letters", "webhooks", "notifications", "result_effects", "arrivals"}
|
||||
var monitoringSections = map[string]monitoringSection{
|
||||
"campaign_reservations": {permission: models.AdminPermViewCampaigns, investigate: "/sends?tab=in-flight", queries: []monitoringQuery{
|
||||
{id: "campaign_reserved", title: "Active campaign progress reservations", unit: "reservations", condition: models.MonitoringInFlight, severity: "info", note: "Native dispatched_at without sent_at progress state, distinct from unknown provider results.", query: `SELECT COUNT(*),COUNT(DISTINCT t.email_account_id),COUNT(DISTINCT c.organization_id),COUNT(*) FILTER(WHERE c.organization_id IS NULL),MIN(p.dispatched_at),MAX(p.dispatched_at),NULL::timestamptz FROM campaign_contact_progress p LEFT JOIN tasks t ON t.id=p.dispatch_task_id JOIN campaigns c ON c.id=p.campaign_id WHERE p.sent_at IS NULL AND p.dispatched_at IS NOT NULL`},
|
||||
{id: "campaign_reserved_aging", title: "Campaign progress reservations older than 30m", unit: "reservations", condition: models.MonitoringRecoveryUnverified, severity: "warning", note: "30m investigation policy, not a native timeout or permission to replay.", query: `SELECT COUNT(*),COUNT(DISTINCT t.email_account_id),COUNT(DISTINCT c.organization_id),COUNT(*) FILTER(WHERE c.organization_id IS NULL),MIN(p.dispatched_at),MAX(p.dispatched_at),NULL::timestamptz FROM campaign_contact_progress p LEFT JOIN tasks t ON t.id=p.dispatch_task_id JOIN campaigns c ON c.id=p.campaign_id WHERE p.sent_at IS NULL AND p.dispatched_at<=$1-INTERVAL '30 minutes'`},
|
||||
}},
|
||||
"worker_samples": {permission: models.AdminPermViewWorkers, investigate: "/fleet", queries: []monitoringQuery{
|
||||
{id: "worker_samples_missing", title: "Active workers without fresh health samples", unit: "workers", condition: models.MonitoringTelemetryStale, severity: "warning", note: "Monitoring 20m receipt-freshness policy; samples are stored with database receipt time, not worker clock. Zero counters in a fresh sample may be uninstrumented.", query: `SELECT COUNT(*),NULL::bigint,NULL::bigint,NULL::bigint,MIN(s.at),MAX(s.at),NULL::timestamptz FROM fleet_nodes n LEFT JOIN LATERAL(SELECT MAX(observed_at) AS at FROM worker_health_samples WHERE worker_id=n.id) s ON true WHERE n.active AND n.role='worker' AND (s.at IS NULL OR s.at<$1-INTERVAL '20 minutes' OR s.at>$1)`},
|
||||
{id: "worker_sample_auth_errors_1h", title: "Reported worker credential errors (1h)", unit: "reports", condition: models.MonitoringRecentFailure, severity: "warning", note: "Worker-reported window counters, independent of mailbox reports. Zero counters do not establish instrumented healthy credentials.", window: time.Hour, query: `SELECT SUM(h.auth_errors)::bigint,NULL::bigint,NULL::bigint,NULL::bigint,MIN(h.observed_at),MAX(h.observed_at),NULL::timestamptz FROM worker_health_samples h WHERE h.observed_at BETWEEN $1-INTERVAL '1 hour' AND $1`},
|
||||
{id: "worker_sample_rate_errors_1h", title: "Reported worker rate-limit errors (1h)", unit: "reports", condition: models.MonitoringRecentFailure, severity: "warning", note: "Reported window counters may be uninstrumented; not cumulative retry attempts.", window: time.Hour, query: `SELECT SUM(h.rate_limit_errors)::bigint,NULL::bigint,NULL::bigint,NULL::bigint,MIN(h.observed_at),MAX(h.observed_at),NULL::timestamptz FROM worker_health_samples h WHERE h.observed_at BETWEEN $1-INTERVAL '1 hour' AND $1`},
|
||||
{id: "worker_sample_sends_1h", title: "Reported worker successful sends (1h)", unit: "reports", condition: models.MonitoringUnknown, severity: "info", note: "Worker-reported counter, not provider-confirmed warmup delivery; token-confirmed tasks are measured separately.", window: time.Hour, query: `SELECT SUM(h.sends_succeeded)::bigint,NULL::bigint,NULL::bigint,NULL::bigint,MIN(h.observed_at),MAX(h.observed_at),NULL::timestamptz FROM worker_health_samples h WHERE h.observed_at BETWEEN $1-INTERVAL '1 hour' AND $1`},
|
||||
}},
|
||||
"send_safety": {permission: models.AdminPermViewUsers, investigate: "/mailboxes", queries: []monitoringQuery{
|
||||
{id: "holds_unknown", title: "Unknown send outcome protection", unit: "mailboxes", condition: "safety_hold", severity: "warning", note: "Protection can be set before execution; not a provider failure or permission to resend.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT e.id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(NULL::timestamptz), MAX(NULL::timestamptz), MIN(NULL::timestamptz) FROM email_accounts e WHERE e.send_recovery_hold AND COALESCE(e.send_recovery_reason,'unknown')='unknown'`},
|
||||
{id: "holds_conflict", title: "conflict send protection", unit: "mailboxes", condition: "safety_hold", severity: "warning", note: "Current persisted safety restriction, not proof of a new rejection.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT e.id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(NULL::timestamptz), MAX(NULL::timestamptz), MIN(NULL::timestamptz) FROM email_accounts e WHERE e.send_recovery_hold AND e.send_recovery_reason='conflict'`},
|
||||
{id: "holds_permanent", title: "permanent send protection", unit: "mailboxes", condition: "safety_hold", severity: "warning", note: "Current persisted safety restriction, not proof of a new rejection.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT e.id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(NULL::timestamptz), MAX(NULL::timestamptz), MIN(NULL::timestamptz) FROM email_accounts e WHERE e.send_recovery_hold AND e.send_recovery_reason='permanent'`},
|
||||
{id: "holds_authentication", title: "authentication send protection", unit: "mailboxes", condition: "safety_hold", severity: "warning", note: "Current persisted safety restriction, not proof of a new rejection.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT e.id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(NULL::timestamptz), MAX(NULL::timestamptz), MIN(NULL::timestamptz) FROM email_accounts e WHERE e.send_recovery_hold AND e.send_recovery_reason='authentication'`},
|
||||
{id: "send_cooldowns", title: "Active provider cooldowns", unit: "mailboxes", condition: "waiting_retry", severity: "info", note: "Expiry is next eligibility, not recovery confirmation.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT e.id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(NULL::timestamptz), MAX(NULL::timestamptz), MIN(e.send_cooldown_until::timestamptz) FROM email_accounts e WHERE e.send_cooldown_until>$1`},
|
||||
{id: "send_paused", title: "Paused warmup", unit: "mailboxes", condition: "waiting_schedule", severity: "info", note: "Explicit operator pause, not a failure.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT e.id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(e.warmup_paused_at::timestamptz), MAX(e.warmup_paused_at::timestamptz), MIN(NULL::timestamptz) FROM email_accounts e WHERE e.status='active' AND e.warmup IS NOT NULL AND e.warmup_paused_at IS NOT NULL`},
|
||||
}},
|
||||
"sends": {permission: models.AdminPermViewCampaigns, queries: []monitoringQuery{
|
||||
{id: "confirmed_warmup_1h", title: "Provider-confirmed warmup tasks (1h)", unit: "tasks", condition: "healthy", severity: "info", note: "Provider acceptance only; legacy completed_at is a time proxy after independent token confirmation.", window: 3600 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(COALESCE(t.send_result_applied_at,t.completed_at)::timestamptz), MAX(COALESCE(t.send_result_applied_at,t.completed_at)::timestamptz), MIN(NULL::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.task_type='warmup' AND EXISTS(SELECT 1 FROM warmup_tokens wt WHERE wt.task_id=t.id AND wt.sent_message_id<>'') AND COALESCE(t.send_result_applied_at,t.completed_at) BETWEEN $1-INTERVAL '1 hour' AND $1`},
|
||||
{id: "confirmed_other_1h", title: "Reconciled non-warmup sent tasks (1h)", unit: "tasks", condition: "healthy", severity: "info", note: "Native reconciliation evidence, not receiver delivery or inbox placement.", window: 3600 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.send_result_applied_at::timestamptz), MAX(t.send_result_applied_at::timestamptz), MIN(NULL::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.task_type<>'warmup' AND t.send_result_state='sent' AND t.send_result_applied_at BETWEEN $1-INTERVAL '1 hour' AND $1`},
|
||||
{id: "confirmed_warmup_proxy_1h", title: "Legacy-clock confirmed warmup tasks (1h)", unit: "tasks", condition: "healthy", severity: "info", note: "Subset of confirmed warmup using completed_at proxy.", window: 3600 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.completed_at::timestamptz), MAX(t.completed_at::timestamptz), MIN(NULL::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.task_type='warmup' AND t.send_result_applied_at IS NULL AND EXISTS(SELECT 1 FROM warmup_tokens wt WHERE wt.task_id=t.id AND wt.sent_message_id<>'') AND t.completed_at BETWEEN $1-INTERVAL '1 hour' AND $1`},
|
||||
{id: "unknown_results", title: "Unverified send results", unit: "tasks", condition: "recovery_unverified", severity: "warning", note: "Includes legacy uncertainty; unknown is not a failed send.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.send_reserved_at::timestamptz), MAX(t.send_reserved_at::timestamptz), MIN(NULL::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.send_result_state='unknown'`},
|
||||
{id: "reservations_under_5m", title: "Outstanding reservation: under_5m", unit: "tasks", condition: "in_flight", severity: "info", note: "Reservation precedes provider execution; age is not a native timeout guarantee.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.send_reserved_at::timestamptz), MAX(t.send_executor_started_at::timestamptz), MIN(NULL::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.send_reserved_at IS NOT NULL AND t.send_released_at IS NULL AND t.send_result_state='unknown' AND t.send_result_applied_at IS NULL AND t.send_reserved_at>$1-INTERVAL '5 minutes' AND t.send_reserved_at<=$1`},
|
||||
{id: "reservations_5_to_30m", title: "Outstanding reservation: 5_to_30m", unit: "tasks", condition: "in_flight", severity: "info", note: "Reservation precedes provider execution; age is not a native timeout guarantee.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.send_reserved_at::timestamptz), MAX(t.send_executor_started_at::timestamptz), MIN(NULL::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.send_reserved_at IS NOT NULL AND t.send_released_at IS NULL AND t.send_result_state='unknown' AND t.send_result_applied_at IS NULL AND t.send_reserved_at<=$1-INTERVAL '5 minutes' AND t.send_reserved_at>$1-INTERVAL '30 minutes'`},
|
||||
{id: "reservations_over_30m", title: "Outstanding reservation: over_30m", unit: "tasks", condition: "recovery_unverified", severity: "warning", note: "Reservation precedes provider execution; age is not a native timeout guarantee.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.send_reserved_at::timestamptz), MAX(t.send_executor_started_at::timestamptz), MIN(NULL::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.send_reserved_at IS NOT NULL AND t.send_released_at IS NULL AND t.send_result_state='unknown' AND t.send_result_applied_at IS NULL AND t.send_reserved_at<=$1-INTERVAL '30 minutes'`},
|
||||
{id: "failed_results_1h", title: "Reconciled failed send results (1h)", unit: "tasks", condition: "recent_failure", severity: "warning", note: "Failure of these tasks, not a continuing mailbox outage.", window: 3600 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.send_result_applied_at::timestamptz), MAX(t.send_result_applied_at::timestamptz), MIN(NULL::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.send_result_state='failed' AND t.send_result_applied_at BETWEEN $1-INTERVAL '1 hour' AND $1`},
|
||||
{id: "failed_results_24h", title: "Reconciled failed send results (24h)", unit: "tasks", condition: "recovery_unverified", severity: "warning", note: "Historical task outcomes; mailbox recovery requires scoped confirmation.", window: 86400 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.send_result_applied_at::timestamptz), MAX(t.send_result_applied_at::timestamptz), MIN(NULL::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.send_result_state='failed' AND t.send_result_applied_at BETWEEN $1-INTERVAL '24 hours' AND $1`},
|
||||
}},
|
||||
"dispatch": {permission: models.AdminPermViewCampaigns, queries: []monitoringQuery{
|
||||
{id: "dispatch_due", title: "Effectively due pending tasks", unit: "tasks", condition: "backlog", severity: "info", note: "Database dispatch backlog, not cloud provider queue depth or outage.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(GREATEST(t.scheduled_at,COALESCE(t.dispatch_retry_at,t.scheduled_at))::timestamptz), MAX(t.dispatch_failure_at::timestamptz), MIN(NULL::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.status='pending' AND GREATEST(t.scheduled_at,COALESCE(t.dispatch_retry_at,t.scheduled_at))<=$1`},
|
||||
{id: "future_schedule", title: "Future scheduled tasks", unit: "tasks", condition: "waiting_schedule", severity: "info", note: "A future slot does not establish whether pacing, daily cap or business hours chose it.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(NULL::timestamptz), MAX(NULL::timestamptz), MIN(GREATEST(t.scheduled_at,COALESCE(t.dispatch_retry_at,t.scheduled_at))::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.status='pending' AND t.scheduled_at>$1`},
|
||||
{id: "retry_wait", title: "Dispatch retry waits", unit: "tasks", condition: "waiting_retry", severity: "info", note: "Local scheduler retry evidence; count is tasks, not attempts.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.dispatch_failure_since::timestamptz), MAX(t.dispatch_failure_at::timestamptz), MIN(t.dispatch_retry_at::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.status='pending' AND t.scheduled_at<=$1 AND t.dispatch_retry_at>$1`},
|
||||
{id: "dispatch_persistent", title: "Repeated recent dispatch failure evidence", unit: "tasks", condition: "persistent_failure", severity: "warning", note: "Monitoring policy: repeated stamped dispatch failure over 10m with recurrence within 15m; not provider failure.", window: 900 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.dispatch_failure_since::timestamptz), MAX(t.dispatch_failure_at::timestamptz), MIN(t.dispatch_retry_at::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.status='pending' AND t.dispatch_failure_since<=$1-INTERVAL '10 minutes' AND t.dispatch_failure_at>$1-INTERVAL '15 minutes' AND t.dispatch_failure_at<=$1 AND t.dispatch_failure_at>t.dispatch_failure_since`},
|
||||
{id: "dispatch_old_failure", title: "Old dispatch failure, recovery unverified", unit: "tasks", condition: "recovery_unverified", severity: "warning", note: "Old failure does not prove continuing failure.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.dispatch_failure_since::timestamptz), MAX(t.dispatch_failure_at::timestamptz), MIN(t.dispatch_retry_at::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.status='pending' AND t.dispatch_failure_at<=$1-INTERVAL '15 minutes'`},
|
||||
{id: "original_slot_overdue_1h", title: "Original warmup slot overdue (1h)", unit: "tasks", condition: "backlog", severity: "info", note: "Original slot delay includes retry waits and persisted safety/cooldown gates; not unexplained dispatcher failure.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.scheduled_at::timestamptz), MAX(NULL::timestamptz), MIN(t.dispatch_retry_at::timestamptz) FROM tasks t JOIN email_accounts e ON e.id=t.email_account_id WHERE t.status='pending' AND t.task_type='warmup' AND t.scheduled_at<=$1-INTERVAL '1 hour' AND e.status='active' AND e.warmup IS NOT NULL AND e.warmup_paused_at IS NULL AND (e.test_mode IS NULL OR e.test_mode='legacy' OR e.test_mode='diagnostic' AND e.test_send_enabled)`},
|
||||
{id: "unscheduled_pending", title: "Pending tasks without a schedule", unit: "tasks", condition: "unknown", severity: "info", note: "No due time means overdue age cannot be inferred.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.created_at::timestamptz), MAX(NULL::timestamptz), MIN(NULL::timestamptz) FROM tasks t LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.status='pending' AND t.scheduled_at IS NULL AND t.dispatch_retry_at IS NULL`},
|
||||
}},
|
||||
"sync": {permission: models.AdminPermViewUsers, investigate: "/sync", queries: []monitoringQuery{
|
||||
{id: "sync_eligible", title: "Active assigned supported mailboxes", unit: "mailboxes", condition: "unknown", severity: "info", note: "Population for sync state coverage, not a throughput assertion.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT e.id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(NULL::timestamptz), MAX(NULL::timestamptz), MIN(NULL::timestamptz) FROM email_accounts e WHERE e.status='active' AND e.worker_id IS NOT NULL AND e.provider::text IN ('gmail','outlook','smtp_imap')`},
|
||||
{id: "sync_missing", title: "Eligible mailboxes without sync state", unit: "mailboxes", condition: "unknown", severity: "warning", note: "Absent worker state is unobserved, not a sync outage.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT e.id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(NULL::timestamptz), MAX(NULL::timestamptz), MIN(NULL::timestamptz) FROM email_accounts e LEFT JOIN email_sync_state s ON s.email_id=e.id WHERE e.status='active' AND e.worker_id IS NOT NULL AND e.provider::text IN ('gmail','outlook','smtp_imap') AND s.email_id IS NULL`},
|
||||
{id: "sync_unassigned", title: "Active supported mailboxes unassigned", unit: "mailboxes", condition: "unknown", severity: "warning", note: "Assignment issue; no provider sync inference.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT e.id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(NULL::timestamptz), MAX(NULL::timestamptz), MIN(NULL::timestamptz) FROM email_accounts e WHERE e.status='active' AND e.worker_id IS NULL AND e.provider::text IN ('gmail','outlook','smtp_imap')`},
|
||||
{id: "sync_state_rows", title: "Persisted sync state rows", unit: "mailboxes", condition: "unknown", severity: "info", note: "Fleet-wide persisted population; zero rows is not healthy sync.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.last_synced_at::timestamptz), MAX(s.last_synced_at::timestamptz), MIN(NULL::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE TRUE`},
|
||||
{id: "sync_hold_burst", title: "Active sync burst hold", unit: "mailboxes", condition: "budget", severity: "info", note: "Persisted governor hold, not transport failure; expiry is eligibility only.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.last_synced_at::timestamptz), MAX(s.last_synced_at::timestamptz), MIN(s.throttled_until::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE s.throttled_until>$1 AND s.throttle_reason='burst'`},
|
||||
{id: "sync_hold_hourly", title: "Active sync hourly hold", unit: "mailboxes", condition: "budget", severity: "info", note: "Persisted governor hold, not transport failure; expiry is eligibility only.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.last_synced_at::timestamptz), MAX(s.last_synced_at::timestamptz), MIN(s.throttled_until::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE s.throttled_until>$1 AND s.throttle_reason='hourly'`},
|
||||
{id: "sync_hold_daily", title: "Active sync daily hold", unit: "mailboxes", condition: "daily_limit", severity: "info", note: "Persisted governor hold, not transport failure; expiry is eligibility only.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.last_synced_at::timestamptz), MAX(s.last_synced_at::timestamptz), MIN(s.throttled_until::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE s.throttled_until>$1 AND s.throttle_reason='daily'`},
|
||||
{id: "sync_hold_org_daily", title: "Active sync org_daily hold", unit: "mailboxes", condition: "daily_limit", severity: "info", note: "Persisted governor hold, not transport failure; expiry is eligibility only.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.last_synced_at::timestamptz), MAX(s.last_synced_at::timestamptz), MIN(s.throttled_until::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE s.throttled_until>$1 AND s.throttle_reason='org_daily'`},
|
||||
{id: "sync_hold_priority_daily", title: "Active sync priority_daily hold", unit: "mailboxes", condition: "daily_limit", severity: "info", note: "Persisted governor hold, not transport failure; expiry is eligibility only.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.last_synced_at::timestamptz), MAX(s.last_synced_at::timestamptz), MIN(s.throttled_until::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE s.throttled_until>$1 AND s.throttle_reason='priority_daily'`},
|
||||
{id: "sync_hold_other", title: "Other active sync holds", unit: "mailboxes", condition: "budget", severity: "info", note: "Unknown reason is not promoted to a provider error.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.last_synced_at::timestamptz), MAX(s.last_synced_at::timestamptz), MIN(s.throttled_until::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE s.throttled_until>$1 AND s.throttle_reason NOT IN ('burst','hourly','daily','org_daily','priority_daily')`},
|
||||
{id: "sync_stale", title: "Eligible sync state older than 20m", unit: "mailboxes", condition: "stale_telemetry", severity: "warning", note: "20m monitoring freshness policy, not provider SLA; base tick timestamps do not prove successful sync.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.last_synced_at::timestamptz), MAX(s.last_synced_at::timestamptz), MIN(NULL::timestamptz) FROM email_sync_state s JOIN email_accounts e ON e.id=s.email_id WHERE e.status='active' AND e.worker_id IS NOT NULL AND e.provider::text IN ('gmail','outlook','smtp_imap') AND (s.last_synced_at IS NULL OR s.last_synced_at<$1-INTERVAL '20 minutes') AND (s.throttled_until IS NULL OR s.throttled_until<=$1)`},
|
||||
{id: "sync_held_stale", title: "Stale sync observations under active holds", unit: "mailboxes", condition: "budget", severity: "info", note: "Hold explains waiting but does not refresh old activity evidence.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.last_synced_at::timestamptz), MAX(s.last_synced_at::timestamptz), MIN(s.throttled_until::timestamptz) FROM email_sync_state s JOIN email_accounts e ON e.id=s.email_id WHERE e.status='active' AND e.worker_id IS NOT NULL AND (s.last_synced_at IS NULL OR s.last_synced_at<$1-INTERVAL '20 minutes') AND s.throttled_until>$1`},
|
||||
{id: "sync_future_clock", title: "Future-dated sync activity", unit: "mailboxes", condition: "unknown", severity: "warning", note: "Clock anomaly; never a negative-age healthy reading.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.last_synced_at::timestamptz), MAX(s.last_synced_at::timestamptz), MIN(NULL::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE s.last_synced_at>$1`},
|
||||
{id: "backfill_running", title: "Backfill running states", unit: "mailboxes", condition: "waiting_schedule", severity: "info", note: "Persisted import state; complete can mean configured cap reached, not current sync health.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.backfill_started_at::timestamptz), MAX(s.updated_at::timestamptz), MIN(NULL::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE s.backfill_status='running'`},
|
||||
{id: "backfill_pending", title: "Backfill pending states", unit: "mailboxes", condition: "waiting_schedule", severity: "info", note: "Persisted import state; complete can mean configured cap reached, not current sync health.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.backfill_started_at::timestamptz), MAX(s.updated_at::timestamptz), MIN(NULL::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE s.backfill_status='pending'`},
|
||||
{id: "backfill_complete", title: "Backfill complete states", unit: "mailboxes", condition: "unknown", severity: "info", note: "Persisted import state; complete can mean configured cap reached, not current sync health.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.backfill_started_at::timestamptz), MAX(s.updated_at::timestamptz), MIN(NULL::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE s.backfill_status='complete'`},
|
||||
{id: "backfill_stale_running", title: "Stale running backfill without active hold", unit: "mailboxes", condition: "stale_telemetry", severity: "warning", note: "Native 1h stale running-state rule, potential stall rather than proved outage.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.updated_at::timestamptz), MAX(s.updated_at::timestamptz), MIN(NULL::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE s.backfill_status='running' AND s.updated_at<$1-INTERVAL '1 hour' AND (s.throttled_until IS NULL OR s.throttled_until<=$1)`},
|
||||
{id: "sync_deferred", title: "Last reported budget-deferred live messages", unit: "messages", condition: "budget", severity: "info", note: "Latest pass count, not remaining provider backlog; incomplete ticks can overwrite it.", window: 0 * time.Second, query: `SELECT COALESCE(SUM(s.deferred),0)::bigint, COUNT(DISTINCT s.email_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(s.last_synced_at::timestamptz), MAX(s.last_synced_at::timestamptz), MIN(s.throttled_until::timestamptz) FROM email_sync_state s LEFT JOIN email_accounts e ON e.id=s.email_id WHERE s.deferred>0`},
|
||||
}},
|
||||
"mailbox_errors": {permission: models.AdminPermViewUsers, investigate: "/mailboxes", queries: []monitoringQuery{
|
||||
{id: "credential_errors", title: "Unresolved mailbox credential reports", unit: "reports", condition: "recovery_unverified", severity: "warning", note: "Deduplicated first stored report, not latest failing attempt or recurrence. Includes inactive accounts.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT r.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(r.created_at::timestamptz), MAX(r.created_at::timestamptz), MIN(NULL::timestamptz) FROM email_account_errors r LEFT JOIN email_accounts e ON e.id=r.email_account_id WHERE r.resolved_at IS NULL AND r.error_code IN ('GOOGLE_AUTHENTICATION_FAILED','AUTHENTICATION_FAILED','AUTHORIZATION_FAILED','INVALID_CREDENTIALS')`},
|
||||
{id: "sync_errors", title: "Unresolved sync transport reports", unit: "reports", condition: "recovery_unverified", severity: "warning", note: "Deduplicated first stored report, not latest failing attempt or recurrence. Includes inactive accounts.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT r.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(r.created_at::timestamptz), MAX(r.created_at::timestamptz), MIN(NULL::timestamptz) FROM email_account_errors r LEFT JOIN email_accounts e ON e.id=r.email_account_id WHERE r.resolved_at IS NULL AND r.error_code IN ('SYNC_ERROR','SYNC_AUTH_ERROR','SYNC_CONNECTION_ERROR')`},
|
||||
{id: "sync_safety_reports", title: "Unresolved sync flood/fair-use reports", unit: "reports", condition: "safety_hold", severity: "warning", note: "Deduplicated first stored report, not latest failing attempt or recurrence. Includes inactive accounts.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT r.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(r.created_at::timestamptz), MAX(r.created_at::timestamptz), MIN(NULL::timestamptz) FROM email_account_errors r LEFT JOIN email_accounts e ON e.id=r.email_account_id WHERE r.resolved_at IS NULL AND r.error_code IN ('SYNC_FLOOD','SYNC_FAIR_USE')`},
|
||||
{id: "other_mailbox_errors", title: "Other unresolved mailbox reports", unit: "reports", condition: "recovery_unverified", severity: "warning", note: "Deduplicated first stored report, not latest failing attempt or recurrence. Includes inactive accounts.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT r.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(r.created_at::timestamptz), MAX(r.created_at::timestamptz), MIN(NULL::timestamptz) FROM email_account_errors r LEFT JOIN email_accounts e ON e.id=r.email_account_id WHERE r.resolved_at IS NULL AND r.error_code NOT IN ('GOOGLE_AUTHENTICATION_FAILED','AUTHENTICATION_FAILED','AUTHORIZATION_FAILED','INVALID_CREDENTIALS','SYNC_ERROR','SYNC_AUTH_ERROR','SYNC_CONNECTION_ERROR','SYNC_FLOOD','SYNC_FAIR_USE')`},
|
||||
{id: "inactive_credential_errors", title: "Inactive mailboxes with unresolved credential reports", unit: "reports", condition: "safety_hold", severity: "warning", note: "Inactive without matching error is not an authentication failure.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT r.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(r.created_at::timestamptz), MAX(r.created_at::timestamptz), MIN(NULL::timestamptz) FROM email_account_errors r JOIN email_accounts e ON e.id=r.email_account_id WHERE r.resolved_at IS NULL AND e.status<>'active' AND r.error_code IN ('GOOGLE_AUTHENTICATION_FAILED','AUTHENTICATION_FAILED','AUTHORIZATION_FAILED','INVALID_CREDENTIALS')`},
|
||||
}},
|
||||
"jobs": {permission: models.AdminPermViewAnalytics, investigate: "/jobs", queries: []monitoringQuery{
|
||||
{id: "job_error_recent", title: "Jobs whose latest completion errored within 1h", unit: "jobs", condition: "recent_failure", severity: "warning", note: "Name-keyed instance rows, not replica heartbeats; wrapper return does not prove subsystem throughput.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, NULL::bigint, NULL::bigint, MIN(j.last_finished_at::timestamptz), MAX(j.last_finished_at::timestamptz), MIN(j.next_run_at::timestamptz) FROM scheduled_job_runs j WHERE j.last_status='error' AND j.last_finished_at BETWEEN $1-INTERVAL '1 hour' AND $1`},
|
||||
{id: "job_error_old", title: "Jobs with older recorded error", unit: "jobs", condition: "recovery_unverified", severity: "warning", note: "Name-keyed instance rows, not replica heartbeats; wrapper return does not prove subsystem throughput.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, NULL::bigint, NULL::bigint, MIN(j.last_finished_at::timestamptz), MAX(j.last_finished_at::timestamptz), MIN(j.next_run_at::timestamptz) FROM scheduled_job_runs j WHERE j.last_status='error' AND (j.last_finished_at IS NULL OR j.last_finished_at<$1-INTERVAL '1 hour')`},
|
||||
{id: "job_running", title: "Recorded running jobs", unit: "jobs", condition: "in_flight", severity: "info", note: "Name-keyed instance rows, not replica heartbeats; wrapper return does not prove subsystem throughput.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, NULL::bigint, NULL::bigint, MIN(j.last_started_at::timestamptz), MAX(j.last_finished_at::timestamptz), MIN(j.next_run_at::timestamptz) FROM scheduled_job_runs j WHERE j.last_status='running'`},
|
||||
{id: "job_future", title: "Recorded future job schedules", unit: "jobs", condition: "waiting_schedule", severity: "info", note: "Name-keyed instance rows, not replica heartbeats; wrapper return does not prove subsystem throughput.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, NULL::bigint, NULL::bigint, MIN(j.last_finished_at::timestamptz), MAX(j.last_finished_at::timestamptz), MIN(j.next_run_at::timestamptz) FROM scheduled_job_runs j WHERE j.last_status<>'running' AND j.next_run_at>$1`},
|
||||
{id: "job_overdue", title: "Recorded job schedule past grace", unit: "jobs", condition: "backlog", severity: "warning", note: "Name-keyed instance rows, not replica heartbeats; wrapper return does not prove subsystem throughput.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, NULL::bigint, NULL::bigint, MIN(j.last_finished_at::timestamptz), MAX(j.last_finished_at::timestamptz), MIN(j.next_run_at::timestamptz) FROM scheduled_job_runs j WHERE j.last_status<>'running' AND j.next_run_at IS NOT NULL AND j.next_run_at+make_interval(secs=>GREATEST(60,LEAST(j.interval_seconds,300)))<$1`},
|
||||
{id: "job_ok", title: "Jobs with latest successful wrapper completion", unit: "jobs", condition: "healthy", severity: "info", note: "Name-keyed instance rows, not replica heartbeats; wrapper return does not prove subsystem throughput.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, NULL::bigint, NULL::bigint, MIN(j.last_finished_at::timestamptz), MAX(j.last_finished_at::timestamptz), MIN(j.next_run_at::timestamptz) FROM scheduled_job_runs j WHERE j.last_status='ok'`},
|
||||
}},
|
||||
"dead_letters": {permission: models.AdminPermViewCampaigns, investigate: "/sends?tab=dead-letters", queries: []monitoringQuery{
|
||||
{id: "dlq_due", title: "Dead letters due for retry", unit: "records", condition: "backlog", severity: "warning", note: "Durable queue state; attempts are delivery/replay attempts, replay is not provider confirmation.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(d.next_retry_at::timestamptz), MAX(d.updated_at::timestamptz), MIN(d.next_retry_at::timestamptz) FROM task_dead_letters d LEFT JOIN tasks t ON t.id=d.task_id LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE d.status='pending' AND d.next_retry_at<=$1 AND d.attempts<d.max_attempts`},
|
||||
{id: "dlq_wait", title: "Dead letters awaiting retry slot", unit: "records", condition: "waiting_retry", severity: "info", note: "Durable queue state; attempts are delivery/replay attempts, replay is not provider confirmation.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(d.created_at::timestamptz), MAX(d.updated_at::timestamptz), MIN(d.next_retry_at::timestamptz) FROM task_dead_letters d LEFT JOIN tasks t ON t.id=d.task_id LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE d.status='pending' AND d.next_retry_at>$1 AND d.attempts<d.max_attempts`},
|
||||
{id: "dlq_exhausted", title: "Dead letters with exhausted attempts", unit: "records", condition: "recovery_unverified", severity: "warning", note: "Durable queue state; attempts are delivery/replay attempts, replay is not provider confirmation.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(d.created_at::timestamptz), MAX(d.updated_at::timestamptz), MIN(d.next_retry_at::timestamptz) FROM task_dead_letters d LEFT JOIN tasks t ON t.id=d.task_id LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE d.status IN ('pending','failed') AND d.attempts>=d.max_attempts`},
|
||||
{id: "dlq_unscheduled", title: "Pending dead letters without retry slot", unit: "records", condition: "unknown", severity: "warning", note: "Durable queue state; attempts are delivery/replay attempts, replay is not provider confirmation.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(d.created_at::timestamptz), MAX(d.updated_at::timestamptz), MIN(d.next_retry_at::timestamptz) FROM task_dead_letters d LEFT JOIN tasks t ON t.id=d.task_id LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE d.status='pending' AND d.next_retry_at IS NULL AND d.attempts<d.max_attempts`},
|
||||
{id: "dlq_replayed", title: "Replayed dead letters", unit: "records", condition: "unknown", severity: "info", note: "Durable queue state; attempts are delivery/replay attempts, replay is not provider confirmation.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(d.created_at::timestamptz), MAX(d.updated_at::timestamptz), MIN(d.next_retry_at::timestamptz) FROM task_dead_letters d LEFT JOIN tasks t ON t.id=d.task_id LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE d.status='replayed'`},
|
||||
{id: "task_failures_recent", title: "Currently failed tasks updated within 1h", unit: "tasks", condition: "recent_failure", severity: "warning", note: "Mutable task update proxy; not immutable failure-event time.", window: 3600 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(t.updated_at::timestamptz), MAX(t.updated_at::timestamptz), MIN(NULL::timestamptz) FROM task_failures f JOIN tasks t ON t.id=f.task_id LEFT JOIN email_accounts e ON e.id=t.email_account_id WHERE t.status='failed' AND t.updated_at BETWEEN $1-INTERVAL '1 hour' AND $1`},
|
||||
}},
|
||||
"webhooks": {permission: models.AdminPermViewOrganizations, investigate: "", queries: []monitoringQuery{
|
||||
{id: "webhook_due", title: "Due webhook deliveries", unit: "deliveries", condition: "backlog", severity: "info", note: "Current row state, not HTTP attempt history; native lease default 5m.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT d.organization_id), COUNT(*) FILTER(WHERE d.organization_id IS NULL), MIN(d.next_attempt_at::timestamptz), MAX(d.updated_at::timestamptz), MIN(d.next_attempt_at::timestamptz) FROM webhook_deliveries d WHERE d.status='pending' AND d.next_attempt_at<=$1`},
|
||||
{id: "webhook_wait", title: "Future webhook retry slots", unit: "deliveries", condition: "waiting_retry", severity: "info", note: "Current row state, not HTTP attempt history; native lease default 5m.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT d.organization_id), COUNT(*) FILTER(WHERE d.organization_id IS NULL), MIN(d.updated_at::timestamptz), MAX(d.updated_at::timestamptz), MIN(d.next_attempt_at::timestamptz) FROM webhook_deliveries d WHERE d.status='pending' AND d.next_attempt_at>$1`},
|
||||
{id: "webhook_retry", title: "Pending webhooks with prior attempts", unit: "deliveries", condition: "waiting_retry", severity: "info", note: "Current row state, not HTTP attempt history; native lease default 5m.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT d.organization_id), COUNT(*) FILTER(WHERE d.organization_id IS NULL), MIN(d.updated_at::timestamptz), MAX(d.updated_at::timestamptz), MIN(d.next_attempt_at::timestamptz) FROM webhook_deliveries d WHERE d.status='pending' AND d.attempt_count>0`},
|
||||
{id: "webhook_stale_claim", title: "Webhook claims older than 5m", unit: "deliveries", condition: "stale_telemetry", severity: "warning", note: "Current row state, not HTTP attempt history; native lease default 5m.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT d.organization_id), COUNT(*) FILTER(WHERE d.organization_id IS NULL), MIN(d.updated_at::timestamptz), MAX(d.updated_at::timestamptz), MIN(d.next_attempt_at::timestamptz) FROM webhook_deliveries d WHERE d.status='in_flight' AND d.updated_at<$1-INTERVAL '5 minutes'`},
|
||||
{id: "webhook_abandoned_24h", title: "Abandoned webhook rows settled within 24h", unit: "deliveries", condition: "recent_failure", severity: "warning", note: "Current row state, not HTTP attempt history; native lease default 5m.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT d.organization_id), COUNT(*) FILTER(WHERE d.organization_id IS NULL), MIN(d.updated_at::timestamptz), MAX(d.updated_at::timestamptz), MIN(d.next_attempt_at::timestamptz) FROM webhook_deliveries d WHERE d.status='abandoned' AND d.updated_at BETWEEN $1-INTERVAL '24 hours' AND $1`},
|
||||
{id: "webhook_delivered_24h", title: "Delivered webhook rows settled within 24h", unit: "deliveries", condition: "healthy", severity: "info", note: "Current row state, not HTTP attempt history; native lease default 5m.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT d.organization_id), COUNT(*) FILTER(WHERE d.organization_id IS NULL), MIN(d.updated_at::timestamptz), MAX(d.updated_at::timestamptz), MIN(d.next_attempt_at::timestamptz) FROM webhook_deliveries d WHERE d.status='delivered' AND d.updated_at BETWEEN $1-INTERVAL '24 hours' AND $1`},
|
||||
{id: "webhook_endpoint_recent", title: "Endpoints with recent failure evidence", unit: "endpoints", condition: "recent_failure", severity: "warning", note: "Endpoint-level evidence is independent of individual delivery outcomes; exact aggregate, not capped examples.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(e.first_failure_at::timestamptz), MAX(e.last_failure_at::timestamptz), MIN(NULL::timestamptz) FROM webhook_endpoints e WHERE e.consecutive_failures>0 AND e.last_failure_at BETWEEN $1-INTERVAL '1 hour' AND $1`},
|
||||
{id: "webhook_endpoint_persistent", title: "Endpoints with repeated sustained recent failures", unit: "endpoints", condition: "persistent_failure", severity: "warning", note: "Endpoint-level evidence is independent of individual delivery outcomes; exact aggregate, not capped examples.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(e.first_failure_at::timestamptz), MAX(e.last_failure_at::timestamptz), MIN(NULL::timestamptz) FROM webhook_endpoints e WHERE e.consecutive_failures>=5 AND e.first_failure_at<=$1-INTERVAL '72 hours' AND e.last_failure_at BETWEEN $1-INTERVAL '1 hour' AND $1`},
|
||||
{id: "webhook_endpoint_old", title: "Endpoints with old unrecovered failure evidence", unit: "endpoints", condition: "recovery_unverified", severity: "warning", note: "Endpoint-level evidence is independent of individual delivery outcomes; exact aggregate, not capped examples.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(e.first_failure_at::timestamptz), MAX(e.last_failure_at::timestamptz), MIN(NULL::timestamptz) FROM webhook_endpoints e WHERE e.consecutive_failures>0 AND (e.last_failure_at IS NULL OR e.last_failure_at<$1-INTERVAL '1 hour')`},
|
||||
{id: "webhook_auto_disabled", title: "Auto-disabled webhook endpoints", unit: "endpoints", condition: "safety_hold", severity: "warning", note: "Endpoint-level evidence is independent of individual delivery outcomes; exact aggregate, not capped examples.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(e.first_failure_at::timestamptz), MAX(e.last_failure_at::timestamptz), MIN(NULL::timestamptz) FROM webhook_endpoints e WHERE e.auto_disabled_at IS NOT NULL AND NOT e.enabled`},
|
||||
{id: "webhook_drop_windows", title: "Rate-limited webhook windows (8 UTC dates)", unit: "windows", condition: "budget", severity: "info", note: "Today plus seven preceding UTC dates; throttled windows, not individual dropped events.", window: 0 * time.Second, query: `SELECT COALESCE(SUM(d.dropped_windows),0)::bigint, NULL::bigint, COUNT(DISTINCT d.organization_id), COUNT(*) FILTER(WHERE d.organization_id IS NULL), MIN(d.last_dropped_at::timestamptz), MAX(d.last_dropped_at::timestamptz), MIN(NULL::timestamptz) FROM webhook_event_drops d WHERE d.day>=($1 AT TIME ZONE 'UTC')::date-7 AND d.day<=($1 AT TIME ZONE 'UTC')::date`},
|
||||
}},
|
||||
"notifications": {permission: models.AdminPermManageSettings, investigate: "/health", queries: []monitoringQuery{
|
||||
{id: "notification_due", title: "Due notification email rows", unit: "rows", condition: "backlog", severity: "info", note: "Digest rows are not physical emails; sent is settlement, skipped reasons/exhaustion time are unavailable.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT n.organization_id), COUNT(*) FILTER(WHERE n.organization_id IS NULL), MIN(n.email_due_at::timestamptz), MAX(n.email_due_at::timestamptz), MIN(CASE WHEN n.email_state='pending' THEN n.email_due_at END::timestamptz) FROM notifications n WHERE n.email_state='pending' AND n.email_due_at<=$1`},
|
||||
{id: "notification_aging", title: "Notification due rows older than 2m", unit: "rows", condition: "backlog", severity: "warning", note: "Digest rows are not physical emails; sent is settlement, skipped reasons/exhaustion time are unavailable.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT n.organization_id), COUNT(*) FILTER(WHERE n.organization_id IS NULL), MIN(n.email_due_at::timestamptz), MAX(n.email_due_at::timestamptz), MIN(CASE WHEN n.email_state='pending' THEN n.email_due_at END::timestamptz) FROM notifications n WHERE n.email_state='pending' AND n.email_due_at<$1-INTERVAL '2 minutes'`},
|
||||
{id: "notification_wait", title: "Future notification email rows", unit: "rows", condition: "waiting_schedule", severity: "info", note: "Digest rows are not physical emails; sent is settlement, skipped reasons/exhaustion time are unavailable.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT n.organization_id), COUNT(*) FILTER(WHERE n.organization_id IS NULL), MIN(n.email_due_at::timestamptz), MAX(n.email_due_at::timestamptz), MIN(CASE WHEN n.email_state='pending' THEN n.email_due_at END::timestamptz) FROM notifications n WHERE n.email_state='pending' AND n.email_due_at>$1`},
|
||||
{id: "notification_retry", title: "Requeued notification email rows", unit: "rows", condition: "waiting_retry", severity: "info", note: "Digest rows are not physical emails; sent is settlement, skipped reasons/exhaustion time are unavailable.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT n.organization_id), COUNT(*) FILTER(WHERE n.organization_id IS NULL), MIN(n.email_due_at::timestamptz), MAX(n.email_due_at::timestamptz), MIN(CASE WHEN n.email_state='pending' THEN n.email_due_at END::timestamptz) FROM notifications n WHERE n.email_state='pending' AND n.email_attempts>0`},
|
||||
{id: "notification_sending", title: "Claimed notification email rows", unit: "rows", condition: "in_flight", severity: "info", note: "Digest rows are not physical emails; sent is settlement, skipped reasons/exhaustion time are unavailable.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT n.organization_id), COUNT(*) FILTER(WHERE n.organization_id IS NULL), MIN(n.email_due_at::timestamptz), MAX(n.email_due_at::timestamptz), MIN(CASE WHEN n.email_state='pending' THEN n.email_due_at END::timestamptz) FROM notifications n WHERE n.email_state='sending'`},
|
||||
{id: "notification_stale", title: "Notification claims older than 10m", unit: "rows", condition: "stale_telemetry", severity: "warning", note: "Digest rows are not physical emails; sent is settlement, skipped reasons/exhaustion time are unavailable.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT n.organization_id), COUNT(*) FILTER(WHERE n.organization_id IS NULL), MIN(n.email_due_at::timestamptz), MAX(n.email_due_at::timestamptz), MIN(CASE WHEN n.email_state='pending' THEN n.email_due_at END::timestamptz) FROM notifications n WHERE n.email_state='sending' AND n.email_due_at<$1-INTERVAL '10 minutes'`},
|
||||
{id: "notification_exhausted", title: "Skipped notification rows with exhausted requeues", unit: "rows", condition: "recovery_unverified", severity: "warning", note: "Digest rows are not physical emails; sent is settlement, skipped reasons/exhaustion time are unavailable.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT n.organization_id), COUNT(*) FILTER(WHERE n.organization_id IS NULL), MIN(NULL::timestamptz), MAX(NULL::timestamptz), MIN(CASE WHEN n.email_state='pending' THEN n.email_due_at END::timestamptz) FROM notifications n WHERE n.email_state='skipped' AND n.email_attempts>=3`},
|
||||
{id: "notification_settled_24h", title: "Notification rows settled within 24h", unit: "rows", condition: "unknown", severity: "info", note: "Digest rows are not physical emails; sent is settlement, skipped reasons/exhaustion time are unavailable.", window: 0 * time.Second, query: `SELECT COUNT(*), NULL::bigint, COUNT(DISTINCT n.organization_id), COUNT(*) FILTER(WHERE n.organization_id IS NULL), MIN(n.email_due_at::timestamptz), MAX(n.email_due_at::timestamptz), MIN(CASE WHEN n.email_state='pending' THEN n.email_due_at END::timestamptz) FROM notifications n WHERE n.email_state='sent' AND n.email_due_at BETWEEN $1-INTERVAL '24 hours' AND $1`},
|
||||
}},
|
||||
"result_effects": {permission: models.AdminPermViewCampaigns, investigate: "/sends?tab=in-flight", queries: []monitoringQuery{
|
||||
{id: "effects_pending", title: "Pending send result effects", unit: "effects", condition: "backlog", severity: "info", note: "Outbox effect delivery, not provider execution or sent email count.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(e.created_at::timestamptz), MAX(NULL::timestamptz), MIN(e.locked_until::timestamptz) FROM send_result_effects e LEFT JOIN tasks t ON t.id=e.task_id WHERE e.delivered_at IS NULL`},
|
||||
{id: "effects_unleased", title: "Pending unleased result effects", unit: "effects", condition: "backlog", severity: "info", note: "Outbox effect delivery, not provider execution or sent email count.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(e.created_at::timestamptz), MAX(NULL::timestamptz), MIN(e.locked_until::timestamptz) FROM send_result_effects e LEFT JOIN tasks t ON t.id=e.task_id WHERE e.delivered_at IS NULL AND (e.locked_until IS NULL OR e.locked_until<=$1)`},
|
||||
{id: "effects_leased", title: "Currently leased result effects", unit: "effects", condition: "in_flight", severity: "info", note: "Outbox effect delivery, not provider execution or sent email count.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(e.created_at::timestamptz), MAX(NULL::timestamptz), MIN(e.locked_until::timestamptz) FROM send_result_effects e LEFT JOIN tasks t ON t.id=e.task_id WHERE e.delivered_at IS NULL AND e.locked_until>$1`},
|
||||
{id: "effects_repeated", title: "Pending effects with repeated attempts", unit: "effects", condition: "waiting_retry", severity: "warning", note: "Outbox effect delivery, not provider execution or sent email count.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT t.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(e.created_at::timestamptz), MAX(NULL::timestamptz), MIN(e.locked_until::timestamptz) FROM send_result_effects e LEFT JOIN tasks t ON t.id=e.task_id WHERE e.delivered_at IS NULL AND e.attempts>1`},
|
||||
}},
|
||||
"arrivals": {permission: models.AdminPermViewUsers, investigate: "/sync", queries: []monitoringQuery{
|
||||
{id: "arrival_pending", title: "Pending durable arrivals", unit: "arrivals", condition: "backlog", severity: "info", note: "Optional outbox coverage excludes legacy published arrivals; stages/leases/slots are not historical retries.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT a.email_id), COUNT(DISTINCT a.organization_id), COUNT(*) FILTER(WHERE a.organization_id IS NULL), MIN(a.created_at::timestamptz), MAX(NULL::timestamptz), MIN(a.retry_at::timestamptz) FROM sync_arrival_outbox a WHERE TRUE`},
|
||||
{id: "arrival_due", title: "Due unleased durable arrivals", unit: "arrivals", condition: "backlog", severity: "info", note: "Optional outbox coverage excludes legacy published arrivals; stages/leases/slots are not historical retries.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT a.email_id), COUNT(DISTINCT a.organization_id), COUNT(*) FILTER(WHERE a.organization_id IS NULL), MIN(a.created_at::timestamptz), MAX(NULL::timestamptz), MIN(a.retry_at::timestamptz) FROM sync_arrival_outbox a WHERE a.retry_at<=$1 AND (a.locked_until IS NULL OR a.locked_until<$1)`},
|
||||
{id: "arrival_leased", title: "Leased durable arrivals", unit: "arrivals", condition: "in_flight", severity: "info", note: "Optional outbox coverage excludes legacy published arrivals; stages/leases/slots are not historical retries.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT a.email_id), COUNT(DISTINCT a.organization_id), COUNT(*) FILTER(WHERE a.organization_id IS NULL), MIN(a.created_at::timestamptz), MAX(NULL::timestamptz), MIN(a.retry_at::timestamptz) FROM sync_arrival_outbox a WHERE a.locked_until>$1`},
|
||||
{id: "arrival_wait", title: "Future arrival retry slots", unit: "arrivals", condition: "waiting_retry", severity: "info", note: "Optional outbox coverage excludes legacy published arrivals; stages/leases/slots are not historical retries.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT a.email_id), COUNT(DISTINCT a.organization_id), COUNT(*) FILTER(WHERE a.organization_id IS NULL), MIN(a.created_at::timestamptz), MAX(NULL::timestamptz), MIN(a.retry_at::timestamptz) FROM sync_arrival_outbox a WHERE a.retry_at>$1`},
|
||||
{id: "arrival_aging", title: "Durable arrivals older than 5m", unit: "arrivals", condition: "backlog", severity: "warning", note: "Optional outbox coverage excludes legacy published arrivals; stages/leases/slots are not historical retries.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT a.email_id), COUNT(DISTINCT a.organization_id), COUNT(*) FILTER(WHERE a.organization_id IS NULL), MIN(a.created_at::timestamptz), MAX(NULL::timestamptz), MIN(a.retry_at::timestamptz) FROM sync_arrival_outbox a WHERE a.created_at<=$1-INTERVAL '5 minutes'`},
|
||||
{id: "arrival_stage_0", title: "Durable arrivals at stage 0", unit: "arrivals", condition: "backlog", severity: "info", note: "Optional outbox coverage excludes legacy published arrivals; stages/leases/slots are not historical retries.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT a.email_id), COUNT(DISTINCT a.organization_id), COUNT(*) FILTER(WHERE a.organization_id IS NULL), MIN(a.created_at::timestamptz), MAX(NULL::timestamptz), MIN(a.retry_at::timestamptz) FROM sync_arrival_outbox a WHERE a.stage=0`},
|
||||
{id: "arrival_stage_1", title: "Durable arrivals at stage 1", unit: "arrivals", condition: "backlog", severity: "info", note: "Optional outbox coverage excludes legacy published arrivals; stages/leases/slots are not historical retries.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT a.email_id), COUNT(DISTINCT a.organization_id), COUNT(*) FILTER(WHERE a.organization_id IS NULL), MIN(a.created_at::timestamptz), MAX(NULL::timestamptz), MIN(a.retry_at::timestamptz) FROM sync_arrival_outbox a WHERE a.stage=1`},
|
||||
{id: "arrival_stage_2", title: "Durable arrivals at stage 2", unit: "arrivals", condition: "backlog", severity: "info", note: "Optional outbox coverage excludes legacy published arrivals; stages/leases/slots are not historical retries.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT a.email_id), COUNT(DISTINCT a.organization_id), COUNT(*) FILTER(WHERE a.organization_id IS NULL), MIN(a.created_at::timestamptz), MAX(NULL::timestamptz), MIN(a.retry_at::timestamptz) FROM sync_arrival_outbox a WHERE a.stage=2`},
|
||||
{id: "arrival_stage_3", title: "Durable arrivals at stage 3", unit: "arrivals", condition: "backlog", severity: "info", note: "Optional outbox coverage excludes legacy published arrivals; stages/leases/slots are not historical retries.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT a.email_id), COUNT(DISTINCT a.organization_id), COUNT(*) FILTER(WHERE a.organization_id IS NULL), MIN(a.created_at::timestamptz), MAX(NULL::timestamptz), MIN(a.retry_at::timestamptz) FROM sync_arrival_outbox a WHERE a.stage=3`},
|
||||
}},
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/warmbly/warmbly/internal/app/monitoring"
|
||||
"github.com/warmbly/warmbly/internal/infrastructure/eventbus"
|
||||
"github.com/warmbly/warmbly/internal/infrastructure/kafka"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
)
|
||||
|
||||
func (r *MonitoringRepository) BrokerSources(bus eventbus.EventBus) []monitoring.Source {
|
||||
collect := func(ctx context.Context, at time.Time, scopes []eventbus.DiagnosticScope) models.MonitoringSource {
|
||||
if diagnostics, ok := bus.(eventbus.Diagnostics); ok {
|
||||
return diagnostics.Diagnose(ctx, at, scopes)
|
||||
}
|
||||
return models.MonitoringSource{Availability: models.MonitoringUnavailable, Coverage: "unavailable", Reason: "unsupported"}
|
||||
}
|
||||
return []monitoring.Source{
|
||||
{ID: "backend_event_broker", Permission: models.AdminPermViewAnalytics, Collect: func(ctx context.Context, at time.Time) (models.MonitoringSource, error) {
|
||||
return collect(ctx, at, []eventbus.DiagnosticScope{{ID: "worker_events", Group: "consumer-group", Topics: []string{kafka.TopicWorkerEvents}}}), nil
|
||||
}},
|
||||
{ID: "worker_command_broker", Permission: models.AdminPermViewWorkers, Collect: func(ctx context.Context, at time.Time) (models.MonitoringSource, error) {
|
||||
if r == nil || r.pool == nil {
|
||||
return models.MonitoringSource{Availability: models.MonitoringUnavailable, Coverage: "unavailable", Reason: "dependency_missing"}, nil
|
||||
}
|
||||
ids, expected, err := r.MonitoringWorkerIDs(ctx)
|
||||
if err != nil {
|
||||
return models.MonitoringSource{}, err
|
||||
}
|
||||
scopes := []eventbus.DiagnosticScope{}
|
||||
for _, id := range ids {
|
||||
scopes = append(scopes, eventbus.DiagnosticScope{ID: id, Group: "worker-" + id, Topics: []string{kafka.GetWorkerTopic(id)}})
|
||||
}
|
||||
out := collect(ctx, at, scopes)
|
||||
out.ExpectedScopes = &expected
|
||||
if int64(len(scopes)) < expected && out.Availability != models.MonitoringUnavailable {
|
||||
out.Coverage = "partial"
|
||||
out.Reason = "scope_limit"
|
||||
}
|
||||
for i := range out.Metrics {
|
||||
out.Metrics[i].Investigate = "/workers/" + out.Metrics[i].ScopeID
|
||||
}
|
||||
return out, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/warmbly/warmbly/internal/app/instancesettings"
|
||||
"github.com/warmbly/warmbly/internal/app/monitoring"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
)
|
||||
|
||||
func TestMonitoringAggregatesDoNotLinkToCampaignOnlyDetails(t *testing.T) {
|
||||
for _, id := range []string{"sends", "dispatch"} {
|
||||
if link := monitoringSections[id].investigate; link != "" {
|
||||
t.Errorf("%s links to %q, which does not cover its measured tasks", id, link)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func collectMonitoring(t *testing.T, r *MonitoringRepository, id string, at time.Time) models.MonitoringSource {
|
||||
t.Helper()
|
||||
for _, source := range r.Sources() {
|
||||
if source.ID == id {
|
||||
ctx, cancel := context.WithTimeout(t.Context(), monitoring.SourceTimeout)
|
||||
defer cancel()
|
||||
out, err := source.Collect(ctx, at)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", id, err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
}
|
||||
t.Fatalf("missing source %s", id)
|
||||
return models.MonitoringSource{}
|
||||
}
|
||||
|
||||
func measuredMonitoring(t *testing.T, source models.MonitoringSource, id string) models.MonitoringMetric {
|
||||
t.Helper()
|
||||
for _, m := range source.Metrics {
|
||||
if m.ID == id {
|
||||
return m
|
||||
}
|
||||
}
|
||||
t.Fatalf("missing metric %s", id)
|
||||
return models.MonitoringMetric{}
|
||||
}
|
||||
|
||||
func monitoringReadOnlyPool(t *testing.T, pool *pgxpool.Pool) *pgxpool.Pool {
|
||||
t.Helper()
|
||||
cfg := pool.Config().Copy()
|
||||
cfg.ConnConfig.RuntimeParams["default_transaction_read_only"] = "on"
|
||||
ro, err := pgxpool.NewWithConfig(t.Context(), cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(ro.Close)
|
||||
return ro
|
||||
}
|
||||
|
||||
func TestLiveMonitoringReadOnlySnapshotAndPrivacy(t *testing.T) {
|
||||
_, pool := liveContactDB(t)
|
||||
requireSchemaVersion(t, pool, 277)
|
||||
f := newWarmupUsageFixture(t, pool)
|
||||
f.exec(`UPDATE email_accounts SET signature_plain='private-monitoring-content',email='private-monitoring@example.test' WHERE id=$1`, f.account)
|
||||
r := NewMonitoringRepository(monitoringReadOnlyPool(t, pool))
|
||||
out := monitoring.New(r.Sources()).Snapshot(t.Context(), models.AdminPermViewAnalytics|models.AdminPermViewUsers|models.AdminPermViewWorkers|models.AdminPermViewCampaigns|models.AdminPermViewOrganizations|models.AdminPermManageSettings)
|
||||
for _, source := range out.Sources {
|
||||
if source.ID == "arrivals" {
|
||||
if source.Reason != "schema_absent" || source.Availability != models.MonitoringUnavailable {
|
||||
t.Fatal(source)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if source.Availability != models.MonitoringFresh {
|
||||
t.Fatalf("source %s unavailable: %+v", source.ID, source)
|
||||
}
|
||||
if source.ID == "worker_samples" {
|
||||
for _, m := range source.Metrics {
|
||||
if m.Count == nil && (m.Availability != models.MonitoringUnavailable || m.Condition != models.MonitoringUnknown) {
|
||||
t.Fatal(m)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
raw, err := json.Marshal(out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, private := range []string{"private-monitoring", f.account.String(), f.org.String(), f.user.String()} {
|
||||
if strings.Contains(string(raw), private) {
|
||||
t.Fatalf("private evidence %q leaked", private)
|
||||
}
|
||||
}
|
||||
if out.Coverage == "complete" {
|
||||
t.Fatal("absent optional arrival source became all-clear")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLiveMonitoringProviderEvidenceHoldsAndDispatch(t *testing.T) {
|
||||
_, pool := liveContactDB(t)
|
||||
requireSchemaVersion(t, pool, 277)
|
||||
f := newWarmupUsageFixture(t, pool)
|
||||
r := NewMonitoringRepository(pool)
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
baseSends, baseDispatch, baseSafety := collectMonitoring(t, r, "sends", now), collectMonitoring(t, r, "dispatch", now), collectMonitoring(t, r, "send_safety", now)
|
||||
count := func(source models.MonitoringSource, id string) int64 {
|
||||
t.Helper()
|
||||
m := measuredMonitoring(t, source, id)
|
||||
if m.Count == nil {
|
||||
t.Fatal(m)
|
||||
}
|
||||
return *m.Count
|
||||
}
|
||||
id := uuid.New()
|
||||
at := now.Add(-10 * time.Minute)
|
||||
f.exec(`INSERT INTO tasks(id,task_type,email_account_id,status,message_id,completed_at,send_result_state,send_result_applied_at) VALUES($1,'warmup',$2,'completed','private-task-message',$3,'sent',$3)`, id, f.account, at)
|
||||
f.exec(`INSERT INTO warmup_tokens(token,task_id,sender_account_id,recipient_account_id,sent_message_id) VALUES($1,$2,$3,$3,'')`, uuid.New(), id, f.account)
|
||||
if source := collectMonitoring(t, r, "sends", now); count(source, "confirmed_warmup_1h") != count(baseSends, "confirmed_warmup_1h") {
|
||||
t.Fatal("completed task or send result without provider token counted")
|
||||
}
|
||||
f.exec(`UPDATE warmup_tokens SET sent_message_id='private-provider-id' WHERE task_id=$1`, id)
|
||||
if source := collectMonitoring(t, r, "sends", now); count(source, "confirmed_warmup_1h") != count(baseSends, "confirmed_warmup_1h")+1 {
|
||||
t.Fatal("provider token not counted")
|
||||
}
|
||||
f.exec(`UPDATE tasks SET send_result_applied_at=NULL WHERE id=$1`, id)
|
||||
if source := collectMonitoring(t, r, "sends", now); count(source, "confirmed_warmup_proxy_1h") != count(baseSends, "confirmed_warmup_proxy_1h")+1 {
|
||||
t.Fatal("legacy clock proxy missing")
|
||||
}
|
||||
for _, reason := range []string{"unknown", "conflict", "permanent", "authentication"} {
|
||||
f.exec(`UPDATE email_accounts SET send_recovery_hold=true,send_recovery_reason=$2 WHERE id=$1`, f.account, reason)
|
||||
if source := collectMonitoring(t, r, "send_safety", now); count(source, "holds_"+reason) != count(baseSafety, "holds_"+reason)+1 {
|
||||
t.Fatal(reason, source)
|
||||
}
|
||||
}
|
||||
f.exec(`UPDATE email_accounts SET send_cooldown_until=$2 WHERE id=$1`, f.account, now)
|
||||
if source := collectMonitoring(t, r, "send_safety", now); count(source, "send_cooldowns") != count(baseSafety, "send_cooldowns") {
|
||||
t.Fatal("expired cooldown counted")
|
||||
}
|
||||
f.exec(`UPDATE email_accounts SET send_cooldown_until=$2 WHERE id=$1`, f.account, now.Add(time.Minute))
|
||||
if source := collectMonitoring(t, r, "send_safety", now); count(source, "send_cooldowns") != count(baseSafety, "send_cooldowns")+1 {
|
||||
t.Fatal(source)
|
||||
}
|
||||
f.exec(`INSERT INTO tasks(id,task_type,email_account_id,status,message_id,scheduled_at,dispatch_retry_at,dispatch_failure_since,dispatch_failure_at) VALUES($1,'warmup',$2,'pending','',$3,$4,$5,$6)`, uuid.New(), f.account, now.Add(-time.Hour), now.Add(time.Minute), now.Add(-20*time.Minute), now.Add(-time.Minute))
|
||||
f.exec(`INSERT INTO tasks(id,task_type,email_account_id,status,message_id,scheduled_at) VALUES($1,'warmup',$2,'pending','',$3),($4,'warmup',$2,'pending','',$5)`, uuid.New(), f.account, now.Add(-time.Minute), uuid.New(), now.Add(time.Hour))
|
||||
dispatch := collectMonitoring(t, r, "dispatch", now)
|
||||
for id, delta := range map[string]int64{"dispatch_due": 1, "retry_wait": 1, "future_schedule": 1, "dispatch_persistent": 1} {
|
||||
if count(dispatch, id) != count(baseDispatch, id)+delta {
|
||||
t.Fatal(id, dispatch)
|
||||
}
|
||||
}
|
||||
if measuredMonitoring(t, dispatch, "retry_wait").Condition != models.MonitoringRetry {
|
||||
t.Fatal("retry became failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLiveMonitoringSyncAuthAndDomainPolicy(t *testing.T) {
|
||||
_, pool := liveContactDB(t)
|
||||
requireSchemaVersion(t, pool, 277)
|
||||
f := newWarmupUsageFixture(t, pool)
|
||||
r := NewMonitoringRepository(pool)
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
f.exec(`UPDATE email_accounts SET status='active' WHERE id=$1`, f.account)
|
||||
baseSync, baseErrors, baseDomain := collectMonitoring(t, r, "sync", now), collectMonitoring(t, r, "mailbox_errors", now), collectMonitoring(t, r, "domain_auth", now)
|
||||
count := func(source models.MonitoringSource, id string) int64 {
|
||||
t.Helper()
|
||||
return *measuredMonitoring(t, source, id).Count
|
||||
}
|
||||
f.exec(`INSERT INTO email_sync_state(email_id,user_id,backfill_status,backfill_synced,deferred,last_synced_at,throttled_until,throttle_reason,updated_at) VALUES($1,$2,'running',900,7,$3,$4,'daily',$3)`, f.account, f.user, now.Add(-2*time.Hour), now.Add(time.Minute))
|
||||
sync := collectMonitoring(t, r, "sync", now)
|
||||
if count(sync, "sync_deferred") != count(baseSync, "sync_deferred")+7 || count(sync, "sync_hold_daily") != count(baseSync, "sync_hold_daily")+1 || measuredMonitoring(t, sync, "sync_hold_daily").Condition != models.MonitoringDailyLimit || count(sync, "backfill_stale_running") != count(baseSync, "backfill_stale_running") {
|
||||
t.Fatal(sync)
|
||||
}
|
||||
f.exec(`UPDATE email_sync_state SET throttled_until=$2 WHERE email_id=$1`, f.account, now)
|
||||
if sync = collectMonitoring(t, r, "sync", now); count(sync, "sync_hold_daily") != count(baseSync, "sync_hold_daily") || count(sync, "backfill_stale_running") != count(baseSync, "backfill_stale_running")+1 {
|
||||
t.Fatal("expired hold remains active", sync)
|
||||
}
|
||||
for _, code := range []string{"AUTHENTICATION_FAILED", "INVALID_CREDENTIALS"} {
|
||||
f.exec(`INSERT INTO email_account_errors(id,email_account_id,user_id,error_code,severity,title,message,created_at) VALUES($1,$2,$3,$4,'CRITICAL','private error','private provider cursor',$5)`, uuid.New(), f.account, f.user, code, now.Add(-2*time.Hour))
|
||||
}
|
||||
errors := collectMonitoring(t, r, "mailbox_errors", now)
|
||||
credential := measuredMonitoring(t, errors, "credential_errors")
|
||||
base := measuredMonitoring(t, baseErrors, "credential_errors")
|
||||
if *credential.Count != *base.Count+2 || *credential.AffectedMailboxes != *base.AffectedMailboxes+1 || credential.Condition != models.MonitoringRecoveryUnverified {
|
||||
t.Fatal(credential)
|
||||
}
|
||||
f.exec(`UPDATE email_accounts SET status='inactive',auth_state='failing',auth_checked_at=$2,auth_failing_since=$3 WHERE id=$1`, f.account, now.Add(-time.Hour), now.Add(-25*time.Hour))
|
||||
if errors = collectMonitoring(t, r, "mailbox_errors", now); count(errors, "inactive_credential_errors") != count(baseErrors, "inactive_credential_errors")+2 {
|
||||
t.Fatal(errors)
|
||||
}
|
||||
store := instancesettings.NewStore(pool)
|
||||
policy, err := store.Get(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var original []byte
|
||||
err = pool.QueryRow(t.Context(), `SELECT doc FROM instance_settings WHERE id=true`).Scan(&original)
|
||||
hadRow := err == nil
|
||||
t.Cleanup(func() {
|
||||
if hadRow {
|
||||
_, _ = pool.Exec(context.Background(), `UPDATE instance_settings SET doc=$1::jsonb WHERE id=true`, original)
|
||||
} else {
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM instance_settings WHERE id=true`)
|
||||
}
|
||||
})
|
||||
policy.Deliverability.EnforceDomainAuth = false
|
||||
if err := store.Put(t.Context(), policy, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if source := collectMonitoring(t, r, "domain_auth", now); count(source, "domain_blocked") != 0 {
|
||||
t.Fatal("disabled enforcement blocked", source)
|
||||
}
|
||||
policy.Deliverability.EnforceDomainAuth = true
|
||||
policy.Deliverability.AuthGraceHours = 24
|
||||
if err := store.Put(t.Context(), policy, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if source := collectMonitoring(t, r, "domain_auth", now); count(source, "domain_blocked") != count(baseDomain, "domain_blocked")+1 {
|
||||
t.Fatal("enforcement/grace ignored", source)
|
||||
}
|
||||
f.exec(`UPDATE email_accounts SET auth_failing_since=$2 WHERE id=$1`, f.account, now.Add(-23*time.Hour))
|
||||
if source := collectMonitoring(t, r, "domain_auth", now); count(source, "domain_blocked") != count(baseDomain, "domain_blocked") {
|
||||
t.Fatal("grace blocked", source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLiveMonitoringLoadingIncidentRecoveryNeedsConfirmedToken(t *testing.T) {
|
||||
_, pool := liveContactDB(t)
|
||||
requireSchemaVersion(t, pool, 277)
|
||||
f := newWarmupUsageFixture(t, pool)
|
||||
r := NewMonitoringRepository(pool)
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
f.exec(`UPDATE email_accounts SET status='active',warmup=$2,test_mode='legacy' WHERE id=$1`, f.account, now.Add(-24*time.Hour))
|
||||
base := collectMonitoring(t, r, "warmup_loading", now)
|
||||
for _, at := range []time.Time{now.Add(-80 * time.Minute), now.Add(-40 * time.Minute), now.Add(-time.Minute)} {
|
||||
id := uuid.New()
|
||||
f.exec(`INSERT INTO tasks(id,task_type,email_account_id,status,message_id,updated_at) VALUES($1,'warmup',$2,'failed','',$3)`, id, f.account, at)
|
||||
f.exec(`INSERT INTO task_failures(task_id,title,message) VALUES($1,'private', $2)`, id, models.MailboxNotLoadedPrefix+"private-provider-cursor")
|
||||
}
|
||||
count := func(source models.MonitoringSource, id string) int64 { return *measuredMonitoring(t, source, id).Count }
|
||||
if source := collectMonitoring(t, r, "warmup_loading", now); count(source, "loading_persistent") != count(base, "loading_persistent")+1 {
|
||||
t.Fatal(source)
|
||||
}
|
||||
id := uuid.New()
|
||||
f.exec(`INSERT INTO tasks(id,task_type,email_account_id,status,message_id,completed_at) VALUES($1,'warmup',$2,'completed','not-confirmed',$3)`, id, f.account, now)
|
||||
f.exec(`INSERT INTO warmup_tokens(token,task_id,sender_account_id,recipient_account_id,sent_message_id) VALUES($1,$2,$3,$3,'')`, uuid.New(), id, f.account)
|
||||
if source := collectMonitoring(t, r, "warmup_loading", now); count(source, "loading_persistent") != count(base, "loading_persistent")+1 {
|
||||
t.Fatal("empty token cleared loading", source)
|
||||
}
|
||||
f.exec(`UPDATE warmup_tokens SET sent_message_id='confirmed' WHERE task_id=$1`, id)
|
||||
if source := collectMonitoring(t, r, "warmup_loading", now); count(source, "loading_persistent") != count(base, "loading_persistent") {
|
||||
t.Fatal("scoped provider confirmation did not close loading", source)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/warmbly/warmbly/internal/app/monitoring"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
)
|
||||
|
||||
func TestLiveMonitoringQueueSettlementAndFailureSemantics(t *testing.T) {
|
||||
_, pool := liveContactDB(t)
|
||||
requireSchemaVersion(t, pool, 277)
|
||||
f := newWarmupUsageFixture(t, pool)
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
r := NewMonitoringRepository(pool)
|
||||
baseline := map[string]models.MonitoringSource{}
|
||||
for _, id := range []string{"jobs", "dead_letters", "webhooks", "notifications", "result_effects"} {
|
||||
baseline[id] = collectMonitoring(t, r, id, now)
|
||||
}
|
||||
id := uuid.New()
|
||||
f.exec(`INSERT INTO tasks(id,task_type,email_account_id,status,message_id)VALUES($1,'email',$2,'active','')`, id, f.account)
|
||||
for _, row := range []struct {
|
||||
status string
|
||||
at time.Time
|
||||
attempts int
|
||||
}{{"pending", now.Add(-time.Minute), 1}, {"pending", now.Add(time.Minute), 1}, {"pending", now.Add(-time.Minute), 5}, {"replayed", now, 1}} {
|
||||
f.exec(`INSERT INTO task_dead_letters(task_id,task_type,status,next_retry_at,attempts,max_attempts,payload,last_error)VALUES($1,'email',$2,$3,$4,5,'{"private":"body"}','private cursor')`, id, row.status, row.at, row.attempts)
|
||||
}
|
||||
t.Cleanup(func() { _, _ = pool.Exec(context.Background(), `DELETE FROM task_dead_letters WHERE task_id=$1`, id) })
|
||||
job := uuid.NewString()
|
||||
f.exec(`INSERT INTO scheduled_job_runs(name,last_status,last_finished_at,error_count,run_count,last_error)VALUES($1,'error',$2,999,1000,'private error')`, job, now.Add(-time.Minute))
|
||||
t.Cleanup(func() { _, _ = pool.Exec(context.Background(), `DELETE FROM scheduled_job_runs WHERE name=$1`, job) })
|
||||
endpoint := uuid.New()
|
||||
f.exec(`INSERT INTO webhook_endpoints(id,organization_id,url,secret,consecutive_failures,first_failure_at,last_failure_at)VALUES($1,$2,'https://private.example.test/secret','private signing key',5,$3,$4)`, endpoint, f.org, now.Add(-73*time.Hour), now.Add(-time.Minute))
|
||||
for _, row := range []struct {
|
||||
status string
|
||||
at, updated time.Time
|
||||
}{{"pending", now.Add(-time.Minute), now}, {"pending", now.Add(time.Minute), now}, {"in_flight", now, now.Add(-6 * time.Minute)}, {"abandoned", now, now}} {
|
||||
f.exec(`INSERT INTO webhook_deliveries(endpoint_id,organization_id,event_type,event_id,payload,status,next_attempt_at,updated_at)VALUES($1,$2,'task.send',$3,'{"private":"body"}',$4,$5,$6)`, endpoint, f.org, uuid.New(), row.status, row.at, row.updated)
|
||||
}
|
||||
for _, row := range []struct {
|
||||
state string
|
||||
at time.Time
|
||||
attempts int
|
||||
}{{"pending", now.Add(-3 * time.Minute), 1}, {"sending", now.Add(-11 * time.Minute), 1}, {"sent", now, 1}, {"skipped", now, 3}} {
|
||||
f.exec(`INSERT INTO notifications(user_id,organization_id,category,title,email_state,email_due_at,email_attempts)VALUES($1,$2,'monitoring','private subject',$3,$4,$5)`, f.user, f.org, row.state, row.at, row.attempts)
|
||||
}
|
||||
f.exec(`INSERT INTO send_result_effects(task_id,effect_key,organization_id,kind,payload,attempts)VALUES($1,'monitoring',$2,'notification','{"private":"body"}',2)`, id, f.org)
|
||||
for source, deltas := range map[string]map[string]int64{
|
||||
"jobs": {"job_error_recent": 1}, "dead_letters": {"dlq_due": 1, "dlq_wait": 1, "dlq_exhausted": 1, "dlq_replayed": 1}, "webhooks": {"webhook_due": 1, "webhook_wait": 1, "webhook_stale_claim": 1, "webhook_abandoned_24h": 1, "webhook_endpoint_persistent": 1}, "notifications": {"notification_aging": 1, "notification_stale": 1, "notification_settled_24h": 1, "notification_exhausted": 1}, "result_effects": {"effects_pending": 1, "effects_repeated": 1},
|
||||
} {
|
||||
got := collectMonitoring(t, NewMonitoringRepository(monitoringReadOnlyPool(t, pool)), source, now)
|
||||
for id, delta := range deltas {
|
||||
m := measuredMonitoring(t, got, id)
|
||||
b := measuredMonitoring(t, baseline[source], id)
|
||||
if *m.Count != *b.Count+delta {
|
||||
t.Fatal(source, id, m)
|
||||
}
|
||||
}
|
||||
}
|
||||
settled := measuredMonitoring(t, collectMonitoring(t, r, "notifications", now), "notification_settled_24h")
|
||||
if settled.Condition == models.MonitoringHealthy {
|
||||
t.Fatal("notification settlement claimed delivery")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLiveMonitoringOptionalSchemaEmptyAndCollectionTimeout(t *testing.T) {
|
||||
_, pool := liveContactDB(t)
|
||||
requireSchemaVersion(t, pool, 277)
|
||||
r := NewMonitoringRepository(pool)
|
||||
now := time.Now().UTC()
|
||||
absent := collectMonitoring(t, r, "arrivals", now)
|
||||
if absent.Reason != "schema_absent" {
|
||||
t.Skip("optional schema already exists")
|
||||
}
|
||||
if _, err := pool.Exec(t.Context(), `CREATE TABLE public.sync_arrival_outbox(email_id uuid,organization_id uuid,created_at timestamptz,retry_at timestamptz,locked_until timestamptz,stage int)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _, _ = pool.Exec(context.Background(), `DROP TABLE public.sync_arrival_outbox`) })
|
||||
present := collectMonitoring(t, NewMonitoringRepository(monitoringReadOnlyPool(t, pool)), "arrivals", now)
|
||||
for _, m := range present.Metrics {
|
||||
if m.Count == nil || *m.Count != 0 || m.Availability != models.MonitoringFresh || m.Condition == models.MonitoringHealthy {
|
||||
t.Fatal("present-empty misrepresented", m)
|
||||
}
|
||||
}
|
||||
tx, err := pool.Begin(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
if _, err := tx.Exec(t.Context(), `LOCK TABLE tasks IN ACCESS EXCLUSIVE MODE`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 25*time.Millisecond)
|
||||
defer cancel()
|
||||
_, err = r.collect(ctx, now, "dispatch", monitoringSections["dispatch"])
|
||||
if err == nil || monitoring.ErrorReason(err) != "timeout" {
|
||||
t.Fatal("blocked query became success", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitoringNilDatabaseCollectorsRemainUnavailable(t *testing.T) {
|
||||
var r *MonitoringRepository
|
||||
for _, source := range r.Sources() {
|
||||
out, err := source.Collect(t.Context(), time.Now())
|
||||
if err != nil || out.Availability != models.MonitoringUnavailable || out.Reason != "dependency_missing" {
|
||||
t.Fatal(source.ID, out, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/warmbly/warmbly/internal/app/instancesettings"
|
||||
"github.com/warmbly/warmbly/internal/app/monitoring"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
)
|
||||
|
||||
func (r *MonitoringRepository) domainSource() monitoring.Source {
|
||||
return monitoring.Source{ID: "domain_auth", Permission: models.AdminPermViewUsers, Collect: func(ctx context.Context, at time.Time) (models.MonitoringSource, error) {
|
||||
if r == nil || r.pool == nil {
|
||||
return models.MonitoringSource{Availability: models.MonitoringUnavailable, Coverage: "unavailable", Reason: "dependency_missing"}, nil
|
||||
}
|
||||
policy, err := instancesettings.NewStore(r.pool).Get(ctx)
|
||||
if err != nil {
|
||||
return models.MonitoringSource{}, err
|
||||
}
|
||||
grace := policy.Deliverability.AuthGrace()
|
||||
queries := []monitoringQuery{
|
||||
{id: "domain_unknown", title: "Unobserved domain authentication", condition: models.MonitoringUnknown, severity: "warning", query: `auth_state='unknown' OR auth_checked_at IS NULL`},
|
||||
{id: "domain_stale", title: "Domain checks older than 26h", condition: models.MonitoringTelemetryStale, severity: "warning", query: `auth_checked_at<$1-INTERVAL '26 hours'`},
|
||||
{id: "domain_failing", title: "Stored failing domain observations", condition: models.MonitoringRecoveryUnverified, severity: "warning", query: `auth_state='failing'`},
|
||||
{id: "domain_passing_fresh", title: "Recently observed passing domain checks", condition: models.MonitoringHealthy, severity: "info", query: `auth_state='passing' AND auth_checked_at BETWEEN $1-INTERVAL '26 hours' AND $1`},
|
||||
{id: "domain_future_clock", title: "Future-dated domain checks", condition: models.MonitoringUnknown, severity: "warning", query: `auth_checked_at>$1`},
|
||||
{id: "domain_blocked", title: "Enforced domain safety blocks", condition: models.MonitoringSafetyHold, severity: "warning", query: `$2 AND auth_state='failing' AND auth_failing_since+make_interval(secs=>$3)<=$1`},
|
||||
}
|
||||
out := models.MonitoringSource{Metrics: []models.MonitoringMetric{}}
|
||||
for _, q := range queries {
|
||||
q.unit, q.note = "mailboxes", "DNS observation, separate from mailbox credentials; 26h freshness is monitoring policy. Missing DKIM at probed selectors is not proof a record is absent."
|
||||
m := monitoringMetric(q, at, "/mailboxes")
|
||||
threshold := int64((26 * time.Hour).Seconds())
|
||||
m.ThresholdSeconds = &threshold
|
||||
if q.id == "domain_blocked" {
|
||||
threshold = int64(grace.Seconds())
|
||||
m.ThresholdSeconds = &threshold
|
||||
m.Note = "Current policy and stored failing-since enforce this restriction independently of check freshness; no holds are changed."
|
||||
}
|
||||
err := r.pool.QueryRow(ctx, `SELECT COUNT(*),COUNT(DISTINCT id),COUNT(DISTINCT organization_id),COUNT(*) FILTER(WHERE organization_id IS NULL),MIN(auth_checked_at),MAX(auth_checked_at),NULL::timestamptz FROM email_accounts WHERE (`+strings.ReplaceAll(q.query, "$1", "$1::timestamptz")+`) AND ($2 OR NOT $2) AND $3::double precision>=0 AND $1::timestamptz IS NOT NULL`, at, policy.Deliverability.EnforceDomainAuth, grace.Seconds()).Scan(&m.Count, &m.AffectedMailboxes, &m.AffectedOrganizations, &m.UnknownOrganizationRows, &m.EvidenceAt, &m.LatestEvidenceAt, &m.NextEligibleAt)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
classifyMonitoringMetric(&m)
|
||||
out.Metrics = append(out.Metrics, m)
|
||||
}
|
||||
return out, nil
|
||||
}}
|
||||
}
|
||||
|
||||
func (r *MonitoringRepository) loadingSource() monitoring.Source {
|
||||
return monitoring.Source{ID: "warmup_loading", Permission: models.AdminPermViewCampaigns, Collect: func(ctx context.Context, at time.Time) (models.MonitoringSource, error) {
|
||||
out := models.MonitoringSource{Metrics: []models.MonitoringMetric{}}
|
||||
if r == nil || r.pool == nil {
|
||||
out.Availability = models.MonitoringUnavailable
|
||||
out.Coverage = "unavailable"
|
||||
out.Reason = "dependency_missing"
|
||||
return out, nil
|
||||
}
|
||||
var expected int64
|
||||
if err := r.pool.QueryRow(ctx, `SELECT COUNT(*) FROM email_accounts WHERE status='active' AND warmup IS NOT NULL AND warmup_paused_at IS NULL AND organization_id IS NOT NULL AND (test_mode IS NULL OR test_mode='legacy' OR test_mode='diagnostic' AND test_send_enabled)`).Scan(&expected); err != nil {
|
||||
return out, err
|
||||
}
|
||||
measured := min(expected, 200)
|
||||
out.MeasuredScopes, out.ExpectedScopes = &measured, &expected
|
||||
if measured < expected {
|
||||
out.Coverage = "partial"
|
||||
out.Reason = "scope_limit"
|
||||
}
|
||||
query := `WITH eligible AS (SELECT id,organization_id,worker_id,status,warmup,warmup_paused_at,test_mode,test_send_enabled FROM email_accounts WHERE status='active' AND warmup IS NOT NULL AND warmup_paused_at IS NULL AND organization_id IS NOT NULL AND (test_mode IS NULL OR test_mode='legacy' OR test_mode='diagnostic' AND test_send_enabled) ORDER BY id LIMIT 200), failures AS (` + strings.Replace(warmupSendFailuresSQL, "FROM email_accounts e", "FROM eligible e", 1) + `) `
|
||||
for _, q := range []monitoringQuery{
|
||||
{id: "loading_recent", title: "Recent unrecovered mailbox-loading reports", condition: models.MonitoringRecentFailure, severity: "warning", query: `at>$3-INTERVAL '1 hour' AND first_at>$3-INTERVAL '1 hour'`},
|
||||
{id: "loading_persistent", title: "Persistent recent mailbox-loading incidents", condition: models.MonitoringPersistentFailure, severity: "warning", query: `first_at<=$3-INTERVAL '1 hour' AND at>$3-INTERVAL '1 hour'`},
|
||||
{id: "loading_old", title: "Old mailbox-loading report, recovery unverified", condition: models.MonitoringRecoveryUnverified, severity: "warning", query: `at<=$3-INTERVAL '1 hour'`},
|
||||
} {
|
||||
q.unit, q.note = "mailboxes", "Bounded eligible-mailbox inspection; counts are lower bounds when coverage is partial. Native task updated/completed clocks are proxies; only scoped nonempty warmup token evidence closes the incident."
|
||||
m := monitoringMetric(q, at, "/sends?tab=failures")
|
||||
threshold := int64(3600)
|
||||
m.ThresholdSeconds = &threshold
|
||||
err := r.pool.QueryRow(ctx, query+`SELECT COUNT(*),COUNT(DISTINCT id),COUNT(DISTINCT organization_id),COUNT(*) FILTER(WHERE organization_id IS NULL),MIN(first_at),MAX(at),NULL::timestamptz FROM failures WHERE message LIKE $2 AND `+q.query, nil, mailboxLoadingPattern, at).Scan(&m.Count, &m.AffectedMailboxes, &m.AffectedOrganizations, &m.UnknownOrganizationRows, &m.EvidenceAt, &m.LatestEvidenceAt, &m.NextEligibleAt)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
classifyMonitoringMetric(&m)
|
||||
out.Metrics = append(out.Metrics, m)
|
||||
}
|
||||
return out, nil
|
||||
}}
|
||||
}
|
||||
|
||||
func (r *MonitoringRepository) workerSource() monitoring.Source {
|
||||
return monitoring.Source{ID: "workers", Permission: models.AdminPermViewWorkers, Collect: func(ctx context.Context, at time.Time) (models.MonitoringSource, error) {
|
||||
out := models.MonitoringSource{Metrics: []models.MonitoringMetric{}}
|
||||
if r == nil || r.pool == nil {
|
||||
out.Availability = models.MonitoringUnavailable
|
||||
out.Coverage = "unavailable"
|
||||
out.Reason = "dependency_missing"
|
||||
return out, nil
|
||||
}
|
||||
q := monitoringQuery{id: "workers_missing_heartbeat", title: "Active workers without recent heartbeat", unit: "workers", condition: models.MonitoringTelemetryStale, severity: "warning", note: "Native node liveness window is 5m. Missing heartbeat is registration evidence, not proof of power failure or lost throughput."}
|
||||
m := monitoringMetric(q, at, "/fleet")
|
||||
threshold := int64(models.NodeLivenessWindow.Seconds())
|
||||
m.ThresholdSeconds = &threshold
|
||||
err := r.pool.QueryRow(ctx, `WITH stale AS(SELECT id,last_seen_at FROM fleet_nodes WHERE role='worker' AND active AND (last_seen_at IS NULL OR last_seen_at<=$1::timestamptz-INTERVAL '5 minutes' OR last_seen_at>$1)), assigned AS(SELECT e.id,e.organization_id FROM email_accounts e JOIN stale s ON s.id=e.worker_id)
|
||||
SELECT (SELECT COUNT(*) FROM stale),(SELECT COUNT(*) FROM assigned),(SELECT COUNT(DISTINCT organization_id) FROM assigned),(SELECT COUNT(*) FROM assigned WHERE organization_id IS NULL),(SELECT MIN(last_seen_at) FROM stale),(SELECT MAX(last_seen_at) FROM stale),NULL::timestamptz`, at).Scan(&m.Count, &m.AffectedMailboxes, &m.AffectedOrganizations, &m.UnknownOrganizationRows, &m.EvidenceAt, &m.LatestEvidenceAt, &m.NextEligibleAt)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
classifyMonitoringMetric(&m)
|
||||
out.Metrics = append(out.Metrics, m)
|
||||
rows, err := r.pool.Query(ctx, `SELECT n.id::text,n.last_seen_at,COUNT(e.id),COUNT(DISTINCT e.organization_id),COUNT(e.id) FILTER(WHERE e.organization_id IS NULL) FROM fleet_nodes n LEFT JOIN email_accounts e ON e.worker_id=n.id WHERE n.role='worker' AND n.active AND (n.last_seen_at IS NULL OR n.last_seen_at<=$1::timestamptz-INTERVAL '5 minutes' OR n.last_seen_at>$1) GROUP BY n.id ORDER BY n.last_seen_at NULLS FIRST,n.id LIMIT 20`, at)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
v := monitoringMetric(q, at, "")
|
||||
one := int64(1)
|
||||
v.Count = &one
|
||||
v.ThresholdSeconds = &threshold
|
||||
if err := rows.Scan(&v.ScopeID, &v.EvidenceAt, &v.AffectedMailboxes, &v.AffectedOrganizations, &v.UnknownOrganizationRows); err != nil {
|
||||
return out, err
|
||||
}
|
||||
v.ID = "worker_missing_heartbeat"
|
||||
v.Investigate = "/workers/" + v.ScopeID
|
||||
out.Metrics = append(out.Metrics, v)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return out, err
|
||||
}
|
||||
expected := *m.Count
|
||||
measured := int64(len(out.Metrics) - 1)
|
||||
out.ExpectedScopes = &expected
|
||||
out.MeasuredScopes = &measured
|
||||
if measured < expected {
|
||||
out.Coverage = "partial"
|
||||
out.Reason = "sample_limit"
|
||||
}
|
||||
return out, nil
|
||||
}}
|
||||
}
|
||||
|
||||
func (r *MonitoringRepository) MonitoringWorkerIDs(ctx context.Context) ([]string, int64, error) {
|
||||
if r == nil || r.pool == nil {
|
||||
return nil, 0, context.Canceled
|
||||
}
|
||||
var expected int64
|
||||
if err := r.pool.QueryRow(ctx, `SELECT COUNT(*) FROM fleet_nodes WHERE role='worker' AND active`).Scan(&expected); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
rows, err := r.pool.Query(ctx, `SELECT id::text FROM fleet_nodes WHERE role='worker' AND active ORDER BY id LIMIT 20`)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
ids := []string{}
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ids, expected, rows.Err()
|
||||
}
|
||||
@@ -420,12 +420,12 @@ const grantAppIsUsable = ` AND EXISTS (SELECT 1 FROM oauth_applications a
|
||||
func (r *oauthRepository) grantByTokenHash(ctx context.Context, column, hash string) (*models.OAuthAccessGrant, error) {
|
||||
var g models.OAuthAccessGrant
|
||||
holder := &models.OrganizationMember{}
|
||||
row := r.db.QueryRow(ctx, `SELECT `+oauthGrantCols+`, holder.role, holder.permissions
|
||||
row := r.db.QueryRow(ctx, `SELECT `+oauthGrantCols+`, holder.role, holder.permissions, holder.access_scope
|
||||
FROM oauth_access_grants
|
||||
CROSS JOIN LATERAL (SELECT m.role, m.permissions FROM organization_members m
|
||||
CROSS JOIN LATERAL (SELECT m.role, m.permissions, m.access_scope FROM organization_members m
|
||||
WHERE m.organization_id = oauth_access_grants.organization_id AND m.user_id = oauth_access_grants.user_id) holder
|
||||
WHERE `+column+` = $1`+grantHolderIsMember+grantHolderNotBanned+grantAppIsUsable, hash)
|
||||
if err := scanOAuthGrant(row, &g, &holder.Role, &holder.Permissions); err != nil {
|
||||
if err := scanOAuthGrant(row, &g, &holder.Role, &holder.Permissions, &holder.AccessScope); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
@@ -59,6 +59,14 @@ type OrganizationRepository interface {
|
||||
SetMemberRoles(ctx context.Context, orgID, userID uuid.UUID, roleIDs []uuid.UUID) error
|
||||
GetMemberRoles(ctx context.Context, orgID, userID uuid.UUID) ([]models.MemberRole, error)
|
||||
HydrateMemberRoles(ctx context.Context, orgID uuid.UUID, members []models.OrganizationMember) error
|
||||
|
||||
// Resource scope: which campaigns, folders and mailboxes a restricted member reaches.
|
||||
GetMemberAccess(ctx context.Context, orgID, userID uuid.UUID) (*models.MemberAccess, error)
|
||||
ListMemberAccess(ctx context.Context, orgID uuid.UUID) (map[uuid.UUID]*models.MemberAccess, error)
|
||||
ResolveMemberScope(ctx context.Context, orgID, userID uuid.UUID) (*models.ResourceScope, error)
|
||||
CountOwnedAccessResources(ctx context.Context, orgID uuid.UUID, a *models.MemberAccess) (folders, campaigns, mailboxes int, err error)
|
||||
SetMemberAccess(ctx context.Context, orgID, userID uuid.UUID, a *models.MemberAccess, grantedBy uuid.UUID) error
|
||||
SuggestCampaignSenders(ctx context.Context, orgID uuid.UUID, campaignIDs, folderIDs []uuid.UUID) ([]models.SuggestedSender, error)
|
||||
SetInvitationRoles(ctx context.Context, invitationID uuid.UUID, roleIDs []uuid.UUID) error
|
||||
GetInvitationRoles(ctx context.Context, invitationID uuid.UUID) ([]uuid.UUID, error)
|
||||
RemoveMember(ctx context.Context, orgID, userID uuid.UUID) error
|
||||
@@ -236,7 +244,7 @@ func (r *organizationRepository) GetUserOrganizations(ctx context.Context, userI
|
||||
query := `
|
||||
SELECT
|
||||
om.id, om.organization_id, om.user_id, om.role, om.permissions,
|
||||
om.invited_by, om.invited_at, om.accepted_at,
|
||||
om.invited_by, om.invited_at, om.accepted_at, om.access_scope,
|
||||
o.id, o.name, o.slug, o.avatar_url, o.owner_user_id, o.created_at, o.updated_at,
|
||||
o.deletion_scheduled_at, o.deletion_scheduled_for, o.category
|
||||
FROM organization_members om
|
||||
@@ -256,7 +264,7 @@ func (r *organizationRepository) GetUserOrganizations(ctx context.Context, userI
|
||||
var org models.Organization
|
||||
err := rows.Scan(
|
||||
&m.ID, &m.OrganizationID, &m.UserID, &m.Role, &m.Permissions,
|
||||
&m.InvitedBy, &m.InvitedAt, &m.AcceptedAt,
|
||||
&m.InvitedBy, &m.InvitedAt, &m.AcceptedAt, &m.AccessScope,
|
||||
&org.ID, &org.Name, &org.Slug, &org.AvatarURL, &org.OwnerUserID, &org.CreatedAt, &org.UpdatedAt,
|
||||
&org.DeletionScheduledAt, &org.DeletionScheduledFor, &org.Category,
|
||||
)
|
||||
@@ -288,7 +296,7 @@ func (r *organizationRepository) GetMembers(ctx context.Context, orgID uuid.UUID
|
||||
query := `
|
||||
SELECT
|
||||
om.id, om.organization_id, om.user_id, om.role, om.role_id, om.permissions,
|
||||
om.invited_by, om.invited_at, om.accepted_at,
|
||||
om.invited_by, om.invited_at, om.accepted_at, om.access_scope,
|
||||
u.id, u.first_name, u.last_name, u.email, u.created_at, u.updated_at
|
||||
FROM organization_members om
|
||||
JOIN users u ON u.id = om.user_id
|
||||
@@ -307,7 +315,7 @@ func (r *organizationRepository) GetMembers(ctx context.Context, orgID uuid.UUID
|
||||
var u models.User
|
||||
err := rows.Scan(
|
||||
&m.ID, &m.OrganizationID, &m.UserID, &m.Role, &m.RoleID, &m.Permissions,
|
||||
&m.InvitedBy, &m.InvitedAt, &m.AcceptedAt,
|
||||
&m.InvitedBy, &m.InvitedAt, &m.AcceptedAt, &m.AccessScope,
|
||||
&u.ID, &u.FirstName, &u.LastName, &u.Email, &u.CreatedAt, &u.UpdatedAt,
|
||||
)
|
||||
if err != nil {
|
||||
@@ -324,13 +332,13 @@ func (r *organizationRepository) GetMembers(ctx context.Context, orgID uuid.UUID
|
||||
// GetMember retrieves a specific member of an organization
|
||||
func (r *organizationRepository) GetMember(ctx context.Context, orgID, userID uuid.UUID) (*models.OrganizationMember, error) {
|
||||
query := `
|
||||
SELECT id, organization_id, user_id, role, role_id, permissions, invited_by, invited_at, accepted_at
|
||||
SELECT id, organization_id, user_id, role, role_id, permissions, invited_by, invited_at, accepted_at, access_scope
|
||||
FROM organization_members
|
||||
WHERE organization_id = $1 AND user_id = $2
|
||||
`
|
||||
row := r.db.QueryRow(ctx, query, orgID, userID)
|
||||
var m models.OrganizationMember
|
||||
err := row.Scan(&m.ID, &m.OrganizationID, &m.UserID, &m.Role, &m.RoleID, &m.Permissions, &m.InvitedBy, &m.InvitedAt, &m.AcceptedAt)
|
||||
err := row.Scan(&m.ID, &m.OrganizationID, &m.UserID, &m.Role, &m.RoleID, &m.Permissions, &m.InvitedBy, &m.InvitedAt, &m.AcceptedAt, &m.AccessScope)
|
||||
if err == pgx.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -343,12 +351,12 @@ func (r *organizationRepository) GetMember(ctx context.Context, orgID, userID uu
|
||||
// GetMemberByID retrieves a member by their membership ID
|
||||
func (r *organizationRepository) GetMemberByID(ctx context.Context, memberID uuid.UUID) (*models.OrganizationMember, error) {
|
||||
query := `
|
||||
SELECT id, organization_id, user_id, role, role_id, permissions, invited_by, invited_at, accepted_at
|
||||
SELECT id, organization_id, user_id, role, role_id, permissions, invited_by, invited_at, accepted_at, access_scope
|
||||
FROM organization_members WHERE id = $1
|
||||
`
|
||||
row := r.db.QueryRow(ctx, query, memberID)
|
||||
var m models.OrganizationMember
|
||||
err := row.Scan(&m.ID, &m.OrganizationID, &m.UserID, &m.Role, &m.RoleID, &m.Permissions, &m.InvitedBy, &m.InvitedAt, &m.AcceptedAt)
|
||||
err := row.Scan(&m.ID, &m.OrganizationID, &m.UserID, &m.Role, &m.RoleID, &m.Permissions, &m.InvitedBy, &m.InvitedAt, &m.AcceptedAt, &m.AccessScope)
|
||||
if err == pgx.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -361,12 +369,12 @@ func (r *organizationRepository) GetMemberByID(ctx context.Context, memberID uui
|
||||
// AddMember adds a member to an organization
|
||||
func (r *organizationRepository) AddMember(ctx context.Context, member *models.OrganizationMember) error {
|
||||
query := `
|
||||
INSERT INTO organization_members (id, organization_id, user_id, role, role_id, permissions, invited_by, invited_at, accepted_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
||||
INSERT INTO organization_members (id, organization_id, user_id, role, role_id, permissions, invited_by, invited_at, accepted_at, access_scope)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
|
||||
`
|
||||
_, err := r.db.Exec(ctx, query,
|
||||
member.ID, member.OrganizationID, member.UserID, member.Role, member.RoleID, member.Permissions,
|
||||
member.InvitedBy, member.InvitedAt, member.AcceptedAt,
|
||||
member.InvitedBy, member.InvitedAt, member.AcceptedAt, memberScope(member),
|
||||
)
|
||||
return err
|
||||
}
|
||||
@@ -397,8 +405,9 @@ func (r *organizationRepository) GetMemberCount(ctx context.Context, orgID uuid.
|
||||
// CreateInvitation creates a new invitation
|
||||
func (r *organizationRepository) CreateInvitation(ctx context.Context, inv *models.OrganizationInvitation) error {
|
||||
query := `
|
||||
INSERT INTO organization_invitations (id, organization_id, email, role, role_id, permissions, invited_by, token, expires_at, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
|
||||
INSERT INTO organization_invitations (id, organization_id, email, role, role_id, permissions, invited_by, token, expires_at, created_at,
|
||||
access_scope, access_folder_ids, access_campaign_ids, access_email_account_ids)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
|
||||
ON CONFLICT (organization_id, email) DO UPDATE SET
|
||||
role = EXCLUDED.role,
|
||||
role_id = EXCLUDED.role_id,
|
||||
@@ -406,13 +415,24 @@ func (r *organizationRepository) CreateInvitation(ctx context.Context, inv *mode
|
||||
invited_by = EXCLUDED.invited_by,
|
||||
token = EXCLUDED.token,
|
||||
link_token_hash = NULL,
|
||||
expires_at = EXCLUDED.expires_at
|
||||
expires_at = EXCLUDED.expires_at,
|
||||
access_scope = EXCLUDED.access_scope,
|
||||
access_folder_ids = EXCLUDED.access_folder_ids,
|
||||
access_campaign_ids = EXCLUDED.access_campaign_ids,
|
||||
access_email_account_ids = EXCLUDED.access_email_account_ids
|
||||
RETURNING id
|
||||
`
|
||||
_, err := r.db.Exec(ctx, query,
|
||||
access := inv.Access
|
||||
if access == nil {
|
||||
access = models.WorkspaceAccess()
|
||||
}
|
||||
access.Normalize()
|
||||
// A re-invite updates the existing row, so the stored id is the one its roles attach to.
|
||||
return r.db.QueryRow(ctx, query,
|
||||
inv.ID, inv.OrganizationID, inv.Email, inv.Role, inv.RoleID, inv.Permissions,
|
||||
inv.InvitedBy, inv.Token, inv.ExpiresAt, inv.CreatedAt,
|
||||
)
|
||||
return err
|
||||
access.Scope, access.FolderIDs, access.CampaignIDs, access.EmailAccountIDs,
|
||||
).Scan(&inv.ID)
|
||||
}
|
||||
|
||||
// SetInvitationLinkToken replaces the digest of an invitation's copied-link token.
|
||||
@@ -431,7 +451,8 @@ func (r *organizationRepository) GetInvitationByToken(ctx context.Context, token
|
||||
SELECT
|
||||
i.id, i.organization_id, i.email, i.role, i.role_id, i.permissions, i.invited_by, i.token, i.expires_at, i.created_at,
|
||||
o.id, o.name, o.slug, o.avatar_url, o.owner_user_id, o.created_at, o.updated_at,
|
||||
o.deletion_scheduled_at, o.deletion_scheduled_for
|
||||
o.deletion_scheduled_at, o.deletion_scheduled_for,
|
||||
` + invitationAccessCols + `
|
||||
FROM organization_invitations i
|
||||
JOIN organizations o ON o.id = i.organization_id
|
||||
WHERE i.token = $1 OR i.link_token_hash = $1
|
||||
@@ -439,11 +460,13 @@ func (r *organizationRepository) GetInvitationByToken(ctx context.Context, token
|
||||
row := r.db.QueryRow(ctx, query, token)
|
||||
var inv models.OrganizationInvitation
|
||||
var org models.Organization
|
||||
var acc invitationAccess
|
||||
err := row.Scan(
|
||||
&inv.ID, &inv.OrganizationID, &inv.Email, &inv.Role, &inv.RoleID, &inv.Permissions,
|
||||
&inv.InvitedBy, &inv.Token, &inv.ExpiresAt, &inv.CreatedAt,
|
||||
&org.ID, &org.Name, &org.Slug, &org.AvatarURL, &org.OwnerUserID, &org.CreatedAt, &org.UpdatedAt,
|
||||
&org.DeletionScheduledAt, &org.DeletionScheduledFor,
|
||||
&acc.scope, &acc.folders, &acc.campaigns, &acc.mailboxes,
|
||||
)
|
||||
if err == pgx.ErrNoRows {
|
||||
return nil, nil
|
||||
@@ -451,6 +474,7 @@ func (r *organizationRepository) GetInvitationByToken(ctx context.Context, token
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
inv.Access = acc.access()
|
||||
inv.Organization = &org
|
||||
return &inv, nil
|
||||
}
|
||||
@@ -461,7 +485,8 @@ func (r *organizationRepository) GetInvitationByID(ctx context.Context, id uuid.
|
||||
SELECT
|
||||
i.id, i.organization_id, i.email, i.role, i.role_id, i.permissions, i.invited_by, i.token, i.expires_at, i.created_at,
|
||||
o.id, o.name, o.slug, o.avatar_url, o.owner_user_id, o.created_at, o.updated_at,
|
||||
o.deletion_scheduled_at, o.deletion_scheduled_for
|
||||
o.deletion_scheduled_at, o.deletion_scheduled_for,
|
||||
` + invitationAccessCols + `
|
||||
FROM organization_invitations i
|
||||
JOIN organizations o ON o.id = i.organization_id
|
||||
WHERE i.id = $1
|
||||
@@ -469,11 +494,13 @@ func (r *organizationRepository) GetInvitationByID(ctx context.Context, id uuid.
|
||||
row := r.db.QueryRow(ctx, query, id)
|
||||
var inv models.OrganizationInvitation
|
||||
var org models.Organization
|
||||
var acc invitationAccess
|
||||
err := row.Scan(
|
||||
&inv.ID, &inv.OrganizationID, &inv.Email, &inv.Role, &inv.RoleID, &inv.Permissions,
|
||||
&inv.InvitedBy, &inv.Token, &inv.ExpiresAt, &inv.CreatedAt,
|
||||
&org.ID, &org.Name, &org.Slug, &org.AvatarURL, &org.OwnerUserID, &org.CreatedAt, &org.UpdatedAt,
|
||||
&org.DeletionScheduledAt, &org.DeletionScheduledFor,
|
||||
&acc.scope, &acc.folders, &acc.campaigns, &acc.mailboxes,
|
||||
)
|
||||
if err == pgx.ErrNoRows {
|
||||
return nil, nil
|
||||
@@ -481,6 +508,7 @@ func (r *organizationRepository) GetInvitationByID(ctx context.Context, id uuid.
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
inv.Access = acc.access()
|
||||
inv.Organization = &org
|
||||
return &inv, nil
|
||||
}
|
||||
@@ -507,8 +535,9 @@ func (r *organizationRepository) GetInvitationByEmail(ctx context.Context, orgID
|
||||
// GetPendingInvitations retrieves all pending invitations for an organization
|
||||
func (r *organizationRepository) GetPendingInvitations(ctx context.Context, orgID uuid.UUID) ([]models.OrganizationInvitation, error) {
|
||||
query := `
|
||||
SELECT id, organization_id, email, role, role_id, permissions, invited_by, token, expires_at, created_at
|
||||
FROM organization_invitations
|
||||
SELECT id, organization_id, email, role, role_id, permissions, invited_by, token, expires_at, created_at,
|
||||
` + invitationAccessCols + `
|
||||
FROM organization_invitations i
|
||||
WHERE organization_id = $1 AND expires_at > NOW()
|
||||
ORDER BY created_at DESC
|
||||
`
|
||||
@@ -521,10 +550,13 @@ func (r *organizationRepository) GetPendingInvitations(ctx context.Context, orgI
|
||||
var invitations []models.OrganizationInvitation
|
||||
for rows.Next() {
|
||||
var inv models.OrganizationInvitation
|
||||
err := rows.Scan(&inv.ID, &inv.OrganizationID, &inv.Email, &inv.Role, &inv.RoleID, &inv.Permissions, &inv.InvitedBy, &inv.Token, &inv.ExpiresAt, &inv.CreatedAt)
|
||||
var acc invitationAccess
|
||||
err := rows.Scan(&inv.ID, &inv.OrganizationID, &inv.Email, &inv.Role, &inv.RoleID, &inv.Permissions, &inv.InvitedBy, &inv.Token, &inv.ExpiresAt, &inv.CreatedAt,
|
||||
&acc.scope, &acc.folders, &acc.campaigns, &acc.mailboxes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
inv.Access = acc.access()
|
||||
invitations = append(invitations, inv)
|
||||
}
|
||||
return invitations, nil
|
||||
|
||||
@@ -46,6 +46,8 @@ type UniboxRepository interface {
|
||||
// the owner even when a different teammate opens it.
|
||||
GetByIDForOrg(ctx context.Context, orgID, id uuid.UUID) (*models.EmailMessageStoreData, uuid.UUID, error)
|
||||
GetByThread(ctx context.Context, orgID, emailID uuid.UUID, threadID string, limit int, cursor string) (*models.MailSearchResult, error)
|
||||
// GetByThreadWithin is GetByThread held to the allowed mailboxes before paging; nil allows every mailbox.
|
||||
GetByThreadWithin(ctx context.Context, orgID, emailID uuid.UUID, threadID string, limit int, cursor string, allowed []uuid.UUID) (*models.MailSearchResult, error)
|
||||
GetBySender(ctx context.Context, userID uuid.UUID, sender string, limit int, cursor string) (*models.MailSearchResult, error)
|
||||
Search(ctx context.Context, orgID uuid.UUID, params *models.MailSearchParams) (*models.MailSearchResult, error)
|
||||
GetUnseenCount(ctx context.Context, orgID uuid.UUID, emailAccountID *uuid.UUID) (int64, error)
|
||||
@@ -530,6 +532,10 @@ func (r *uniboxRepository) GetByIDForOrg(ctx context.Context, orgID, id uuid.UUI
|
||||
// not user_id, so any member with unibox access sees the whole conversation,
|
||||
// matching the org-scoped inbox list.
|
||||
func (r *uniboxRepository) GetByThread(ctx context.Context, orgID, emailID uuid.UUID, threadID string, limit int, cursor string) (*models.MailSearchResult, error) {
|
||||
return r.GetByThreadWithin(ctx, orgID, emailID, threadID, limit, cursor, nil)
|
||||
}
|
||||
|
||||
func (r *uniboxRepository) GetByThreadWithin(ctx context.Context, orgID, emailID uuid.UUID, threadID string, limit int, cursor string, allowed []uuid.UUID) (*models.MailSearchResult, error) {
|
||||
query := fmt.Sprintf(`
|
||||
SELECT %s
|
||||
FROM unibox_emails
|
||||
@@ -545,6 +551,11 @@ func (r *uniboxRepository) GetByThread(ctx context.Context, orgID, emailID uuid.
|
||||
args = append(args, emailID)
|
||||
argPos++
|
||||
}
|
||||
if allowed != nil {
|
||||
query += fmt.Sprintf(` AND email_id = ANY($%d::uuid[])`, argPos)
|
||||
args = append(args, allowed)
|
||||
argPos++
|
||||
}
|
||||
|
||||
if cursor != "" {
|
||||
cursorID, err := uuid.Parse(cursor)
|
||||
@@ -1718,7 +1729,7 @@ func (r *uniboxRepository) overview(ctx context.Context, orgID uuid.UUID, accoun
|
||||
COUNT(*) FILTER (WHERE NOT t.is_snoozed AND t.working AND t.is_automated AND t.has_unread) AS automated_unread,
|
||||
(SELECT COUNT(*) FROM latest_per_thread l WHERE `+ownAddressSQL("l.from_addr", "$1")+`) AS awaiting,
|
||||
(SELECT COUNT(*) FROM ai_thread_drafts d
|
||||
WHERE d.organization_id = $1 AND d.status = 'pending') AS awaiting_agent_draft
|
||||
WHERE d.organization_id = $1 AND d.status = 'pending'`+onlyMailboxes("d.email_account_id")+`) AS awaiting_agent_draft
|
||||
FROM threads t
|
||||
`, orgID, todayStart, weekStart).Scan(
|
||||
&overview.Total,
|
||||
@@ -1815,6 +1826,11 @@ func (r *uniboxRepository) overview(ctx context.Context, orgID uuid.UUID, accoun
|
||||
overview.Mailboxes = append(overview.Mailboxes, m)
|
||||
}
|
||||
|
||||
// A mailbox-limited view lists only the tags on its own mailboxes.
|
||||
tagsOnMailboxes := ""
|
||||
if len(accountIDs) > 0 {
|
||||
tagsOnMailboxes = " AND EXISTS (SELECT 1 FROM email_tags et2 WHERE et2.tag_id = t.id" + onlyMailboxes("et2.email_id") + ")"
|
||||
}
|
||||
// Per-tag counters. Mailbox tags live in `tags` + `email_tags`.
|
||||
// Per THREAD (distinct thread key) so threads aren't over-counted by
|
||||
// message multiplicity or the email_tags fan-out. Every join is on the
|
||||
@@ -1841,7 +1857,7 @@ func (r *uniboxRepository) overview(ctx context.Context, orgID uuid.UUID, accoun
|
||||
LEFT JOIN unibox_emails ue ON ue.email_id = ea.id
|
||||
AND ue.folder NOT IN `+foldersOutsideWorkingViews+`
|
||||
AND NOT `+automatedThreadSQL("ue.thread_id", "ue.automated", "$1")+onlyMailboxes("ue.email_id")+`
|
||||
WHERE t.organization_id = $1
|
||||
WHERE t.organization_id = $1`+tagsOnMailboxes+`
|
||||
GROUP BY t.id, t.title, t.color, t.position
|
||||
ORDER BY t.position ASC, t.title ASC
|
||||
`, orgID)
|
||||
|
||||
@@ -93,7 +93,7 @@ func TestLiveUsageOverviewCountsTheSelectedOrganization(t *testing.T) {
|
||||
}
|
||||
|
||||
from, to := time.Now().Add(-24*time.Hour), time.Now().Add(24*time.Hour)
|
||||
overall, xerr := repo.GetDashboardOverallStats(ctx, orgID, from, to, nil)
|
||||
overall, xerr := repo.GetDashboardOverallStats(ctx, orgID, from, to, nil, nil)
|
||||
if xerr != nil {
|
||||
t.Fatalf("GetDashboardOverallStats: %v", xerr)
|
||||
}
|
||||
@@ -152,7 +152,7 @@ func TestLiveUsageOverviewCountsTheSelectedOrganization(t *testing.T) {
|
||||
|
||||
exec(`INSERT INTO warmup_pool_participants (pool_id, email_account_id, health_state)
|
||||
VALUES ($1, $2, 'watch')`, models.WarmupPoolPremiumID, accountID)
|
||||
health, xerr := repo.GetAccountHealthSummary(ctx, orgID)
|
||||
health, xerr := repo.GetAccountHealthSummary(ctx, orgID, nil)
|
||||
if xerr != nil {
|
||||
t.Fatalf("GetAccountHealthSummary watch: %v", xerr)
|
||||
}
|
||||
@@ -160,7 +160,7 @@ func TestLiveUsageOverviewCountsTheSelectedOrganization(t *testing.T) {
|
||||
t.Fatalf("account health for watch state = %+v, want one warning", health)
|
||||
}
|
||||
exec(`UPDATE warmup_pool_participants SET health_state = 'throttled' WHERE email_account_id = $1`, accountID)
|
||||
health, xerr = repo.GetAccountHealthSummary(ctx, orgID)
|
||||
health, xerr = repo.GetAccountHealthSummary(ctx, orgID, nil)
|
||||
if xerr != nil {
|
||||
t.Fatalf("GetAccountHealthSummary throttled: %v", xerr)
|
||||
}
|
||||
@@ -173,7 +173,7 @@ func TestLiveUsageOverviewCountsTheSelectedOrganization(t *testing.T) {
|
||||
(email_account_id, user_id, error_code, severity, title, message)
|
||||
VALUES ($1, $2, 'usage-warning', 'WARNING', 'Warning', 'Warning'),
|
||||
($1, $2, 'usage-critical', 'CRITICAL', 'Critical', 'Critical')`, accountID, userID)
|
||||
health, xerr = repo.GetAccountHealthSummary(ctx, orgID)
|
||||
health, xerr = repo.GetAccountHealthSummary(ctx, orgID, nil)
|
||||
if xerr != nil {
|
||||
t.Fatalf("GetAccountHealthSummary: %v", xerr)
|
||||
}
|
||||
|
||||
@@ -161,6 +161,19 @@ Run `QA_FLEET_FIXTURES=1 pnpm proof fleet-telemetry` in `qa/`. Stop the addition
|
||||
admin server as well as the stack afterward. This proof does not validate real
|
||||
worker sampling, network egress, mail delivery or inbox placement.
|
||||
|
||||
## Member access proof fixtures
|
||||
|
||||
`flows/member-access.flow.ts` is opt-in: apply `fixtures/member-access.sql` to
|
||||
this worktree's rich-seed database, which adds a teammate restricted to the
|
||||
Outbound folder and one mailbox (`client@acme-agency.test`, `password123`).
|
||||
The first flow saves a change to that scope, so apply the fixture again before
|
||||
every recording.
|
||||
|
||||
```bash
|
||||
docker exec -i warmbly-postgres-1 psql -U warmbly -d warmbly_qa_<worktree> < fixtures/member-access.sql
|
||||
QA_MEMBER_ACCESS_FIXTURES=1 pnpm proof member-access
|
||||
```
|
||||
|
||||
## Tester sample data proof
|
||||
|
||||
The `tester-sample-data` flow uses the real local tester-provisioning and sample-data endpoints. Start the worktree's lite stack, then serve `admin/` separately against its API. Set `QA_WEB_URL` to that localhost admin origin. Prepare a seeded admin holding `view_users` and `manage_testers`, with a locally MFA-verified session; clear only that user's and session's isolated Redis entries after SQL fixture changes. These authentication fixtures are not proof of real MFA. Run `QA_TESTER_SAMPLE_FIXTURES=1 pnpm proof tester-sample-data`.
|
||||
@@ -198,6 +211,9 @@ test("search contacts and open a contact's details", async ({ page, proof }) =>
|
||||
data. A flow is skipped, with the command that fixes it, when the stack holds
|
||||
the other seed
|
||||
- `test.use({ signedIn: false })` for a flow that starts on the sign-in pages
|
||||
- `test.use({ account: { email, password } })` for a flow recorded as another
|
||||
seeded or fixture account. It signs in once through Mailpit like the stack's
|
||||
own account, and keeps its session in a file of its own
|
||||
|
||||
Conventions:
|
||||
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
-- A restricted client teammate for the isolated rich-seed QA database, never production.
|
||||
BEGIN;
|
||||
DO $$
|
||||
BEGIN
|
||||
IF left(current_database(), 11) <> 'warmbly_qa_' THEN
|
||||
RAISE EXCEPTION 'member access proof fixtures require an isolated warmbly_qa_ database';
|
||||
END IF;
|
||||
IF NOT EXISTS (SELECT 1 FROM users WHERE email = 'dev@warmbly.com') THEN
|
||||
RAISE EXCEPTION 'member access proof fixtures require the rich QA seed';
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
-- Same password as the seeded owner (password123); onboarding already done.
|
||||
INSERT INTO users (id, first_name, last_name, email, password_hash, onboarding_completed_at)
|
||||
SELECT '86100000-0000-0000-0000-000000000001', 'Casey', 'Client', 'client@acme-agency.test', password_hash, now()
|
||||
FROM users WHERE email = 'dev@warmbly.com'
|
||||
ON CONFLICT (id) DO NOTHING;
|
||||
|
||||
INSERT INTO organization_roles (id, organization_id, name, description, color, permissions)
|
||||
VALUES ('86100000-0000-0000-0000-000000000002', '22222222-0000-0000-0000-000000000001', 'Viewer',
|
||||
'Read-only access to campaigns, contacts, reports and the inbox.', '#f59e0b', 3232)
|
||||
ON CONFLICT (id) DO NOTHING;
|
||||
|
||||
-- Restricted from the moment the row exists, so its permissions are masked on insert.
|
||||
INSERT INTO organization_members (organization_id, user_id, role, role_id, permissions, invited_by, invited_at, accepted_at, access_scope)
|
||||
VALUES ('22222222-0000-0000-0000-000000000001', '86100000-0000-0000-0000-000000000001', 'Viewer',
|
||||
'86100000-0000-0000-0000-000000000002', 3232, '11111111-0000-0000-0000-000000000001', now(), now(), 'restricted')
|
||||
ON CONFLICT (organization_id, user_id) DO UPDATE SET access_scope = 'restricted';
|
||||
INSERT INTO organization_member_roles (organization_id, user_id, role_id)
|
||||
VALUES ('22222222-0000-0000-0000-000000000001', '86100000-0000-0000-0000-000000000001', '86100000-0000-0000-0000-000000000002')
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
-- Their grants: the Outbound folder and the mailbox its campaign replies arrive in.
|
||||
DELETE FROM organization_member_campaign_access WHERE user_id = '86100000-0000-0000-0000-000000000001';
|
||||
DELETE FROM organization_member_mailbox_access WHERE user_id = '86100000-0000-0000-0000-000000000001';
|
||||
INSERT INTO organization_member_campaign_access (organization_id, user_id, folder_id, granted_by)
|
||||
SELECT f.organization_id, '86100000-0000-0000-0000-000000000001', f.id, '11111111-0000-0000-0000-000000000001'
|
||||
FROM folders f WHERE f.organization_id = '22222222-0000-0000-0000-000000000001' AND f.title = 'Outbound';
|
||||
INSERT INTO organization_member_mailbox_access (organization_id, user_id, email_account_id, granted_by)
|
||||
SELECT ea.organization_id, '86100000-0000-0000-0000-000000000001', ea.id, '11111111-0000-0000-0000-000000000001'
|
||||
FROM email_accounts ea WHERE ea.organization_id = '22222222-0000-0000-0000-000000000001' AND ea.email = 'dev.outbound@warmbly.test';
|
||||
COMMIT;
|
||||
@@ -0,0 +1,85 @@
|
||||
import type { Page } from "@playwright/test";
|
||||
import { expect, test } from "../lib/proof.ts";
|
||||
|
||||
// Requires ../fixtures/member-access.sql applied to this worktree's rich-seed database.
|
||||
const FIXTURES = process.env.QA_MEMBER_ACCESS_FIXTURES === "1";
|
||||
const CLIENT = { email: "client@acme-agency.test", password: "password123" };
|
||||
|
||||
// Saving access needs a recent sign-in; the dashboard asks for the password when the session is older.
|
||||
async function confirmIfAsked(page: Page) {
|
||||
const dialog = page.getByRole("dialog").filter({ hasText: "Confirm it is you" });
|
||||
if (await dialog.isVisible({ timeout: 2000 }).catch(() => false)) {
|
||||
await dialog.getByPlaceholder("Password").fill("password123");
|
||||
await dialog.getByPlaceholder("Password").press("Enter");
|
||||
}
|
||||
}
|
||||
|
||||
test.describe("member access scope", () => {
|
||||
test.skip(!FIXTURES, "Apply qa/fixtures/member-access.sql to this stack's database and set QA_MEMBER_ACCESS_FIXTURES=1.");
|
||||
|
||||
test("restrict a teammate to one folder, a campaign and the mailboxes behind them", async ({ page, proof }) => {
|
||||
await page.goto("/app/settings/members");
|
||||
const row = page.getByRole("row").filter({ hasText: CLIENT.email });
|
||||
await expect(row.getByRole("button", { name: /1 folder, 1 mailbox/ })).toBeVisible();
|
||||
await proof.chapter("Access scope", "Roles say what a member may do; access says which campaigns and mailboxes it applies to");
|
||||
await proof.shot("members-access-column", { caption: "The roster shows each member's access next to their roles" });
|
||||
|
||||
await row.getByRole("button", { name: /1 folder, 1 mailbox/ }).click();
|
||||
const dialog = page.getByRole("dialog", { name: "Member access" });
|
||||
await expect(dialog.getByText("Campaign folders")).toBeVisible();
|
||||
await expect(dialog.getByText("Outbound", { exact: true })).toBeVisible();
|
||||
await proof.shot("access-dialog", { caption: "Selected resources: a folder grants whatever campaigns it holds, mailboxes are granted one by one" });
|
||||
|
||||
await dialog.getByText("Choose individual campaigns…").click();
|
||||
await page.getByPlaceholder("Search…").pressSequentially("Agency", { delay: 60 });
|
||||
await page.getByRole("button", { name: "Agency partnerships" }).click();
|
||||
await page.keyboard.press("Escape");
|
||||
await expect(dialog.getByText("Senders of the selected campaigns")).toBeVisible();
|
||||
await proof.shot("suggested-senders", { caption: "Campaign access never includes a mailbox; the campaigns' senders are suggested to review" });
|
||||
|
||||
await dialog.getByRole("listitem").filter({ hasText: "dev.send@warmbly.test" }).getByRole("button", { name: "Add" }).click();
|
||||
await expect(dialog.getByText("dev.send@warmbly.test")).toBeVisible();
|
||||
await dialog.getByRole("button", { name: "Save access" }).click();
|
||||
await confirmIfAsked(page);
|
||||
await expect(row.getByRole("button", { name: /1 folder, 1 campaign, 2 mailboxes/ })).toBeVisible();
|
||||
await proof.dwell();
|
||||
await proof.shot("access-saved", { caption: "Saved: the member's dashboard, API keys and live updates follow the new scope on their next request" });
|
||||
});
|
||||
|
||||
test.describe("as the restricted teammate", () => {
|
||||
test.use({ account: CLIENT });
|
||||
|
||||
test("a restricted teammate sees only their campaigns and mailboxes, read-only", async ({ page, proof }) => {
|
||||
await page.goto("/app");
|
||||
await expect(page).toHaveURL(/\/app\/campaigns/);
|
||||
await expect(page.getByRole("link", { name: /RevOps outreach - July/ }).first()).toBeVisible();
|
||||
await expect(page.getByRole("link", { name: /Q1 Outreach/ })).toHaveCount(0);
|
||||
await expect(page.getByRole("link", { name: "Contacts" })).toHaveCount(0);
|
||||
await proof.chapter("Restricted teammate", "Only the granted campaigns and mailboxes, and only the pages that can be narrowed to them");
|
||||
await proof.shot("restricted-campaigns", { caption: "Inbox, Campaigns and Analytics stay open, every other page is locked; the list holds only the granted campaigns" });
|
||||
|
||||
await page.getByRole("complementary").getByRole("button", { name: "Contacts", exact: true }).click();
|
||||
await expect(page.getByText("Your access to this workspace is limited to selected campaigns and mailboxes")).toBeVisible();
|
||||
await proof.shot("restricted-locked", { caption: "A locked page says the access scope is why, and who can change it" });
|
||||
await page.getByRole("button", { name: "Got it" }).click();
|
||||
|
||||
await page.getByRole("link", { name: /RevOps outreach - July/ }).first().click();
|
||||
const main = page.getByRole("main");
|
||||
await expect(main.getByRole("link", { name: "Steps" })).toBeVisible();
|
||||
await expect(main.getByRole("link", { name: "Leads" })).toHaveCount(0);
|
||||
await proof.shot("restricted-campaign", { caption: "Campaign analytics and steps, with no tab that reads contacts or settings" });
|
||||
|
||||
await page.goto("/app/unibox");
|
||||
const thread = page.getByRole("button", { name: /Quick question about outbound at Fieldstone/ }).first();
|
||||
await expect(thread).toBeVisible();
|
||||
await thread.click();
|
||||
await expect(page.getByRole("heading", { name: /Quick question about outbound at Fieldstone/ })).toBeVisible();
|
||||
await expect(page.getByRole("button", { name: "Reply", exact: true })).toHaveCount(0);
|
||||
await proof.dwell();
|
||||
await proof.shot("restricted-inbox", { caption: "The inbox shows only the granted mailbox's conversations, to read without replying or changing them" });
|
||||
|
||||
await page.goto("/app/contacts");
|
||||
await expect(page).toHaveURL(/\/app\/campaigns/);
|
||||
});
|
||||
});
|
||||
});
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user