Files
openobserve/tests/ui-testing/playwright-tests/utils/alerts-api-helpers.js
T
4772df1723 feat: per-group and per-series alerting, and SLO measurement (#13547)
Design at: alerts_2.md

Per-group and per-series alerting, SLO measurement, and the UI for both.

## Alerts

**Multi-alerts (Feature 3).** A grouped alert can opt in to evaluating
and paging **per group** rather than collapsing to one verdict. Opt-in
only (`multi_alert`), so no existing alert changes behaviour: the flag
cannot be present in JSON written before it existed. Per-group state,
transitions, silence fingerprints, disappearance/reaping and a
cardinality cap, with the count-gate rules that make "most severe group"
and the legacy collapsed level provably the same verdict.

**Per-series alerting for PromQL.** The same feature for the metrics
family, where a group is one returned **series** keyed by its full label
set — that is a series' identity in Prometheus, and the expression's own
`by (…)` clause is already where the label set is chosen, so a second
picker could only disagree with it. One additive nullable column; NULL
means off, so stored alerts are untouched.

**Alert detail page**, replacing the row-click side panel: stat strip,
evaluation chart with threshold marklines, per-group table and history.
Simple alerts get a real history too — evaluations (one record per run)
by default, level changes alongside.

## SLOs

Count / time-slice / alert SLIs, error-budget and burn-rate arithmetic,
rolling windows, per-group slices with an exact rollup, coverage gating,
backfill, and SLO-based alerts.

Two capabilities the model could not express, both added tests-first:

- **Metrics-native counting** (`CountSource::PromQl`) — counter metrics
have no rows for a predicate to classify, so "good" exists only as
arithmetic between series, and correct counter arithmetic is
`increase()`.
- **Freshness semantics** (`absent_is_bad`) — for a pipeline SLO,
silence IS the failure, which the default gap rule can never report. A
*failed* query still freezes rather than pages; only a query that PROVED
emptiness counts as bad.

**SLO form**: live good/bad preview so a wrong predicate is caught
before saving rather than days later, real query editors with field
autocomplete, folders shared with alerts, and a move endpoint.

## Notes for review

- The SLO and composite features are **deferred (TODO)**:
`ZO_SLO_ENABLED` stays default **false** and the SLOs menu entry is
hidden until they ship; the SLO backend and UI remain in the tree behind
the flag, and composites stay pure logic + tests.
- One migration (schema 61): a nullable column for the PromQL opt-in. No
backfill.
- Two fixes outside the feature that it surfaced: query-editor
completions were duplicated per mounted editor (Monaco registers
providers per *language*), and the alert detail chart had no PromQL
branch, so it charted row counts against the series-count gate.
- Lands together with the matching o2-enterprise PR — `QueryType::Slo`
is new here and the enterprise dedup matches are exhaustive over it.
- `ratelimit::test_add_batch_missing_rule_id` fails intermittently on
`main` as well; unrelated to this branch.

---------

Co-authored-by: Shrinath Rao <shnath@openobserve.ai>
Co-authored-by: hengfeiyang <hengfei.yang@gmail.com>
Co-authored-by: sai nikhil kethe <nikhil@openobserve.ai>
2026-07-31 07:23:22 +00:00

206 lines
7.8 KiB
JavaScript

// Copyright 2026 OpenObserve Inc.
/**
* Shared API helpers for the Alerts 4.0 (multi-alert) specs.
*
* The `-api`, `-ui` and `-regression` specs all drive the same v1/v2 alert
* endpoints, seed the same fixtures, and build the same canonical payloads.
* This module is the single source of truth for that plumbing so the three
* specs stay in lockstep. It deliberately contains NO test/expect calls —
* assertions belong in the spec files.
*/
const { getAuthHeaders, getOrgIdentifier } = require('./cloud-auth.js');
const BASE = process.env.ZO_BASE_URL || 'http://localhost:5080';
const STREAM = 'alerts_p0_stream';
const SINK = 'alerts_notify_sink'; // dogfood destination target — this instance's own ingest
const TMPL = 'auto_p0_tmpl';
const DEST = 'auto_p0_dest';
/** Unique, human-readable name so parallel/repeat runs never collide. */
const uniq = (p) => `${p}_${Date.now().toString(36)}${Math.random().toString(36).slice(2, 6)}`;
function urls() {
const org = getOrgIdentifier();
return { org, v1: `${BASE}/api/${org}`, v2: `${BASE}/api/v2/${org}` };
}
/** page.request wrapper that always carries Basic auth. */
async function api(page, method, url, data) {
const opts = { headers: getAuthHeaders() };
if (data !== undefined) opts.data = data;
return page.request[method](url, opts);
}
// ---- canonical payloads (validated live against the branch) ----------------
/** A pre-feature scheduled alert: no aggregation, no priority/tags, no warning family. */
function simpleAlert(name) {
return {
name,
stream_type: 'logs',
stream_name: STREAM,
is_real_time: false,
query_condition: {
type: 'custom',
conditions: { version: 2, conditions: { filterType: 'group', logicalOperator: 'AND', conditions: [] } },
sql: null, promql: null, promql_condition: null, aggregation: null,
vrl_function: null, search_event_type: null, multi_time_range: [],
},
trigger_condition: {
period: 10, operator: '>=', threshold: 3, frequency: 10, cron: '',
frequency_type: 'minutes', silence: 10, timezone: 'UTC', align_time: true,
},
destinations: [DEST], context_attributes: {}, row_template: '', enabled: true,
};
}
/** A grouped alert with the per-group opt-in ON (the M-10 "any breaching group" shape). */
function multiAlert(name) {
const a = simpleAlert(name);
a.query_condition.aggregation = {
group_by: ['city'], function: 'avg',
having: { column: 'latency', operator: '>', value: 500 },
multi_alert: true,
};
a.trigger_condition.threshold = 1; // the "any breaching group" gate (M-10)
return a;
}
/** Grouped, but the multi_alert flag is deliberately absent — must stay a simple alert. */
function groupedSimpleAlert(name) {
const a = simpleAlert(name);
a.query_condition.aggregation = { group_by: ['city'], function: 'avg', having: { column: 'latency', operator: '>', value: 500 } };
return a;
}
function realtimeAlert(name) {
const a = simpleAlert(name);
a.is_real_time = true;
return a;
}
// ---- CRUD helpers ----------------------------------------------------------
async function createAlert(page, payload) {
return api(page, 'post', `${urls().v2}/alerts?folder=default`, payload);
}
async function listAlerts(page) {
return (await (await api(page, 'get', `${urls().v2}/alerts?folder=default&page_size=100`)).json()).list || [];
}
async function findAlertId(page, name) {
return (await listAlerts(page)).find((a) => a.name === name)?.alert_id;
}
/** Best-effort delete of the given alert_ids (used in afterEach). */
async function deleteAlerts(page, ids) {
const { v2 } = urls();
for (const id of ids) {
if (id) await api(page, 'delete', `${v2}/alerts/${id}?folder=default`).catch(() => {});
}
}
/**
* Idempotently seed the template + a "dogfood" destination + a 3-group stream.
*
* The destination points back at THIS OpenObserve instance's own ingest endpoint
* (a dedicated sink stream) rather than an external webhook, so a firing alert
* actually delivers with no third-party dependency or rate limit. The self-call
* authenticates via the same Basic-auth headers the specs already use.
*/
async function seedAlertFixtures(page) {
const { v1 } = urls();
await api(page, 'post', `${v1}/alerts/templates`, {
name: TMPL, body: '{"text":"{alert_name} {alert_level}"}', type: 'http', title: '',
}).catch(() => {});
const destination = {
name: DEST,
url: `${v1}/${SINK}/_json`, // this instance's own ingest -> self-contained delivery
method: 'post', template: TMPL, type: 'http',
headers: getAuthHeaders(), // Basic auth so the self-call is authorized
};
// create-if-absent, then update so a stale definition on a persistent env is corrected.
await api(page, 'post', `${v1}/alerts/destinations`, destination).catch(() => {});
await api(page, 'put', `${v1}/alerts/destinations/${DEST}`, destination).catch(() => {});
// city = group key, latency = the measure; three groups so a multi-alert can fan out.
await api(page, 'post', `${v1}/${STREAM}/_json`, [
{ city: 'bangalore', latency: 890, status: 500 },
{ city: 'mumbai', latency: 950, status: 500 },
{ city: 'delhi', latency: 990, status: 500 },
]).catch(() => {});
}
/** Ingest rows into a stream (creates it on first write). */
async function ingest(page, stream, rows) {
return api(page, 'post', `${urls().v1}/${stream}/_json`, rows);
}
/** Per-group state of a multi-alert (empty list on a simple alert). */
async function getAlertGroups(page, alertId) {
const r = await api(page, 'get', `${urls().v2}/alerts/${alertId}/groups`);
return r.ok() ? r.json() : { list: [] };
}
/** Durable level-change history for an alert (from_level -> to_level, newest first). */
async function getAlertTransitions(page, alertId, { limit = 20 } = {}) {
const r = await api(page, 'get', `${urls().v2}/alerts/${alertId}/groups/transitions?limit=${limit}`);
return r.ok() ? r.json() : { list: [] };
}
/** Full alert object as stored (includes context_attributes, description, priority, tags). */
async function getAlert(page, alertId) {
const r = await api(page, 'get', `${urls().v2}/alerts/${alertId}?folder=default`);
return r.ok() ? r.json() : null;
}
/**
* Poll the alert list until `name` has a run outcome (i.e. the scheduler evaluated it),
* or the timeout elapses. Returns the list item (or null). Scheduled alerts are picked
* up within ~15s, so 60s is a generous ceiling.
*/
async function waitForAlertOutcome(page, name, { timeoutMs = 60000, pollMs = 5000 } = {}) {
const deadline = Date.now() + timeoutMs;
let item = null;
for (;;) {
item = (await listAlerts(page)).find((a) => a.name === name) || null;
if (item && item.last_outcome) return item;
if (Date.now() >= deadline) return item;
await page.waitForTimeout(pollMs);
}
}
/**
* Poll the alert list until `name` reaches `level` (ok|warning|critical|no_data),
* or the timeout elapses. Returns the matching list item, or the last item seen
* (or null) so the caller can assert a helpful message on timeout.
*/
async function waitForAlertLevel(page, name, level, { timeoutMs = 120000, pollMs = 5000 } = {}) {
const deadline = Date.now() + timeoutMs;
let item = null;
for (;;) {
item = (await listAlerts(page)).find((a) => a.name === name) || null;
if (item && item.level === level) return item;
if (Date.now() >= deadline) return item;
await page.waitForTimeout(pollMs);
}
}
/** True for outcomes that mean the alert fired (delivery success is a separate axis). */
function isFiringOutcome(outcome) {
return outcome === 'firing' || outcome === 'notify_failed';
}
module.exports = {
BASE, STREAM, SINK, TMPL, DEST,
uniq, urls, api,
simpleAlert, multiAlert, groupedSimpleAlert, realtimeAlert,
createAlert, listAlerts, findAlertId, getAlert, deleteAlerts, seedAlertFixtures,
ingest, getAlertGroups, getAlertTransitions,
waitForAlertOutcome, waitForAlertLevel, isFiringOutcome,
};