mirror of
https://github.com/openobserve/openobserve.git
synced 2026-11-05 16:10:43 +00:00
1120 lines
41 KiB
JavaScript
1120 lines
41 KiB
JavaScript
// Copyright 2026 OpenObserve Inc.
|
|
|
|
/**
|
|
* On-Call seeding + teardown helpers.
|
|
*
|
|
* Plan: docs/test_generator/features/oncall-test-plan.md
|
|
*
|
|
* On-Call is ENTERPRISE-GATED: `isOnCallAvailable()` skips on an OSS build and throws when an enterprise build has it off.
|
|
*
|
|
* Three environment facts this module encodes, each of which fails QUIETLY if
|
|
* you get it wrong:
|
|
*
|
|
* ORG IDENTIFIER Every route is `/api/{org_identifier}/oncall/...` and the
|
|
* identifier is a ksuid. Passing the org's DISPLAY NAME
|
|
* returns "Organization not found" — a 404 indistinguishable
|
|
* from the feature being off.
|
|
* BASIC AUTH Ingestion rejects the session cookie. Management routes
|
|
* accept either. Everything here therefore carries Basic
|
|
* auth, which is the one credential both halves take.
|
|
* MICROSECONDS Every timestamp on every on-call route — `after_micros`,
|
|
* `anchor_micros`, `shift_micros`, `opened_at`, `_timestamp`
|
|
* on the logs ingest path — is MICROseconds. Milliseconds
|
|
* ingest happily and land in 1970.
|
|
*
|
|
* Self-cleaning. Everything created here is named from a caller-supplied
|
|
* prefix, and `deleteOnCallFixturesByPrefix` sweeps exactly that prefix, so
|
|
* parallel workers never delete each other's fixtures. Specs scope the prefix
|
|
* to the WORKER (`_w<index>_`) — see the docblock in SLO/slo-crud.spec.js.
|
|
*
|
|
* This module contains NO expect() calls: assertions belong in spec files.
|
|
*/
|
|
|
|
const { randomBytes } = require('node:crypto');
|
|
const testLogger = require('./test-logger.js');
|
|
const { getAuthHeaders, getOrgIdentifier } = require('./cloud-auth.js');
|
|
const { ingestBase } = require('./ingest-url.js');
|
|
|
|
/** Every on-call timestamp is microseconds. Named so the call sites read. */
|
|
const MICROS = 1_000_000;
|
|
|
|
/** One hour and one day, in micros — the units rotations and ladders are built from. */
|
|
const HOUR_MICROS = 3600 * MICROS;
|
|
const DAY_MICROS = 24 * HOUR_MICROS;
|
|
|
|
/** Dimension values the seeded log rows carry, so ownership rules have something to match. */
|
|
const SEED_SERVICES = ['checkout', 'search', 'payments'];
|
|
|
|
/** Ladder delays for a two-rung policy: page now, escalate a minute later. */
|
|
const FIRST_RUNG_MICROS = 0;
|
|
const SECOND_RUNG_MICROS = 60 * MICROS;
|
|
|
|
/** A new alert is due at once and the scheduler ticks every 10s, so its page opens in about 10-20s. */
|
|
const PAGE_WAIT_MS = 90_000;
|
|
|
|
function baseUrl() {
|
|
const url = process.env.ZO_BASE_URL || 'http://localhost:5080';
|
|
return url.endsWith('/') ? url.slice(0, -1) : url;
|
|
}
|
|
|
|
/**
|
|
* Ingestion has its own base URL on cloud, and it is NOT interchangeable with
|
|
* the management one — pointing ingest at the management host writes nothing
|
|
* and still answers 200.
|
|
*/
|
|
function ingestUrl() {
|
|
const url = process.env.INGESTION_URL || process.env.ZO_BASE_URL || baseUrl();
|
|
return url.endsWith('/') ? url.slice(0, -1) : url;
|
|
}
|
|
|
|
/** The org KSUID. Never the display name — see the module note. */
|
|
function orgId() {
|
|
return getOrgIdentifier();
|
|
}
|
|
|
|
/**
|
|
* Unique per call so parallel specs and repeat runs never collide on a name.
|
|
*
|
|
* `crypto`, not `Math.random`: these names become fixture email addresses and
|
|
* flow into user-creation payloads, which CodeQL reads as a security context
|
|
* (js/insecure-randomness). Nothing here is a secret — the seeded password is a
|
|
* constant and is not derived from this — but a CSPRNG costs nothing and keeps
|
|
* the finding off a suite nobody should have to triage.
|
|
*
|
|
* FOUR hex characters is not enough: the original took four base36 characters
|
|
* (1.7e6) and a naive 4-hex swap drops that to 65536, which collided 22 times in
|
|
* 5000 names generated inside the same millisecond. Fixture-name collisions are
|
|
* cross-test interference, so the width is load-bearing — keep 8.
|
|
*/
|
|
function uniqueName(prefix) {
|
|
return `${prefix}_${Date.now().toString(36)}${randomBytes(4).toString('hex')}`;
|
|
}
|
|
|
|
/** A transient meta-store error worth retrying rather than failing the test on. */
|
|
function isTransientStoreError(body) {
|
|
return /database is locked|SqlxError|SeaORMError|Execution Error/i.test(String(body ?? ''));
|
|
}
|
|
|
|
/**
|
|
* Both readings of "on-call is not here": the feature flag is off (404) or this
|
|
* is an OSS build (403 "Not Supported"). The product collapses them deliberately
|
|
* — §G.8.1 — because neither is something a reader can act on.
|
|
*/
|
|
function isUnavailableStatus(status, body) {
|
|
if (status === 404) return true;
|
|
return status === 403 && /not supported/i.test(String(body ?? ''));
|
|
}
|
|
|
|
/**
|
|
* Authenticated request with a retry on a transient meta-store error.
|
|
*
|
|
* The single-node build keeps metadata in SQLite, which serialises writers.
|
|
* Several Playwright workers creating teams, rotations and ownership rules at
|
|
* once collide and get `database is locked` — a contention signal, not a
|
|
* product failure. Without this the suite fails intermittently on an
|
|
* environment characteristic and sends the reader hunting a bug that is not there.
|
|
*/
|
|
async function api(page, method, url, data, { attempts = 5, baseDelayMs = 400 } = {}) {
|
|
const opts = { headers: getAuthHeaders() };
|
|
if (data !== undefined) opts.data = data;
|
|
|
|
let last = null;
|
|
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
const res = await page.request[method](url, opts);
|
|
if (res.ok()) return res;
|
|
|
|
const body = await res.text().catch(() => '');
|
|
last = { status: res.status(), body };
|
|
if (!isTransientStoreError(body)) return res;
|
|
|
|
await page.waitForTimeout(baseDelayMs * attempt);
|
|
testLogger.debug('oncall seed: retrying after a transient store error', {
|
|
url, attempt, status: res.status(),
|
|
});
|
|
}
|
|
throw new Error(
|
|
`Still failing after ${attempts} attempts on a transient store error.\n` +
|
|
` ${url}\n last: HTTP ${last?.status} — ${String(last?.body).slice(0, 200)}`,
|
|
);
|
|
}
|
|
|
|
/** Throw with the payload attached — a 400 that names only a field is unreadable without it. */
|
|
async function must(res, what, payload) {
|
|
if (res.ok()) return res.json().catch(() => ({}));
|
|
const body = await res.text().catch(() => '<unreadable>');
|
|
throw new Error(
|
|
`${what} failed: HTTP ${res.status()} — ${body}` +
|
|
(payload === undefined ? '' : `\nPayload: ${JSON.stringify(payload)}`),
|
|
);
|
|
}
|
|
|
|
/** GET one resource: null only when it does not exist (404), a throw on any other failure. */
|
|
async function getOrNull(page, url, what) {
|
|
const res = await page.request.get(url, { headers: getAuthHeaders() });
|
|
if (res.status() === 404) return null;
|
|
const text = await res.text().catch(() => '');
|
|
if (!res.ok()) throw new Error(`${what} failed: HTTP ${res.status()} — ${text.slice(0, 200)}`);
|
|
return text.trim() ? JSON.parse(text) : null;
|
|
}
|
|
|
|
// ------------------------------------------------------------- capability probe
|
|
|
|
/** The server's `build_type` from the unauthenticated `/config`, or a throw when it cannot be read. */
|
|
async function readBuildType(page) {
|
|
const res = await page.request.get(`${baseUrl()}/config`);
|
|
const text = await res.text().catch(() => '');
|
|
if (!res.ok()) {
|
|
throw new Error(`on-call probe: could not read GET /config (HTTP ${res.status()}) — ${text.slice(0, 200)}`);
|
|
}
|
|
let buildType;
|
|
try {
|
|
buildType = JSON.parse(text)?.build_type;
|
|
} catch {
|
|
buildType = undefined;
|
|
}
|
|
if (!buildType) throw new Error(`on-call probe: GET /config carries no build_type — ${text.slice(0, 200)}`);
|
|
return buildType;
|
|
}
|
|
|
|
/**
|
|
* Is on-call reachable here? Skips only on an OSS build; anything else that is not a working on-call throws.
|
|
*
|
|
* @returns {Promise<{available: boolean, reason: string}>}
|
|
*/
|
|
async function isOnCallAvailable(page) {
|
|
const org = orgId();
|
|
if (!org) throw new Error('on-call probe: no org identifier — ORGNAME / cloud-config is unset');
|
|
|
|
const res = await page.request.get(`${baseUrl()}/api/${org}/oncall/teams`, {
|
|
headers: getAuthHeaders(),
|
|
});
|
|
if (res.ok()) return { available: true, reason: '' };
|
|
|
|
const body = await res.text().catch(() => '');
|
|
// Every spec skips on `available: false`, so only a genuinely absent feature may return it.
|
|
if (!isUnavailableStatus(res.status(), body)) {
|
|
throw new Error(
|
|
`on-call probe failed: HTTP ${res.status()} on /oncall/teams — ${body.slice(0, 200)}`,
|
|
);
|
|
}
|
|
|
|
const buildType = await readBuildType(page);
|
|
if (buildType === 'opensource') {
|
|
return {
|
|
available: false,
|
|
reason: `on-call is enterprise-only and this is an OSS build (HTTP ${res.status()} on /oncall/teams)`,
|
|
};
|
|
}
|
|
throw new Error(
|
|
`on-call is switched off on this ${buildType} build: /oncall/teams answered HTTP ${res.status()}. ` +
|
|
'Start the server with O2_ONCALL_ENABLED=true; skipping here would report the on-call suite green ' +
|
|
'without running it.',
|
|
);
|
|
}
|
|
|
|
// -------------------------------------------------------------------- teams
|
|
|
|
/**
|
|
* Create a team.
|
|
*
|
|
* A new team is auto-staffed with a `source: "default"` rotation covering the
|
|
* whole roster weekly, so `GET /schedule` does NOT return null afterwards.
|
|
* "Never configured" has to be read from `source`, not from absence.
|
|
*
|
|
* @returns {Promise<{id: string, name: string}>}
|
|
*/
|
|
async function createTeam(page, { name, timezone = 'UTC', description = null } = {}) {
|
|
const payload = { name, timezone, description };
|
|
const res = await api(page, 'post', `${baseUrl()}/api/${orgId()}/oncall/teams`, payload);
|
|
const body = await must(res, `On-call team create "${name}"`, payload);
|
|
const id = body?.id ?? body?.data?.id;
|
|
if (!id) throw new Error(`Team created but no id returned: ${JSON.stringify(body)}`);
|
|
testLogger.info('On-call team created', { id, name });
|
|
return { id, name };
|
|
}
|
|
|
|
async function listTeams(page) {
|
|
const res = await page.request.get(`${baseUrl()}/api/${orgId()}/oncall/teams`, {
|
|
headers: getAuthHeaders(),
|
|
});
|
|
// An error is not an empty list. Swallowing it here is how an assertion
|
|
// like `expect(list).toEqual([])` passes the moment the API stops answering.
|
|
if (!res.ok()) {
|
|
throw new Error(
|
|
`listing on-call teams failed: ${res.status()} ${(await res.text()).slice(0, 200)}`,
|
|
);
|
|
}
|
|
const body = await res.json().catch(() => ([]));
|
|
return Array.isArray(body) ? body : (body?.list ?? []);
|
|
}
|
|
|
|
async function getTeam(page, teamId) {
|
|
return await getOrNull(
|
|
page, `${baseUrl()}/api/${orgId()}/oncall/teams/${encodeURIComponent(teamId)}`,
|
|
`Reading team ${teamId}`,
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Put people on the team.
|
|
*
|
|
* Membership carries no level: which rung somebody covers is a property of the
|
|
* ROTATION, never of belonging. Seeding a member does not put them on call.
|
|
*/
|
|
async function addTeamMembers(page, teamId, userEmails) {
|
|
const payload = { user_emails: userEmails };
|
|
const res = await api(
|
|
page, 'post',
|
|
`${baseUrl()}/api/${orgId()}/oncall/teams/${encodeURIComponent(teamId)}/members`,
|
|
payload,
|
|
);
|
|
const body = await must(res, `Add members to team ${teamId}`, payload);
|
|
testLogger.info('On-call members added', { teamId, count: userEmails.length });
|
|
return body;
|
|
}
|
|
|
|
async function listTeamMembers(page, teamId) {
|
|
const res = await page.request.get(
|
|
`${baseUrl()}/api/${orgId()}/oncall/teams/${encodeURIComponent(teamId)}/members`,
|
|
{ headers: getAuthHeaders() },
|
|
);
|
|
// An error is not an empty list. Swallowing it here is how an assertion
|
|
// like `expect(list).toEqual([])` passes the moment the API stops answering.
|
|
if (!res.ok()) {
|
|
throw new Error(
|
|
`listing team members failed: ${res.status()} ${(await res.text()).slice(0, 200)}`,
|
|
);
|
|
}
|
|
const body = await res.json().catch(() => ([]));
|
|
return Array.isArray(body) ? body : (body?.list ?? []);
|
|
}
|
|
|
|
// ----------------------------------------------------------------- schedule
|
|
|
|
/**
|
|
* One rotation, one shift rule, whole roster in turn.
|
|
*
|
|
* `anchor_micros` is the WHOLE of the secondary mechanism: two rotations with
|
|
* the same roster and anchors one shift apart can never resolve to the same
|
|
* person. Nothing at resolution time knows the two are related, so a secondary
|
|
* is built by passing `anchorMicros` one shift earlier — not by a flag.
|
|
*
|
|
* A rotation with no shift rules is rejected (§8.6) and is the one state that
|
|
* looks configured on a calendar and pages nobody.
|
|
*/
|
|
function rotation({
|
|
id,
|
|
name = id,
|
|
members,
|
|
shiftMicros = DAY_MICROS,
|
|
anchorMicros = null,
|
|
priority = 0,
|
|
restrictions = null,
|
|
} = {}) {
|
|
const rule = {
|
|
name: `${name} shift`,
|
|
members,
|
|
shift_micros: shiftMicros,
|
|
// Anchored in the past so somebody is already on call when the spec reads it.
|
|
anchor_micros: anchorMicros ?? (Date.now() * 1000 - shiftMicros),
|
|
priority,
|
|
};
|
|
if (restrictions) rule.restrictions = restrictions;
|
|
return { id, name, shift_rules: [rule] };
|
|
}
|
|
|
|
/**
|
|
* An all-day restriction.
|
|
*
|
|
* Minute 1440, never 0, for the closing edge: a restriction ending at minute 0
|
|
* applies at no instant at all and the server 400s it (§8.2), while the UI must
|
|
* render this one as `00:00 / 24:00` rather than `00:00 / 00:00` (§8.1).
|
|
*/
|
|
function allDayRestriction(days = [0, 1, 2, 3, 4, 5, 6]) {
|
|
return { days, start_minute: 0, end_minute: 1440 };
|
|
}
|
|
|
|
/**
|
|
* Replace the team's schedule.
|
|
*
|
|
* A FULL REPLACE, not a merge. Replacing a schedule whose rotation the
|
|
* escalation policy names is refused with a 400 telling the caller to keep the
|
|
* rotation or edit the policy first (§8.5), so order matters: set the schedule
|
|
* BEFORE the policy that points at it, and when changing rotation ids, clear
|
|
* the policy first.
|
|
*
|
|
* Add the people BEFORE writing a schedule that names them: a rotation naming
|
|
* somebody not on the team is refused with a 400. The default rotation and
|
|
* ladder the first member provisions are regenerated by the schedule write.
|
|
*/
|
|
async function setTeamSchedule(page, teamId, { timezone = 'UTC', rotations }) {
|
|
const payload = { timezone, rotations };
|
|
const res = await api(
|
|
page, 'put',
|
|
`${baseUrl()}/api/${orgId()}/oncall/teams/${encodeURIComponent(teamId)}/schedule`,
|
|
payload,
|
|
);
|
|
const body = await must(res, `Set schedule for team ${teamId}`, payload);
|
|
testLogger.info('On-call schedule set', { teamId, rotations: rotations.length });
|
|
return body;
|
|
}
|
|
|
|
async function getTeamSchedule(page, teamId) {
|
|
return await getOrNull(
|
|
page, `${baseUrl()}/api/${orgId()}/oncall/teams/${encodeURIComponent(teamId)}/schedule`,
|
|
`Reading the schedule of team ${teamId}`,
|
|
);
|
|
}
|
|
|
|
/** Who the engine says is on call, optionally at a future instant (micros). */
|
|
async function whoIsOnCall(page, teamId, atMicros = undefined) {
|
|
const query = atMicros === undefined ? '' : `?at=${atMicros}`;
|
|
const res = await page.request.get(
|
|
`${baseUrl()}/api/${orgId()}/oncall/teams/${encodeURIComponent(teamId)}/on-call${query}`,
|
|
{ headers: getAuthHeaders() },
|
|
);
|
|
// An error is not an empty list. Swallowing it here is how an assertion
|
|
// like `expect(list).toEqual([])` passes the moment the API stops answering.
|
|
if (!res.ok()) {
|
|
throw new Error(
|
|
`reading who is on call failed: ${res.status()} ${(await res.text()).slice(0, 200)}`,
|
|
);
|
|
}
|
|
const body = await res.json().catch(() => ([]));
|
|
return Array.isArray(body) ? body : (body?.list ?? []);
|
|
}
|
|
|
|
/**
|
|
* Hand one rotation's shift to somebody else for a window.
|
|
*
|
|
* `start_at` / `end_at`, NOT the plural `starts_at` / `ends_at` — the server
|
|
* rejects the plural form with a 422. `end_at` is exclusive, so a cover ending
|
|
* exactly when the next begins does not overlap it. `reason`, never `note`: a
|
|
* `note` key is dropped in silence.
|
|
*
|
|
* `rotation_id` must be populated on the stored record (§8.3) — a cover with a
|
|
* null rotation is the `oncall_overrides.rotation_id` outage this suite
|
|
* smoke-tests for. Omitting it targets the team's PRIMARY, which on a
|
|
* multi-rotation team silently evicts whoever held it.
|
|
*/
|
|
async function createCover(page, teamId, {
|
|
userEmail, rotationId, fromMicros, toMicros, coveringFor = undefined, reason = undefined,
|
|
}) {
|
|
const payload = {
|
|
user_email: userEmail,
|
|
rotation_id: rotationId,
|
|
start_at: fromMicros,
|
|
end_at: toMicros,
|
|
};
|
|
if (coveringFor !== undefined) payload.covering_for = coveringFor;
|
|
if (reason !== undefined) payload.reason = reason;
|
|
const res = await api(
|
|
page, 'post',
|
|
`${baseUrl()}/api/${orgId()}/oncall/teams/${encodeURIComponent(teamId)}/overrides`,
|
|
payload,
|
|
);
|
|
return await must(res, `Create cover on team ${teamId}`, payload);
|
|
}
|
|
|
|
// ------------------------------------------------------------------- policy
|
|
|
|
/**
|
|
* A ladder rung that pages whoever a rotation puts on call.
|
|
*
|
|
* `rotation_id`, never the rotation's NAME: a rotation is renameable, and a
|
|
* stored policy that keyed on the name would start paging a different position
|
|
* the moment somebody fixed a typo on a calendar.
|
|
*
|
|
* `mode` is omitted when it is `on_call` so a rung written without it
|
|
* round-trips unchanged — never send `"on_call"` explicitly.
|
|
*/
|
|
function rungPagingRotation(priority, rotationId, { afterMicros = FIRST_RUNG_MICROS, channels = ['email'] } = {}) {
|
|
return {
|
|
priority,
|
|
steps: [{ after_micros: afterMicros, targets: [{ kind: 'rotation', rotation_id: rotationId }] }],
|
|
channels,
|
|
};
|
|
}
|
|
|
|
/** A two-step ladder: the rotation now, then everybody a minute later. Exhausts fast enough to assert on. */
|
|
function rungEscalatingToTeam(priority, rotationId, { channels = ['email'] } = {}) {
|
|
return {
|
|
priority,
|
|
steps: [
|
|
{ after_micros: FIRST_RUNG_MICROS, targets: [{ kind: 'rotation', rotation_id: rotationId }] },
|
|
{ after_micros: SECOND_RUNG_MICROS, targets: [{ kind: 'whole_team' }] },
|
|
],
|
|
channels,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Write the escalation policy.
|
|
*
|
|
* `l0` is deliberately omitted unless a caller passes it: absent means
|
|
* unchanged, and sending it on an ordinary rung edit would silently
|
|
* un-configure the AI-SRE gate.
|
|
*/
|
|
async function setTeamPolicy(page, teamId, {
|
|
rungs,
|
|
destinations = undefined,
|
|
repeatCount = undefined,
|
|
finalAction = undefined,
|
|
l0 = undefined,
|
|
} = {}) {
|
|
const payload = { rungs };
|
|
if (destinations !== undefined) payload.destinations = destinations;
|
|
if (repeatCount !== undefined) payload.repeat_count = repeatCount;
|
|
if (finalAction !== undefined) payload.final_action = finalAction;
|
|
if (l0 !== undefined) payload.l0 = l0;
|
|
|
|
const res = await api(
|
|
page, 'put',
|
|
`${baseUrl()}/api/${orgId()}/oncall/teams/${encodeURIComponent(teamId)}/policy`,
|
|
payload,
|
|
);
|
|
const body = await must(res, `Set policy for team ${teamId}`, payload);
|
|
testLogger.info('On-call policy set', { teamId, rungs: rungs.length });
|
|
return body;
|
|
}
|
|
|
|
/**
|
|
* Point the policy at the team instead of at any rotation, so the schedule can
|
|
* then be replaced.
|
|
*
|
|
* This is the server's own advice — "keep the rotation, or edit the policy
|
|
* first" — made callable, and it is a SEEDING primitive, not an assertion
|
|
* helper: it exists so a spec can reach the state it wants to test, never so a
|
|
* §8.5 refusal can be swallowed. `setTeamSchedule` still surfaces a real 400.
|
|
*
|
|
* A single whole-team rung is the smallest policy the route accepts and names
|
|
* no rotation, which is the whole point.
|
|
*/
|
|
async function detachPolicyFromRotations(page, teamId) {
|
|
return await setTeamPolicy(page, teamId, {
|
|
rungs: [{
|
|
priority: 1,
|
|
steps: [{ after_micros: FIRST_RUNG_MICROS, targets: [{ kind: 'whole_team' }] }],
|
|
channels: ['email'],
|
|
}],
|
|
});
|
|
}
|
|
|
|
async function getTeamPolicy(page, teamId) {
|
|
return await getOrNull(
|
|
page, `${baseUrl()}/api/${orgId()}/oncall/teams/${encodeURIComponent(teamId)}/policy`,
|
|
`Reading the policy of team ${teamId}`,
|
|
);
|
|
}
|
|
|
|
// ---------------------------------------------------------------- ownership
|
|
|
|
/**
|
|
* Claim an identity path for a team.
|
|
*
|
|
* `dimensions` is `{name: value}` and EVERY pair must match for the rule to
|
|
* apply — which is also what makes one rule more specific than another, and so
|
|
* what decides which team is woken when two match (§5.5).
|
|
*
|
|
* A 409 means another team already owns this exact path. Repointing is a PUT on
|
|
* the existing rule, never delete-then-create: creating the replacement while
|
|
* the original still holds the path is a duplicate, and the server refuses it.
|
|
*/
|
|
async function createOwnershipRule(page, { teamId, dimensions }) {
|
|
const payload = { team_id: teamId, dimensions };
|
|
const res = await api(page, 'post', `${baseUrl()}/api/${orgId()}/oncall/ownership`, payload);
|
|
const body = await must(res, 'Create ownership rule', payload);
|
|
const id = body?.id ?? body?.rule_id;
|
|
testLogger.info('Ownership rule created', { id, teamId, dimensions });
|
|
return { id, teamId, dimensions };
|
|
}
|
|
|
|
async function listOwnershipRules(page, teamId = undefined) {
|
|
const query = teamId ? `?team_id=${encodeURIComponent(teamId)}` : '';
|
|
const res = await page.request.get(`${baseUrl()}/api/${orgId()}/oncall/ownership${query}`, {
|
|
headers: getAuthHeaders(),
|
|
});
|
|
// An error is not an empty list. Swallowing it here is how an assertion
|
|
// like `expect(list).toEqual([])` passes the moment the API stops answering.
|
|
if (!res.ok()) {
|
|
throw new Error(
|
|
`listing ownership rules failed: ${res.status()} ${(await res.text()).slice(0, 200)}`,
|
|
);
|
|
}
|
|
const body = await res.json().catch(() => ([]));
|
|
return Array.isArray(body) ? body : (body?.list ?? []);
|
|
}
|
|
|
|
/**
|
|
* Rules plus what each one CAUGHT — and the server's shadowing verdict.
|
|
*
|
|
* The analysis compares every rule against every other, so only the server can
|
|
* make it: a client that can see one team's rules cannot know another team's
|
|
* rule outranks it. A spec asserting "shadowed" must read `health` from here
|
|
* rather than recomputing it.
|
|
*/
|
|
async function ownershipStats(page, { teamId = undefined, days = undefined } = {}) {
|
|
const params = new URLSearchParams();
|
|
if (teamId) params.set('team_id', teamId);
|
|
if (days !== undefined) params.set('days', String(days));
|
|
const query = params.toString() ? `?${params}` : '';
|
|
return await getOrNull(
|
|
page, `${baseUrl()}/api/${orgId()}/oncall/ownership/stats${query}`,
|
|
'Reading ownership-rule stats',
|
|
);
|
|
}
|
|
|
|
/** Nominate the org's catch-all team, or clear it with null. */
|
|
async function setDefaultTeam(page, teamId) {
|
|
const payload = { default_team_id: teamId };
|
|
const res = await api(page, 'put', `${baseUrl()}/api/${orgId()}/oncall/routing/config`, payload);
|
|
return await must(res, 'Set default routing team', payload);
|
|
}
|
|
|
|
async function getRoutingConfig(page) {
|
|
return await getOrNull(
|
|
page, `${baseUrl()}/api/${orgId()}/oncall/routing/config`,
|
|
'Reading the routing config',
|
|
);
|
|
}
|
|
|
|
// ------------------------------------------------------------------ log data
|
|
|
|
/**
|
|
* Ingest rows carrying the dimensions ownership rules match on.
|
|
*
|
|
* Basic auth, not the cookie: the ingest endpoint rejects the session. Recent
|
|
* rather than backdated — an alert evaluates a trailing window, so unlike the
|
|
* SLO seed there is nothing to gain from reaching days back, and
|
|
* `ZO_INGEST_ALLOWED_UPTO` (default 5 hours) would drop it while still
|
|
* answering 200.
|
|
*
|
|
* @returns {Promise<{streamName: string, records: number}>}
|
|
*/
|
|
async function seedOnCallStream(page, streamName, {
|
|
minutes = 60,
|
|
services = SEED_SERVICES,
|
|
extraDimensions = {},
|
|
} = {}) {
|
|
const nowSecs = Math.floor(Date.now() / 1000);
|
|
const rows = [];
|
|
for (let i = 0; i < minutes; i++) {
|
|
rows.push({
|
|
_timestamp: (nowSecs - i * 60) * MICROS,
|
|
service: services[i % services.length],
|
|
latency: 100 + (i % 10) * 100,
|
|
status_code: i % 5 === 0 ? 500 : 200,
|
|
job: 'e2e-oncall',
|
|
...extraDimensions,
|
|
});
|
|
}
|
|
|
|
const res = await api(page, 'post', `${ingestBase(ingestUrl())}/api/${orgId()}/${streamName}/_json`, rows);
|
|
await must(res, `On-call stream ingest "${streamName}"`);
|
|
await waitForStreamSearchable(page, streamName, rows.length);
|
|
testLogger.info('On-call stream seeded', { streamName, records: rows.length });
|
|
return { streamName, records: rows.length };
|
|
}
|
|
|
|
/** Block until the rows are queryable, so an alert created next can actually match them. */
|
|
async function waitForStreamSearchable(page, streamName, expected, { timeout = 120_000 } = {}) {
|
|
const nowSecs = Math.floor(Date.now() / 1000);
|
|
const deadline = Date.now() + timeout;
|
|
|
|
while (Date.now() < deadline) {
|
|
const res = await page.request.post(`${baseUrl()}/api/${orgId()}/_search?type=logs`, {
|
|
headers: getAuthHeaders(),
|
|
data: {
|
|
query: {
|
|
sql: `SELECT COUNT(*) AS c FROM "${streamName}"`,
|
|
start_time: (nowSecs - 86400) * MICROS,
|
|
end_time: (nowSecs + 3600) * MICROS,
|
|
size: 1,
|
|
},
|
|
},
|
|
});
|
|
if (res.ok()) {
|
|
const body = await res.json().catch(() => ({}));
|
|
// 90% is enough: demanding an exact count stalls on ordinary indexing lag.
|
|
if (Number(body?.hits?.[0]?.c ?? 0) >= expected * 0.9) return;
|
|
}
|
|
await page.waitForTimeout(3000);
|
|
}
|
|
throw new Error(`Seeded rows for "${streamName}" never became searchable within ${timeout}ms.`);
|
|
}
|
|
|
|
// -------------------------------------------------------------- destinations
|
|
|
|
/**
|
|
* A notification template + destination pointed at this instance's OWN ingest.
|
|
*
|
|
* Delivery is then self-contained and needs no external service, and a page's
|
|
* delivery ledger still records real attempts. Same trick
|
|
* `slo-seed.seedNotificationDestination` uses.
|
|
*
|
|
* @returns {Promise<string>} the destination name
|
|
*/
|
|
async function seedNotificationDestination(page, baseName) {
|
|
const v1 = `${baseUrl()}/api/${orgId()}`;
|
|
const template = `${baseName}_tmpl`;
|
|
const destination = `${baseName}_dest`;
|
|
|
|
const tmplRes = await api(page, 'post', `${v1}/alerts/templates`, {
|
|
name: template,
|
|
body: '{"text":"{alert_name} {alert_level}"}',
|
|
type: 'http',
|
|
title: '',
|
|
});
|
|
if (!tmplRes.ok() && tmplRes.status() !== 409) {
|
|
await must(tmplRes, `Create alert template "${template}"`);
|
|
}
|
|
|
|
const destRes = await api(page, 'post', `${v1}/alerts/destinations`, {
|
|
name: destination,
|
|
url: `${v1}/${baseName}_sink/_json`,
|
|
method: 'post',
|
|
template,
|
|
type: 'http',
|
|
headers: getAuthHeaders(),
|
|
});
|
|
if (!destRes.ok() && destRes.status() !== 409) {
|
|
await must(destRes, `Create alert destination "${destination}"`);
|
|
}
|
|
|
|
testLogger.info('On-call notification destination seeded', { destination });
|
|
return destination;
|
|
}
|
|
|
|
// ------------------------------------------------------------------- alerts
|
|
|
|
/**
|
|
* A scheduled alert that pages a team.
|
|
*
|
|
* `oncall_team` is an on-call team's **id**. It is a destination in its own
|
|
* right (§4.1): an alert carrying it saves with `destinations: []` and
|
|
* `workflows: []`.
|
|
*
|
|
* `multiAlert` is the §5 distinction and it is NOT cosmetic. With
|
|
* `aggregation.multi_alert: true` a result set spanning two owning teams opens
|
|
* one page PER group; with `aggregation: null` — Simple — the groups are
|
|
* collapsed into a single result and exactly one page is correct. Both halves
|
|
* are pinned so the distinction cannot rot.
|
|
*
|
|
* @returns {Promise<{id: string, name: string}>}
|
|
*/
|
|
async function createPagingAlert(page, {
|
|
name,
|
|
stream,
|
|
teamId = undefined,
|
|
priority = undefined,
|
|
destinations = [],
|
|
multiAlert = false,
|
|
groupBy = ['service'],
|
|
aggregateFunction = 'avg',
|
|
having = { column: 'latency', operator: '>', value: 0 },
|
|
frequencyMinutes = 1,
|
|
periodMinutes = 10,
|
|
folder = 'default',
|
|
} = {}) {
|
|
const payload = {
|
|
name,
|
|
stream_type: 'logs',
|
|
stream_name: stream,
|
|
is_real_time: false,
|
|
query_condition: {
|
|
type: 'custom',
|
|
conditions: {
|
|
version: 2,
|
|
conditions: { filterType: 'group', logicalOperator: 'AND', conditions: [] },
|
|
},
|
|
sql: null, promql: null, promql_condition: null,
|
|
aggregation: multiAlert
|
|
? { group_by: groupBy, function: aggregateFunction, having, multi_alert: true }
|
|
: null,
|
|
vrl_function: null, search_event_type: null, multi_time_range: [],
|
|
},
|
|
trigger_condition: {
|
|
period: periodMinutes,
|
|
operator: '>=',
|
|
// 1 is the "any breaching group" gate a multi-alert needs; a simple alert
|
|
// is happy with it too, and a higher threshold would make the fire flaky.
|
|
threshold: 1,
|
|
frequency: frequencyMinutes,
|
|
frequency_type: 'minutes',
|
|
cron: '',
|
|
// No silence: a silenced alert fires once and then goes quiet for the
|
|
// rest of the run, which reads exactly like the ladder failing to fire.
|
|
silence: 0,
|
|
timezone: 'UTC',
|
|
align_time: true,
|
|
},
|
|
destinations,
|
|
context_attributes: {},
|
|
row_template: '',
|
|
enabled: true,
|
|
};
|
|
if (teamId) payload.oncall_team = teamId;
|
|
// An alert with no priority opens its page at the server's default, not P1, so a
|
|
// severity-specific assertion has to name one or it silently tests another rung.
|
|
if (priority !== undefined) payload.priority = priority;
|
|
|
|
const res = await api(
|
|
page, 'post',
|
|
`${baseUrl()}/api/v2/${orgId()}/alerts?folder=${encodeURIComponent(folder)}`,
|
|
payload,
|
|
);
|
|
const body = await must(res, `Create paging alert "${name}"`, payload);
|
|
const id = body?.id ?? body?.alert_id;
|
|
if (!id) throw new Error(`Alert created but no id returned: ${JSON.stringify(body)}`);
|
|
testLogger.info('Paging alert created', { id, name, teamId, priority, multiAlert });
|
|
return { id, name };
|
|
}
|
|
|
|
// -------------------------------------------------------------------- pages
|
|
|
|
/**
|
|
* Read the pages list.
|
|
*
|
|
* `include_resolved` is off by default server-side — the screen is what still
|
|
* needs somebody — so a spec looking for a page it already closed must ask for
|
|
* it. The endpoint has NO total count and caps at 200: never present the
|
|
* returned length as the total (§G.5).
|
|
*/
|
|
async function listResponses(page, {
|
|
teamId = undefined,
|
|
includeResolved = false,
|
|
cause = undefined,
|
|
subjectType = undefined,
|
|
sourceId = undefined,
|
|
ownershipPath = undefined,
|
|
limit = 200,
|
|
offset = undefined,
|
|
} = {}) {
|
|
const params = new URLSearchParams();
|
|
if (teamId) params.set('team_id', teamId);
|
|
if (includeResolved) params.set('include_resolved', 'true');
|
|
if (cause) params.set('cause', cause);
|
|
if (subjectType) params.set('subject_type', subjectType);
|
|
if (sourceId) params.set('source_id', sourceId);
|
|
if (ownershipPath) params.set('ownership_path', ownershipPath);
|
|
if (limit !== undefined) params.set('limit', String(limit));
|
|
if (offset !== undefined) params.set('offset', String(offset));
|
|
|
|
const query = params.toString() ? `?${params}` : '';
|
|
const res = await page.request.get(`${baseUrl()}/api/${orgId()}/oncall/responses${query}`, {
|
|
headers: getAuthHeaders(),
|
|
});
|
|
// An error is not an empty list. Swallowing it here is how an assertion
|
|
// like `expect(list).toEqual([])` passes the moment the API stops answering.
|
|
if (!res.ok()) {
|
|
throw new Error(
|
|
`listing pages failed: ${res.status()} ${(await res.text()).slice(0, 200)}`,
|
|
);
|
|
}
|
|
const body = await res.json().catch(() => ([]));
|
|
return Array.isArray(body) ? body : (body?.list ?? []);
|
|
}
|
|
|
|
async function getResponse(page, responseId) {
|
|
return await getOrNull(
|
|
page, `${baseUrl()}/api/${orgId()}/oncall/responses/${encodeURIComponent(responseId)}`,
|
|
`Reading page ${responseId}`,
|
|
);
|
|
}
|
|
|
|
/** The routing evidence on a page that landed on the wrong team, for the failure message. */
|
|
async function describeMisroutedPage(page, record, expectTeamId) {
|
|
const lines = [
|
|
`Page ${record.id} opened on team ${record.team_id ?? 'NONE (unrouted)'}; ` +
|
|
`expected ${expectTeamId ?? 'NONE (unrouted)'}.`,
|
|
];
|
|
try {
|
|
const detail = await getResponse(page, record.id);
|
|
for (const e of detail?.events ?? []) {
|
|
lines.push(` event ${e.kind} by ${e.actor}: ${String(e.body ?? '').slice(0, 300)}`);
|
|
}
|
|
const progress = await getEscalationProgress(page, record.id);
|
|
if (progress?.stopped_because) lines.push(` escalation stopped because: ${progress.stopped_because}`);
|
|
} catch (e) {
|
|
lines.push(` (could not read the record's events: ${e.message})`);
|
|
}
|
|
lines.push(
|
|
'Check the ownership rule claims the dimension the alert\'s rows carry, and that nothing ' +
|
|
'fired the alert by hand: a manual trigger opens a teamless page that swallows every scheduled firing.',
|
|
);
|
|
return lines.join('\n');
|
|
}
|
|
|
|
/**
|
|
* Wait for the pages an alert opens, matched on `source_id` because each firing is its own record.
|
|
*
|
|
* `expectTeamId` (a team id, or null for an unrouted page) fails at once when any page lands elsewhere.
|
|
*
|
|
* @returns {Promise<object[]>} every page opened by this alert, newest first
|
|
*/
|
|
async function waitForPages(page, {
|
|
alertId,
|
|
expected = 1,
|
|
teamId = undefined,
|
|
expectTeamId = undefined,
|
|
timeout = PAGE_WAIT_MS,
|
|
pollMs = 2000,
|
|
} = {}) {
|
|
const deadline = Date.now() + timeout;
|
|
let seen = [];
|
|
|
|
while (Date.now() < deadline) {
|
|
seen = await listResponses(page, {
|
|
subjectType: 'alert', sourceId: alertId, teamId, includeResolved: true,
|
|
});
|
|
if (expectTeamId !== undefined) {
|
|
const stray = seen.find((r) => (r.team_id ?? null) !== expectTeamId);
|
|
if (stray) throw new Error(await describeMisroutedPage(page, stray, expectTeamId));
|
|
}
|
|
if (seen.length >= expected) {
|
|
testLogger.info('On-call page(s) observed', { alertId, count: seen.length });
|
|
return seen;
|
|
}
|
|
await page.waitForTimeout(pollMs);
|
|
}
|
|
|
|
throw new Error(
|
|
`Alert ${alertId} opened ${seen.length} of ${expected} on-call page(s) within ${timeout}ms.\n` +
|
|
'A firing opens a page even when nothing routes it, so a shortfall means the alert never FIRED: ' +
|
|
'check the stream has rows inside the alert\'s window and that the alert scheduler is running.',
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Create the alert and wait for the scheduler to fire it into a page; never fire it by hand.
|
|
*
|
|
* `expectTeamId` defaults to the alert's own `teamId`; pass the team an ownership rule should pick otherwise.
|
|
*
|
|
* @returns {Promise<{alert: {id: string, name: string}, pages: object[]}>}
|
|
*/
|
|
async function firePageAndWait(page, {
|
|
alertOptions,
|
|
expected = 1,
|
|
expectTeamId = alertOptions?.teamId,
|
|
timeout = PAGE_WAIT_MS,
|
|
} = {}) {
|
|
const alert = await createPagingAlert(page, alertOptions);
|
|
const pages = await waitForPages(page, { alertId: alert.id, expected, expectTeamId, timeout });
|
|
return { alert, pages };
|
|
}
|
|
|
|
/**
|
|
* Send a REAL page to whoever the ladder would reach, and report who it got to.
|
|
*
|
|
* `reached_anyone: false` carries `not_sent_because` — the honest answer to
|
|
* "would a page actually land", which is not otherwise knowable without waiting
|
|
* for an incident. Cheaper than firePageAndWait when the subject is coverage
|
|
* rather than a record on the pages list.
|
|
*/
|
|
async function sendTestPage(page, teamId, { priority = undefined } = {}) {
|
|
const payload = priority === undefined ? {} : { priority };
|
|
const res = await api(
|
|
page, 'post',
|
|
`${baseUrl()}/api/${orgId()}/oncall/teams/${encodeURIComponent(teamId)}/test-page`,
|
|
payload,
|
|
);
|
|
return await must(res, `Test page for team ${teamId}`, payload);
|
|
}
|
|
|
|
/**
|
|
* How far the ladder got, who it reached, and whether it is exhausted (§2, §3).
|
|
*
|
|
* The route is `/escalation`, not `/progress` — the shape it returns is named
|
|
* `EscalationProgress`, which is the trap.
|
|
*/
|
|
async function getEscalationProgress(page, responseId) {
|
|
return await getOrNull(
|
|
page, `${baseUrl()}/api/${orgId()}/oncall/responses/${encodeURIComponent(responseId)}/escalation`,
|
|
`Reading the escalation progress of page ${responseId}`,
|
|
);
|
|
}
|
|
|
|
/** The delivery ledger — the ground truth `progress.reached` must agree with (§2.1). */
|
|
async function getDeliveries(page, responseId) {
|
|
return await getOrNull(
|
|
page, `${baseUrl()}/api/${orgId()}/oncall/responses/${encodeURIComponent(responseId)}/deliveries`,
|
|
`Reading the delivery ledger of page ${responseId}`,
|
|
);
|
|
}
|
|
|
|
// ------------------------------------------------------------------ teardown
|
|
|
|
/**
|
|
* Remove everything a run created under one prefix.
|
|
*
|
|
* Best-effort throughout: cleanup must never mask a test result, and a
|
|
* half-deleted fixture is better than a failed teardown hiding a real failure.
|
|
*
|
|
* ORDER IS LOAD-BEARING:
|
|
* 1. Alerts, because a scheduled alert whose stream is gone is retried every
|
|
* scheduler cycle forever and starves the same worker budget the on-call
|
|
* escalation lane runs on — leaving them behind breaks LATER runs.
|
|
* 2. Ownership rules, before the teams they point at.
|
|
* 3. Teams.
|
|
* 4. Streams, destinations, then templates — a template in use by a
|
|
* destination cannot be removed.
|
|
*/
|
|
async function deleteOnCallFixturesByPrefix(page, prefix) {
|
|
const org = orgId();
|
|
const base = baseUrl();
|
|
const headers = getAuthHeaders();
|
|
|
|
const tryDelete = async (url) => {
|
|
try { await page.request.delete(url, { headers }); } catch { /* best effort */ }
|
|
};
|
|
|
|
try {
|
|
const res = await page.request.get(
|
|
`${base}/api/v2/${org}/alerts?folder=default&page_size=200`, { headers },
|
|
);
|
|
if (res.ok()) {
|
|
const body = await res.json().catch(() => ({}));
|
|
for (const a of body?.list ?? []) {
|
|
if (typeof a?.name === 'string' && a.name.startsWith(prefix)) {
|
|
const id = a.alert_id ?? a.id;
|
|
if (id) await tryDelete(`${base}/api/v2/${org}/alerts/${id}`);
|
|
}
|
|
}
|
|
}
|
|
} catch { /* best effort */ }
|
|
|
|
// Rules are keyed by id and carry no name, so they are swept by the team they
|
|
// point at — which is why teams must be identified before they are deleted.
|
|
let doomedTeamIds = [];
|
|
try {
|
|
const teams = await listTeams(page);
|
|
doomedTeamIds = teams
|
|
.filter((t) => typeof t?.name === 'string' && t.name.startsWith(prefix))
|
|
.map((t) => t.id);
|
|
} catch { /* best effort */ }
|
|
|
|
try {
|
|
const rules = await listOwnershipRules(page);
|
|
for (const rule of rules) {
|
|
const id = rule?.id ?? rule?.rule_id;
|
|
if (id && doomedTeamIds.includes(rule?.team_id)) {
|
|
await tryDelete(`${base}/api/${org}/oncall/ownership/${encodeURIComponent(id)}`);
|
|
}
|
|
}
|
|
} catch { /* best effort */ }
|
|
|
|
for (const teamId of doomedTeamIds) {
|
|
await tryDelete(`${base}/api/${org}/oncall/teams/${encodeURIComponent(teamId)}`);
|
|
}
|
|
|
|
try {
|
|
const res = await page.request.get(`${base}/api/${org}/streams?type=logs`, { headers });
|
|
if (res.ok()) {
|
|
const body = await res.json().catch(() => ({}));
|
|
for (const stream of body?.list ?? []) {
|
|
if (typeof stream?.name === 'string' && stream.name.startsWith(prefix)) {
|
|
await tryDelete(`${base}/api/${org}/streams/${stream.name}?type=logs`);
|
|
}
|
|
}
|
|
}
|
|
} catch { /* best effort */ }
|
|
|
|
for (const kind of ['destinations', 'templates']) {
|
|
try {
|
|
const res = await page.request.get(`${base}/api/${org}/alerts/${kind}`, { headers });
|
|
if (!res.ok()) continue;
|
|
const body = await res.json().catch(() => ([]));
|
|
for (const item of (Array.isArray(body) ? body : body.list ?? [])) {
|
|
if (typeof item?.name === 'string' && item.name.startsWith(prefix)) {
|
|
await tryDelete(`${base}/api/${org}/alerts/${kind}/${item.name}`);
|
|
}
|
|
}
|
|
} catch { /* best effort */ }
|
|
}
|
|
|
|
// Users LAST, and they are swept at all because nothing else sweeps them: the
|
|
// schedule timeline labels its bands by fetching the whole org's user list, so
|
|
// every account a run leaves behind widens the window in which a bar shows a
|
|
// raw address instead of a name. Left unswept the org reached ~1500 accounts
|
|
// and that window grew long enough to fail an assertion that had been passing.
|
|
let sweptUsers = 0;
|
|
try {
|
|
const res = await page.request.get(`${base}/api/${org}/users`, { headers });
|
|
if (res.ok()) {
|
|
const body = await res.json().catch(() => ({}));
|
|
for (const user of (body?.data ?? [])) {
|
|
const email = user?.email;
|
|
if (typeof email === 'string' && email.startsWith(prefix)) {
|
|
await tryDelete(`${base}/api/${org}/users/${encodeURIComponent(email)}`);
|
|
sweptUsers += 1;
|
|
}
|
|
}
|
|
}
|
|
} catch { /* best effort */ }
|
|
|
|
testLogger.debug('on-call fixture cleanup swept', {
|
|
prefix, teams: doomedTeamIds.length, users: sweptUsers,
|
|
});
|
|
}
|
|
|
|
/** Delete one team by id, tolerating an already-gone one. */
|
|
async function deleteTeamById(page, teamId) {
|
|
try {
|
|
await page.request.delete(
|
|
`${baseUrl()}/api/${orgId()}/oncall/teams/${encodeURIComponent(teamId)}`,
|
|
{ headers: getAuthHeaders() },
|
|
);
|
|
} catch (e) {
|
|
testLogger.debug('On-call team cleanup failed (non-fatal)', { teamId, error: e.message });
|
|
}
|
|
}
|
|
|
|
module.exports = {
|
|
MICROS,
|
|
HOUR_MICROS,
|
|
DAY_MICROS,
|
|
api,
|
|
FIRST_RUNG_MICROS,
|
|
SECOND_RUNG_MICROS,
|
|
SEED_SERVICES,
|
|
baseUrl,
|
|
ingestUrl,
|
|
orgId,
|
|
uniqueName,
|
|
isOnCallAvailable,
|
|
createTeam,
|
|
listTeams,
|
|
getTeam,
|
|
addTeamMembers,
|
|
listTeamMembers,
|
|
rotation,
|
|
allDayRestriction,
|
|
setTeamSchedule,
|
|
getTeamSchedule,
|
|
whoIsOnCall,
|
|
createCover,
|
|
rungPagingRotation,
|
|
rungEscalatingToTeam,
|
|
setTeamPolicy,
|
|
detachPolicyFromRotations,
|
|
getTeamPolicy,
|
|
createOwnershipRule,
|
|
listOwnershipRules,
|
|
ownershipStats,
|
|
setDefaultTeam,
|
|
getRoutingConfig,
|
|
seedOnCallStream,
|
|
waitForStreamSearchable,
|
|
seedNotificationDestination,
|
|
createPagingAlert,
|
|
listResponses,
|
|
getResponse,
|
|
waitForPages,
|
|
firePageAndWait,
|
|
sendTestPage,
|
|
getEscalationProgress,
|
|
getDeliveries,
|
|
deleteOnCallFixturesByPrefix,
|
|
deleteTeamById,
|
|
};
|