mirror of
https://github.com/stablyai/orca.git
synced 2026-11-04 11:07:42 +00:00
Jinwoo Hong
8cf6228655
* feat(relay): alert on cell disconnect bursts and database pool herds Two alert policies that separate a load-balancer mass disconnect from a database stall, after the 2026-10-06 18:34Z asia-east2 drop: - Disconnect burst: over 120 code-1006 desktop control closes on one cell in a minute, counted from the cell's own close line. The LB's internal_error log entries carry the stream start as their timestamp (median 2.4 h before the drop), so an LB-log rate never shows the pulse. - Pool herd: a cell runtime sample with more than 50 requests queued for a PostgreSQL connection (interval maximum, bar in the filter). Also publishes orca_relay_host_hellos_shed for the refusal counter from #26362, and pins both filters to the relay's text in a contract test. * fix(relay): raise the cell pool-herd alert bar to 200 waiters asia-east2 cells reach 184-196 waiters outside any herd, so >50 would have fired in 84 episodes since 2026-10-03. >200 fires in 10, covering every known herd. Drops the false 'healthy peak is 2 waiters' baseline from the policy text. * docs(relay): describe pool-herd baseline as stalls, not routine load A healthy cell peaks at 2 waiters (p99); the 50-196 readings are single-cell stalls that already time out. The 200 bar is unchanged.
46 lines
2.2 KiB
JavaScript
46 lines
2.2 KiB
JavaScript
import assert from 'node:assert/strict'
|
|
import { readFileSync } from 'node:fs'
|
|
import test from 'node:test'
|
|
import { fileURLToPath } from 'node:url'
|
|
|
|
// Why: both herd alerts key on text the relay writes (a log line and a runtime field). A rename on
|
|
// either side silences the alert without failing anything.
|
|
|
|
const read = (relative) => readFileSync(fileURLToPath(new URL(relative, import.meta.url)), 'utf8')
|
|
const terraform = read('../../infra/terraform/relay-observability.tf')
|
|
|
|
const block = (kind, name) => {
|
|
const body = new RegExp(`resource "${kind}" "${name}" \\{([\\s\\S]*?)\\n\\}`).exec(terraform)?.[1]
|
|
assert.ok(body, `${kind}.${name} not found in relay-observability.tf`)
|
|
return body
|
|
}
|
|
|
|
const incidentMetric = (key) => {
|
|
const body = new RegExp(`\\n ${key} = \\{([\\s\\S]*?)\\n \\}`).exec(terraform)?.[1]
|
|
assert.ok(body, `relay_incident_metrics.${key} not found`)
|
|
return body
|
|
}
|
|
|
|
test('the disconnect-burst metric matches the line a cell writes per control close', () => {
|
|
const emitter = read('../../apps/relay/src/host-session-registry.ts')
|
|
assert.match(emitter, /`\[orca-relay\] control closed host=\$\{/)
|
|
assert.match(emitter, /` code=\$\{code\} reason=\$\{/)
|
|
const metric = incidentMetric('cell_control_abnormal_closes')
|
|
assert.match(metric, /jsonPayload\.message:\\"\[orca-relay\] control closed \\"/)
|
|
assert.match(metric, /jsonPayload\.message:\\" code=1006 \\"/)
|
|
assert.match(metric, /logs\/cos_containers/)
|
|
const policy = block('google_monitoring_alert_policy', 'relay_cell_control_close_burst')
|
|
assert.match(policy, /orca_relay_cell_control_abnormal_closes/)
|
|
assert.match(policy, /group_by_fields\s*=\s*\["resource\.label\.\\"instance_id\\""\]/)
|
|
})
|
|
|
|
test('the pool-herd metric thresholds the interval maximum the relay emits', () => {
|
|
const emitter = read('../../apps/relay/src/postgres-pool-pressure.ts')
|
|
assert.match(emitter, /\bdatabasePoolWaitersMax: number\b/)
|
|
const metric = block('google_logging_metric', 'relay_cell_pool_herd')
|
|
// 200, not lower: single-cell stalls of 50-196 waiters recur daily; herds go past 200.
|
|
assert.match(metric, /jsonPayload\.databasePoolWaitersMax>200"/)
|
|
const policy = block('google_monitoring_alert_policy', 'relay_cell_pool_herd')
|
|
assert.match(policy, /threshold_value\s*=\s*0\n/)
|
|
})
|