mirror of
https://github.com/stablyai/orca.git
synced 2026-10-30 16:47:34 +00:00
Jinwoo Hong
e9dc9534d7
A reserve cell's dead-man is fed only by director polls through the load balancer. With SQL readiness the balancer drops the cell during a database outage, the polls get 503 failed_to_pick_backend, and the cell trips (staging drill 2026-10-11, c2 at 11:47:59Z). admitMode=reserve now needs readinessLocal=true in the object being written, already there or set in the same write, and a reserve cell refuses readinessLocal=false unless the same write flips it back to db.
563 lines
26 KiB
JavaScript
563 lines
26 KiB
JavaScript
import assert from 'node:assert/strict'
|
|
import { test } from 'node:test'
|
|
import {
|
|
assertSupportedObject,
|
|
desiredObject,
|
|
operateCellFlags,
|
|
parseCellFlagsRequest,
|
|
parseFlagChanges,
|
|
READ_BACK_TIMEOUT_MS,
|
|
REREGISTER_WAIT_MAX_MS,
|
|
REREGISTER_WAIT_MIN_MS,
|
|
reregisterLanes,
|
|
reregisterWaitMs,
|
|
WRITE_CONFIRMATION
|
|
} from './operate-relay-cell-flags.mjs'
|
|
|
|
const CELL = 'production-gce-c26'
|
|
const RESERVE = { 'director-origin': 'https://relay.onorca.dev', 'confirm-reserve': `RESERVE ${CELL}` }
|
|
const values = (overrides = {}) => ({
|
|
mode: 'write',
|
|
'cell-id': CELL,
|
|
'cell-origin': 'https://c26.relay.onorca.dev',
|
|
'project-id': 'onorca-cloud',
|
|
set: 'readinessLocal=true',
|
|
'expected-generation': '0',
|
|
confirmation: `${WRITE_CONFIRMATION} ${CELL}`,
|
|
...overrides
|
|
})
|
|
|
|
// A cell whose runtime status applies whatever was written, after `applyAfterPolls` polls.
|
|
const SUPPORTED = {
|
|
readinessLocal: { type: 'boolean' },
|
|
ticketCheck: { type: 'enum', values: ['off', 'shadow', 'enforce'] },
|
|
admitMode: { type: 'enum', values: ['db', 'reserve'] },
|
|
intakePerSec: { type: 'number', min: 0, max: 1_000 },
|
|
reserveDryRun: { type: 'boolean' },
|
|
rejectionFence: { type: 'boolean' },
|
|
readTimeoutMarginMs: { type: 'number', min: 1_000, max: 60_000, integer: true },
|
|
reregisterInFlight: { type: 'number', min: 1, max: 16, integer: true }
|
|
}
|
|
|
|
function fakeGoogleAndCell({
|
|
stored = null,
|
|
applyAfterPolls = 2,
|
|
role = 'cell',
|
|
cellId = CELL,
|
|
supportedFlags = SUPPORTED,
|
|
// Polls that still report admitModeEffective=reserve after a flip back.
|
|
reregisteringPolls = 0,
|
|
controls,
|
|
// Forces admitModeEffective, as a tripped dead-man reports it.
|
|
effectiveOverride,
|
|
heartbeatFresh = false,
|
|
// The director refuses this numbered reserve post (1-based) with this error.
|
|
refuseReservePost
|
|
} = {}) {
|
|
const state = {
|
|
stored,
|
|
writes: [],
|
|
polls: 0,
|
|
// A cell that has applied whatever object is already there.
|
|
applied: stored
|
|
? { generation: Number(stored.generation), flags: stored.object.flags }
|
|
: { generation: 0, flags: { readinessLocal: false, ticketCheck: 'off' } },
|
|
directorAdmitMode: 'db',
|
|
reservePosts: 0,
|
|
order: [],
|
|
reregisteringPolls
|
|
}
|
|
const fetchImpl = async (url, init = {}) => {
|
|
const target = new URL(url)
|
|
if (target.pathname === '/v1/admin/cell-admit-mode') {
|
|
const body = JSON.parse(String(init.body))
|
|
if (body.admitMode === 'reserve') {
|
|
state.reservePosts += 1
|
|
if (refuseReservePost?.post === state.reservePosts) {
|
|
return Response.json({ error: refuseReservePost.error }, { status: 409 })
|
|
}
|
|
}
|
|
if (body.admitMode) {
|
|
state.directorAdmitMode = body.admitMode
|
|
state.order.push(`director:${body.admitMode}`)
|
|
}
|
|
return Response.json({ v: 1, cellId: body.cellId, admitMode: state.directorAdmitMode, updatedAt: 1, heartbeatFresh })
|
|
}
|
|
if (target.pathname === '/v1/admin/runtime-status') {
|
|
state.polls += 1
|
|
if (state.stored && state.polls > applyAfterPolls) {
|
|
state.applied = { generation: Number(state.stored.generation), flags: state.stored.object.flags }
|
|
}
|
|
const reserveApplied = state.applied.flags.admitMode === 'reserve'
|
|
const effective = reserveApplied || (state.writes.length > 0 && state.reregisteringPolls-- > 0) ? 'reserve' : 'db'
|
|
return Response.json({
|
|
v: 1,
|
|
role,
|
|
cellId,
|
|
flagsApplied: state.applied,
|
|
admitModeEffective: effectiveOverride ?? effective,
|
|
...(controls === undefined ? {} : { runtime: { controls } }),
|
|
...(supportedFlags ? { supportedFlags } : {})
|
|
})
|
|
}
|
|
if (target.pathname.startsWith('/upload/')) {
|
|
const expected = target.searchParams.get('ifGenerationMatch')
|
|
if (expected !== (state.stored?.generation ?? '0')) return new Response(null, { status: 412 })
|
|
const parts = String(init.body).split('\r\n')
|
|
const object = JSON.parse(parts[parts.indexOf('Content-Type: application/json') + 2])
|
|
const metadata = JSON.parse(parts[3])
|
|
const generation = String(Number(state.stored?.generation ?? 1000) + 1)
|
|
state.stored = { generation, object }
|
|
state.writes.push({ generation, object, metadata })
|
|
state.order.push(`file:${object.flags.admitMode ?? 'db'}`)
|
|
return Response.json({ generation })
|
|
}
|
|
if (!state.stored) return new Response(null, { status: 404 })
|
|
if (target.searchParams.get('alt') === 'media') return Response.json(state.stored.object)
|
|
return Response.json({ generation: state.stored.generation })
|
|
}
|
|
return { state, fetchImpl }
|
|
}
|
|
|
|
const run = (request, fake, extra = {}) => {
|
|
let clock = 0
|
|
const lines = []
|
|
return {
|
|
lines,
|
|
result: operateCellFlags(request, {
|
|
fetchImpl: fake.fetchImpl,
|
|
accessToken: 'access',
|
|
idToken: 'id',
|
|
audit: { writtenBy: 'octocat', runId: '42', runAttempt: '1' },
|
|
log: (line) => lines.push(JSON.parse(line)),
|
|
now: () => clock,
|
|
sleep: async (ms) => {
|
|
clock += ms
|
|
},
|
|
...extra
|
|
})
|
|
}
|
|
}
|
|
|
|
test('refuses malformed requests before touching anything', () => {
|
|
assert.throws(() => parseCellFlagsRequest(values({ mode: 'apply' })), /mode/)
|
|
assert.throws(() => parseCellFlagsRequest(values({ 'cell-id': 'all' })), /cell id/)
|
|
assert.throws(() => parseCellFlagsRequest(values({ set: 'ticketCheck=always' })), /ticketCheck/)
|
|
assert.throws(() => parseCellFlagsRequest(values({ set: 'placerMode=on' })), /unknown switch/)
|
|
assert.throws(() => parseCellFlagsRequest(values({ set: '' })), /names no switch/)
|
|
assert.throws(() => parseCellFlagsRequest(values({ set: 'admitMode=db,admitMode=reserve' })), /twice/)
|
|
assert.throws(() => parseCellFlagsRequest(values({ set: 'readTimeoutMarginMs=500' })), /cannot be/)
|
|
assert.throws(() => parseCellFlagsRequest(values({ 'expected-generation': '' })), /expected/)
|
|
assert.throws(() => parseCellFlagsRequest(values({ confirmation: WRITE_CONFIRMATION })), /confirmation/)
|
|
assert.throws(
|
|
() => parseCellFlagsRequest(values({ 'cell-origin': 'https://c27.relay.onorca.dev' })),
|
|
/another cell/
|
|
)
|
|
assert.throws(() => parseCellFlagsRequest(values({ 'cell-origin': 'https://evil.example' })), /origin/)
|
|
// A dry run needs no confirmation.
|
|
assert.equal(parseCellFlagsRequest(values({ mode: 'dry-run', confirmation: '' })).mode, 'dry-run')
|
|
})
|
|
|
|
test('a dry run reports the change and writes nothing', async () => {
|
|
const fake = fakeGoogleAndCell()
|
|
const { result, lines } = run(parseCellFlagsRequest(values({ mode: 'dry-run' })), fake)
|
|
const outcome = await result
|
|
assert.equal(outcome.written, false)
|
|
assert.equal(fake.state.writes.length, 0)
|
|
assert.equal(lines[0].event, 'orca_relay_cell_flags_plan')
|
|
assert.deepEqual(lines[0].desired.flags, { readinessLocal: true })
|
|
})
|
|
|
|
test('writes with compare-and-swap, records who wrote it, and waits for the cell to apply it', async () => {
|
|
const fake = fakeGoogleAndCell()
|
|
const { result, lines } = run(parseCellFlagsRequest(values()), fake)
|
|
const outcome = await result
|
|
assert.equal(outcome.written, true)
|
|
assert.equal(outcome.generation, '1001')
|
|
assert.deepEqual(outcome.applied, { generation: 1001, flags: { readinessLocal: true } })
|
|
assert.deepEqual(fake.state.writes[0].metadata.metadata, { writtenBy: 'octocat', runId: '42', runAttempt: '1' })
|
|
assert.deepEqual(fake.state.writes[0].object, {
|
|
v: 1,
|
|
cellId: CELL,
|
|
flags: { readinessLocal: true }
|
|
})
|
|
assert.deepEqual(lines.map((line) => line.event), [
|
|
'orca_relay_cell_flags_plan',
|
|
'orca_relay_cell_flags_written',
|
|
'orca_relay_cell_flags_applied_read_back'
|
|
])
|
|
// Flip back: the next write must name the generation just written.
|
|
const back = run(
|
|
parseCellFlagsRequest(values({ set: 'readinessLocal=false', 'expected-generation': '1001' })),
|
|
fake
|
|
)
|
|
assert.equal((await back.result).generation, '1002')
|
|
})
|
|
|
|
test('refuses a stale expected generation, and a concurrent writer between read and write', async () => {
|
|
const fake = fakeGoogleAndCell({
|
|
stored: { generation: '7', object: { v: 1, cellId: CELL, flags: {} } }
|
|
})
|
|
await assert.rejects(run(parseCellFlagsRequest(values()), fake).result, /current generation is 7/)
|
|
assert.equal(fake.state.writes.length, 0)
|
|
const racing = fakeGoogleAndCell()
|
|
const write = racing.fetchImpl
|
|
racing.fetchImpl = async (url, init) => {
|
|
if (new URL(url).pathname.startsWith('/upload/')) racing.state.stored = { generation: '9', object: {} }
|
|
return await write(url, init)
|
|
}
|
|
await assert.rejects(run(parseCellFlagsRequest(values()), racing).result, /changed since/)
|
|
})
|
|
|
|
test('a dry run against a stale generation still shows the plan, then refuses', async () => {
|
|
const fake = fakeGoogleAndCell({
|
|
stored: { generation: '7', object: { v: 1, cellId: CELL, flags: {} } }
|
|
})
|
|
const { result, lines } = run(parseCellFlagsRequest(values({ mode: 'dry-run' })), fake)
|
|
await assert.rejects(result, /current generation is 7/)
|
|
assert.equal(lines[0].event, 'orca_relay_cell_flags_plan')
|
|
assert.equal(lines[0].currentGeneration, '7')
|
|
})
|
|
|
|
test('rides out a transient 5xx on the first read and during the read-back', async () => {
|
|
const fake = fakeGoogleAndCell()
|
|
const inner = fake.fetchImpl
|
|
let runtimeCalls = 0
|
|
fake.fetchImpl = async (url, init) => {
|
|
if (new URL(url).pathname === '/v1/admin/runtime-status') {
|
|
runtimeCalls += 1
|
|
// The first call, and the first read-back poll after the write.
|
|
if (runtimeCalls === 1 || runtimeCalls === 3) return new Response(null, { status: 503 })
|
|
}
|
|
return await inner(url, init)
|
|
}
|
|
const outcome = await run(parseCellFlagsRequest(values()), fake).result
|
|
assert.equal(outcome.written, true)
|
|
assert.equal(outcome.generation, '1001')
|
|
})
|
|
|
|
test('refuses an origin that is not the named cell, or an image without the flag channel', async () => {
|
|
await assert.rejects(
|
|
run(parseCellFlagsRequest(values()), fakeGoogleAndCell({ cellId: 'production-gce-c25' })).result,
|
|
/not production-gce-c26/
|
|
)
|
|
const old = fakeGoogleAndCell()
|
|
const fetchImpl = old.fetchImpl
|
|
old.fetchImpl = async (url, init) =>
|
|
new URL(url).pathname === '/v1/admin/runtime-status'
|
|
? Response.json({ v: 1, role: 'cell', cellId: CELL })
|
|
: await fetchImpl(url, init)
|
|
await assert.rejects(run(parseCellFlagsRequest(values()), old).result, /no flag channel/)
|
|
})
|
|
|
|
test('fails loudly when the cell does not apply the write within the read-back window', async () => {
|
|
const fake = fakeGoogleAndCell({ applyAfterPolls: READ_BACK_TIMEOUT_MS })
|
|
await assert.rejects(run(parseCellFlagsRequest(values()), fake).result, /did not apply generation 1001/)
|
|
assert.equal(fake.state.writes.length, 1)
|
|
})
|
|
|
|
test('changes only the named switches and keeps every other key the object already has', () => {
|
|
const current = {
|
|
v: 1,
|
|
cellId: CELL,
|
|
flags: { readinessLocal: true, ticketCheck: 'shadow', readTimeoutMarginMs: 5_000 }
|
|
}
|
|
const request = parseCellFlagsRequest(
|
|
values({ set: 'admitMode=reserve, intakePerSec=40, readTimeoutMarginMs=default', ...RESERVE })
|
|
)
|
|
assert.deepEqual(desiredObject(request, current), {
|
|
v: 1,
|
|
cellId: CELL,
|
|
flags: { readinessLocal: true, ticketCheck: 'shadow', admitMode: 'reserve', intakePerSec: 40 }
|
|
})
|
|
assert.deepEqual(desiredObject(request, null).flags, { admitMode: 'reserve', intakePerSec: 40 })
|
|
assert.throws(() => desiredObject(request, { v: 1, cellId: 'production-gce-c27', flags: {} }), /by hand/)
|
|
assert.deepEqual(parseFlagChanges('rejectionFence=false'), { rejectionFence: false })
|
|
assert.deepEqual(parseFlagChanges('reregisterInFlight=8'), { reregisterInFlight: 8 })
|
|
assert.throws(() => parseFlagChanges('reregisterInFlight=32'), /cannot be/)
|
|
})
|
|
|
|
test('a write keeps the switches it did not name, end to end', async () => {
|
|
const fake = fakeGoogleAndCell({
|
|
stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { ticketCheck: 'shadow' } } }
|
|
})
|
|
const outcome = await run(parseCellFlagsRequest(values({ 'expected-generation': '7' })), fake).result
|
|
assert.equal(outcome.written, true)
|
|
assert.deepEqual(fake.state.writes[0].object.flags, { ticketCheck: 'shadow', readinessLocal: true })
|
|
})
|
|
|
|
test('the read-back compares every switch, not only the one this run changed', async () => {
|
|
const fake = fakeGoogleAndCell({
|
|
stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { ticketCheck: 'shadow' } } }
|
|
})
|
|
const inner = fake.fetchImpl
|
|
fake.fetchImpl = async (url, init) => {
|
|
const response = await inner(url, init)
|
|
if (new URL(url).pathname !== '/v1/admin/runtime-status') return response
|
|
const body = await response.json()
|
|
// The cell reports the new generation but still runs the old ticketCheck.
|
|
if (body.flagsApplied.generation === 8) body.flagsApplied.flags = { readinessLocal: true }
|
|
return Response.json(body)
|
|
}
|
|
await assert.rejects(
|
|
run(parseCellFlagsRequest(values({ 'expected-generation': '7' })), fake).result,
|
|
/did not apply generation 8/
|
|
)
|
|
})
|
|
|
|
test('a switch the cell reports on its own is compared only when the object names it', async () => {
|
|
const fake = fakeGoogleAndCell()
|
|
const inner = fake.fetchImpl
|
|
fake.fetchImpl = async (url, init) => {
|
|
const response = await inner(url, init)
|
|
if (new URL(url).pathname !== '/v1/admin/runtime-status') return response
|
|
const body = await response.json()
|
|
body.flagsApplied.flags = { ...body.flagsApplied.flags, intakePerSec: 50, readTimeoutMarginMs: 10_000 }
|
|
return Response.json(body)
|
|
}
|
|
assert.equal((await run(parseCellFlagsRequest(values()), fake).result).written, true)
|
|
})
|
|
|
|
test('fails the read-back when the cell ignores a switch the object names', async () => {
|
|
const fake = fakeGoogleAndCell()
|
|
const inner = fake.fetchImpl
|
|
fake.fetchImpl = async (url, init) => {
|
|
const response = await inner(url, init)
|
|
if (new URL(url).pathname !== '/v1/admin/runtime-status') return response
|
|
const body = await response.json()
|
|
if (body.flagsApplied.generation === 1001) body.flagsApplied.ignoredKeys = ['admitMode']
|
|
return Response.json(body)
|
|
}
|
|
await assert.rejects(
|
|
run(parseCellFlagsRequest(values({ set: 'readinessLocal=true,admitMode=reserve', ...RESERVE })), fake).result,
|
|
/ignores admitMode/
|
|
)
|
|
})
|
|
|
|
test('refuses a switch the cell image does not support, before writing anything', async () => {
|
|
const old = fakeGoogleAndCell({ supportedFlags: null })
|
|
await assert.rejects(
|
|
run(parseCellFlagsRequest(values({ set: 'ticketCheck=enforce' })), old).result,
|
|
/does not support ticketCheck="enforce"/
|
|
)
|
|
await assert.rejects(
|
|
run(parseCellFlagsRequest(values({ set: 'admitMode=reserve', ...RESERVE })), old).result,
|
|
/does not support admitMode/
|
|
)
|
|
assert.equal(old.state.writes.length, 0)
|
|
// An image from before supportedFlags still takes the two switches it always had.
|
|
assert.equal((await run(parseCellFlagsRequest(values()), old).result).written, true)
|
|
assert.throws(
|
|
() => assertSupportedObject({ readTimeoutMarginMs: 1_500.5 }, { supportedFlags: SUPPORTED }),
|
|
/readTimeoutMarginMs/
|
|
)
|
|
assertSupportedObject({ ticketCheck: 'enforce' }, { supportedFlags: SUPPORTED })
|
|
})
|
|
|
|
test('refuses to carry a bad value already in the file', async () => {
|
|
const bad = fakeGoogleAndCell({
|
|
supportedFlags: null,
|
|
stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { ticketCheck: 'enforce' } } }
|
|
})
|
|
// The old image voided generation 7 and still runs generation 0.
|
|
bad.state.applied = { generation: 0, flags: { readinessLocal: false, ticketCheck: 'off' } }
|
|
await assert.rejects(
|
|
run(parseCellFlagsRequest(values({ 'expected-generation': '7' })), bad).result,
|
|
/does not support ticketCheck="enforce"/
|
|
)
|
|
assert.equal(bad.state.writes.length, 0)
|
|
})
|
|
|
|
const DIRECTOR = { 'director-origin': RESERVE['director-origin'] }
|
|
|
|
test('admitMode=reserve needs a typed confirmation, and Postgres records reserve before the file', async () => {
|
|
assert.throws(
|
|
() => parseCellFlagsRequest(values({ ...DIRECTOR, set: 'admitMode=reserve' })),
|
|
/--confirm-reserve "RESERVE production-gce-c26"/
|
|
)
|
|
assert.throws(() => parseCellFlagsRequest(values({ set: 'admitMode=db' })), /--director-origin/)
|
|
const fake = fakeGoogleAndCell()
|
|
const request = parseCellFlagsRequest(
|
|
values({ ...DIRECTOR, set: 'readinessLocal=true,admitMode=reserve', 'confirm-reserve': `RESERVE ${CELL}` })
|
|
)
|
|
assert.equal((await run(request, fake).result).written, true)
|
|
// Recorded, re-checked after every director's cache has expired, then the file.
|
|
assert.deepEqual(fake.state.order, ['director:reserve', 'director:reserve', 'file:reserve'])
|
|
})
|
|
|
|
test('a refused file write after Postgres says reserve puts Postgres back to db', async () => {
|
|
const fake = fakeGoogleAndCell()
|
|
const inner = fake.fetchImpl
|
|
fake.fetchImpl = async (url, init) => {
|
|
if (new URL(url).pathname.startsWith('/upload/')) return new Response(null, { status: 403 })
|
|
return await inner(url, init)
|
|
}
|
|
const request = parseCellFlagsRequest(values({ set: 'readinessLocal=true,admitMode=reserve', ...RESERVE }))
|
|
await assert.rejects(run(request, fake).result, /returned 403/)
|
|
assert.deepEqual(fake.state.order, ['director:reserve', 'director:reserve', 'director:db'])
|
|
})
|
|
|
|
test('repairs an object the cell did not apply when the whole new object is valid', async () => {
|
|
const voided = fakeGoogleAndCell({
|
|
stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { readinessLocal: true } } }
|
|
})
|
|
voided.state.applied = { generation: 3, flags: { readinessLocal: false } }
|
|
const { result, lines } = run(parseCellFlagsRequest(values({ 'expected-generation': '7' })), voided)
|
|
assert.equal((await result).written, true)
|
|
assert.ok(lines.some((line) => line.event === 'orca_relay_cell_flags_repairing_unapplied_object'))
|
|
})
|
|
|
|
test('a flip back records db in Postgres only after the cell has leased every control', async () => {
|
|
const fake = fakeGoogleAndCell({
|
|
stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve' } } },
|
|
reregisteringPolls: 3
|
|
})
|
|
fake.state.directorAdmitMode = 'reserve'
|
|
const request = parseCellFlagsRequest(values({ ...DIRECTOR, set: 'admitMode=default', 'expected-generation': '7' }))
|
|
assert.equal((await run(request, fake).result).written, true)
|
|
assert.deepEqual(fake.state.order, ['file:db', 'director:db'])
|
|
assert.equal(fake.state.reregisteringPolls, -1)
|
|
})
|
|
|
|
test('the flip-back wait scales with the control count and the switch-file pace', () => {
|
|
assert.equal(reregisterWaitMs({ runtime: { controls: 100 } }), REREGISTER_WAIT_MIN_MS)
|
|
// 1,500 controls at the default 3 lanes: 1,667 s.
|
|
assert.equal(reregisterWaitMs({ runtime: { controls: 1_500 } }), 1_666_667)
|
|
assert.equal(
|
|
reregisterWaitMs({ runtime: { controls: 1_500 }, flagsApplied: { flags: { reregisterInFlight: 10 } } }),
|
|
// The cell caps 10 at its US pool of 10 less two.
|
|
625_000
|
|
)
|
|
assert.equal(reregisterWaitMs({ runtime: { controls: 3_000 } }), REREGISTER_WAIT_MAX_MS)
|
|
})
|
|
|
|
test('the wait counts the lanes the cell actually uses', () => {
|
|
assert.equal(reregisterLanes({ region: 'us-central1' }), 3)
|
|
assert.equal(reregisterLanes({ region: 'asia-east2' }), 5)
|
|
assert.equal(reregisterLanes({ region: 'us-central1', flagsApplied: { flags: { reregisterInFlight: 16 } } }), 8)
|
|
assert.equal(reregisterLanes({ region: 'asia-east2', flagsApplied: { flags: { reregisterInFlight: 16 } } }), 14)
|
|
// The pool the cell reports wins over the deployed-size table.
|
|
assert.equal(reregisterLanes({ region: 'us-central1', databasePoolMax: 22 }), 7)
|
|
assert.equal(reregisterLanes({ region: 'asia-east2', databasePoolMax: 12, flagsApplied: { flags: { reregisterInFlight: 16 } } }), 10)
|
|
assert.equal(reregisterLanes({ region: 'asia-east2', databasePoolMax: 'x' }), 5)
|
|
})
|
|
|
|
test('a tripped cell takes no write that keeps reserve, only one that also says db', async () => {
|
|
const stored = { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve' } } }
|
|
const tripped = () => fakeGoogleAndCell({ stored, effectiveOverride: 'db' })
|
|
for (const set of ['readinessLocal=true', 'reregisterInFlight=4']) {
|
|
const fake = tripped()
|
|
const request = parseCellFlagsRequest(values({ set, 'expected-generation': '7' }))
|
|
await assert.rejects(run(request, fake).result, /tripped its dead-man[\s\S]*admitMode=db,reregisterInFlight=N/)
|
|
assert.deepEqual(fake.state.writes, [])
|
|
}
|
|
const fake = tripped()
|
|
fake.state.directorAdmitMode = 'reserve'
|
|
const request = parseCellFlagsRequest(
|
|
values({ ...DIRECTOR, set: 'admitMode=db,reregisterInFlight=4', 'expected-generation': '7' })
|
|
)
|
|
assert.equal((await run(request, fake).result).written, true)
|
|
assert.deepEqual(fake.state.writes[0].object.flags, { admitMode: 'db', reregisterInFlight: 4 })
|
|
})
|
|
|
|
test('reserve requires readinessLocal=true, already written or in the same write', async () => {
|
|
const refused = fakeGoogleAndCell()
|
|
await assert.rejects(
|
|
run(parseCellFlagsRequest(values({ set: 'admitMode=reserve', ...RESERVE })), refused).result,
|
|
/admitMode=reserve requires readinessLocal=true/
|
|
)
|
|
assert.deepEqual(refused.state.order, [])
|
|
assert.deepEqual(refused.state.writes, [])
|
|
|
|
// Already written by an earlier run: reserve alone is enough.
|
|
const stored = { generation: '7', object: { v: 1, cellId: CELL, flags: { readinessLocal: true } } }
|
|
const ready = fakeGoogleAndCell({ stored })
|
|
const request = parseCellFlagsRequest(
|
|
values({ set: 'admitMode=reserve', 'expected-generation': '7', ...RESERVE })
|
|
)
|
|
assert.equal((await run(request, ready).result).written, true)
|
|
assert.deepEqual(ready.state.writes[0].object.flags, { readinessLocal: true, admitMode: 'reserve' })
|
|
})
|
|
|
|
test('a reserve cell refuses readinessLocal=false, but takes it together with the flip back', async () => {
|
|
const stored = {
|
|
generation: '7',
|
|
object: { v: 1, cellId: CELL, flags: { readinessLocal: true, admitMode: 'reserve' } }
|
|
}
|
|
for (const set of ['readinessLocal=false', 'readinessLocal=default']) {
|
|
const fake = fakeGoogleAndCell({ stored })
|
|
const request = parseCellFlagsRequest(values({ set, 'expected-generation': '7' }))
|
|
await assert.rejects(run(request, fake).result, /readinessLocal must stay true while the cell is reserve/)
|
|
assert.deepEqual(fake.state.writes, [])
|
|
}
|
|
const fake = fakeGoogleAndCell({ stored })
|
|
fake.state.directorAdmitMode = 'reserve'
|
|
const request = parseCellFlagsRequest(
|
|
values({ ...DIRECTOR, set: 'admitMode=db,readinessLocal=false', 'expected-generation': '7' })
|
|
)
|
|
assert.equal((await run(request, fake).result).written, true)
|
|
assert.deepEqual(fake.state.writes[0].object.flags, { readinessLocal: false, admitMode: 'db' })
|
|
})
|
|
|
|
test('a migration opened during the cache wait puts Postgres back to db', async () => {
|
|
const fake = fakeGoogleAndCell({ refuseReservePost: { post: 2, error: 'cell_has_open_migration' } })
|
|
const request = parseCellFlagsRequest(values({ ...RESERVE, set: 'readinessLocal=true,admitMode=reserve' }))
|
|
await assert.rejects(run(request, fake).result, /cell_has_open_migration/)
|
|
assert.deepEqual(fake.state.order, ['director:reserve', 'director:db'])
|
|
assert.deepEqual(fake.state.writes, [])
|
|
})
|
|
|
|
test('break glass refuses a cell that answers or that the director still hears', async () => {
|
|
const glass = { ...RESERVE, set: 'admitMode=default', 'expected-generation': '7', 'break-glass': `BREAK GLASS ${CELL}` }
|
|
const stored = { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve' } } }
|
|
const answering = fakeGoogleAndCell({ stored })
|
|
await assert.rejects(run(parseCellFlagsRequest(values(glass)), answering).result, /answers runtime-status/)
|
|
const heard = fakeGoogleAndCell({ stored, heartbeatFresh: true })
|
|
const inner = heard.fetchImpl
|
|
heard.fetchImpl = async (url, init) => {
|
|
if (new URL(url).pathname === '/v1/admin/runtime-status') throw new Error('connect refused')
|
|
return await inner(url, init)
|
|
}
|
|
await assert.rejects(run(parseCellFlagsRequest(values(glass)), heard).result, /still sees heartbeats/)
|
|
assert.deepEqual(heard.state.writes, [])
|
|
assert.deepEqual(answering.state.writes, [])
|
|
})
|
|
|
|
test('a flip back that outlasts its wait keeps Postgres on reserve and says how to finish', async () => {
|
|
const fake = fakeGoogleAndCell({
|
|
stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve' } } },
|
|
reregisteringPolls: Number.POSITIVE_INFINITY,
|
|
controls: 1_500
|
|
})
|
|
fake.state.directorAdmitMode = 'reserve'
|
|
const request = parseCellFlagsRequest(values({ ...DIRECTOR, set: 'admitMode=default', 'expected-generation': '7' }))
|
|
const { lines, result } = run(request, fake)
|
|
await assert.rejects(result, /after 1666667 ms[\s\S]*Run again with --set admitMode=db[\s\S]*--set admitMode=db,reregisterInFlight=N/)
|
|
assert.deepEqual(fake.state.order, ['file:db'])
|
|
assert.equal(lines.find((line) => line.event === 'orca_relay_cell_reregistration_wait')?.waitMs, 1_666_667)
|
|
})
|
|
|
|
test('break glass records db for a cell that cannot answer, typed for that cell only', async () => {
|
|
const glass = { ...RESERVE, set: 'admitMode=default', 'expected-generation': '7' }
|
|
assert.throws(() => parseCellFlagsRequest(values({ ...glass, 'break-glass': 'BREAK GLASS production-gce-c27' })), /break glass requires/)
|
|
assert.throws(
|
|
() => parseCellFlagsRequest(values({ ...glass, set: 'admitMode=default,readinessLocal=true', 'break-glass': `BREAK GLASS ${CELL}` })),
|
|
/on its own/
|
|
)
|
|
const fake = fakeGoogleAndCell({
|
|
stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve', readinessLocal: true } } }
|
|
})
|
|
fake.state.directorAdmitMode = 'reserve'
|
|
const inner = fake.fetchImpl
|
|
fake.fetchImpl = async (url, init) => {
|
|
if (new URL(url).pathname === '/v1/admin/runtime-status') throw new Error('connect refused')
|
|
return await inner(url, init)
|
|
}
|
|
const { result, lines } = run(parseCellFlagsRequest(values({ ...glass, 'break-glass': `BREAK GLASS ${CELL}` })), fake)
|
|
assert.equal((await result).written, true)
|
|
assert.deepEqual(fake.state.writes[0].object.flags, { readinessLocal: true })
|
|
assert.deepEqual(fake.state.order, ['file:db', 'director:db'])
|
|
assert.equal(lines.at(-1).event, 'orca_relay_cell_admit_mode_break_glass')
|
|
})
|
|
|