orca/cloud/dev/scripts/operate-relay-cell-flags.test.mjs

import assert from 'node:assert/strict'
import { test } from 'node:test'
import {
  assertSupportedObject,
  desiredObject,
  operateCellFlags,
  parseCellFlagsRequest,
  parseFlagChanges,
  READ_BACK_TIMEOUT_MS,
  REREGISTER_WAIT_MAX_MS,
  REREGISTER_WAIT_MIN_MS,
  reregisterLanes,
  reregisterWaitMs,
  WRITE_CONFIRMATION
} from './operate-relay-cell-flags.mjs'

const CELL = 'production-gce-c26'
const RESERVE = { 'director-origin': 'https://relay.onorca.dev', 'confirm-reserve': `RESERVE ${CELL}` }
const values = (overrides = {}) => ({
  mode: 'write',
  'cell-id': CELL,
  'cell-origin': 'https://c26.relay.onorca.dev',
  'project-id': 'onorca-cloud',
  set: 'readinessLocal=true',
  'expected-generation': '0',
  confirmation: `${WRITE_CONFIRMATION} ${CELL}`,
  ...overrides
})

// A cell whose runtime status applies whatever was written, after `applyAfterPolls` polls.
const SUPPORTED = {
  readinessLocal: { type: 'boolean' },
  ticketCheck: { type: 'enum', values: ['off', 'shadow', 'enforce'] },
  admitMode: { type: 'enum', values: ['db', 'reserve'] },
  intakePerSec: { type: 'number', min: 0, max: 1_000 },
  reserveDryRun: { type: 'boolean' },
  rejectionFence: { type: 'boolean' },
  readTimeoutMarginMs: { type: 'number', min: 1_000, max: 60_000, integer: true },
  reregisterInFlight: { type: 'number', min: 1, max: 16, integer: true },
  phonePath: { type: 'enum', values: ['db', 'memory'] }
}

function fakeGoogleAndCell({
  stored = null,
  applyAfterPolls = 2,
  role = 'cell',
  cellId = CELL,
  supportedFlags = SUPPORTED,
  // Polls that still report admitModeEffective=reserve after a flip back.
  reregisteringPolls = 0,
  controls,
  // Forces admitModeEffective, as a tripped dead-man reports it.
  effectiveOverride,
  heartbeatFresh = false,
  // The director refuses this numbered reserve post (1-based) with this error.
  refuseReservePost
} = {}) {
  const state = {
    stored,
    writes: [],
    polls: 0,
    // A cell that has applied whatever object is already there.
    applied: stored
      ? { generation: Number(stored.generation), flags: stored.object.flags }
      : { generation: 0, flags: { readinessLocal: false, ticketCheck: 'off' } },
    directorAdmitMode: 'db',
    reservePosts: 0,
    order: [],
    reregisteringPolls
  }
  const fetchImpl = async (url, init = {}) => {
    const target = new URL(url)
    if (target.pathname === '/v1/admin/cell-admit-mode') {
      const body = JSON.parse(String(init.body))
      if (body.admitMode === 'reserve') {
        state.reservePosts += 1
        if (refuseReservePost?.post === state.reservePosts) {
          return Response.json({ error: refuseReservePost.error }, { status: 409 })
        }
      }
      if (body.admitMode) {
        state.directorAdmitMode = body.admitMode
        state.order.push(`director:${body.admitMode}`)
      }
      return Response.json({ v: 1, cellId: body.cellId, admitMode: state.directorAdmitMode, updatedAt: 1, heartbeatFresh })
    }
    if (target.pathname === '/v1/admin/runtime-status') {
      state.polls += 1
      if (state.stored && state.polls > applyAfterPolls) {
        state.applied = { generation: Number(state.stored.generation), flags: state.stored.object.flags }
      }
      const reserveApplied = state.applied.flags.admitMode === 'reserve'
      const effective = reserveApplied || (state.writes.length > 0 && state.reregisteringPolls-- > 0) ? 'reserve' : 'db'
      return Response.json({
        v: 1,
        role,
        cellId,
        flagsApplied: state.applied,
        admitModeEffective: effectiveOverride ?? effective,
        ...(controls === undefined ? {} : { runtime: { controls } }),
        ...(supportedFlags ? { supportedFlags } : {})
      })
    }
    if (target.pathname.startsWith('/upload/')) {
      const expected = target.searchParams.get('ifGenerationMatch')
      if (expected !== (state.stored?.generation ?? '0')) return new Response(null, { status: 412 })
      const parts = String(init.body).split('\r\n')
      const object = JSON.parse(parts[parts.indexOf('Content-Type: application/json') + 2])
      const metadata = JSON.parse(parts[3])
      const generation = String(Number(state.stored?.generation ?? 1000) + 1)
      state.stored = { generation, object }
      state.writes.push({ generation, object, metadata })
      state.order.push(`file:${object.flags.admitMode ?? 'db'}`)
      return Response.json({ generation })
    }
    if (!state.stored) return new Response(null, { status: 404 })
    if (target.searchParams.get('alt') === 'media') return Response.json(state.stored.object)
    return Response.json({ generation: state.stored.generation })
  }
  return { state, fetchImpl }
}

const run = (request, fake, extra = {}) => {
  let clock = 0
  const lines = []
  return {
    lines,
    result: operateCellFlags(request, {
      fetchImpl: fake.fetchImpl,
      accessToken: 'access',
      idToken: 'id',
      audit: { writtenBy: 'octocat', runId: '42', runAttempt: '1' },
      log: (line) => lines.push(JSON.parse(line)),
      now: () => clock,
      sleep: async (ms) => {
        clock += ms
      },
      ...extra
    })
  }
}

test('refuses malformed requests before touching anything', () => {
  assert.throws(() => parseCellFlagsRequest(values({ mode: 'apply' })), /mode/)
  assert.throws(() => parseCellFlagsRequest(values({ 'cell-id': 'all' })), /cell id/)
  assert.throws(() => parseCellFlagsRequest(values({ set: 'ticketCheck=always' })), /ticketCheck/)
  assert.throws(() => parseCellFlagsRequest(values({ set: 'placerMode=on' })), /unknown switch/)
  assert.throws(() => parseCellFlagsRequest(values({ set: '' })), /names no switch/)
  assert.throws(() => parseCellFlagsRequest(values({ set: 'admitMode=db,admitMode=reserve' })), /twice/)
  assert.throws(() => parseCellFlagsRequest(values({ set: 'readTimeoutMarginMs=500' })), /cannot be/)
  assert.throws(() => parseCellFlagsRequest(values({ 'expected-generation': '' })), /expected/)
  assert.throws(() => parseCellFlagsRequest(values({ confirmation: WRITE_CONFIRMATION })), /confirmation/)
  assert.throws(
    () => parseCellFlagsRequest(values({ 'cell-origin': 'https://c27.relay.onorca.dev' })),
    /another cell/
  )
  assert.throws(() => parseCellFlagsRequest(values({ 'cell-origin': 'https://evil.example' })), /origin/)
  // A dry run needs no confirmation.
  assert.equal(parseCellFlagsRequest(values({ mode: 'dry-run', confirmation: '' })).mode, 'dry-run')
})

test('a dry run reports the change and writes nothing', async () => {
  const fake = fakeGoogleAndCell()
  const { result, lines } = run(parseCellFlagsRequest(values({ mode: 'dry-run' })), fake)
  const outcome = await result
  assert.equal(outcome.written, false)
  assert.equal(fake.state.writes.length, 0)
  assert.equal(lines[0].event, 'orca_relay_cell_flags_plan')
  assert.deepEqual(lines[0].desired.flags, { readinessLocal: true })
})

test('writes with compare-and-swap, records who wrote it, and waits for the cell to apply it', async () => {
  const fake = fakeGoogleAndCell()
  const { result, lines } = run(parseCellFlagsRequest(values()), fake)
  const outcome = await result
  assert.equal(outcome.written, true)
  assert.equal(outcome.generation, '1001')
  assert.deepEqual(outcome.applied, { generation: 1001, flags: { readinessLocal: true } })
  assert.deepEqual(fake.state.writes[0].metadata.metadata, { writtenBy: 'octocat', runId: '42', runAttempt: '1' })
  assert.deepEqual(fake.state.writes[0].object, {
    v: 1,
    cellId: CELL,
    flags: { readinessLocal: true }
  })
  assert.deepEqual(lines.map((line) => line.event), [
    'orca_relay_cell_flags_plan',
    'orca_relay_cell_flags_written',
    'orca_relay_cell_flags_applied_read_back'
  ])
  // Flip back: the next write must name the generation just written.
  const back = run(
    parseCellFlagsRequest(values({ set: 'readinessLocal=false', 'expected-generation': '1001' })),
    fake
  )
  assert.equal((await back.result).generation, '1002')
})

test('refuses a stale expected generation, and a concurrent writer between read and write', async () => {
  const fake = fakeGoogleAndCell({
    stored: { generation: '7', object: { v: 1, cellId: CELL, flags: {} } }
  })
  await assert.rejects(run(parseCellFlagsRequest(values()), fake).result, /current generation is 7/)
  assert.equal(fake.state.writes.length, 0)
  const racing = fakeGoogleAndCell()
  const write = racing.fetchImpl
  racing.fetchImpl = async (url, init) => {
    if (new URL(url).pathname.startsWith('/upload/')) racing.state.stored = { generation: '9', object: {} }
    return await write(url, init)
  }
  await assert.rejects(run(parseCellFlagsRequest(values()), racing).result, /changed since/)
})

test('a dry run against a stale generation still shows the plan, then refuses', async () => {
  const fake = fakeGoogleAndCell({
    stored: { generation: '7', object: { v: 1, cellId: CELL, flags: {} } }
  })
  const { result, lines } = run(parseCellFlagsRequest(values({ mode: 'dry-run' })), fake)
  await assert.rejects(result, /current generation is 7/)
  assert.equal(lines[0].event, 'orca_relay_cell_flags_plan')
  assert.equal(lines[0].currentGeneration, '7')
})

test('rides out a transient 5xx on the first read and during the read-back', async () => {
  const fake = fakeGoogleAndCell()
  const inner = fake.fetchImpl
  let runtimeCalls = 0
  fake.fetchImpl = async (url, init) => {
    if (new URL(url).pathname === '/v1/admin/runtime-status') {
      runtimeCalls += 1
      // The first call, and the first read-back poll after the write.
      if (runtimeCalls === 1 || runtimeCalls === 3) return new Response(null, { status: 503 })
    }
    return await inner(url, init)
  }
  const outcome = await run(parseCellFlagsRequest(values()), fake).result
  assert.equal(outcome.written, true)
  assert.equal(outcome.generation, '1001')
})

test('refuses an origin that is not the named cell, or an image without the flag channel', async () => {
  await assert.rejects(
    run(parseCellFlagsRequest(values()), fakeGoogleAndCell({ cellId: 'production-gce-c25' })).result,
    /not production-gce-c26/
  )
  const old = fakeGoogleAndCell()
  const fetchImpl = old.fetchImpl
  old.fetchImpl = async (url, init) =>
    new URL(url).pathname === '/v1/admin/runtime-status'
      ? Response.json({ v: 1, role: 'cell', cellId: CELL })
      : await fetchImpl(url, init)
  await assert.rejects(run(parseCellFlagsRequest(values()), old).result, /no flag channel/)
})

test('fails loudly when the cell does not apply the write within the read-back window', async () => {
  const fake = fakeGoogleAndCell({ applyAfterPolls: READ_BACK_TIMEOUT_MS })
  await assert.rejects(run(parseCellFlagsRequest(values()), fake).result, /did not apply generation 1001/)
  assert.equal(fake.state.writes.length, 1)
})

test('changes only the named switches and keeps every other key the object already has', () => {
  const current = {
    v: 1,
    cellId: CELL,
    flags: { readinessLocal: true, ticketCheck: 'shadow', readTimeoutMarginMs: 5_000 }
  }
  const request = parseCellFlagsRequest(
    values({ set: 'admitMode=reserve, intakePerSec=40, readTimeoutMarginMs=default', ...RESERVE })
  )
  assert.deepEqual(desiredObject(request, current), {
    v: 1,
    cellId: CELL,
    flags: { readinessLocal: true, ticketCheck: 'shadow', admitMode: 'reserve', intakePerSec: 40 }
  })
  assert.deepEqual(desiredObject(request, null).flags, { admitMode: 'reserve', intakePerSec: 40 })
  assert.throws(() => desiredObject(request, { v: 1, cellId: 'production-gce-c27', flags: {} }), /by hand/)
  assert.deepEqual(parseFlagChanges('rejectionFence=false'), { rejectionFence: false })
  assert.deepEqual(parseFlagChanges('reregisterInFlight=8'), { reregisterInFlight: 8 })
  assert.throws(() => parseFlagChanges('reregisterInFlight=32'), /cannot be/)
})

test('a write keeps the switches it did not name, end to end', async () => {
  const fake = fakeGoogleAndCell({
    stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { ticketCheck: 'shadow' } } }
  })
  const outcome = await run(parseCellFlagsRequest(values({ 'expected-generation': '7' })), fake).result
  assert.equal(outcome.written, true)
  assert.deepEqual(fake.state.writes[0].object.flags, { ticketCheck: 'shadow', readinessLocal: true })
})

test('the read-back compares every switch, not only the one this run changed', async () => {
  const fake = fakeGoogleAndCell({
    stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { ticketCheck: 'shadow' } } }
  })
  const inner = fake.fetchImpl
  fake.fetchImpl = async (url, init) => {
    const response = await inner(url, init)
    if (new URL(url).pathname !== '/v1/admin/runtime-status') return response
    const body = await response.json()
    // The cell reports the new generation but still runs the old ticketCheck.
    if (body.flagsApplied.generation === 8) body.flagsApplied.flags = { readinessLocal: true }
    return Response.json(body)
  }
  await assert.rejects(
    run(parseCellFlagsRequest(values({ 'expected-generation': '7' })), fake).result,
    /did not apply generation 8/
  )
})

test('a switch the cell reports on its own is compared only when the object names it', async () => {
  const fake = fakeGoogleAndCell()
  const inner = fake.fetchImpl
  fake.fetchImpl = async (url, init) => {
    const response = await inner(url, init)
    if (new URL(url).pathname !== '/v1/admin/runtime-status') return response
    const body = await response.json()
    body.flagsApplied.flags = { ...body.flagsApplied.flags, intakePerSec: 50, readTimeoutMarginMs: 10_000 }
    return Response.json(body)
  }
  assert.equal((await run(parseCellFlagsRequest(values()), fake).result).written, true)
})

test('fails the read-back when the cell ignores a switch the object names', async () => {
  const fake = fakeGoogleAndCell()
  const inner = fake.fetchImpl
  fake.fetchImpl = async (url, init) => {
    const response = await inner(url, init)
    if (new URL(url).pathname !== '/v1/admin/runtime-status') return response
    const body = await response.json()
    if (body.flagsApplied.generation === 1001) body.flagsApplied.ignoredKeys = ['admitMode']
    return Response.json(body)
  }
  await assert.rejects(
    run(parseCellFlagsRequest(values({ set: 'readinessLocal=true,admitMode=reserve', ...RESERVE })), fake).result,
    /ignores admitMode/
  )
})

test('refuses a switch the cell image does not support, before writing anything', async () => {
  const old = fakeGoogleAndCell({ supportedFlags: null })
  await assert.rejects(
    run(parseCellFlagsRequest(values({ set: 'ticketCheck=enforce' })), old).result,
    /does not support ticketCheck="enforce"/
  )
  await assert.rejects(
    run(parseCellFlagsRequest(values({ set: 'admitMode=reserve', ...RESERVE })), old).result,
    /does not support admitMode/
  )
  assert.equal(old.state.writes.length, 0)
  // An image from before supportedFlags still takes the two switches it always had.
  assert.equal((await run(parseCellFlagsRequest(values()), old).result).written, true)
  assert.throws(
    () => assertSupportedObject({ readTimeoutMarginMs: 1_500.5 }, { supportedFlags: SUPPORTED }),
    /readTimeoutMarginMs/
  )
  assertSupportedObject({ ticketCheck: 'enforce' }, { supportedFlags: SUPPORTED })
})

test('refuses to carry a bad value already in the file', async () => {
  const bad = fakeGoogleAndCell({
    supportedFlags: null,
    stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { ticketCheck: 'enforce' } } }
  })
  // The old image voided generation 7 and still runs generation 0.
  bad.state.applied = { generation: 0, flags: { readinessLocal: false, ticketCheck: 'off' } }
  await assert.rejects(
    run(parseCellFlagsRequest(values({ 'expected-generation': '7' })), bad).result,
    /does not support ticketCheck="enforce"/
  )
  assert.equal(bad.state.writes.length, 0)
})

const DIRECTOR = { 'director-origin': RESERVE['director-origin'] }

test('admitMode=reserve needs a typed confirmation, and Postgres records reserve before the file', async () => {
  assert.throws(
    () => parseCellFlagsRequest(values({ ...DIRECTOR, set: 'admitMode=reserve' })),
    /--confirm-reserve "RESERVE production-gce-c26"/
  )
  assert.throws(() => parseCellFlagsRequest(values({ set: 'admitMode=db' })), /--director-origin/)
  const fake = fakeGoogleAndCell()
  const request = parseCellFlagsRequest(
    values({ ...DIRECTOR, set: 'readinessLocal=true,admitMode=reserve', 'confirm-reserve': `RESERVE ${CELL}` })
  )
  assert.equal((await run(request, fake).result).written, true)
  // Recorded, re-checked after every director's cache has expired, then the file.
  assert.deepEqual(fake.state.order, ['director:reserve', 'director:reserve', 'file:reserve'])
})

test('a refused file write after Postgres says reserve puts Postgres back to db', async () => {
  const fake = fakeGoogleAndCell()
  const inner = fake.fetchImpl
  fake.fetchImpl = async (url, init) => {
    if (new URL(url).pathname.startsWith('/upload/')) return new Response(null, { status: 403 })
    return await inner(url, init)
  }
  const request = parseCellFlagsRequest(values({ set: 'readinessLocal=true,admitMode=reserve', ...RESERVE }))
  await assert.rejects(run(request, fake).result, /returned 403/)
  assert.deepEqual(fake.state.order, ['director:reserve', 'director:reserve', 'director:db'])
})

test('repairs an object the cell did not apply when the whole new object is valid', async () => {
  const voided = fakeGoogleAndCell({
    stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { readinessLocal: true } } }
  })
  voided.state.applied = { generation: 3, flags: { readinessLocal: false } }
  const { result, lines } = run(parseCellFlagsRequest(values({ 'expected-generation': '7' })), voided)
  assert.equal((await result).written, true)
  assert.ok(lines.some((line) => line.event === 'orca_relay_cell_flags_repairing_unapplied_object'))
})

test('a flip back records db in Postgres only after the cell has leased every control', async () => {
  const fake = fakeGoogleAndCell({
    stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve' } } },
    reregisteringPolls: 3
  })
  fake.state.directorAdmitMode = 'reserve'
  const request = parseCellFlagsRequest(values({ ...DIRECTOR, set: 'admitMode=default', 'expected-generation': '7' }))
  assert.equal((await run(request, fake).result).written, true)
  assert.deepEqual(fake.state.order, ['file:db', 'director:db'])
  assert.equal(fake.state.reregisteringPolls, -1)
})

test('the flip-back wait scales with the control count and the switch-file pace', () => {
  assert.equal(reregisterWaitMs({ runtime: { controls: 100 } }), REREGISTER_WAIT_MIN_MS)
  // 1,500 controls at the default 3 lanes: 1,667 s.
  assert.equal(reregisterWaitMs({ runtime: { controls: 1_500 } }), 1_666_667)
  assert.equal(
    reregisterWaitMs({ runtime: { controls: 1_500 }, flagsApplied: { flags: { reregisterInFlight: 10 } } }),
    // The cell caps 10 at its US pool of 10 less two.
    625_000
  )
  assert.equal(reregisterWaitMs({ runtime: { controls: 3_000 } }), REREGISTER_WAIT_MAX_MS)
})

test('the wait counts the lanes the cell actually uses', () => {
  assert.equal(reregisterLanes({ region: 'us-central1' }), 3)
  assert.equal(reregisterLanes({ region: 'asia-east2' }), 5)
  assert.equal(reregisterLanes({ region: 'us-central1', flagsApplied: { flags: { reregisterInFlight: 16 } } }), 8)
  assert.equal(reregisterLanes({ region: 'asia-east2', flagsApplied: { flags: { reregisterInFlight: 16 } } }), 14)
  // The pool the cell reports wins over the deployed-size table.
  assert.equal(reregisterLanes({ region: 'us-central1', databasePoolMax: 22 }), 7)
  assert.equal(reregisterLanes({ region: 'asia-east2', databasePoolMax: 12, flagsApplied: { flags: { reregisterInFlight: 16 } } }), 10)
  assert.equal(reregisterLanes({ region: 'asia-east2', databasePoolMax: 'x' }), 5)
})

test('a tripped cell takes no write that keeps reserve, only one that also says db', async () => {
  const stored = { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve' } } }
  const tripped = () => fakeGoogleAndCell({ stored, effectiveOverride: 'db' })
  for (const set of ['readinessLocal=true', 'reregisterInFlight=4']) {
    const fake = tripped()
    const request = parseCellFlagsRequest(values({ set, 'expected-generation': '7' }))
    await assert.rejects(run(request, fake).result, /tripped its dead-man[\s\S]*admitMode=db,reregisterInFlight=N/)
    assert.deepEqual(fake.state.writes, [])
  }
  const fake = tripped()
  fake.state.directorAdmitMode = 'reserve'
  const request = parseCellFlagsRequest(
    values({ ...DIRECTOR, set: 'admitMode=db,reregisterInFlight=4', 'expected-generation': '7' })
  )
  assert.equal((await run(request, fake).result).written, true)
  assert.deepEqual(fake.state.writes[0].object.flags, { admitMode: 'db', reregisterInFlight: 4 })
})

test('reserve requires readinessLocal=true, already written or in the same write', async () => {
  const refused = fakeGoogleAndCell()
  await assert.rejects(
    run(parseCellFlagsRequest(values({ set: 'admitMode=reserve', ...RESERVE })), refused).result,
    /admitMode=reserve requires readinessLocal=true/
  )
  assert.deepEqual(refused.state.order, [])
  assert.deepEqual(refused.state.writes, [])

  // Already written by an earlier run: reserve alone is enough.
  const stored = { generation: '7', object: { v: 1, cellId: CELL, flags: { readinessLocal: true } } }
  const ready = fakeGoogleAndCell({ stored })
  const request = parseCellFlagsRequest(
    values({ set: 'admitMode=reserve', 'expected-generation': '7', ...RESERVE })
  )
  assert.equal((await run(request, ready).result).written, true)
  assert.deepEqual(ready.state.writes[0].object.flags, { readinessLocal: true, admitMode: 'reserve' })
})

test('a reserve cell refuses readinessLocal=false, but takes it together with the flip back', async () => {
  const stored = {
    generation: '7',
    object: { v: 1, cellId: CELL, flags: { readinessLocal: true, admitMode: 'reserve' } }
  }
  for (const set of ['readinessLocal=false', 'readinessLocal=default']) {
    const fake = fakeGoogleAndCell({ stored })
    const request = parseCellFlagsRequest(values({ set, 'expected-generation': '7' }))
    await assert.rejects(run(request, fake).result, /readinessLocal must stay true while the cell is reserve/)
    assert.deepEqual(fake.state.writes, [])
  }
  const fake = fakeGoogleAndCell({ stored })
  fake.state.directorAdmitMode = 'reserve'
  const request = parseCellFlagsRequest(
    values({ ...DIRECTOR, set: 'admitMode=db,readinessLocal=false', 'expected-generation': '7' })
  )
  assert.equal((await run(request, fake).result).written, true)
  assert.deepEqual(fake.state.writes[0].object.flags, { readinessLocal: false, admitMode: 'db' })
})

test('phonePath=memory needs reserve, and a reserve flip never turns phones over unnamed', async () => {
  const onDatabase = { generation: '7', object: { v: 1, cellId: CELL, flags: { readinessLocal: true } } }
  const refused = fakeGoogleAndCell({ stored: onDatabase })
  await assert.rejects(
    run(parseCellFlagsRequest(values({ set: 'phonePath=memory', 'expected-generation': '7' })), refused).result,
    /phonePath=memory requires admitMode=reserve/
  )
  assert.deepEqual(refused.state.writes, [])

  const together = fakeGoogleAndCell({ stored: onDatabase })
  const both = parseCellFlagsRequest(
    values({ set: 'admitMode=reserve,phonePath=memory', 'expected-generation': '7', ...RESERVE })
  )
  assert.equal((await run(both, together).result).written, true)
  assert.deepEqual(together.state.writes[0].object.flags, {
    readinessLocal: true,
    admitMode: 'reserve',
    phonePath: 'memory'
  })

  const reserve = { readinessLocal: true, admitMode: 'reserve' }
  const onReserve = fakeGoogleAndCell({ stored: { ...onDatabase, object: { ...onDatabase.object, flags: reserve } } })
  const alone = parseCellFlagsRequest(values({ set: 'phonePath=memory', 'expected-generation': '7' }))
  assert.equal((await run(alone, onReserve).result).written, true)

  // Rolling back with admitMode=db leaves phonePath=memory in the object, which the cell ignores.
  const memory = { ...reserve, phonePath: 'memory' }
  const rollback = fakeGoogleAndCell({ stored: { ...onDatabase, object: { ...onDatabase.object, flags: memory } } })
  rollback.state.directorAdmitMode = 'reserve'
  const toDatabase = parseCellFlagsRequest(values({ ...DIRECTOR, set: 'admitMode=db', 'expected-generation': '7' }))
  assert.equal((await run(toDatabase, rollback).result).written, true)
  assert.deepEqual(rollback.state.writes[0].object.flags, { ...memory, admitMode: 'db' })

  // phonePath=db removes the key, so an image without the switch still takes later writes.
  const off = fakeGoogleAndCell({ stored: { ...onDatabase, object: { ...onDatabase.object, flags: memory } } })
  assert.equal((await run(parseCellFlagsRequest(values({ set: 'phonePath=db', 'expected-generation': '7' })), off).result).written, true)
  assert.deepEqual(off.state.writes[0].object.flags, reserve)

  const lingering = { readinessLocal: true, admitMode: 'db', phonePath: 'memory' }
  const unnamed = fakeGoogleAndCell({ stored: { ...onDatabase, object: { ...onDatabase.object, flags: lingering } } })
  await assert.rejects(
    run(parseCellFlagsRequest(values({ set: 'admitMode=reserve', 'expected-generation': '7', ...RESERVE })), unnamed)
      .result,
    /still says phonePath=memory/
  )
  assert.deepEqual(unnamed.state.writes, [])
})

test('a migration opened during the cache wait puts Postgres back to db', async () => {
  const fake = fakeGoogleAndCell({ refuseReservePost: { post: 2, error: 'cell_has_open_migration' } })
  const request = parseCellFlagsRequest(values({ ...RESERVE, set: 'readinessLocal=true,admitMode=reserve' }))
  await assert.rejects(run(request, fake).result, /cell_has_open_migration/)
  assert.deepEqual(fake.state.order, ['director:reserve', 'director:db'])
  assert.deepEqual(fake.state.writes, [])
})

test('break glass refuses a cell that answers or that the director still hears', async () => {
  const glass = { ...RESERVE, set: 'admitMode=default', 'expected-generation': '7', 'break-glass': `BREAK GLASS ${CELL}` }
  const stored = { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve' } } }
  const answering = fakeGoogleAndCell({ stored })
  await assert.rejects(run(parseCellFlagsRequest(values(glass)), answering).result, /answers runtime-status/)
  const heard = fakeGoogleAndCell({ stored, heartbeatFresh: true })
  const inner = heard.fetchImpl
  heard.fetchImpl = async (url, init) => {
    if (new URL(url).pathname === '/v1/admin/runtime-status') throw new Error('connect refused')
    return await inner(url, init)
  }
  await assert.rejects(run(parseCellFlagsRequest(values(glass)), heard).result, /still sees heartbeats/)
  assert.deepEqual(heard.state.writes, [])
  assert.deepEqual(answering.state.writes, [])
})

test('a flip back that outlasts its wait keeps Postgres on reserve and says how to finish', async () => {
  const fake = fakeGoogleAndCell({
    stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve' } } },
    reregisteringPolls: Number.POSITIVE_INFINITY,
    controls: 1_500
  })
  fake.state.directorAdmitMode = 'reserve'
  const request = parseCellFlagsRequest(values({ ...DIRECTOR, set: 'admitMode=default', 'expected-generation': '7' }))
  const { lines, result } = run(request, fake)
  await assert.rejects(result, /after 1666667 ms[\s\S]*Run again with --set admitMode=db[\s\S]*--set admitMode=db,reregisterInFlight=N/)
  assert.deepEqual(fake.state.order, ['file:db'])
  assert.equal(lines.find((line) => line.event === 'orca_relay_cell_reregistration_wait')?.waitMs, 1_666_667)
})

test('break glass records db (and drops phonePath) for a cell that cannot answer, typed for that cell only', async () => {
  const glass = { ...RESERVE, set: 'admitMode=default', 'expected-generation': '7' }
  assert.throws(() => parseCellFlagsRequest(values({ ...glass, 'break-glass': 'BREAK GLASS production-gce-c27' })), /break glass requires/)
  assert.throws(
    () => parseCellFlagsRequest(values({ ...glass, set: 'admitMode=default,readinessLocal=true', 'break-glass': `BREAK GLASS ${CELL}` })),
    /on its own/
  )
  const fake = fakeGoogleAndCell({
    stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve', readinessLocal: true, phonePath: 'memory' } } }
  })
  fake.state.directorAdmitMode = 'reserve'
  const inner = fake.fetchImpl
  fake.fetchImpl = async (url, init) => {
    if (new URL(url).pathname === '/v1/admin/runtime-status') throw new Error('connect refused')
    return await inner(url, init)
  }
  const { result, lines } = run(parseCellFlagsRequest(values({ ...glass, 'break-glass': `BREAK GLASS ${CELL}` })), fake)
  assert.equal((await result).written, true)
  assert.deepEqual(fake.state.writes[0].object.flags, { readinessLocal: true })
  assert.deepEqual(fake.state.order, ['file:db', 'director:db'])
  assert.equal(lines.at(-1).event, 'orca_relay_cell_admit_mode_break_glass')
})