orca/cloud/dev/scripts/relay-asia-rollout-evidence.test.mjs

import assert from 'node:assert/strict'
import { test } from 'node:test'
import { RELAY_GITHUB_REPOSITORY, relayWorkflowPath } from './relay-repository.mjs'
import {
  buildProductionCanaryEvidence,
  buildStagingEvidence,
  verifyRolloutEvidence
} from './relay-asia-rollout-evidence.mjs'

const digest = `sha256:${'a'.repeat(64)}`
const commitSha = 'b'.repeat(40)
const repository = RELAY_GITHUB_REPOSITORY
const start = new Date('2026-08-13T12:00:00.000Z')
const end = new Date(start.valueOf() + 15 * 60_000)
const canaryEnd = new Date(start.valueOf() + 5 * 60_000)

function sourceInput(overrides = {}) {
  return {
    repository, runId: 123, runAttempt: 2, commitSha, imageDigest: digest,
    selectorGeneration: 9, ...overrides
  }
}

function metricLog(timestamp, role, cellId, overrides = {}) {
  return {
    timestamp: timestamp.toISOString(),
    resource: { labels: role === 'director' ? { instance_id: 'director-1' } : {} },
    jsonPayload: {
      role, cellId, region: role === 'cell' ? 'asia-east2' : 'us-central1',
      controls: role === 'cell' ? 2_840 : 0,
      splices: role === 'cell' ? 120 : 0,
      selectedRegionsDelta: role === 'director' ? { 'asia-east2': 1 } : {},
      regionFallbacksDelta: {}, unavailableRegionsDelta: {}, sqlFailuresDelta: 0,
      databasePoolWaiting: 0, databasePoolWaitersMax: 0, databasePoolWaitMsMax: 0,
      ...overrides
    }
  }
}

function completeLogs(cellId = 'production-gce-c27', windowEnd = end) {
  const samples = (windowEnd.valueOf() - start.valueOf()) / 60_000 + 1
  return Array.from({ length: samples }, (_, minute) => {
    const timestamp = new Date(start.valueOf() + minute * 60_000)
    return [
      metricLog(timestamp, 'director', 'production-director'),
      metricLog(timestamp, 'cell', cellId)
    ]
  }).flat()
}

function cloudSql(max = 319, windowEnd = end) {
  const samples = (windowEnd.valueOf() - start.valueOf()) / 60_000
  return {
    timeSeries: [{
      points: Array.from({ length: samples }, (_, minute) => ({
        interval: { endTime: new Date(start.valueOf() + (minute + 1) * 60_000).toISOString() },
        value: { int64Value: String(minute === samples - 1 ? max : 300) }
      }))
    }]
  }
}

function loadReport({
  controls, shardIndex, splices = 0, slow = 0, wedged = 0,
  launch = false
}) {
  return {
    event: 'relay_load_complete', controls, shardCount: 4, shardIndex,
    configuredRampSeconds: 180, configuredSteadySeconds: 210, requiredLeaseHorizons: 2,
    configuredSpliceHoldSeconds: 210,
    configuredSplices: splices, configuredSlowReaderSplices: slow,
    configuredWedgedReaderSplices: wedged,
    peakActive: controls, steadyMinimumActive: controls,
    connectionFailures: 0, rampConnectionFailures: 0, steadyConnectionFailures: 0,
    transitionConnectionFailures: 0, unexpectedCloses: 0, protocolErrors: 0,
    refreshErrors: 0, socketErrors: 0, failedSplices: 0,
    regionalFallbacksProved: 0,
    oldClientUsFirstProved: launch ? 1 : 0,
    stickyAssignmentProved: launch ? 1 : 0,
    requestUnitInvitesOpened: 0,
    requestUnitPrincipalCount: 0,
    relayAsiaLoadPrincipalCount: 32,
    requestUnitOverflowReason: null,
    requestUnitCleanupProved: 0,
    phaseBarrierPassed: true,
    rebindProbesOpened: launch ? 2 : 0,
    rebindOverflowReason: null,
    peakActiveSplices: splices, completedSplices: splices - wedged,
    slowReaderSplicesCompleted: slow, wedgedReaderSplicesClosed: wedged,
    readerQueuedBytesPeak: slow > 0 ? 1_024 : 0,
    generatorCpuPercent: 25, generatorEventLoopP99Ms: 20, generatorRssGrowthMiB: 10,
    readerQueueEvidence: slow > 0 ? [{
      origin: 'https://c4.relay-staging.onorca.dev',
      baselineBytes: 128,
      peakBytes: 1_152,
      increaseBytes: 1_024
    }] : [],
    shutdownEvidence: {
      peerShutdowns: controls, activeControls: 0, activeSplices: 0, reconnectTimers: 0
    }
  }
}

function stagingInput(overrides = {}) {
  const logs = completeLogs('staging-gce-c4')
  logs[0].jsonPayload.regionFallbacksDelta = { 'us-central1': 1 }
  return {
    ...sourceInput({ selectorGeneration: 4 }),
    startedAt: start.toISOString(), endedAt: end.toISOString(),
    launchReport: Array.from({ length: 4 }, (_, shardIndex) => loadReport({
      controls: 5, shardIndex, splices: 5, launch: shardIndex === 0,
      slow: shardIndex === 0 ? 4 : 0, wedged: shardIndex === 0 ? 1 : 0
    })),
    logs, cloudSql: cloudSql(), ...overrides
  }
}

function canaryInput(overrides = {}, cellId = 'production-gce-c27') {
  const load = loadReport({ controls: 1, shardIndex: 0, splices: 1 })
  Object.assign(load, {
    shardCount: 1,
    configuredSteadySeconds: 300,
    configuredSpliceHoldSeconds: 60,
    relayAsiaLoadPrincipalCount: 1,
    assignedCellOrigins: [`https://${cellId.split('-').at(-1)}.relay.onorca.dev`]
  })
  return {
    ...sourceInput(), cellId, startedAt: start.toISOString(), endedAt: canaryEnd.toISOString(),
    loadReport: load,
    logs: completeLogs(cellId, canaryEnd), cloudSql: cloudSql(319, canaryEnd),
    ...overrides
  }
}

function workflowRun(evidence, overrides = {}) {
  return {
    id: evidence.source.runId, run_attempt: evidence.source.runAttempt,
    conclusion: 'success', event: 'workflow_dispatch', head_branch: 'main',
    head_sha: evidence.source.commitSha,
    repository: { full_name: evidence.source.repository }, path: evidence.source.workflow,
    ...overrides
  }
}

function verifyExpected(kind, overrides = {}) {
  const staging = kind === 'staging-asia-readiness'
  return {
    kind, repository,
    workflow: staging
      ? relayWorkflowPath('prove-relay-asia-staging.yml')
      : relayWorkflowPath('operate-relay-asia-admission.yml'),
    environment: staging ? 'staging' : 'production', commitSha, imageDigest: digest,
    cellIds: [staging ? 'staging-gce-c4' : 'production-gce-c27'],
    now: new Date(end.valueOf() + 60_000).toISOString(), ...overrides
  }
}

test('accepts the exact sharded staging load, telemetry, and provenance proof', () => {
  const evidence = buildStagingEvidence(stagingInput())
  assert.equal(evidence.load.launch.controls, 20)
  assert.equal(evidence.load.launch.peakActiveSplices, 20)
  assert.equal(evidence.load.launch.readerQueueEvidence.length, 1)
  assert.equal(verifyRolloutEvidence(
    evidence, workflowRun(evidence), verifyExpected('staging-asia-readiness')
  ), evidence)
})

test('ignores unrelated legacy cell metrics outside the Asia proof', () => {
  const input = stagingInput()
  const legacy = metricLog(start, 'cell', 'staging-gce-c1', {
    region: undefined,
    sqlFailuresDelta: 1
  })
  delete legacy.jsonPayload.databasePoolWaiting
  delete legacy.jsonPayload.databasePoolWaitersMax
  delete legacy.jsonPayload.databasePoolWaitMsMax
  input.logs.push(legacy)

  assert.equal(buildStagingEvidence(input).metrics.relaySqlFailures, 0)
})

test('accepts bounded transient pool waits without a sampled queue', () => {
  const input = stagingInput()
  input.logs[0].jsonPayload.databasePoolWaitersMax = 4
  input.logs[0].jsonPayload.databasePoolWaitMsMax = 50

  const metrics = buildStagingEvidence(input).metrics
  assert.equal(metrics.databasePoolWaitingMax, 0)
  assert.equal(metrics.databasePoolWaitersMax, 4)
  assert.equal(metrics.databasePoolWaitMsMax, 50)
})

test('requires exactly the intentional sticky-assignment fallback in staging', () => {
  const missing = stagingInput()
  missing.logs[0].jsonPayload.regionFallbacksDelta = {}
  assert.throws(() => buildStagingEvidence(missing), /intentional probes/)

  const extra = stagingInput()
  extra.logs[2].jsonPayload.regionFallbacksDelta = { 'asia-east2': 1 }
  assert.throws(() => buildStagingEvidence(extra), /intentional probes/)

  const substituted = stagingInput()
  substituted.logs[0].jsonPayload.regionFallbacksDelta = { 'asia-east2': 1 }
  assert.throws(() => buildStagingEvidence(substituted), /intentional probes/)
})

test('accepts the wedged splice outside the sustained non-wedged peak', () => {
  const input = stagingInput()
  input.launchReport[0].peakActiveSplices = 4
  input.logs.filter((entry) => entry.jsonPayload.role === 'cell')
    .forEach((entry) => { entry.jsonPayload.splices = 19 })
  const evidence = buildStagingEvidence(input)
  assert.equal(evidence.load.launch.peakActiveSplices, 19)

  input.launchReport[0].peakActiveSplices = 3
  assert.throws(() => buildStagingEvidence(input), /mixed load evidence/)
})

test('rejects staging evidence with incomplete load or cleanup', () => {
  const input = stagingInput()
  input.launchReport[0].steadyMinimumActive--
  assert.throws(() => buildStagingEvidence(input), /required controls/)
  const cleanup = stagingInput()
  cleanup.launchReport[0].shutdownEvidence.activeControls = 1
  assert.throws(() => buildStagingEvidence(cleanup), /cleanup/)
  const nonCausal = stagingInput()
  nonCausal.launchReport[0].readerQueueEvidence[0].increaseBytes = 0
  assert.throws(() => buildStagingEvidence(nonCausal), /not causal/)
  const multipleOwners = stagingInput()
  multipleOwners.launchReport[0].configuredSlowReaderSplices--
  multipleOwners.launchReport[0].slowReaderSplicesCompleted--
  multipleOwners.launchReport[1] = loadReport({
    controls: 5, shardIndex: 1, splices: 5, slow: 1
  })
  assert.throws(() => buildStagingEvidence(multipleOwners), /one causal owner/)
  const overloaded = stagingInput()
  overloaded.launchReport[0].generatorCpuPercent = 80
  assert.throws(() => buildStagingEvidence(overloaded), /insufficient headroom/)
  const missingLaunchProof = stagingInput()
  missingLaunchProof.launchReport[0].rebindProbesOpened = 0
  assert.throws(() => buildStagingEvidence(missingLaunchProof), /launch-path/)
  const offTarget = stagingInput()
  offTarget.logs.filter((entry) => entry.jsonPayload.role === 'cell')
    .forEach((entry) => { entry.jsonPayload.controls = 19 })
  assert.throws(() => buildStagingEvidence(offTarget), /did not reach C4/)
})

test('rejects staging evidence with a shortened splice hold', () => {
  const input = stagingInput()
  input.launchReport[0].configuredSpliceHoldSeconds = 60
  assert.throws(() => buildStagingEvidence(input), /load profile does not match/)
})

for (const [label, value] of [['missing', undefined], ['malformed', 'invalid']]) {
  test(`rejects staging evidence with a ${label} splice hold`, () => {
    const input = stagingInput()
    input.launchReport[0].configuredSpliceHoldSeconds = value
    assert.throws(() => buildStagingEvidence(input), /staging splice hold seconds is invalid/)
  })
}

for (const [label, mutate, message] of [
  ['phase barrier', (input) => { input.launchReport[0].phaseBarrierPassed = false }, /profile/],
  ['old-client routing', (input) => { input.launchReport[0].oldClientUsFirstProved = 0 }, /launch-path/],
  ['sticky routing', (input) => { input.launchReport[0].stickyAssignmentProved = 0 }, /launch-path/]
]) {
  test(`rejects staging evidence without ${label} proof`, () => {
    const input = stagingInput()
    mutate(input)
    assert.throws(() => buildStagingEvidence(input), message)
  })
}

test('rejects staging evidence from a non-canonical repository', () => {
  assert.throws(() => buildStagingEvidence(stagingInput({ repository: 'fork/orca-cloud' })), /repository/)
})

test('rejects mismatched staging provenance, digest, topology, or age', () => {
  const evidence = buildStagingEvidence(stagingInput())
  const expected = verifyExpected('staging-asia-readiness')
  assert.throws(() => verifyRolloutEvidence(evidence, workflowRun(evidence, {
    head_sha: 'c'.repeat(40)
  }), expected), /provenance/)
  assert.throws(() => verifyRolloutEvidence(evidence, workflowRun(evidence), {
    ...expected, commitSha: 'c'.repeat(40)
  }), /provenance/)
  assert.throws(() => verifyRolloutEvidence(evidence, workflowRun(evidence), {
    ...expected, imageDigest: `sha256:${'c'.repeat(64)}`
  }), /image digest/)
  assert.throws(() => verifyRolloutEvidence({
    ...evidence, topology: { ...evidence.topology, cellIds: ['staging-gce-c3'] }
  }, workflowRun(evidence), expected), /topology/)
  assert.throws(() => verifyRolloutEvidence(evidence, workflowRun(evidence), {
    ...expected, now: new Date(end.valueOf() + 25 * 60 * 60_000).toISOString()
  }), /stale/)
})

test('builds and verifies a passing continuous 5-minute C27 canary', () => {
  const evidence = buildProductionCanaryEvidence(canaryInput())
  assert.equal(evidence.load.completedSplices, 1)
  assert.equal(evidence.metrics.asiaSelections, 6)
  assert.equal(evidence.metrics.cloudSqlBackendsMax, 319)
  assert.equal(verifyRolloutEvidence(
    evidence, workflowRun(evidence),
    verifyExpected('production-c27-canary', { selectorGeneration: 9 })
  ), evidence)
})

test('rejects short, sparse, or unrelated-cell-only C27 coverage', () => {
  assert.throws(() => buildProductionCanaryEvidence(canaryInput({
    endedAt: new Date(canaryEnd.valueOf() - 1).toISOString()
  })), /shorter than 5 minutes/)
  const sparse = completeLogs('production-gce-c27', canaryEnd).filter((entry) =>
    entry.timestamp === start.toISOString() || entry.timestamp === canaryEnd.toISOString()
  )
  assert.throws(() => buildProductionCanaryEvidence(canaryInput({ logs: sparse })), /sampling gap/)
  assert.throws(() => buildProductionCanaryEvidence(canaryInput({
    logs: completeLogs('production-gce-c26', canaryEnd)
  })), /production-gce-c27 metrics has no samples/)
})

test('rejects a C27 canary without a real control and splice', () => {
  const input = canaryInput()
  input.loadReport.completedSplices = 0
  assert.throws(() => buildProductionCanaryEvidence(input), /did not match/)
  const shortHold = canaryInput()
  shortHold.loadReport.configuredSpliceHoldSeconds = 59
  assert.throws(() => buildProductionCanaryEvidence(shortHold), /did not match/)
})

for (const [label, mutation, message] of [
  ['Asia selections', (input) => input.logs.forEach((entry) => { entry.jsonPayload.selectedRegionsDelta = {} }), /no Asia selections/],
  ['Asia-targeted region fallbacks', (input) => { input.logs[0].jsonPayload.regionFallbacksDelta = { 'asia-east2': 1 } }, /C27 canary asiaRegionFallbacks must be zero/],
  ['unavailable regions', (input) => { input.logs[0].jsonPayload.unavailableRegionsDelta = { 'asia-east2': 1 } }, /unavailableRegions/],
  ['C27 SQL failures', (input) => { input.logs[1].jsonPayload.sqlFailuresDelta = 1 }, /C27 canary relaySqlFailures must be zero/],
  ['C27 pool waiting', (input) => { input.logs[1].jsonPayload.databasePoolWaiting = 1 }, /databasePoolWaitingMax/],
  ['C27 pool waiters', (input) => { input.logs[1].jsonPayload.databasePoolWaitersMax = 5 }, /transient database pool pressure/],
  ['C27 pool wait time', (input) => { input.logs[1].jsonPayload.databasePoolWaitMsMax = 51 }, /transient database pool pressure/],
  ['C27 controls', (input) => input.logs.filter((entry) => entry.jsonPayload.role === 'cell')
    .forEach((entry) => { entry.jsonPayload.controls = 0 }), /did not reach C27/],
  ['C27 splices', (input) => input.logs.filter((entry) => entry.jsonPayload.role === 'cell')
    .forEach((entry) => { entry.jsonPayload.splices = 0 }), /did not reach C27/],
  ['Cloud SQL headroom', (input) => { input.cloudSql = cloudSql(320, canaryEnd) }, /below 320/]
]) {
  test(`rejects C27 evidence with ${label}`, () => {
    const input = canaryInput()
    mutation(input)
    assert.throws(() => buildProductionCanaryEvidence(input), message)
  })
}

test('rejects C27 evidence from a different selector generation', () => {
  const evidence = buildProductionCanaryEvidence(canaryInput())
  assert.throws(() => verifyRolloutEvidence(
    evidence, workflowRun(evidence),
    verifyExpected('production-c27-canary', { selectorGeneration: 10 })
  ), /selector generation/)
})

test('rejects a C27 canary whose control was placed on another cell', () => {
  for (const origins of [
    ['https://c28.relay.onorca.dev'],
    ['https://c27.relay.onorca.dev', 'https://c28.relay.onorca.dev'],
    [],
    undefined
  ]) {
    const input = canaryInput()
    input.loadReport.assignedCellOrigins = origins
    assert.throws(() => buildProductionCanaryEvidence(input), /C27 canary load was not placed only on C27/)
  }
})

const c30 = 'production-gce-c30'

test('builds and verifies a C30 canary from C30 runtime metrics and placement', () => {
  const evidence = buildProductionCanaryEvidence(canaryInput({}, c30))
  assert.equal(evidence.kind, 'production-c30-canary')
  assert.deepEqual(evidence.topology, { cellIds: [c30], selectorGeneration: 9 })
  assert.equal(verifyRolloutEvidence(
    evidence, workflowRun(evidence),
    verifyExpected('production-c30-canary', { cellIds: [c30], selectorGeneration: 9 })
  ), evidence)
  assert.throws(() => verifyRolloutEvidence(
    evidence, workflowRun(evidence),
    verifyExpected('production-c30-canary', { cellIds: [c30], selectorGeneration: 10 })
  ), /selector generation/)
  assert.throws(() => verifyRolloutEvidence(
    evidence, workflowRun(evidence), verifyExpected('production-c27-canary', { selectorGeneration: 9 })
  ), /evidence kind is invalid/)
})

test('builds a C31 canary that only C31 placement satisfies', () => {
  const c31 = 'production-gce-c31'
  const evidence = buildProductionCanaryEvidence(canaryInput({}, c31))
  assert.equal(evidence.kind, 'production-c31-canary')
  assert.equal(verifyRolloutEvidence(
    evidence, workflowRun(evidence),
    verifyExpected('production-c31-canary', { cellIds: [c31], selectorGeneration: 9 })
  ), evidence)
  const onC30 = canaryInput({}, c31)
  onC30.loadReport.assignedCellOrigins = ['https://c30.relay.onorca.dev']
  assert.throws(() => buildProductionCanaryEvidence(onC30), /C31 canary load was not placed only on C31/)
})

test('builds a C34 canary that only C34 placement satisfies', () => {
  const c34 = 'production-gce-c34'
  const evidence = buildProductionCanaryEvidence(canaryInput({}, c34))
  assert.equal(evidence.kind, 'production-c34-canary')
  assert.equal(verifyRolloutEvidence(
    evidence, workflowRun(evidence),
    verifyExpected('production-c34-canary', { cellIds: [c34], selectorGeneration: 9 })
  ), evidence)
  const onC31 = canaryInput({}, c34)
  onC31.loadReport.assignedCellOrigins = ['https://c31.relay.onorca.dev']
  assert.throws(() => buildProductionCanaryEvidence(onC31), /C34 canary load was not placed only on C34/)
})

test('records but does not gate organic US-targeted fallbacks during a canary', () => {
  // Mirrors C31's 2026-10-01 canary: US-targeted fallbacks, none targeting Asia.
  const input = canaryInput({}, 'production-gce-c31')
  input.logs.filter((entry) => entry.jsonPayload.role === 'director').forEach((entry) => {
    entry.jsonPayload.regionFallbacksDelta = { 'us-central1': 24 }
  })
  const evidence = buildProductionCanaryEvidence(input)
  assert.equal(evidence.metrics.regionFallbacks, 144)
  assert.equal(evidence.metrics.usRegionFallbacks, 144)
  assert.equal(evidence.metrics.asiaRegionFallbacks, 0)
  input.logs[0].jsonPayload.regionFallbacksDelta = { 'us-central1': 24, 'asia-east2': 1 }
  assert.throws(() => buildProductionCanaryEvidence(input), /C31 canary asiaRegionFallbacks must be zero/)
})

// A root-region cell logs region us-central1, and the directors place US hosts too.
function usCanaryInput(cellId) {
  const input = canaryInput({}, cellId)
  for (const entry of input.logs) {
    if (entry.jsonPayload.role === 'cell') entry.jsonPayload.region = 'us-central1'
    else entry.jsonPayload.selectedRegionsDelta = { 'asia-east2': 1, 'us-central1': 3 }
  }
  return input
}

test('builds and verifies C32 and C33 canaries from their own us-central1 metrics', () => {
  for (const cellId of ['production-gce-c32', 'production-gce-c33']) {
    const label = cellId.split('-').at(-1).toUpperCase()
    const kind = `production-${label.toLowerCase()}-canary`
    const evidence = buildProductionCanaryEvidence(usCanaryInput(cellId))
    assert.equal(evidence.kind, kind)
    assert.equal(evidence.metrics.usSelections, 18)
    assert.equal(verifyRolloutEvidence(
      evidence, workflowRun(evidence), verifyExpected(kind, { cellIds: [cellId], selectorGeneration: 9 })
    ), evidence)
    // The Asia-region filter would find none of a US cell's samples.
    assert.throws(
      () => buildProductionCanaryEvidence(canaryInput({}, cellId)),
      new RegExp(`${cellId} metrics`)
    )
    const elsewhere = usCanaryInput(cellId)
    elsewhere.loadReport.assignedCellOrigins = ['https://c26.relay.onorca.dev']
    assert.throws(
      () => buildProductionCanaryEvidence(elsewhere),
      new RegExp(`${label} canary load was not placed only on ${label}`)
    )
    const noUs = usCanaryInput(cellId)
    for (const entry of noUs.logs) {
      if (entry.jsonPayload.role === 'director') entry.jsonPayload.selectedRegionsDelta = { 'asia-east2': 1 }
    }
    assert.throws(() => buildProductionCanaryEvidence(noUs), /observed no US selections/)
  }
})

test('records but does not gate either region\'s fallbacks during a US canary', () => {
  const input = usCanaryInput('production-gce-c32')
  for (const entry of input.logs.filter(({ jsonPayload }) => jsonPayload.role === 'director')) {
    entry.jsonPayload.regionFallbacksDelta = { 'us-central1': 24, 'asia-east2': 1 }
  }
  const evidence = buildProductionCanaryEvidence(input)
  assert.equal(evidence.metrics.usRegionFallbacks, 144)
  assert.equal(evidence.metrics.asiaRegionFallbacks, 6)
  // Every other gate still binds a US canary.
  input.logs.find(({ jsonPayload }) => jsonPayload.role === 'cell').jsonPayload.sqlFailuresDelta = 1
  assert.throws(() => buildProductionCanaryEvidence(input), /relaySqlFailures must be zero/)
})

test('keeps the Asia canary evidence field set unchanged', () => {
  const evidence = buildProductionCanaryEvidence(canaryInput({}, 'production-gce-c31'))
  assert.equal('usSelections' in evidence.metrics, false)
})

test('rejects a C30 canary that C30 did not serve', () => {
  const onLaunchCell = canaryInput({}, c30)
  onLaunchCell.loadReport.assignedCellOrigins = ['https://c27.relay.onorca.dev']
  assert.throws(() => buildProductionCanaryEvidence(onLaunchCell), /C30 canary load was not placed only on C30/)
  assert.throws(() => buildProductionCanaryEvidence(canaryInput({
    logs: completeLogs('production-gce-c27', canaryEnd)
  }, c30)), /production-gce-c30 metrics has no samples/)
  const idle = canaryInput({}, c30)
  idle.logs.filter((entry) => entry.jsonPayload.role === 'cell')
    .forEach((entry) => { entry.jsonPayload.splices = 0 })
  assert.throws(() => buildProductionCanaryEvidence(idle), /C30 canary traffic did not reach C30/)
  const poolPressure = canaryInput({}, c30)
  poolPressure.logs.at(-1).jsonPayload.databasePoolWaiting = 1
  assert.throws(() => buildProductionCanaryEvidence(poolPressure), /C30 canary databasePoolWaitingMax/)
  assert.throws(() => buildProductionCanaryEvidence(canaryInput({
    endedAt: new Date(canaryEnd.valueOf() - 1).toISOString()
  }, c30)), /C30 canary window is shorter than 5 minutes/)
})

test('gates a C30 canary on C30 pool pressure and only reports the director baseline', () => {
  const directorPressure = canaryInput({}, c30)
  const directorLogs = directorPressure.logs.filter((entry) => entry.jsonPayload.role === 'director')
  Object.assign(directorLogs[0].jsonPayload, { databasePoolWaiting: 3 })
  Object.assign(directorLogs[1].jsonPayload, { databasePoolWaitersMax: 9, databasePoolWaitMsMax: 240 })
  const evidence = buildProductionCanaryEvidence(directorPressure)
  assert.deepEqual([
    evidence.metrics.databasePoolWaitingMax,
    evidence.metrics.databasePoolWaitersMax,
    evidence.metrics.databasePoolWaitMsMax
  ], [0, 0, 0])
  assert.deepEqual([
    evidence.metrics.directorDatabasePoolWaitingMax,
    evidence.metrics.directorDatabasePoolWaitersMax,
    evidence.metrics.directorDatabasePoolWaitMsMax
  ], [3, 9, 240])

  for (const [field, value, message] of [
    ['databasePoolWaiting', 1, /C30 canary databasePoolWaitingMax must be zero/],
    ['databasePoolWaitersMax', 5, /C30 canary transient database pool pressure/],
    ['databasePoolWaitMsMax', 51, /C30 canary transient database pool pressure/]
  ]) {
    const cellPressure = canaryInput({}, c30)
    cellPressure.logs.find((entry) => entry.jsonPayload.role === 'cell').jsonPayload[field] = value
    assert.throws(() => buildProductionCanaryEvidence(cellPressure), message)
  }
})

test('gates a C30 canary on C30 SQL failures and only reports the director baseline', () => {
  const directorNoise = canaryInput({}, c30)
  directorNoise.logs.filter((entry) => entry.jsonPayload.role === 'director')
    .slice(0, 3).forEach((entry) => { entry.jsonPayload.sqlFailuresDelta = 2 })
  const evidence = buildProductionCanaryEvidence(directorNoise)
  assert.equal(evidence.metrics.relaySqlFailures, 0)
  assert.equal(evidence.metrics.directorSqlFailures, 6)
  assert.equal(verifyRolloutEvidence(
    evidence, workflowRun(evidence),
    verifyExpected('production-c30-canary', { cellIds: [c30], selectorGeneration: 9 })
  ), evidence)

  const cellFailure = canaryInput({}, c30)
  cellFailure.logs.find((entry) => entry.jsonPayload.role === 'cell').jsonPayload.sqlFailuresDelta = 1
  assert.throws(() => buildProductionCanaryEvidence(cellFailure), /C30 canary relaySqlFailures must be zero/)
})

test('keeps staging gated on director database metrics without director-only fields', () => {
  const metrics = buildStagingEvidence(stagingInput()).metrics
  assert.equal(metrics.relaySqlFailures, 0)
  assert.equal(Object.keys(metrics).some((key) => key.startsWith('director')), false)
  for (const [field, value, message] of [
    ['databasePoolWaiting', 1, /staging proof databasePoolWaitingMax must be zero/],
    ['databasePoolWaitersMax', 5, /staging proof transient database pool pressure/],
    ['databasePoolWaitMsMax', 51, /staging proof transient database pool pressure/]
  ]) {
    const directorPressure = stagingInput()
    directorPressure.logs[0].jsonPayload[field] = value
    assert.throws(() => buildStagingEvidence(directorPressure), message)
  }

  const directorFailure = stagingInput()
  directorFailure.logs[0].jsonPayload.sqlFailuresDelta = 1
  assert.throws(() => buildStagingEvidence(directorFailure), /staging proof relaySqlFailures must be zero/)
})

test('accepts canaries only for the reviewed production Asia cells', () => {
  for (const cellId of ['production-gce-c28', 'production-gce-c29', 'staging-gce-c4', undefined]) {
    assert.throws(
      () => buildProductionCanaryEvidence({ ...canaryInput(), cellId }),
      /not a reviewed production Asia canary/
    )
  }
})