fix(relay-ops): pin the capacity identity so a stale same-cap template can roll (#21314)

c17's canary-apply failed closed at plan validation. Its instance template is from 2026-08-07 and predates the ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT line that every cell rolled since already carries, so the plan legitimately added it. The same-cap validator holds the whole startup script identical before and after except the image, and that line is not one it excluded, so the wave stopped with nothing applied. Pin the line for same-cap-cell exactly as bootstrap-cell already does, and exclude it from the before/after comparison. The cell may gain it; the pin is what refuses a roll that drops it or rewrites it to another identity. Both plan validations in the job now pass the capacity identity the job already requires. The same-cap contract is otherwise unchanged: any other stale line still fails closed, and needs a convergence apply before the cell can roll.
2026-09-22 00:02:31 +00:00 · 2026-09-17 20:42:34 -04:00
parent 1cd2964501
commit acedcf2a97
6 changed files with 245 additions and 12 deletions
@@ -67,11 +67,11 @@ test('same-cap wrapper is reusable, canary-bound, and sequential', () => {
  )
  assert.match(
    job,
-    /Require converged Terraform state and a stable MIG on resume[\s\S]{0,200}DIRECTOR_RUNTIME_SERVICE_ACCOUNT: \$\{\{ vars\.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT \}\}/
+    /Require converged Terraform state and a stable MIG on resume[\s\S]{0,300}CAPACITY_SERVICE_ACCOUNT: \$\{\{ vars\.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT \}\}\n {10}DIRECTOR_RUNTIME_SERVICE_ACCOUNT: \$\{\{ vars\.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT \}\}/
  )
  assert.match(
    job,
-    /--rollback-image "\$\{DESIRED_IMAGE\}" \\\n {16}--rehome-director-service-account "\$\{DIRECTOR_RUNTIME_SERVICE_ACCOUNT\}"/
+    /--rollback-image "\$\{DESIRED_IMAGE\}" \\\n {16}--capacity-service-account "\$\{CAPACITY_SERVICE_ACCOUNT\}" \\\n {16}--rehome-director-service-account "\$\{DIRECTOR_RUNTIME_SERVICE_ACCOUNT\}"/
  )
  assert.match(
    job,
@@ -20,6 +20,7 @@ const production = readFileSync(
 )
 const REHOME_SOURCE_CELLS = rehomeSourceCells()
 const DIRECTOR_IDENTITY = 'relay-director@onorca-cloud.iam.gserviceaccount.com'
+const CAPACITY_IDENTITY = 'orca-cloud-gha-cap@onorca-cloud.iam.gserviceaccount.com'
 const AUDIENCE = 'https://relay.onorca.dev/v1/admin/host-drain'
 const ROLLBACK_IMAGE = `us-central1-docker.pkg.dev/p/orca-cloud/relay@sha256:${'d'.repeat(64)}`
 const TARGET_IMAGE = `us-central1-docker.pkg.dev/p/orca-cloud/relay@sha256:${'e'.repeat(64)}`
@@ -53,10 +54,13 @@ function tfvarsCellBlock(cellId) {
  return production.slice(start, production.indexOf('\n  }', start))
 }

-function startupScript({ cap, image, trusted, pool }) {
+function startupScript({ cap, image, trusted, pool, capacityIdentity = CAPACITY_IDENTITY }) {
  return [
    `  printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`,
    `  printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
+    ...(capacityIdentity === null
+      ? []
+      : [`  printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`]),
    ...(pool === undefined
      ? []
      : [`  printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '${pool}'`]),
@@ -73,7 +77,11 @@ function startupScript({ cap, image, trusted, pool }) {
 }

 // The exact shape the apply step's plan has: template replaced, MIG rebound to it.
-function rollPlan({ cellId, cap, protocol, pool }) {
+function rollPlan({
+  cellId, cap, protocol, pool,
+  beforeCapacityIdentity = CAPACITY_IDENTITY,
+  afterCapacityIdentity = CAPACITY_IDENTITY
+}) {
  return {
    configuration: {
      root_module: {
@@ -104,7 +112,8 @@ function rollPlan({ cellId, cap, protocol, pool }) {
              image: ROLLBACK_IMAGE,
              trusted: protocol >= 1,
              // The live template predates the reviewed pool raise, as every asia cell's does.
-              pool: pool === undefined ? undefined : '10'
+              pool: pool === undefined ? undefined : '10',
+              capacityIdentity: beforeCapacityIdentity
            })
          },
          after: {
@@ -112,7 +121,8 @@ function rollPlan({ cellId, cap, protocol, pool }) {
              cap,
              image: TARGET_IMAGE,
              trusted: protocol >= 1,
-              pool
+              pool,
+              capacityIdentity: afterCapacityIdentity
            }),
            self_link: null
          },
@@ -298,6 +308,7 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
        unobservedBound: 60,
        image: TARGET_IMAGE,
        rollbackImage: ROLLBACK_IMAGE,
+        capacityServiceAccount: CAPACITY_IDENTITY,
        rehomeDirectorServiceAccount: DIRECTOR_IDENTITY,
        rehomeAudience: AUDIENCE,
        regionalRehomeProtocol: String(protocol),
@@ -340,6 +351,7 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
      unobservedBound: 60,
      image: TARGET_IMAGE,
      rollbackImage: ROLLBACK_IMAGE,
+      capacityServiceAccount: CAPACITY_IDENTITY,
      rehomeDirectorServiceAccount: DIRECTOR_IDENTITY,
      rehomeAudience: AUDIENCE,
      regionalRehomeProtocol: '0'
@@ -443,6 +455,67 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
    )
  })

+  it('rolls a template stale enough to predate the pinned capacity identity', () => {
+    // Exactly c17's shape on 2026-09-18: its live template is from 2026-08-07 and has no
+    // capacity identity line, so the roll adds one. Run 35290908836 failed closed here.
+    const cellId = 'production-gce-c17'
+    const config = {
+      mode: 'same-cap-cell',
+      cellId,
+      hardCap: 600,
+      unobservedBound: 60,
+      image: TARGET_IMAGE,
+      rollbackImage: ROLLBACK_IMAGE,
+      capacityServiceAccount: CAPACITY_IDENTITY,
+      rehomeDirectorServiceAccount: DIRECTOR_IDENTITY,
+      rehomeAudience: AUDIENCE,
+      regionalRehomeProtocol: '0'
+    }
+    const stale = rollPlan({ cellId, cap: 600, protocol: 0, beforeCapacityIdentity: null })
+    assert.deepEqual(validateCapacityPlan(stale, config), { mode: 'same-cap-cell', changes: 2 })
+    // The line may only be gained. A roll may not rewrite it,
+    assert.throws(
+      () => validateCapacityPlan(stale, {
+        ...config,
+        capacityServiceAccount: 'orca-cloud-gha-other@onorca-cloud.iam.gserviceaccount.com'
+      }),
+      /reviewed image and capacity/
+    )
+    // nor drop it from a template that already carries one.
+    assert.throws(
+      () => validateCapacityPlan(
+        rollPlan({ cellId, cap: 600, protocol: 0, afterCapacityIdentity: null }),
+        config
+      ),
+      /reviewed image and capacity/
+    )
+    // A same-cap roll cannot run without the identity pinned at all.
+    assert.throws(
+      () => validateCapacityPlan(stale, { ...config, capacityServiceAccount: undefined }),
+      /invalid service account/
+    )
+  })
+
+  it('pins the capacity identity on every plan validation the job runs', () => {
+    const invocations = workflow.split('validate-relay-capacity-plan.mjs').slice(1)
+    assert.equal(invocations.length, 2)
+    for (const invocation of invocations) {
+      const lines = invocation.split('\n')
+      const end = lines.findIndex((line) => !line.trimEnd().endsWith('\\'))
+      assert.match(
+        lines.slice(0, end + 1).join(' '),
+        /--capacity-service-account "\$\{CAPACITY_SERVICE_ACCOUNT\}"/
+      )
+    }
+    // Both steps must read it from the same repository variable the job already requires.
+    assert.equal(
+      workflow.split(
+        'CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}'
+      ).length,
+      4
+    )
+  })
+
  it('leaves the US-only capacity job on the default allowlist', () => {
    assert.doesNotMatch(capacityWorkflow, /--approved-cells/)
  })
@@ -47,7 +47,10 @@ export function parseCapacityPlanArguments(argv) {
    return value
  }
  if (!values.image) throw new Error('missing --image')
-  if (values.mode === 'bootstrap-cell' && !values['capacity-service-account']) {
+  if (
+    ['bootstrap-cell', 'same-cap-cell'].includes(values.mode) &&
+    !values['capacity-service-account']
+  ) {
    throw new Error('missing --capacity-service-account')
  }
  if (
@@ -239,7 +242,10 @@ function requireDesiredStartupScript(script, config) {
      `  printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '${config.unobservedBound}'`
    ]
  ]
-  if (config.mode === 'bootstrap-cell') {
+  // A same-cap cell whose template predates this line gains it on its next roll, so the
+  // before/after comparison ignores it; pinning the exact identity here is what reviews it,
+  // and what stops a roll dropping or rewriting the line it lets through.
+  if (['bootstrap-cell', 'same-cap-cell'].includes(config.mode)) {
    expected.push([
      /^  printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '[a-z][a-z0-9-]{4,28}[a-z0-9]@[a-z0-9-]+\.iam\.gserviceaccount\.com'$/,
      `  printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${config.capacityServiceAccount}'`
@@ -454,18 +460,22 @@ function cellPlan(plan, changes, config) {
  const sameCap = ['same-cap-cell', 'same-cap-image'].includes(config.mode)
  // Only a pinned pool may move here; requireDesiredStartupScript holds the after value exactly.
  const stripPool = config.mode === 'same-cap-cell' && config.databasePoolMax !== undefined
+  // Only a template stale enough to predate the line may move it, and only by gaining it;
+  // requireDesiredStartupScript holds the after value to the exact reviewed identity.
+  const stripCapacityIdentity =
+    ['bootstrap-cell', 'same-cap-cell'].includes(config.mode)
  if (
    typeof beforeScript !== 'string' ||
    (sameCap && relayImage(beforeScript) !== config.rollbackImage) ||
    normalizedStartupScript(
      beforeScript,
-      config.mode === 'bootstrap-cell',
+      stripCapacityIdentity,
      config.mode === 'same-cap-cell',
      sameCap,
      stripPool
    ) !== normalizedStartupScript(
      script,
-      config.mode === 'bootstrap-cell',
+      stripCapacityIdentity,
      config.mode === 'same-cap-cell',
      sameCap,
      stripPool
@@ -501,7 +511,7 @@ export function validateCapacityPlan(plan, config) {
    throw new Error('capacity Terraform plans may change only a cell')
  }
  if (
-    config.mode === 'bootstrap-cell' &&
+    ['bootstrap-cell', 'same-cap-cell'].includes(config.mode) &&
    !SERVICE_ACCOUNT_EMAIL.test(config.capacityServiceAccount ?? '')
  ) {
    throw new Error('capacity Terraform plan has an invalid service account')
@@ -428,10 +428,14 @@ test('same-cap mode preserves 1000/60 while adding only the reviewed trust confi
  const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
  const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
  const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
+  const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
  const audience = 'https://relay.example.com/v1/admin/host-drain'
-  const startup = ({ selectedImage, cap = 1_000, trust = false }) => [
+  const startup = ({ selectedImage, cap = 1_000, trust = false, capacity = capacityIdentity }) => [
    `  printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${cap}'`,
    `  printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
+    ...(capacity === null
+      ? []
+      : [`  printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacity}'`]),
    ...(trust ? [
      `  printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
      `  printf 'ORCA_RELAY_REHOME_AUDIENCE=%s\\n' '${audience}'`
@@ -468,6 +472,7 @@ test('same-cap mode preserves 1000/60 while adding only the reviewed trust confi
    mode: 'same-cap-cell',
    image,
    rollbackImage,
+    capacityServiceAccount: capacityIdentity,
    rehomeDirectorServiceAccount: directorIdentity,
    rehomeAudience: audience,
    regionalRehomeProtocol: '1'
@@ -653,10 +658,12 @@ test('protocol-0 same-cap cells roll without rehome trust lines', () => {
  const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
  const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
  const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
+  const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
  const audience = 'https://relay.example.com/v1/admin/host-drain'
  const startup = ({ selectedImage, trust = false }) => [
    `  printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`,
    `  printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
+    `  printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
    `  printf 'ORCA_RELAY_CELL_REGION=%s\\n' 'asia-east2'`,
    ...(trust ? [
      `  printf 'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT=%s\\n' '${directorIdentity}'`,
@@ -693,6 +700,7 @@ test('protocol-0 same-cap cells roll without rehome trust lines', () => {
    mode: 'same-cap-cell',
    image,
    rollbackImage,
+    capacityServiceAccount: capacityIdentity,
    rehomeDirectorServiceAccount: directorIdentity,
    rehomeAudience: audience,
    regionalRehomeProtocol: '0'
@@ -737,6 +745,133 @@ test('protocol-0 same-cap cells roll without rehome trust lines', () => {
  }
 })

+test('a same-cap roll may gain the pinned capacity identity but never move it', () => {
+  const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
+  const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
+  const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
+  const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
+  const audience = 'https://relay.example.com/v1/admin/host-drain'
+  const startup = ({ selectedImage, capacity }) => [
+    `  printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '600'`,
+    `  printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
+    ...(capacity === null
+      ? []
+      : [`  printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacity}'`]),
+    `printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${selectedImage.split('@')[1]}'`,
+    `docker pull '${selectedImage}'`,
+    'docker run --detach \\',
+    '  --name orca-relay \\',
+    `  '${selectedImage}'`
+  ].join('\n')
+  const plan = (beforeCapacity, afterCapacity) => ({
+    resource_changes: [
+      {
+        address: 'google_compute_instance_template.relay_gce_cell["production-gce-c17"]',
+        change: {
+          actions: ['create', 'delete'],
+          before: {
+            metadata_startup_script: startup({
+              selectedImage: rollbackImage,
+              capacity: beforeCapacity
+            })
+          },
+          after: {
+            metadata_startup_script: startup({ selectedImage: image, capacity: afterCapacity }),
+            self_link: null
+          },
+          after_unknown: { self_link: true }
+        }
+      },
+      {
+        address: 'google_compute_instance_group_manager.relay_gce_cell["production-gce-c17"]',
+        change: {
+          actions: ['update'],
+          before: { target_size: 1, version: [{ instance_template: 'old' }] },
+          after: { target_size: 1, version: [{ instance_template: null }] },
+          after_unknown: { version: [{ instance_template: true }] }
+        }
+      }
+    ]
+  })
+  const config = {
+    cellId: 'production-gce-c17',
+    hardCap: 600,
+    unobservedBound: 60,
+    mode: 'same-cap-cell',
+    image,
+    rollbackImage,
+    capacityServiceAccount: capacityIdentity,
+    rehomeDirectorServiceAccount: directorIdentity,
+    rehomeAudience: audience,
+    regionalRehomeProtocol: '0'
+  }
+  // A template old enough to predate the line gains it, which is the only move allowed.
+  assert.deepEqual(
+    validateCapacityPlan(plan(null, capacityIdentity), config),
+    { mode: 'same-cap-cell', changes: 2 }
+  )
+  assert.deepEqual(
+    validateCapacityPlan(plan(capacityIdentity, capacityIdentity), config),
+    { mode: 'same-cap-cell', changes: 2 }
+  )
+  for (const [before, after] of [
+    [capacityIdentity, null],
+    [null, null],
+    [capacityIdentity, 'orca-cloud-gha-other@project.iam.gserviceaccount.com'],
+    [null, 'orca-cloud-gha-other@project.iam.gserviceaccount.com']
+  ]) {
+    assert.throws(
+      () => validateCapacityPlan(plan(before, after), config),
+      /reviewed image and capacity/,
+      `${before} -> ${after}`
+    )
+  }
+  // Without the pin there is nothing reviewing the line the comparison now ignores.
+  assert.throws(
+    () => validateCapacityPlan(plan(null, capacityIdentity), {
+      ...config,
+      capacityServiceAccount: undefined
+    }),
+    /invalid service account/
+  )
+  assert.throws(
+    () => validateCapacityPlan(plan(null, capacityIdentity), {
+      ...config,
+      capacityServiceAccount: 'not-an-email'
+    }),
+    /invalid service account/
+  )
+})
+
+test('the capacity identity argument is required by same-cap-cell mode', () => {
+  const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
+  const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
+  const base = [
+    '--mode', 'same-cap-cell',
+    '--cell-id', 'production-gce-c17',
+    '--hard-cap', '600',
+    '--unobserved-bound', '60',
+    '--image', image,
+    '--rollback-image', rollbackImage,
+    '--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
+    '--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
+    '--regional-rehome-protocol', '0'
+  ]
+  assert.throws(() => parseCapacityPlanArguments(base), /missing --capacity-service-account/)
+  assert.throws(
+    () => parseCapacityPlanArguments([...base, '--capacity-service-account', 'nope']),
+    /--capacity-service-account is invalid/
+  )
+  assert.equal(
+    parseCapacityPlanArguments([
+      ...base,
+      '--capacity-service-account',
+      'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
+    ]).capacityServiceAccount,
+    'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
+  )
+})
+
 test('the rehome protocol argument is required by same-cap-cell mode alone', () => {
  const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
  const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
@@ -747,6 +882,7 @@ test('the rehome protocol argument is required by same-cap-cell mode alone', ()
    '--unobserved-bound', '60',
    '--image', image,
    '--rollback-image', rollbackImage,
+    '--capacity-service-account', 'orca-cloud-gha-cap@project.iam.gserviceaccount.com',
    '--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
    '--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
    ...extra
@@ -788,10 +924,12 @@ test('the reviewed database pool is pinned for the cells that emit one', () => {
  const rollbackImage = `us-docker.pkg.dev/project/relay/image@sha256:${'d'.repeat(64)}`
  const image = `us-docker.pkg.dev/project/relay/image@sha256:${'e'.repeat(64)}`
  const directorIdentity = 'relay-director@project.iam.gserviceaccount.com'
+  const capacityIdentity = 'orca-cloud-gha-cap@project.iam.gserviceaccount.com'
  const audience = 'https://relay.example.com/v1/admin/host-drain'
  const startup = ({ selectedImage, pool }) => [
    `  printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '3000'`,
    `  printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '60'`,
+    `  printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${capacityIdentity}'`,
    ...(pool === undefined
      ? []
      : [`  printf 'ORCA_RELAY_DATABASE_POOL_MAX=%s\\n' '${pool}'`]),
@@ -837,6 +975,7 @@ test('the reviewed database pool is pinned for the cells that emit one', () => {
    mode: 'same-cap-cell',
    image,
    rollbackImage,
+    capacityServiceAccount: capacityIdentity,
    rehomeDirectorServiceAccount: directorIdentity,
    rehomeAudience: audience,
    regionalRehomeProtocol: '1'
@@ -891,6 +1030,7 @@ test('the database pool argument is accepted by same-cap-cell mode alone', () =>
    '--unobserved-bound', '60',
    '--image', image,
    '--rollback-image', rollbackImage,
+    '--capacity-service-account', 'orca-cloud-gha-cap@project.iam.gserviceaccount.com',
    '--rehome-director-service-account', 'relay-director@project.iam.gserviceaccount.com',
    '--rehome-audience', 'https://relay.onorca.dev/v1/admin/host-drain',
    '--regional-rehome-protocol', '1',