fix(relay): accept MIG version-name reconciliation and recreate stranded cells without rewriting the MIG (#24373)

* fix(relay): accept MIG version-name reconciliation and recreate stranded cells without rewriting the MIG

The stranded-rollback recovery ran a gcloud rolling action, which renames the
MIG version outside Terraform. Every later plan for that cell then reverted the
label, and the capacity-plan validator refused the revert as an unreviewed MIG
change, so the cell could be neither rolled nor rolled back.

The validator now accepts a MIG field moving back to what relay-gce-cells.tf
declares (version name and update policy), in every mode, and a test pins those
values to the Terraform file. The stranded branch recreates the cell's single
instance with recreate-instances, which leaves the MIG untouched.

Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010

* fix(relay): let a label-only MIG plan through and recreate on it in a stranded rollback

A stranded rollback whose template is already in place plans only the version
name revert. The validator still required the MIG template to move, so that
plan was refused, and the recreate gate (changes == 0) would have skipped a
plan of one change and left the drain flag set. Require the template move only
when no declared field reconciles, and recreate whenever the template was not
replaced (changes < 2).

Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010

cloud/dev/scripts/relay-same-cap-script-census.test.mjs

@@ -188,34 +188,6 @@ function drainingBlock() {
   )}\necho "\${PRECHECK_ADMISSION} \${PRECHECK_DRAINING} \${PREDECESSOR_DRAINING_OK}"`
 }
 
-// The three fields gcloud would otherwise default, as the MIG resource declares them.
-function migUpdatePolicy() {
-  const terraform = readFileSync(
-    new URL('../../infra/terraform/relay-gce-cells.tf', import.meta.url),
-    'utf8'
-  )
-  const policy = terraform.split('  update_policy {')[1]?.split('\n  }')[0] ?? ''
-  const method = /replacement_method\s+= "([A-Z]+)"/.exec(policy)?.[1]
-  assert.notEqual(method, undefined, 'the MIG declares no replacement method')
-  // Both fixed bounds come from the topology locals the MIG resource points at.
-  const surgeLocal = /max_surge_fixed\s+= local\.relay_gce_topology\.(\w+)/.exec(policy)?.[1]
-  const unavailableLocal =
-    /max_unavailable_fixed\s+= local\.relay_gce_topology\.(\w+)/.exec(policy)?.[1]
-  assert.notEqual(surgeLocal, undefined, 'the MIG pins no surge local')
-  assert.notEqual(unavailableLocal, undefined, 'the MIG pins no unavailable local')
-  const topology = terraform.split('  relay_gce_topology = {')[1]?.split('\n  }')[0] ?? ''
-  const local = (name) => {
-    const value = new RegExp(`${name}\\s+= (\\d+)`).exec(topology)?.[1]
-    assert.notEqual(value, undefined, `the topology locals pin no ${name}`)
-    return value
-  }
-  return {
-    replacementMethod: method.toLowerCase(),
-    maxSurge: local(surgeLocal),
-    maxUnavailable: local(unavailableLocal)
-  }
-}
-
 // The stage decides the predecessor, the plan's reviewed rollback image, and whether the
 // MIG is rolled explicitly, so run the real block rather than restating its rule.
 function stageBlock() {
@@ -605,38 +577,51 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
     assert.equal(missing, 'resume')
   })
 
-  it('rolls the MIG itself when a stranded plan changes nothing', () => {
+  it('recreates the one stranded instance when a stranded plan changes nothing', () => {
     const apply = workflow
       .split('name: Apply only the selected same-cap template and MIG')[1]
       .split('\n      - id:')[0]
     // The plan is reviewed against the image the cell serves, not an assumed predecessor.
     assert.match(apply, /--rollback-image "\$\{PLAN_ROLLBACK_IMAGE\}"/)
     assert.doesNotMatch(apply, /--rollback-image "\$\{IMAGE_REPOSITORY\}/)
+    // A rolling action rewrites the MIG's version name outside Terraform, and the validator then
+    // refuses every later plan for the cell; recreating the instance leaves the MIG untouched.
+    assert.doesNotMatch(apply, /rolling-action/)
     assert.match(
       apply,
-      /test "\$\{ROLLBACK_STAGE\}" = stranded \\\n\s+&& test "\$\(jq -er '\.changes' <<< "\$\{PLAN_REVIEW\}"\)" = 0/
+      /list-instances \\\n\s+"\$\{MIG_NAME\}"[\s\S]*?if length == 1 then \.\[0\]\.instance/
     )
-    // gcloud persists all three fields into the MIG's update policy and defaults the
-    // method to substitute here, so every one has to match what Terraform declares or the
-    // recovery drifts the policy and the next targeted plan is refused as an unreviewed
-    // MIG change. Read the declared values rather than restating them.
-    assert.match(apply, /rolling-action replace "\$\{MIG_NAME\}"/)
-    const declared = migUpdatePolicy()
-    assert.deepEqual(declared, {
-      replacementMethod: 'recreate',
-      maxSurge: '0',
-      maxUnavailable: '1'
-    })
     assert.match(
       apply,
-      new RegExp(
-        `--replacement-method ${declared.replacementMethod}` +
-          ` --max-surge ${declared.maxSurge} --max-unavailable ${declared.maxUnavailable}`
-      )
+      /recreate-instances "\$\{MIG_NAME\}" \\\n\s+--instances "\$\{STRANDED_INSTANCE\}"/
     )
-    // Nothing else may reach the group, and the roll has to be waited on.
-    assert.equal(apply.split('rolling-action').length, 2)
+    assert.equal(apply.split('recreate-instances').length, 2)
+    // The recreate has to be waited on, after the apply's own wait.
+    const recreate = apply.indexOf('recreate-instances')
     assert.equal(apply.split('wait-until "${MIG_NAME}" --stable').length, 3)
+    assert.ok(apply.indexOf('wait-until "${MIG_NAME}" --stable', recreate) > recreate)
+  })
+
+  // The stranded branch's instance pick has to refuse anything but exactly one instance.
+  it('picks the stranded instance only from a one-instance MIG', () => {
+    const apply = workflow
+      .split('name: Apply only the selected same-cap template and MIG')[1]
+      .split('\n      - id:')[0]
+    const filter = /jq -er '(if length == 1[\s\S]*?end)'\)"/.exec(apply)?.[1]
+    assert.notEqual(filter, undefined, 'the stranded branch no longer asserts one instance')
+    const pick = (instances) =>
+      spawnSync('jq', ['-er', filter], { input: JSON.stringify(instances), encoding: 'utf8' })
+    const link = (name) =>
+      `https://www.googleapis.com/compute/v1/projects/p/zones/z/instances/${name}`
+    const one = pick([{ instance: link('relay-c29-abcd') }])
+    assert.equal(one.error, undefined)
+    assert.equal(one.status, 0, one.stderr)
+    assert.equal(one.stdout.trim(), 'relay-c29-abcd')
+    assert.notEqual(pick([]).status, 0)
+    assert.notEqual(
+      pick([{ instance: link('relay-c29-abcd') }, { instance: link('relay-c29-efgh') }]).status,
+      0
+    )
   })
 
   // Run the predicate the job ships rather than restating it, because restating it is how the
@@ -674,7 +659,7 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
       .split('name: Apply only the selected same-cap template and MIG')[1]
       .split('\n      - id:')[0]
     const condition =
-      /if test "\$\{ROLLBACK_STAGE\}" = stranded \\\n\s+(&& test "\$\(jq -er '\.changes' <<< "\$\{PLAN_REVIEW\}"\)" = 0); then/
+      /if test "\$\{ROLLBACK_STAGE\}" = stranded \\\n\s+(&& jq -e '\.changes < 2' <<< "\$\{PLAN_REVIEW\}" >\/dev\/null); then/
         .exec(apply)
     assert.notEqual(condition, null, 'the stranded roll no longer gates on the plan review')
     const rolls = (stage, review) => {
@@ -688,6 +673,8 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
       return resolved.stdout.trim()
     }
     assert.equal(rolls('stranded', { changes: 0 }), 'replace')
+    // A MIG-only reconciliation (a version label revert) leaves the template, so no restart.
+    assert.equal(rolls('stranded', { changes: 1 }), 'replace')
     // A real template replacement already restarts the instance; rolling again would be a second.
     assert.equal(rolls('stranded', { changes: 2 }), 'no-replace')
     assert.equal(rolls('resume', { changes: 0 }), 'no-replace')

cloud/dev/scripts/validate-relay-capacity-plan.mjs

@@ -160,19 +160,24 @@ function canonicalResourcePaths(change, paths) {
   )
 }
 
-function bootstrapRestartNormalizationPaths(change, mode) {
-  if (!['bootstrap-cell', 'same-cap-cell', 'same-cap-image'].includes(mode)) return []
-  const policyMatches =
-    valueAtPath(change.change.before, 'update_policy.0.minimal_action') === 'RESTART' &&
-    valueAtPath(change.change.after, 'update_policy.0.minimal_action') === 'REPLACE'
-  const priorVersion = valueAtPath(change.change.before, 'version.0.name')
-  const versionMatches =
-    typeof priorVersion === 'string' &&
-    /^0\/\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6}\+00:00$/.test(priorVersion) &&
-    valueAtPath(change.change.after, 'version.0.name') === 'primary'
-  return policyMatches && versionMatches
-    ? ['update_policy.0.minimal_action', 'version.0.name']
-    : []
+// What relay-gce-cells.tf declares for these MIG fields (a test pins the two together). A gcloud
+// rolling action, like the stranded recovery's old one, rewrites the version label and persists
+// its flags into the update policy outside Terraform; a later plan then reverts them. Moving back
+// to the declared value is reconciliation, never a capacity change, so it is the only move allowed.
+export const DECLARED_MANAGER_FIELDS = Object.freeze({
+  'version.0.name': 'primary',
+  'update_policy.0.type': 'PROACTIVE',
+  'update_policy.0.minimal_action': 'REPLACE',
+  'update_policy.0.most_disruptive_allowed_action': 'REPLACE',
+  'update_policy.0.replacement_method': 'RECREATE',
+  'update_policy.0.max_surge_fixed': 0,
+  'update_policy.0.max_unavailable_fixed': 1
+})
+
+function declaredManagerReconciliationPaths(change) {
+  return Object.entries(DECLARED_MANAGER_FIELDS)
+    .filter(([path, declared]) => valueAtPath(change.change.after, path) === declared)
+    .map(([path]) => path)
 }
 
 function requireOnlyPaths(change, allowed, required = [], allowedUnknown = new Set()) {
@@ -381,7 +386,7 @@ function requireDesiredPlannedCell(plan, config) {
   }
 }
 
-function validateManagerUpdate(manager, config) {
+function validateManagerUpdate(manager) {
   if (!sameActions(manager, ['update'])) {
     throw new Error('cell plan has unexpected MIG actions')
   }
@@ -392,14 +397,18 @@ function validateManagerUpdate(manager, config) {
     'version.0.instance_template'
   ])
   const managerUnknown = unknownPaths(manager.change.after_unknown)
+  const moved = changedPaths(manager.change.before, manager.change.after)
+  const reconciled = declaredManagerReconciliationPaths(manager).filter((path) =>
+    moved.includes(path))
+  // A plan that only reconciles declared fields leaves the template where it is.
   requireOnlyPaths(
     manager,
     new Set([
       'version.0.instance_template',
-      ...bootstrapRestartNormalizationPaths(manager, config.mode),
+      ...reconciled,
       ...managerUnknown.filter((path) => managerComputed.has(path))
     ]),
-    ['version.0.instance_template'],
+    reconciled.length > 0 ? [] : ['version.0.instance_template'],
     managerComputed
   )
 }
@@ -514,7 +523,7 @@ function cellPlan(plan, changes, config) {
     ['metadata_startup_script'],
     templateComputed
   )
-  validateManagerUpdate(manager, config)
+  validateManagerUpdate(manager)
   requireReplacementTemplateDependency(plan, template, manager)
   const beforeScript = template.change.before?.metadata_startup_script
   const script = template.change.after?.metadata_startup_script
@@ -561,7 +570,7 @@ function convergenceCellPlan(plan, changes, config) {
   ) {
     throw new Error('cell convergence plan changes outside the exact template and MIG')
   }
-  if (manager) validateManagerUpdate(manager, config)
+  if (manager) validateManagerUpdate(manager)
   if (obsoleteTemplates.some((change) => !sameActions(change, ['delete']))) {
     throw new Error('cell convergence plan has unexpected obsolete-template actions')
   }

cloud/dev/scripts/validate-relay-capacity-plan.test.mjs

@@ -4,7 +4,9 @@ import {
   RELAY_CELL_CONNECTION_DRAIN_SECONDS,
   RELAY_CELL_LOG_SAMPLE_RATE
 } from './validate-relay-asia-topology-plan.mjs'
+import { readFileSync } from 'node:fs'
 import {
+  DECLARED_MANAGER_FIELDS,
   parseCapacityPlanArguments,
   validateCapacityPlan as validateCapacityPlanRaw
 } from './validate-relay-capacity-plan.mjs'
@@ -240,6 +242,7 @@ test('accepts only the exact canary template replacement and MIG update', () =>
     ),
     /no exact planned C26 state/
   )
+  // A MIG moving back to its declared label and update policy is reconciliation in every mode.
   const restartedBootstrapManager = structuredClone(bootstrapManager)
   restartedBootstrapManager.change.before.update_policy = [{ minimal_action: 'RESTART' }]
   restartedBootstrapManager.change.after.update_policy = [{ minimal_action: 'REPLACE' }]
@@ -253,33 +256,20 @@ test('accepts only the exact canary template replacement and MIG update', () =>
     ),
     { mode: 'bootstrap-cell', changes: 2 }
   )
-  assert.throws(
-    () =>
-      validateCapacityPlan(
-        { resource_changes: [template, restartedBootstrapManager] },
-        cellConfig
-      ),
-    /outside the reviewed capacity fields/
+  assert.deepEqual(
+    validateCapacityPlan({ resource_changes: [template, restartedBootstrapManager] }, cellConfig),
+    { mode: 'cell', changes: 2 }
   )
-  const unrecognizedRestartManager = structuredClone(restartedBootstrapManager)
-  unrecognizedRestartManager.change.before.version[0].name = 'operator-version'
-  assert.throws(
-    () =>
-      validateCapacityPlan(
-        { resource_changes: [bootstrapTemplate, unrecognizedRestartManager] },
-        bootstrapConfig
-      ),
-    /outside the reviewed capacity fields/
-  )
-  const unrecognizedRestartPolicy = structuredClone(restartedBootstrapManager)
-  unrecognizedRestartPolicy.change.before.update_policy[0].minimal_action = 'REFRESH'
-  assert.throws(
-    () =>
-      validateCapacityPlan(
-        { resource_changes: [bootstrapTemplate, unrecognizedRestartPolicy] },
-        bootstrapConfig
-      ),
-    /outside the reviewed capacity fields/
+  const operatorLabelManager = structuredClone(restartedBootstrapManager)
+  operatorLabelManager.change.before.version[0].name = 'operator-version'
+  delete operatorLabelManager.change.before.update_policy
+  delete operatorLabelManager.change.after.update_policy
+  assert.deepEqual(
+    validateCapacityPlan(
+      { resource_changes: [bootstrapTemplate, operatorLabelManager] },
+      bootstrapConfig
+    ),
+    { mode: 'bootstrap-cell', changes: 2 }
   )
   const unrecognizedPrimaryVersion = structuredClone(restartedBootstrapManager)
   unrecognizedPrimaryVersion.change.after.version[0].name = 'other'
@@ -301,23 +291,13 @@ test('accepts only the exact canary template replacement and MIG update', () =>
       ),
     /outside the reviewed capacity fields/
   )
-  const missingRestartPolicy = structuredClone(restartedBootstrapManager)
-  delete missingRestartPolicy.change.before.update_policy
-  delete missingRestartPolicy.change.after.update_policy
+  // Gaining a whole policy block is not a field reverting, so it is not reconciliation.
+  const addedPolicy = structuredClone(restartedBootstrapManager)
+  delete addedPolicy.change.before.update_policy
   assert.throws(
     () =>
       validateCapacityPlan(
-        { resource_changes: [bootstrapTemplate, missingRestartPolicy] },
-        bootstrapConfig
-      ),
-    /outside the reviewed capacity fields/
-  )
-  const missingRestartVersion = structuredClone(restartedBootstrapManager)
-  missingRestartVersion.change.before.version[0].name = 'primary'
-  assert.throws(
-    () =>
-      validateCapacityPlan(
-        { resource_changes: [bootstrapTemplate, missingRestartVersion] },
+        { resource_changes: [bootstrapTemplate, addedPolicy] },
         bootstrapConfig
       ),
     /outside the reviewed capacity fields/
@@ -485,6 +465,127 @@ test('same-cap mode preserves 1000/60 while adding only the reviewed trust confi
     validateCapacityPlan({ resource_changes: [template, manager] }, sameCapConfig),
     { mode: 'same-cap-cell', changes: 2 }
   )
+  // A gcloud rolling action on a stranded cell renamed its MIG version; the next roll reverts it.
+  const relabelledManager = structuredClone(manager)
+  relabelledManager.change.before.version[0].name = '0/2026-10-01 10:41:28.681518+00:00'
+  relabelledManager.change.after.version[0].name = 'primary'
+  assert.deepEqual(
+    validateCapacityPlan({ resource_changes: [template, relabelledManager] }, sameCapConfig),
+    { mode: 'same-cap-cell', changes: 2 }
+  )
+  for (const [field, moved] of [
+    ['target_size', 0],
+    ['base_instance_name', 'relay-other'],
+    ['named_port', [{ name: 'relay', port: 9090 }]]
+  ]) {
+    const unrelated = structuredClone(relabelledManager)
+    unrelated.change.after[field] = moved
+    assert.throws(
+      () => validateCapacityPlan({ resource_changes: [template, unrelated] }, sameCapConfig),
+      /outside the reviewed capacity fields/,
+      field
+    )
+  }
+  const declaredPolicy = {
+    type: 'PROACTIVE',
+    minimal_action: 'REPLACE',
+    most_disruptive_allowed_action: 'REPLACE',
+    replacement_method: 'RECREATE',
+    max_surge_fixed: 0,
+    max_unavailable_fixed: 1
+  }
+  for (const [field, drifted, away] of [
+    ['type', 'OPPORTUNISTIC', 'OPPORTUNISTIC'],
+    ['minimal_action', 'RESTART', 'RESTART'],
+    ['most_disruptive_allowed_action', 'RESTART', 'RESTART'],
+    ['replacement_method', 'SUBSTITUTE', 'SUBSTITUTE'],
+    ['max_surge_fixed', 1, 1],
+    ['max_unavailable_fixed', 0, 0]
+  ]) {
+    const reverted = structuredClone(relabelledManager)
+    reverted.change.before.update_policy = [{ ...declaredPolicy, [field]: drifted }]
+    reverted.change.after.update_policy = [{ ...declaredPolicy }]
+    assert.deepEqual(
+      validateCapacityPlan({ resource_changes: [template, reverted] }, sameCapConfig),
+      { mode: 'same-cap-cell', changes: 2 },
+      field
+    )
+    const leaving = structuredClone(relabelledManager)
+    leaving.change.before.update_policy = [{ ...declaredPolicy }]
+    leaving.change.after.update_policy = [{ ...declaredPolicy, [field]: away }]
+    assert.throws(
+      () => validateCapacityPlan({ resource_changes: [template, leaving] }, sameCapConfig),
+      /outside the reviewed capacity fields/,
+      field
+    )
+  }
+  // A second version is a canary split, never a label revert.
+  const secondVersion = structuredClone(relabelledManager)
+  secondVersion.change.after.version.push({
+    name: 'primary',
+    instance_template: 'projects/project/global/instanceTemplates/canary',
+    target_size: [{ fixed: 1 }]
+  })
+  assert.throws(
+    () => validateCapacityPlan({ resource_changes: [template, secondVersion] }, sameCapConfig),
+    /outside the reviewed capacity fields/
+  )
+  // A stranded rollback whose template is already in place plans only the label revert.
+  const plannedCell = (startupScript, templateLink) => ({
+    root_module: {
+      resources: [
+        {
+          address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]',
+          values: { metadata_startup_script: startupScript, self_link: templateLink }
+        },
+        {
+          address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]',
+          values: { version: [{ instance_template: templateLink, name: 'primary' }] }
+        }
+      ]
+    }
+  })
+  const reviewedLink = 'projects/project/global/instanceTemplates/reviewed'
+  const labelOnly = {
+    address: manager.address,
+    change: {
+      actions: ['update'],
+      before: {
+        target_size: 1,
+        version: [{ instance_template: reviewedLink, name: '0/2026-10-01 10:41:28.681518+00:00' }]
+      },
+      after: { target_size: 1, version: [{ instance_template: reviewedLink, name: 'primary' }] },
+      after_unknown: {}
+    }
+  }
+  const strandedPlan = (managerChange) => ({
+    resource_changes: [managerChange],
+    planned_values: plannedCell(template.change.after.metadata_startup_script, reviewedLink)
+  })
+  assert.deepEqual(
+    validateCapacityPlan(strandedPlan(labelOnly), sameCapConfig),
+    { mode: 'same-cap-cell', changes: 1 }
+  )
+  const labelAway = structuredClone(labelOnly)
+  labelAway.change.before.version[0].name = 'primary'
+  labelAway.change.after.version[0].name = 'other'
+  assert.throws(
+    () => validateCapacityPlan(strandedPlan(labelAway), sameCapConfig),
+    /outside the reviewed capacity fields/
+  )
+  const labelAndResize = structuredClone(labelOnly)
+  labelAndResize.change.after.target_size = 0
+  assert.throws(
+    () => validateCapacityPlan(strandedPlan(labelAndResize), sameCapConfig),
+    /outside the reviewed capacity fields/
+  )
+  const percentSurge = structuredClone(relabelledManager)
+  percentSurge.change.before.update_policy = [{ ...declaredPolicy, max_surge_percent: 0 }]
+  percentSurge.change.after.update_policy = [{ ...declaredPolicy, max_surge_percent: 50 }]
+  assert.throws(
+    () => validateCapacityPlan({ resource_changes: [template, percentSurge] }, sameCapConfig),
+    /outside the reviewed capacity fields/
+  )
   // A pre-template-apply rollback resume validates drift for the image the
   // cell already serves: the template leaves and re-enters the rollback image.
   const resumeTemplate = structuredClone(template)
@@ -1300,3 +1401,30 @@ test('a same-cap roll may carry only this cell backend drain and request logging
     /only the exact instance template and MIG/
   )
 })
+
+// The reconciliation allowance is only safe while it names exactly what Terraform declares.
+test('the declared MIG fields match relay-gce-cells.tf', () => {
+  const terraform = readFileSync(
+    new URL('../../infra/terraform/relay-gce-cells.tf', import.meta.url),
+    'utf8'
+  )
+  const manager = terraform
+    .split('resource "google_compute_instance_group_manager" "relay_gce_cell" {')[1]
+    ?.split('\n}')[0] ?? ''
+  const block = (name) => manager.split(`  ${name} {`)[1]?.split('\n  }')[0] ?? ''
+  const topology = terraform.split('  relay_gce_topology = {')[1]?.split('\n  }')[0] ?? ''
+  const attribute = (source, name) => {
+    const raw = new RegExp(`\\n\\s+${name}\\s+= (.+)`).exec(source)?.[1]
+    assert.notEqual(raw, undefined, `relay-gce-cells.tf declares no ${name}`)
+    const local = /^local\.relay_gce_topology\.(\w+)$/.exec(raw)?.[1]
+    if (local) return Number(new RegExp(`${local}\\s+= (\\d+)`).exec(topology)?.[1])
+    return JSON.parse(raw)
+  }
+  const declared = Object.fromEntries(
+    Object.keys(DECLARED_MANAGER_FIELDS).map((path) => {
+      const [blockName, , field] = path.split('.')
+      return [path, attribute(block(blockName), field)]
+    })
+  )
+  assert.deepEqual(declared, { ...DECLARED_MANAGER_FIELDS })
+})