diff --git a/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml b/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml index 7aafd33cab0..4ff59d35f81 100644 --- a/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml +++ b/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml @@ -709,15 +709,20 @@ jobs: --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900 # A stranded cell already runs the reviewed template, so the apply above replaces # no instance and the drain flag, which only a restart clears, would survive the - # whole wave. Roll the MIG explicitly on exactly the policy a template change uses. - # Every field is passed: gcloud persists these into the MIG's update policy, and it - # defaults the method to substitute on a group with no stateful config, so omitting - # one drifts the policy off the reviewed one and fails every later targeted plan. + # whole wave. Fewer than the template-and-MIG pair means the template stayed put, + # including a MIG-only reconciliation. Recreate its one instance from the MIG's + # current template; a rolling action would rewrite the MIG's version name outside + # Terraform, and every later targeted plan would carry that revert. if test "${ROLLBACK_STAGE}" = stranded \ - && test "$(jq -er '.changes' <<< "${PLAN_REVIEW}")" = 0; then - gcloud compute instance-groups managed rolling-action replace "${MIG_NAME}" \ - --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" \ - --replacement-method recreate --max-surge 0 --max-unavailable 1 + && jq -e '.changes < 2' <<< "${PLAN_REVIEW}" >/dev/null; then + STRANDED_INSTANCE="$(gcloud compute instance-groups managed list-instances \ + "${MIG_NAME}" --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" \ + --format=json \ + | jq -er 'if length == 1 then .[0].instance | split("/") | last + else error("stranded cell MIG does not have exactly one instance") end')" + gcloud compute instance-groups managed recreate-instances "${MIG_NAME}" \ + --instances "${STRANDED_INSTANCE}" \ + --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --quiet gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \ --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900 fi diff --git a/cloud/dev/scripts/relay-same-cap-script-census.test.mjs b/cloud/dev/scripts/relay-same-cap-script-census.test.mjs index 77a006cec80..24176a25bae 100644 --- a/cloud/dev/scripts/relay-same-cap-script-census.test.mjs +++ b/cloud/dev/scripts/relay-same-cap-script-census.test.mjs @@ -188,34 +188,6 @@ function drainingBlock() { )}\necho "\${PRECHECK_ADMISSION} \${PRECHECK_DRAINING} \${PREDECESSOR_DRAINING_OK}"` } -// The three fields gcloud would otherwise default, as the MIG resource declares them. -function migUpdatePolicy() { - const terraform = readFileSync( - new URL('../../infra/terraform/relay-gce-cells.tf', import.meta.url), - 'utf8' - ) - const policy = terraform.split(' update_policy {')[1]?.split('\n }')[0] ?? '' - const method = /replacement_method\s+= "([A-Z]+)"/.exec(policy)?.[1] - assert.notEqual(method, undefined, 'the MIG declares no replacement method') - // Both fixed bounds come from the topology locals the MIG resource points at. - const surgeLocal = /max_surge_fixed\s+= local\.relay_gce_topology\.(\w+)/.exec(policy)?.[1] - const unavailableLocal = - /max_unavailable_fixed\s+= local\.relay_gce_topology\.(\w+)/.exec(policy)?.[1] - assert.notEqual(surgeLocal, undefined, 'the MIG pins no surge local') - assert.notEqual(unavailableLocal, undefined, 'the MIG pins no unavailable local') - const topology = terraform.split(' relay_gce_topology = {')[1]?.split('\n }')[0] ?? '' - const local = (name) => { - const value = new RegExp(`${name}\\s+= (\\d+)`).exec(topology)?.[1] - assert.notEqual(value, undefined, `the topology locals pin no ${name}`) - return value - } - return { - replacementMethod: method.toLowerCase(), - maxSurge: local(surgeLocal), - maxUnavailable: local(unavailableLocal) - } -} - // The stage decides the predecessor, the plan's reviewed rollback image, and whether the // MIG is rolled explicitly, so run the real block rather than restating its rule. function stageBlock() { @@ -605,38 +577,51 @@ describe('same-cap roll scripts accept every same-cap cell', () => { assert.equal(missing, 'resume') }) - it('rolls the MIG itself when a stranded plan changes nothing', () => { + it('recreates the one stranded instance when a stranded plan changes nothing', () => { const apply = workflow .split('name: Apply only the selected same-cap template and MIG')[1] .split('\n - id:')[0] // The plan is reviewed against the image the cell serves, not an assumed predecessor. assert.match(apply, /--rollback-image "\$\{PLAN_ROLLBACK_IMAGE\}"/) assert.doesNotMatch(apply, /--rollback-image "\$\{IMAGE_REPOSITORY\}/) + // A rolling action rewrites the MIG's version name outside Terraform, and the validator then + // refuses every later plan for the cell; recreating the instance leaves the MIG untouched. + assert.doesNotMatch(apply, /rolling-action/) assert.match( apply, - /test "\$\{ROLLBACK_STAGE\}" = stranded \\\n\s+&& test "\$\(jq -er '\.changes' <<< "\$\{PLAN_REVIEW\}"\)" = 0/ + /list-instances \\\n\s+"\$\{MIG_NAME\}"[\s\S]*?if length == 1 then \.\[0\]\.instance/ ) - // gcloud persists all three fields into the MIG's update policy and defaults the - // method to substitute here, so every one has to match what Terraform declares or the - // recovery drifts the policy and the next targeted plan is refused as an unreviewed - // MIG change. Read the declared values rather than restating them. - assert.match(apply, /rolling-action replace "\$\{MIG_NAME\}"/) - const declared = migUpdatePolicy() - assert.deepEqual(declared, { - replacementMethod: 'recreate', - maxSurge: '0', - maxUnavailable: '1' - }) assert.match( apply, - new RegExp( - `--replacement-method ${declared.replacementMethod}` + - ` --max-surge ${declared.maxSurge} --max-unavailable ${declared.maxUnavailable}` - ) + /recreate-instances "\$\{MIG_NAME\}" \\\n\s+--instances "\$\{STRANDED_INSTANCE\}"/ ) - // Nothing else may reach the group, and the roll has to be waited on. - assert.equal(apply.split('rolling-action').length, 2) + assert.equal(apply.split('recreate-instances').length, 2) + // The recreate has to be waited on, after the apply's own wait. + const recreate = apply.indexOf('recreate-instances') assert.equal(apply.split('wait-until "${MIG_NAME}" --stable').length, 3) + assert.ok(apply.indexOf('wait-until "${MIG_NAME}" --stable', recreate) > recreate) + }) + + // The stranded branch's instance pick has to refuse anything but exactly one instance. + it('picks the stranded instance only from a one-instance MIG', () => { + const apply = workflow + .split('name: Apply only the selected same-cap template and MIG')[1] + .split('\n - id:')[0] + const filter = /jq -er '(if length == 1[\s\S]*?end)'\)"/.exec(apply)?.[1] + assert.notEqual(filter, undefined, 'the stranded branch no longer asserts one instance') + const pick = (instances) => + spawnSync('jq', ['-er', filter], { input: JSON.stringify(instances), encoding: 'utf8' }) + const link = (name) => + `https://www.googleapis.com/compute/v1/projects/p/zones/z/instances/${name}` + const one = pick([{ instance: link('relay-c29-abcd') }]) + assert.equal(one.error, undefined) + assert.equal(one.status, 0, one.stderr) + assert.equal(one.stdout.trim(), 'relay-c29-abcd') + assert.notEqual(pick([]).status, 0) + assert.notEqual( + pick([{ instance: link('relay-c29-abcd') }, { instance: link('relay-c29-efgh') }]).status, + 0 + ) }) // Run the predicate the job ships rather than restating it, because restating it is how the @@ -674,7 +659,7 @@ describe('same-cap roll scripts accept every same-cap cell', () => { .split('name: Apply only the selected same-cap template and MIG')[1] .split('\n - id:')[0] const condition = - /if test "\$\{ROLLBACK_STAGE\}" = stranded \\\n\s+(&& test "\$\(jq -er '\.changes' <<< "\$\{PLAN_REVIEW\}"\)" = 0); then/ + /if test "\$\{ROLLBACK_STAGE\}" = stranded \\\n\s+(&& jq -e '\.changes < 2' <<< "\$\{PLAN_REVIEW\}" >\/dev\/null); then/ .exec(apply) assert.notEqual(condition, null, 'the stranded roll no longer gates on the plan review') const rolls = (stage, review) => { @@ -688,6 +673,8 @@ describe('same-cap roll scripts accept every same-cap cell', () => { return resolved.stdout.trim() } assert.equal(rolls('stranded', { changes: 0 }), 'replace') + // A MIG-only reconciliation (a version label revert) leaves the template, so no restart. + assert.equal(rolls('stranded', { changes: 1 }), 'replace') // A real template replacement already restarts the instance; rolling again would be a second. assert.equal(rolls('stranded', { changes: 2 }), 'no-replace') assert.equal(rolls('resume', { changes: 0 }), 'no-replace') diff --git a/cloud/dev/scripts/validate-relay-capacity-plan.mjs b/cloud/dev/scripts/validate-relay-capacity-plan.mjs index 8ce27e6b27c..5da51abe95c 100644 --- a/cloud/dev/scripts/validate-relay-capacity-plan.mjs +++ b/cloud/dev/scripts/validate-relay-capacity-plan.mjs @@ -160,19 +160,24 @@ function canonicalResourcePaths(change, paths) { ) } -function bootstrapRestartNormalizationPaths(change, mode) { - if (!['bootstrap-cell', 'same-cap-cell', 'same-cap-image'].includes(mode)) return [] - const policyMatches = - valueAtPath(change.change.before, 'update_policy.0.minimal_action') === 'RESTART' && - valueAtPath(change.change.after, 'update_policy.0.minimal_action') === 'REPLACE' - const priorVersion = valueAtPath(change.change.before, 'version.0.name') - const versionMatches = - typeof priorVersion === 'string' && - /^0\/\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6}\+00:00$/.test(priorVersion) && - valueAtPath(change.change.after, 'version.0.name') === 'primary' - return policyMatches && versionMatches - ? ['update_policy.0.minimal_action', 'version.0.name'] - : [] +// What relay-gce-cells.tf declares for these MIG fields (a test pins the two together). A gcloud +// rolling action, like the stranded recovery's old one, rewrites the version label and persists +// its flags into the update policy outside Terraform; a later plan then reverts them. Moving back +// to the declared value is reconciliation, never a capacity change, so it is the only move allowed. +export const DECLARED_MANAGER_FIELDS = Object.freeze({ + 'version.0.name': 'primary', + 'update_policy.0.type': 'PROACTIVE', + 'update_policy.0.minimal_action': 'REPLACE', + 'update_policy.0.most_disruptive_allowed_action': 'REPLACE', + 'update_policy.0.replacement_method': 'RECREATE', + 'update_policy.0.max_surge_fixed': 0, + 'update_policy.0.max_unavailable_fixed': 1 +}) + +function declaredManagerReconciliationPaths(change) { + return Object.entries(DECLARED_MANAGER_FIELDS) + .filter(([path, declared]) => valueAtPath(change.change.after, path) === declared) + .map(([path]) => path) } function requireOnlyPaths(change, allowed, required = [], allowedUnknown = new Set()) { @@ -381,7 +386,7 @@ function requireDesiredPlannedCell(plan, config) { } } -function validateManagerUpdate(manager, config) { +function validateManagerUpdate(manager) { if (!sameActions(manager, ['update'])) { throw new Error('cell plan has unexpected MIG actions') } @@ -392,14 +397,18 @@ function validateManagerUpdate(manager, config) { 'version.0.instance_template' ]) const managerUnknown = unknownPaths(manager.change.after_unknown) + const moved = changedPaths(manager.change.before, manager.change.after) + const reconciled = declaredManagerReconciliationPaths(manager).filter((path) => + moved.includes(path)) + // A plan that only reconciles declared fields leaves the template where it is. requireOnlyPaths( manager, new Set([ 'version.0.instance_template', - ...bootstrapRestartNormalizationPaths(manager, config.mode), + ...reconciled, ...managerUnknown.filter((path) => managerComputed.has(path)) ]), - ['version.0.instance_template'], + reconciled.length > 0 ? [] : ['version.0.instance_template'], managerComputed ) } @@ -514,7 +523,7 @@ function cellPlan(plan, changes, config) { ['metadata_startup_script'], templateComputed ) - validateManagerUpdate(manager, config) + validateManagerUpdate(manager) requireReplacementTemplateDependency(plan, template, manager) const beforeScript = template.change.before?.metadata_startup_script const script = template.change.after?.metadata_startup_script @@ -561,7 +570,7 @@ function convergenceCellPlan(plan, changes, config) { ) { throw new Error('cell convergence plan changes outside the exact template and MIG') } - if (manager) validateManagerUpdate(manager, config) + if (manager) validateManagerUpdate(manager) if (obsoleteTemplates.some((change) => !sameActions(change, ['delete']))) { throw new Error('cell convergence plan has unexpected obsolete-template actions') } diff --git a/cloud/dev/scripts/validate-relay-capacity-plan.test.mjs b/cloud/dev/scripts/validate-relay-capacity-plan.test.mjs index e323a0e110e..bbd6220423e 100644 --- a/cloud/dev/scripts/validate-relay-capacity-plan.test.mjs +++ b/cloud/dev/scripts/validate-relay-capacity-plan.test.mjs @@ -4,7 +4,9 @@ import { RELAY_CELL_CONNECTION_DRAIN_SECONDS, RELAY_CELL_LOG_SAMPLE_RATE } from './validate-relay-asia-topology-plan.mjs' +import { readFileSync } from 'node:fs' import { + DECLARED_MANAGER_FIELDS, parseCapacityPlanArguments, validateCapacityPlan as validateCapacityPlanRaw } from './validate-relay-capacity-plan.mjs' @@ -240,6 +242,7 @@ test('accepts only the exact canary template replacement and MIG update', () => ), /no exact planned C26 state/ ) + // A MIG moving back to its declared label and update policy is reconciliation in every mode. const restartedBootstrapManager = structuredClone(bootstrapManager) restartedBootstrapManager.change.before.update_policy = [{ minimal_action: 'RESTART' }] restartedBootstrapManager.change.after.update_policy = [{ minimal_action: 'REPLACE' }] @@ -253,33 +256,20 @@ test('accepts only the exact canary template replacement and MIG update', () => ), { mode: 'bootstrap-cell', changes: 2 } ) - assert.throws( - () => - validateCapacityPlan( - { resource_changes: [template, restartedBootstrapManager] }, - cellConfig - ), - /outside the reviewed capacity fields/ + assert.deepEqual( + validateCapacityPlan({ resource_changes: [template, restartedBootstrapManager] }, cellConfig), + { mode: 'cell', changes: 2 } ) - const unrecognizedRestartManager = structuredClone(restartedBootstrapManager) - unrecognizedRestartManager.change.before.version[0].name = 'operator-version' - assert.throws( - () => - validateCapacityPlan( - { resource_changes: [bootstrapTemplate, unrecognizedRestartManager] }, - bootstrapConfig - ), - /outside the reviewed capacity fields/ - ) - const unrecognizedRestartPolicy = structuredClone(restartedBootstrapManager) - unrecognizedRestartPolicy.change.before.update_policy[0].minimal_action = 'REFRESH' - assert.throws( - () => - validateCapacityPlan( - { resource_changes: [bootstrapTemplate, unrecognizedRestartPolicy] }, - bootstrapConfig - ), - /outside the reviewed capacity fields/ + const operatorLabelManager = structuredClone(restartedBootstrapManager) + operatorLabelManager.change.before.version[0].name = 'operator-version' + delete operatorLabelManager.change.before.update_policy + delete operatorLabelManager.change.after.update_policy + assert.deepEqual( + validateCapacityPlan( + { resource_changes: [bootstrapTemplate, operatorLabelManager] }, + bootstrapConfig + ), + { mode: 'bootstrap-cell', changes: 2 } ) const unrecognizedPrimaryVersion = structuredClone(restartedBootstrapManager) unrecognizedPrimaryVersion.change.after.version[0].name = 'other' @@ -301,23 +291,13 @@ test('accepts only the exact canary template replacement and MIG update', () => ), /outside the reviewed capacity fields/ ) - const missingRestartPolicy = structuredClone(restartedBootstrapManager) - delete missingRestartPolicy.change.before.update_policy - delete missingRestartPolicy.change.after.update_policy + // Gaining a whole policy block is not a field reverting, so it is not reconciliation. + const addedPolicy = structuredClone(restartedBootstrapManager) + delete addedPolicy.change.before.update_policy assert.throws( () => validateCapacityPlan( - { resource_changes: [bootstrapTemplate, missingRestartPolicy] }, - bootstrapConfig - ), - /outside the reviewed capacity fields/ - ) - const missingRestartVersion = structuredClone(restartedBootstrapManager) - missingRestartVersion.change.before.version[0].name = 'primary' - assert.throws( - () => - validateCapacityPlan( - { resource_changes: [bootstrapTemplate, missingRestartVersion] }, + { resource_changes: [bootstrapTemplate, addedPolicy] }, bootstrapConfig ), /outside the reviewed capacity fields/ @@ -485,6 +465,127 @@ test('same-cap mode preserves 1000/60 while adding only the reviewed trust confi validateCapacityPlan({ resource_changes: [template, manager] }, sameCapConfig), { mode: 'same-cap-cell', changes: 2 } ) + // A gcloud rolling action on a stranded cell renamed its MIG version; the next roll reverts it. + const relabelledManager = structuredClone(manager) + relabelledManager.change.before.version[0].name = '0/2026-10-01 10:41:28.681518+00:00' + relabelledManager.change.after.version[0].name = 'primary' + assert.deepEqual( + validateCapacityPlan({ resource_changes: [template, relabelledManager] }, sameCapConfig), + { mode: 'same-cap-cell', changes: 2 } + ) + for (const [field, moved] of [ + ['target_size', 0], + ['base_instance_name', 'relay-other'], + ['named_port', [{ name: 'relay', port: 9090 }]] + ]) { + const unrelated = structuredClone(relabelledManager) + unrelated.change.after[field] = moved + assert.throws( + () => validateCapacityPlan({ resource_changes: [template, unrelated] }, sameCapConfig), + /outside the reviewed capacity fields/, + field + ) + } + const declaredPolicy = { + type: 'PROACTIVE', + minimal_action: 'REPLACE', + most_disruptive_allowed_action: 'REPLACE', + replacement_method: 'RECREATE', + max_surge_fixed: 0, + max_unavailable_fixed: 1 + } + for (const [field, drifted, away] of [ + ['type', 'OPPORTUNISTIC', 'OPPORTUNISTIC'], + ['minimal_action', 'RESTART', 'RESTART'], + ['most_disruptive_allowed_action', 'RESTART', 'RESTART'], + ['replacement_method', 'SUBSTITUTE', 'SUBSTITUTE'], + ['max_surge_fixed', 1, 1], + ['max_unavailable_fixed', 0, 0] + ]) { + const reverted = structuredClone(relabelledManager) + reverted.change.before.update_policy = [{ ...declaredPolicy, [field]: drifted }] + reverted.change.after.update_policy = [{ ...declaredPolicy }] + assert.deepEqual( + validateCapacityPlan({ resource_changes: [template, reverted] }, sameCapConfig), + { mode: 'same-cap-cell', changes: 2 }, + field + ) + const leaving = structuredClone(relabelledManager) + leaving.change.before.update_policy = [{ ...declaredPolicy }] + leaving.change.after.update_policy = [{ ...declaredPolicy, [field]: away }] + assert.throws( + () => validateCapacityPlan({ resource_changes: [template, leaving] }, sameCapConfig), + /outside the reviewed capacity fields/, + field + ) + } + // A second version is a canary split, never a label revert. + const secondVersion = structuredClone(relabelledManager) + secondVersion.change.after.version.push({ + name: 'primary', + instance_template: 'projects/project/global/instanceTemplates/canary', + target_size: [{ fixed: 1 }] + }) + assert.throws( + () => validateCapacityPlan({ resource_changes: [template, secondVersion] }, sameCapConfig), + /outside the reviewed capacity fields/ + ) + // A stranded rollback whose template is already in place plans only the label revert. + const plannedCell = (startupScript, templateLink) => ({ + root_module: { + resources: [ + { + address: 'google_compute_instance_template.relay_gce_cell["staging-gce-c3"]', + values: { metadata_startup_script: startupScript, self_link: templateLink } + }, + { + address: 'google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]', + values: { version: [{ instance_template: templateLink, name: 'primary' }] } + } + ] + } + }) + const reviewedLink = 'projects/project/global/instanceTemplates/reviewed' + const labelOnly = { + address: manager.address, + change: { + actions: ['update'], + before: { + target_size: 1, + version: [{ instance_template: reviewedLink, name: '0/2026-10-01 10:41:28.681518+00:00' }] + }, + after: { target_size: 1, version: [{ instance_template: reviewedLink, name: 'primary' }] }, + after_unknown: {} + } + } + const strandedPlan = (managerChange) => ({ + resource_changes: [managerChange], + planned_values: plannedCell(template.change.after.metadata_startup_script, reviewedLink) + }) + assert.deepEqual( + validateCapacityPlan(strandedPlan(labelOnly), sameCapConfig), + { mode: 'same-cap-cell', changes: 1 } + ) + const labelAway = structuredClone(labelOnly) + labelAway.change.before.version[0].name = 'primary' + labelAway.change.after.version[0].name = 'other' + assert.throws( + () => validateCapacityPlan(strandedPlan(labelAway), sameCapConfig), + /outside the reviewed capacity fields/ + ) + const labelAndResize = structuredClone(labelOnly) + labelAndResize.change.after.target_size = 0 + assert.throws( + () => validateCapacityPlan(strandedPlan(labelAndResize), sameCapConfig), + /outside the reviewed capacity fields/ + ) + const percentSurge = structuredClone(relabelledManager) + percentSurge.change.before.update_policy = [{ ...declaredPolicy, max_surge_percent: 0 }] + percentSurge.change.after.update_policy = [{ ...declaredPolicy, max_surge_percent: 50 }] + assert.throws( + () => validateCapacityPlan({ resource_changes: [template, percentSurge] }, sameCapConfig), + /outside the reviewed capacity fields/ + ) // A pre-template-apply rollback resume validates drift for the image the // cell already serves: the template leaves and re-enters the rollback image. const resumeTemplate = structuredClone(template) @@ -1300,3 +1401,30 @@ test('a same-cap roll may carry only this cell backend drain and request logging /only the exact instance template and MIG/ ) }) + +// The reconciliation allowance is only safe while it names exactly what Terraform declares. +test('the declared MIG fields match relay-gce-cells.tf', () => { + const terraform = readFileSync( + new URL('../../infra/terraform/relay-gce-cells.tf', import.meta.url), + 'utf8' + ) + const manager = terraform + .split('resource "google_compute_instance_group_manager" "relay_gce_cell" {')[1] + ?.split('\n}')[0] ?? '' + const block = (name) => manager.split(` ${name} {`)[1]?.split('\n }')[0] ?? '' + const topology = terraform.split(' relay_gce_topology = {')[1]?.split('\n }')[0] ?? '' + const attribute = (source, name) => { + const raw = new RegExp(`\\n\\s+${name}\\s+= (.+)`).exec(source)?.[1] + assert.notEqual(raw, undefined, `relay-gce-cells.tf declares no ${name}`) + const local = /^local\.relay_gce_topology\.(\w+)$/.exec(raw)?.[1] + if (local) return Number(new RegExp(`${local}\\s+= (\\d+)`).exec(topology)?.[1]) + return JSON.parse(raw) + } + const declared = Object.fromEntries( + Object.keys(DECLARED_MANAGER_FIELDS).map((path) => { + const [blockName, , field] = path.split('.') + return [path, attribute(block(blockName), field)] + }) + ) + assert.deepEqual(declared, { ...DECLARED_MANAGER_FIELDS }) +}) diff --git a/cloud/docs/relay-workflows.md b/cloud/docs/relay-workflows.md index 94df67f2bb7..a82f32d43c0 100644 --- a/cloud/docs/relay-workflows.md +++ b/cloud/docs/relay-workflows.md @@ -472,8 +472,14 @@ drained. Then read the cell's live runtime image from 3. The job classifies the cell itself and needs no extra input: - serving the **rollback** image and draining, it is `stranded`. The wave stopped before or during its template apply. The job re-isolates, re-drains, applies the reviewed - template, and rolls the MIG explicitly if that template was already in place. The cell - comes back on a new instance, so the drain clears, and it is restored to its entry class. + template, and, if that template was already in place, recreates the cell's one instance + with `recreate-instances`. The cell comes back on a new instance, so the drain clears, and + it is restored to its entry class. The recovery does not use a rolling action: that + rewrites the MIG's version name outside Terraform. The plan validator does accept a MIG + moving back to the version name and update policy `relay-gce-cells.tf` declares, so a cell + an older rolling action left relabelled reconciles on its next apply, roll, or stranded + rollback. The recreate refuses a MIG that does not hold exactly one instance, such as a + fenced cell; that failure is the guard, not a fault, so unfence before dispatching. - serving the **target** image, it is `roll`, the ordinary rollback. The template applied and the instance was replaced. - serving the **rollback** image and not draining, it is `resume`: a rollback that failed