import assert from 'node:assert/strict' import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { test } from 'node:test' import { allocateTargetQuotas, assertCutoverCellReady, cutoverMembership, parseMultiTargetArguments, pruneIncompatibleDirectorRevisions, runMultiTargetDeployment, sameBackendServiceResource, selectMultiTargetDeployments, verifySelectorCompatibleDirector } from './deploy-relay-gce-multi-target.mjs' const runtimeServiceAccount = 'orca-relay@example.iam.gserviceaccount.com' const digest = (value) => `sha256:${value.repeat(64)}` test('matches only exact canonical backend-service resource forms', () => { const resource = 'projects/onorca-cloud/global/backendServices/orca-cloud-relay-gce-c11' assert.equal( sameBackendServiceResource( `https://www.googleapis.com/compute/v1/${resource}`, resource ), true ) assert.equal( sameBackendServiceResource( `https://www.googleapis.com/compute/v1/${resource}`, resource.replace('c11', 'c12') ), false ) assert.equal( sameBackendServiceResource( `https://compute.example/compute/v1/${resource}`, resource ), false ) }) test('requires only compatible active and rollback director revisions', () => { let rollbackInventory = ['c1'] const revision = (minimum = 1, inventory = ['c1']) => ({ metadata: { annotations: { 'autoscaling.knative.dev/minScale': String(minimum) } }, spec: { containers: [ { image: 'registry.example/relay@sha256:abc', env: [ { name: 'ORCA_RELAY_ROLE', value: 'director' }, { name: 'ORCA_RELAY_ADMISSION_SELECTOR_VERSION', value: '3' }, { name: 'ORCA_RELAY_CELLS_JSON', value: JSON.stringify(inventory.map((id) => ({ id }))) } ] } ] } }) let names = ['active', 'rollback'] const deps = { command: (args) => { const revision = args[3] names = names.filter((name) => name !== revision) }, commandJson: (args) => { if (args.includes('services')) { return { status: { traffic: [ { percent: 100, revisionName: 'active' }, { tag: 'selector-rollback', revisionName: 'rollback' } ] } } } if (args.includes('list')) { return names.map((name) => ({ metadata: { name } })) } return args.includes('rollback') ? revision(0, rollbackInventory) : revision(1) } } const config = { project: 'project', directorRegion: 'region', directorService: 'service', directorMinimumInstances: 1 } assert.deepEqual(verifySelectorCompatibleDirector(config, deps), { activeRevision: 'active', rollbackRevision: 'rollback' }) rollbackInventory = ['c1', 'c2'] assert.throws( () => verifySelectorCompatibleDirector(config, deps), /director inventories do not match/ ) assert.throws( () => pruneIncompatibleDirectorRevisions(config, deps), /director compatibility pair failed/ ) rollbackInventory = ['c1'] names = ['active', 'rollback', 'legacy'] assert.throws( () => verifySelectorCompatibleDirector(config, deps), /old or pre-selector director revisions/ ) pruneIncompatibleDirectorRevisions(config, deps) assert.deepEqual(names, ['active', 'rollback']) assert.deepEqual(verifySelectorCompatibleDirector(config, deps), { activeRevision: 'active', rollbackRevision: 'rollback' }) names = ['active', 'rollback', null] assert.throws( () => verifySelectorCompatibleDirector(config, deps), /old or pre-selector director revisions/ ) assert.throws( () => pruneIncompatibleDirectorRevisions(config, deps), /unnamed revision/ ) }) test('requires the active director to meet the configured floor', () => { let activeMinimum = 5 let rollbackMinimum = 0 const revision = (minimum) => ({ metadata: { annotations: { 'autoscaling.knative.dev/minScale': String(minimum) } }, spec: { containers: [ { image: 'registry.example/relay@sha256:abc', env: [ { name: 'ORCA_RELAY_ROLE', value: 'director' }, { name: 'ORCA_RELAY_ADMISSION_SELECTOR_VERSION', value: '3' }, { name: 'ORCA_RELAY_CELLS_JSON', value: JSON.stringify([{ id: 'c1' }]) } ] } ] } }) const deps = { command: () => {}, commandJson: (args) => { if (args.includes('services')) { return { status: { traffic: [ { percent: 100, revisionName: 'active' }, { tag: 'selector-rollback', revisionName: 'rollback' } ] } } } if (args.includes('list')) { return ['active', 'rollback'].map((name) => ({ metadata: { name } })) } return args.includes('rollback') ? revision(rollbackMinimum) : revision(activeMinimum) } } const config = { project: 'project', directorRegion: 'region', directorService: 'service', directorMinimumInstances: 5 } assert.deepEqual(verifySelectorCompatibleDirector(config, deps), { activeRevision: 'active', rollbackRevision: 'rollback' }) pruneIncompatibleDirectorRevisions(config, deps) activeMinimum = 4 assert.throws( () => verifySelectorCompatibleDirector(config, deps), /active selector revision is below the required floor/ ) assert.throws( () => pruneIncompatibleDirectorRevisions(config, deps), /director compatibility pair failed/ ) // Every comparison against NaN is false, so negating the minimum check rejects it. activeMinimum = 'warm' assert.throws( () => verifySelectorCompatibleDirector(config, deps), /active selector revision is below the required floor/ ) assert.throws( () => pruneIncompatibleDirectorRevisions(config, deps), /director compatibility pair failed/ ) activeMinimum = 5 rollbackMinimum = 1 assert.throws( () => verifySelectorCompatibleDirector(config, deps), /selector rollback revision is not scale-to-zero/ ) assert.throws( () => pruneIncompatibleDirectorRevisions(config, deps), /director compatibility pair failed/ ) }) function topology() { const cell = (id, hostname, initiallyEnabled) => ({ origin: `https://${hostname}.relay.example.com`, zone: `us-central1-${hostname}`, mig_name: `relay-${hostname}`, instance_group: `https://compute.example/instanceGroups/relay-${hostname}`, backend_name: `relay-${hostname}`, backend_id: `https://compute.example/backendServices/relay-${hostname}`, url_map_name: 'orca-relay', generation_identity: `https://compute.example/instanceTemplates/relay-${hostname}-abc`, image: `us-central1-docker.pkg.dev/project/repo/relay@${digest(id)}`, capacity_requests: 4_000, connection_hard_cap: 600, connection_unobserved_bound: 40, initially_enabled: initiallyEnabled, fenced: false, desired_target_size: 1, target_size: 1 }) return { source: cell('a', 'a', true), target1: cell('b', 'b', false), target2: cell('c', 'c', false), general: cell('d', 'd', true) } } function withTopology(operation, value = topology()) { const directory = mkdtempSync(join(tmpdir(), 'relay-gce-multi-')) const file = join(directory, 'topology.json') writeFileSync(file, JSON.stringify(value)) return Promise.resolve(operation(file)).finally(() => rmSync(directory, { recursive: true })) } function config(topologyFile, mode = 'preflight') { const selectorMutation = [ 'cutover-admission', 'add-migration-cells', 'promote-general-cell', 'retire-migration-cell' ].includes(mode) return { project: 'test-project', directorOrigin: 'https://relay.example.com', adminAudience: 'https://relay.example.com/v1/admin/drain', topologyFile, sourceCellId: 'source', targetCellIds: ['promote-general-cell', 'retire-migration-cell'].includes(mode) ? ['target1'] : ['target1', 'target2'], generalCellIds: mode === 'cutover-admission' ? ['general'] : [], directorRegion: selectorMutation ? 'us-central1' : undefined, directorService: selectorMutation ? 'relay-director' : undefined, directorMinimumInstances: selectorMutation ? 1 : undefined, selectorAttemptId: mode === 'add-migration-cells' ? 'add_cells_test' : mode === 'promote-general-cell' ? 'promote_cell_test' : mode === 'retire-migration-cell' ? 'retire_cell_test' : undefined, unobservedConnectionBound: selectorMutation || mode === 'fence-source' ? 40 : undefined, failedTargetCellId: mode === 'supersede-target' ? 'target1' : undefined, replacementTargetCellId: mode === 'supersede-target' ? 'target2' : undefined, runtimeServiceAccount, environment: 'production', fenceCommit: 'a'.repeat(40), terraformDir: 'infra/terraform', terraformVarFile: 'environments/production.tfvars', mode, batchSize: 2, connectionCeiling: mode === 'fence-source' ? 600 : 6, minimumLeaseRemainingMs: 600_000, drainGraceMs: 120_000, pollIntervalMs: 1, timeoutMs: 1_000 } } function response(body, status = 200) { return new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } }) } function harness({ leaseRemainingMs = 900_000, refreshedLeaseRemainingMs = leaseRemainingMs, failAfterDrain = false, failEvacuationStatus = false, fence = false, allowPreFenceCompletion = false, loseResizeResponse = false, resumeNeedsPreApply = false, loseDrainSendResponse = false, loseDrainBeforeAccept = false, preparedDrainAttempt = false, recoverableDrain = false, recoveryAlreadyAttempted = false, preexistingRegisteredMigrations = 0, supersede = false, cleanupBeforeSupersession = false, offlineTargetMigrations = 0, offlineTargetMigrationsByTarget = {}, unregisteredTargetMigrations = 0, registeredSourceActive = 0, recoveryRegistrationDelayReads = 0, failedTargetEnabled = false, replacementHeartbeatFresh = true, replacementRuntimeCellUrl, legacySource = false, legacyMetricAgeMs = 0, selectorMembership = null, capacitySourceAssignments = [], capacityRequiredTargetUnits = [], headroomFailureTarget = null, sourceAssignments = fence ? 0 : 4, sourceRequiredTargetUnits = fence ? 0 : 8, sourceObservedRequests = null, sourceOutgoingMigrations = null, existingFenceAttempt = false, alreadyFencedSource = false, supersededFenceAttempt = false, completedFenceAttempt = false, activeDirectorMinimum = 1, misroutedCell = null, routeOverrideCell = null, frontendMisbound = false, templateHardCap = 600, templateUnobservedBound = 40, directorHardCap = templateHardCap, directorUnobservedBound = templateUnobservedBound, directorCapacityOverrides = {}, liveMigTemplateOverrides = {}, topologyValue = topology() } = {}) { const directorCapacity = (cellId) => { const hardCap = directorCapacityOverrides[cellId]?.hardCap ?? directorHardCap const unobservedBound = directorCapacityOverrides[cellId]?.unobservedBound ?? directorUnobservedBound return { hardCap, controlRebindReserve: 100, ordinaryConnectionLimit: hardCap - 100, unobservedBound, normalAdmissionPause: hardCap - 100 - unobservedBound } } const cells = { source: { enabled: !fence && !supersede, assignments: 4, activityLeases: fence ? 0 : 4, totalConnections: fence ? 1 : 5, controls: fence ? 0 : 4, observedRequests: sourceObservedRequests ?? (fence ? 0 : 4), heartbeatFresh: !alreadyFencedSource, draining: fence }, target1: { enabled: failedTargetEnabled || (fence && !supersede), assignments: fence ? 2 : 0, activityLeases: fence ? 2 : 0, totalConnections: fence ? 2 : 0, controls: fence ? 2 : 0, heartbeatFresh: !supersede, draining: false }, target2: { enabled: fence && !supersede, assignments: fence ? 2 : 0, activityLeases: fence ? 2 : 0, totalConnections: fence ? 2 : 0, controls: fence ? 2 : 0, heartbeatFresh: replacementHeartbeatFresh, draining: false }, general: { enabled: true, assignments: 0, activityLeases: 0, totalConnections: 0, controls: 0, heartbeatFresh: true, draining: false } } for (const cellId of Object.keys(topologyValue)) { cells[cellId] ??= { enabled: fence && !supersede, assignments: fence ? 2 : 0, activityLeases: fence ? 2 : 0, totalConnections: fence ? 2 : 0, controls: fence ? 2 : 0, heartbeatFresh: true, draining: false } } const events = [] const stateChanges = [] const batches = [] const publishedMigrations = new Map() const runtimeInspections = new Map() let drained = fence let remainingSourceAssignments = sourceAssignments let remainingRequiredTargetUnits = sourceRequiredTargetUnits const migSizes = Object.fromEntries( Object.keys(topologyValue).map((cellId) => [ cellId, cellId === 'source' && alreadyFencedSource ? 0 : cellId === 'target1' && completedFenceAttempt ? 0 : 1 ]) ) const instanceCounts = { ...migSizes } let supersessionRemaining = supersede ? 2 : 0 let remainingOfflineTargetMigrations = offlineTargetMigrations const initialOfflineTargetMigrationsByTarget = new Map( Object.entries(offlineTargetMigrationsByTarget) ) const remainingOfflineTargetMigrationsByTarget = new Map( initialOfflineTargetMigrationsByTarget ) const targetMigrationTotal = (cellId) => initialOfflineTargetMigrationsByTarget.has(cellId) ? initialOfflineTargetMigrationsByTarget.get(cellId) : 2 const remainingOfflineTargetMigrationCount = (cellId) => remainingOfflineTargetMigrationsByTarget.has(cellId) ? remainingOfflineTargetMigrationsByTarget.get(cellId) : remainingOfflineTargetMigrations const clearOfflineTargetMigrations = (cellId) => { if (remainingOfflineTargetMigrationsByTarget.has(cellId)) { remainingOfflineTargetMigrationsByTarget.set(cellId, 0) return } remainingOfflineTargetMigrations = 0 } let drainReceiptRecorded = recoverableDrain let drainSendStarted = false let recoveryDrainPrepared = recoveryAlreadyAttempted let recoveryLeasesRefreshed = false const recoveryRegistrationReads = new Map() let headroomFailureInjected = false let currentLeaseRemainingMs = leaseRemainingMs const drainGraces = [] const timeline = [] let fencedCompletions = 0 let resizeAttempts = 0 let capacityReads = 0 let addedCells = [] let fenceAttested = false let fenceAttempt = existingFenceAttempt || supersededFenceAttempt || completedFenceAttempt ? { attemptId: '44444444-4444-4444-8444-444444444444', environment: 'production', cellId: 'source', cellIncarnation: '11111111-1111-4111-8111-111111111111', migName: topologyValue.source.mig_name, instanceGroup: topologyValue.source.instance_group, generationIdentity: topologyValue.source.generation_identity, fenceCommit: (supersededFenceAttempt || completedFenceAttempt ? 'b' : 'a').repeat(40), planSha256: 'd'.repeat(64), planObjectName: 'terraform/state/relay-fence-plans/production/44444444-4444-4444-8444-444444444444.tfplan', ...(supersededFenceAttempt && !completedFenceAttempt ? {} : { planObjectGeneration: '123456789' }), varFileSha256: 'e'.repeat(64), terraformStateLineage: '55555555-5555-4555-8555-555555555555', terraformStateSerial: 7, terraformStateObjectGeneration: '987654321', terraformStateObjectSha256: 'f'.repeat(64), requestReason: 'orca-relay-fence/44444444-4444-4444-8444-444444444444', createdAt: Date.now(), expiresAt: Date.now() + 3_600_000, ...(completedFenceAttempt ? { applyStartedAt: 101, applyInvocations: [ { invocationId: '66666666-6666-4666-8666-666666666666', requestReason: 'orca-relay-fence/44444444-4444-4444-8444-444444444444/66666666-6666-4666-8666-666666666666', startedAt: 101 } ] } : {}) } : null let selector = selectorMembership ? { generation: 1, attemptId: 'existing-selector', membership: selectorMembership } : null const selectorIntents = new Map() const cellForOrigin = (origin) => { const entry = Object.entries(topologyValue).find(([, value]) => value.origin === origin) return entry?.[0] } const cellForMig = (name) => { const entry = Object.entries(topologyValue).find(([, value]) => value.mig_name === name) return entry?.[0] } const commandJson = (args) => { if (args[0] === 'run') { if (args.includes('services')) { return { status: { traffic: [ { percent: 100, revisionName: 'active' }, { percent: 0, tag: 'selector-rollback', revisionName: 'rollback' } ] } } } if (args.includes('list')) { return ['active', 'rollback'].map((name) => ({ metadata: { name } })) } const revisionName = args[3] return { metadata: { annotations: { 'autoscaling.knative.dev/minScale': revisionName === 'rollback' ? '0' : String(activeDirectorMinimum) } }, spec: { containers: [ { image: 'registry.example/relay@sha256:abc', env: [ { name: 'ORCA_RELAY_ROLE', value: 'director' }, { name: 'ORCA_RELAY_ADMISSION_SELECTOR_VERSION', value: '3' }, { name: 'ORCA_RELAY_CELLS_JSON', value: JSON.stringify( Object.keys(topologyValue).map((id) => ({ id })) ) } ] } ] } } } if (args.includes('instance-templates')) { const templateName = args[args.indexOf('describe') + 1] const expected = Object.values(topologyValue).find((cell) => cell.generation_identity.endsWith(`/instanceTemplates/${templateName}`) ) return { selfLink: expected.generation_identity, properties: { metadata: { items: [ { key: 'startup-script', value: [ `printf 'ORCA_RELAY_IMAGE_DIGEST=%s\\n' '${expected.image.split('@')[1]}'`, ` printf 'ORCA_RELAY_CELL_CONNECTION_HARD_CAP=%s\\n' '${templateHardCap}'`, ` printf 'ORCA_RELAY_CELL_CONNECTION_UNOBSERVED_BOUND=%s\\n' '${templateUnobservedBound}'`, `docker pull '${expected.image}'`, `docker run '${expected.image}'` ].join('\n') } ] } } } } if (args[0] === 'logging') { return [ { timestamp: new Date(Date.now() - legacyMetricAgeMs).toISOString(), jsonPayload: { totalConnections: cells.source.totalConnections, preAuthConnections: 0, controls: cells.source.controls, splices: 0, pendingSplices: 0, queuedBytes: 0 } } ] } const describeIndex = args.indexOf('describe') const listIndex = args.indexOf('list-instances') const name = args[describeIndex >= 0 ? describeIndex + 1 : listIndex + 1] const migCell = cellForMig(name) if (args.includes('list-instances')) { return instanceCounts[migCell] === 0 ? [] : [ { instance: `https://compute.example/instances/${name}-vm`, instanceStatus: 'RUNNING', currentAction: 'NONE', version: { name: 'primary', instanceTemplate: topologyValue[migCell].generation_identity } } ] } if (args.includes('instance-groups')) { return { targetSize: migSizes[migCell], instanceTemplate: liveMigTemplateOverrides[migCell] ?? topologyValue[migCell].generation_identity, updatePolicy: { replacementMethod: 'RECREATE', maxSurge: { fixed: 0 }, maxUnavailable: { fixed: 1 } } } } if (args.includes('instances')) { return { networkInterfaces: [{ networkIP: '10.42.0.2' }], serviceAccounts: [{ email: runtimeServiceAccount }] } } if (args.includes('target-https-proxies')) { return { name: 'orca-relay', selfLink: 'https://www.googleapis.com/compute/v1/projects/test-project/global/targetHttpsProxies/orca-relay', urlMap: frontendMisbound ? 'https://www.googleapis.com/compute/v1/projects/test-project/global/urlMaps/other' : 'https://www.googleapis.com/compute/v1/projects/test-project/global/urlMaps/orca-relay' } } if (args.includes('forwarding-rules')) { return { name: 'orca-relay', target: 'https://www.googleapis.com/compute/v1/projects/test-project/global/targetHttpsProxies/orca-relay', IPAddress: '203.0.113.10', portRange: '443-443', loadBalancingScheme: 'EXTERNAL_MANAGED' } } if (args.includes('addresses')) { return { name: 'orca-relay', address: '203.0.113.10' } } if (args.includes('url-maps')) { return { name: 'orca-relay', selfLink: 'https://www.googleapis.com/compute/v1/projects/test-project/global/urlMaps/orca-relay', hostRules: Object.entries(topologyValue).map(([cellId, cell]) => ({ hosts: [new URL(cell.origin).hostname], pathMatcher: cellId })), pathMatchers: Object.entries(topologyValue).map(([cellId, cell]) => ({ name: cellId, defaultService: cellId === misroutedCell ? topologyValue.general.backend_id : cell.backend_id, ...(cellId === routeOverrideCell ? { pathRules: [ { paths: ['/v1/*'], service: topologyValue.general.backend_id } ] } : {}) })) } } const id = Object.entries(topologyValue).find(([, value]) => value.backend_name === name)?.[0] return { protocol: 'HTTP', timeoutSec: 86_400, backends: [{ group: topologyValue[id].instance_group }] } } const fetch = async (url, options = {}) => { const parsed = new URL(url) if (parsed.pathname === '/health' || parsed.pathname === '/ready') return response({ ok: true }) const body = JSON.parse(options.body ?? '{}') if (parsed.pathname === '/v1/admin/runtime-status') { const id = cellForOrigin(parsed.origin) const cell = cells[id] const capacity = directorCapacity(id) runtimeInspections.set(id, (runtimeInspections.get(id) ?? 0) + 1) return response({ v: 1, role: 'cell', cellId: id, cellUrl: topologyValue[id].origin, imageDigest: topologyValue[id].image.split('@')[1], draining: cell.draining, connectionCapacity: { ...capacity }, runtime: legacySource && id === 'source' ? null : { totalConnections: cell.totalConnections, preAuthConnections: 0, enforcedConnectionUnits: cell.totalConnections, controls: cell.controls, splices: 0, pendingSplices: 0, queuedBytes: 0 } }) } if (parsed.pathname === '/v1/admin/cell-status') { const cell = cells[body.cellId] const capacity = directorCapacity(body.cellId) return response({ v: 1, status: { cellId: body.cellId, cellUrl: topologyValue[body.cellId].origin, enabled: cell.enabled, assignments: cell.assignments, activityLeases: cell.activityLeases, activityRequestUnits: cell.activityLeases, reservedRequests: cell.activityLeases, connectionCapacity: { ...capacity, observedConnections: cell.totalConnections, inFlightConnections: 0, reservedConnectionUnits: 0, enforcedConnectionUnits: cell.totalConnections, pendingControlReservations: 0, heartbeatFresh: cell.heartbeatFresh }, outgoingMigrations: sourceOutgoingMigrations ?? (fence ? Object.keys(topologyValue) .filter((cellId) => cellId !== 'source' && cellId !== 'general') .reduce((total, cellId) => total + targetMigrationTotal(cellId), 0) : 0), incomingMigrations: 0, runtime: { cellUrl: body.cellId === 'target2' && replacementRuntimeCellUrl ? replacementRuntimeCellUrl : topologyValue[body.cellId].origin, cellIncarnation: '11111111-1111-4111-8111-111111111111', ready: true, heartbeatFresh: cell.heartbeatFresh, observedRequests: cell.observedRequests ?? cell.controls } } }) } if (parsed.pathname === '/v1/admin/admission-selector/status') { selector ??= { generation: 0, attemptId: null, membership: { existingOnly: Object.keys(cells).filter((cellId) => !cells[cellId].enabled), migrationOnly: [], general: Object.keys(cells).filter((cellId) => cells[cellId].enabled) } } return response({ v: 1, selector, intent: body.attemptId ? selectorIntents.get(body.attemptId) ?? null : null }) } if (parsed.pathname === '/v1/admin/admission-selector/apply') { selector = { generation: body.expectedGeneration + 1, attemptId: body.attemptId, membership: body.membership } selectorIntents.set(body.attemptId, { attemptId: body.attemptId, expectedGeneration: body.expectedGeneration, intendedGeneration: selector.generation, membership: body.membership, state: 'committed' }) return response({ v: 1, changed: true, selector }) } if (parsed.pathname === '/v1/admin/admission-selector/add-migration-cells') { addedCells = body.cells selector = { generation: body.expectedGeneration + 1, attemptId: body.attemptId, membership: { ...selector.membership, migrationOnly: [ ...selector.membership.migrationOnly, ...body.cells.map(({ cellId }) => cellId) ].sort() } } selectorIntents.set(body.attemptId, { attemptId: body.attemptId, expectedGeneration: body.expectedGeneration, intendedGeneration: selector.generation, membership: selector.membership, state: 'committed' }) return response({ v: 1, changed: true, selector }) } if (parsed.pathname === '/v1/admin/evacuation-capacity') { const read = capacityReads++ return response({ v: 1, sourceAssignments: capacitySourceAssignments[read] ?? remainingSourceAssignments, requiredTargetUnits: capacityRequiredTargetUnits[read] ?? remainingRequiredTargetUnits, availableTargetUnits: 4_000 }) } if (parsed.pathname === '/v1/admin/cell-state') { cells[body.cellId].enabled = body.enabled stateChanges.push([body.cellId, body.enabled]) return response({ ok: true }) } if (parsed.pathname === '/v1/admin/drain-attempt-prepare') { return response({ v: 1, state: 'prepared' }) } if (parsed.pathname === '/v1/admin/drain-attempt-send') { const shouldSend = !drainSendStarted drainSendStarted = true if (loseDrainSendResponse) throw new Error('injected_send_transition_response_loss') return response({ v: 1, attempt: { state: 'send-may-have-started', shouldSend, sendPermitExpiresAt: Date.now() + 30_000 } }) } if (parsed.pathname === '/v1/admin/drain-attempt-receipt') { drainReceiptRecorded = true return response({ v: 1, attempt: { state: 'application-receipt' } }) } if (parsed.pathname === '/v1/admin/drain-attempt-recover-forward') { recoveryLeasesRefreshed = true currentLeaseRemainingMs = refreshedLeaseRemainingMs if (!drainReceiptRecorded) { if (preparedDrainAttempt && !drainSendStarted) { return response({ v: 1, shouldSend: false, retryAfter: Date.now(), preparedAttempt: { attemptId: '33333333-3333-4333-8333-333333333333', cellId: 'source', cellIncarnation: '11111111-1111-4111-8111-111111111111', traceValue: '44444444-4444-4444-8444-444444444444', plannedGraceMs: 120_000, state: 'prepared' } }) } return response({ error: 'drain_application_receipt_missing' }, 409) } const shouldSend = !recoveryDrainPrepared recoveryDrainPrepared = true return response({ v: 1, shouldSend, retryAfter: Date.now() }) } if (parsed.pathname === '/v1/admin/evacuate-cell') { if (body.targetCellId === headroomFailureTarget && !headroomFailureInjected) { headroomFailureInjected = true const partiallyStarted = Math.min(1, remainingSourceAssignments) publishedMigrations.set( body.targetCellId, (publishedMigrations.get(body.targetCellId) ?? 0) + partiallyStarted ) remainingSourceAssignments -= partiallyStarted return response({ error: 'relay_connection_headroom_exhausted' }, 409) } const started = Math.min(body.limit, remainingSourceAssignments) batches.push([body.targetCellId, started]) publishedMigrations.set( body.targetCellId, (publishedMigrations.get(body.targetCellId) ?? 0) + started ) remainingSourceAssignments -= started if (remainingSourceAssignments === 0) remainingRequiredTargetUnits = 0 return response({ v: 1, started }) } if (parsed.pathname === '/v1/admin/cell-fence-attest') { fenceAttested = true return response({ v: 1, cellId: body.cellId, expiresAt: Date.now() + 300_000 }) } if (parsed.pathname === '/v1/admin/cell-fence-adopt-legacy') { fenceAttested = true return response({ v: 1, cellId: body.cellId, expiresAt: Date.now() + 300_000 }) } if (parsed.pathname === '/v1/admin/cell-fence-commit-legacy-adoption') { return response({ v: 1, cellId: body.cellId, committed: true }) } if (parsed.pathname === '/v1/admin/cell-fence-attempt-prepare') { fenceAttempt = body return response({ v: 1, attempt: body }) } if (parsed.pathname === '/v1/admin/cell-fence-attempt-start') { fenceAttempt = { ...body, applyStartedAt: Date.now() } return response({ v: 1, attempt: fenceAttempt, invocation: { invocationId: body.invocationId, requestReason: body.invocationRequestReason, startedAt: fenceAttempt.applyStartedAt } }) } if (parsed.pathname === '/v1/admin/cell-fence-attempt-operation') { fenceAttempt = body return response({ v: 1, attempt: body, invocation: { invocationId: body.invocationId, requestReason: body.invocationRequestReason, startedAt: Date.now(), gceOperation: body.gceOperation } }) } if (parsed.pathname === '/v1/admin/cell-fence-attempt-status') { return response({ v: 1, attempt: fenceAttempt }) } if (parsed.pathname === '/v1/admin/cell-fence-attempt-abort') { fenceAttempt = { ...body, abortedAt: Date.now() } return response({ v: 1, attempt: fenceAttempt }) } if (parsed.pathname === '/v1/admin/evacuation-status') { if (failEvacuationStatus) { return response({ error: 'injected_status_failure' }, 500) } const completed = body.completeReady && (!fence || fenceAttested || allowPreFenceCompletion) if (completed && fenceAttested) clearOfflineTargetMigrations(body.targetCellId) const migrationTotal = targetMigrationTotal(body.targetCellId) const remainingOfflineTargetMigrationsForCell = remainingOfflineTargetMigrationCount(body.targetCellId) const supersededTarget = supersede && body.targetCellId === 'target1' const recoveryRegistrationRead = recoveryRegistrationReads.get(body.targetCellId) ?? 0 if (recoveryLeasesRefreshed) { recoveryRegistrationReads.set(body.targetCellId, recoveryRegistrationRead + 1) } const recoveryRegistrationSettled = recoveryRegistrationRead >= recoveryRegistrationDelayReads const remainingMigrations = supersededTarget ? supersessionRemaining : completed ? remainingOfflineTargetMigrationsForCell + unregisteredTargetMigrations : migrationTotal const registeredMigrations = supersededTarget ? supersessionRemaining : drained ? remainingMigrations - unregisteredTargetMigrations : Math.min( remainingMigrations, preexistingRegisteredMigrations + (recoveryLeasesRefreshed && recoveryRegistrationSettled ? publishedMigrations.get(body.targetCellId) ?? 0 : 0) ) const completableMigrations = drained && !completed ? migrationTotal - remainingOfflineTargetMigrationsForCell - unregisteredTargetMigrations - registeredSourceActive : recoveryLeasesRefreshed ? Math.max( 0, registeredMigrations - remainingOfflineTargetMigrationsForCell - registeredSourceActive ) : 0 if (completed) { if (failAfterDrain) return response({ error: 'injected_completion_failure' }, 500) if (fenceAttested) fencedCompletions++ cells.source.activityLeases = 0 for (const targetCellId of Object.keys(cells).filter((id) => id !== 'source')) { cells[targetCellId].activityLeases = 2 } } timeline.push({ kind: 'evacuation_status', targetCellId: body.targetCellId, completeReady: body.completeReady, inProgress: remainingMigrations, registeredTargetInactive: remainingOfflineTargetMigrationsForCell }) return response({ v: 1, inProgress: remainingMigrations, oldestExpiresAt: remainingMigrations === 0 ? null : Date.now() + currentLeaseRemainingMs, oldestRemainingMs: remainingMigrations === 0 ? null : currentLeaseRemainingMs, targetRegistered: registeredMigrations, registeredSourceActive, registeredCompletable: completableMigrations, registeredTargetInactive: remainingOfflineTargetMigrationsForCell, completed: completed ? migrationTotal - remainingOfflineTargetMigrationsForCell - unregisteredTargetMigrations : 0, blocked: completed ? remainingOfflineTargetMigrationsForCell : 0, expiredUnregistered: 0, repairableExpiredUnregistered: 0, abortableExpiredUnregistered: 0, blockedExpiredUnregistered: 0, blockedExpiredOnNewerTargetAssignment: 0 }) } if (parsed.pathname === '/v1/admin/migration-supersede-cell') { const superseded = cleanupBeforeSupersession ? 0 : supersessionRemaining supersessionRemaining = 0 return response({ v: 1, superseded }) } if (parsed.pathname === '/v1/admin/drain') { drainGraces.push(body.graceMs) if (loseDrainBeforeAccept && body.graceMs === 120_000) { throw new Error('injected_drain_response_loss') } drained = true cells.source.draining = true cells.source.activityLeases = 0 cells.source.controls = 0 for (const targetCellId of Object.keys(cells).filter((id) => id !== 'source')) { cells[targetCellId].totalConnections = 2 } return response({ ok: true }) } return response({ error: 'unexpected_request' }, 500) } return { overrides: { commandJson, terraform: (args) => { if (args.includes('console')) return 'true\n' if (args.includes('plan')) return '' if (args.includes('show') && args.length > 3) { return JSON.stringify({ resource_changes: [] }) } if (args.includes('show')) { return JSON.stringify({ values: { root_module: { resources: [ { address: 'google_compute_instance_group_manager.relay_gce_cell["source"]', values: { name: topologyValue.source.mig_name, zone: topologyValue.source.zone, instance_group: topologyValue.source.instance_group, target_size: migSizes.source, version: [ { instance_template: topologyValue.source.generation_identity } ] } } ] } } }) } throw new Error(`unexpected terraform command: ${args.join(' ')}`) }, command: (args) => { throw new Error(`unexpected direct gcloud mutation: ${args.join(' ')}`) }, terraformFenceApply: async (fenceConfig, callbacks) => { const attempt = { attemptId: '44444444-4444-4444-8444-444444444444', environment: fenceConfig.environment, cellId: fenceConfig.cell.cellId, cellIncarnation: fenceConfig.cellIncarnation, migName: fenceConfig.cell.migName, instanceGroup: fenceConfig.cell.instanceGroup, generationIdentity: fenceConfig.cell.generationIdentity, fenceCommit: fenceConfig.fenceCommit, planSha256: 'd'.repeat(64), planObjectName: `terraform/state/relay-fence-plans/${fenceConfig.environment}/44444444-4444-4444-8444-444444444444.tfplan`, planObjectGeneration: '123456789', varFileSha256: 'e'.repeat(64), terraformStateLineage: '55555555-5555-4555-8555-555555555555', terraformStateSerial: 7, terraformStateObjectGeneration: '987654321', terraformStateObjectSha256: 'f'.repeat(64), requestReason: 'orca-relay-fence/44444444-4444-4444-8444-444444444444' } await callbacks.prepareAttempt(attempt) await callbacks.preApplyGuard() const invocation = { invocationId: '66666666-6666-4666-8666-666666666666', requestReason: `${attempt.requestReason}/66666666-6666-4666-8666-666666666666` } await callbacks.markApplyStarted(attempt, invocation) const cellId = fenceConfig.cell.cellId migSizes[cellId] = 0 instanceCounts[cellId] = 0 cells[cellId].heartbeatFresh = false resizeAttempts++ if (loseResizeResponse && resizeAttempts === 1) { throw new Error('injected_resize_response_loss') } const completed = { ...attempt, gceOperation: 'operation-1' } await callbacks.markOperation(completed, invocation) await callbacks.postApplyGuard(attempt.cellIncarnation) await callbacks.attest(completed) }, terraformFenceAdopt: async (fenceConfig, callbacks) => { assert.equal(await callbacks.loadAttempt(), null) await callbacks.assertCommittedFenceSet() await callbacks.preApplyGuard() await callbacks.assertStateFenced() await callbacks.postApplyGuard(fenceConfig.cellIncarnation) assert.equal(await callbacks.loadAttempt(), null) await callbacks.attest(fenceConfig.cellIncarnation) await callbacks.postApplyGuard(fenceConfig.cellIncarnation) await callbacks.commitAdoption(fenceConfig.cellIncarnation) callbacks.emit({ event: 'terraform_cell_fence_legacy_adopted', cellId: fenceConfig.cell.cellId }) }, terraformFenceResume: async (fenceConfig, callbacks) => { const attempt = await callbacks.loadAttempt() if (resumeNeedsPreApply) await callbacks.preApplyGuard() const completed = { ...attempt, cellIncarnation: fenceConfig.cellIncarnation, gceOperation: 'operation-1' } await callbacks.postApplyGuard(completed.cellIncarnation) await callbacks.attest(completed) }, terraformFenceRecoverCompleted: async ( fenceConfig, callbacks, recovery ) => { const attempt = await callbacks.loadAttempt() callbacks.emit({ event: 'terraform_cell_fence_completed_attempt_recovered', cellId: fenceConfig.cell.cellId, attemptId: attempt.attemptId, gceOperation: recovery.gceOperation }) }, terraformFenceAbort: async (fenceConfig, callbacks) => { await callbacks.abortAttempt() callbacks.emit({ event: 'terraform_fence_aborted_before_apply', cellId: fenceConfig.cell.cellId }) }, terraformFenceSupersede: async (fenceConfig, callbacks) => { const attempt = await callbacks.loadAttempt() await callbacks.abortAttempt(attempt) callbacks.emit({ event: 'terraform_fence_superseded_before_upload', cellId: fenceConfig.cell.cellId, previousFenceCommit: attempt.fenceCommit, fenceCommit: fenceConfig.fenceCommit }) }, identityToken: () => 'aaa.bbb.ccc', mutationIdentityToken: () => 'ddd.eee.fff', resolve4: async () => ['203.0.113.10'], fetch, emit: (event) => { events.push(event) timeline.push({ kind: 'event', event }) }, wait: async () => undefined, random: () => 0 }, batches, events, stateChanges, drainGraces, timeline, fencedCompletions: () => fencedCompletions, capacityReads: () => capacityReads, selector: () => selector, addedCells: () => addedCells, runtimeInspections } } test('parses deterministic target sets with single-cell selector exceptions', () => { const common = [ '--project', 'project', '--director-origin', 'https://relay.example.com', '--admin-audience', 'https://relay.example.com/v1/admin/drain', '--topology-file', 'topology.json', '--source-cell-id', 'source', '--runtime-service-account', runtimeServiceAccount, '--mode', 'preflight' ] assert.deepEqual( parseMultiTargetArguments([...common, '--target-cell-ids', 'target2,target1']).targetCellIds, ['target1', 'target2'] ) assert.throws( () => parseMultiTargetArguments([...common, '--target-cell-ids', 'target1']), /at least 2/ ) const cutover = [ ...common.slice(0, -2), '--mode', 'cutover-admission', '--target-cell-ids', 'target1,target2', '--general-cell-ids', 'general', '--director-region', 'us-central1', '--director-service', 'relay-director', '--director-min-instances', '5' ] assert.throws( () => parseMultiTargetArguments(cutover), /unobserved-connection-bound/ ) assert.equal( parseMultiTargetArguments([ ...cutover, '--unobserved-connection-bound', '40' ]).unobservedConnectionBound, 40 ) const recovery = [ ...common.slice(0, -2), '--mode', 'recover-forward', '--target-cell-ids', 'target1,target2' ] assert.throws( () => parseMultiTargetArguments(recovery), /unobserved-connection-bound/ ) assert.equal( parseMultiTargetArguments([ ...recovery, '--unobserved-connection-bound', '60' ]).unobservedConnectionBound, 60 ) const fenceSource = [ ...common.slice(0, -2), '--mode', 'fence-source', '--target-cell-ids', 'target1,target2', '--fence-commit', 'a'.repeat(40) ] assert.throws( () => parseMultiTargetArguments(fenceSource), /unobserved-connection-bound/ ) assert.equal( parseMultiTargetArguments([ ...fenceSource, '--unobserved-connection-bound', '60' ]).unobservedConnectionBound, 60 ) const addCells = [ ...common.slice(0, -2), '--mode', 'add-migration-cells', '--target-cell-ids', 'target1,target2', '--director-region', 'us-central1', '--director-service', 'relay-director', '--director-min-instances', '5', '--unobserved-connection-bound', '40' ] assert.throws( () => parseMultiTargetArguments(addCells), /selector-attempt-id/ ) assert.equal( parseMultiTargetArguments([ ...addCells, '--target-cell-ids', 'target1', '--selector-attempt-id', 'add_cells_parse' ]).targetCellIds.length, 1 ) const retireCell = [ ...common.slice(0, -2), '--mode', 'retire-migration-cell', '--target-cell-ids', 'target1', '--director-region', 'us-central1', '--director-service', 'relay-director', '--director-min-instances', '5', '--selector-attempt-id', 'retire_cell_parse' ] assert.equal( parseMultiTargetArguments(retireCell).mode, 'retire-migration-cell' ) assert.throws( () => parseMultiTargetArguments([ ...retireCell, '--target-cell-ids', 'target1,target2' ]), /exactly one/ ) const promoteCell = [ ...common.slice(0, -2), '--mode', 'promote-general-cell', '--target-cell-ids', 'target1', '--director-region', 'us-central1', '--director-service', 'relay-director', '--director-min-instances', '5', '--selector-attempt-id', 'promote_cell_parse' ] assert.equal( parseMultiTargetArguments(promoteCell).mode, 'promote-general-cell' ) assert.throws( () => parseMultiTargetArguments([ ...promoteCell, '--target-cell-ids', 'target1,target2' ]), /exactly one/ ) }) test('requires a complete exact completed-fence recovery pin set', () => { const args = [ '--project', 'project', '--director-origin', 'https://relay.example.com', '--admin-audience', 'https://relay.example.com/v1/admin/drain', '--topology-file', 'topology.json', '--source-cell-id', 'source', '--target-cell-ids', 'target1,target2', '--runtime-service-account', runtimeServiceAccount, '--mode', 'supersede-target', '--failed-target-cell-id', 'target1', '--replacement-target-cell-id', 'target2', '--fence-commit', 'a'.repeat(40), '--completed-fence-attempt-id', '44444444-4444-4444-8444-444444444444', '--completed-fence-commit', 'b'.repeat(40), '--completed-fence-operation', 'operation-1', '--completed-fence-state-serial', '61', '--completed-fence-plan-generation', '123', '--completed-fence-state-generation', '456', '--completed-fence-state-sha256', 'c'.repeat(64), '--fence-broker-service-account', 'fence-broker@example.gserviceaccount.com' ] assert.equal( parseMultiTargetArguments(args).completedFenceRecovery .terraformStateSerial, 61 ) assert.throws( () => parseMultiTargetArguments(args.slice(0, -2)), /recovery inputs are invalid/ ) }) test('requires the cutover source to remain existing-only', () => { assert.throws( () => cutoverMembership(topology(), { sourceCellId: 'source', targetCellIds: ['target1'], generalCellIds: ['source', 'target2'] }), /source must remain existing-only/ ) }) test('requires exact cutover connection-capacity evidence and live headroom', () => { const status = { draining: false, connectionCapacity: { hardCap: 600, controlRebindReserve: 100, ordinaryConnectionLimit: 500, unobservedBound: 40, normalAdmissionPause: 460, pendingControlReservations: 20, heartbeatFresh: true }, runtimeConnectionCapacity: { hardCap: 600, controlRebindReserve: 100, ordinaryConnectionLimit: 500, unobservedBound: 40, normalAdmissionPause: 460 }, process: { enforcedConnectionUnits: 400, preAuthConnections: 3 } } assert.doesNotThrow(() => assertCutoverCellReady('target1', status, 40)) assert.throws( () => assertCutoverCellReady( 'target1', { ...status, runtimeConnectionCapacity: { ...status.runtimeConnectionCapacity, unobservedBound: 39 } }, 40 ), /reviewed connection-capacity policy/ ) assert.throws( () => assertCutoverCellReady( 'target1', { ...status, connectionCapacity: { ...status.connectionCapacity, pendingControlReservations: 60 } }, 40 ), /normal-admission connection headroom/ ) assert.throws( () => assertCutoverCellReady( 'target1', { ...status, process: { ...status.process, preAuthConnections: 45 } }, 40 ), /pre-auth connection headroom/ ) }) test('cuts over only after every proposed cell passes exact readiness evidence', async () => { await withTopology(async (file) => { const testHarness = harness() await runMultiTargetDeployment( config(file, 'cutover-admission'), testHarness.overrides ) assert.deepEqual(testHarness.selector(), { generation: 1, attemptId: testHarness.selector().attemptId, membership: { existingOnly: ['source'], migrationOnly: ['target1', 'target2'], general: ['general'] } }) assert.equal(testHarness.events.at(-1).event, 'admission_selector_cutover') assert.deepEqual(Object.fromEntries(testHarness.runtimeInspections), { target1: 2, target2: 2, general: 2 }) }) }) test('rejects a different active selector before readiness checks or revision mutation', async () => { await withTopology(async (file) => { const testHarness = harness({ selectorMembership: { existingOnly: ['source', 'target2'], migrationOnly: ['target1'], general: ['general'] } }) await assert.rejects( runMultiTargetDeployment( config(file, 'cutover-admission'), testHarness.overrides ), /already active with different membership/ ) assert.equal(testHarness.runtimeInspections.size, 0) }) }) test('registers new Terraform targets as one selector generation', async () => { await withTopology(async (file) => { const reviewed = topology() reviewed.target1.connection_hard_cap = 1_000 reviewed.target1.connection_unobserved_bound = 60 writeFileSync(file, JSON.stringify(reviewed)) const testHarness = harness({ selectorMembership: { existingOnly: ['source'], migrationOnly: [], general: ['general'] }, activeDirectorMinimum: 5 }) await runMultiTargetDeployment( { ...config(file, 'add-migration-cells'), directorMinimumInstances: 5 }, testHarness.overrides ) assert.deepEqual(testHarness.selector(), { generation: 2, attemptId: 'add_cells_test', membership: { existingOnly: ['source'], migrationOnly: ['target1', 'target2'], general: ['general'] } }) assert.equal(testHarness.events.at(-1).event, 'migration_cells_added') assert.deepEqual(testHarness.addedCells(), [ { cellId: 'target1', cellUrl: topology().target1.origin, capacityRequests: 4_000, region: 'us-central1', connectionHardCap: 1_000, connectionUnobservedBound: 60 }, { cellId: 'target2', cellUrl: topology().target2.origin, capacityRequests: 4_000, region: 'us-central1', connectionHardCap: 600, connectionUnobservedBound: 40 } ]) assert.equal(testHarness.runtimeInspections.size, 0) }) }) test('promotes exactly one healthy migration cell to general admission', async () => { await withTopology(async (file) => { const testHarness = harness({ selectorMembership: { existingOnly: ['source'], migrationOnly: ['target1', 'target2'], general: ['general'] } }) await runMultiTargetDeployment( config(file, 'promote-general-cell'), testHarness.overrides ) assert.deepEqual(testHarness.selector(), { generation: 2, attemptId: 'promote_cell_test', membership: { existingOnly: ['source'], migrationOnly: ['target2'], general: ['general', 'target1'] } }) assert.equal(testHarness.events.at(-1).event, 'migration_cell_promoted_general') }) }) test('rejects promotion below the configured director floor', async () => { await withTopology(async (file) => { const testHarness = harness({ selectorMembership: { existingOnly: ['source'], migrationOnly: ['target1', 'target2'], general: ['general'] } }) await assert.rejects( runMultiTargetDeployment( { ...config(file, 'promote-general-cell'), directorMinimumInstances: 2 }, testHarness.overrides ), /active director is not compatible/ ) assert.equal(testHarness.events.length, 0) }) }) test('rejects general promotion unless the exact cell is migration-only', async () => { await withTopology(async (file) => { const testHarness = harness({ selectorMembership: { existingOnly: ['source'], migrationOnly: ['target2'], general: ['general', 'target1'] } }) await assert.rejects( runMultiTargetDeployment( config(file, 'promote-general-cell'), testHarness.overrides ), /must be migration-only/ ) assert.equal(testHarness.events.length, 0) }) }) test('retires exactly one migration cell without changing other membership', async () => { await withTopology(async (file) => { const testHarness = harness({ selectorMembership: { existingOnly: ['source'], migrationOnly: ['target1', 'target2'], general: ['general'] } }) await runMultiTargetDeployment( config(file, 'retire-migration-cell'), testHarness.overrides ) assert.deepEqual(testHarness.selector(), { generation: 2, attemptId: 'retire_cell_test', membership: { existingOnly: ['source', 'target1'], migrationOnly: ['target2'], general: ['general'] } }) assert.equal(testHarness.events.at(-1).event, 'migration_cell_retired') assert.equal(testHarness.runtimeInspections.size, 0) }) }) test('rejects retirement unless the exact cell is migration-only', async () => { await withTopology(async (file) => { const testHarness = harness({ selectorMembership: { existingOnly: ['source'], migrationOnly: ['target2'], general: ['general', 'target1'] } }) await assert.rejects( runMultiTargetDeployment( config(file, 'retire-migration-cell'), testHarness.overrides ), /must be migration-only/ ) assert.equal(testHarness.events.length, 0) }) }) test('rejects overlapping generations and allocates below the connection ceiling', () => { const selected = selectMultiTargetDeployments(topology(), 'source', ['target1', 'target2']) assert.equal(selected.targets.length, 2) const planned = allocateTargetQuotas({ sourceAssignments: 4, sourceConnections: 5, requiredTargetUnits: 8, connectionCeiling: 6, targets: [ { cellId: 'target1', currentConnections: 0, availableConnectionReservations: 10, availableTargetUnits: 10 }, { cellId: 'target2', currentConnections: 0, availableConnectionReservations: 10, availableTargetUnits: 10 } ] }) assert.deepEqual( planned.map(({ cellId, quota, projectedConnections }) => ({ cellId, quota, projectedConnections })), [ { cellId: 'target1', quota: 2, projectedConnections: 3 }, { cellId: 'target2', quota: 2, projectedConnections: 3 } ] ) assert.throws(() => allocateTargetQuotas({ sourceAssignments: 4, sourceConnections: 20, requiredTargetUnits: 8, connectionCeiling: 4, targets: [ { cellId: 'target1', currentConnections: 0, availableConnectionReservations: 10, availableTargetUnits: 10 }, { cellId: 'target2', currentConnections: 0, availableConnectionReservations: 10, availableTargetUnits: 10 } ] }) ) assert.deepEqual( allocateTargetQuotas({ sourceAssignments: 1, sourceConnections: 700, requiredTargetUnits: 1, connectionCeiling: 1_000, targets: [ { cellId: 'target-600', currentConnections: 0, connectionCeiling: 600, availableConnectionReservations: 1, availableTargetUnits: 1 }, { cellId: 'target-1000', currentConnections: 0, connectionCeiling: 1_000, availableConnectionReservations: 1, availableTargetUnits: 1 } ] }).map(({ cellId, projectedConnections }) => ({ cellId, projectedConnections })), [ { cellId: 'target-600', projectedConnections: 699 }, { cellId: 'target-1000', projectedConnections: 700 } ] ) }) test('assumes every unbound source connection can land on one target', () => { const targets = [ { cellId: 'target1', currentConnections: 0, availableConnectionReservations: 10, availableTargetUnits: 10 }, { cellId: 'target2', currentConnections: 0, availableConnectionReservations: 10, availableTargetUnits: 10 } ] const planned = allocateTargetQuotas({ sourceAssignments: 4, sourceConnections: 9, requiredTargetUnits: 8, connectionCeiling: 8, targets }) assert.deepEqual( planned.map(({ quota, projectedConnections }) => ({ quota, projectedConnections })), [ { quota: 2, projectedConnections: 7 }, { quota: 2, projectedConnections: 7 } ] ) assert.throws(() => allocateTargetQuotas({ sourceAssignments: 4, sourceConnections: 9, requiredTargetUnits: 8, connectionCeiling: 7, targets }) ) }) test('serializes deterministic target quotas and completes after drain acceptance', async () => { await withTopology(async (file) => { const testHarness = harness() await runMultiTargetDeployment(config(file, 'execute'), testHarness.overrides) assert.deepEqual(testHarness.batches, [ ['target1', 2], ['target2', 2] ]) assert.deepEqual(testHarness.stateChanges.slice(0, 3), [ ['source', false], ['target1', true], ['target2', true] ]) assert.equal(testHarness.events.some((event) => event.event === 'source_drain_accepted'), true) assert.equal(testHarness.events.at(-1).event, 'multi_target_complete') }) }) test('uses fresh aggregate logs for a legacy source without runtime counts', async () => { await withTopology(async (file) => { const testHarness = harness({ legacySource: true }) await runMultiTargetDeployment(config(file), testHarness.overrides) assert.equal(testHarness.events.at(-1).sourceConnections, 5) }) }) test('rejects stale legacy source telemetry', async () => { await withTopology(async (file) => { const testHarness = harness({ legacySource: true, legacyMetricAgeMs: 90_001 }) await assert.rejects( runMultiTargetDeployment(config(file), testHarness.overrides), /runtime metrics are stale/ ) }) }) test('refuses a drain with an expiring lease and restores pre-drain admission', async () => { await withTopology(async (file) => { const testHarness = harness({ leaseRemainingMs: 599_999 }) await assert.rejects( runMultiTargetDeployment(config(file, 'execute'), testHarness.overrides), /insufficient time/ ) assert.deepEqual(testHarness.stateChanges.slice(-3), [ ['source', true], ['target1', false], ['target2', false] ]) assert.equal( testHarness.events.some((event) => event.event === 'source_drain_accepted'), false ) }) }) test('preserves forward recovery when target registration status is unavailable', async () => { await withTopology(async (file) => { const testHarness = harness({ failEvacuationStatus: true }) await assert.rejects( runMultiTargetDeployment(config(file, 'execute'), testHarness.overrides), /cannot prove zero target registrations/ ) assert.equal( testHarness.stateChanges.some(([cellId, enabled]) => cellId === 'source' && enabled), false ) assert.equal( testHarness.stateChanges.some( ([cellId, enabled]) => cellId.startsWith('target') && !enabled ), false ) assert.deepEqual(testHarness.events.at(-1), { event: 'multi_forward_recovery_required', targetRegistered: null, reason: 'registration_status_unavailable' }) }) }) test('audits partial migration state without requiring new migration headroom', async () => { await withTopology(async (file) => { const testHarness = harness({ capacitySourceAssignments: [1_000], capacityRequiredTargetUnits: [2_000] }) await runMultiTargetDeployment(config(file, 'audit'), testHarness.overrides) assert.equal(testHarness.capacityReads(), 0) assert.deepEqual(testHarness.events.at(-1), { event: 'multi_target_audit', inProgress: 4, targetRegistered: 0, registeredSourceActive: 0, registeredCompletable: 0, registeredTargetInactive: 0, completed: 0, blocked: 0, expiredUnregistered: 0, repairableExpiredUnregistered: 0, abortableExpiredUnregistered: 0, blockedExpiredUnregistered: 0, blockedExpiredOnNewerTargetAssignment: 0 }) }) }) test('never restores source admission after drain acceptance', async () => { await withTopology(async (file) => { const testHarness = harness({ failAfterDrain: true }) await assert.rejects( runMultiTargetDeployment(config(file, 'execute'), testHarness.overrides), /injected_completion_failure/ ) assert.equal(testHarness.events.some((event) => event.event === 'source_drain_accepted'), true) assert.equal( testHarness.stateChanges.some(([cellId, enabled]) => cellId === 'source' && enabled), false ) assert.equal(testHarness.events.at(-1).event, 'multi_forward_recovery_required') }) }) test('never restores source admission after the send transition becomes ambiguous', async () => { await withTopology(async (file) => { const testHarness = harness({ loseDrainSendResponse: true }) await assert.rejects( runMultiTargetDeployment(config(file, 'execute'), testHarness.overrides), /injected_send_transition_response_loss/ ) assert.equal( testHarness.stateChanges.some(([cellId, enabled]) => cellId === 'source' && enabled), false ) assert.equal(testHarness.events.at(-1).event, 'multi_forward_recovery_required') }) }) test('freezes forward recovery when a legacy drain response is ambiguous', async () => { await withTopology(async (file) => { const testHarness = harness({ loseDrainBeforeAccept: true }) await assert.rejects( runMultiTargetDeployment(config(file, 'execute'), testHarness.overrides), /injected_drain_response_loss/ ) assert.equal( testHarness.stateChanges.some(([cellId, enabled]) => cellId === 'source' && enabled), false ) await assert.rejects( runMultiTargetDeployment(config(file, 'recover-forward'), testHarness.overrides), /drain_application_receipt_missing/ ) assert.deepEqual(testHarness.drainGraces, [120_000]) }) }) test('resumes an exact prepared drain without allocating new migrations', async () => { await withTopology(async (file) => { const testHarness = harness({ preparedDrainAttempt: true, preexistingRegisteredMigrations: 2, sourceAssignments: 0, sourceRequiredTargetUnits: 0, selectorMembership: { existingOnly: ['source'], migrationOnly: ['target1', 'target2'], general: ['general'] } }) await runMultiTargetDeployment( config(file, 'recover-forward'), testHarness.overrides ) assert.deepEqual(testHarness.drainGraces, [120_000]) assert.deepEqual(testHarness.batches, []) assert.equal(testHarness.capacityReads(), 8) assert.equal( testHarness.events.some( (event) => event.event === 'source_prepared_drain_recovered' ), true ) assert.deepEqual(testHarness.events.at(-1), { event: 'multi_target_complete', sourceCellId: 'source' }) }) }) test('registers only remaining assignments before recovering a replacement drain', async () => { await withTopology(async (file) => { const testHarness = harness({ recoverableDrain: true, preexistingRegisteredMigrations: 1, selectorMembership: { existingOnly: ['source'], migrationOnly: ['target1', 'target2'], general: ['general'] } }) await runMultiTargetDeployment( config(file, 'recover-forward'), testHarness.overrides ) assert.deepEqual(testHarness.batches, [ ['target1', 2], ['target2', 2] ]) assert.equal(testHarness.capacityReads(), 8) assert.deepEqual(testHarness.drainGraces, [0]) const eventNames = testHarness.events.map((event) => event.event) assert.equal( testHarness.events.find( (event) => event.event === 'multi_forward_recovery_preflight' ).targetRegistered, 2 ) assert.ok( eventNames.lastIndexOf('multi_migration_batch') < eventNames.indexOf('multi_forward_recovery_ready_to_drain') ) assert.ok( eventNames.indexOf('multi_forward_recovery_ready_to_drain') < eventNames.indexOf('source_recovery_drain_accepted') ) }) }) test('refreshes registered migration leases before the recovery drain gate', async () => { await withTopology(async (file) => { const testHarness = harness({ recoverableDrain: true, preexistingRegisteredMigrations: 2, sourceAssignments: 0, sourceRequiredTargetUnits: 0, leaseRemainingMs: 599_999, refreshedLeaseRemainingMs: 900_000, selectorMembership: { existingOnly: ['source'], migrationOnly: ['target1', 'target2'], general: ['general'] } }) await runMultiTargetDeployment( config(file, 'recover-forward'), testHarness.overrides ) assert.deepEqual(testHarness.drainGraces, [0]) }) }) test('waits for catch-up migrations to gain durable target ownership', async () => { await withTopology(async (file) => { const testHarness = harness({ recoverableDrain: true, recoveryRegistrationDelayReads: 2, selectorMembership: { existingOnly: ['source'], migrationOnly: ['target1', 'target2'], general: ['general'] } }) await runMultiTargetDeployment( config(file, 'recover-forward'), testHarness.overrides ) const ownershipEvents = testHarness.events.filter( (event) => event.event === 'multi_recovery_target_ownership' ) assert.equal(ownershipEvents.length, 3) assert.equal(ownershipEvents[0].inProgress > ownershipEvents[0].targetRegistered, true) assert.equal(ownershipEvents.at(-1).inProgress, ownershipEvents.at(-1).targetRegistered) assert.deepEqual(testHarness.drainGraces, [0]) }) }) test('times out before drain when catch-up migrations remain unregistered', async () => { await withTopology(async (file) => { const testHarness = harness({ recoverableDrain: true, recoveryRegistrationDelayReads: Number.POSITIVE_INFINITY, selectorMembership: { existingOnly: ['source'], migrationOnly: ['target1', 'target2'], general: ['general'] } }) let now = 0 testHarness.overrides.now = () => now testHarness.overrides.wait = async (ms) => { now += ms } await assert.rejects( runMultiTargetDeployment( { ...config(file, 'recover-forward'), timeoutMs: 3 }, testHarness.overrides ), /timed out waiting for recovery target ownership/ ) assert.deepEqual(testHarness.drainGraces, []) }) }) test('drains a fully unregistered recovery within the reviewed bound', async () => { await withTopology(async (file) => { const testHarness = harness({ recoverableDrain: true, recoveryRegistrationDelayReads: Number.POSITIVE_INFINITY, selectorMembership: { existingOnly: ['source'], migrationOnly: ['target1', 'target2'], general: ['general'] } }) await runMultiTargetDeployment( { ...config(file, 'recover-forward'), unobservedConnectionBound: 4 }, testHarness.overrides ) assert.deepEqual(testHarness.drainGraces, [0]) assert.equal( testHarness.events.some( (event) => event.event === 'multi_recovery_bounded_unregistered' ), true ) }) }) test('drains mixed target registrations within the unobserved bound', async () => { await withTopology(async (file) => { const testHarness = harness({ recoverableDrain: true, preexistingRegisteredMigrations: 1, recoveryRegistrationDelayReads: Number.POSITIVE_INFINITY, selectorMembership: { existingOnly: ['source'], migrationOnly: ['target1', 'target2'], general: ['general'] } }) await runMultiTargetDeployment( { ...config(file, 'recover-forward'), unobservedConnectionBound: 2 }, testHarness.overrides ) assert.deepEqual(testHarness.drainGraces, [0]) const event = testHarness.events.find( (entry) => entry.event === 'multi_recovery_bounded_mixed_registration' ) assert.equal(event.unregistered, 2) assert.equal(event.targetRegistered, 2) }) }) test('reissues a proven non-delivered recovery drain and settles bounded offline clients', async () => { await withTopology(async (file) => { const testHarness = harness({ recoverableDrain: true, recoveryAlreadyAttempted: true, preexistingRegisteredMigrations: 1, unregisteredTargetMigrations: 1, selectorMembership: { existingOnly: ['source'], migrationOnly: ['target1', 'target2'], general: ['general'] } }) await runMultiTargetDeployment( { ...config(file, 'recover-forward'), unobservedConnectionBound: 2 }, testHarness.overrides ) assert.deepEqual(testHarness.drainGraces, [0]) assert.equal( testHarness.events.some( (event) => event.event === 'source_recovery_drain_reissued_after_non_delivery' ), true ) assert.equal( testHarness.events.some( (event) => event.event === 'multi_migration_bounded_offline_complete' ), true ) }) }) test('rejects mixed target registrations above the unobserved bound', async () => { await withTopology(async (file) => { const testHarness = harness({ recoverableDrain: true, preexistingRegisteredMigrations: 1, recoveryRegistrationDelayReads: Number.POSITIVE_INFINITY, selectorMembership: { existingOnly: ['source'], migrationOnly: ['target1', 'target2'], general: ['general'] } }) let now = 0 testHarness.overrides.now = () => now testHarness.overrides.wait = async (ms) => { now += ms } await assert.rejects( runMultiTargetDeployment( { ...config(file, 'recover-forward'), timeoutMs: 3, unobservedConnectionBound: 1 }, testHarness.overrides ), /timed out waiting for recovery target ownership/ ) assert.deepEqual(testHarness.drainGraces, []) }) }) test('times out before drain while a target registration remains source-active', async () => { await withTopology(async (file) => { const testHarness = harness({ recoverableDrain: true, preexistingRegisteredMigrations: 1, registeredSourceActive: 1, recoveryRegistrationDelayReads: Number.POSITIVE_INFINITY, selectorMembership: { existingOnly: ['source'], migrationOnly: ['target1', 'target2'], general: ['general'] } }) let now = 0 testHarness.overrides.now = () => now testHarness.overrides.wait = async (ms) => { now += ms } await assert.rejects( runMultiTargetDeployment( { ...config(file, 'recover-forward'), timeoutMs: 3, unobservedConnectionBound: 2 }, testHarness.overrides ), /timed out waiting for recovery target ownership/ ) assert.deepEqual(testHarness.drainGraces, []) }) }) test('allows recovery drain with durable offline target registrations', async () => { await withTopology(async (file) => { const testHarness = harness({ recoverableDrain: true, offlineTargetMigrations: 1, selectorMembership: { existingOnly: ['source'], migrationOnly: ['target1', 'target2'], general: ['general'] } }) await runMultiTargetDeployment( config(file, 'recover-forward'), testHarness.overrides ) assert.deepEqual(testHarness.drainGraces, [0]) }) }) test('catches up source assignments that arrive during recovery publication', async () => { await withTopology(async (file) => { const testHarness = harness({ recoverableDrain: true, sourceAssignments: 5, sourceRequiredTargetUnits: 10, capacitySourceAssignments: [4, 4, 4, 4, 1, 1, 1, 1], capacityRequiredTargetUnits: [8, 8, 8, 8, 2, 2, 2, 2], selectorMembership: { existingOnly: ['source'], migrationOnly: ['target1', 'target2'], general: ['general'] } }) await runMultiTargetDeployment( config(file, 'recover-forward'), testHarness.overrides ) assert.equal( testHarness.batches.reduce((total, [, limit]) => total + limit, 0), 5 ) assert.equal( testHarness.events.some( (event) => event.event === 'multi_forward_recovery_catch_up' ), true ) assert.deepEqual( testHarness.events .filter((event) => event.event === 'multi_target_preflight') .map((event) => ({ sourceConnections: event.sourceConnections, observedSourceConnections: event.observedSourceConnections })), [ { sourceConnections: 5, observedSourceConnections: 5 }, { sourceConnections: 1, observedSourceConnections: 5 } ] ) assert.deepEqual(testHarness.drainGraces, [0]) }) }) test('replans recover-forward after transactional target headroom rejection', async () => { await withTopology(async (file) => { const testHarness = harness({ recoverableDrain: true, headroomFailureTarget: 'target1', selectorMembership: { existingOnly: ['source'], migrationOnly: ['target1', 'target2'], general: ['general'] } }) await runMultiTargetDeployment( config(file, 'recover-forward'), testHarness.overrides ) assert.equal( testHarness.events.some( (event) => event.event === 'multi_recovery_target_headroom_paused' && event.targetCellId === 'target1' ), true ) assert.equal( testHarness.events.some( (event) => event.event === 'multi_forward_recovery_catch_up' ), true ) assert.deepEqual(testHarness.drainGraces, [0]) }) }) test('uses conservative changing capacity samples during recovery', async () => { await withTopology(async (file) => { const testHarness = harness({ recoverableDrain: true, sourceAssignments: 5, sourceRequiredTargetUnits: 10, capacitySourceAssignments: [5, 4, 5, 4], capacityRequiredTargetUnits: [10, 8, 10, 8], selectorMembership: { existingOnly: ['source'], migrationOnly: ['target1', 'target2'], general: ['general'] } }) await runMultiTargetDeployment( config(file, 'recover-forward'), testHarness.overrides ) assert.equal( testHarness.batches.reduce((total, [, limit]) => total + limit, 0), 5 ) assert.deepEqual(testHarness.drainGraces, [0]) }) }) test('requires every final recovery sample to prove zero source assignments', async () => { await withTopology(async (file) => { const testHarness = harness({ recoverableDrain: true, preexistingRegisteredMigrations: 2, sourceAssignments: 1, sourceRequiredTargetUnits: 2, capacitySourceAssignments: [1, 1, 1, 1, 0, 0, 0, 1], capacityRequiredTargetUnits: [2, 2, 2, 2, 0, 0, 0, 2], selectorMembership: { existingOnly: ['source'], migrationOnly: ['target1', 'target2'], general: ['general'] } }) await runMultiTargetDeployment( config(file, 'recover-forward'), testHarness.overrides ) assert.equal( testHarness.events.some( (event) => event.event === 'multi_forward_recovery_catch_up' ), true ) assert.deepEqual(testHarness.drainGraces, [0]) }) }) test('stops after bounded recovery catch-up cannot quiesce the source', async () => { await withTopology(async (file) => { const testHarness = harness({ recoverableDrain: true, sourceAssignments: 10, sourceRequiredTargetUnits: 20, capacitySourceAssignments: Array.from({ length: 40 }, () => 1), capacityRequiredTargetUnits: Array.from({ length: 40 }, () => 2), selectorMembership: { existingOnly: ['source'], migrationOnly: ['target1', 'target2'], general: ['general'] } }) await assert.rejects( runMultiTargetDeployment( config(file, 'recover-forward'), testHarness.overrides ), /did not quiesce within bounded recovery catch-up/ ) assert.equal( testHarness.batches.reduce((total, [, limit]) => total + limit, 0), 5 ) assert.deepEqual(testHarness.drainGraces, []) }) }) test('settles a drained source while registered target users remain offline', async () => { await withTopology(async (file) => { const testHarness = harness({ fence: true, allowPreFenceCompletion: true, offlineTargetMigrations: 1, selectorMembership: { existingOnly: ['source'], migrationOnly: ['target1', 'target2'], general: ['general'] } }) await runMultiTargetDeployment( config(file, 'recover-forward'), testHarness.overrides ) assert.deepEqual(testHarness.drainGraces, []) assert.deepEqual(testHarness.events.at(-1), { event: 'multi_target_complete', sourceCellId: 'source' }) }) }) test('fences a quiescent disabled source and completes through stale-source checks', async () => { await withTopology(async (file) => { const testHarness = harness({ fence: true }) const authorizationByOrigin = new Map() const fetch = testHarness.overrides.fetch testHarness.overrides.fetch = async (url, options) => { const parsed = new URL(url) if (parsed.pathname.startsWith('/v1/admin/')) { authorizationByOrigin.set( parsed.origin, new Set([ ...(authorizationByOrigin.get(parsed.origin) ?? []), options?.headers?.authorization ]) ) } return await fetch(url, options) } await runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides) assert.equal(testHarness.fencedCompletions(), 2) assert.deepEqual( authorizationByOrigin.get('https://relay.example.com'), new Set(['Bearer aaa.bbb.ccc', 'Bearer ddd.eee.fff']) ) assert.equal(authorizationByOrigin.has('https://a.relay.example.com'), false) assert.equal(authorizationByOrigin.has('https://b.relay.example.com'), false) assert.equal(authorizationByOrigin.has('https://c.relay.example.com'), false) assert.deepEqual(testHarness.events.at(-1), { event: 'source_fenced', sourceCellId: 'source', targetSize: 0 }) }) }) test('adopts an already-fenced source only through the legacy no-op path', async () => { await withTopology(async (file) => { const testHarness = harness({ fence: true, alreadyFencedSource: true }) await runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides) assert.equal(testHarness.fencedCompletions(), 2) assert.deepEqual( testHarness.events.find( ({ event }) => event === 'terraform_cell_fence_legacy_adopted' ), { event: 'terraform_cell_fence_legacy_adopted', cellId: 'source' } ) assert.deepEqual(testHarness.events.at(-1), { event: 'source_fenced', sourceCellId: 'source', targetSize: 0 }) }) }) test('refuses to attest a fence when selected targets omit an outgoing migration', async () => { await withTopology(async (file) => { const testHarness = harness({ fence: true, alreadyFencedSource: true, sourceOutgoingMigrations: 5 }) await assert.rejects( runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), /full migration coverage/ ) assert.equal(testHarness.fencedCompletions(), 0) }) }) test('clears the complete 156-row legacy backlog before reporting the source fenced', async () => { const rollout = topology() const additionalTarget = (id, hostname) => ({ ...rollout.target2, origin: `https://${hostname}.relay.example.com`, zone: `us-central1-${hostname}`, mig_name: `relay-${hostname}`, instance_group: `https://compute.example/instanceGroups/relay-${hostname}`, backend_name: `relay-${hostname}`, backend_id: `https://compute.example/backendServices/relay-${hostname}`, generation_identity: `https://compute.example/instanceTemplates/relay-${hostname}-abc`, image: `us-central1-docker.pkg.dev/project/repo/relay@${digest(id)}` }) rollout.target3 = additionalTarget('e', 'e') rollout.target4 = additionalTarget('f', 'f') rollout.target5 = additionalTarget('1', 'g') rollout.target6 = additionalTarget('2', 'h') const migrationCounts = { target1: 58, target2: 63, target3: 24, target4: 10, target5: 1, target6: 0 } await withTopology(async (file) => { const testHarness = harness({ fence: true, alreadyFencedSource: true, offlineTargetMigrationsByTarget: migrationCounts, topologyValue: rollout }) const fenceConfig = config(file, 'fence-source') fenceConfig.targetCellIds = Object.keys(migrationCounts) await runMultiTargetDeployment(fenceConfig, testHarness.overrides) const sourceFencedIndex = testHarness.timeline.findIndex( ({ kind, event }) => kind === 'event' && event.event === 'source_fenced' ) assert.notEqual(sourceFencedIndex, -1) let observedInitialTotal = 0 for (const [targetCellId, initialCount] of Object.entries(migrationCounts)) { const initialIndex = testHarness.timeline.findIndex( (entry) => entry.kind === 'evacuation_status' && entry.targetCellId === targetCellId && entry.inProgress === initialCount && entry.registeredTargetInactive === initialCount ) const clearedIndex = testHarness.timeline.findIndex( (entry) => entry.kind === 'evacuation_status' && entry.targetCellId === targetCellId && entry.inProgress === 0 ) assert.notEqual(initialIndex, -1) observedInitialTotal += testHarness.timeline[initialIndex].inProgress if (initialCount > 0) assert.ok(clearedIndex > initialIndex) else assert.ok(clearedIndex >= initialIndex) assert.ok(clearedIndex < sourceFencedIndex) } assert.equal(observedInitialTotal, 156) assert.equal(testHarness.fencedCompletions(), 6) }, rollout) }) test('refuses legacy adoption when the live MIG template changed', async () => { await withTopology(async (file) => { const testHarness = harness({ fence: true, alreadyFencedSource: true, liveMigTemplateOverrides: { source: `${topology().source.generation_identity}-other` } }) await assert.rejects( runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), /source MIG fence topology is unsafe/ ) assert.equal(testHarness.fencedCompletions(), 0) }) }) test('fences with reviewed 600 templates during the declared 1000 rollout', async () => { const rollout = topology() for (const [cellId, cell] of Object.entries(rollout)) { cell.connection_unobserved_bound = 60 if (['target1', 'target2'].includes(cellId)) cell.connection_hard_cap = 1_000 } await withTopology(async (file) => { const testHarness = harness({ fence: true, templateUnobservedBound: 60, directorUnobservedBound: 60 }) const fenceConfig = config(file, 'fence-source') fenceConfig.unobservedConnectionBound = 60 await runMultiTargetDeployment(fenceConfig, testHarness.overrides) assert.equal(testHarness.fencedCompletions(), 2) }, rollout) }) test('fences exact configured capacity when the saved Terraform output is legacy', async () => { const rollout = topology() for (const cell of Object.values(rollout)) { delete cell.connection_hard_cap delete cell.connection_unobserved_bound } await withTopology(async (file) => { const testHarness = harness({ fence: true, templateUnobservedBound: 60, directorUnobservedBound: 60 }) const fenceConfig = config(file, 'fence-source') fenceConfig.unobservedConnectionBound = 60 await runMultiTargetDeployment(fenceConfig, testHarness.overrides) assert.equal(testHarness.fencedCompletions(), 2) }, rollout) }) test('refuses legacy Terraform output when live capacity differs from broker config', async () => { const rollout = topology() for (const cell of Object.values(rollout)) { delete cell.connection_hard_cap delete cell.connection_unobserved_bound } await withTopology(async (file) => { const testHarness = harness({ fence: true }) const fenceConfig = config(file, 'fence-source') fenceConfig.unobservedConnectionBound = 60 await assert.rejects( runMultiTargetDeployment(fenceConfig, testHarness.overrides), /instance template capacity differs from Terraform/ ) assert.equal(testHarness.fencedCompletions(), 0) }, rollout) }) test('refuses explicit null capacity as a legacy Terraform output', async () => { const rollout = topology() for (const cell of Object.values(rollout)) { cell.connection_hard_cap = null cell.connection_unobserved_bound = null } await withTopology(async (file) => { const testHarness = harness({ fence: true, templateUnobservedBound: 60, directorUnobservedBound: 60 }) const fenceConfig = config(file, 'fence-source') fenceConfig.unobservedConnectionBound = 60 await assert.rejects( runMultiTargetDeployment(fenceConfig, testHarness.overrides), /instance template capacity differs from Terraform/ ) assert.equal(testHarness.fencedCompletions(), 0) }, rollout) }) test('refuses a mixed legacy and declared capacity topology', async () => { const rollout = topology() for (const cell of Object.values(rollout)) { delete cell.connection_hard_cap delete cell.connection_unobserved_bound } rollout.general.connection_hard_cap = null rollout.general.connection_unobserved_bound = null await withTopology(async (file) => { const testHarness = harness({ fence: true, templateUnobservedBound: 60, directorUnobservedBound: 60 }) const fenceConfig = config(file, 'fence-source') fenceConfig.unobservedConnectionBound = 60 await assert.rejects( runMultiTargetDeployment(fenceConfig, testHarness.overrides), /instance template capacity differs from Terraform/ ) assert.equal(testHarness.fencedCompletions(), 0) }, rollout) }) test('fences after the active template reaches the declared 1000 capacity', async () => { const rollout = topology() for (const cellId of Object.keys(rollout)) { rollout[cellId].connection_hard_cap = 1_000 } await withTopology(async (file) => { const testHarness = harness({ fence: true, templateHardCap: 1_000, directorHardCap: 1_000 }) await runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides) assert.equal(testHarness.fencedCompletions(), 2) }, rollout) }) test('refuses a rollout predecessor whose template has the wrong bound', async () => { const rollout = topology() for (const cellId of ['target1', 'target2']) { rollout[cellId].connection_hard_cap = 1_000 rollout[cellId].connection_unobserved_bound = 60 } await withTopology(async (file) => { const testHarness = harness({ fence: true }) await assert.rejects( runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), /instance template capacity is outside reviewed rollout/ ) assert.equal(testHarness.fencedCompletions(), 0) }, rollout) }) test('refuses a rollout predecessor when the director reports another cap', async () => { const rollout = topology() for (const cellId of ['target1', 'target2']) { rollout[cellId].connection_hard_cap = 1_000 } await withTopology(async (file) => { const testHarness = harness({ fence: true, directorCapacityOverrides: { target1: { hardCap: 1_000, unobservedBound: 40 } } }) await assert.rejects( runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), /connection capacity differs from Terraform/ ) assert.equal(testHarness.fencedCompletions(), 0) }, rollout) }) test('preserves direct target runtime reads outside fence mode', async () => { await withTopology(async (file) => { const testHarness = harness({}) const fetch = testHarness.overrides.fetch const authorizationByOrigin = new Map() testHarness.overrides.fetch = async (url, options) => { const parsed = new URL(url) if (parsed.pathname.startsWith('/v1/admin/')) { authorizationByOrigin.set( parsed.origin, new Set([ ...(authorizationByOrigin.get(parsed.origin) ?? []), options?.headers?.authorization ]) ) } return await fetch(url, options) } await runMultiTargetDeployment(config(file, 'preflight'), testHarness.overrides) assert.deepEqual( authorizationByOrigin.get('https://relay.example.com'), new Set(['Bearer aaa.bbb.ccc']) ) assert.deepEqual( authorizationByOrigin.get('https://b.relay.example.com'), new Set(['Bearer aaa.bbb.ccc']) ) assert.deepEqual( authorizationByOrigin.get('https://c.relay.example.com'), new Set(['Bearer aaa.bbb.ccc']) ) }) }) test('refuses to fence when a cell hostname routes to the wrong backend', async () => { await withTopology(async (file) => { const testHarness = harness({ fence: true, misroutedCell: 'target1' }) await assert.rejects( runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), /retained route topology mismatch/ ) assert.equal(testHarness.fencedCompletions(), 0) }) }) test('refuses to fence when a cell route overrides the reviewed backend', async () => { await withTopology(async (file) => { const testHarness = harness({ fence: true, routeOverrideCell: 'target1' }) await assert.rejects( runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), /retained route topology mismatch/ ) assert.equal(testHarness.fencedCompletions(), 0) }) }) test('refuses to fence when the live HTTPS frontend uses another URL map', async () => { await withTopology(async (file) => { const testHarness = harness({ fence: true, frontendMisbound: true }) await assert.rejects( runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), /live frontend topology mismatch/ ) assert.equal(testHarness.fencedCompletions(), 0) }) }) test('refuses to fence when the fresh source heartbeat reports active work', async () => { await withTopology(async (file) => { const testHarness = harness({ fence: true, sourceObservedRequests: 1 }) await assert.rejects( runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), /source fencing requires zero source-owned work/ ) assert.equal(testHarness.fencedCompletions(), 0) }) }) test('fences a quiescent source while registered target users remain offline', async () => { await withTopology(async (file) => { const testHarness = harness({ fence: true, alreadyFencedSource: true, offlineTargetMigrations: 1 }) await runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides) assert.equal(testHarness.fencedCompletions(), 2) assert.deepEqual(testHarness.events.at(-1), { event: 'source_fenced', sourceCellId: 'source', targetSize: 0 }) }) }) test('refuses to fence source-active or unregistered migrations', async () => { for (const migrationState of [ { registeredSourceActive: 1 }, { unregisteredTargetMigrations: 1 } ]) { await withTopology(async (file) => { const testHarness = harness({ fence: true, ...migrationState }) await assert.rejects( runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), /full migration coverage and durable target ownership/ ) }) } }) test('resumes fenced completion after losing the resize response', async () => { await withTopology(async (file) => { const testHarness = harness({ fence: true, loseResizeResponse: true, resumeNeedsPreApply: true }) await assert.rejects( runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides), /injected_resize_response_loss/ ) await runMultiTargetDeployment(config(file, 'fence-source'), testHarness.overrides) assert.equal(testHarness.fencedCompletions(), 2) assert.deepEqual(testHarness.events.at(-1), { event: 'source_fenced', sourceCellId: 'source', targetSize: 0 }) }) }) test('records a proven pre-apply Terraform fence abort', async () => { await withTopology(async (file) => { const testHarness = harness({ existingFenceAttempt: true }) await runMultiTargetDeployment( config(file, 'abort-fence-source'), testHarness.overrides ) assert.deepEqual(testHarness.events.at(-1), { event: 'terraform_fence_aborted_before_apply', cellId: 'source' }) }) }) test('fences a failed registered target before aggregate supersession', async () => { await withTopology(async (file) => { const testHarness = harness({ supersede: true }) await runMultiTargetDeployment( config(file, 'supersede-target'), testHarness.overrides ) assert.equal( testHarness.stateChanges.some( ([cellId, enabled]) => cellId === 'target2' && enabled ), true ) assert.deepEqual(testHarness.events.at(-1), { event: 'registered_target_superseded', sourceCellId: 'source', failedTargetCellId: 'target1', replacementTargetCellId: 'target2', superseded: 2, remainingUnregistered: 0 }) assert.equal(testHarness.runtimeInspections.get('target2') ?? 0, 0) }) }) test('fails closed when cleanup wins before aggregate supersession selects rows', async () => { await withTopology(async (file) => { const testHarness = harness({ supersede: true, cleanupBeforeSupersession: true }) await assert.rejects( runMultiTargetDeployment(config(file, 'supersede-target'), testHarness.overrides), /registered target supersession count changed/ ) assert.deepEqual(testHarness.events, []) }) }) test('does not accept a capacity predecessor for target supersession', async () => { const rollout = topology() rollout.target2.connection_hard_cap = 1_000 await withTopology(async (file) => { const testHarness = harness({ supersede: true }) await assert.rejects( runMultiTargetDeployment( config(file, 'supersede-target'), testHarness.overrides ), /instance template capacity is outside reviewed rollout/ ) }, rollout) }) test('does not accept capacity-bearing templates from legacy output for supersession', async () => { const rollout = topology() for (const cell of Object.values(rollout)) { delete cell.connection_hard_cap delete cell.connection_unobserved_bound } await withTopology(async (file) => { const testHarness = harness({ supersede: true }) await assert.rejects( runMultiTargetDeployment( config(file, 'supersede-target'), testHarness.overrides ), /instance template capacity differs from Terraform/ ) }, rollout) }) test('replaces a superseded unuploaded fence attempt before target fencing', async () => { await withTopology(async (file) => { const testHarness = harness({ supersede: true, supersededFenceAttempt: true }) await runMultiTargetDeployment( config(file, 'supersede-target'), testHarness.overrides ) assert.deepEqual(testHarness.events[0], { event: 'terraform_fence_superseded_before_upload', cellId: 'target1', previousFenceCommit: 'b'.repeat(40), fenceCommit: 'a'.repeat(40) }) assert.equal(testHarness.events.at(-1).event, 'registered_target_superseded') }) }) test('recovers a pinned completed older fence before registered supersession', async () => { await withTopology(async (file) => { const testHarness = harness({ supersede: true, completedFenceAttempt: true }) const deploymentConfig = config(file, 'supersede-target') deploymentConfig.completedFenceRecovery = { attemptId: '44444444-4444-4444-8444-444444444444', fenceCommit: 'b'.repeat(40), gceOperation: 'operation-1', terraformStateSerial: 7, planObjectGeneration: '123456789', terraformStateObjectGeneration: '222222222', terraformStateObjectSha256: 'c'.repeat(64), principalEmail: 'fence-broker@example.gserviceaccount.com' } await runMultiTargetDeployment(deploymentConfig, testHarness.overrides) assert.equal( testHarness.events[0].event, 'terraform_cell_fence_completed_attempt_recovered' ) assert.equal(testHarness.events.at(-1).event, 'registered_target_superseded') }) }) test('fails supersession on stale replacement heartbeat without calling its admin API', async () => { await withTopology(async (file) => { const testHarness = harness({ supersede: true, replacementHeartbeatFresh: false }) await assert.rejects( runMultiTargetDeployment(config(file, 'supersede-target'), testHarness.overrides), /no fresh matching director runtime snapshot/ ) assert.equal(testHarness.runtimeInspections.get('target2') ?? 0, 0) assert.deepEqual(testHarness.events, []) }) }) test('fails supersession on mismatched director runtime without calling cell admin', async () => { await withTopology(async (file) => { const testHarness = harness({ supersede: true, replacementRuntimeCellUrl: 'https://wrong.relay.example.com' }) await assert.rejects( runMultiTargetDeployment(config(file, 'supersede-target'), testHarness.overrides), /no fresh matching director runtime snapshot/ ) assert.equal(testHarness.runtimeInspections.get('target2') ?? 0, 0) assert.deepEqual(testHarness.events, []) }) }) test('reads the broker mutation token without treating the audience as the environment', async () => { await withTopology(async (file) => { const testHarness = harness({ supersede: true }) delete testHarness.overrides.mutationIdentityToken const previous = process.env.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN process.env.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN = 'ddd.eee.fff' try { await runMultiTargetDeployment( config(file, 'supersede-target'), testHarness.overrides ) } finally { if (previous === undefined) { delete process.env.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN } else { process.env.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN = previous } } assert.equal(testHarness.events.at(-1).event, 'registered_target_superseded') }) }) test('fails closed when the broker child has no mutation token', async () => { await withTopology(async (file) => { const testHarness = harness({ supersede: true }) delete testHarness.overrides.mutationIdentityToken const previous = process.env.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN delete process.env.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN try { await assert.rejects( runMultiTargetDeployment( config(file, 'supersede-target'), testHarness.overrides ), /requires a broker mutation identity token/ ) } finally { if (previous !== undefined) { process.env.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN = previous } } assert.equal(testHarness.events.length, 0) }) }) test('uses Terraform fencing for selector-era target supersession', async () => { await withTopology(async (file) => { const testHarness = harness({ supersede: true, selectorMembership: { existingOnly: ['source', 'target1'], migrationOnly: ['target2'], general: ['general'] } }) await runMultiTargetDeployment( config(file, 'supersede-target'), testHarness.overrides ) assert.deepEqual(testHarness.stateChanges, []) assert.equal(testHarness.events.at(-1).event, 'registered_target_superseded') }) }) test('resumes failed-target supersession after losing the resize response', async () => { await withTopology(async (file) => { const testHarness = harness({ supersede: true, loseResizeResponse: true }) await assert.rejects( runMultiTargetDeployment(config(file, 'supersede-target'), testHarness.overrides), /injected_resize_response_loss/ ) await runMultiTargetDeployment( config(file, 'supersede-target'), testHarness.overrides ) assert.equal(testHarness.events.at(-1).event, 'registered_target_superseded') }) }) test('refuses to fence a failed target while its admission remains enabled', async () => { await withTopology(async (file) => { const testHarness = harness({ supersede: true, failedTargetEnabled: true }) await assert.rejects( runMultiTargetDeployment(config(file, 'supersede-target'), testHarness.overrides), /failed target admission must be disabled/ ) assert.equal(testHarness.events.length, 0) }) }) test('retries until two complete target-capacity rounds agree', async () => { await withTopology(async (file) => { const testHarness = harness({ capacitySourceAssignments: [4, 3] }) await runMultiTargetDeployment(config(file), testHarness.overrides) assert.equal(testHarness.capacityReads(), 8) assert.equal(testHarness.events.at(-1).sourceAssignments, 4) }) }) test('fails closed after bounded inconsistent target-capacity rounds', async () => { await withTopology(async (file) => { const testHarness = harness({ capacitySourceAssignments: Array.from( { length: 12 }, (_, index) => index % 2 === 0 ? 4 : 3 ) }) await assert.rejects( runMultiTargetDeployment(config(file), testHarness.overrides), /target capacity snapshots disagree/ ) assert.equal(testHarness.capacityReads(), 12) assert.deepEqual(testHarness.stateChanges, []) }) }) test('emits aggregate capacity inputs before rejecting insufficient headroom', async () => { await withTopology(async (file) => { const testHarness = harness({ capacityRequiredTargetUnits: Array.from({ length: 4 }, () => 20_000) }) await assert.rejects( runMultiTargetDeployment(config(file), testHarness.overrides), /multi-target connection or request-unit headroom exhausted/ ) const snapshot = testHarness.events.at(-1) assert.equal(snapshot.event, 'multi_target_capacity_snapshot') assert.equal(snapshot.sourceConnections, 5) assert.equal(snapshot.sourceAssignments, 4) assert.equal(snapshot.requiredTargetUnits, 20_000) assert.deepEqual(snapshot.targets, [ { cellId: 'target1', currentConnections: 0, availableConnectionReservations: 460, availableTargetUnits: 4_000 }, { cellId: 'target2', currentConnections: 0, availableConnectionReservations: 460, availableTargetUnits: 4_000 } ]) }) }) test('rejects quotas above enforced target reservation headroom', () => { assert.throws( () => allocateTargetQuotas({ sourceAssignments: 3, sourceConnections: 3, requiredTargetUnits: 3, connectionCeiling: 600, targets: [ { cellId: 'target1', currentConnections: 0, availableConnectionReservations: 1, availableTargetUnits: 10 }, { cellId: 'target2', currentConnections: 0, availableConnectionReservations: 1, availableTargetUnits: 10 } ] }), /multi-target connection or request-unit headroom exhausted/ ) })