test: remove classification inventories and verify settled boundaries (#32216)

Co-authored-by: OrcaWin <293788423+OrcaWin@users.noreply.github.com>
This commit is contained in:
OrcaWin
2026-10-23 00:03:24 -07:00
committed by GitHub
co-authored by OrcaWin
parent a5b2543d85
commit 0841e09b64
8 changed files with 129 additions and 709 deletions
@@ -1,183 +0,0 @@
import { readdirSync, readFileSync } from 'node:fs'
import { join, sep } from 'node:path'
import { fileURLToPath } from 'node:url'
import ts from 'typescript-api'
import { afterEach, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import type { RelayCellConfig } from './config.js'
import { openInMemoryRelayDatabase, type RelayDatabase } from './database.js'
// Step 6 (design §3.5): a phone on a `phonePath=memory` cell writes no `confirmation:` or
// `splice:` lease. Every reader of the lease table is classified by why that is safe; a new
// reader fails this test until someone decides. While the cell is recorded as reserve its
// control holds no lease either, so step 5's guards already cover it; after a flip back a
// memory phone looks like today's phone once its 90 s lease has expired.
const ACTIVITY_LEASE_READERS: Record<string, string> = {
// Reads control leases only, so phone leases never change the answer.
'assignment-store.ts:activateControlOnce': 'control-only',
'assignment-store.ts:reconcileIdleRegionalRehome': 'control-only',
'assignment-store.ts:renewTransactionalControlActivity': 'control-only',
'assignment-store.ts:startRegionalRehomeCandidate': 'control-only',
'control-renewal-statement.ts:CONTROL_RENEWAL_BATCH_SQL': 'control-only',
'idle-regional-rehome-selection.ts:selectIdleRegionalRehomes': 'control-only',
// Refuses or skips a cell recorded as reserve (or an unreadable set).
'assignment-store.ts:cellDeploymentStatus': 'refuses-or-skips-reserve-cells',
'assignment-store.ts:cellEvacuationCapacity': 'refuses-or-skips-reserve-cells',
'assignment-store.ts:releaseExpiredActivity': 'refuses-or-skips-reserve-cells',
'assignment-store.ts:releaseExpiredActivityLeases': 'refuses-or-skips-reserve-cells',
'assignment-store.ts:startEvacuation': 'refuses-or-skips-reserve-cells',
// Its only caller drops candidates whose source or target is a reserve cell.
'idle-regional-rehome-selection.ts:sourceRequestUnits': 'refuses-or-skips-reserve-cells',
// Only hosts in an open drain, migration or rehome, which no reserve cell can have:
// setCellAdmitMode refuses reserve while one is open, and every start refuses a reserve cell.
'assignment-store.ts:activeCellMigrations': 'open-flow-only',
'assignment-store.ts:additionalActivityCellIds': 'open-flow-only',
'assignment-store.ts:assertCellReservationAccounting': 'open-flow-only',
'assignment-store.ts:beginCellDrainSend': 'open-flow-only',
'assignment-store.ts:beginCellDrainSendOnce': 'open-flow-only',
'assignment-store.ts:completeReadyEvacuations': 'open-flow-only',
'assignment-store.ts:completeReadyRegionalRehomes': 'open-flow-only',
'assignment-store.ts:reconcileDrainMigrationAccounting': 'open-flow-only',
'assignment-store.ts:reconcileReservationAccounting': 'open-flow-only',
'assignment-store.ts:refreshDrainMigrationLeasesOnce': 'open-flow-only',
'registered-migration-abandonment.ts:ABANDONED_REGISTERED_MIGRATION': 'open-flow-only',
// One host's rows under lock; each caller is classified in the reserve-mode census.
'assignment-store.ts:lockAssignmentActivities': 'host-lock',
// Metrics and previews: they report, and act on nothing.
'assignment/assignment-inventory-snapshot.ts:readAssignmentInventorySnapshot': 'report-only',
'region-correction-preview.ts:previewRegionalRehomeEligibility': 'report-only',
'registered-migration-inventory.ts:readRegisteredMigrationInventory': 'report-only',
// Step 5's existing-only stranded release, fixed separately in #30609 before the first flip.
'assignment-store.ts:assignmentStrandedOnUnservedCell': 'step-5-stranded-release'
}
// A read is FROM, JOIN or USING on the table; `DELETE FROM` alone only writes. Not caught: a
// second read inside an already listed function, and counters derived from leases elsewhere.
const LEASE_TABLE = 'relay_assignment_activity_leases'
// Bare table-name values that never reach SQL.
const BARE_TABLE_NAME_USES: Record<string, string> = {
'database.ts:POSTGRES_TRANSACTION_PHASES': 'labels a failed statement by the table it names'
}
const LEASE_READ = /(?<!DELETE\s+)\b(?:FROM|JOIN|USING)\s+relay_assignment_activity_leases\b/
const CELLS: RelayCellConfig[] = [
{ id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
{ id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
]
describe('activity-lease reader census', () => {
let database: RelayDatabase | undefined
afterEach(async () => {
await database?.close()
database = undefined
})
it('classifies every reader of the activity-lease table', () => {
const { readers, bareTableNames } = activityLeaseReaders()
// A bare name could reach `FROM ${table}`, which the scan cannot see: write the table into the SQL.
expect(bareTableNames.filter((use) => !(use in BARE_TABLE_NAME_USES))).toEqual([])
expect(readers.filter((name) => !(name in ACTIVITY_LEASE_READERS))).toEqual([])
expect(Object.keys(ACTIVITY_LEASE_READERS).filter((name) => !readers.includes(name))).toEqual([])
})
async function setup(now: () => number) {
database = await openInMemoryRelayDatabase()
const store = new RelayAssignmentStore(database, now)
await store.reconcileCells(CELLS)
return store
}
// A host whose source cell kept no lease for it, as a memory cell's control and phones do.
async function registeredEvacuationWithoutSourceLeases(store: RelayAssignmentStore) {
const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
const first = await store.assign(identity)
expect(first.cellId).toBe('cell-a')
const migration = await store.startEvacuation(identity, 'cell-b')
await store.activateControl(identity, {
cellId: 'cell-b',
assignmentEpoch: migration.assignmentEpoch,
generation: 1
})
await store.markMigrationTargetRegistered(identity, {
cellId: 'cell-b',
assignmentEpoch: migration.assignmentEpoch
})
await store.releaseActivity(identity, `control-pending:${first.assignmentEpoch}`)
return { identity, migration }
}
it('completes no evacuation from a reserve source, though it holds no source lease', async () => {
const now = () => 1_000
const store = await setup(now)
const { identity, migration } = await registeredEvacuationWithoutSourceLeases(store)
// setCellAdmitMode refuses this with the migration open; written directly to reach the second guard.
await database!.query(
`INSERT INTO relay_cell_admit_modes (cell_id, admit_mode, updated_at) VALUES ('cell-a', 'reserve', 1000)`
)
const fresh = new RelayAssignmentStore(database!, now)
await expect(fresh.completeEvacuation(identity, migration.assignmentEpoch)).rejects.toThrow(
'cell_in_reserve_mode'
)
await expect(fresh.completeReadyEvacuations()).rejects.toThrow('cell_in_reserve_mode')
expect(
await database!.query(`SELECT completed_at FROM relay_assignment_migrations`)
).toEqual([{ completed_at: null }])
// The same host on a db source completes: the guard, not the fixture, held it.
await store.setCellAdmitMode('cell-a', 'db')
expect(await store.completeReadyEvacuations()).toBe(1)
})
it('refuses evacuation status for a reserve cell, where activeCellMigrations would count no units', async () => {
const store = await setup(() => 1_000)
await store.setCellAdmitMode('cell-a', 'reserve')
await expect(store.cellEvacuationStatus('cell-a', 'cell-b', true)).rejects.toThrow(
'cell_in_reserve_mode'
)
await expect(store.cellEvacuationStatus('cell-b', 'cell-a', false)).rejects.toThrow(
'cell_in_reserve_mode'
)
})
})
function activityLeaseReaders(): { readers: string[]; bareTableNames: string[] } {
const directory = fileURLToPath(new URL('.', import.meta.url))
const readers = new Set<string>()
const bareTableNames: string[] = []
const files = readdirSync(directory, { recursive: true, encoding: 'utf8' }).filter(
(file) => file.endsWith('.ts') && !file.includes('.test.') && !file.split(/[\\/]/).includes('test-fixtures')
)
for (const relativeFile of files) {
const file = relativeFile.split(sep).join('/')
const text = readFileSync(join(directory, file), 'utf8')
if (!text.includes(LEASE_TABLE)) continue
const source = ts.createSourceFile(file, text, ts.ScriptTarget.ES2022, true)
const visit = (node: ts.Node): void => {
if (ts.isNoSubstitutionTemplateLiteral(node) || ts.isTemplateExpression(node) || ts.isStringLiteral(node)) {
if ((ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node)) && node.text.trim() === LEASE_TABLE) {
bareTableNames.push(`${file}:${enclosingName(node)}`)
}
if (LEASE_READ.test(node.getText())) {
readers.add(`${file}:${enclosingName(node)}`)
return
}
}
ts.forEachChild(node, visit)
}
visit(source)
}
return { readers: [...readers].sort(), bareTableNames }
}
function enclosingName(node: ts.Node): string {
for (let current = node.parent; current; current = current.parent) {
if ((ts.isMethodDeclaration(current) || ts.isFunctionDeclaration(current)) && current.name) {
return current.name.getText()
}
// A module-level SQL constant, never a local `const rows = ...` inside a method.
if (ts.isVariableStatement(current) && ts.isSourceFile(current.parent)) {
return current.declarationList.declarations.map((declaration) => declaration.name.getText()).join(',')
}
}
return '<module>'
}
@@ -0,0 +1,76 @@
import { afterEach, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import type { RelayCellConfig } from './config.js'
import { openInMemoryRelayDatabase, type RelayDatabase } from './database.js'
const CELLS: RelayCellConfig[] = [
{ id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
{ id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
]
describe('evacuation from reserve-mode cells', () => {
let database: RelayDatabase | undefined
afterEach(async () => {
await database?.close()
database = undefined
})
async function setup(now: () => number) {
database = await openInMemoryRelayDatabase()
const store = new RelayAssignmentStore(database, now)
await store.reconcileCells(CELLS)
return store
}
// A host whose source cell kept no lease for it, as a memory cell's control and phones do.
async function registeredEvacuationWithoutSourceLeases(store: RelayAssignmentStore) {
const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
const first = await store.assign(identity)
expect(first.cellId).toBe('cell-a')
const migration = await store.startEvacuation(identity, 'cell-b')
await store.activateControl(identity, {
cellId: 'cell-b',
assignmentEpoch: migration.assignmentEpoch,
generation: 1
})
await store.markMigrationTargetRegistered(identity, {
cellId: 'cell-b',
assignmentEpoch: migration.assignmentEpoch
})
await store.releaseActivity(identity, `control-pending:${first.assignmentEpoch}`)
return { identity, migration }
}
it('completes no evacuation from a reserve source, though it holds no source lease', async () => {
const now = () => 1_000
const store = await setup(now)
const { identity, migration } = await registeredEvacuationWithoutSourceLeases(store)
// setCellAdmitMode refuses this with the migration open; written directly to reach the second guard.
await database!.query(
`INSERT INTO relay_cell_admit_modes (cell_id, admit_mode, updated_at) VALUES ('cell-a', 'reserve', 1000)`
)
const fresh = new RelayAssignmentStore(database!, now)
await expect(fresh.completeEvacuation(identity, migration.assignmentEpoch)).rejects.toThrow(
'cell_in_reserve_mode'
)
await expect(fresh.completeReadyEvacuations()).rejects.toThrow('cell_in_reserve_mode')
expect(
await database!.query(`SELECT completed_at FROM relay_assignment_migrations`)
).toEqual([{ completed_at: null }])
// The same host on a db source completes: the guard, not the fixture, held it.
await store.setCellAdmitMode('cell-a', 'db')
expect(await store.completeReadyEvacuations()).toBe(1)
})
it('refuses evacuation status for a reserve cell, where activeCellMigrations would count no units', async () => {
const store = await setup(() => 1_000)
await store.setCellAdmitMode('cell-a', 'reserve')
await expect(store.cellEvacuationStatus('cell-a', 'cell-b', true)).rejects.toThrow(
'cell_in_reserve_mode'
)
await expect(store.cellEvacuationStatus('cell-b', 'cell-a', false)).rejects.toThrow(
'cell_in_reserve_mode'
)
})
})
@@ -1,109 +1,15 @@
import { readFileSync } from 'node:fs'
import { fileURLToPath } from 'node:url'
import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract'
import ts from 'typescript-api'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { RelayAssignmentStore, RelayCellInReserveModeError } from './assignment-store.js'
import type { RelayCellConfig } from './config.js'
import { openInMemoryRelayDatabase, type RelayDatabase } from './database.js'
// Step 5 census (design §6.3): a reserve-mode cell keeps its hosts in memory, so Postgres's
// per-host state for it is stale. Every store method is classified below; a new one fails
// this test until someone decides how it treats a reserve-mode cell.
const RESERVE_MODE_CENSUS: Record<string, string> = {
// One host's connect, resolve or lease: a reserve-mode cell answers these from memory, and
// its ledger row is written after the fact.
acquireActivity: 'connect-path',
activateControl: 'connect-path',
// A flip back's re-registration: the cell is leaving reserve mode, so its rows are rebuilt.
activateControlDeferringCell: 'connect-path',
assign: 'connect-path',
changeActivity: 'connect-path',
exchangeRegionCorrection: 'connect-path',
hostWhereabouts: 'connect-path',
markMigrationTargetRegistered: 'connect-path',
releaseActivity: 'connect-path',
renewControlActivities: 'connect-path',
renewControlActivity: 'connect-path',
resolve: 'connect-path',
verifyCellAssignment: 'connect-path',
// Sweeps and per-cell host readers: skip reserve-mode cells, or refuse them.
cellDeploymentStatus: 'refuses-reserve-cells',
cellEvacuationCapacity: 'refuses-reserve-cells',
cellEvacuationStatus: 'refuses-reserve-cells',
evacuateDeadCells: 'skips-reserve-cells',
prepareCellDrainAttempt: 'refuses-reserve-cells',
rebalanceDormant: 'refuses-reserve-cells',
releaseExpiredActivity: 'skips-reserve-cells',
releaseExpiredActivityLeases: 'skips-reserve-cells',
selectIdleRegionalRehomeCandidates: 'skips-reserve-cells',
startActiveCellEvacuations: 'refuses-reserve-cells',
startEvacuation: 'refuses-reserve-cells',
// Steps of a drain, migration or rehome that only exist once a refused start above began one.
abortExpiredEvacuations: 'follows-a-refused-start',
abortExpiredRegionalRehomes: 'follows-a-refused-start',
abortUnarrivedRegionalRehomes: 'follows-a-refused-start',
beginCellDrainSend: 'follows-a-refused-start',
commitIdleRegionalRehome: 'follows-a-refused-start',
// A reserve source keeps no activity rows, so its "no source units" check proves nothing.
completeEvacuation: 'refuses-reserve-cells',
completeEvacuationFromDeadSource: 'refuses-reserve-cells',
completeReadyEvacuations: 'follows-a-refused-start',
completeReadyRegionalRehomes: 'follows-a-refused-start',
prepareCellDrainRecovery: 'follows-a-refused-start',
proveCellDrainNotDelivered: 'follows-a-refused-start',
reapRegionalRehomeAttempts: 'follows-a-refused-start',
reconcileIdleRegionalRehome: 'follows-a-refused-start',
recordCellDrainApplicationReceipt: 'follows-a-refused-start',
refreshRegionalRehomeLeases: 'follows-a-refused-start',
supersedeRegisteredCellEvacuations: 'follows-a-refused-start',
supersedeRegisteredEvacuation: 'follows-a-refused-start',
// Cell-level state, fences and controls: no per-host rows.
abortCellFenceAttempt: 'cell-level',
addMigrationCells: 'cell-level',
adoptLegacyCellFence: 'cell-level',
applyCellAdmissionSelector: 'cell-level',
applyRegionalRehomeControl: 'cell-level',
cellAdmitMode: 'cell-level',
cellHeartbeatFresh: 'cell-level',
recordCellAdmitEffective: 'cell-level',
commitCellReservationDelta: 'cell-level',
attestCellFence: 'cell-level',
attestCellFenceAttempt: 'cell-level',
bindCellFencePlanGeneration: 'cell-level',
cellFenceAttempt: 'cell-level',
commitLegacyCellFenceAdoption: 'cell-level',
configureCell: 'cell-level',
disableRegionalRehomeControl: 'cell-level',
inspectCellAdmissionSelector: 'cell-level',
inspectRegionalRehomeControl: 'cell-level',
prepareCellFenceAttempt: 'cell-level',
previewRegionCorrection: 'cell-level',
previewRegionalRehomeEligibility: 'cell-level',
pruneReleasedControlReservations: 'cell-level',
reconcileCells: 'cell-level',
reconcileCellsAtStartup: 'cell-level',
reserveModeCells: 'cell-level',
recordCellFenceOperation: 'cell-level',
recordCellHeartbeat: 'cell-level',
recordCellRegionalRehomeStatus: 'cell-level',
regionCatalog: 'cell-level',
regionCorrectionOutcomes: 'cell-level',
regionalRehomeFleetSafety: 'cell-level',
releaseExpiredRegionPreferences: 'cell-level',
seatFeedCells: 'cell-level',
setCellAdmissionState: 'cell-level',
setCellAdmitMode: 'cell-level',
setCellEnabled: 'cell-level',
startCellFenceApply: 'cell-level'
}
const CELLS: RelayCellConfig[] = [
{ id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
{ id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
]
describe('reserve-mode census', () => {
describe('assignment reserve-mode behavior', () => {
let database: RelayDatabase | undefined
afterEach(async () => {
@@ -111,12 +17,6 @@ describe('reserve-mode census', () => {
database = undefined
})
it('classifies every public store method for reserve-mode cells', () => {
const methods = publicStoreMethods()
expect(methods.filter((name) => !(name in RESERVE_MODE_CENSUS))).toEqual([])
expect(Object.keys(RESERVE_MODE_CENSUS).filter((name) => !methods.includes(name))).toEqual([])
})
async function setup(now: () => number, reserve: string[] = []) {
database = await openInMemoryRelayDatabase()
const store = new RelayAssignmentStore(database, now)
@@ -385,24 +285,3 @@ describe('reserve-mode census', () => {
expect(await store.cellAdmitMode('cell-a')).toMatchObject({ admitMode: 'reserve', updatedAt: 1_000 })
})
})
function publicStoreMethods(): string[] {
const path = fileURLToPath(new URL('./assignment-store.ts', import.meta.url))
const source = ts.createSourceFile(path, readFileSync(path, 'utf8'), ts.ScriptTarget.ES2022, true)
const names = new Set<string>()
source.forEachChild((node) => {
if (!ts.isClassDeclaration(node) || node.name?.text !== 'RelayAssignmentStore') return
for (const member of node.members) {
if (!ts.isMethodDeclaration(member) || !ts.isIdentifier(member.name)) continue
const hidden = ts
.getModifiers(member)
?.some(
(modifier) =>
modifier.kind === ts.SyntaxKind.PrivateKeyword ||
modifier.kind === ts.SyntaxKind.ProtectedKeyword
)
if (!hidden) names.add(member.name.text)
}
})
return [...names].sort()
}