mirror of
https://github.com/stablyai/orca.git
synced 2026-11-17 19:34:29 +00:00
* test: remove low-value fixture suites and custom scenarios * test: prune relay and recovery matrices and refresh retained host journey --------- Co-authored-by: OrcaWin <293788423+OrcaWin@users.noreply.github.com>
572 lines
20 KiB
TypeScript
572 lines
20 KiB
TypeScript
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest'
|
|
import { RelayAssignmentStore } from '../assignment/assignment-store.js'
|
|
import {
|
|
openRelayDatabase,
|
|
type RelayDatabase,
|
|
type RelayLockOptions,
|
|
type SqlRow
|
|
} from './database.js'
|
|
|
|
function postgresTestDatabaseUrl(value: string | undefined): string | undefined {
|
|
if (!value) return undefined
|
|
const url = new URL(value)
|
|
// Detect the intended deadlock before the one-second runtime lock deadline.
|
|
url.searchParams.set('options', '-c deadlock_timeout=100ms')
|
|
return url.toString()
|
|
}
|
|
|
|
const databaseUrl = postgresTestDatabaseUrl(process.env.ORCA_RELAY_TEST_POSTGRES_URL)
|
|
const describePostgres = databaseUrl ? describe : describe.skip
|
|
|
|
type QueryLockHook = (phase: 'before' | 'after', sql: string) => Promise<void>
|
|
|
|
class TransactionProbeDatabase implements RelayDatabase {
|
|
attempts = 0
|
|
|
|
constructor(
|
|
private readonly database: RelayDatabase,
|
|
private readonly hook: QueryLockHook,
|
|
private readonly probeQueries = false
|
|
) {}
|
|
|
|
async query(sql: string, params?: unknown[]): Promise<SqlRow[]> {
|
|
return await this.database.query(sql, params)
|
|
}
|
|
|
|
async queryLocked(
|
|
sql: string,
|
|
params?: unknown[],
|
|
options?: RelayLockOptions
|
|
): Promise<SqlRow[]> {
|
|
return await this.database.queryLocked(sql, params, options)
|
|
}
|
|
|
|
async transaction<T>(operation: (transaction: RelayDatabase) => Promise<T>): Promise<T> {
|
|
return await this.database.transaction(async (transaction) => {
|
|
this.attempts++
|
|
return await operation(
|
|
new QueryLockProbeTransaction(transaction, this.hook, this.probeQueries)
|
|
)
|
|
})
|
|
}
|
|
|
|
async close(): Promise<void> {}
|
|
}
|
|
|
|
class QueryLockProbeTransaction implements RelayDatabase {
|
|
constructor(
|
|
private readonly transactionDatabase: RelayDatabase,
|
|
private readonly hook: QueryLockHook,
|
|
private readonly probeQueries: boolean
|
|
) {}
|
|
|
|
async query(sql: string, params?: unknown[]): Promise<SqlRow[]> {
|
|
if (this.probeQueries) await this.hook('before', sql)
|
|
const rows = await this.transactionDatabase.query(sql, params)
|
|
if (this.probeQueries) await this.hook('after', sql)
|
|
return rows
|
|
}
|
|
|
|
async queryLocked(
|
|
sql: string,
|
|
params?: unknown[],
|
|
options?: RelayLockOptions
|
|
): Promise<SqlRow[]> {
|
|
await this.hook('before', sql)
|
|
const rows = await this.transactionDatabase.queryLocked(sql, params, options)
|
|
await this.hook('after', sql)
|
|
return rows
|
|
}
|
|
|
|
async transaction<T>(operation: (transaction: RelayDatabase) => Promise<T>): Promise<T> {
|
|
return await operation(this)
|
|
}
|
|
|
|
async close(): Promise<void> {}
|
|
}
|
|
|
|
function signal(): { promise: Promise<void>; resolve: () => void } {
|
|
let resolve!: () => void
|
|
return { promise: new Promise<void>((done) => (resolve = done)), resolve }
|
|
}
|
|
|
|
describePostgres('PostgreSQL transaction recovery', () => {
|
|
let database: RelayDatabase
|
|
|
|
beforeAll(async () => {
|
|
database = await openRelayDatabase({ databaseUrl, dataDir: '' })
|
|
})
|
|
|
|
afterAll(async () => {
|
|
await database.close()
|
|
})
|
|
|
|
afterEach(async () => {
|
|
vi.restoreAllMocks()
|
|
for (const table of [
|
|
'relay_assignment_migration_incarnations',
|
|
'relay_control_connection_reservations'
|
|
]) {
|
|
await database.query(`DELETE FROM ${table} WHERE user_id IN (?, ?)`, [
|
|
'completion-race-user',
|
|
'completion-contention-user'
|
|
])
|
|
}
|
|
for (const table of ['relay_cell_admission', 'relay_cell_regions']) {
|
|
await database.query(`DELETE FROM ${table} WHERE cell_id IN (?, ?, ?, ?)`, [
|
|
'completion-race-source',
|
|
'completion-race-target',
|
|
'completion-contention-source',
|
|
'completion-contention-target'
|
|
])
|
|
}
|
|
for (const identity of [
|
|
{ userId: 'released-order-user', relayHostId: 'releasedorder001' },
|
|
{ userId: 'normalized-order-user', relayHostId: 'normalizedorder1' },
|
|
{ userId: 'atomic-final-user-a', relayHostId: 'atomicfinalhosta' },
|
|
{ userId: 'atomic-final-user-b', relayHostId: 'atomicfinalhostb' },
|
|
{ userId: 'lease-assignment-contention-user', relayHostId: 'leaseassignment1' }
|
|
]) {
|
|
await database.query(
|
|
`DELETE FROM relay_assignment_activity_leases WHERE user_id = ? AND relay_host_id = ?`,
|
|
[identity.userId, identity.relayHostId]
|
|
)
|
|
await database.query(
|
|
`DELETE FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
|
|
[identity.userId, identity.relayHostId]
|
|
)
|
|
}
|
|
await database.query(`DELETE FROM relay_cells WHERE cell_id IN (?, ?)`, [
|
|
'released-order-cell',
|
|
'normalized-order-cell'
|
|
])
|
|
await database.query(`DELETE FROM relay_cells WHERE cell_id = ?`, ['atomic-final-cell'])
|
|
await database.query(`DELETE FROM relay_cells WHERE cell_id = ?`, [
|
|
'lease-assignment-contention-cell'
|
|
])
|
|
await database.query(
|
|
`DELETE FROM relay_assignment_activity_leases WHERE user_id LIKE 'reconcile-user-%'`
|
|
)
|
|
await database.query(`DELETE FROM relay_assignments WHERE user_id LIKE 'reconcile-user-%'`)
|
|
await database.query(`DELETE FROM relay_cells WHERE cell_id IN (?, ?)`, [
|
|
'reconcile-cell-a',
|
|
'reconcile-cell-b'
|
|
])
|
|
await database.query(
|
|
`DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`,
|
|
['completion-race-user']
|
|
)
|
|
await database.query(`DELETE FROM relay_assignment_migrations WHERE user_id = ?`, [
|
|
'completion-race-user'
|
|
])
|
|
await database.query(`DELETE FROM relay_assignments WHERE user_id = ?`, [
|
|
'completion-race-user'
|
|
])
|
|
await database.query(`DELETE FROM relay_cells WHERE cell_id IN (?, ?)`, [
|
|
'completion-race-source',
|
|
'completion-race-target'
|
|
])
|
|
await database.query(
|
|
`DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`,
|
|
['completion-contention-user']
|
|
)
|
|
await database.query(`DELETE FROM relay_assignment_migrations WHERE user_id = ?`, [
|
|
'completion-contention-user'
|
|
])
|
|
await database.query(`DELETE FROM relay_assignments WHERE user_id = ?`, [
|
|
'completion-contention-user'
|
|
])
|
|
await database.query(`DELETE FROM relay_cell_runtime WHERE cell_id IN (?, ?)`, [
|
|
'completion-contention-source',
|
|
'completion-contention-target'
|
|
])
|
|
await database.query(`DELETE FROM relay_cells WHERE cell_id IN (?, ?)`, [
|
|
'completion-contention-source',
|
|
'completion-contention-target'
|
|
])
|
|
await database.query(`DELETE FROM relay_cell_fences WHERE cell_id = ?`, [
|
|
'dead-source-contention-source'
|
|
])
|
|
await database.query(
|
|
`DELETE FROM relay_control_connection_reservations WHERE user_id = ?`,
|
|
['dead-source-contention-user']
|
|
)
|
|
await database.query(
|
|
`DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`,
|
|
['dead-source-contention-user']
|
|
)
|
|
await database.query(
|
|
`DELETE FROM relay_assignment_migration_incarnations WHERE user_id = ?`,
|
|
['dead-source-contention-user']
|
|
)
|
|
await database.query(`DELETE FROM relay_assignment_migrations WHERE user_id = ?`, [
|
|
'dead-source-contention-user'
|
|
])
|
|
await database.query(`DELETE FROM relay_assignments WHERE user_id = ?`, [
|
|
'dead-source-contention-user'
|
|
])
|
|
await database.query(`DELETE FROM relay_cell_runtime WHERE cell_id IN (?, ?)`, [
|
|
'dead-source-contention-source',
|
|
'dead-source-contention-target'
|
|
])
|
|
await database.query(`DELETE FROM relay_cell_admission WHERE cell_id IN (?, ?)`, [
|
|
'dead-source-contention-source',
|
|
'dead-source-contention-target'
|
|
])
|
|
await database.query(`DELETE FROM relay_cells WHERE cell_id IN (?, ?)`, [
|
|
'dead-source-contention-source',
|
|
'dead-source-contention-target'
|
|
])
|
|
await database.query(
|
|
`DELETE FROM relay_assignment_activity_leases
|
|
WHERE user_id IN (?, ?)`,
|
|
['lease-contention-user', 'aggregate-contention-user']
|
|
)
|
|
await database.query(
|
|
`DELETE FROM relay_assignments
|
|
WHERE user_id IN (?, ?)`,
|
|
['lease-contention-user', 'aggregate-contention-user']
|
|
)
|
|
await database.query(`DELETE FROM relay_cells WHERE cell_id IN (?, ?)`, [
|
|
'lease-contention-cell',
|
|
'aggregate-contention-cell'
|
|
])
|
|
await database.query(
|
|
`DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`,
|
|
['sustained-lock-user']
|
|
)
|
|
await database.query(`DELETE FROM relay_assignments WHERE user_id = ?`, [
|
|
'sustained-lock-user'
|
|
])
|
|
await database.query(`DELETE FROM relay_cells WHERE cell_id = ?`, [
|
|
'sustained-lock-cell'
|
|
])
|
|
await database.query(
|
|
`DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`,
|
|
['sticky-cell-contention-user']
|
|
)
|
|
await database.query(
|
|
`DELETE FROM relay_assignments WHERE user_id = ?`,
|
|
['sticky-cell-contention-user']
|
|
)
|
|
await database.query(`DELETE FROM relay_cells WHERE cell_id = ?`, [
|
|
'sticky-cell-contention-cell'
|
|
])
|
|
await database.query(
|
|
`DELETE FROM relay_assignment_activity_leases WHERE user_id LIKE 'postgres-user-%'`
|
|
)
|
|
await database.query(`DELETE FROM relay_assignments WHERE user_id LIKE 'postgres-user-%'`)
|
|
await database.query(`DELETE FROM relay_cells WHERE cell_id IN (?, ?, ?)`, [
|
|
'cell-a',
|
|
'cell-b',
|
|
'cell-c'
|
|
])
|
|
await database.query(
|
|
`DELETE FROM relay_assignment_activity_leases WHERE user_id LIKE 'sticky-fast-user-%'`
|
|
)
|
|
await database.query(
|
|
`DELETE FROM relay_assignments WHERE user_id LIKE 'sticky-fast-user-%'`
|
|
)
|
|
await database.query(`DELETE FROM relay_cells WHERE cell_id = ?`, [
|
|
'sticky-fast-cell'
|
|
])
|
|
})
|
|
|
|
it('retries an entire deadlock victim transaction on a fresh attempt', async () => {
|
|
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
|
const keys = ['deadlock-a', 'deadlock-b']
|
|
for (const key of keys) {
|
|
await database.query(
|
|
`INSERT INTO relay_rate_windows
|
|
(scope_key, window_kind, window_started_at, count) VALUES (?, ?, ?, ?)
|
|
ON CONFLICT (scope_key, window_kind, window_started_at) DO UPDATE SET count = ?`,
|
|
[key, 'transaction-recovery', 1, 0, 0]
|
|
)
|
|
}
|
|
|
|
let firstAttemptArrivals = 0
|
|
let releaseFirstAttempts!: () => void
|
|
const firstAttemptsReady = new Promise<void>((resolve) => {
|
|
releaseFirstAttempts = resolve
|
|
})
|
|
const attempts = [0, 0]
|
|
const mutateWithOppositeLockOrder = async (
|
|
operationIndex: number,
|
|
firstKey: string,
|
|
secondKey: string
|
|
): Promise<void> => {
|
|
await database.transaction(async (transaction) => {
|
|
attempts[operationIndex] = (attempts[operationIndex] ?? 0) + 1
|
|
await transaction.queryLocked(
|
|
`SELECT * FROM relay_rate_windows
|
|
WHERE scope_key = ? AND window_kind = ? AND window_started_at = ?`,
|
|
[firstKey, 'transaction-recovery', 1]
|
|
)
|
|
if (attempts[operationIndex] === 1) {
|
|
firstAttemptArrivals++
|
|
if (firstAttemptArrivals === 2) releaseFirstAttempts()
|
|
await firstAttemptsReady
|
|
}
|
|
await transaction.queryLocked(
|
|
`SELECT * FROM relay_rate_windows
|
|
WHERE scope_key = ? AND window_kind = ? AND window_started_at = ?`,
|
|
[secondKey, 'transaction-recovery', 1]
|
|
)
|
|
await transaction.query(
|
|
`UPDATE relay_rate_windows SET count = count + 1
|
|
WHERE scope_key = ? AND window_kind = ? AND window_started_at = ?`,
|
|
[firstKey, 'transaction-recovery', 1]
|
|
)
|
|
})
|
|
}
|
|
|
|
await Promise.all([
|
|
mutateWithOppositeLockOrder(0, keys[0]!, keys[1]!),
|
|
mutateWithOppositeLockOrder(1, keys[1]!, keys[0]!)
|
|
])
|
|
|
|
expect([...attempts].sort()).toEqual([1, 2])
|
|
const rows = await database.query(
|
|
`SELECT scope_key, count FROM relay_rate_windows
|
|
WHERE window_kind = ? ORDER BY scope_key`,
|
|
['transaction-recovery']
|
|
)
|
|
expect(rows).toEqual([
|
|
{ scope_key: keys[0], count: '1' },
|
|
{ scope_key: keys[1], count: '1' }
|
|
])
|
|
expect(warn).toHaveBeenCalledWith(
|
|
expect.stringContaining('"event":"orca_relay_postgres_transaction_retry"')
|
|
)
|
|
expect(warn).toHaveBeenCalledWith(expect.stringContaining('"phase":"rate-limit"'))
|
|
expect(warn).not.toHaveBeenCalledWith(
|
|
expect.stringContaining('"event":"orca_relay_postgres_transaction_exhausted"')
|
|
)
|
|
warn.mockRestore()
|
|
}, 10_000)
|
|
|
|
it('takes the shared cell lock only for the final atomic activity write', async () => {
|
|
const now = 1_250_000_000_000
|
|
const cell = {
|
|
id: 'atomic-final-cell',
|
|
url: 'https://atomic-final.example.com',
|
|
capacityRequests: 4
|
|
}
|
|
const identities = [
|
|
{ userId: 'atomic-final-user-a', relayHostId: 'atomicfinalhosta' },
|
|
{ userId: 'atomic-final-user-b', relayHostId: 'atomicfinalhostb' }
|
|
]
|
|
const seedStore = new RelayAssignmentStore(database, () => now)
|
|
await seedStore.reconcileCells([cell])
|
|
await Promise.all(identities.map(async (identity) => await seedStore.assign(identity)))
|
|
|
|
const firstAtCellWrite = signal()
|
|
const secondAtCellWrite = signal()
|
|
const releaseFirst = signal()
|
|
const releaseSecond = signal()
|
|
const isAtomicCellWrite = (sql: string): boolean =>
|
|
sql.includes('UPDATE relay_cells SET reserved_requests') &&
|
|
sql.includes('RETURNING cell_id')
|
|
const firstDatabase = new TransactionProbeDatabase(
|
|
database,
|
|
async (phase, sql) => {
|
|
if (phase !== 'before' || !isAtomicCellWrite(sql)) return
|
|
firstAtCellWrite.resolve()
|
|
await releaseFirst.promise
|
|
},
|
|
true
|
|
)
|
|
const secondDatabase = new TransactionProbeDatabase(
|
|
database,
|
|
async (phase, sql) => {
|
|
if (phase !== 'before' || !isAtomicCellWrite(sql)) return
|
|
secondAtCellWrite.resolve()
|
|
await releaseSecond.promise
|
|
},
|
|
true
|
|
)
|
|
const first = new RelayAssignmentStore(firstDatabase, () => now).acquireActivity(
|
|
identities[0]!,
|
|
{ activityId: 'splice:atomic-final-a', kind: 'splice', cellId: cell.id }
|
|
)
|
|
await firstAtCellWrite.promise
|
|
const second = new RelayAssignmentStore(secondDatabase, () => now).acquireActivity(
|
|
identities[1]!,
|
|
{ activityId: 'splice:atomic-final-b', kind: 'splice', cellId: cell.id }
|
|
)
|
|
let reachTimeout: ReturnType<typeof setTimeout> | undefined
|
|
const secondReachedCellWrite = await Promise.race([
|
|
secondAtCellWrite.promise.then(() => true),
|
|
new Promise<false>((resolve) => {
|
|
reachTimeout = setTimeout(() => resolve(false), 2_000)
|
|
})
|
|
])
|
|
if (reachTimeout) clearTimeout(reachTimeout)
|
|
if (!secondReachedCellWrite) {
|
|
releaseFirst.resolve()
|
|
releaseSecond.resolve()
|
|
await Promise.allSettled([first, second])
|
|
}
|
|
expect(secondReachedCellWrite).toBe(true)
|
|
|
|
releaseFirst.resolve()
|
|
await expect(first).resolves.toBeUndefined()
|
|
releaseSecond.resolve()
|
|
await expect(second).rejects.toThrow('relay_capacity_exhausted')
|
|
await expect(
|
|
database.query(
|
|
`SELECT cell.reserved_requests, COUNT(lease.activity_id) AS leases,
|
|
COALESCE(SUM(lease.request_units), 0) AS lease_units
|
|
FROM relay_cells cell
|
|
LEFT JOIN relay_assignment_activity_leases lease ON lease.cell_id = cell.cell_id
|
|
WHERE cell.cell_id = ? GROUP BY cell.cell_id, cell.reserved_requests`,
|
|
[cell.id]
|
|
)
|
|
).resolves.toEqual([{ reserved_requests: '4', leases: '3', lease_units: '4' }])
|
|
await expect(
|
|
database.query(
|
|
`SELECT user_id, reserved_splices FROM relay_assignments
|
|
WHERE user_id IN (?, ?) ORDER BY user_id`,
|
|
[identities[0]!.userId, identities[1]!.userId]
|
|
)
|
|
).resolves.toEqual([
|
|
{ user_id: identities[0]!.userId, reserved_splices: '1' },
|
|
{ user_id: identities[1]!.userId, reserved_splices: '0' }
|
|
])
|
|
}, 15_000)
|
|
|
|
it('keeps a unit placed between reconciliation reading leases and locking cells', async () => {
|
|
const now = 1_350_000_000_000
|
|
const cells = [
|
|
{ id: 'reconcile-cell-a', url: 'https://reconcile-a.example.com', capacityRequests: 100 },
|
|
{ id: 'reconcile-cell-b', url: 'https://reconcile-b.example.com', capacityRequests: 100 }
|
|
]
|
|
const seedStore = new RelayAssignmentStore(database, () => now)
|
|
await seedStore.reconcileCells(cells)
|
|
|
|
const leasesRead = signal()
|
|
const continueReconciliation = signal()
|
|
let gateReconciliation = true
|
|
const reconcileDatabase = new TransactionProbeDatabase(database, async (phase, sql) => {
|
|
if (
|
|
gateReconciliation &&
|
|
phase === 'after' &&
|
|
sql.includes('SELECT lease.* FROM relay_assignment_activity_leases lease')
|
|
) {
|
|
gateReconciliation = false
|
|
leasesRead.resolve()
|
|
await continueReconciliation.promise
|
|
}
|
|
})
|
|
const reconciliation = new RelayAssignmentStore(
|
|
reconcileDatabase,
|
|
() => now
|
|
).cellEvacuationStatus('reconcile-cell-a', 'reconcile-cell-b', true)
|
|
await leasesRead.promise
|
|
// A brand-new host holds no row reconciliation locked, so it commits in the gap.
|
|
const grant = await seedStore.assign({
|
|
userId: 'reconcile-user-gap',
|
|
relayHostId: 'reconcilehostgap'
|
|
})
|
|
continueReconciliation.resolve()
|
|
await reconciliation
|
|
|
|
expect(['reconcile-cell-a', 'reconcile-cell-b']).toContain(grant.cellId)
|
|
const reservations = await database.query(
|
|
`SELECT cell.cell_id, cell.reserved_requests,
|
|
COALESCE(SUM(lease.request_units), 0) AS lease_units
|
|
FROM relay_cells cell
|
|
LEFT JOIN relay_assignment_activity_leases lease ON lease.cell_id = cell.cell_id
|
|
WHERE cell.cell_id = ?
|
|
GROUP BY cell.cell_id, cell.reserved_requests`,
|
|
[grant.cellId]
|
|
)
|
|
expect(reservations).toEqual([
|
|
{ cell_id: grant.cellId, reserved_requests: '1', lease_units: '1' }
|
|
])
|
|
}, 15_000)
|
|
|
|
it('fences a source activity queued behind evacuation completion', async () => {
|
|
const now = 1_400_000_000_000
|
|
const identity = {
|
|
userId: 'completion-race-user',
|
|
relayHostId: 'completionrace01'
|
|
}
|
|
const sourceCellId = 'completion-race-source'
|
|
const targetCellId = 'completion-race-target'
|
|
const seedStore = new RelayAssignmentStore(database, () => now)
|
|
await seedStore.reconcileCells([
|
|
{
|
|
id: sourceCellId,
|
|
url: 'https://completion-source.example.com',
|
|
capacityRequests: 100
|
|
},
|
|
{
|
|
id: targetCellId,
|
|
url: 'https://completion-target.example.com',
|
|
capacityRequests: 100
|
|
}
|
|
])
|
|
const assignment = await seedStore.assign(identity)
|
|
const sourceControl = await seedStore.activateControl(identity, {
|
|
cellId: sourceCellId,
|
|
assignmentEpoch: assignment.assignmentEpoch,
|
|
generation: 1
|
|
})
|
|
const migration = await seedStore.startEvacuation(identity, targetCellId)
|
|
await seedStore.activateControl(identity, {
|
|
cellId: targetCellId,
|
|
assignmentEpoch: migration.assignmentEpoch,
|
|
generation: 1
|
|
})
|
|
await seedStore.markMigrationTargetRegistered(identity, {
|
|
cellId: targetCellId,
|
|
assignmentEpoch: migration.assignmentEpoch
|
|
})
|
|
await seedStore.releaseActivity(identity, sourceControl)
|
|
|
|
const assignmentLocked = signal()
|
|
const activityReachedAssignment = signal()
|
|
const continueCompletion = signal()
|
|
let gateCompletion = true
|
|
const completionDatabase = new TransactionProbeDatabase(database, async (phase, sql) => {
|
|
if (
|
|
gateCompletion &&
|
|
phase === 'after' &&
|
|
sql.includes('FROM relay_assignments WHERE user_id')
|
|
) {
|
|
gateCompletion = false
|
|
assignmentLocked.resolve()
|
|
await continueCompletion.promise
|
|
}
|
|
})
|
|
const activityDatabase = new TransactionProbeDatabase(database, async (phase, sql) => {
|
|
if (phase === 'before' && sql.includes('FROM relay_assignments WHERE user_id')) {
|
|
activityReachedAssignment.resolve()
|
|
}
|
|
})
|
|
const completionStore = new RelayAssignmentStore(completionDatabase, () => now)
|
|
const activityStore = new RelayAssignmentStore(activityDatabase, () => now)
|
|
|
|
const completion = completionStore.completeReadyEvacuations()
|
|
await assignmentLocked.promise
|
|
const activity = activityStore.acquireActivity(identity, {
|
|
activityId: 'install:queued-source-work',
|
|
kind: 'install',
|
|
cellId: sourceCellId
|
|
})
|
|
const activityRejected = expect(activity).rejects.toThrow(
|
|
'activity_cell_not_authoritative'
|
|
)
|
|
await activityReachedAssignment.promise
|
|
continueCompletion.resolve()
|
|
|
|
await expect(completion).resolves.toBe(1)
|
|
await activityRejected
|
|
await expect(
|
|
seedStore.cellEvacuationStatus(sourceCellId, targetCellId, false)
|
|
).resolves.toMatchObject({ inProgress: 0 })
|
|
}, 15_000)
|
|
|
|
})
|