mirror of
https://github.com/stablyai/orca.git
synced 2026-09-30 00:03:15 +00:00
* fix(relay): rehome hosts to their preferred region in either direction The regional-rehome worker only moved hosts from a us-central1 cell to an asia-east2 one, so a host whose desktop later records us-central1 stays where it was put. Rehoming now compares the fresh preference against the region of the cell the host is on and moves it to a general cell in the preferred region either way, through the same drain, migrate, safety, and rate-limit machinery. - relay_region_rehome_attempts.preferred_region accepts both regions; existing databases are upgraded in place by an idempotent named-constraint swap that is safe when several directors start at once. - A target must carry the drain protocol too: moving a host onto a cell it can never be drained off again is the trap this change exists to undo. The fleet whose health gates a rehome is now every general drainable cell, which is exactly the set of legal sources and targets. - The trust probe accepts a source cell in any region. No wire change, and no behaviour change while the durable control is off. * fix(relay): bound bidirectional rehoming with a per-host cooldown Moving hosts in both directions removed the property that made the old one-way worker self-terminating: a desktop whose region probe flips would be dragged back and forth, one full drain and migrate per flip, because the preference age never expires while the host keeps reconnecting. - relay_region_rehome_control gains host_cooldown_ms, an operator input plumbed like preference_max_age_ms (workflow, ops script, admin route, durable row) and defaulted to seven days. A host with any attempt row inside the window, whichever way that move went, is not a candidate; the claim re-reads it under lock so an attempt landing between scan and claim cannot start a second move. Skips are named host_cooldown, and the lookup rides a new index on (user_id, relay_host_id, created_at). - The candidate scan now also requires the target cell to be enabled, so it mirrors the claim-time filter exactly and stops spending batch slots on candidates that are certain to be skipped. - Region CHECK lists are rendered from the shared region list instead of being written out four times. - The operations runbook states that cells without the drain protocol are neither sources, targets, nor members of the safety gate. * fix(relay): keep rehome reads and brakes working across the cooldown rollout The ops script validated hostCooldownMs on every inspected control, so against any director image predating the field inspect, pause, disable, and failed-enable recovery all threw client-side. The workflow always runs from main while the director image is operator-supplied, so that window opened at merge and reopened on every rollback: the operator lost read-only visibility and both emergency brakes while the worker could still be enabled. The field is now validated only when the director reports it, and every apply body that echoes an inspected control omits the key when that control lacks it, so a legacy director never sees an unknown key. The write path stays fail-closed the other way: enable refuses up front, before any mutation, when the director does not report a cooldown it could honour. Also replaces two bare 'us-central1' defaults with RELAY_DEFAULT_REGION.
195 lines
7.1 KiB
TypeScript
195 lines
7.1 KiB
TypeScript
import { mkdtempSync, rmSync } from 'node:fs'
|
|
import { tmpdir } from 'node:os'
|
|
import { join } from 'node:path'
|
|
import { afterEach, describe, expect, it } from 'vitest'
|
|
import {
|
|
openInMemoryRelayDatabase,
|
|
openRelayDatabase,
|
|
POSTGRES_SCHEMA_MIGRATIONS,
|
|
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS
|
|
} from './database.js'
|
|
|
|
const temporaryDirectories: string[] = []
|
|
|
|
afterEach(() => {
|
|
for (const directory of temporaryDirectories.splice(0)) {
|
|
rmSync(directory, { recursive: true, force: true })
|
|
}
|
|
})
|
|
|
|
describe('relay database', () => {
|
|
it('creates every durable relay state table', async () => {
|
|
const database = await openInMemoryRelayDatabase()
|
|
const rows = await database.query(
|
|
`SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'relay_%' ORDER BY name`
|
|
)
|
|
expect(rows.map((row) => row.name)).toEqual([
|
|
'relay_admission_selector_cell_additions',
|
|
'relay_admission_selector_intents',
|
|
'relay_admission_selectors',
|
|
'relay_assignment_activity_leases',
|
|
'relay_assignment_migration_incarnations',
|
|
'relay_assignment_migrations',
|
|
'relay_assignment_region_preferences',
|
|
'relay_assignments',
|
|
'relay_audit_events',
|
|
'relay_cell_admission',
|
|
'relay_cell_capabilities',
|
|
'relay_cell_committed_fences',
|
|
'relay_cell_connection_limits',
|
|
'relay_cell_connection_runtime',
|
|
'relay_cell_connection_snapshots',
|
|
'relay_cell_drain_attempt_states',
|
|
'relay_cell_drain_attempts',
|
|
'relay_cell_drain_recovery_attempts',
|
|
'relay_cell_fence_apply_invocations',
|
|
'relay_cell_fence_attempts',
|
|
'relay_cell_fence_plan_bindings',
|
|
'relay_cell_fences',
|
|
'relay_cell_legacy_fence_adoptions',
|
|
'relay_cell_regions',
|
|
'relay_cell_rehome_safety',
|
|
'relay_cell_runtime',
|
|
'relay_cells',
|
|
'relay_confirm_results',
|
|
'relay_confirmable_splices',
|
|
'relay_connection_bases',
|
|
'relay_control_connection_reservations',
|
|
'relay_devices',
|
|
'relay_direct_authorizations',
|
|
'relay_install_results',
|
|
'relay_invites',
|
|
'relay_migration_leases',
|
|
'relay_post_drain_migration_pins',
|
|
'relay_rate_windows',
|
|
'relay_region_rehome_attempts',
|
|
'relay_region_rehome_control',
|
|
'relay_region_rehome_worker_state'
|
|
])
|
|
await database.close()
|
|
})
|
|
|
|
it('rolls back every effect and serializes concurrent transactions', async () => {
|
|
const database = await openInMemoryRelayDatabase()
|
|
await expect(
|
|
database.transaction(async (transaction) => {
|
|
await transaction.query(
|
|
`INSERT INTO relay_rate_windows
|
|
(scope_key, window_kind, window_started_at, count) VALUES (?, ?, ?, ?)`,
|
|
['scope', 'invite', 1, 1]
|
|
)
|
|
throw new Error('injected failure')
|
|
})
|
|
).rejects.toThrow('injected failure')
|
|
expect(await database.query(`SELECT * FROM relay_rate_windows`)).toEqual([])
|
|
|
|
await Promise.all([
|
|
database.transaction(async (transaction) => {
|
|
await transaction.query(
|
|
`INSERT INTO relay_rate_windows
|
|
(scope_key, window_kind, window_started_at, count) VALUES (?, ?, ?, ?)`,
|
|
['scope', 'invite', 1, 1]
|
|
)
|
|
}),
|
|
database.transaction(async (transaction) => {
|
|
const rows = await transaction.query(
|
|
`SELECT count FROM relay_rate_windows
|
|
WHERE scope_key = ? AND window_kind = ? AND window_started_at = ?`,
|
|
['scope', 'invite', 1]
|
|
)
|
|
await transaction.query(
|
|
`UPDATE relay_rate_windows SET count = ?
|
|
WHERE scope_key = ? AND window_kind = ? AND window_started_at = ?`,
|
|
[Number(rows[0]?.count ?? 0) + 1, 'scope', 'invite', 1]
|
|
)
|
|
})
|
|
])
|
|
const rows = await database.query(`SELECT count FROM relay_rate_windows`)
|
|
expect(Number(rows[0]?.count)).toBe(2)
|
|
await database.close()
|
|
})
|
|
|
|
it('persists SQLite state across process-style reopen', async () => {
|
|
const dataDir = mkdtempSync(join(tmpdir(), 'orca-relay-db-'))
|
|
temporaryDirectories.push(dataDir)
|
|
const first = await openRelayDatabase({ dataDir })
|
|
await first.query(
|
|
`INSERT INTO relay_rate_windows
|
|
(scope_key, window_kind, window_started_at, count) VALUES (?, ?, ?, ?)`,
|
|
['scope', 'connect', 1, 7]
|
|
)
|
|
await first.close()
|
|
const second = await openRelayDatabase({ dataDir })
|
|
const rows = await second.query(`SELECT count FROM relay_rate_windows`)
|
|
expect(Number(rows[0]?.count)).toBe(7)
|
|
await second.close()
|
|
})
|
|
|
|
it('defaults cells created by an older schema user to the US on reopen', async () => {
|
|
const dataDir = mkdtempSync(join(tmpdir(), 'orca-relay-region-db-'))
|
|
temporaryDirectories.push(dataDir)
|
|
const first = await openRelayDatabase({ dataDir })
|
|
await first.query(
|
|
`INSERT INTO relay_cells
|
|
(cell_id, cell_url, enabled, capacity_requests, reserved_requests,
|
|
observed_requests, last_heartbeat_at, updated_at)
|
|
VALUES (?, ?, 1, 10, 0, 0, 1, 1)`,
|
|
['legacy-cell', 'https://legacy.relay.example.com']
|
|
)
|
|
await first.close()
|
|
|
|
const second = await openRelayDatabase({ dataDir })
|
|
expect(
|
|
await second.query(`SELECT region FROM relay_cell_regions WHERE cell_id = ?`, [
|
|
'legacy-cell'
|
|
])
|
|
).toEqual([{ region: 'us-central1' }])
|
|
await second.close()
|
|
})
|
|
|
|
it('renders every region check from the shared region list', async () => {
|
|
// Derived, not hand-written: a third region must not leave one column
|
|
// rejecting a value the rest of the relay already accepts.
|
|
const database = await openInMemoryRelayDatabase()
|
|
const checked = await database.query(
|
|
`SELECT name, sql FROM sqlite_master
|
|
WHERE type = 'table'
|
|
AND name IN ('relay_assignment_region_preferences', 'relay_cell_regions',
|
|
'relay_region_rehome_attempts')
|
|
ORDER BY name`
|
|
)
|
|
const list = `IN ('us-central1', 'asia-east2')`
|
|
expect(checked.map((row) => row.name)).toEqual([
|
|
'relay_assignment_region_preferences',
|
|
'relay_cell_regions',
|
|
'relay_region_rehome_attempts'
|
|
])
|
|
expect(checked.every((row) => String(row.sql).includes(list))).toBe(true)
|
|
expect(
|
|
POSTGRES_SCHEMA_MIGRATIONS.some((statement) => statement.includes(list))
|
|
).toBe(true)
|
|
await database.close()
|
|
})
|
|
|
|
it('indexes rehome attempts by host recency for the per-host cooldown', async () => {
|
|
const database = await openInMemoryRelayDatabase()
|
|
const rows = await database.query(
|
|
`SELECT sql FROM sqlite_master
|
|
WHERE type = 'index' AND name = 'relay_region_rehome_attempts_host_recency'`
|
|
)
|
|
expect(rows[0]?.sql).toContain('(user_id, relay_host_id, created_at)')
|
|
expect(REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS).toBe(7 * 24 * 60 * 60_000)
|
|
await database.close()
|
|
})
|
|
|
|
it('indexes region preference expiry by observation time', async () => {
|
|
const database = await openInMemoryRelayDatabase()
|
|
const rows = await database.query(
|
|
`SELECT sql FROM sqlite_master
|
|
WHERE type = 'index' AND name = 'relay_assignment_region_preferences_observed'`
|
|
)
|
|
expect(rows[0]?.sql).toContain('(observed_at)')
|
|
await database.close()
|
|
})
|
|
})
|