diff --git a/cloud/apps/relay/src/admin-token-verifier.test.ts b/cloud/apps/relay/src/admin-token-verifier.test.ts index f0870dbb09c..342144c4bff 100644 --- a/cloud/apps/relay/src/admin-token-verifier.test.ts +++ b/cloud/apps/relay/src/admin-token-verifier.test.ts @@ -67,6 +67,13 @@ describe('Relay admin route authorization', () => { expect(relayAdminIdentityMayAccess('monitor', route)).toBe(false) } expect(RELAY_MONITOR_ADMIN_ROUTES).toContain('/v1/admin/evacuation-status') + // Read-only on-call lookup: monitor and fence read it, capacity and asia-proof do not. + for (const identity of ['monitor', 'fence', 'fence-broker', 'deploy'] as const) { + expect(relayAdminIdentityMayAccess(identity, '/v1/admin/host-whereabouts')).toBe(true) + } + for (const identity of ['capacity', 'asia-proof'] as const) { + expect(relayAdminIdentityMayAccess(identity, '/v1/admin/host-whereabouts')).toBe(false) + } }) it('allows only reviewed fence evidence mutations beyond aggregate reads', () => { diff --git a/cloud/apps/relay/src/admin-token-verifier.ts b/cloud/apps/relay/src/admin-token-verifier.ts index 8b236d58473..badb65d797b 100644 --- a/cloud/apps/relay/src/admin-token-verifier.ts +++ b/cloud/apps/relay/src/admin-token-verifier.ts @@ -5,6 +5,7 @@ export const RELAY_MONITOR_ADMIN_ROUTES = [ '/v1/admin/admission-selector/status', '/v1/admin/cell-status', '/v1/admin/evacuation-status', + '/v1/admin/host-whereabouts', '/v1/admin/regional-rehome-control', '/v1/admin/regional-rehome-preview', '/v1/admin/runtime-status' diff --git a/cloud/apps/relay/src/app.ts b/cloud/apps/relay/src/app.ts index 4793dfc3fe5..ecb2d5a7f45 100644 --- a/cloud/apps/relay/src/app.ts +++ b/cloud/apps/relay/src/app.ts @@ -56,10 +56,12 @@ import { } from './public-assignment-admission.js' import { relayHostLogDigest } from './relay-host-log-digest.js' import type { RelayLocalReadinessVerdict } from './relay-local-readiness.js' -import type { - ShadowCompareDatabaseAnswer, - ShadowCompareRoute +import { + classifyShadowSeat, + type ShadowCompareDatabaseAnswer, + type ShadowCompareRoute } from './shadow-directory-compare.js' +import type { ShadowSeatDirectory } from './shadow-seat-directory.js' import type { RelayReadinessDependency } from './relay-readiness.js' import type { AssignmentAdmissionLane, @@ -143,6 +145,7 @@ export function createRelayApp( recordAssignmentUnavailable?: (cause: AssignmentUnavailableCause) => void recordRegionRequest?: (region: RelayRegion | undefined) => void // Step 3 shadow directory; absent unless the director polls seat feeds. + shadowSeats?: ShadowSeatDirectory compareShadowSeats?: ( route: ShadowCompareRoute, identity: { userId: string; relayHostId: string }, @@ -1530,6 +1533,85 @@ export function createRelayApp( return context.json({ error: 'database_temporarily_unavailable' }, 503) } }) + // Per director instance: tooling may ask more than one to see every map. + app.post('/v1/admin/host-whereabouts', async (context) => { + const bearer = readBearer(context.req.header('authorization')) + if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) + if (!bearer || !(await verifyAdminToken(bearer, context.req.path))) { + return context.json({ error: 'invalid_token' }, 401) + } + if (requestTooLarge(context.req.header('content-length'))) { + return context.json({ error: 'request_too_large' }, 413) + } + const body = AdminHostWhereaboutsSchema.safeParse(await context.req.json().catch(() => null)) + if (!body.success) return context.json({ error: 'invalid_request' }, 400) + const { relayHostId } = body.data + const shadow = operations.shadowSeats + const now = Date.now() + // Without a user id only the map can name one: relay_assignments has no host-id index. + const userIds = body.data.userId ? [body.data.userId] : (shadow?.userIdsOf(relayHostId) ?? []) + try { + const cellIds = new Set() + const users = [] + for (const userId of userIds) { + const identity = { userId, relayHostId } + const database = await operations.assignments.hostWhereabouts(identity) + const seats = shadow?.seatsOf(userId, relayHostId) ?? [] + const recentlyLeft = shadow?.recentlyLeftOf(userId, relayHostId, now) ?? [] + if (database) cellIds.add(database.cellId) + if (database?.openMigration) { + cellIds.add(database.openMigration.sourceCellId) + cellIds.add(database.openMigration.targetCellId) + } + for (const entry of [...seats, ...recentlyLeft]) cellIds.add(entry.cellId) + users.push({ + userId, + database, + seats, + recentlyLeft, + // As /v1/resolve sees it: a row on a cell that is not live reads as no answer. + verdict: shadow + ? classifyShadowSeat( + shadow, + identity, + database && shadow.cellLiveness(database.cellId, now) !== 'unlive' + ? database + : null, + now + ) + : null + }) + } + const cells = Object.fromEntries( + [...cellIds].sort().map((cellId) => { + const state = shadow?.cellState(cellId) + return [ + cellId, + state + ? { + status: state.status, + pollAgeMs: state.lastLiveAt === undefined ? null : now - state.lastLiveAt, + lastFailure: state.lastFailure ?? null, + draining: state.draining ?? null, + flagsApplied: state.flagsApplied ?? null + } + : null + ] + }) + ) + return context.json({ + v: 1, + directorIncarnation: operations.cellIncarnation ?? null, + relayHostId, + shadow: shadow ? { state: 'on', complete: shadow.isComplete() } : { state: 'off' }, + users, + cells + }) + } catch (error) { + if (!isRelayDatabaseTransientError(error)) throw error + return context.json({ error: 'database_temporarily_unavailable' }, 503) + } + }) app.post('/v1/admin/cell-status', async (context) => { const bearer = readBearer(context.req.header('authorization')) if (config.role !== 'director') return context.json({ error: 'director_only' }, 404) @@ -1551,6 +1633,14 @@ export function createRelayApp( return app } +const AdminHostWhereaboutsSchema = z + .object({ + v: z.literal(1), + relayHostId: z.string().regex(/^[A-Za-z0-9_-]{16}$/), + userId: z.string().min(1).max(256).optional() + }) + .strict() + const AdminAssignmentMoveSchema = z .object({ v: z.literal(1), diff --git a/cloud/apps/relay/src/assignment-store.ts b/cloud/apps/relay/src/assignment-store.ts index c6c98dfea5b..f0812babc6d 100644 --- a/cloud/apps/relay/src/assignment-store.ts +++ b/cloud/apps/relay/src/assignment-store.ts @@ -152,6 +152,20 @@ export type RelayAssignment = AssignmentIdentity & { // a roll right now: its next assign re-places it rather than re-granting the pin. export type ResolvedRelayAssignment = RelayAssignment & { homeCellRollIsolated?: true } +export type HostWhereaboutsRow = { + cellId: string + assignmentEpoch: number + leaseExpiresAt: number + openMigration: { + sourceCellId: string + targetCellId: string + previousEpoch: number + assignmentEpoch: number + expiresAt: number + targetRegisteredAt: number | null + } | null +} + export type RelayRegionCatalogEntry = { region: RelayRegion probeOrigins: string[] @@ -7861,6 +7875,42 @@ export class RelayAssignmentStore { }) } + // On-call "where is host X: primary-key reads only, no liveness filter. + async hostWhereabouts(identity: AssignmentIdentity): Promise { + const row = ( + await this.database.query( + `SELECT cell_id, assignment_epoch, lease_expires_at FROM relay_assignments + WHERE user_id = ? AND relay_host_id = ?`, + [identity.userId, identity.relayHostId] + ) + )[0] + if (!row) return null + const migrations = await this.database.query( + `SELECT source_cell_id, target_cell_id, previous_epoch, assignment_epoch, expires_at, + target_registered_at + FROM relay_assignment_migrations + WHERE user_id = ? AND relay_host_id = ? AND completed_at IS NULL AND aborted_at IS NULL + ORDER BY assignment_epoch DESC LIMIT 1`, + [identity.userId, identity.relayHostId] + ) + const migration = migrations[0] + return { + cellId: text(row, 'cell_id'), + assignmentEpoch: integer(row, 'assignment_epoch'), + leaseExpiresAt: integer(row, 'lease_expires_at'), + openMigration: migration + ? { + sourceCellId: text(migration, 'source_cell_id'), + targetCellId: text(migration, 'target_cell_id'), + previousEpoch: integer(migration, 'previous_epoch'), + assignmentEpoch: integer(migration, 'assignment_epoch'), + expiresAt: integer(migration, 'expires_at'), + targetRegisteredAt: optionalInteger(migration, 'target_registered_at') ?? null + } + : null + } + } + async regionCatalog(): Promise { const rows = await this.database.query( `SELECT cell.cell_id, cell.cell_url, region.region diff --git a/cloud/apps/relay/src/host-whereabouts-app.test.ts b/cloud/apps/relay/src/host-whereabouts-app.test.ts new file mode 100644 index 00000000000..8888ee2db32 --- /dev/null +++ b/cloud/apps/relay/src/host-whereabouts-app.test.ts @@ -0,0 +1,302 @@ +import { describe, expect, it, vi } from 'vitest' +import type { HostWhereaboutsRow } from './assignment-store.js' +import { RelayAssignmentStore } from './assignment-store.js' +import type { RelayConfig } from './config.js' +import { openInMemoryRelayDatabase } from './database.js' +import { ShadowSeatDirectory } from './shadow-seat-directory.js' + +vi.mock('./admin-token-verifier.js', async (importOriginal) => { + const actual = await importOriginal() + const identities = new Map[0]>([ + ['deploy-token', 'deploy'], + ['monitor-token', 'monitor'], + ['fence-token', 'fence'], + ['capacity-token', 'capacity'] + ]) + return { + ...actual, + // The real route allow-list behind a token-to-identity stub. + createAdminTokenVerifier: () => async (token: string, route?: string) => { + const identity = identities.get(token) + if (identity === undefined) return false + return !route || actual.relayAdminIdentityMayAccess(identity, route) + }, + createReadOnlyAdminTokenVerifier: () => async () => false, + createRegionalRehomeControlApplyTokenVerifier: () => async () => false, + createRegionalRehomeRuntimeTokenVerifier: () => async () => false, + createRegionalRehomeTokenVerifier: () => async () => false, + createRuntimeTokenVerifier: () => async () => false + } +}) + +vi.mock('./relay-token-verifier.js', () => ({ + createRelayTokenVerifier: () => async () => null, + readBearer: (value: string | undefined) => value?.replace(/^Bearer /, '') ?? null +})) + +import { createRelayApp } from './app.js' + +const HOST = 'hhhhhhhhhhhhhhhh' +const ROW: HostWhereaboutsRow = { + cellId: 'cell-a', + assignmentEpoch: 3, + leaseExpiresAt: 5_000, + openMigration: null +} + +function request(body: unknown, token = 'deploy-token'): RequestInit { + return { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify(body) + } +} + +function directory(): ShadowSeatDirectory { + const shadow = new ShadowSeatDirectory() + shadow.setCells(['cell-a', 'cell-b']) + shadow.apply( + 'cell-a', + { + v: 1, + cellId: 'cell-a', + incarnation: 'inc-a', + seq: 1, + at: 1, + flagsApplied: { generation: '7', flags: { readinessLocal: false } }, + full: [ + { + userId: 'user-a', + relayHostId: HOST, + epoch: 3, + generation: 1, + state: 'active', + joinedAt: 1 + } + ] + }, + Date.now() + ) + shadow.markNoFeed('cell-b', Date.now()) + return shadow +} + +function appWith( + hostWhereabouts: (identity: { userId: string; relayHostId: string }) => Promise, + options: { shadowSeats?: ShadowSeatDirectory; role?: RelayConfig['role'] } = {} +) { + return createRelayApp(config(options.role), { + store: {} as never, + // SAFETY: the whereabouts route reads only hostWhereabouts. + assignments: { hostWhereabouts } as never, + drain: vi.fn(), + ready: vi.fn(async () => true), + cellIncarnation: 'director-incarnation', + shadowSeats: options.shadowSeats + }) +} + +describe('POST /v1/admin/host-whereabouts', () => { + it('requires admin auth and a director', async () => { + const lookup = vi.fn(async () => ROW) + const app = appWith(lookup) + const body = { v: 1, relayHostId: HOST, userId: 'user-a' } + + expect((await app.request('/v1/admin/host-whereabouts', request(body, 'other'))).status).toBe( + 401 + ) + const cell = appWith(lookup, { role: 'cell' }) + expect((await cell.request('/v1/admin/host-whereabouts', request(body))).status).toBe(404) + expect( + (await app.request('/v1/admin/host-whereabouts', request({ ...body, extra: 1 }))).status + ).toBe(400) + expect(lookup).not.toHaveBeenCalled() + }) + + it('lets the monitor and fence accounts read it, and not the capacity account', async () => { + const app = appWith(async () => ROW) + const body = { v: 1, relayHostId: HOST, userId: 'user-a' } + for (const token of ['deploy-token', 'monitor-token', 'fence-token']) { + const response = await app.request('/v1/admin/host-whereabouts', request(body, token)) + expect(response.status).toBe(200) + } + const capacity = await app.request( + '/v1/admin/host-whereabouts', + request(body, 'capacity-token') + ) + expect(capacity.status).toBe(401) + }) + + it('returns the database row, the map seats, the verdict and the named cells', async () => { + const lookup = vi.fn(async () => ROW) + const app = appWith(lookup, { shadowSeats: directory() }) + + const response = await app.request( + '/v1/admin/host-whereabouts', + request({ v: 1, relayHostId: HOST, userId: 'user-a' }) + ) + + expect(response.status).toBe(200) + expect(lookup).toHaveBeenCalledWith({ userId: 'user-a', relayHostId: HOST }) + expect(await response.json()).toMatchObject({ + v: 1, + directorIncarnation: 'director-incarnation', + relayHostId: HOST, + shadow: { state: 'on', complete: true }, + users: [ + { + userId: 'user-a', + database: ROW, + seats: [{ cellId: 'cell-a', epoch: 3, state: 'active' }], + recentlyLeft: [], + verdict: { class: 'agree', explained: true } + } + ], + cells: { + 'cell-a': { + status: 'live', + lastFailure: null, + flagsApplied: { generation: '7', flags: { readinessLocal: false } } + } + } + }) + }) + + it('names the user from the map when the request has only a host id', async () => { + const lookup = vi.fn(async () => ROW) + const app = appWith(lookup, { shadowSeats: directory() }) + + const response = await app.request( + '/v1/admin/host-whereabouts', + request({ v: 1, relayHostId: HOST }) + ) + + expect(lookup).toHaveBeenCalledWith({ userId: 'user-a', relayHostId: HOST }) + expect(await response.json()).toMatchObject({ users: [{ userId: 'user-a', database: ROW }] }) + }) + + it('judges the verdict as /v1/resolve would when the row names a dead cell', async () => { + const shadow = directory() + // cell-a's heartbeat had already run out when the cell list was read. + shadow.setCells(['cell-a', 'cell-b'], ['cell-a', 'cell-b'], { + readAt: Date.now(), + expiresAt: new Map([ + ['cell-a', Date.now() - 1], + ['cell-b', Date.now() + 60_000] + ]) + }) + const app = appWith(async () => ROW, { shadowSeats: shadow }) + + const response = await app.request( + '/v1/admin/host-whereabouts', + request({ v: 1, relayHostId: HOST, userId: 'user-a' }) + ) + + expect(await response.json()).toMatchObject({ + users: [{ database: ROW, verdict: { class: 'map-only-cell-unlive', explained: true } }] + }) + }) + + it('answers an unknown host with an empty result, not a 404', async () => { + const lookup = vi.fn(async () => null) + const shadowOff = appWith(lookup) + const unknown = await shadowOff.request( + '/v1/admin/host-whereabouts', + request({ v: 1, relayHostId: 'uuuuuuuuuuuuuuuu' }) + ) + expect(unknown.status).toBe(200) + expect(await unknown.json()).toEqual({ + v: 1, + directorIncarnation: 'director-incarnation', + relayHostId: 'uuuuuuuuuuuuuuuu', + shadow: { state: 'off' }, + users: [], + cells: {} + }) + expect(lookup).not.toHaveBeenCalled() + + const named = await shadowOff.request( + '/v1/admin/host-whereabouts', + request({ v: 1, relayHostId: 'uuuuuuuuuuuuuuuu', userId: 'user-z' }) + ) + expect(await named.json()).toMatchObject({ + users: [{ userId: 'user-z', database: null, seats: [], recentlyLeft: [], verdict: null }] + }) + }) + + it('answers a briefly unreachable database with a retryable 503', async () => { + const app = appWith(async () => { + throw new Error('Connection terminated due to connection timeout') + }) + const response = await app.request( + '/v1/admin/host-whereabouts', + request({ v: 1, relayHostId: HOST, userId: 'user-a' }) + ) + expect(response.status).toBe(503) + }) +}) + +describe('RelayAssignmentStore.hostWhereabouts', () => { + it('reads the assignment row and its open migration by primary key', async () => { + const database = await openInMemoryRelayDatabase() + const store = new RelayAssignmentStore(database, () => 100) + await store.reconcileCells([ + { id: 'cell-a', url: 'https://cell-a.example.test', capacityRequests: 10 }, + { id: 'cell-b', url: 'https://cell-b.example.test', capacityRequests: 10 } + ]) + const identity = { userId: 'user-a', relayHostId: HOST } + expect(await store.hostWhereabouts(identity)).toBeNull() + + const assigned = await store.assign(identity) + expect(await store.hostWhereabouts(identity)).toEqual({ + cellId: assigned.cellId, + assignmentEpoch: assigned.assignmentEpoch, + leaseExpiresAt: assigned.leaseExpiresAt, + openMigration: null + }) + + await database.query( + `INSERT INTO relay_assignment_migrations + (user_id, relay_host_id, source_cell_id, target_cell_id, previous_epoch, assignment_epoch, + source_request_units, target_reserved_units, expires_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ['user-a', HOST, 'cell-a', 'cell-b', 1, 2, 1, 1, 900, 100, 100] + ) + expect((await store.hostWhereabouts(identity))?.openMigration).toEqual({ + sourceCellId: 'cell-a', + targetCellId: 'cell-b', + previousEpoch: 1, + assignmentEpoch: 2, + expiresAt: 900, + targetRegisteredAt: null + }) + }) +}) + +function config(role: RelayConfig['role'] = 'director'): RelayConfig { + return { + port: 8080, + publicUrl: 'https://relay.example.test', + cellUrl: 'https://relay.example.test', + authIssuer: 'https://auth.example.test', + authAudience: 'orca-relay', + jwksUrl: 'https://auth.example.test/jwks', + assignmentSigningKey: new TextEncoder().encode('assignment-key-with-at-least-32-bytes'), + role, + cellId: role === 'cell' ? 'cell-a' : 'director', + cells: [], + adminAudience: 'https://relay.example.test/v1/admin/drain', + deployServiceAccount: 'deploy@example.test', + runtimeServiceAccount: 'runtime@example.test', + adminJwksUrl: 'https://auth.example.test/jwks', + databasePoolMax: 3, + publicAssignmentsEnabled: true, + publicAssignmentConcurrency: 2, + publicAssignmentQueueMax: 128, + publicAssignmentWaitMs: 4_000, + publicResolveConcurrency: 1, + publicResolveWaitMs: 5_000, + publicAssignmentRetryAfterSeconds: 5, + dataDir: './data' + } +} diff --git a/cloud/apps/relay/src/relay-server.ts b/cloud/apps/relay/src/relay-server.ts index 61d3da6f34b..821ac12df6d 100644 --- a/cloud/apps/relay/src/relay-server.ts +++ b/cloud/apps/relay/src/relay-server.ts @@ -227,6 +227,7 @@ export function createRelayServer( observability.recordAdmissionServiceMs?.(lane, durationMs), recordAssignmentUnavailable: (cause) => observability.recordAssignmentUnavailable?.(cause), recordRegionRequest: (region) => observability.recordRegionRequest?.(region), + shadowSeats: shadowSeatPoller?.directory, compareShadowSeats: shadowCompare ? (route, identity, answer) => { shadowCompare.compare(route, identity, answer) diff --git a/cloud/apps/relay/src/shadow-seat-directory.ts b/cloud/apps/relay/src/shadow-seat-directory.ts index 9a784f48533..4785dbe4766 100644 --- a/cloud/apps/relay/src/shadow-seat-directory.ts +++ b/cloud/apps/relay/src/shadow-seat-directory.ts @@ -258,6 +258,18 @@ export class ShadowSeatDirectory { return [...(this.hosts.get(hostKey(userId, relayHostId))?.values() ?? [])] } + // Users the map has seen with this host id; a linear scan, for admin reads only. + userIdsOf(relayHostId: string): string[] { + const suffix = `\u0000${relayHostId}` + const userIds = new Set() + for (const keys of [this.hosts.keys(), this.recentlyLeft.keys()]) { + for (const key of keys) { + if (key.endsWith(suffix)) userIds.add(key.slice(0, -suffix.length)) + } + } + return [...userIds].sort() + } + recentlyLeftOf(userId: string, relayHostId: string, now: number): RecentlyLeftSeat[] { return (this.recentlyLeft.get(hostKey(userId, relayHostId)) ?? []).filter( (entry) => entry.at > now - SHADOW_SEAT_RECENTLY_LEFT_TTL_MS