feat(relay): read-only /v1/admin/host-whereabouts on directors (step 3) (#26816)

On-call can ask a director where a host is: its relay_assignments row and
open migration (primary-key reads only), the shadow map's seats and recent
leaves with close codes, the compare verdict (pure classification, judged as
/v1/resolve sees the host, recording nothing), and each named cell's poll
status and applied flags. shadow is {state:'off'} when the poller is off.
Deploy, monitor and fence identities may read it; capacity and asia-proof
may not. With no user id, the map names the user; an unknown host answers
empty.

cloud/apps/relay/src/admin-token-verifier.test.ts

@@ -67,6 +67,13 @@ describe('Relay admin route authorization', () => {
       expect(relayAdminIdentityMayAccess('monitor', route)).toBe(false)
     }
     expect(RELAY_MONITOR_ADMIN_ROUTES).toContain('/v1/admin/evacuation-status')
+    // Read-only on-call lookup: monitor and fence read it, capacity and asia-proof do not.
+    for (const identity of ['monitor', 'fence', 'fence-broker', 'deploy'] as const) {
+      expect(relayAdminIdentityMayAccess(identity, '/v1/admin/host-whereabouts')).toBe(true)
+    }
+    for (const identity of ['capacity', 'asia-proof'] as const) {
+      expect(relayAdminIdentityMayAccess(identity, '/v1/admin/host-whereabouts')).toBe(false)
+    }
   })
 
   it('allows only reviewed fence evidence mutations beyond aggregate reads', () => {

cloud/apps/relay/src/admin-token-verifier.ts

@@ -5,6 +5,7 @@ export const RELAY_MONITOR_ADMIN_ROUTES = [
   '/v1/admin/admission-selector/status',
   '/v1/admin/cell-status',
   '/v1/admin/evacuation-status',
+  '/v1/admin/host-whereabouts',
   '/v1/admin/regional-rehome-control',
   '/v1/admin/regional-rehome-preview',
   '/v1/admin/runtime-status'

cloud/apps/relay/src/app.ts

@@ -56,10 +56,12 @@ import {
 } from './public-assignment-admission.js'
 import { relayHostLogDigest } from './relay-host-log-digest.js'
 import type { RelayLocalReadinessVerdict } from './relay-local-readiness.js'
-import type {
-  ShadowCompareDatabaseAnswer,
-  ShadowCompareRoute
+import {
+  classifyShadowSeat,
+  type ShadowCompareDatabaseAnswer,
+  type ShadowCompareRoute
 } from './shadow-directory-compare.js'
+import type { ShadowSeatDirectory } from './shadow-seat-directory.js'
 import type { RelayReadinessDependency } from './relay-readiness.js'
 import type {
   AssignmentAdmissionLane,
@@ -143,6 +145,7 @@ export function createRelayApp(
     recordAssignmentUnavailable?: (cause: AssignmentUnavailableCause) => void
     recordRegionRequest?: (region: RelayRegion | undefined) => void
     // Step 3 shadow directory; absent unless the director polls seat feeds.
+    shadowSeats?: ShadowSeatDirectory
     compareShadowSeats?: (
       route: ShadowCompareRoute,
       identity: { userId: string; relayHostId: string },
@@ -1530,6 +1533,85 @@ export function createRelayApp(
       return context.json({ error: 'database_temporarily_unavailable' }, 503)
     }
   })
+  // Per director instance: tooling may ask more than one to see every map.
+  app.post('/v1/admin/host-whereabouts', async (context) => {
+    const bearer = readBearer(context.req.header('authorization'))
+    if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+    if (!bearer || !(await verifyAdminToken(bearer, context.req.path))) {
+      return context.json({ error: 'invalid_token' }, 401)
+    }
+    if (requestTooLarge(context.req.header('content-length'))) {
+      return context.json({ error: 'request_too_large' }, 413)
+    }
+    const body = AdminHostWhereaboutsSchema.safeParse(await context.req.json().catch(() => null))
+    if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+    const { relayHostId } = body.data
+    const shadow = operations.shadowSeats
+    const now = Date.now()
+    // Without a user id only the map can name one: relay_assignments has no host-id index.
+    const userIds = body.data.userId ? [body.data.userId] : (shadow?.userIdsOf(relayHostId) ?? [])
+    try {
+      const cellIds = new Set<string>()
+      const users = []
+      for (const userId of userIds) {
+        const identity = { userId, relayHostId }
+        const database = await operations.assignments.hostWhereabouts(identity)
+        const seats = shadow?.seatsOf(userId, relayHostId) ?? []
+        const recentlyLeft = shadow?.recentlyLeftOf(userId, relayHostId, now) ?? []
+        if (database) cellIds.add(database.cellId)
+        if (database?.openMigration) {
+          cellIds.add(database.openMigration.sourceCellId)
+          cellIds.add(database.openMigration.targetCellId)
+        }
+        for (const entry of [...seats, ...recentlyLeft]) cellIds.add(entry.cellId)
+        users.push({
+          userId,
+          database,
+          seats,
+          recentlyLeft,
+          // As /v1/resolve sees it: a row on a cell that is not live reads as no answer.
+          verdict: shadow
+            ? classifyShadowSeat(
+                shadow,
+                identity,
+                database && shadow.cellLiveness(database.cellId, now) !== 'unlive'
+                  ? database
+                  : null,
+                now
+              )
+            : null
+        })
+      }
+      const cells = Object.fromEntries(
+        [...cellIds].sort().map((cellId) => {
+          const state = shadow?.cellState(cellId)
+          return [
+            cellId,
+            state
+              ? {
+                  status: state.status,
+                  pollAgeMs: state.lastLiveAt === undefined ? null : now - state.lastLiveAt,
+                  lastFailure: state.lastFailure ?? null,
+                  draining: state.draining ?? null,
+                  flagsApplied: state.flagsApplied ?? null
+                }
+              : null
+          ]
+        })
+      )
+      return context.json({
+        v: 1,
+        directorIncarnation: operations.cellIncarnation ?? null,
+        relayHostId,
+        shadow: shadow ? { state: 'on', complete: shadow.isComplete() } : { state: 'off' },
+        users,
+        cells
+      })
+    } catch (error) {
+      if (!isRelayDatabaseTransientError(error)) throw error
+      return context.json({ error: 'database_temporarily_unavailable' }, 503)
+    }
+  })
   app.post('/v1/admin/cell-status', async (context) => {
     const bearer = readBearer(context.req.header('authorization'))
     if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
@@ -1551,6 +1633,14 @@ export function createRelayApp(
   return app
 }
 
+const AdminHostWhereaboutsSchema = z
+  .object({
+    v: z.literal(1),
+    relayHostId: z.string().regex(/^[A-Za-z0-9_-]{16}$/),
+    userId: z.string().min(1).max(256).optional()
+  })
+  .strict()
+
 const AdminAssignmentMoveSchema = z
   .object({
     v: z.literal(1),

cloud/apps/relay/src/assignment-store.ts

@@ -152,6 +152,20 @@ export type RelayAssignment = AssignmentIdentity & {
 // a roll right now: its next assign re-places it rather than re-granting the pin.
 export type ResolvedRelayAssignment = RelayAssignment & { homeCellRollIsolated?: true }
 
+export type HostWhereaboutsRow = {
+  cellId: string
+  assignmentEpoch: number
+  leaseExpiresAt: number
+  openMigration: {
+    sourceCellId: string
+    targetCellId: string
+    previousEpoch: number
+    assignmentEpoch: number
+    expiresAt: number
+    targetRegisteredAt: number | null
+  } | null
+}
+
 export type RelayRegionCatalogEntry = {
   region: RelayRegion
   probeOrigins: string[]
@@ -7861,6 +7875,42 @@ export class RelayAssignmentStore {
     })
   }
 
+  // On-call "where is host X: primary-key reads only, no liveness filter.
+  async hostWhereabouts(identity: AssignmentIdentity): Promise<HostWhereaboutsRow | null> {
+    const row = (
+      await this.database.query(
+        `SELECT cell_id, assignment_epoch, lease_expires_at FROM relay_assignments
+         WHERE user_id = ? AND relay_host_id = ?`,
+        [identity.userId, identity.relayHostId]
+      )
+    )[0]
+    if (!row) return null
+    const migrations = await this.database.query(
+      `SELECT source_cell_id, target_cell_id, previous_epoch, assignment_epoch, expires_at,
+              target_registered_at
+       FROM relay_assignment_migrations
+       WHERE user_id = ? AND relay_host_id = ? AND completed_at IS NULL AND aborted_at IS NULL
+       ORDER BY assignment_epoch DESC LIMIT 1`,
+      [identity.userId, identity.relayHostId]
+    )
+    const migration = migrations[0]
+    return {
+      cellId: text(row, 'cell_id'),
+      assignmentEpoch: integer(row, 'assignment_epoch'),
+      leaseExpiresAt: integer(row, 'lease_expires_at'),
+      openMigration: migration
+        ? {
+            sourceCellId: text(migration, 'source_cell_id'),
+            targetCellId: text(migration, 'target_cell_id'),
+            previousEpoch: integer(migration, 'previous_epoch'),
+            assignmentEpoch: integer(migration, 'assignment_epoch'),
+            expiresAt: integer(migration, 'expires_at'),
+            targetRegisteredAt: optionalInteger(migration, 'target_registered_at') ?? null
+          }
+        : null
+    }
+  }
+
   async regionCatalog(): Promise<RelayRegionCatalogEntry[]> {
     const rows = await this.database.query(
       `SELECT cell.cell_id, cell.cell_url, region.region

cloud/apps/relay/src/host-whereabouts-app.test.ts

@@ -0,0 +1,302 @@
+import { describe, expect, it, vi } from 'vitest'
+import type { HostWhereaboutsRow } from './assignment-store.js'
+import { RelayAssignmentStore } from './assignment-store.js'
+import type { RelayConfig } from './config.js'
+import { openInMemoryRelayDatabase } from './database.js'
+import { ShadowSeatDirectory } from './shadow-seat-directory.js'
+
+vi.mock('./admin-token-verifier.js', async (importOriginal) => {
+  const actual = await importOriginal<typeof import('./admin-token-verifier.js')>()
+  const identities = new Map<string, Parameters<typeof actual.relayAdminIdentityMayAccess>[0]>([
+    ['deploy-token', 'deploy'],
+    ['monitor-token', 'monitor'],
+    ['fence-token', 'fence'],
+    ['capacity-token', 'capacity']
+  ])
+  return {
+    ...actual,
+    // The real route allow-list behind a token-to-identity stub.
+    createAdminTokenVerifier: () => async (token: string, route?: string) => {
+      const identity = identities.get(token)
+      if (identity === undefined) return false
+      return !route || actual.relayAdminIdentityMayAccess(identity, route)
+    },
+    createReadOnlyAdminTokenVerifier: () => async () => false,
+    createRegionalRehomeControlApplyTokenVerifier: () => async () => false,
+    createRegionalRehomeRuntimeTokenVerifier: () => async () => false,
+    createRegionalRehomeTokenVerifier: () => async () => false,
+    createRuntimeTokenVerifier: () => async () => false
+  }
+})
+
+vi.mock('./relay-token-verifier.js', () => ({
+  createRelayTokenVerifier: () => async () => null,
+  readBearer: (value: string | undefined) => value?.replace(/^Bearer /, '') ?? null
+}))
+
+import { createRelayApp } from './app.js'
+
+const HOST = 'hhhhhhhhhhhhhhhh'
+const ROW: HostWhereaboutsRow = {
+  cellId: 'cell-a',
+  assignmentEpoch: 3,
+  leaseExpiresAt: 5_000,
+  openMigration: null
+}
+
+function request(body: unknown, token = 'deploy-token'): RequestInit {
+  return {
+    method: 'POST',
+    headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
+    body: JSON.stringify(body)
+  }
+}
+
+function directory(): ShadowSeatDirectory {
+  const shadow = new ShadowSeatDirectory()
+  shadow.setCells(['cell-a', 'cell-b'])
+  shadow.apply(
+    'cell-a',
+    {
+      v: 1,
+      cellId: 'cell-a',
+      incarnation: 'inc-a',
+      seq: 1,
+      at: 1,
+      flagsApplied: { generation: '7', flags: { readinessLocal: false } },
+      full: [
+        {
+          userId: 'user-a',
+          relayHostId: HOST,
+          epoch: 3,
+          generation: 1,
+          state: 'active',
+          joinedAt: 1
+        }
+      ]
+    },
+    Date.now()
+  )
+  shadow.markNoFeed('cell-b', Date.now())
+  return shadow
+}
+
+function appWith(
+  hostWhereabouts: (identity: { userId: string; relayHostId: string }) => Promise<unknown>,
+  options: { shadowSeats?: ShadowSeatDirectory; role?: RelayConfig['role'] } = {}
+) {
+  return createRelayApp(config(options.role), {
+    store: {} as never,
+    // SAFETY: the whereabouts route reads only hostWhereabouts.
+    assignments: { hostWhereabouts } as never,
+    drain: vi.fn(),
+    ready: vi.fn(async () => true),
+    cellIncarnation: 'director-incarnation',
+    shadowSeats: options.shadowSeats
+  })
+}
+
+describe('POST /v1/admin/host-whereabouts', () => {
+  it('requires admin auth and a director', async () => {
+    const lookup = vi.fn(async () => ROW)
+    const app = appWith(lookup)
+    const body = { v: 1, relayHostId: HOST, userId: 'user-a' }
+
+    expect((await app.request('/v1/admin/host-whereabouts', request(body, 'other'))).status).toBe(
+      401
+    )
+    const cell = appWith(lookup, { role: 'cell' })
+    expect((await cell.request('/v1/admin/host-whereabouts', request(body))).status).toBe(404)
+    expect(
+      (await app.request('/v1/admin/host-whereabouts', request({ ...body, extra: 1 }))).status
+    ).toBe(400)
+    expect(lookup).not.toHaveBeenCalled()
+  })
+
+  it('lets the monitor and fence accounts read it, and not the capacity account', async () => {
+    const app = appWith(async () => ROW)
+    const body = { v: 1, relayHostId: HOST, userId: 'user-a' }
+    for (const token of ['deploy-token', 'monitor-token', 'fence-token']) {
+      const response = await app.request('/v1/admin/host-whereabouts', request(body, token))
+      expect(response.status).toBe(200)
+    }
+    const capacity = await app.request(
+      '/v1/admin/host-whereabouts',
+      request(body, 'capacity-token')
+    )
+    expect(capacity.status).toBe(401)
+  })
+
+  it('returns the database row, the map seats, the verdict and the named cells', async () => {
+    const lookup = vi.fn(async () => ROW)
+    const app = appWith(lookup, { shadowSeats: directory() })
+
+    const response = await app.request(
+      '/v1/admin/host-whereabouts',
+      request({ v: 1, relayHostId: HOST, userId: 'user-a' })
+    )
+
+    expect(response.status).toBe(200)
+    expect(lookup).toHaveBeenCalledWith({ userId: 'user-a', relayHostId: HOST })
+    expect(await response.json()).toMatchObject({
+      v: 1,
+      directorIncarnation: 'director-incarnation',
+      relayHostId: HOST,
+      shadow: { state: 'on', complete: true },
+      users: [
+        {
+          userId: 'user-a',
+          database: ROW,
+          seats: [{ cellId: 'cell-a', epoch: 3, state: 'active' }],
+          recentlyLeft: [],
+          verdict: { class: 'agree', explained: true }
+        }
+      ],
+      cells: {
+        'cell-a': {
+          status: 'live',
+          lastFailure: null,
+          flagsApplied: { generation: '7', flags: { readinessLocal: false } }
+        }
+      }
+    })
+  })
+
+  it('names the user from the map when the request has only a host id', async () => {
+    const lookup = vi.fn(async () => ROW)
+    const app = appWith(lookup, { shadowSeats: directory() })
+
+    const response = await app.request(
+      '/v1/admin/host-whereabouts',
+      request({ v: 1, relayHostId: HOST })
+    )
+
+    expect(lookup).toHaveBeenCalledWith({ userId: 'user-a', relayHostId: HOST })
+    expect(await response.json()).toMatchObject({ users: [{ userId: 'user-a', database: ROW }] })
+  })
+
+  it('judges the verdict as /v1/resolve would when the row names a dead cell', async () => {
+    const shadow = directory()
+    // cell-a's heartbeat had already run out when the cell list was read.
+    shadow.setCells(['cell-a', 'cell-b'], ['cell-a', 'cell-b'], {
+      readAt: Date.now(),
+      expiresAt: new Map([
+        ['cell-a', Date.now() - 1],
+        ['cell-b', Date.now() + 60_000]
+      ])
+    })
+    const app = appWith(async () => ROW, { shadowSeats: shadow })
+
+    const response = await app.request(
+      '/v1/admin/host-whereabouts',
+      request({ v: 1, relayHostId: HOST, userId: 'user-a' })
+    )
+
+    expect(await response.json()).toMatchObject({
+      users: [{ database: ROW, verdict: { class: 'map-only-cell-unlive', explained: true } }]
+    })
+  })
+
+  it('answers an unknown host with an empty result, not a 404', async () => {
+    const lookup = vi.fn(async () => null)
+    const shadowOff = appWith(lookup)
+    const unknown = await shadowOff.request(
+      '/v1/admin/host-whereabouts',
+      request({ v: 1, relayHostId: 'uuuuuuuuuuuuuuuu' })
+    )
+    expect(unknown.status).toBe(200)
+    expect(await unknown.json()).toEqual({
+      v: 1,
+      directorIncarnation: 'director-incarnation',
+      relayHostId: 'uuuuuuuuuuuuuuuu',
+      shadow: { state: 'off' },
+      users: [],
+      cells: {}
+    })
+    expect(lookup).not.toHaveBeenCalled()
+
+    const named = await shadowOff.request(
+      '/v1/admin/host-whereabouts',
+      request({ v: 1, relayHostId: 'uuuuuuuuuuuuuuuu', userId: 'user-z' })
+    )
+    expect(await named.json()).toMatchObject({
+      users: [{ userId: 'user-z', database: null, seats: [], recentlyLeft: [], verdict: null }]
+    })
+  })
+
+  it('answers a briefly unreachable database with a retryable 503', async () => {
+    const app = appWith(async () => {
+      throw new Error('Connection terminated due to connection timeout')
+    })
+    const response = await app.request(
+      '/v1/admin/host-whereabouts',
+      request({ v: 1, relayHostId: HOST, userId: 'user-a' })
+    )
+    expect(response.status).toBe(503)
+  })
+})
+
+describe('RelayAssignmentStore.hostWhereabouts', () => {
+  it('reads the assignment row and its open migration by primary key', async () => {
+    const database = await openInMemoryRelayDatabase()
+    const store = new RelayAssignmentStore(database, () => 100)
+    await store.reconcileCells([
+      { id: 'cell-a', url: 'https://cell-a.example.test', capacityRequests: 10 },
+      { id: 'cell-b', url: 'https://cell-b.example.test', capacityRequests: 10 }
+    ])
+    const identity = { userId: 'user-a', relayHostId: HOST }
+    expect(await store.hostWhereabouts(identity)).toBeNull()
+
+    const assigned = await store.assign(identity)
+    expect(await store.hostWhereabouts(identity)).toEqual({
+      cellId: assigned.cellId,
+      assignmentEpoch: assigned.assignmentEpoch,
+      leaseExpiresAt: assigned.leaseExpiresAt,
+      openMigration: null
+    })
+
+    await database.query(
+      `INSERT INTO relay_assignment_migrations
+       (user_id, relay_host_id, source_cell_id, target_cell_id, previous_epoch, assignment_epoch,
+        source_request_units, target_reserved_units, expires_at, created_at, updated_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      ['user-a', HOST, 'cell-a', 'cell-b', 1, 2, 1, 1, 900, 100, 100]
+    )
+    expect((await store.hostWhereabouts(identity))?.openMigration).toEqual({
+      sourceCellId: 'cell-a',
+      targetCellId: 'cell-b',
+      previousEpoch: 1,
+      assignmentEpoch: 2,
+      expiresAt: 900,
+      targetRegisteredAt: null
+    })
+  })
+})
+
+function config(role: RelayConfig['role'] = 'director'): RelayConfig {
+  return {
+    port: 8080,
+    publicUrl: 'https://relay.example.test',
+    cellUrl: 'https://relay.example.test',
+    authIssuer: 'https://auth.example.test',
+    authAudience: 'orca-relay',
+    jwksUrl: 'https://auth.example.test/jwks',
+    assignmentSigningKey: new TextEncoder().encode('assignment-key-with-at-least-32-bytes'),
+    role,
+    cellId: role === 'cell' ? 'cell-a' : 'director',
+    cells: [],
+    adminAudience: 'https://relay.example.test/v1/admin/drain',
+    deployServiceAccount: 'deploy@example.test',
+    runtimeServiceAccount: 'runtime@example.test',
+    adminJwksUrl: 'https://auth.example.test/jwks',
+    databasePoolMax: 3,
+    publicAssignmentsEnabled: true,
+    publicAssignmentConcurrency: 2,
+    publicAssignmentQueueMax: 128,
+    publicAssignmentWaitMs: 4_000,
+    publicResolveConcurrency: 1,
+    publicResolveWaitMs: 5_000,
+    publicAssignmentRetryAfterSeconds: 5,
+    dataDir: './data'
+  }
+}

cloud/apps/relay/src/relay-server.ts

@@ -227,6 +227,7 @@ export function createRelayServer(
       observability.recordAdmissionServiceMs?.(lane, durationMs),
     recordAssignmentUnavailable: (cause) => observability.recordAssignmentUnavailable?.(cause),
     recordRegionRequest: (region) => observability.recordRegionRequest?.(region),
+    shadowSeats: shadowSeatPoller?.directory,
     compareShadowSeats: shadowCompare
       ? (route, identity, answer) => {
           shadowCompare.compare(route, identity, answer)

cloud/apps/relay/src/shadow-seat-directory.ts

@@ -258,6 +258,18 @@ export class ShadowSeatDirectory {
     return [...(this.hosts.get(hostKey(userId, relayHostId))?.values() ?? [])]
   }
 
+  // Users the map has seen with this host id; a linear scan, for admin reads only.
+  userIdsOf(relayHostId: string): string[] {
+    const suffix = `\u0000${relayHostId}`
+    const userIds = new Set<string>()
+    for (const keys of [this.hosts.keys(), this.recentlyLeft.keys()]) {
+      for (const key of keys) {
+        if (key.endsWith(suffix)) userIds.add(key.slice(0, -suffix.length))
+      }
+    }
+    return [...userIds].sort()
+  }
+
   recentlyLeftOf(userId: string, relayHostId: string, now: number): RecentlyLeftSeat[] {
     return (this.recentlyLeft.get(hostKey(userId, relayHostId)) ?? []).filter(
       (entry) => entry.at > now - SHADOW_SEAT_RECENTLY_LEFT_TTL_MS