starred/orca
1
0
Fork 0
mirror of https://github.com/stablyai/orca.git synced 2026-10-03 08:02:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

merge main and refresh mobile route parity

Browse Source
This commit is contained in:
Merge Sim
2026-09-11 12:10:23 -07:00
parent 37568ef729 729491597f
commit b08ec19dc8
414 changed files with 22449 additions and 6183 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitattributes
+4
View File
@@ -31,6 +31,10 @@
# the reviewable change, and pin LF because they are compared byte-for-byte.
# Not -diff: the shell diff is the review surface when a wrapper does change.
/src/main/__fixtures__/shell-wrapper-snapshots/*.txt linguist-generated=true text eol=lf
# Captured agent PTY transcripts. -text, not `text eol=lf` like the wrapper snapshots above:
# these carry real CR and CRLF bytes as the terminal emitted them, and line-ending
# normalisation on a Windows checkout would rewrite the evidence the fixture exists to be.
/src/main/runtime/__fixtures__/*.txt -text
# Generated runtime English subset: compared byte-for-byte by
# verify:localization-runtime-catalog, so a CRLF checkout would fail the gate.
/src/renderer/src/i18n/en-runtime-required.json linguist-generated=true text eol=lf
.github/workflows/cloud-deploy-relay-production-director.yml
+12 -2
View File
@@ -13,6 +13,11 @@ on:
default: preserve
type: choice
options: [preserve, enable, disable]
region-correction-cohort-percent:
description: 'Preserve the measured-correction cohort, or set an integer 0–100; durable rehome stays disabled'
required: true
default: preserve
type: string
prune-incompatible-revisions:
description: Retain only the newly verified serving and rollback revisions
required: true
@@ -62,6 +67,7 @@ jobs:
REGIONAL_PLACEMENT_SECRET: orca-cloud-relay-regional-placement-enabled
IMAGE_DIGEST: ${{ inputs.image-digest }}
REGIONAL_PLACEMENT_MODE: ${{ inputs.regional-placement-mode }}
REGION_CORRECTION_COHORT_PERCENT: ${{ inputs.region-correction-cohort-percent }}
PRUNE_INCOMPATIBLE_REVISIONS: ${{ inputs.prune-incompatible-revisions }}
# Floor the served revision must keep, matching relay_min_instances in
# environments/production.tfvars. This gate only fails a bad deploy; Terraform
@@ -106,8 +112,11 @@ jobs:
echo "image-digest must be an immutable lowercase sha256 digest" >&2
exit 1
fi
if test "${REGION_CORRECTION_COHORT_PERCENT}" != preserve; then
[[ "${REGION_CORRECTION_COHORT_PERCENT}" =~ ^([0-9]|[1-9][0-9]|100)$ ]]
fi
IMAGE="${IMAGE_REPOSITORY}@${IMAGE_DIGEST}"
SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" --format='value(image_summary.digest)')"
SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
test "${SERVED_DIGEST}" = "${IMAGE_DIGEST}"
[[ "${PRUNE_INCOMPATIBLE_REVISIONS}" =~ ^(true|false)$ ]]
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
@@ -218,7 +227,8 @@ jobs:
--max-instances "${DIRECTOR_MAX_INSTANCES}" \
--prune-revisions "${PRUNE_INCOMPATIBLE_REVISIONS}" \
--release-id "${RELEASE_ID}" \
--regional-placement-secret-version "${target_version}"
--regional-placement-secret-version "${target_version}" \
--region-correction-cohort-percent "${REGION_CORRECTION_COHORT_PERCENT}"
echo "REGIONAL_PLACEMENT_ENABLED=${desired}" >> "${GITHUB_ENV}"
echo "REGIONAL_PLACEMENT_VERSION=${target_version}" >> "${GITHUB_ENV}"
.github/workflows/pi-owner-runtime.yml
+29
View File
@@ -0,0 +1,29 @@
name: Pi owner runtime verification
on:
pull_request:
paths:
- 'src/main/pi/agent-status-handler-source.ts'
- 'tests/tools/pi-owner-runtime-smoke.mjs'
- '.github/workflows/pi-owner-runtime.yml'
workflow_dispatch:
permissions:
contents: read
jobs:
runtime:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
timeout-minutes: 20
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
- name: Install pinned extension loader
run: npm install --prefix .cache/pi-owner --ignore-scripts --no-audit --no-fund @earendil-works/pi-coding-agent@0.83.0
- name: Verify real owner exit and hook delivery
run: node tests/tools/pi-owner-runtime-smoke.mjs .cache/pi-owner/node_modules/@earendil-works/pi-coding-agent
.github/workflows/pi-provider-runtime.yml
+28
View File
@@ -0,0 +1,28 @@
name: Pi extension provider verification
on:
pull_request:
paths:
- 'src/shared/commit-message-agent-specs-primary.ts'
- 'tests/tools/pi-provider-runtime-smoke.mjs'
- '.github/workflows/pi-provider-runtime.yml'
permissions:
contents: read
jobs:
runtime:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
timeout-minutes: 20
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
- name: Install pinned Pi runtime
run: npm install --prefix .cache/pi-provider --ignore-scripts --no-audit --no-fund @earendil-works/pi-coding-agent@0.84.2
- name: Verify extension model generation before and after
run: node tests/tools/pi-provider-runtime-smoke.mjs .cache/pi-provider/node_modules/@earendil-works/pi-coding-agent/dist/cli.js
.github/workflows/unit-tests.yml
+7
View File
@@ -37,6 +37,13 @@ jobs:
- name: Install Electron package binary for tests
run: node config/scripts/install-electron-package-binary.mjs
# The real two-cell transport test imports cloud relay source and its contracts.
- name: Install relay integration dependencies
working-directory: cloud
run: |
npx --yes pnpm@10.24.0 --filter '@orca-cloud/relay...' install --frozen-lockfile --ignore-scripts
npx --yes pnpm@10.24.0 --filter '@orca-cloud/relay^...' build
- name: Test shard
run: |
pnpm exec vitest run --config config/vitest.config.ts \
.gitignore
+2
View File
@@ -103,7 +103,9 @@ docs/**
!docs/agent-skill-sharing-implementation-checklist.md
!docs/mobile-terminal-shortcut-bar.md
!docs/reference/
!docs/reference/agent-pty-transcript-capture.md
!docs/reference/agent-status-store.md
!docs/reference/antigravity-readiness-evidence.md
!docs/reference/git-compatibility.md
!docs/reference/headless-linux-server.md
!docs/reference/ime-regression-checklist.md
AGENTS.md
+4
View File
@@ -72,6 +72,10 @@ All changes must consider folder workspaces as well as git worktrees. Don't assu
The execution host owns agent status in one store, the hook server's, and every reader (sidebar, `worktree ps`, mobile, dashboard) subscribes to it. Before adding a producer, a cache, or a reader-side precedence rule, read [`docs/reference/agent-status-store.md`](./docs/reference/agent-status-store.md): new producers write into that store, and readers keep only presentation policy.
## Agent Terminal Screens
A rule that reads what an agent CLI paints on a terminal — readiness, blocked prompts, idle — must be written against a captured transcript, not a remembered screen. Record one with [`docs/reference/agent-pty-transcript-capture.md`](./docs/reference/agent-pty-transcript-capture.md), which keeps escapes and wrapping intact and scrubs account identifiers before they reach git. Antigravity readiness has no transcript yet and five failed attempts without one; before touching it, read [`docs/reference/antigravity-readiness-evidence.md`](./docs/reference/antigravity-readiness-evidence.md).
## Remote Wire Compatibility
Clients and remote Orca servers update independently, so mixed versions are the normal state. Before changing anything a paired client and host exchange — RPC params, stream frames, or the content either side publishes over them — follow [`docs/reference/remote-wire-compatibility.md`](./docs/reference/remote-wire-compatibility.md). A new optional field is safe; a new stream opcode must be capability-negotiated because decoders drop unknown opcodes silently; and changing what the host publishes reaches old clients even with no wire change.
cloud/apps/push/src/fcm-client.test.ts
+13 -24
View File
@@ -1,6 +1,6 @@
import { createHash } from 'node:crypto'
import { describe, expect, it } from 'vitest'
import { fcmCollapseKey, FcmClient, type FcmRequest, type FcmResponse } from './fcm-client.js'
import { FcmClient, type FcmRequest, type FcmResponse } from './fcm-client.js'
import { buildPushDelivery } from './push-delivery-message.js'
const NOW = 1_700_000_000_000
@@ -20,6 +20,7 @@ function delivery(agentState: 'needs-input' | null = 'needs-input') {
agentState,
title: 'Agent needs input',
body: 'Waiting on your answer',
paneKey: 'tab-b:pane-1',
worktreeId: 'wt-1'
}
})
@@ -59,27 +60,14 @@ describe('fcm client', () => {
expect(JSON.parse(request.body)).toEqual({
message: {
token: TOKEN,
notification: { title: 'Agent needs input', body: 'Waiting on your answer' },
android: {
priority: 'HIGH',
ttl: '300s',
collapse_key: createHash('sha256')
.update(
createHash('sha256')
.update(JSON.stringify([HOST, 'note-1']))
.digest('hex')
)
.digest('hex')
.slice(0, 32),
notification: {
channel_id: 'orca-desktop',
tag: createHash('sha256')
.update(JSON.stringify([HOST, 'note-1']))
.digest('hex')
}
},
android: { priority: 'HIGH', ttl: '300s' },
data: {
title: 'Agent needs input',
message: 'Waiting on your answer',
tag: delivery().collapseId,
channelId: 'orca-desktop',
hostFingerprint: HOST,
paneKey: 'tab-b:pane-1',
worktreeId: 'wt-1',
notificationId: 'note-1',
notificationSeq: '7',
@@ -96,7 +84,7 @@ describe('fcm client', () => {
await fcm.send(delivery(null), { token: TOKEN })
const message = JSON.parse(fake.requests[0]!.body) as {
message: {
android: { collapse_key: string; notification: { tag: string } }
android: Record<string, unknown>
data: Record<string, string>
}
}
@@ -108,9 +96,10 @@ describe('fcm client', () => {
.update(JSON.stringify([HOST, 'note-1']))
.digest('hex')
expect(message.message.data.coalescedCount).toBeUndefined()
expect(message.message.android.notification.tag).toBe(tag)
expect(message.message.android.collapse_key).toBe(fcmCollapseKey(tag))
expect(message.message.android.collapse_key).toHaveLength(32)
expect(message.message.data.tag).toBe(tag)
expect(message.message.android).not.toHaveProperty('collapse_key')
expect(message.message).not.toHaveProperty('notification')
expect(message.message.data).not.toHaveProperty('body')
})
it('marks an unregistered token dead from the status or the error detail', async () => {
cloud/apps/push/src/fcm-client.ts
+11 -19
View File
@@ -1,5 +1,4 @@
import { providerRetryAfter } from './provider-retry-delay.js'
import { createHash } from 'node:crypto'
import { PUSH_DEFAULTS } from '@orca-cloud/push-contract'
import { orcaDataStrings, type PushDelivery } from './push-delivery-message.js'
import type { PushProviderOutcome } from './push-provider-outcome.js'
@@ -22,12 +21,6 @@ type FcmErrorBody = {
error?: { status?: unknown; message?: unknown; details?: { errorCode?: unknown }[] }
}
// FCM collapse_key is a short opaque string, so the collapse id is hashed
// rather than truncated: truncation would merge unrelated notifications.
export function fcmCollapseKey(collapseId: string): string {
return createHash('sha256').update(collapseId).digest('hex').slice(0, 32)
}
export function fcmMessageBody(input: {
delivery: PushDelivery
token: string
@@ -39,24 +32,23 @@ export function fcmMessageBody(input: {
return JSON.stringify({
message: {
token: input.token,
...(delivery.orca.kind === 'dismiss'
? {}
: { notification: { title: delivery.title, body: delivery.body } }),
android: {
priority: 'HIGH',
ttl: `${Math.max(0, Math.ceil((delivery.expiresAt - now) / 1000))}s`,
collapse_key: fcmCollapseKey(delivery.collapseId),
ttl: `${Math.max(0, Math.ceil((delivery.expiresAt - now) / 1000))}s`
},
// Notification payloads collapse offline; Expo renders these data messages natively.
data: {
...orcaDataStrings(delivery.orca),
...(delivery.orca.kind === 'dismiss'
? {}
: {
notification: {
channel_id:
delivery.sound === false ? `${input.channelId}-silent` : input.channelId,
tag: delivery.collapseId
}
title: delivery.title,
message: delivery.body,
tag: delivery.collapseId,
channelId: delivery.sound === false ? `${input.channelId}-silent` : input.channelId,
...(delivery.sound === false ? { sound: '' } : {})
})
},
data: orcaDataStrings(delivery.orca)
}
}
})
}
cloud/apps/push/src/push-delivery-message.ts
+2
View File
@@ -5,6 +5,7 @@ export type PushOrcaData = {
kind?: 'alert' | 'dismiss'
hostFingerprint: string
worktreeId?: string
paneKey?: string
notificationId?: string
notificationSeq: number
notificationEpoch: string
@@ -51,6 +52,7 @@ export function buildPushDelivery(input: {
orca: {
...(notification.kind ? { kind: notification.kind } : {}),
hostFingerprint,
...(notification.paneKey === undefined ? {} : { paneKey: notification.paneKey }),
...(notification.worktreeId === undefined ? {} : { worktreeId: notification.worktreeId }),
...(notification.notificationId === undefined
? {}
cloud/apps/push/src/push-dismissal-provider.test.ts
+4
View File
@@ -23,5 +23,9 @@ it('dismissal provider payloads cannot display a new alert or play a sound', ()
const android = JSON.parse(fcmMessageBody({ delivery, token: 'test', channelId: 'test' })).message
expect(android).not.toHaveProperty('notification')
expect(android.android).not.toHaveProperty('notification')
expect(android.android).not.toHaveProperty('collapse_key')
expect(android.data).not.toHaveProperty('title')
expect(android.data).not.toHaveProperty('message')
expect(android.data).not.toHaveProperty('sound')
expect(android.data.kind).toBe('dismiss')
})
cloud/apps/push/src/push-notification-sound.test.ts
+5 -1
View File
@@ -23,7 +23,11 @@ it('carries a silent preference through validation to APNs and Android payloads'
expect(JSON.parse(apnsBody(delivery)).aps).not.toHaveProperty('sound')
expect(
JSON.parse(fcmMessageBody({ delivery, token: 'test-token', channelId: 'orca-desktop' })).message
.android.notification.channel_id
.data.channelId
).toBe('orca-desktop-silent')
expect(
JSON.parse(fcmMessageBody({ delivery, token: 'test-token', channelId: 'orca-desktop' })).message
.data.sound
).toBe('')
expect(JSON.parse(apnsBody({ ...delivery, sound: undefined })).aps.sound).toBe('default')
})
cloud/apps/push/src/push-pane-routing.test.ts
+27
View File
@@ -0,0 +1,27 @@
import { expect, it } from 'vitest'
import { PushNotificationSchema } from '@orca-cloud/push-contract'
import { buildPushDelivery, orcaDataStrings } from './push-delivery-message.js'
it('preserves pane identity for both APNs and FCM, and accepts older workspace-only messages', () => {
const base = {
notificationSeq: 1,
notificationEpoch: 'epoch',
source: 'agent-task-complete',
agentState: 'finished',
title: 'Done',
body: '',
worktreeId: 'folder:/work'
}
const paneKey = 'tab-b:11111111-1111-4111-8111-111111111111'
for (const extra of [{}, { paneKey }]) {
const notification = PushNotificationSchema.parse({ ...base, ...extra })
const delivery = buildPushDelivery({
notification,
hostFingerprint: 'host',
registrationId: 'phone',
expiresAt: Date.now() + 300000
})
expect(delivery.orca.paneKey).toBe('paneKey' in extra ? paneKey : undefined)
expect(orcaDataStrings(delivery.orca).paneKey).toBe('paneKey' in extra ? paneKey : undefined)
}
})
cloud/apps/push/src/push-server-send.test.ts
+2 -2
View File
@@ -56,7 +56,7 @@ describe('push gateway send route', () => {
await harness.flushDeliveries()
expect(harness.fcmRequests).toHaveLength(1)
expect(JSON.parse(harness.fcmRequests[0]!.body)).toMatchObject({
message: { token: FCM_TOKEN, notification: { title: 'Agent needs input' } }
message: { token: FCM_TOKEN, data: { title: 'Agent needs input' } }
})
const afterDeath = await harness.post(
@@ -179,7 +179,7 @@ describe('push gateway send route', () => {
const message = JSON.parse(harness.fcmRequests[0]!.body) as {
message: { android: { notification: { tag: string } }; data: Record<string, string> }
}
expect(message.message.android.notification.tag).toMatch(/^[a-f0-9]{64}$/)
expect(message.message.data.tag).toMatch(/^[a-f0-9]{64}$/)
expect(message.message.data.coalescedCount).toBeUndefined()
})
cloud/apps/relay/src/admin-token-verifier.ts
+1
View File
@@ -6,6 +6,7 @@ export const RELAY_MONITOR_ADMIN_ROUTES = [
'/v1/admin/cell-status',
'/v1/admin/evacuation-status',
'/v1/admin/regional-rehome-control',
'/v1/admin/regional-rehome-preview',
'/v1/admin/runtime-status'
] as const
cloud/apps/relay/src/app.ts
+137 -51
View File
@@ -1,5 +1,9 @@
import {
AssignmentRequestSchema,
IdleRegionalRehomeRequestSchema,
type IdleRegionalRehomeRequest,
type IdleRegionalRehomeOutcome,
type RegionCorrectionResponse,
isRelayCellConnectionHardCap,
RELAY_ADMISSION_BUDGETS,
RELAY_DEFAULT_REGION,
@@ -39,7 +43,7 @@ import {
type AssignmentAdmissionRejection
} from './public-assignment-admission.js'
import { relayHostLogDigest } from './relay-host-log-digest.js'
import type { RelayRuntimeCounts } from './relay-observability.js'
import type { RegionalRehomeSafetySnapshot, RelayRuntimeCounts } from './relay-observability.js'
import {
isRegionalRehomeTrustProbe,
probeRegionalRehomeTrust
@@ -68,21 +72,28 @@ export function createRelayApp(
store: RelayCredentialStore
assignments: RelayAssignmentStore
drain: (graceMs: number) => void
idleRehome?: (input: IdleRegionalRehomeRequest & {
cohortPercent: number
directorSafety: RegionalRehomeSafetySnapshot
}) => Promise<{ outcome: IdleRegionalRehomeOutcome }>
drainHost?: (input: {
attemptId: string
userId: string
relayHostId: string
sourceAssignmentEpoch: number
sourceCellIncarnation: string
graceMs: number
}) => 'accepted' | 'already-accepted' | 'host-not-connected'
}) =>
| 'accepted'
| 'already-accepted'
| 'host-not-connected'
| Promise<'accepted' | 'already-accepted' | 'host-not-connected'>
regionalRehomeIdentityToken?: (audience: string) => Promise<string>
regionalRehomeFetch?: typeof fetch
regionalRehomeTrustProbeHostExists?: (input: {
userId: string
relayHostId: string
}) => boolean
regionalRehomeTrustProbeHostExists?: (input: { userId: string; relayHostId: string }) => boolean
cellIncarnation?: string
isDraining?: () => boolean
regionalRehomeSafetySnapshot?: () => RegionalRehomeSafetySnapshot
runtimeCounts?: () => RelayRuntimeCounts
ready: () => Promise<boolean>
recordAssignmentAdmission?: (
@@ -226,7 +237,8 @@ export function createRelayApp(
return context.json({ error: 'host_identity_mismatch' }, 403)
}
const identity = { userId: claims.sub, relayHostId: claims.relayHostId }
const requestedRegion = body.data.preferredRegion
const requestedRegion =
body.data.regionCorrection?.action === 'report' ? undefined : body.data.preferredRegion
const targetRegion =
config.regionalPlacementEnabled !== false && requestedRegion
? requestedRegion
@@ -295,10 +307,30 @@ export function createRelayApp(
}
}
let assignment: RelayAssignment
let regionCorrection: RegionCorrectionResponse | undefined
try {
assignment = requestedRegion
? await operations.assignments.assign(identity, requestedRegion, targetRegion)
: await operations.assignments.assign(identity)
if (body.data.regionCorrection?.action === 'report') {
const current = await operations.assignments.resolve(identity)
if (!current) return context.json({ error: 'assignment_not_found' }, 409)
assignment = current
} else {
assignment = requestedRegion
? await operations.assignments.assign(identity, requestedRegion, targetRegion)
: await operations.assignments.assign(identity)
}
if (body.data.regionCorrection) {
try {
regionCorrection = await operations.assignments.exchangeRegionCorrection(
identity,
body.data.regionCorrection,
assignment.assignmentEpoch
)
} catch (error) {
if (body.data.regionCorrection.action === 'report') throw error
// Optional measurement setup must not discard an otherwise valid placement.
console.warn(JSON.stringify({ event: 'orca_relay_region_window_unavailable' }))
}
}
} catch (error) {
if (isRelayAssignmentCapacityError(error) || isRelayDatabaseTransientError(error)) {
logAssignmentRejection({
@@ -353,13 +385,16 @@ export function createRelayApp(
v: 1,
cellUrl: assignment.cellUrl,
assignmentEpoch: assignment.assignmentEpoch,
lease
lease,
...(regionCorrection ? { regionCorrection } : {})
})
})
app.post('/v1/resolve', async (context) => {
if (config.role === 'cell') return context.json({ error: 'director_only' }, 404)
if (!config.publicAssignmentsEnabled) return rejectPublicAssignment(context)
if (Number(context.req.header('content-length') ?? 0) > RELAY_PROTOCOL_LIMITS.maxHttpBodyBytes) {
if (
Number(context.req.header('content-length') ?? 0) > RELAY_PROTOCOL_LIMITS.maxHttpBodyBytes
) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = ResolveRequestSchema.safeParse(await context.req.json().catch(() => null))
@@ -433,6 +468,34 @@ export function createRelayApp(
operations.drain(body.data.graceMs)
return context.json({ ok: true })
})
app.post('/v1/admin/host-idle-rehome', async (context) => {
if (config.role !== 'cell' || !operations.idleRehome) {
return context.json({ error: 'cell_only' }, 404)
}
const bearer = readBearer(context.req.header('authorization'))
if (!bearer || !(await verifyRegionalRehomeToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = IdleRegionalRehomeCommandSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
if (
body.data.sourceCellId !== config.cellId ||
!operations.cellIncarnation ||
body.data.sourceCellIncarnation !== operations.cellIncarnation
) {
return context.json({ error: 'regional_rehome_source_generation_mismatch' }, 409)
}
try {
return context.json({ v: 1, ...(await operations.idleRehome(body.data)) })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
}
})
app.post('/v1/admin/host-drain', async (context) => {
if (config.role !== 'cell' || !operations.drainHost) {
return context.json({ error: 'cell_only' }, 404)
@@ -474,7 +537,7 @@ export function createRelayApp(
}
sharedRuntimeIdentityRejected = true
}
const outcome = operations.drainHost(body.data)
const outcome = await operations.drainHost(body.data)
return context.json({
v: 1,
outcome,
@@ -502,8 +565,7 @@ export function createRelayApp(
region: config.region ?? RELAY_DEFAULT_REGION,
imageDigest: config.imageDigest ?? null,
draining: operations.isDraining?.() ?? false,
regionalRehomeProtocol:
config.rehomeAudience && config.rehomeDirectorServiceAccount ? 1 : 0,
regionalRehomeProtocol: config.rehomeAudience && config.rehomeDirectorServiceAccount ? 3 : 0,
connectionCapacity:
config.connectionHardCap === undefined
? null
@@ -559,6 +621,18 @@ export function createRelayApp(
return context.json({ error: operationError(error) }, 409)
}
})
app.get('/v1/admin/regional-rehome-preview', async (context) => {
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
const bearer = readBearer(context.req.header('authorization'))
if (!bearer || !(await verifyAdminToken(bearer, context.req.path))) {
return context.json({ error: 'invalid_token' }, 401)
}
const preview = await operations.assignments.previewRegionalRehomeEligibility(
operations.regionalRehomeSafetySnapshot?.()
)
const outcomes = await operations.assignments.regionCorrectionOutcomes()
return context.json({ v: 1, preview, outcomes })
})
app.post('/v1/admin/regional-rehome-control', async (context) => {
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
const bearer = readBearer(context.req.header('authorization'))
@@ -1295,6 +1369,11 @@ const RegionalRehomeSafetySchema = z
})
.strict()
const IdleRegionalRehomeCommandSchema = IdleRegionalRehomeRequestSchema.extend({
cohortPercent: z.number().int().min(0).max(100),
directorSafety: RegionalRehomeSafetySchema
})
const CellHeartbeatSchema = z
.object({
v: z.literal(1),
@@ -1394,45 +1473,48 @@ const CellRegionalRehomeStatusSchema = z
v: z.literal(1),
cellId: z.string().min(1).max(128),
cellIncarnation: z.string().uuid(),
regionalRehomeProtocol: z.number().int().min(0).max(1),
regionalRehomeProtocol: z.number().int().min(0).max(3),
safety: RegionalRehomeSafetySchema
})
.strict()
const RegionalRehomeControlSchema = z.discriminatedUnion('action', [
z.object({ v: z.literal(1), action: z.literal('inspect') }).strict(),
z.object({
v: z.literal(1),
action: z.literal('apply'),
expectedGeneration: z.number().int().nonnegative(),
enabled: z.boolean(),
notBefore: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
ratePerMinute: z.number().int().min(1).max(120),
preferenceMaxAgeMs: z
.number()
.int()
.min(60_000)
.max(30 * 24 * 60 * 60_000),
hostCooldownMs: z
.number()
.int()
.min(60_000)
.max(30 * 24 * 60 * 60_000),
drainGraceMs: z.number().int().min(60_000).max(60 * 60_000),
confirmation: z.enum([
'ENABLE_REGIONAL_REHOMING',
'DISABLE_REGIONAL_REHOMING'
])
}).strict()
]).superRefine((value, context) => {
if (value.action !== 'apply') return
const expected = value.enabled
? 'ENABLE_REGIONAL_REHOMING'
: 'DISABLE_REGIONAL_REHOMING'
if (value.confirmation !== expected) {
context.addIssue({ code: 'custom', message: 'confirmation does not match state' })
}
})
const RegionalRehomeControlSchema = z
.discriminatedUnion('action', [
z.object({ v: z.literal(1), action: z.literal('inspect') }).strict(),
z
.object({
v: z.literal(1),
action: z.literal('apply'),
expectedGeneration: z.number().int().nonnegative(),
enabled: z.boolean(),
notBefore: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
ratePerMinute: z.number().int().min(1).max(120),
preferenceMaxAgeMs: z
.number()
.int()
.min(60_000)
.max(30 * 24 * 60 * 60_000),
hostCooldownMs: z
.number()
.int()
.min(60_000)
.max(30 * 24 * 60 * 60_000),
drainGraceMs: z
.number()
.int()
.min(60_000)
.max(60 * 60_000),
confirmation: z.enum(['ENABLE_REGIONAL_REHOMING', 'DISABLE_REGIONAL_REHOMING'])
})
.strict()
])
.superRefine((value, context) => {
if (value.action !== 'apply') return
const expected = value.enabled ? 'ENABLE_REGIONAL_REHOMING' : 'DISABLE_REGIONAL_REHOMING'
if (value.confirmation !== expected) {
context.addIssue({ code: 'custom', message: 'confirmation does not match state' })
}
})
const RegionalRehomeTrustProbeSchema = z
.object({
@@ -1776,7 +1858,11 @@ const RegionalHostDrainSchema = z
sourceCellId: z.string().min(1).max(128),
sourceCellIncarnation: z.string().uuid(),
sourceAssignmentEpoch: z.number().int().positive(),
graceMs: z.number().int().nonnegative().max(60 * 60 * 1000)
graceMs: z
.number()
.int()
.nonnegative()
.max(60 * 60 * 1000)
})
.strict()
cloud/apps/relay/src/assignment-store.ts
+556 -775
View File
File diff suppressed because it is too large Load Diff
cloud/apps/relay/src/cell-heartbeat-client.test.ts
+27 -21
View File
@@ -92,7 +92,7 @@ describe('cell heartbeat client', () => {
client.stop()
expect(JSON.parse(String(requests[1]!.body))).toMatchObject({
regionalRehomeProtocol: 1,
regionalRehomeProtocol: 3,
safety: {
observedAt: 120,
sqlFailures: 0,
@@ -149,28 +149,34 @@ describe('cell heartbeat client', () => {
it('does not start outside an explicitly configured cell role', () => {
expect(
startCellHeartbeat({ ...CONFIG, role: 'director' }, {
ready: async () => true,
observedRequests: () => 0,
connectionCounts: () => ({
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0
})
})
startCellHeartbeat(
{ ...CONFIG, role: 'director' },
{
ready: async () => true,
observedRequests: () => 0,
connectionCounts: () => ({
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0
})
}
)
).toBeNull()
expect(
startCellHeartbeat({ ...CONFIG, directorUrl: undefined }, {
ready: async () => true,
observedRequests: () => 0,
connectionCounts: () => ({
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0
})
})
startCellHeartbeat(
{ ...CONFIG, directorUrl: undefined },
{
ready: async () => true,
observedRequests: () => 0,
connectionCounts: () => ({
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0
})
}
)
).toBeNull()
})
})
cloud/apps/relay/src/cell-heartbeat-client.ts
+6 -4
View File
@@ -70,8 +70,7 @@ export function startCellHeartbeat(
inFlightConnections: connectionCounts!.inFlightConnections,
reservedConnectionUnits: connectionCounts!.reservedConnectionUnits,
enforcedConnectionUnits: connectionCounts!.enforcedConnectionUnits,
connectionInclusionWatermark:
connectionCounts!.inclusionWatermark,
connectionInclusionWatermark: connectionCounts!.inclusionWatermark,
connectionHardCap: config.connectionHardCap,
connectionUnobservedBound: config.connectionUnobservedBound
})
@@ -94,7 +93,7 @@ export function startCellHeartbeat(
cellId: config.cellId,
cellIncarnation,
regionalRehomeProtocol:
config.rehomeAudience && config.rehomeDirectorServiceAccount ? 1 : 0,
config.rehomeAudience && config.rehomeDirectorServiceAccount ? 3 : 0,
safety: options.regionalRehomeSafety()
}),
signal: AbortSignal.timeout(10_000)
@@ -106,7 +105,10 @@ export function startCellHeartbeat(
}
} catch (error) {
// A heartbeat must fail closed without ever logging its bearer token.
console.warn('[orca-relay] cell heartbeat failed', error instanceof Error ? error.message : '')
console.warn(
'[orca-relay] cell heartbeat failed',
error instanceof Error ? error.message : ''
)
} finally {
inFlight = false
}
cloud/apps/relay/src/cell-inventory-lock-census.test.ts
+7 -9
View File
@@ -43,14 +43,13 @@ const CENSUS: CensusEntry[] = [
{ method: 'completeEvacuation', mode: 'nowait', reach: 'both' },
{ method: 'completeEvacuation', mode: 'pool-default', reach: 'both' },
{ method: 'rebalanceDormant', mode: 'request', reach: 'request' },
{ method: 'startRegionalRehomeCandidate', mode: 'nowait', reach: 'sweep' },
{ method: 'lockedRegionalRehomeFleetSafety', mode: 'nowait', reach: 'sweep' },
{ method: 'startRegionalRehomeCandidate', mode: 'nowait', reach: 'request' },
{ method: 'completeRegionalRehomeCandidate', mode: 'nowait', reach: 'sweep' },
{ method: 'abortExpiredRegionalRehomes', mode: 'nowait', reach: 'sweep' },
{ method: 'abortExpiredEvacuations', mode: 'nowait', reach: 'sweep' },
{ method: 'abortExpiredEvacuations', mode: 'nowait', reach: 'sweep' },
{ method: 'releaseExpiredActivityLeases', mode: 'nowait', reach: 'sweep' },
{ method: 'releaseExpiredActivity', mode: 'nowait', reach: 'sweep' },
{ method: 'releaseExpiredActivity', mode: 'nowait', reach: 'sweep' }
// reconcileReservationAccounting and leastLoadedCell are gone too: the first
// repairs exactly two cells' counters and now holds only those rows, and the
// second selects from the inventory its single caller has already locked.
@@ -119,9 +118,10 @@ function storeCallGraph(lines: string[]): Map<string, Set<string>> {
bounds.forEach((method, index) => {
const end = bounds[index + 1]?.start ?? lines.length
const names = callees.get(method.name) ?? new Set<string>()
for (const call of lines.slice(method.start, end).join('\n').matchAll(
/this\.([A-Za-z_][\w]*)\s*\(/g
)) {
for (const call of lines
.slice(method.start, end)
.join('\n')
.matchAll(/this\.([A-Za-z_][\w]*)\s*\(/g)) {
names.add(call[1]!)
}
callees.set(method.name, names)
@@ -174,9 +174,7 @@ function readCallSites(): { method: string; mode: CensusMode }[] {
describe('cell inventory lock call-site census', () => {
it('classifies every call site exactly as recorded', () => {
expect(readCallSites()).toEqual(
CENSUS.map(({ method, mode }) => ({ method, mode }))
)
expect(readCallSites()).toEqual(CENSUS.map(({ method, mode }) => ({ method, mode })))
})
// Why: the census only sees lockCellInventory calls, so a hand-written
cloud/apps/relay/src/config.test.ts
+11
View File
@@ -25,6 +25,17 @@ function cellEnvironment(capacity: number): NodeJS.ProcessEnv {
}
describe('GCE relay capacity configuration', () => {
it('defaults optional region correction off and bounds the cohort', () => {
const env = cellEnvironment(4_000)
expect(loadRelayConfig(env).regionCorrectionCohortPercent).toBe(0)
env.ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT = '5'
expect(loadRelayConfig(env).regionCorrectionCohortPercent).toBe(5)
for (const invalid of ['-1', '101', '1.5', 'not-a-number']) {
env.ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT = invalid
expect(() => loadRelayConfig(env)).toThrow()
}
})
it('requires distinct dedicated admin identities and accepts omitted values', () => {
const env = cellEnvironment(4_000)
expect(loadRelayConfig(env)).toMatchObject({
cloud/apps/relay/src/config.ts
+7 -1
View File
@@ -75,11 +75,15 @@ const EnvSchema = z.object({
ORCA_RELAY_RUNTIME_SERVICE_ACCOUNT: z.string().email().optional(),
ORCA_RELAY_DIRECTOR_URL: z.string().url().optional(),
ORCA_RELAY_HEARTBEAT_AUDIENCE: z.string().url().optional(),
ORCA_RELAY_IMAGE_DIGEST: z.string().regex(/^sha256:[a-f0-9]{64}$/).optional(),
ORCA_RELAY_IMAGE_DIGEST: z
.string()
.regex(/^sha256:[a-f0-9]{64}$/)
.optional(),
ORCA_RELAY_ADMIN_JWKS_URL: z.string().url().default('https://www.googleapis.com/oauth2/v3/certs'),
ORCA_RELAY_DATABASE_POOL_MAX: z.coerce.number().int().positive().max(100).optional(),
ORCA_RELAY_PUBLIC_ASSIGNMENTS_ENABLED: EnvironmentBooleanSchema,
ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED: EnvironmentBooleanSchema,
ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT: z.coerce.number().int().min(0).max(100).default(0),
ORCA_RELAY_PUBLIC_ASSIGNMENT_CONCURRENCY: z.coerce.number().int().positive().max(100).default(2),
ORCA_RELAY_PUBLIC_STICKY_CONCURRENCY: z.coerce.number().int().positive().max(100).default(1),
ORCA_RELAY_PUBLIC_STICKY_QUEUE_MAX: z.coerce.number().int().positive().max(4_096).default(64),
@@ -185,6 +189,7 @@ export type RelayConfig = {
databasePoolMax: number
publicAssignmentsEnabled: boolean
regionalPlacementEnabled?: boolean
regionCorrectionCohortPercent?: number
publicAssignmentConcurrency: number
publicAssignmentQueueMax: number
publicAssignmentWaitMs: number
@@ -332,6 +337,7 @@ export function loadRelayConfig(env: NodeJS.ProcessEnv = process.env): RelayConf
databasePoolMax,
publicAssignmentsEnabled: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENTS_ENABLED,
regionalPlacementEnabled: parsed.ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED,
regionCorrectionCohortPercent: parsed.ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT,
publicAssignmentConcurrency: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENT_CONCURRENCY,
publicAssignmentQueueMax: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENT_QUEUE_MAX,
publicAssignmentWaitMs: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENT_WAIT_MS,
cloud/apps/relay/src/database.test.ts
+32 -6
View File
@@ -18,6 +18,34 @@ afterEach(() => {
})
describe('relay database', () => {
it('upgrades an existing SQLite relay without treating legacy controls as idle-capable', async () => {
const dataDir = mkdtempSync(join(tmpdir(), 'orca-idle-schema-'))
temporaryDirectories.push(dataDir)
const legacy = await openRelayDatabase({ dataDir })
await legacy.query('ALTER TABLE relay_control_capabilities DROP COLUMN idle_regional_rehome')
await legacy.query('ALTER TABLE relay_region_rehome_attempts DROP COLUMN source_generation')
await legacy.query(
`INSERT INTO relay_control_capabilities
(user_id, relay_host_id, activity_id, cell_id, cell_incarnation, assignment_epoch, generation, finish_existing)
VALUES ('legacy-user', 'abcdefghijklmnop', 'control:source:1', 'source', 'legacy-incarnation', 1, 1, 1)`
)
await legacy.close()
const upgraded = await openRelayDatabase({ dataDir })
try {
expect(
await upgraded.query('SELECT idle_regional_rehome FROM relay_control_capabilities')
).toEqual([{ idle_regional_rehome: 0 }])
const columns = await upgraded.query(
"SELECT * FROM pragma_table_info('relay_region_rehome_attempts')"
)
expect(columns.find((column) => column.name === 'source_generation')).toMatchObject({
dflt_value: '0'
})
} finally {
await upgraded.close()
}
})
it('creates every durable relay state table', async () => {
const database = await openInMemoryRelayDatabase()
const rows = await database.query(
@@ -54,6 +82,7 @@ describe('relay database', () => {
'relay_confirm_results',
'relay_confirmable_splices',
'relay_connection_bases',
'relay_control_capabilities',
'relay_control_connection_reservations',
'relay_devices',
'relay_direct_authorizations',
@@ -62,6 +91,7 @@ describe('relay database', () => {
'relay_migration_leases',
'relay_post_drain_migration_pins',
'relay_rate_windows',
'relay_region_decisions',
'relay_region_rehome_attempts',
'relay_region_rehome_control',
'relay_region_rehome_worker_state'
@@ -140,9 +170,7 @@ describe('relay database', () => {
const second = await openRelayDatabase({ dataDir })
expect(
await second.query(`SELECT region FROM relay_cell_regions WHERE cell_id = ?`, [
'legacy-cell'
])
await second.query(`SELECT region FROM relay_cell_regions WHERE cell_id = ?`, ['legacy-cell'])
).toEqual([{ region: 'us-central1' }])
await second.close()
})
@@ -165,9 +193,7 @@ describe('relay database', () => {
'relay_region_rehome_attempts'
])
expect(checked.every((row) => String(row.sql).includes(list))).toBe(true)
expect(
POSTGRES_SCHEMA_MIGRATIONS.some((statement) => statement.includes(list))
).toBe(true)
expect(POSTGRES_SCHEMA_MIGRATIONS.some((statement) => statement.includes(list))).toBe(true)
await database.close()
})
cloud/apps/relay/src/database.ts
+33 -1
View File
@@ -197,6 +197,24 @@ CREATE TABLE IF NOT EXISTS relay_assignment_region_preferences (
CREATE INDEX IF NOT EXISTS relay_assignment_region_preferences_observed
ON relay_assignment_region_preferences(observed_at);
CREATE TABLE IF NOT EXISTS relay_region_decisions (
user_id TEXT NOT NULL, relay_host_id TEXT NOT NULL,
generation BIGINT NOT NULL, expires_at BIGINT NOT NULL,
assignment_epoch BIGINT NOT NULL, incumbent_region TEXT NOT NULL,
policy_version BIGINT NOT NULL, outcome TEXT NOT NULL,
cohort_bucket BIGINT NOT NULL DEFAULT 0,
last_considered_at BIGINT NOT NULL DEFAULT 0,
preferred_region TEXT, observed_at BIGINT NOT NULL, report_json TEXT,
PRIMARY KEY (user_id, relay_host_id)
);
CREATE TABLE IF NOT EXISTS relay_control_capabilities (
user_id TEXT NOT NULL, relay_host_id TEXT NOT NULL, activity_id TEXT NOT NULL,
cell_id TEXT NOT NULL, cell_incarnation TEXT NOT NULL,
assignment_epoch BIGINT NOT NULL, generation BIGINT NOT NULL,
finish_existing BIGINT NOT NULL,
idle_regional_rehome BIGINT NOT NULL DEFAULT 0,
PRIMARY KEY (user_id, relay_host_id, activity_id)
);
CREATE TABLE IF NOT EXISTS relay_region_rehome_worker_state (
worker_id TEXT PRIMARY KEY,
next_dispatch_at BIGINT NOT NULL,
@@ -228,6 +246,7 @@ CREATE TABLE IF NOT EXISTS relay_region_rehome_attempts (
CHECK (preferred_region IN (${REGION_LIST})),
source_cell_id TEXT NOT NULL,
source_cell_incarnation TEXT NOT NULL,
source_generation BIGINT NOT NULL DEFAULT 0,
target_cell_id TEXT NOT NULL,
target_cell_incarnation TEXT NOT NULL,
previous_epoch BIGINT NOT NULL,
@@ -600,6 +619,8 @@ CREATE INDEX IF NOT EXISTS relay_audit_events_at ON relay_audit_events(at);
// auto-named; the replacement is named, so both statements are no-ops on a
// database the current schema created and neither can drop the other.
export const POSTGRES_SCHEMA_MIGRATIONS = [
`ALTER TABLE relay_region_decisions ADD COLUMN IF NOT EXISTS last_considered_at BIGINT NOT NULL DEFAULT 0`,
`ALTER TABLE relay_region_decisions ADD COLUMN IF NOT EXISTS cohort_bucket BIGINT NOT NULL DEFAULT 0`,
`ALTER TABLE relay_region_rehome_attempts
DROP CONSTRAINT IF EXISTS relay_region_rehome_attempts_preferred_region_check`,
`ALTER TABLE relay_region_rehome_attempts
@@ -607,7 +628,9 @@ export const POSTGRES_SCHEMA_MIGRATIONS = [
CHECK (preferred_region IN (${REGION_LIST}))`,
`ALTER TABLE relay_region_rehome_control
ADD COLUMN IF NOT EXISTS host_cooldown_ms BIGINT NOT NULL
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS}`
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS}`,
`ALTER TABLE relay_control_capabilities ADD COLUMN IF NOT EXISTS idle_regional_rehome BIGINT NOT NULL DEFAULT 0`,
`ALTER TABLE relay_region_rehome_attempts ADD COLUMN IF NOT EXISTS source_generation BIGINT NOT NULL DEFAULT 0`
]
function postgresSql(sql: string): string {
@@ -1013,6 +1036,15 @@ async function applySchema(database: RelayDatabase): Promise<void> {
for (const statement of SCHEMA.split(';')) {
if (statement.trim()) await database.query(statement)
}
for (const [table, column] of [
['relay_control_capabilities', 'idle_regional_rehome'],
['relay_region_rehome_attempts', 'source_generation']
]) {
const columns = await database.query('SELECT name FROM pragma_table_info(?)', [table])
if (!columns.some((existing) => existing.name === column)) {
await database.query(`ALTER TABLE ${table} ADD COLUMN ${column} BIGINT NOT NULL DEFAULT 0`)
}
}
}
// Why: DDL is not a request. A CREATE INDEX on a grown table legitimately runs
cloud/apps/relay/src/host-session-client-accept.test.ts
+64
View File
@@ -155,6 +155,66 @@ describe('client accept abandoned mid-DB-phase', () => {
vi.useRealTimers()
})
it('does not admit new source work after a drain crosses activity acquisition', async () => {
const h = harness()
const control = await activeHost(h)
const slow = deferred<void>()
h.acquireActivity.mockReturnValueOnce(slow.promise)
const client = new FakeSocket()
const capacity = { bind: vi.fn(), release: vi.fn() }
const accepting = h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential',
capacity
)
await vi.advanceTimersByTimeAsync(0)
h.registry.drainHost({
attemptId: 'attempt',
userId: identity.sub,
relayHostId: identity.relayHostId,
sourceAssignmentEpoch: 1,
graceMs: 60_000
})
slow.resolve()
await accepting
expect(control.send).not.toHaveBeenCalledWith(expect.stringContaining('conn-open'))
expect(capacity.bind).not.toHaveBeenCalled()
expect(client.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, expect.any(String))
expect(h.releaseActivity).toHaveBeenCalled()
})
it('does not splice an attachment whose generation retired during basis persistence', async () => {
const h = harness()
await activeHost(h)
const client = new FakeSocket()
await h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential'
)
const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
const pending = [...session.pendingConns.values()][0]!
const slow = deferred<void>()
h.store.recordConnectionBasis.mockReturnValueOnce(slow.promise)
const host = new FakeSocket()
const attaching = h.registry.acceptHostData(
host as unknown as WebSocket,
pending.connId,
pending.connTicket,
1
)
await vi.advanceTimersByTimeAsync(0)
h.registry.drain(0)
await vi.advanceTimersByTimeAsync(0)
slow.resolve()
expect(await attaching).toBe(false)
expect(session.activeSplices.size).toBe(0)
expect(h.store.deactivateBasis).toHaveBeenCalledWith(pending.connId)
expect(client.send).not.toHaveBeenCalledWith(expect.stringContaining('\"ok\":true'))
expect(host.close).toHaveBeenCalled()
})
it('stops after a slow activity acquire when the phone already hung up', async () => {
const h = harness()
const control = await activeHost(h)
@@ -390,6 +450,7 @@ describe('successful client accept timing', () => {
relayHostIdDigest: string
}
expect(event.credentialKind).toBe('resume')
expect(event).toMatchObject({ assignmentEpoch: 1, controlGeneration: 1, drainMode: 'none' })
// Joins the line back to the emitting process, like the runtime metrics event.
expect(event).toMatchObject({ role: 'cell', cellId: config.cellId, region: 'us-central1' })
expect(Object.keys(event.stageMs).sort()).toEqual([
@@ -460,6 +521,9 @@ describe('control round-trip sampling', () => {
cellId: config.cellId,
region: 'us-central1',
rttMsMedian: 40,
assignmentEpoch: 1,
controlGeneration: 1,
drainMode: 'none',
sampleCount: 4
})
expect(rttLines()[0]).not.toContain(identity.relayHostId)
cloud/apps/relay/src/host-session-registry.test.ts
+345 -42
View File
@@ -4,6 +4,7 @@ import {
CONTROL_CONTINUITY_LIMITS,
RELAY_CLOSE_CODE,
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS,
RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME,
RELAY_PROTOCOL_LIMITS
} from '@orca-cloud/relay-contract'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
@@ -140,7 +141,10 @@ function createRegistry(
store as RelayCredentialStore,
assignments,
new ProcessQueuedByteBudget(),
observer
observer,
Date.now,
Math.random,
'incarnation-1'
)
// Mirrors the production signature exactly so a future positional shift fails to compile.
const bound = (
@@ -166,7 +170,14 @@ function createRegistry(
assignmentEpoch,
appVersion = '1.4.173'
) => bound(socket, identity, existing, generation, rebind, assignmentEpoch, appVersion)
return { registry, activate, acquireActivity, renewControlActivity, releaseActivity, observer }
return {
registry,
activate,
acquireActivity,
renewControlActivity,
releaseActivity,
observer
}
}
describe('host session cleanup races', () => {
@@ -398,26 +409,20 @@ describe('host session cleanup races', () => {
attemptId: '22222222-2222-4222-8222-222222222222'
})
).toThrow('regional_rehome_attempt_conflict')
expect(() =>
registry.drainHost({ ...request, sourceAssignmentEpoch: 8 })
).toThrow('regional_rehome_assignment_epoch_mismatch')
expect(() => registry.drainHost({ ...request, sourceAssignmentEpoch: 8 })).toThrow(
'regional_rehome_assignment_epoch_mismatch'
)
const rebound = new FakeSocket()
await activate(
rebound as unknown as WebSocket,
identity,
registry.get(request),
1,
true,
7
)
await activate(rebound as unknown as WebSocket, identity, registry.get(request), 1, true, 7)
expect(registry.get(request)?.state).toBe('drain-only')
expect(rebound.send).toHaveBeenCalledWith(expect.stringContaining('"type":"drain"'))
await vi.advanceTimersByTimeAsync(30_000)
expect(registry.get(request)).toBeNull()
expect(registry.get({ userId: secondIdentity.sub, relayHostId: secondIdentity.relayHostId }))
.not.toBeNull()
expect(
registry.get({ userId: secondIdentity.sub, relayHostId: secondIdentity.relayHostId })
).not.toBeNull()
expect(secondSocket.close).not.toHaveBeenCalled()
})
@@ -513,14 +518,7 @@ describe('host session cleanup races', () => {
expect(original).not.toBeNull()
const rebindSocket = new FakeSocket()
const rebinding = activate(
rebindSocket as unknown as WebSocket,
identity,
original,
1,
true,
1
)
const rebinding = activate(rebindSocket as unknown as WebSocket, identity, original, 1, true, 1)
rebindSocket.close()
blocked.resolve('control:production-gce-c3:1')
await rebinding
@@ -659,14 +657,7 @@ describe('host session cleanup races', () => {
originalSocket.close()
const replacementSocket = new FakeSocket()
await activate(
replacementSocket as unknown as WebSocket,
identity,
original,
2,
false,
1
)
await activate(replacementSocket as unknown as WebSocket, identity, original, 2, false, 1)
const replacement = registry.get({
userId: identity.sub,
relayHostId: identity.relayHostId
@@ -696,14 +687,7 @@ describe('host session cleanup races', () => {
})
expect(original).not.toBeNull()
await activate(
new FakeSocket() as unknown as WebSocket,
identity,
original,
2,
false,
1
)
await activate(new FakeSocket() as unknown as WebSocket, identity, original, 2, false, 1)
vi.advanceTimersByTime(15_000)
expect(renewControlActivity).toHaveBeenCalledOnce()
@@ -718,6 +702,53 @@ describe('host session cleanup races', () => {
vi.advanceTimersByTime(0)
})
it('ignores a denial belonging to the socket before a same-generation rebind', async () => {
const h = createRegistry(vi.fn().mockResolvedValue('control:production-gce-c3:1'))
const oldSocket = new FakeSocket()
await h.activate(oldSocket as unknown as WebSocket, identity, null, 1, false, 1)
const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
let reject!: (error: Error) => void
h.renewControlActivity.mockReturnValueOnce(
new Promise<void>((_, fail) => {
reject = fail
})
)
await vi.advanceTimersByTimeAsync(15_000)
const replacement = new FakeSocket()
await h.activate(replacement as unknown as WebSocket, identity, session, 1, true, 1)
reject(new Error('activity_cell_not_authoritative'))
await vi.advanceTimersByTimeAsync(0)
expect(replacement.close).not.toHaveBeenCalled()
expect(session.socket).toBe(replacement)
expect(session.generation).toBe(1)
})
it('ignores missing-activity recovery denial after an authority transition', async () => {
const h = createRegistry(vi.fn().mockResolvedValue('control:production-gce-c3:1'))
const socket = new FakeSocket()
await h.activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
h.renewControlActivity.mockRejectedValueOnce(new Error('control_activity_not_found'))
let reject!: (error: Error) => void
h.acquireActivity.mockReturnValueOnce(
new Promise<void>((_, fail) => {
reject = fail
})
)
await vi.advanceTimersByTimeAsync(15_000)
h.registry.drainHost({
attemptId: 'attempt',
userId: identity.sub,
relayHostId: identity.relayHostId,
sourceAssignmentEpoch: 1,
graceMs: 60_000
})
reject(new Error('activity_cell_not_authoritative'))
await vi.advanceTimersByTimeAsync(0)
expect(socket.close).not.toHaveBeenCalled()
expect(session.state).toBe('drain-only')
})
it('keeps 15s pings while halving steady-state control renewals', async () => {
const activateControl = vi
.fn<RelayAssignmentStore['activateControl']>()
@@ -732,9 +763,7 @@ describe('host session cleanup races', () => {
socket.emit('message', Buffer.from(JSON.stringify({ type: 'pong' })), false)
}
const pings = socket.send.mock.calls.filter((call) =>
String(call[0]).includes('"ping"')
)
const pings = socket.send.mock.calls.filter((call) => String(call[0]).includes('"ping"'))
expect(pings).toHaveLength(4)
expect(renewControlActivity).toHaveBeenCalledTimes(2)
const firstExpiry = Number(renewControlActivity.mock.calls[0]![1].expiresAt)
@@ -1118,3 +1147,277 @@ describe('host hello ack pending connections', () => {
expect(rebound.pendingConns).toEqual([DETAILED_ENTRY])
})
})
describe('source-owned idle cutover', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
const request = {
attemptId: 'idle-1',
userId: identity.sub,
relayHostId: identity.relayHostId,
sourceAssignmentEpoch: 1,
sourceGeneration: 1,
sourceCellIncarnation: 'incarnation-1',
targetCellId: 'target'
}
async function source(store: Partial<RelayCredentialStore> = {}) {
const h = createRegistry(vi.fn().mockResolvedValue('control:1'), store)
const socket = new FakeSocket()
h.registry.acceptControl(
socket as unknown as WebSocket,
identity,
undefined,
new Set([RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME])
)
socket.removeAllListeners('message')
await h.activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
return { ...h, socket, session: h.registry.get(request)! }
}
it('keeps either established client busy until both actually leave', async () => {
const h = await source()
h.session.activeConnIds.add('phone')
h.session.activeConnIds.add('ipad')
const commit = vi.fn().mockResolvedValue({ outcome: 'committed' })
h.session.activeConnIds.delete('ipad')
expect(
await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('not-committed'))
).toEqual({ outcome: 'busy' })
expect(commit).not.toHaveBeenCalled()
h.session.activeConnIds.delete('phone')
expect(
await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('not-committed'))
).toEqual({ outcome: 'committed' })
expect(h.socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, expect.any(String))
expect(h.releaseActivity).toHaveBeenCalled()
})
it.each([
{ userId: 'other-user' },
{ sourceAssignmentEpoch: 2 },
{ sourceGeneration: 2 },
{ sourceCellIncarnation: 'other-incarnation' },
{ targetCellId: 'other-target' }
])('rejects a reused operation ID with changed authority %j', async (change) => {
const h = await source()
const result = deferred<{ outcome: 'deferred' }>()
const commit = vi.fn().mockReturnValue(result.promise)
const reconcile = vi.fn().mockResolvedValue('not-committed')
const moving = h.registry.idleRehome(request, commit, reconcile)
const conflicting = h.registry.idleRehome({ ...request, ...change }, commit, reconcile)
result.resolve({ outcome: 'deferred' })
expect(await conflicting).toEqual({ outcome: 'stale' })
expect(await moving).toEqual({ outcome: 'deferred' })
expect(commit).toHaveBeenCalledOnce()
expect(h.socket.close).not.toHaveBeenCalled()
})
it('accounts for accepts before credential identity resolves', async () => {
const lookup = deferred<null>()
const h = await source({
resolveResume: vi.fn().mockReturnValue(lookup.promise),
resolveInviteForMove: vi.fn().mockResolvedValue(null)
})
const client = new FakeSocket()
const accept = h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential'
)
expect(
await h.registry.idleRehome(request, vi.fn(), vi.fn().mockResolvedValue('not-committed'))
).toEqual({ outcome: 'busy' })
lookup.resolve(null)
await accept
expect(h.socket.close).not.toHaveBeenCalled()
})
it('rejects new accepts and replacements synchronously while a commit awaits', async () => {
const h = await source()
const result = deferred<{ outcome: 'deferred' }>()
const commit = vi.fn().mockReturnValue(result.promise)
const moving = h.registry.idleRehome(
request,
commit,
vi.fn().mockResolvedValue('not-committed')
)
const duplicate = h.registry.idleRehome(
request,
commit,
vi.fn().mockResolvedValue('not-committed')
)
const client = new FakeSocket()
const release = vi.fn()
await h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential',
{ release } as never
)
expect(client.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, expect.any(String))
expect(release).toHaveBeenCalledOnce()
const replacement = new FakeSocket()
await h.activate(replacement as unknown as WebSocket, identity, h.session, 2, false, 1)
expect(replacement.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, expect.any(String))
result.resolve({ outcome: 'deferred' })
await moving
await duplicate
expect(commit).toHaveBeenCalledOnce()
expect(h.socket.close).not.toHaveBeenCalled()
expect(
await h.registry.idleRehome(
{ ...request, attemptId: 'next' },
vi.fn().mockResolvedValue({ outcome: 'committed' }),
vi.fn().mockResolvedValue('not-committed')
)
).toEqual({ outcome: 'committed' })
})
it.each(['ambiguous', 'deferred'])(
'keeps %s outcomes fenced until locked reconciliation succeeds',
async (claim) => {
const h = await source()
const reconcile = vi
.fn()
.mockRejectedValueOnce(new Error('database unavailable'))
.mockRejectedValueOnce(new Error('database unavailable'))
.mockResolvedValue('not-committed')
const moving = h.registry.idleRehome(
request,
claim === 'ambiguous'
? vi.fn().mockRejectedValue(new Error('lost commit reply'))
: vi.fn().mockResolvedValue({ outcome: 'deferred' }),
reconcile
)
await vi.advanceTimersByTimeAsync(50)
expect(
await h.registry.idleRehome(
{ ...request, attemptId: 'other' },
vi.fn(),
vi.fn().mockResolvedValue('not-committed')
)
).toEqual({ outcome: 'busy' })
expect(h.socket.close).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(300)
expect(await moving).toEqual({ outcome: 'deferred' })
expect(reconcile).toHaveBeenCalledTimes(3)
expect(h.socket.close).not.toHaveBeenCalled()
}
)
it('owns accepted control mutations before the handler first awaits', async () => {
const mutation = deferred<RelayTokenClaims | null>()
const h = await source()
;(h.registry as unknown as { verifyRelayToken: unknown }).verifyRelayToken = vi
.fn()
.mockReturnValue(mutation.promise)
h.socket.emit(
'message',
Buffer.from(JSON.stringify({ type: 'auth-refresh', relayJwt: 'token' })),
false
)
const commit = vi.fn().mockResolvedValue({ outcome: 'deferred' })
expect(
await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('not-committed'))
).toEqual({ outcome: 'busy' })
mutation.resolve(identity)
await vi.advanceTimersByTimeAsync(0)
expect(
await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('not-committed'))
).toEqual({ outcome: 'deferred' })
})
it('owns queued replacement activation before its first persistence await', async () => {
const h = await source()
const activation = deferred<string>()
const assignments = (h.registry as unknown as { assignments: { activateControl: unknown } })
.assignments
assignments.activateControl = vi.fn().mockReturnValue(activation.promise)
const replacement = new FakeSocket()
const activating = h.activate(
replacement as unknown as WebSocket,
identity,
h.session,
2,
false,
1
)
expect(
await h.registry.idleRehome(request, vi.fn(), vi.fn().mockResolvedValue('not-committed'))
).toEqual({ outcome: 'busy' })
activation.resolve('control:2')
await activating
})
it('retires changed authority even when the claim definitively deferred', async () => {
const h = await source()
expect(
await h.registry.idleRehome(
request,
vi.fn().mockResolvedValue({ outcome: 'deferred' }),
vi.fn().mockResolvedValue('stale')
)
).toEqual({ outcome: 'stale' })
expect(h.session.state).toBe('closed')
expect(h.releaseActivity).toHaveBeenCalled()
})
it('holds attach ownership through basis failure reservation cleanup', async () => {
const basis = deferred<void>()
const cleanup = deferred<void>()
const h = await source({
recordConnectionBasis: vi.fn().mockImplementation(async () => {
await basis.promise
throw new Error('basis failed')
}),
failReservation: vi.fn().mockReturnValue(cleanup.promise)
})
const client = new FakeSocket()
h.session.pendingConns.set('conn', {
connId: 'conn',
connTicket: 'ticket',
client: client as unknown as WebSocket,
reservation: {
userId: identity.sub,
relayHostId: identity.relayHostId,
credentialKind: 'invite',
leaseExpiresAt: Date.now() + 1000
},
attachTimer: setTimeout(() => {}, 1000),
credentialActivityId: null
} as never)
const attached = h.registry.acceptHostData(
new FakeSocket() as unknown as WebSocket,
'conn',
'ticket',
1
)
const commit = vi.fn().mockResolvedValue({ outcome: 'deferred' })
expect(await h.registry.idleRehome(request, commit, vi.fn())).toEqual({ outcome: 'busy' })
basis.resolve()
await vi.advanceTimersByTimeAsync(0)
expect(h.session.activeConnIds.size).toBe(0)
expect(await h.registry.idleRehome(request, commit, vi.fn())).toEqual({ outcome: 'busy' })
expect(commit).not.toHaveBeenCalled()
cleanup.resolve()
await attached
})
it('returns the durable operation outcome after source retirement', async () => {
const h = await source()
const commit = vi.fn().mockResolvedValue({ outcome: 'committed' })
await h.registry.idleRehome(request, commit, vi.fn())
expect(
await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('committed'))
).toEqual({ outcome: 'committed' })
expect(commit).toHaveBeenCalledOnce()
})
it('does not reopen a source overtaken by emergency drain', async () => {
const h = await source()
const result = deferred<{ outcome: 'deferred' }>()
const moving = h.registry.idleRehome(
request,
() => result.promise,
vi.fn().mockResolvedValue('not-committed')
)
h.registry.drain(0)
await vi.advanceTimersByTimeAsync(0)
result.resolve({ outcome: 'deferred' })
await moving
expect(h.session.state).toBe('closed')
expect(h.registry.get(request)).toBeNull()
})
})
cloud/apps/relay/src/host-session-registry.ts
+331 -53
View File
@@ -15,6 +15,7 @@ import {
HostHelloSchema,
InviteCreateSchema,
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS,
RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME,
RELAY_PROTOCOL_LIMITS,
RELAY_CLOSE_CODE,
type RelayHostCloseReason,
@@ -25,10 +26,7 @@ import type WebSocket from 'ws'
import type { RawData } from 'ws'
import type { RelayConfig } from './config.js'
import type { RelayAssignmentStore } from './assignment-store.js'
import {
RelayCredentialStore,
type CredentialReservation
} from './credential-store.js'
import { RelayCredentialStore, type CredentialReservation } from './credential-store.js'
import { HostCloseReasonMemory } from './host-close-reason-memory.js'
import { relayHostLogDigest } from './relay-host-log-digest.js'
import type { RelayTokenClaims } from './relay-token-verifier.js'
@@ -78,7 +76,7 @@ export type HostSession = {
identity: RelayTokenClaims
readonly relayHostId: string
readonly generation: number
readonly assignmentEpoch: number
assignmentEpoch: number
readonly controlActivityId: string | null
readonly controlResumeSecret: string
// Why: reconnect churn is only actionable once it can be pinned to a client build.
@@ -94,6 +92,7 @@ export type HostSession = {
pendingPingAt: number | null
controlRttSamplesMs: number[]
controlRttLoggedAt: number | null
authorityRevision: number
activityRenewalDueAt: number
activityRenewalAttempt: number
activityRenewalCompletedAttempt: number
@@ -108,10 +107,7 @@ export type HostSession = {
regionalDrainExpiresAt: number | null
}
export type RegionalHostDrainOutcome =
| 'accepted'
| 'already-accepted'
| 'host-not-connected'
export type RegionalHostDrainOutcome = 'accepted' | 'already-accepted' | 'host-not-connected'
type PendingConnection = {
connId: string
@@ -184,6 +180,113 @@ export class HostSessionRegistry {
private readonly hostCapabilities = new WeakMap<WebSocket, ReadonlySet<string>>()
private draining = false
private readonly idleWork = new Map<string, number>()
private readonly idleAttempts = new Map<
string,
{
attemptId: string
authorityKey: string
promise: Promise<{ outcome: 'committed' | 'deferred' | 'stale' }>
}
>()
async idleRehome(
input: {
attemptId: string
userId: string
relayHostId: string
sourceAssignmentEpoch: number
sourceGeneration: number
sourceCellIncarnation: string
targetCellId: string
},
commit: () => Promise<{ outcome: 'committed' | 'deferred' | 'stale' }>,
reconcile: () => Promise<'committed' | 'not-committed' | 'stale'>
): Promise<{ outcome: 'busy' | 'committed' | 'deferred' | 'stale' }> {
const authorityKey = JSON.stringify([
input.userId,
input.sourceAssignmentEpoch,
input.sourceGeneration,
input.sourceCellIncarnation,
input.targetCellId
])
const prior = this.idleAttempts.get(input.relayHostId)
if (prior) {
if (prior.attemptId !== input.attemptId) return { outcome: 'busy' }
return prior.authorityKey === authorityKey ? prior.promise : { outcome: 'stale' }
}
const session = this.get(input)
if (
this.draining ||
!session ||
session.state !== 'active' ||
session.generation !== input.sourceGeneration ||
session.assignmentEpoch !== input.sourceAssignmentEpoch ||
this.cellIncarnation !== input.sourceCellIncarnation
) {
const durable = await reconcile()
return { outcome: durable === 'committed' ? 'committed' : 'stale' }
}
if (
!session.socket ||
!this.hostCapabilities.get(session.socket)?.has(RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME)
)
return { outcome: 'deferred' }
if (
(this.idleWork.get(input.relayHostId) ?? 0) !== 0 ||
session.activeConnIds.size !== 0 ||
session.activeSplices.size !== 0 ||
session.pendingConns.size !== 0
)
return { outcome: 'busy' }
const revision = session.authorityRevision
const promise = Promise.resolve().then(async () => {
let outcome: 'committed' | 'deferred' | 'stale'
try {
outcome = (await commit()).outcome
if (outcome === 'deferred') {
const durable = await reconcile()
outcome = durable === 'not-committed' ? 'deferred' : durable
}
} catch {
let delay = 100
for (;;) {
try {
const durable = await reconcile()
outcome = durable === 'not-committed' ? 'deferred' : durable
break
} catch {
await new Promise<void>((resolve) => {
const timer = setTimeout(resolve, delay)
timer.unref?.()
})
delay = Math.min(delay * 2, 5000)
}
}
}
if (this.get(input) === session) {
if (outcome !== 'deferred' || this.draining || session.authorityRevision !== revision) {
this.closeDrainedSession(session)
}
}
if (this.idleAttempts.get(input.relayHostId)?.promise === promise)
this.idleAttempts.delete(input.relayHostId)
return { outcome }
})
this.idleAttempts.set(input.relayHostId, { attemptId: input.attemptId, authorityKey, promise })
return promise
}
private beginIdleWork(hostId: string): (() => void) | null {
if (this.idleAttempts.has(hostId)) return null
this.idleWork.set(hostId, (this.idleWork.get(hostId) ?? 0) + 1)
return () => {
const remaining = (this.idleWork.get(hostId) ?? 1) - 1
if (remaining === 0) this.idleWork.delete(hostId)
else this.idleWork.set(hostId, remaining)
}
}
constructor(
private readonly config: RelayConfig,
private readonly verifyRelayToken: VerifyRelayToken,
@@ -192,7 +295,8 @@ export class HostSessionRegistry {
private readonly queuedByteBudget: ProcessQueuedByteBudget,
private readonly observer: RelayRuntimeObserver,
private readonly now: () => number = Date.now,
private readonly random: () => number = Math.random
private readonly random: () => number = Math.random,
private readonly cellIncarnation?: string
) {}
// Uniform over [CONTROL_LEASE_MS - jitter, CONTROL_LEASE_MS + jitter).
@@ -206,6 +310,25 @@ export class HostSessionRegistry {
hostId: string,
credential: string,
capacityReservation?: PendingHostDataReservation
): Promise<void> {
const release = this.beginIdleWork(hostId)
if (!release) {
capacityReservation?.release()
this.rejectClient(socket, RELAY_CLOSE_CODE.WRONG_CELL)
return
}
try {
await this.acceptClientUnfenced(socket, hostId, credential, capacityReservation)
} finally {
release()
}
}
private async acceptClientUnfenced(
socket: WebSocket,
hostId: string,
credential: string,
capacityReservation?: PendingHostDataReservation
): Promise<void> {
if (this.draining) {
capacityReservation?.release()
@@ -295,6 +418,7 @@ export class HostSessionRegistry {
this.rejectClient(socket, RELAY_CLOSE_CODE.LIMIT_EXCEEDED)
return
}
const admittingSocket = session.socket
const connId = randomUUID()
const connTicket = randomBytes(32).toString('base64url')
const identity = { userId: reservation.userId, relayHostId: hostId }
@@ -324,6 +448,20 @@ export class HostSessionRegistry {
) {
return
}
// Admission may have crossed a drain or control replacement while persisting activity.
if (
this.draining ||
this.sessions.get(sessionKey) !== session ||
session.state !== 'active' ||
session.socket !== admittingSocket ||
admittingSocket.readyState !== admittingSocket.OPEN
) {
capacityReservation?.release()
this.failReservationBestEffort(reservation)
if (credentialActivityId) this.releaseActivityBestEffort(identity, credentialActivityId)
this.rejectClient(socket, RELAY_CLOSE_CODE.WRONG_CELL)
return
}
markStage('activity')
const attachTimer = setTimeout(() => {
session.pendingConns.delete(connId)
@@ -372,6 +510,27 @@ export class HostSessionRegistry {
connId: string,
connTicket: string,
generation: number
): Promise<boolean> {
const owner = [...this.sessions.values()].find((candidate) =>
candidate.pendingConns.has(connId)
)
const release = owner ? this.beginIdleWork(owner.relayHostId) : () => {}
if (!release) {
socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress')
return false
}
try {
return await this.acceptHostDataUnfenced(socket, connId, connTicket, generation)
} finally {
release()
}
}
private async acceptHostDataUnfenced(
socket: WebSocket,
connId: string,
connTicket: string,
generation: number
): Promise<boolean> {
const session = [...this.sessions.values()].find((candidate) =>
candidate.pendingConns.has(connId)
@@ -427,6 +586,27 @@ export class HostSessionRegistry {
socket.close(RELAY_CLOSE_CODE.LIMIT_EXCEEDED, 'basis persistence failed')
return false
}
// Already admitted attachments may finish a regional drain, but never a retired generation.
if (
this.draining ||
this.sessions.get(this.key(identity.userId, identity.relayHostId)) !== session ||
this.get(identity)?.state === 'closed' ||
!session.activeConnIds.has(connId) ||
socket.readyState !== socket.OPEN ||
pending.client.readyState !== pending.client.OPEN
) {
session.activeConnIds.delete(connId)
pending.capacityReservation?.release()
this.deactivateBasisBestEffort(connId)
this.failReservationBestEffort(pending.reservation)
if (spliceActivityId) this.releaseActivityBestEffort(identity, spliceActivityId)
if (pending.credentialActivityId) {
this.releaseActivityBestEffort(identity, pending.credentialActivityId)
}
this.rejectClient(pending.client, RELAY_CLOSE_CODE.DRAINING)
socket.close(RELAY_CLOSE_CODE.DRAINING, 'host retired during attachment')
return false
}
const close = wireSplice({
client: pending.client,
host: socket,
@@ -505,6 +685,7 @@ export class HostSessionRegistry {
JSON.stringify({
event: 'orca_relay_client_accept_completed',
...this.logIdentity(),
...this.sessionPlacementLogFields(session),
credentialKind: pending.reservation.credentialKind,
stageMs,
totalMs,
@@ -513,6 +694,14 @@ export class HostSessionRegistry {
)
}
private sessionPlacementLogFields(session: HostSession) {
return {
assignmentEpoch: session.assignmentEpoch,
controlGeneration: session.generation,
drainMode: session.regionalDrainAttemptId ? 'deadline' : 'none'
}
}
// Matches the runtime metrics event so a log line and a metric point can be
// joined back to the process that emitted them.
private logIdentity(): { role: string; cellId: string; region: RelayRegion } {
@@ -549,6 +738,7 @@ export class HostSessionRegistry {
JSON.stringify({
event: 'orca_relay_host_control_rtt',
...this.logIdentity(),
...this.sessionPlacementLogFields(session),
relayHostIdDigest: relayHostLogDigest(session.relayHostId),
rttMsMedian: percentile(samples, 0.5),
sampleCount: samples.length
@@ -562,6 +752,10 @@ export class HostSessionRegistry {
connectionInclusionWatermark?: number,
hostCapabilities?: ReadonlySet<string>
): void {
if (this.idleAttempts.has(identity.relayHostId)) {
socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress')
return
}
// Keyed by socket, not session: a rebind swaps the session's socket, and the
// successor's own advertisement is the only one that describes its decoder.
if (hostCapabilities?.size) this.hostCapabilities.set(socket, hostCapabilities)
@@ -602,8 +796,7 @@ export class HostSessionRegistry {
socket: WebSocket | null,
context: string
): void {
void Promise.resolve()
.then(task)
void (async () => task())()
.catch((error: unknown) => {
const message = (error instanceof Error ? error.message : 'unknown')
// Untruncated, unlike peer-supplied close reasons: this is the
@@ -653,6 +846,7 @@ export class HostSessionRegistry {
this.draining = true
for (const session of this.sessions.values()) {
if (session.state === 'closed') continue
session.authorityRevision += 1
session.state = 'drain-only'
if (session.socket) send(session.socket, 'drain', { graceMs, recovery: 'resolve-director' })
setTimeout(() => this.closeDrainedSession(session), graceMs)
@@ -665,7 +859,8 @@ export class HostSessionRegistry {
relayHostId: string
sourceAssignmentEpoch: number
graceMs: number
}): RegionalHostDrainOutcome {
sourceCellIncarnation?: string
}): RegionalHostDrainOutcome | Promise<RegionalHostDrainOutcome> {
const session = this.get(input)
if (!session || session.state === 'closed') return 'host-not-connected'
if (session.assignmentEpoch !== input.sourceAssignmentEpoch) {
@@ -678,13 +873,11 @@ export class HostSessionRegistry {
this.reassertRegionalDrain(session)
return 'already-accepted'
}
session.authorityRevision += 1
session.regionalDrainAttemptId = input.attemptId
session.regionalDrainExpiresAt = this.now() + input.graceMs
this.reassertRegionalDrain(session)
session.regionalDrainTimer = setTimeout(
() => this.closeDrainedSession(session),
input.graceMs
)
session.regionalDrainTimer = setTimeout(() => this.closeDrainedSession(session), input.graceMs)
return 'accepted'
}
@@ -740,9 +933,9 @@ export class HostSessionRegistry {
const existing = this.sessions.get(key)
const rebind = Boolean(
existing &&
hello.data.controlResumeSecret &&
hello.data.controlResumeSecret === existing.controlResumeSecret &&
(existing.state === 'orphaned' || existing.state === 'active')
hello.data.controlResumeSecret &&
hello.data.controlResumeSecret === existing.controlResumeSecret &&
(existing.state === 'orphaned' || existing.state === 'active')
)
const generation = rebind ? existing!.generation : (existing?.generation ?? 0) + 1
const ephemeral = nacl.box.keyPair()
@@ -784,7 +977,9 @@ export class HostSessionRegistry {
}, 10_000)
socket.once('message', (raw, isBinary) => {
clearTimeout(proofTimer)
const ack = isBinary ? null : HostChallengeAckSchema.safeParse(payload(raw, 'host-challenge-ack'))
const ack = isBinary
? null
: HostChallengeAckSchema.safeParse(payload(raw, 'host-challenge-ack'))
const proof = ack?.success ? decodeCanonicalBase64(ack.data.proofB64, 32) : null
if (
!ack?.success ||
@@ -826,6 +1021,11 @@ export class HostSessionRegistry {
appVersion: string,
connectionInclusionWatermark?: number
): Promise<void> {
const release = this.beginIdleWork(identity.relayHostId)
if (!release) {
socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress')
return Promise.resolve()
}
const key = this.key(identity.sub, identity.relayHostId)
const previous = this.activationQueues.get(key) ?? Promise.resolve()
// The timeout only fails this waiting socket; the queue entry still chains
@@ -836,26 +1036,29 @@ export class HostSessionRegistry {
socket.close(RELAY_CLOSE_CODE.LIMIT_EXCEEDED, 'control activation queue stalled')
}, ACTIVATION_QUEUE_WAIT_MS)
queueWaitTimer.unref?.()
const activation = previous.catch(() => undefined).then(async () => {
clearTimeout(queueWaitTimer)
if (queueWaitExpired) return
if ((this.sessions.get(key) ?? null) !== existing) {
socket.close(RELAY_CLOSE_CODE.PEER_DROPPED, 'control activation superseded')
return
}
await this.activateCurrent(
socket,
identity,
existing,
generation,
rebind,
assignmentEpoch,
appVersion,
connectionInclusionWatermark
)
})
const activation = previous
.catch(() => undefined)
.then(async () => {
clearTimeout(queueWaitTimer)
if (queueWaitExpired) return
if ((this.sessions.get(key) ?? null) !== existing) {
socket.close(RELAY_CLOSE_CODE.PEER_DROPPED, 'control activation superseded')
return
}
await this.activateCurrent(
socket,
identity,
existing,
generation,
rebind,
assignmentEpoch,
appVersion,
connectionInclusionWatermark
)
})
this.activationQueues.set(key, activation)
const cleanup = (): void => {
release()
if (this.activationQueues.get(key) === activation) this.activationQueues.delete(key)
}
void activation.then(cleanup, cleanup)
@@ -881,7 +1084,11 @@ export class HostSessionRegistry {
cellId: this.config.cellId,
assignmentEpoch,
generation,
connectionInclusionWatermark
connectionInclusionWatermark,
idleRegionalRehome:
this.hostCapabilities.get(socket)?.has(RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME) ??
false,
cellIncarnation: this.cellIncarnation
}
)
await this.assignments.markMigrationTargetRegistered(
@@ -918,14 +1125,15 @@ export class HostSessionRegistry {
const previousSocket = existing.socket
if (existing.orphanTimer) clearTimeout(existing.orphanTimer)
existing.orphanTimer = null
existing.authorityRevision += 1
existing.assignmentEpoch = assignmentEpoch
existing.socket = socket
existing.state = existing.regionalDrainAttemptId ? 'drain-only' : 'active'
existing.appVersion = appVersion
existing.leaseExpiresAt = this.controlLeaseExpiresAt()
existing.lastPongAt = this.now()
existing.pendingPingAt = null
existing.activityRenewalDueAt =
this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs
existing.activityRenewalDueAt = this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs
this.wireActiveControl(existing)
this.sendHelloAck(existing)
if (existing.regionalDrainAttemptId) this.reassertRegionalDrain(existing)
@@ -980,6 +1188,7 @@ export class HostSessionRegistry {
pendingPingAt: null,
controlRttSamplesMs: [],
controlRttLoggedAt: null,
authorityRevision: 0,
activityRenewalDueAt: this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs,
activityRenewalAttempt: 0,
activityRenewalCompletedAttempt: 0,
@@ -1028,7 +1237,9 @@ export class HostSessionRegistry {
` splices=${session.closingCounts?.splices ?? session.activeSplices.size}` +
` pending=${session.closingCounts?.pending ?? session.pendingConns.size}` +
` code=${code} reason=${JSON.stringify(printableCloseReason(reason))}` +
(socketError === null ? '' : ` error=${JSON.stringify(printableCloseReason(socketError))}`)
(socketError === null
? ''
: ` error=${JSON.stringify(printableCloseReason(socketError))}`)
)
})
socket.on('message', (raw, isBinary) => {
@@ -1077,6 +1288,19 @@ export class HostSessionRegistry {
}
private async acceptRefresh(session: HostSession, raw: RawData): Promise<void> {
const release = this.beginIdleWork(session.relayHostId)
if (!release) {
session.socket?.close(RELAY_CLOSE_CODE.DRAINING, 'resolve-director')
return
}
try {
await this.acceptRefreshUnfenced(session, raw)
} finally {
release()
}
}
private async acceptRefreshUnfenced(session: HostSession, raw: RawData): Promise<void> {
const parsed = AuthRefreshSchema.safeParse(payload(raw, 'auth-refresh'))
if (!parsed.success) return
const refreshed = await this.verifyRelayToken(parsed.data.relayJwt)
@@ -1107,6 +1331,16 @@ export class HostSessionRegistry {
if (controlActivityId && now >= session.activityRenewalDueAt) {
const attempt = ++session.activityRenewalAttempt
const startedAt = now
const socket = session.socket
const authorityRevision = session.authorityRevision
const current = (): boolean =>
this.sessions.get(key) === session &&
session.state !== 'closed' &&
session.socket === socket &&
socket.readyState === socket.OPEN &&
session.controlActivityId === controlActivityId &&
session.authorityRevision === authorityRevision &&
attempt > session.activityRenewalCompletedAttempt
void this.assignments
.renewControlActivity(
{ userId: session.identity.sub, relayHostId: session.relayHostId },
@@ -1117,11 +1351,16 @@ export class HostSessionRegistry {
}
)
.then(() => {
if (attempt <= session.activityRenewalCompletedAttempt) return
if (!current()) return
session.activityRenewalCompletedAttempt = attempt
session.activityRenewalDueAt = startedAt + CONTROL_ACTIVITY_RENEWAL_INTERVAL_MS
})
.catch(async (error: unknown) => {
if (!current()) return
if (error instanceof Error && error.message === 'assignment_not_found') {
socket.close(RELAY_CLOSE_CODE.DRAINING, 'control assignment missing')
return
}
if (error instanceof Error && error.message === 'activity_cell_not_authoritative') {
// Completion fences a late drain-only heartbeat after all source work is gone.
session.socket?.close(RELAY_CLOSE_CODE.DRAINING, 'control migration completed')
@@ -1144,8 +1383,22 @@ export class HostSessionRegistry {
cellId: this.config.cellId
}
)
if (!current()) {
// A replaced activity must not remain leased after its owner disappears.
if (
!this.sessions.get(key) ||
this.sessions.get(key)?.controlActivityId !== controlActivityId
) {
this.releaseActivityBestEffort(
{ userId: session.identity.sub, relayHostId: session.relayHostId },
controlActivityId
)
}
return
}
this.observer.recordControlActivityRecovery?.(true)
} catch (acquireError: unknown) {
if (!current()) return
this.observer.recordControlActivityRecovery?.(false)
if (
acquireError instanceof Error &&
@@ -1218,6 +1471,21 @@ export class HostSessionRegistry {
private closeDrainedSession(session: HostSession): void {
if (session.state === 'closed') return
const forcedConnections = session.activeConnIds.size + session.pendingConns.size
if (forcedConnections > 0) {
console.warn(
JSON.stringify({
event: 'orca_relay_host_drain_forced_close',
...this.logIdentity(),
...this.sessionPlacementLogFields(session),
relayHostIdDigest: relayHostLogDigest(session.relayHostId),
reason: this.draining ? 'emergency' : 'regional-deadline',
forcedConnections,
splices: session.activeSplices.size,
pending: session.pendingConns.size
})
)
}
if (session.heartbeatTimer) clearInterval(session.heartbeatTimer)
if (session.orphanTimer) clearTimeout(session.orphanTimer)
if (session.regionalDrainTimer) clearTimeout(session.regionalDrainTimer)
@@ -1250,11 +1518,7 @@ export class HostSessionRegistry {
session.pendingConns.clear()
session.state = 'closed'
if (session.socket) {
closeRelayWebSocket(
session.socket,
RELAY_CLOSE_CODE.DRAINING,
'resolve configured director'
)
closeRelayWebSocket(session.socket, RELAY_CLOSE_CODE.DRAINING, 'resolve configured director')
}
const key = this.key(session.identity.sub, session.relayHostId)
if (this.sessions.get(key) === session) this.sessions.delete(key)
@@ -1278,6 +1542,23 @@ export class HostSessionRegistry {
session: HostSession,
type: unknown,
raw: RawData
): Promise<void> {
const release = this.beginIdleWork(session.relayHostId)
if (!release) {
session.socket?.close(RELAY_CLOSE_CODE.DRAINING, 'resolve-director')
return
}
try {
await this.acceptControlCommandUnfenced(session, type, raw)
} finally {
release()
}
}
private async acceptControlCommandUnfenced(
session: HostSession,
type: unknown,
raw: RawData
): Promise<void> {
if (typeof type !== 'string' || !session.socket) return
try {
@@ -1315,10 +1596,7 @@ export class HostSessionRegistry {
}
if (type === 'device-credential-install') {
const request = DeviceCredentialInstallSchema.parse(payload(raw, type))
if (
session.state !== 'active' &&
request.authorization.mode === 'authenticated-direct'
) {
if (session.state !== 'active' && request.authorization.mode === 'authenticated-direct') {
throw new Error('authorization_expired')
}
const installActivityId = `install:${request.reqId}`
cloud/apps/relay/src/idle-regional-rehome-reconciliation.test.ts
+103
View File
@@ -0,0 +1,103 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import type { RelayDatabase } from './database.js'
import { openIdleRehomeTestDatabase } from './idle-regional-rehome-test-database.js'
const request = {
v: 1 as const,
attemptId: '33333333-3333-4333-8333-333333333333',
userId: 'idle-reconciliation-test',
relayHostId: 'abcdefghijklmnop',
sourceCellId: 'source',
sourceCellIncarnation: '11111111-1111-4111-8111-111111111111',
sourceAssignmentEpoch: 1,
sourceGeneration: 7,
targetCellId: 'target'
}
const databases: RelayDatabase[] = []
afterEach(async () => {
vi.restoreAllMocks()
for (const database of databases.splice(0)) await database.close()
})
async function setup() {
const database = await openIdleRehomeTestDatabase()
databases.push(database)
const store = new RelayAssignmentStore(database, () => 100_000_000)
await store.reconcileCells([
{ id: 'source', url: 'https://source.example.test', capacityRequests: 100 },
{ id: 'target', url: 'https://target.example.test', capacityRequests: 100 }
])
await store.assign(request)
await store.activateControl(request, {
cellId: request.sourceCellId,
assignmentEpoch: request.sourceAssignmentEpoch,
generation: request.sourceGeneration,
cellIncarnation: request.sourceCellIncarnation
})
return { database, store }
}
describe('idle cutover durable reconciliation', () => {
it('only permits reopening when the exact source still owns the assignment', async () => {
const { store } = await setup()
expect(await store.reconcileIdleRegionalRehome(request)).toBe('not-committed')
await store.activateControl(request, {
cellId: request.sourceCellId,
assignmentEpoch: request.sourceAssignmentEpoch,
generation: request.sourceGeneration + 1,
cellIncarnation: request.sourceCellIncarnation
})
expect(await store.reconcileIdleRegionalRehome(request)).toBe('stale')
})
it('does not reopen an obsolete source after an assignment change', async () => {
const { store } = await setup()
await store.startEvacuation(request, request.targetCellId)
expect(await store.reconcileIdleRegionalRehome(request)).toBe('stale')
})
it('propagates unavailable durable state instead of declaring rollback', async () => {
const { database, store } = await setup()
vi.spyOn(database, 'transaction').mockRejectedValue(new Error('database_unavailable'))
await expect(store.reconcileIdleRegionalRehome(request)).rejects.toThrow('database_unavailable')
})
it('waits for an outstanding assignment transaction before deciding authority', async () => {
const { database, store } = await setup()
let release!: () => void
let entered!: () => void
const locked = new Promise<void>((resolve) => {
entered = resolve
})
const gate = new Promise<void>((resolve) => {
release = resolve
})
const commit = database.transaction(async (transaction) => {
await transaction.queryLocked(
'SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?',
[request.userId, request.relayHostId]
)
entered()
await gate
await transaction.query(
'UPDATE relay_assignments SET assignment_epoch = assignment_epoch + 1 WHERE user_id = ? AND relay_host_id = ?',
[request.userId, request.relayHostId]
)
})
await locked
let settled = false
const reconciliation = store.reconcileIdleRegionalRehome(request).finally(() => {
settled = true
})
try {
await new Promise<void>((resolve) => setImmediate(resolve))
expect(settled).toBe(false)
} finally {
release()
await commit
await reconciliation
}
expect(await reconciliation).toBe('stale')
})
})
cloud/apps/relay/src/idle-regional-rehome-selection.ts
+117
View File
@@ -0,0 +1,117 @@
import { createHash } from 'node:crypto'
import type { IdleRegionalRehomeRequest } from '@orca-cloud/relay-contract'
import type { RelayDatabase, SqlRow } from './database.js'
export const IDLE_REHOME_PAGE_SIZE = 100
export async function selectIdleRegionalRehomes(input: {
database: RelayDatabase
now: number
heartbeatTtlMs: number
cohortPercent: number
offset: number
connectionHeadroom: Map<string, boolean>
cellIsClean: (safety: SqlRow | undefined, runtime: SqlRow, now: number) => boolean
}): Promise<Array<IdleRegionalRehomeRequest & { sourceCellUrl: string }>> {
const [runtimes, safetyRows] = await Promise.all([
input.database.query('SELECT * FROM relay_cell_runtime'),
input.database.query('SELECT * FROM relay_cell_rehome_safety')
])
const cleanCells = runtimes
.filter((runtime) =>
input.cellIsClean(
safetyRows.find((safety) => safety.cell_id === runtime.cell_id),
runtime,
input.now
)
)
.map((runtime) => String(runtime.cell_id))
const targetCells = cleanCells.filter((id) => input.connectionHeadroom.get(id) !== false)
if (!cleanCells.length || !targetCells.length) return []
const rows = await input.database.query(
`SELECT a.user_id, a.relay_host_id, a.cell_id AS source_cell_id,
a.assignment_epoch, host.generation, r.cell_incarnation,
s.cell_url, target.cell_id AS target_cell_id
FROM relay_region_rehome_control policy
JOIN relay_region_decisions d ON d.outcome = 'conclusive'
JOIN relay_assignments a ON a.user_id = d.user_id AND a.relay_host_id = d.relay_host_id
JOIN relay_cells s ON s.cell_id = a.cell_id AND s.enabled = 1
JOIN relay_cell_regions sr ON sr.cell_id = a.cell_id
JOIN relay_cell_admission sa ON sa.cell_id = a.cell_id AND sa.admission_state = 'general'
JOIN relay_cell_runtime r ON r.cell_id = a.cell_id AND r.ready = 1
JOIN relay_cell_capabilities c ON c.cell_id = r.cell_id AND c.cell_incarnation = r.cell_incarnation
JOIN relay_control_capabilities host ON host.user_id = a.user_id AND host.relay_host_id = a.relay_host_id
AND host.cell_id = a.cell_id AND host.assignment_epoch = a.assignment_epoch
AND host.cell_incarnation = r.cell_incarnation AND host.idle_regional_rehome = 1
JOIN relay_assignment_activity_leases lease ON lease.user_id = host.user_id
AND lease.relay_host_id = host.relay_host_id AND lease.activity_id = host.activity_id
AND lease.cell_id = a.cell_id AND lease.activity_kind = 'control'
JOIN relay_cell_regions tr ON tr.region = d.preferred_region
JOIN relay_cells target ON target.cell_id = tr.cell_id AND target.enabled = 1
JOIN relay_cell_admission ta ON ta.cell_id = target.cell_id AND ta.admission_state = 'general'
JOIN relay_cell_runtime rt ON rt.cell_id = target.cell_id AND rt.ready = 1
JOIN relay_cell_capabilities ct ON ct.cell_id = rt.cell_id AND ct.cell_incarnation = rt.cell_incarnation
WHERE policy.control_id = 'global' AND policy.enabled = 1 AND policy.not_before <= ?
AND d.preferred_region <> sr.region AND d.incumbent_region = sr.region
AND d.assignment_epoch = a.assignment_epoch AND d.policy_version = 1
AND d.expires_at > ? AND d.observed_at >= ? - policy.preference_max_age_ms
AND d.cohort_bucket < ? AND lease.expires_at > ? AND lease.updated_at >= r.started_at
AND r.last_heartbeat_at > ? AND rt.last_heartbeat_at > ?
AND s.cell_id IN (${cleanCells.map(() => '?').join(',')})
AND target.cell_id IN (${targetCells.map(() => '?').join(',')})
-- Reserve the moving host's source activity plus its assignment on the target.
AND target.reserved_requests + 1 + (
SELECT COALESCE(SUM(activity.request_units), 0)
FROM relay_assignment_activity_leases activity
WHERE activity.user_id = a.user_id AND activity.relay_host_id = a.relay_host_id
AND activity.cell_id = a.cell_id
) <= target.capacity_requests
AND c.regional_rehome_protocol >= 3 AND ct.regional_rehome_protocol >= 3
AND NOT EXISTS (SELECT 1 FROM relay_assignment_migrations migration
WHERE migration.user_id = a.user_id AND migration.relay_host_id = a.relay_host_id
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL)
AND NOT EXISTS (SELECT 1 FROM relay_region_rehome_attempts attempt
WHERE attempt.user_id = a.user_id AND attempt.relay_host_id = a.relay_host_id
AND attempt.created_at > ? - policy.host_cooldown_ms)
ORDER BY a.user_id, a.relay_host_id, host.generation DESC,
(target.reserved_requests + rt.observed_requests) * 1.0 / target.capacity_requests,
target.cell_id
LIMIT ? OFFSET ?`,
[
input.now,
input.now,
input.now,
input.cohortPercent,
input.now,
input.now - input.heartbeatTtlMs,
input.now - input.heartbeatTtlMs,
...cleanCells,
...targetCells,
input.now,
IDLE_REHOME_PAGE_SIZE,
input.offset
]
)
return rows.map((row) => {
const request = {
v: 1 as const,
userId: String(row.user_id),
relayHostId: String(row.relay_host_id),
sourceCellId: String(row.source_cell_id),
sourceCellIncarnation: String(row.cell_incarnation),
sourceAssignmentEpoch: Number(row.assignment_epoch),
sourceGeneration: Number(row.generation),
targetCellId: String(row.target_cell_id)
}
// UUIDv5 keeps retries on every director bound to the same source authority and target.
const digest = createHash('sha1')
.update(Buffer.from('0a1c5a9b197b4ea8b6f1f3bcaa3d712c', 'hex'))
.update(JSON.stringify(request))
.digest()
digest[6] = (digest[6]! & 0x0f) | 0x50
digest[8] = (digest[8]! & 0x3f) | 0x80
const hex = digest.subarray(0, 16).toString('hex')
const attemptId = `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`
return { ...request, attemptId, sourceCellUrl: String(row.cell_url) }
})
}
cloud/apps/relay/src/idle-regional-rehome-store.test.ts
+350
View File
@@ -0,0 +1,350 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import type { RelayDatabase, RelayLockOptions } from './database.js'
import { openIdleRehomeTestDatabase } from './idle-regional-rehome-test-database.js'
const identity = { userId: 'idle-store-test', relayHostId: 'abcdefghijklmnop' }
const incarnations = [
'11111111-1111-4111-8111-111111111111',
'22222222-2222-4222-8222-222222222222'
]
const cells = [
{
id: 'source',
url: 'https://source.example.test',
region: 'us-central1' as const,
capacityRequests: 100
},
{
id: 'target',
url: 'https://target.example.test',
region: 'asia-east2' as const,
capacityRequests: 100
}
]
const databases: RelayDatabase[] = []
afterEach(async () => {
vi.restoreAllMocks()
for (const database of databases.splice(0)) await database.close()
})
async function setup() {
const database = await openIdleRehomeTestDatabase()
databases.push(database)
let now = 100_000_000
const store = new RelayAssignmentStore(database, () => now, { regionalRehomeCohortPercent: 100 })
await store.inspectRegionalRehomeControl()
now += 86_400_000
await store.applyRegionalRehomeControl({
expectedGeneration: 0,
enabled: true,
notBefore: now,
ratePerMinute: 10,
preferenceMaxAgeMs: 86_400_000,
hostCooldownMs: 604_800_000,
drainGraceMs: 60_000
})
await store.reconcileCells(cells)
const safety = {
observedAt: now,
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
}
for (const [index, cell] of cells.entries()) {
await store.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
region: cell.region,
cellIncarnation: incarnations[index]!,
startedAt: now - 1_000,
ready: true,
observedRequests: 0
})
await store.recordCellRegionalRehomeStatus({
cellId: cell.id,
cellIncarnation: incarnations[index]!,
regionalRehomeProtocol: 3,
safety
})
}
const assignment = await store.assign(identity, undefined, 'us-central1')
await store.activateControl(identity, {
cellId: cells[0]!.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 7,
cellIncarnation: incarnations[0],
idleRegionalRehome: true
})
const issued = await store.exchangeRegionCorrection(
identity,
{ v: 1, action: 'issue-window' },
assignment.assignmentEpoch
)
await store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: issued.window!.generation,
assignmentEpoch: assignment.assignmentEpoch,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 180, 'asia-east2': 40 }
},
assignment.assignmentEpoch
)
const request = {
v: 1 as const,
...identity,
attemptId: '33333333-3333-4333-8333-333333333333',
sourceCellId: cells[0]!.id,
sourceCellIncarnation: incarnations[0]!,
sourceAssignmentEpoch: assignment.assignmentEpoch,
sourceGeneration: 7,
targetCellId: cells[1]!.id
}
return { store, database, safety, request }
}
describe('constrained idle regional assignment transaction', () => {
it.each([10, 11])('reserves source activity plus assignment at target capacity %i', async (capacity) => {
const { store, database, safety, request } = await setup()
// Model three source activity units and seven units already reserved at the target.
await database.query(
'UPDATE relay_assignment_activity_leases SET request_units = 3 WHERE user_id = ? AND relay_host_id = ?',
[identity.userId, identity.relayHostId]
)
await database.query("UPDATE relay_cells SET reserved_requests = 4 WHERE cell_id = 'source'")
await database.query(
"UPDATE relay_cells SET reserved_requests = 7, capacity_requests = ? WHERE cell_id = 'target'",
[capacity]
)
const candidates = await store.selectIdleRegionalRehomeCandidates(safety)
expect(candidates).toHaveLength(capacity === 11 ? 1 : 0)
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({
outcome: capacity === 11 ? 'committed' : 'deferred'
})
const [target] = await database.query("SELECT reserved_requests FROM relay_cells WHERE cell_id = 'target'")
expect(Number(target!.reserved_requests)).toBe(capacity === 11 ? 11 : 7)
expect(await store.resolve(identity)).toMatchObject({
cellId: capacity === 11 ? 'target' : 'source',
assignmentEpoch: capacity === 11 ? 2 : 1
})
})
it('progresses past a full page of busy candidates without writing eligibility state', async () => {
const { store, database, safety } = await setup()
for (const table of [
'relay_assignments',
'relay_assignment_activity_leases',
'relay_control_capabilities',
'relay_region_decisions'
]) {
const template = (
await database.query(`SELECT * FROM ${table} WHERE user_id = ? AND relay_host_id = ?`, [
identity.userId,
identity.relayHostId
])
)[0]!
const columns = Object.keys(template)
for (let index = 0; index < 100; index++) {
const values = columns.map((column) =>
column === 'user_id' || column === 'relay_host_id' ? '?' : column
)
await database.query(
`INSERT INTO ${table} (${columns.join(', ')}) SELECT ${values.join(', ')} FROM ${table}
WHERE user_id = ? AND relay_host_id = ?`,
[
`idle-store-test-${String(index).padStart(3, '0')}`,
`pagehost${String(index).padStart(8, '0')}`,
identity.userId,
identity.relayHostId
]
)
}
}
const first = await store.selectIdleRegionalRehomeCandidates(safety)
const next = await store.selectIdleRegionalRehomeCandidates(safety)
expect(first).toHaveLength(100)
expect(next).toHaveLength(1)
expect(next[0]!.relayHostId).toBe('pagehost00000099')
const restarted = new RelayAssignmentStore(database, () => safety.observedAt, {
regionalRehomeCohortPercent: 100
})
expect(await restarted.selectIdleRegionalRehomeCandidates(safety)).toEqual(first)
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
const decisions = await database.query('SELECT last_considered_at FROM relay_region_decisions')
expect(decisions.every((decision) => Number(decision.last_considered_at) === 0)).toBe(true)
})
it.runIf(Boolean(process.env.ORCA_IDLE_REHOME_POSTGRES_URL))(
'rechecks generation when replacement wins after the initial authority lookup',
async () => {
const { store, safety, request, database } = await setup()
const held = holdStatement(database, 'SELECT * FROM relay_region_rehome_control')
const commit = store.commitIdleRegionalRehome(request, safety)
await held.entered
try {
await store.activateControl(identity, {
cellId: 'source',
assignmentEpoch: 1,
generation: 8,
cellIncarnation: incarnations[0],
idleRegionalRehome: true
})
} finally {
held.release()
}
expect(await commit).toEqual({ outcome: 'deferred' })
expect(await store.reconcileIdleRegionalRehome(request)).toBe('stale')
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
}
)
it('rejects source replacement when the cutover already holds assignment authority', async () => {
const { store, safety, request, database } = await setup()
const held = holdStatement(database, 'UPDATE relay_assignments SET cell_id')
const commit = store.commitIdleRegionalRehome(request, safety)
await held.entered
const replacement = store.activateControl(identity, {
cellId: 'source',
assignmentEpoch: 1,
generation: 8,
cellIncarnation: incarnations[0],
idleRegionalRehome: true
})
const rejected = expect(replacement).rejects.toThrow('wrong_assignment')
held.release()
expect(await commit).toEqual({ outcome: 'committed' })
await rejected
expect(await store.resolve(identity)).toMatchObject({ cellId: 'target', assignmentEpoch: 2 })
})
it('finds the committed attempt after its database reply is lost', async () => {
const { store, safety, request, database } = await setup()
const transaction = database.transaction.bind(database)
const intercepted = vi
.spyOn(database, 'transaction')
.mockImplementation(async (operation, options) => {
let changed = false
const result = await transaction(
async (tx) =>
operation(
new Proxy(tx, {
get(target, key) {
if (key === 'query')
return async (sql: string, params?: unknown[]) => {
if (sql.includes('INSERT INTO relay_region_rehome_attempts')) changed = true
return target.query(sql, params)
}
const value = Reflect.get(target, key)
return typeof value === 'function' ? value.bind(target) : value
}
})
),
options
)
if (changed) throw new Error('simulated_commit_reply_lost')
return result
})
await expect(store.commitIdleRegionalRehome(request, safety)).rejects.toThrow(
'simulated_commit_reply_lost'
)
intercepted.mockRestore()
expect(await store.reconcileIdleRegionalRehome(request)).toBe('committed')
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({ outcome: 'committed' })
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toHaveLength(1)
})
it('commits the requested move once and records its outcome without source retention', async () => {
const { store, database, safety, request } = await setup()
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({ outcome: 'committed' })
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({ outcome: 'committed' })
expect(await store.resolve(identity)).toMatchObject({ cellId: 'target', assignmentEpoch: 2 })
const attempts = await database.query('SELECT * FROM relay_region_rehome_attempts')
expect(attempts).toHaveLength(1)
expect(attempts[0]!.attempt_id).toBe(request.attemptId)
expect(Number(attempts[0]!.source_generation)).toBe(7)
})
it('rejects a replaced control and never substitutes a different target', async () => {
const { store, safety, request } = await setup()
expect(
await store.commitIdleRegionalRehome({ ...request, targetCellId: 'missing' }, safety)
).toEqual({ outcome: 'deferred' })
await store.activateControl(identity, {
cellId: 'source',
assignmentEpoch: 1,
generation: 8,
cellIncarnation: incarnations[0],
idleRegionalRehome: true
})
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({ outcome: 'stale' })
expect(await store.resolve(identity)).toMatchObject({ cellId: 'source', assignmentEpoch: 1 })
})
it('does not commit without process safety or cohort authorization', async () => {
const { store, safety, request, database } = await setup()
expect(await store.commitIdleRegionalRehome(request)).toEqual({ outcome: 'deferred' })
expect(await store.commitIdleRegionalRehome(request, safety, 0)).toEqual({
outcome: 'deferred'
})
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
})
it('selects read-only with stable identity and the control generation, not probe generation', async () => {
const { store, safety, database } = await setup()
const before = await database.query('SELECT * FROM relay_assignments')
const candidates = await store.selectIdleRegionalRehomeCandidates(safety)
expect(candidates).toHaveLength(1)
expect(candidates[0]).toMatchObject({
sourceGeneration: 7,
sourceCellId: 'source',
targetCellId: 'target'
})
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual(candidates)
expect(await database.query('SELECT * FROM relay_assignments')).toEqual(before)
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
})
})
function holdStatement(database: RelayDatabase, fragment: string) {
let entered!: () => void
let release!: () => void
const arrival = new Promise<void>((resolve) => {
entered = resolve
})
const gate = new Promise<void>((resolve) => {
release = resolve
})
let held = false
const transaction = database.transaction.bind(database)
vi.spyOn(database, 'transaction').mockImplementation((operation, options) =>
transaction(async (tx) => {
return operation(
new Proxy(tx, {
get(target, key) {
if (key === 'query' || key === 'queryLocked')
return async (sql: string, params?: unknown[], lockOptions?: RelayLockOptions) => {
if (!held && sql.includes(fragment)) {
held = true
entered()
await gate
}
return key === 'queryLocked'
? target.queryLocked(sql, params, lockOptions)
: target.query(sql, params)
}
const value = Reflect.get(target, key)
return typeof value === 'function' ? value.bind(target) : value
}
})
)
}, options)
)
return { entered: arrival, release }
}
cloud/apps/relay/src/idle-regional-rehome-test-database.ts
+40
View File
@@ -0,0 +1,40 @@
import { randomUUID } from 'node:crypto'
import pg from 'pg'
import { openInMemoryRelayDatabase, openRelayDatabase, type RelayDatabase } from './database.js'
export async function openIdleRehomeTestDatabase(): Promise<RelayDatabase> {
const configured = process.env.ORCA_IDLE_REHOME_POSTGRES_URL
if (!configured) return openInMemoryRelayDatabase()
const url = new URL(configured)
if (url.port !== '55440' || !['localhost', '127.0.0.1', '[::1]'].includes(url.hostname)) {
throw new Error('idle_rehome_tests_require_local_postgres_55440')
}
const schema = `idle_rehome_${randomUUID().replaceAll('-', '')}`
const admin = new pg.Client({ connectionString: configured })
await admin.connect()
try {
await admin.query(`CREATE SCHEMA ${schema}`)
url.searchParams.set('options', `-c search_path=${schema}`)
const database = await openRelayDatabase({ databaseUrl: url.toString(), dataDir: '' })
const close = database.close.bind(database)
database.close = async () => {
try {
await close()
} finally {
try {
await admin.query(`DROP SCHEMA ${schema} CASCADE`)
} finally {
await admin.end()
}
}
}
return database
} catch (error) {
try {
await admin.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
} finally {
await admin.end()
}
throw error
}
}
cloud/apps/relay/src/idle-regional-rehome-worker.test.ts
+105
View File
@@ -0,0 +1,105 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { RelayAssignmentStore } from './assignment-store.js'
import type { RelayConfig } from './config.js'
import { startRegionalRehomeWorker } from './regional-rehome-worker.js'
const candidate = {
v: 1,
attemptId: '11111111-1111-4111-8111-111111111111',
userId: 'private-user',
relayHostId: 'abcdefghijklmnop',
sourceCellId: 'source',
sourceCellUrl: 'https://source.example.test',
sourceCellIncarnation: '22222222-2222-4222-8222-222222222222',
sourceAssignmentEpoch: 7,
sourceGeneration: 3,
targetCellId: 'target'
}
const config = {
role: 'director',
regionCorrectionCohortPercent: 100,
rehomeAudience: 'https://relay.example.test/v1/admin/host-drain',
rehomeDirectorServiceAccount: 'director@example.test'
} as RelayConfig
function setup(fetch: typeof globalThis.fetch) {
const selectIdleRegionalRehomeCandidates = vi
.fn()
.mockResolvedValueOnce([])
.mockResolvedValue([candidate])
const claimRegionalRehome = vi.fn()
const recordRegionalRehomeDispatchFailure = vi.fn()
const worker = startRegionalRehomeWorker(
config,
{
selectIdleRegionalRehomeCandidates,
claimRegionalRehome,
recordRegionalRehomeDispatchFailure
} as unknown as RelayAssignmentStore,
{
safetySnapshot: () => ({ observedAt: 100 }) as never,
intervalMs: 60_000,
identityToken: async () => 'private-token',
fetch
}
)!
return {
worker,
selectIdleRegionalRehomeCandidates,
claimRegionalRehome,
recordRegionalRehomeDispatchFailure
}
}
describe('idle regional worker dispatch', () => {
afterEach(() => vi.restoreAllMocks())
it('sends an idle request without claiming an assignment first', async () => {
const fetch = vi.fn<typeof globalThis.fetch>(async () =>
Response.json({ v: 1, outcome: 'committed' })
)
const c = setup(fetch)
await vi.waitFor(() => expect(c.selectIdleRegionalRehomeCandidates).toHaveBeenCalledOnce())
await c.worker.run()
c.worker.stop()
expect(c.claimRegionalRehome).not.toHaveBeenCalled()
expect(fetch).toHaveBeenCalledOnce()
const [url, init] = fetch.mock.calls[0]!
expect(String(url)).toBe('https://source.example.test/v1/admin/host-idle-rehome')
const { sourceCellUrl: _, ...request } = candidate
expect(JSON.parse(String(init?.body))).toEqual({
...request,
cohortPercent: 100,
directorSafety: { observedAt: 100 }
})
})
it('progresses past busy hosts without charging a dispatch failure', async () => {
const fetch = vi
.fn<typeof globalThis.fetch>()
.mockResolvedValueOnce(Response.json({ v: 1, outcome: 'busy' }))
.mockResolvedValueOnce(Response.json({ v: 1, outcome: 'committed' }))
const c = setup(fetch)
await vi.waitFor(() => expect(c.selectIdleRegionalRehomeCandidates).toHaveBeenCalledOnce())
c.selectIdleRegionalRehomeCandidates.mockResolvedValue([
candidate,
{
...candidate,
relayHostId: 'ponmlkjihgfedcba',
attemptId: '33333333-3333-4333-8333-333333333333'
}
])
await c.worker.run()
c.worker.stop()
expect(fetch).toHaveBeenCalledTimes(2)
expect(c.recordRegionalRehomeDispatchFailure).not.toHaveBeenCalled()
})
it('does not charge a lost response as a claimed migration failure', async () => {
const fetch = vi.fn<typeof globalThis.fetch>(async () => {
throw new Error('response lost')
})
const c = setup(fetch)
await vi.waitFor(() => expect(c.selectIdleRegionalRehomeCandidates).toHaveBeenCalledOnce())
await c.worker.run()
c.worker.stop()
expect(c.recordRegionalRehomeDispatchFailure).not.toHaveBeenCalled()
})
})
cloud/apps/relay/src/index.ts
+8
View File
@@ -2,6 +2,7 @@ import {
formatAssignmentInventorySnapshot,
readAssignmentInventorySnapshot
} from './assignment-inventory-snapshot.js'
import { readRegionCorrectionOutcomes } from './region-correction-outcomes.js'
import { RelayAssignmentStore } from './assignment-store.js'
import { loadRelayConfig } from './config.js'
import { startCellHeartbeat } from './cell-heartbeat-client.js'
@@ -71,6 +72,13 @@ const migrationInventoryTimer = roleOwnsAssignmentMaintenance(config.role)
void runRelayBackgroundOperation(async () => {
const inventory = await readRegisteredMigrationInventory(database, Date.now())
for (const line of formatRegisteredMigrationInventory(inventory)) console.warn(line)
console.log(
JSON.stringify({
event: 'orca_relay_region_correction_outcomes',
observedAt: Date.now(),
outcomes: await readRegionCorrectionOutcomes(database, Date.now())
})
)
}, '[orca-relay] migration inventory failed')
}, 5 * 60_000)
: null
cloud/apps/relay/src/region-correction-outcomes.ts
+29
View File
@@ -0,0 +1,29 @@
import type { RelayDatabase } from './database.js'
export async function readRegionCorrectionOutcomes(database: RelayDatabase, now: number) {
const rows = await database.query(
`SELECT attempt.source_cell_id, attempt.target_cell_id,
CASE WHEN attempt.aborted_at IS NOT NULL THEN 'aborted'
WHEN attempt.completed_at IS NOT NULL THEN 'completed'
WHEN migration.target_registered_at IS NOT NULL THEN 'registered' ELSE 'registering' END AS state,
COUNT(*) AS count,
COALESCE(MAX(CASE WHEN attempt.completed_at IS NULL AND attempt.aborted_at IS NULL
THEN ? - attempt.created_at ELSE 0 END), 0) AS oldest_open_ms,
COALESCE(SUM(CASE WHEN attempt.completed_at IS NULL AND attempt.aborted_at IS NULL
THEN migration.target_reserved_units ELSE 0 END), 0) AS target_reserved_units
FROM relay_region_rehome_attempts attempt
JOIN relay_assignment_migrations migration ON migration.user_id = attempt.user_id
AND migration.relay_host_id = attempt.relay_host_id AND migration.assignment_epoch = attempt.assignment_epoch
GROUP BY attempt.source_cell_id, attempt.target_cell_id, state
ORDER BY attempt.source_cell_id, attempt.target_cell_id, state`,
[now]
)
return rows.map((row) => ({
sourceCellId: String(row.source_cell_id),
targetCellId: String(row.target_cell_id),
state: String(row.state),
count: Number(row.count),
oldestOpenMs: Number(row.oldest_open_ms),
targetReservedUnits: Number(row.target_reserved_units)
}))
}
cloud/apps/relay/src/region-correction-preview.ts
+157
View File
@@ -0,0 +1,157 @@
import type { RelayDatabase, SqlRow } from './database.js'
import { REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS } from './database.js'
import {
REGIONAL_REHOME_CONCURRENT_LIMIT,
REGION_DECISION_TTL_MS
} from './region-correction-state.js'
export type RegionCorrectionPreview = {
observedAt: number
newClaimsEnabled: boolean
cohortPercent: number
openMigrations: number
availableMigrationSlots: number
globalSafetyFailure: string | null
counts: Record<string, number>
}
export async function previewRegionalRehomeEligibility(input: {
database: RelayDatabase
now: number
heartbeatTtlMs: number
cohortPercent: number
globalSafetyFailure: string | null
connectionHeadroom: ReadonlyMap<string, boolean>
cellIsClean: (safety: SqlRow | undefined, runtime: SqlRow, now: number) => boolean
}): Promise<RegionCorrectionPreview> {
const { database, now } = input
const [hosts, cells, runtimeRows, capabilityRows, safetyRows, controls, migrations] =
await Promise.all([
database.query(
`SELECT assignment.cell_id, assignment.assignment_epoch,
decision.generation, decision.assignment_epoch AS decision_epoch, decision.expires_at,
decision.incumbent_region, decision.preferred_region, decision.outcome, decision.policy_version,
decision.observed_at, decision.cohort_bucket,
(SELECT MAX(attempt.created_at) FROM relay_region_rehome_attempts attempt
WHERE attempt.user_id = assignment.user_id AND attempt.relay_host_id = assignment.relay_host_id) AS last_attempt_at,
(SELECT COUNT(*) FROM relay_assignment_migrations migration
WHERE migration.user_id = assignment.user_id AND migration.relay_host_id = assignment.relay_host_id
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL) AS open_migrations,
(SELECT COALESCE(SUM(lease.request_units),0) FROM relay_assignment_activity_leases lease
WHERE lease.user_id = assignment.user_id AND lease.relay_host_id = assignment.relay_host_id
AND lease.cell_id = assignment.cell_id) AS source_units,
(SELECT COUNT(*) FROM relay_control_capabilities host_capability
JOIN relay_assignment_activity_leases lease ON lease.user_id = host_capability.user_id
AND lease.relay_host_id = host_capability.relay_host_id AND lease.activity_id = host_capability.activity_id
JOIN relay_cell_runtime runtime ON runtime.cell_id = host_capability.cell_id
AND runtime.cell_incarnation = host_capability.cell_incarnation
WHERE host_capability.user_id = assignment.user_id AND host_capability.relay_host_id = assignment.relay_host_id
AND host_capability.cell_id = assignment.cell_id AND host_capability.assignment_epoch = assignment.assignment_epoch
AND host_capability.idle_regional_rehome = 1 AND lease.activity_kind = 'control'
AND lease.activity_id NOT LIKE 'control-pending:%' AND lease.expires_at > ?
AND lease.updated_at >= runtime.started_at) AS capable_controls
FROM relay_assignments assignment LEFT JOIN relay_region_decisions decision
ON decision.user_id = assignment.user_id AND decision.relay_host_id = assignment.relay_host_id`,
[now]
),
database.query(`SELECT cell.*, region.region, admission.admission_state FROM relay_cells cell
LEFT JOIN relay_cell_regions region ON region.cell_id = cell.cell_id
LEFT JOIN relay_cell_admission admission ON admission.cell_id = cell.cell_id`),
database.query(`SELECT * FROM relay_cell_runtime`),
database.query(`SELECT * FROM relay_cell_capabilities`),
database.query(`SELECT * FROM relay_cell_rehome_safety`),
database.query(`SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'`),
database.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE completed_at IS NULL AND aborted_at IS NULL`
)
])
const byCell = (rows: SqlRow[]) => new Map(rows.map((row) => [String(row.cell_id), row]))
const runtimes = byCell(runtimeRows)
const capabilities = byCell(capabilityRows)
const safety = byCell(safetyRows)
const inventory = byCell(cells)
const control = controls[0]
const cooldown = Number(control?.host_cooldown_ms ?? REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS)
const maxAge = Number(control?.preference_max_age_ms ?? REGION_DECISION_TTL_MS)
const openMigrations = Number(migrations[0]?.count ?? 0)
const counts: Record<string, number> = {}
const count = (reason: string) => {
counts[reason] = (counts[reason] ?? 0) + 1
}
const available = (cell: SqlRow): boolean => {
const id = String(cell.cell_id)
const runtime = runtimes.get(id)
const capability = capabilities.get(id)
return (
Number(cell.enabled) === 1 &&
cell.admission_state === 'general' &&
cell.region != null &&
runtime !== undefined &&
Number(runtime.ready) === 1 &&
Number(runtime.last_heartbeat_at) > now - input.heartbeatTtlMs &&
capability !== undefined &&
capability.cell_incarnation === runtime.cell_incarnation &&
Number(capability.regional_rehome_protocol) >= 3
)
}
for (const host of hosts) {
let reason: string | null = null
const source = inventory.get(String(host.cell_id))
if (host.generation == null) reason = 'no-verified-decision'
else if (Number(host.expires_at) <= now || Number(host.observed_at) < now - maxAge)
reason = 'expired'
else if (
Number(host.decision_epoch) !== Number(host.assignment_epoch) ||
host.incumbent_region !== source?.region
)
reason = 'basis-changed'
else if (
host.outcome !== 'conclusive' ||
Number(host.policy_version) !== 1 ||
host.preferred_region == null
)
reason = 'inconclusive-or-insufficient-improvement'
else if (Number(host.cohort_bucket) >= input.cohortPercent) reason = 'outside-cohort'
else if (Number(host.open_migrations) > 0) reason = 'migration-open'
else if (host.last_attempt_at != null && Number(host.last_attempt_at) > now - cooldown)
reason = 'host-cooldown'
else if (!source || !available(source)) reason = 'source-ineligible'
else if (Number(host.capable_controls) === 0) reason = 'source-control-unsupported-or-inactive'
else if (
!input.cellIsClean(safety.get(String(host.cell_id)), runtimes.get(String(host.cell_id))!, now)
)
reason = 'source-unclean'
if (reason) {
count(reason)
continue
}
const targets = cells.filter(
(cell) =>
cell.cell_id !== host.cell_id && cell.region === host.preferred_region && available(cell)
)
const clean = targets.filter((cell) =>
input.cellIsClean(safety.get(String(cell.cell_id)), runtimes.get(String(cell.cell_id))!, now)
)
const capacity = clean.filter(
(cell) =>
input.connectionHeadroom.get(String(cell.cell_id)) !== false &&
Number(cell.reserved_requests) + Number(host.source_units) + 1 <=
Number(cell.capacity_requests)
)
if (targets.length === 0) count('no-eligible-target')
else if (clean.length === 0) count('target-unclean')
else if (capacity.length === 0) count('no-target-headroom')
else if (input.globalSafetyFailure) count('global-safety-blocked')
else if (openMigrations >= REGIONAL_REHOME_CONCURRENT_LIMIT) count('concurrent-migration-cap')
else count(`eligible:${host.incumbent_region}-to-${host.preferred_region}`)
}
return {
observedAt: now,
newClaimsEnabled: Number(control?.enabled ?? 0) === 1,
cohortPercent: input.cohortPercent,
openMigrations,
availableMigrationSlots: Math.max(0, REGIONAL_REHOME_CONCURRENT_LIMIT - openMigrations),
globalSafetyFailure: input.globalSafetyFailure,
counts
}
}
cloud/apps/relay/src/region-correction-restart.test.ts
+119
View File
@@ -0,0 +1,119 @@
import { mkdtemp, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { openRelayDatabase, type RelayDatabase } from './database.js'
const identity = { userId: 'restart-test-user', relayHostId: 'abcdefghijklmnop' }
const paths: string[] = []
const databases = new Set<RelayDatabase>()
afterEach(async () => {
for (const database of databases) await database.close()
databases.clear()
for (const path of paths.splice(0)) await rm(path, { recursive: true, force: true })
})
async function setup() {
const dataDir = await mkdtemp(join(tmpdir(), 'relay-region-restart-'))
paths.push(dataDir)
let now = 1_000_000_000
const open = async () => {
const database = await openRelayDatabase({ dataDir })
databases.add(database)
return { database, store: new RelayAssignmentStore(database, () => now) }
}
const first = await open()
const cell = {
id: 'restart-us',
url: 'https://restart-us.example.test',
region: 'us-central1' as const,
capacityRequests: 100
}
await first.store.reconcileCells([cell])
await first.store.setCellEnabled(cell.id, true)
await first.store.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
cellIncarnation: '11111111-1111-4111-8111-111111111111',
region: cell.region,
startedAt: now - 1_000,
ready: true,
observedRequests: 0
})
const assignment = await first.store.assign(identity)
const issue = (store: RelayAssignmentStore) =>
store.exchangeRegionCorrection(identity, { v: 1, action: 'issue-window' }, assignment.assignmentEpoch)
const window = (await issue(first.store)).window!
const report = {
v: 1 as const,
action: 'report' as const,
generation: window.generation,
assignmentEpoch: window.assignmentEpoch,
policyVersion: 1 as const,
outcome: 'conclusive' as const,
measurements: { 'us-central1': 200, 'asia-east2': 40 }
}
const restart = async () => {
await first.database.close()
databases.delete(first.database)
return open()
}
return {
...first, window, report, issue, restart,
setNow: (value: number) => { now = value }
}
}
describe('persisted region decisions across director restart', () => {
it('keeps tombstones and fixed expiry, then invalidates the prior generation after restart', async () => {
const context = await setup()
const epoch = context.window.assignmentEpoch
await context.store.exchangeRegionCorrection(identity, {
v: 1, action: 'report', generation: context.window.generation,
assignmentEpoch: epoch, policyVersion: 1, outcome: 'inconclusive', reason: 'jitter'
}, epoch)
const restarted = await context.restart()
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, epoch))
.toMatchObject({ reportStatus: 'duplicate' })
const row = (await restarted.database.query('SELECT * FROM relay_region_decisions'))[0]!
expect(row.outcome).toBe('inconclusive')
expect(Number(row.expires_at)).toBe(context.window.expiresAt)
const successor = (await context.issue(restarted.store)).window!
expect(successor.generation).toBe(context.window.generation + 1)
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, epoch))
.toMatchObject({ reportStatus: 'stale' })
})
it('uses server expiry after a restart regardless of an old client report', async () => {
const context = await setup()
context.setNow(context.window.expiresAt)
const restarted = await context.restart()
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, context.window.assignmentEpoch))
.toMatchObject({ reportStatus: 'expired' })
expect(await restarted.store.previewRegionCorrection()).toEqual({ expired: 1 })
})
it('does not interpret a persisted future-policy window using the old policy after rollback', async () => {
const context = await setup()
await context.database.query('UPDATE relay_region_decisions SET policy_version = 2')
const restarted = await context.restart()
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, context.window.assignmentEpoch))
.toMatchObject({ reportStatus: 'stale' })
const row = (await restarted.database.query('SELECT * FROM relay_region_decisions'))[0]!
expect(row.outcome).toBe('pending')
expect(row.preferred_region).toBeNull()
expect(row.report_json).toBeNull()
})
it('keeps generation ordering when the server clock moves backwards across restart', async () => {
const context = await setup()
context.setNow(1_000_000_000 - 60_000)
const restarted = await context.restart()
const successor = (await context.issue(restarted.store)).window!
expect(successor.generation).toBe(context.window.generation + 1)
expect(successor.expiresAt).toBe(context.window.expiresAt - 60_000)
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, context.window.assignmentEpoch))
.toMatchObject({ reportStatus: 'stale' })
})
})
cloud/apps/relay/src/region-correction-state.ts
+158
View File
@@ -0,0 +1,158 @@
import { relayHostLogDigest } from './relay-host-log-digest.js'
import { createHash } from 'node:crypto'
import type {
RegionCorrectionRequest,
RegionCorrectionResponse,
RelayRegion
} from '@orca-cloud/relay-contract'
import type { RelayDatabase } from './database.js'
type Identity = { userId: string; relayHostId: string }
export const REGION_DECISION_TTL_MS = 24 * 60 * 60_000
export const REGIONAL_REHOME_CONCURRENT_LIMIT = 8
export async function exchangeRegionCorrection(
database: RelayDatabase,
identity: Identity,
request: RegionCorrectionRequest,
assignmentEpoch: number,
now: number
): Promise<RegionCorrectionResponse> {
const result: RegionCorrectionResponse = await database.transaction(async (transaction) => {
const assignment = (
await transaction.queryLocked(
`SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
)[0]
const region =
assignment &&
(
await transaction.query(`SELECT region FROM relay_cell_regions WHERE cell_id = ?`, [
assignment.cell_id
])
)[0]
if (!assignment || !region || Number(assignment.assignment_epoch) !== assignmentEpoch) {
return { v: 1, reportStatus: 'basis-changed' }
}
const prior = (
await transaction.queryLocked(
`SELECT * FROM relay_region_decisions WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
)[0]
if (request.action === 'issue-window') {
const generation = Number(prior?.generation ?? 0) + 1
if (!Number.isSafeInteger(generation)) throw new Error('region_generation_exhausted')
const expiresAt = now + REGION_DECISION_TTL_MS
const cohortBucket =
createHash('sha256')
.update(JSON.stringify([identity.userId, identity.relayHostId]))
.digest()
.readUInt32BE(0) % 100
await transaction.query(
`INSERT INTO relay_region_decisions
(user_id, relay_host_id, generation, expires_at, assignment_epoch, incumbent_region,
policy_version, outcome, preferred_region, observed_at, report_json, cohort_bucket)
VALUES (?, ?, ?, ?, ?, ?, 1, 'pending', NULL, ?, NULL, ?)
ON CONFLICT (user_id, relay_host_id) DO UPDATE SET
generation = excluded.generation, expires_at = excluded.expires_at,
assignment_epoch = excluded.assignment_epoch, incumbent_region = excluded.incumbent_region,
policy_version = 1, outcome = 'pending', preferred_region = NULL,
observed_at = excluded.observed_at, report_json = NULL, cohort_bucket = excluded.cohort_bucket`,
[
identity.userId,
identity.relayHostId,
generation,
expiresAt,
assignmentEpoch,
region.region,
now,
cohortBucket
]
)
return {
v: 1,
window: {
generation,
expiresAt,
assignmentEpoch,
incumbentRegion: region.region as RelayRegion,
policyVersion: 1
}
}
}
if (!prior || Number(prior.generation) !== request.generation)
return { v: 1, reportStatus: 'stale' }
if (Number(prior.policy_version) !== request.policyVersion)
return { v: 1, reportStatus: 'stale' }
if (Number(prior.expires_at) <= now) return { v: 1, reportStatus: 'expired' }
if (
request.assignmentEpoch !== assignmentEpoch ||
Number(prior.assignment_epoch) !== assignmentEpoch ||
prior.incumbent_region !== region.region
) {
return { v: 1, reportStatus: 'basis-changed' }
}
// The first report wins, including an inconclusive tombstone.
if (prior.outcome !== 'pending') return { v: 1, reportStatus: 'duplicate' }
let preferredRegion: RelayRegion | null = null
if (request.outcome === 'conclusive') {
const incumbent = request.measurements[region.region as RelayRegion]
const target: RelayRegion = region.region === 'us-central1' ? 'asia-east2' : 'us-central1'
const targetRtt = request.measurements[target]
if (incumbent - targetRtt >= 25 && targetRtt <= incumbent * 0.8) preferredRegion = target
}
await transaction.query(
`UPDATE relay_region_decisions SET outcome = ?, preferred_region = ?, report_json = ?
WHERE user_id = ? AND relay_host_id = ? AND generation = ?`,
[
request.outcome,
preferredRegion,
JSON.stringify(request),
identity.userId,
identity.relayHostId,
request.generation
]
)
return { v: 1, reportStatus: 'accepted' }
})
if (request.action === 'report' && result.reportStatus === 'accepted') {
const digest = relayHostLogDigest(identity.relayHostId)
// Stable sampling includes unchanged hosts for before/after comparisons.
if (Number.parseInt(digest.slice(0, 8), 16) % 10 === 0) {
console.log(
JSON.stringify({
event: 'orca_relay_region_comparison',
relayHostIdDigest: digest,
assignmentEpoch,
generation: request.generation,
policyVersion: request.policyVersion,
outcome: request.outcome,
...(request.outcome === 'conclusive' ? { measurements: request.measurements } : {})
})
)
}
}
return result
}
export async function previewRegionCorrection(
database: RelayDatabase,
now: number
): Promise<Record<string, number>> {
const rows = await database.query(
`SELECT CASE WHEN decision.expires_at <= ? THEN 'expired'
WHEN decision.assignment_epoch <> assignment.assignment_epoch THEN 'basis-changed'
WHEN decision.outcome = 'pending' THEN 'pending'
WHEN decision.preferred_region IS NULL THEN 'ineligible'
ELSE decision.incumbent_region || '-to-' || decision.preferred_region END AS reason,
COUNT(*) AS count
FROM relay_region_decisions decision
JOIN relay_assignments assignment ON assignment.user_id = decision.user_id
AND assignment.relay_host_id = decision.relay_host_id
GROUP BY reason`,
[now]
)
return Object.fromEntries(rows.map((row) => [String(row.reason), Number(row.count)]))
}
cloud/apps/relay/src/region-correction-store.test.ts
+359
View File
@@ -0,0 +1,359 @@
import { afterEach, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { openInMemoryRelayDatabase, openRelayDatabase, type RelayDatabase } from './database.js'
const identity = { userId: 'region-correction-test-user', relayHostId: 'abcdefghijklmnop' }
const cells = [
{
id: 'decision-us',
url: 'https://decision-us.example.test',
region: 'us-central1' as const,
capacityRequests: 100
},
{
id: 'decision-asia',
url: 'https://decision-asia.example.test',
region: 'asia-east2' as const,
capacityRequests: 100
}
]
const incarnations = [
'11111111-1111-4111-8111-111111111111',
'22222222-2222-4222-8222-222222222222'
]
const opened: RelayDatabase[] = []
afterEach(async () => {
for (const database of opened.splice(0)) {
if (database.dialect === 'postgres') await cleanupPostgres(database)
await database.close()
}
})
async function cleanupPostgres(database: RelayDatabase) {
for (const table of [
'relay_control_connection_reservations',
'relay_region_decisions',
'relay_control_capabilities',
'relay_assignment_activity_leases',
'relay_assignment_migrations',
'relay_assignment_migration_incarnations',
'relay_assignment_region_preferences',
'relay_region_rehome_attempts',
'relay_assignments'
]) {
await database.query(`DELETE FROM ${table} WHERE user_id = ?`, [identity.userId])
}
for (const table of [
'relay_cell_rehome_safety',
'relay_cell_capabilities',
'relay_cell_connection_snapshots',
'relay_cell_connection_runtime',
'relay_cell_runtime',
'relay_cell_connection_limits',
'relay_cell_admission',
'relay_cell_regions',
'relay_cells'
]) {
await database.query(
`DELETE FROM ${table} WHERE cell_id IN (?, ?)`,
cells.map((cell) => cell.id)
)
}
}
async function setup() {
const database =
process.env.ORCA_REGION_CORRECTION_POSTGRES === '1'
? await openRelayDatabase({
databaseUrl: requiredPostgresUrl(),
dataDir: '/tmp/orca-region-correction-unused'
})
: await openInMemoryRelayDatabase()
opened.push(database)
if (database.dialect === 'postgres') await cleanupPostgres(database)
let clock = 100_000_000
const store = new RelayAssignmentStore(database, () => clock, {
regionalRehomeCohortPercent: 100
})
await store.reconcileCells(cells)
for (const cell of cells) await store.setCellEnabled(cell.id, true)
for (const [index, cell] of cells.entries()) {
await store.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
region: cell.region,
cellIncarnation: incarnations[index]!,
startedAt: clock - 1_000,
ready: true,
observedRequests: 0
})
}
const assignment = await store.assign(identity, undefined, 'us-central1')
const activityId = await store.activateControl(identity, {
cellId: cells[0]!.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 7,
cellIncarnation: incarnations[0],
idleRegionalRehome: true
})
return {
database,
store,
assignment,
activityId,
now: () => clock,
advance: (ms: number) => {
clock += ms
}
}
}
function requiredPostgresUrl(): string {
const url = process.env.ORCA_RELAY_TEST_POSTGRES_URL
if (!url || new URL(url).port !== '55440')
throw new Error('PostgreSQL tests require configured port 55440')
return url
}
async function window(context: Awaited<ReturnType<typeof setup>>) {
const result = await context.store.exchangeRegionCorrection(
identity,
{ v: 1, action: 'issue-window' },
context.assignment.assignmentEpoch
)
return result.window!
}
async function regionalMigration(context: Awaited<ReturnType<typeof setup>>) {
const migration = await context.store.startEvacuation(identity, cells[1]!.id)
const attemptId = '33333333-3333-4333-8333-333333333333'
await context.database.query(
`INSERT INTO relay_region_rehome_attempts
(attempt_id,user_id,relay_host_id,preferred_region,source_cell_id,source_cell_incarnation,
target_cell_id,target_cell_incarnation,previous_epoch,assignment_epoch,drain_grace_ms,send_attempts,created_at,updated_at)
VALUES (?,?,?,'asia-east2',?,?,?,?,?,?,60000,1,?,?)`,
[
attemptId,
identity.userId,
identity.relayHostId,
cells[0]!.id,
incarnations[0],
cells[1]!.id,
incarnations[1],
migration.previousEpoch,
migration.assignmentEpoch,
context.now(),
context.now()
]
)
return { migration }
}
describe('ordered region decisions and migration outcomes', () => {
it('reports aggregate migration lifecycle and reservations without identity disclosure or writes', async () => {
const context = await setup()
const { migration } = await regionalMigration(context)
context.advance(1_000)
const before = await context.database.query('SELECT * FROM relay_region_rehome_attempts')
const outcomes = await context.store.regionCorrectionOutcomes()
expect(outcomes).toEqual([
expect.objectContaining({
sourceCellId: cells[0]!.id,
targetCellId: cells[1]!.id,
state: 'registering',
count: 1,
oldestOpenMs: 1_000
})
])
expect(outcomes[0]!.targetReservedUnits).toBeGreaterThan(0)
expect(JSON.stringify(outcomes)).not.toContain(identity.relayHostId)
expect(JSON.stringify(outcomes)).not.toContain(identity.userId)
expect(await context.database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual(
before
)
await context.store.activateControl(identity, {
cellId: cells[1]!.id,
assignmentEpoch: migration.assignmentEpoch,
generation: 1
})
await context.store.markMigrationTargetRegistered(identity, {
cellId: cells[1]!.id,
assignmentEpoch: migration.assignmentEpoch
})
expect(await context.store.regionCorrectionOutcomes()).toEqual([
expect.objectContaining({ state: 'registered' })
])
await context.store.releaseActivity(identity, context.activityId)
expect(await context.store.completeReadyRegionalRehomes()).toBe(1)
expect(await context.store.regionCorrectionOutcomes()).toEqual([
expect.objectContaining({
state: 'completed',
targetReservedUnits: 0,
oldestOpenMs: 0
})
])
})
it('supersedes prior windows and keeps an inconclusive tombstone immutable', async () => {
const context = await setup()
const first = await window(context)
const second = await window(context)
expect(second.generation).toBe(first.generation + 1)
const report = {
v: 1 as const,
action: 'report' as const,
assignmentEpoch: first.assignmentEpoch,
policyVersion: 1 as const,
outcome: 'conclusive' as const,
measurements: { 'us-central1': 200, 'asia-east2': 40 }
}
expect(
await context.store.exchangeRegionCorrection(
identity,
{ ...report, generation: first.generation },
first.assignmentEpoch
)
).toMatchObject({ reportStatus: 'stale' })
expect(
await context.store.exchangeRegionCorrection(
identity,
{ ...report, generation: second.generation, outcome: 'inconclusive', reason: 'jitter' },
second.assignmentEpoch
)
).toMatchObject({ reportStatus: 'accepted' })
expect(
await context.store.exchangeRegionCorrection(
identity,
{ ...report, generation: second.generation },
second.assignmentEpoch
)
).toMatchObject({ reportStatus: 'duplicate' })
expect(await context.store.previewRegionCorrection()).toEqual({ ineligible: 1 })
})
it('previews the uncapped fleet without writes, claims, or locked reads', async () => {
const context = await setup()
const query = context.database.query.bind(context.database)
const transaction = context.database.transaction.bind(context.database)
const queryLocked = context.database.queryLocked.bind(context.database)
context.database.query = async (sql, params) => {
expect(sql.trim()).toMatch(/^(SELECT|WITH)/i)
return query(sql, params)
}
context.database.transaction = async () => {
throw new Error('preview_must_not_open_mutating_transaction')
}
context.database.queryLocked = async () => {
throw new Error('preview_must_not_lock')
}
try {
const preview = await context.store.previewRegionalRehomeEligibility()
expect(preview.counts['no-verified-decision']).toBeGreaterThanOrEqual(1)
expect(preview.globalSafetyFailure).toBe('process-safety-unavailable')
expect(JSON.stringify(preview)).not.toContain(identity.relayHostId)
expect(JSON.stringify(preview)).not.toContain(identity.userId)
} finally {
context.database.query = query
context.database.transaction = transaction
context.database.queryLocked = queryLocked
}
})
it('allocates distinct ordered generations for concurrent window issuers', async () => {
const context = await setup()
const replies = await Promise.all([window(context), window(context), window(context)])
expect(replies.map((reply) => reply.generation).sort((a, b) => a - b)).toEqual([1, 2, 3])
const older = replies.find((reply) => reply.generation === 2)!
expect(
await context.store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: older.generation,
assignmentEpoch: older.assignmentEpoch,
policyVersion: 1,
outcome: 'inconclusive',
reason: 'delayed'
},
older.assignmentEpoch
)
).toMatchObject({ reportStatus: 'stale' })
})
it('compares with assigned region, preserves hints, and never extends a window on report', async () => {
const context = await setup()
await context.store.assign(identity, 'asia-east2')
const issued = await window(context)
expect(issued.incumbentRegion).toBe('us-central1')
context.advance(50)
await context.store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: issued.generation,
assignmentEpoch: issued.assignmentEpoch,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 110, 'asia-east2': 90 }
},
issued.assignmentEpoch
)
expect(await context.store.previewRegionCorrection()).toEqual({ ineligible: 1 })
const row = (await context.database.query(`SELECT * FROM relay_region_decisions`))[0]!
expect(Number(row.expires_at)).toBe(issued.expiresAt)
const hint = (
await context.database.query(
`SELECT preferred_region FROM relay_assignment_region_preferences WHERE user_id = ?`,
[identity.userId]
)
)[0]
expect(hint?.preferred_region).toBe('asia-east2')
context.advance(24 * 60 * 60_000)
expect(
await context.store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: issued.generation,
assignmentEpoch: issued.assignmentEpoch,
policyVersion: 1,
outcome: 'inconclusive',
reason: 'late'
},
issued.assignmentEpoch
)
).toMatchObject({ reportStatus: 'expired' })
})
it('rejects stale assignment basis and requires both thresholds', async () => {
const context = await setup()
const issued = await window(context)
await context.store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: issued.generation,
assignmentEpoch: issued.assignmentEpoch,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 150, 'asia-east2': 100 }
},
issued.assignmentEpoch
)
expect(await context.store.previewRegionCorrection()).toEqual({
'us-central1-to-asia-east2': 1
})
await context.store.startEvacuation(identity, cells[1]!.id)
expect(
await context.store.exchangeRegionCorrection(
identity,
{ v: 1, action: 'issue-window' },
issued.assignmentEpoch
)
).toMatchObject({ reportStatus: 'basis-changed' })
})
})
cloud/apps/relay/src/regional-host-drain-app.test.ts
+207 -94
View File
@@ -5,7 +5,9 @@ vi.mock('./admin-token-verifier.js', () => ({
createAdminTokenVerifier: () => async (token: string, route?: string) =>
token === 'deploy-token' ||
(token === 'monitor-token' &&
(!route || route === '/v1/admin/regional-rehome-control')),
(!route ||
route === '/v1/admin/regional-rehome-control' ||
route === '/v1/admin/regional-rehome-preview')),
createReadOnlyAdminTokenVerifier: () => async () => false,
createRegionalRehomeControlApplyTokenVerifier: () => async (token: string) =>
token === 'deploy-token',
@@ -41,7 +43,83 @@ const request = {
graceMs: 60_000
}
describe('idle regional cutover endpoint', () => {
it('authenticates and fences the source before invoking a cutover', async () => {
const idleRehome = vi.fn(async () => ({ outcome: 'busy' }))
const app = createRelayApp(config(), {
store: {} as never,
assignments: {} as never,
drain: vi.fn(),
idleRehome,
cellIncarnation,
ready: vi.fn(async () => true)
} as Parameters<typeof createRelayApp>[1])
const input = {
v: 1,
attemptId: request.attemptId,
userId: request.userId,
relayHostId: request.relayHostId,
sourceCellId: request.sourceCellId,
sourceCellIncarnation: cellIncarnation,
sourceAssignmentEpoch: 7,
sourceGeneration: 1,
targetCellId: 'target-cell',
cohortPercent: 100,
directorSafety: {
observedAt: 100, sqlFailures: 0, reconnects: 0, controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0, databasePoolWaitersMax: 0, databasePoolWaitMsMax: 0
}
}
const path = '/v1/admin/host-idle-rehome'
expect((await postPath(app, path, 'runtime-token', input)).status).toBe(401)
expect(
(
await postPath(app, path, 'rehome-token', {
...input,
sourceCellIncarnation: '33333333-3333-4333-8333-333333333333'
})
).status
).toBe(409)
expect(idleRehome).not.toHaveBeenCalled()
const response = await postPath(app, path, 'rehome-token', input)
expect(response.status).toBe(200)
expect(await response.json()).toEqual({ v: 1, outcome: 'busy' })
expect(idleRehome).toHaveBeenCalledExactlyOnceWith(input)
})
})
describe('regional host drain endpoint', () => {
it('exposes aggregate preview to monitors without a mutation path', async () => {
const preview = { counts: { 'eligible:asia-east2-to-us-central1': 2 } }
const safety = { observedAt: 100 }
const previewRegionalRehomeEligibility = vi.fn(async () => preview)
const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
store: {} as never,
assignments: {
previewRegionalRehomeEligibility,
regionCorrectionOutcomes: async () => []
} as never,
regionalRehomeSafetySnapshot: () => safety as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
const path = '/v1/admin/regional-rehome-preview'
expect((await app.request(path)).status).toBe(401)
expect(previewRegionalRehomeEligibility).not.toHaveBeenCalled()
const response = await app.request(path, { headers: { authorization: 'Bearer monitor-token' } })
expect(response.status).toBe(200)
expect(await response.json()).toEqual({ v: 1, preview, outcomes: [] })
expect(previewRegionalRehomeEligibility).toHaveBeenCalledExactlyOnceWith(safety)
expect(
(
await app.request(path, {
method: 'POST',
headers: { authorization: 'Bearer deploy-token' }
})
).status
).toBe(404)
})
it('accepts only the dedicated identity and exact cell generation', async () => {
const drainHost = vi.fn(() => 'accepted' as const)
const app = createRelayApp(config(), {
@@ -60,14 +138,66 @@ describe('regional host drain endpoint', () => {
expect((await post(app, 'deploy-token', request)).status).toBe(401)
expect(
(await post(app, 'rehome-token', {
...request,
sourceCellIncarnation: '33333333-3333-4333-8333-333333333333'
})).status
(
await post(app, 'rehome-token', {
...request,
sourceCellIncarnation: '33333333-3333-4333-8333-333333333333'
})
).status
).toBe(409)
expect(drainHost).toHaveBeenCalledOnce()
})
it('waits for an asynchronous drain operation before acknowledging', async () => {
let grant!: (value: 'accepted') => void
let entered!: () => void
const started = new Promise<void>((resolve) => {
entered = resolve
})
const drainHost = vi.fn(() => {
entered()
return new Promise<'accepted'>((resolve) => {
grant = resolve
})
})
const app = createRelayApp(config(), {
store: {} as never,
assignments: {} as never,
drain: vi.fn(),
drainHost,
cellIncarnation,
ready: vi.fn(async () => true)
})
const pending = post(app, 'rehome-token', request)
let acknowledged = false
void pending.then(() => {
acknowledged = true
})
await started
await Promise.resolve()
expect(acknowledged).toBe(false)
grant('accepted')
const response = await pending
expect(response.status).toBe(200)
expect(await response.json()).toEqual({ v: 1, outcome: 'accepted' })
})
it('rejects a failed asynchronous drain instead of acknowledging it', async () => {
const app = createRelayApp(config(), {
store: {} as never,
assignments: {} as never,
drain: vi.fn(),
drainHost: async () => {
throw new Error('activity_cell_not_authoritative')
},
cellIncarnation,
ready: vi.fn(async () => true)
})
const response = await post(app, 'rehome-token', request)
expect(response.status).toBe(409)
expect(await response.json()).toEqual({ error: 'activity_cell_not_authoritative' })
})
it('rejects malformed identities before touching the session registry', async () => {
const drainHost = vi.fn(() => 'accepted' as const)
const app = createRelayApp(config(), {
@@ -224,7 +354,7 @@ describe('regional rehome director controls', () => {
v: 1,
cellId: 'production-gce-c7',
cellIncarnation,
regionalRehomeProtocol: 1,
regionalRehomeProtocol: 2,
safety: {
observedAt: 100,
sqlFailures: 0,
@@ -235,12 +365,7 @@ describe('regional rehome director controls', () => {
databasePoolWaitMsMax: 0
}
}
const response = await postPath(
app,
'/v1/admin/cell-rehome-status',
'runtime-token',
body
)
const response = await postPath(app, '/v1/admin/cell-rehome-status', 'runtime-token', body)
expect(response.status).toBe(200)
expect(recordCellRegionalRehomeStatus).toHaveBeenCalledWith(body)
@@ -266,18 +391,13 @@ describe('regional rehome director controls', () => {
v: 1,
cellId: 'production-gce-c7',
cellIncarnation,
regionalRehomeProtocol: 1,
regionalRehomeProtocol: 2,
safety: {
...observability.regionalRehomeRuntimeSafety(),
...emptyPostgresPoolPressureCounts()
}
}
const response = await postPath(
app,
'/v1/admin/cell-rehome-status',
'runtime-token',
body
)
const response = await postPath(app, '/v1/admin/cell-rehome-status', 'runtime-token', body)
expect(response.status).toBe(200)
expect(recordCellRegionalRehomeStatus).toHaveBeenCalledWith(body)
@@ -301,12 +421,14 @@ describe('regional rehome director controls', () => {
drain: vi.fn(),
ready: vi.fn(async () => true)
})
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'deploy-token',
{ v: 1, action: 'inspect' }
)).status).toBe(200)
expect(
(
await postPath(app, '/v1/admin/regional-rehome-control', 'deploy-token', {
v: 1,
action: 'inspect'
})
).status
).toBe(200)
const apply = {
v: 1,
action: 'apply',
@@ -319,39 +441,35 @@ describe('regional rehome director controls', () => {
drainGraceMs: 60_000,
confirmation: 'ENABLE_REGIONAL_REHOMING'
}
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'deploy-token',
apply
)).status).toBe(200)
expect(
(await postPath(app, '/v1/admin/regional-rehome-control', 'deploy-token', apply)).status
).toBe(200)
expect(applyRegionalRehomeControl).toHaveBeenCalledOnce()
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'monitor-token',
{ v: 1, action: 'inspect' }
)).status).toBe(200)
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'monitor-token',
apply
)).status).toBe(403)
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'deploy-token',
{ ...apply, confirmation: 'DISABLE_REGIONAL_REHOMING' }
)).status).toBe(400)
expect(
(
await postPath(app, '/v1/admin/regional-rehome-control', 'monitor-token', {
v: 1,
action: 'inspect'
})
).status
).toBe(200)
expect(
(await postPath(app, '/v1/admin/regional-rehome-control', 'monitor-token', apply)).status
).toBe(403)
expect(
(
await postPath(app, '/v1/admin/regional-rehome-control', 'deploy-token', {
...apply,
confirmation: 'DISABLE_REGIONAL_REHOMING'
})
).status
).toBe(400)
// The per-host cooldown is part of the durable shape an operator must state.
const { hostCooldownMs: _omitted, ...withoutCooldown } = apply
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'deploy-token',
withoutCooldown
)).status).toBe(400)
expect(
(await postPath(app, '/v1/admin/regional-rehome-control', 'deploy-token', withoutCooldown))
.status
).toBe(400)
})
it('probes dedicated trust twice and returns only aggregate proof', async () => {
@@ -382,12 +500,11 @@ describe('regional rehome director controls', () => {
}) as typeof fetch,
ready: vi.fn(async () => true)
})
const response = await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ v: 1, sourceCellId: 'production-gce-c7', sourceCellIncarnation: cellIncarnation }
)
const response = await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', {
v: 1,
sourceCellId: 'production-gce-c7',
sourceCellIncarnation: cellIncarnation
})
expect(response.status).toBe(200)
const responseBody = await response.json()
@@ -448,12 +565,11 @@ describe('regional rehome director controls', () => {
ready: vi.fn(async () => true)
})
const response = await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ v: 1, sourceCellId: 'production-gce-c27', sourceCellIncarnation: cellIncarnation }
)
const response = await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', {
v: 1,
sourceCellId: 'production-gce-c27',
sourceCellIncarnation: cellIncarnation
})
expect(response.status).toBe(200)
expect(await response.json()).toMatchObject({ proven: true })
@@ -481,12 +597,11 @@ describe('regional rehome director controls', () => {
ready: vi.fn(async () => true)
})
const response = await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ v: 1, sourceCellId: 'production-gce-c27', sourceCellIncarnation: cellIncarnation }
)
const response = await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', {
v: 1,
sourceCellId: 'production-gce-c27',
sourceCellIncarnation: cellIncarnation
})
expect(response.status).toBe(409)
expect(sourceFetch).not.toHaveBeenCalled()
@@ -504,18 +619,17 @@ describe('regional rehome director controls', () => {
sourceCellId: 'production-gce-c7',
sourceCellIncarnation: cellIncarnation
}
expect((await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'monitor-token',
body
)).status).toBe(401)
expect((await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ ...body, unexpected: true }
)).status).toBe(400)
expect(
(await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'monitor-token', body)).status
).toBe(401)
expect(
(
await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', {
...body,
unexpected: true
})
).status
).toBe(400)
})
it('fails closed when the source rejects the dedicated identity', async () => {
@@ -529,9 +643,9 @@ describe('regional rehome director controls', () => {
regionalRehomeProtocol: 1
}
})
const sourceFetch = vi.fn<typeof fetch>().mockResolvedValue(
Response.json({ error: 'invalid_token' }, { status: 401 })
)
const sourceFetch = vi
.fn<typeof fetch>()
.mockResolvedValue(Response.json({ error: 'invalid_token' }, { status: 401 }))
const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
store: {} as never,
assignments: { cellDeploymentStatus } as never,
@@ -540,12 +654,11 @@ describe('regional rehome director controls', () => {
regionalRehomeFetch: sourceFetch,
ready: vi.fn(async () => true)
})
const response = await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ v: 1, sourceCellId: 'production-gce-c7', sourceCellIncarnation: cellIncarnation }
)
const response = await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', {
v: 1,
sourceCellId: 'production-gce-c7',
sourceCellIncarnation: cellIncarnation
})
expect(response.status).toBe(409)
expect(sourceFetch).toHaveBeenCalledOnce()
cloud/apps/relay/src/regional-rehome-postgres.test.ts
+260 -233
View File
@@ -29,6 +29,9 @@ describePostgres('PostgreSQL regional rehoming', () => {
})
async function cleanup(): Promise<void> {
for (const table of ['relay_region_decisions', 'relay_control_capabilities']) {
await primary.query(`DELETE FROM ${table} WHERE user_id LIKE 'pg-rehome-user-%'`)
}
await primary.query(
`DELETE FROM relay_region_rehome_attempts WHERE user_id LIKE 'pg-rehome-user-%'`
)
@@ -69,6 +72,81 @@ describePostgres('PostgreSQL regional rehoming', () => {
}
}
it('defaults to a closed correction cohort even with enabled durable control', async () => {
const context = await fixture()
const closed = new RelayAssignmentStore(primary, context.now, {
requireLiveCells: true,
heartbeatTtlMs: 45_000
})
expect(await cutover(closed, context.now())).toBeNull()
const preview = await closed.previewRegionalRehomeEligibility({
observedAt: context.now(),
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
})
expect(preview.cohortPercent).toBe(0)
expect(preview.counts['outside-cohort']).toBe(1)
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('counts existing generic migrations against the optimization cap and preview', async () => {
const context = await fixture()
const safety = {
observedAt: context.now(),
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
}
const before = await context.store.previewRegionalRehomeEligibility(safety)
expect(before.counts['eligible:us-central1-to-asia-east2']).toBe(1)
for (let index = 0; index < 8; index++) {
const identity = {
userId: `pg-rehome-user-budget-${sequence}-${index}`,
relayHostId: `budgethost${String(index).padStart(6, '0')}`
}
await context.store.assign(identity, undefined, 'us-central1')
await context.store.startEvacuation(identity, context.target.id)
}
const preview = await context.store.previewRegionalRehomeEligibility(safety)
expect(preview.openMigrations).toBe(8)
expect(preview.availableMigrationSlots).toBe(0)
expect(preview.counts['concurrent-migration-cap']).toBe(1)
expect(await cutover(context.store, context.now())).toBeNull()
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('preview excludes request capacity exhaustion before a claim', async () => {
const context = await fixture()
await primary.query(
`UPDATE relay_cells SET capacity_requests = reserved_requests + 1 WHERE cell_id = ?`,
[context.target.id]
)
const preview = await context.store.previewRegionalRehomeEligibility({
observedAt: context.now(),
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
})
expect(preview.counts['no-target-headroom']).toBe(1)
expect(await cutover(context.store, context.now())).toBeNull()
})
it('claims through ambient per-cell sql retry noise', async () => {
const context = await fixture()
await primary.query(
@@ -78,27 +156,31 @@ describePostgres('PostgreSQL regional rehoming', () => {
[context.source.id, context.target.id]
)
expect(await context.store.claimRegionalRehome()).not.toBeNull()
expect(await cutover(context.store, context.now())).not.toBeNull()
})
it('moves a us-central1 host onto a cell in its preferred asia-east2 region', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
const attempt = await cutover(context.store, context.now())
expect(attempt).toMatchObject({
preferredRegion: 'asia-east2',
sourceCellId: context.source.id,
targetCellId: context.target.id
})
expect(await primary.query(
`SELECT preferred_region, source_cell_id, target_cell_id
expect(
await primary.query(
`SELECT preferred_region, source_cell_id, target_cell_id
FROM relay_region_rehome_attempts WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{
preferred_region: 'asia-east2',
source_cell_id: context.source.id,
target_cell_id: context.target.id
}])
[context.identity.userId]
)
).toEqual([
{
preferred_region: 'asia-east2',
source_cell_id: context.source.id,
target_cell_id: context.target.id
}
])
})
it('moves an asia-east2 host back onto a cell in its preferred us-central1 region', async () => {
@@ -107,32 +189,37 @@ describePostgres('PostgreSQL regional rehoming', () => {
targetRegion: 'us-central1'
})
const attempt = await context.store.claimRegionalRehome()
const attempt = await cutover(context.store, context.now())
expect(attempt).toMatchObject({
preferredRegion: 'us-central1',
sourceCellId: context.source.id,
targetCellId: context.target.id
})
// The durable attempt row must accept the reverse direction too.
expect(await primary.query(
`SELECT preferred_region, source_cell_id, target_cell_id
expect(
await primary.query(
`SELECT preferred_region, source_cell_id, target_cell_id
FROM relay_region_rehome_attempts WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{
preferred_region: 'us-central1',
source_cell_id: context.source.id,
target_cell_id: context.target.id
}])
expect(await primary.query(
`SELECT cell_id FROM relay_assignments WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ cell_id: context.target.id }])
[context.identity.userId]
)
).toEqual([
{
preferred_region: 'us-central1',
source_cell_id: context.source.id,
target_cell_id: context.target.id
}
])
expect(
await primary.query(`SELECT cell_id FROM relay_assignments WHERE user_id = ?`, [
context.identity.userId
])
).toEqual([{ cell_id: context.target.id }])
})
it('leaves a host whose preference already matches its own region', async () => {
const context = await fixture({ preferredRegion: 'us-central1' })
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(cutover(context.store, context.now())).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true
@@ -146,16 +233,12 @@ describePostgres('PostgreSQL regional rehoming', () => {
it('leaves a host whose preference is older than the configured max age', async () => {
const context = await fixture()
await primary.query(
`UPDATE relay_assignment_region_preferences SET observed_at = ?
`UPDATE relay_region_decisions SET observed_at = ?
WHERE user_id = ? AND relay_host_id = ?`,
[
context.now() - 24 * 60 * 60_000 - 1,
context.identity.userId,
context.identity.relayHostId
]
[context.now() - 24 * 60 * 60_000 - 1, context.identity.userId, context.identity.relayHostId]
)
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(cutover(context.store, context.now())).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true
@@ -190,7 +273,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
]
)
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(cutover(context.store, context.now())).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true,
@@ -206,7 +289,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
`UPDATE relay_region_rehome_attempts SET created_at = ? WHERE user_id = ?`,
[context.now() - 3 * 24 * 60 * 60_000, context.identity.userId]
)
await expect(context.store.claimRegionalRehome()).resolves.toMatchObject({
await expect(cutover(context.store, context.now())).resolves.toMatchObject({
sourceCellId: context.source.id,
targetCellId: context.target.id
})
@@ -217,7 +300,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
// where no later rehome could move it out again.
const context = await fixture({ targetProtocol: 0 })
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(cutover(context.store, context.now())).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true
@@ -237,119 +320,39 @@ describePostgres('PostgreSQL regional rehoming', () => {
[context.target.id]
)
expect(await context.store.claimRegionalRehome()).toBeNull()
expect(await cutover(context.store, context.now())).toBeNull()
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
generation: 1,
enabled: true
})
expect(await primary.query(
`SELECT next_dispatch_at FROM relay_region_rehome_worker_state`
)).toEqual([{ next_dispatch_at: String(context.now() + 6_000) }])
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('lets only one director claim a host', async () => {
const context = await fixture()
const claims = await Promise.all([
context.store.claimRegionalRehome(),
context.competingStore.claimRegionalRehome()
cutover(context.store, context.now()),
cutover(context.competingStore, context.now())
])
expect(claims.filter(Boolean)).toHaveLength(1)
expect(await primary.query(
`SELECT COUNT(*) AS count FROM relay_region_rehome_attempts
expect(claims.filter(Boolean).length).toBeGreaterThanOrEqual(1)
expect(
await primary.query(
`SELECT COUNT(*) AS count FROM relay_region_rehome_attempts
WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ count: '1' }])
expect(await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations
[context.identity.userId]
)
).toEqual([{ count: '1' }])
expect(
await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations
WHERE user_id = ? AND completed_at IS NULL AND aborted_at IS NULL`,
[context.identity.userId]
)).toEqual([{ count: '1' }])
})
it('serializes an enable with a budget-exhausting failure without retries', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
await context.store.recordRegionalRehomeDispatchFailure(attempt!.attemptId)
await context.store.recordRegionalRehomeDispatchFailure(attempt!.attemptId)
const locked = Promise.withResolvers<void>()
const release = Promise.withResolvers<void>()
const primaryTransaction = primary.transaction.bind(primary)
const secondaryTransaction = secondary.transaction.bind(secondary)
let enableTransactions = 0
let failureTransactions = 0
let enablePid = 0
let failurePid = 0
const enableSpy = vi.spyOn(primary, 'transaction').mockImplementation((operation, options) =>
primaryTransaction(async (transaction) => {
enableTransactions++
enablePid = Number((await transaction.query('SELECT pg_backend_pid() AS pid'))[0]!.pid)
return await operation({
dialect: 'postgres',
query: transaction.query.bind(transaction),
queryLocked: async (sql, params, lockOptions) => {
const rows = await transaction.queryLocked(sql, params, lockOptions)
if (sql.includes('FROM relay_region_rehome_control')) {
locked.resolve()
await release.promise
}
return rows
},
transaction: transaction.transaction.bind(transaction),
close: transaction.close.bind(transaction)
})
}, options)
)
const failureSpy = vi.spyOn(secondary, 'transaction').mockImplementation((operation, options) =>
secondaryTransaction(async (transaction) => {
failureTransactions++
failurePid = Number((await transaction.query('SELECT pg_backend_pid() AS pid'))[0]!.pid)
return await operation(transaction)
}, options)
)
const enable = context.store.applyRegionalRehomeControl({
expectedGeneration: 1,
enabled: true,
notBefore: context.now(),
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: 7 * 24 * 60 * 60_000,
drainGraceMs: 60_000
})
let failure: Promise<void> | undefined
let outcomes: PromiseSettledResult<unknown>[] = []
try {
await Promise.race([
locked.promise,
enable.then(() => {
throw new Error('enable completed before the control lock')
})
])
failure = context.competingStore.recordRegionalRehomeDispatchFailure(attempt!.attemptId)
// Observe the actual PostgreSQL wait before letting enable acquire the worker row.
await vi.waitFor(async () => {
expect(failurePid).not.toBe(0)
const rows = await primary.query('SELECT pg_blocking_pids(?) AS blockers', [failurePid])
expect(rows[0]!.blockers).toContain(enablePid)
}, { interval: 10, timeout: 800 })
} finally {
release.resolve()
outcomes = await Promise.allSettled([enable, ...(failure ? [failure] : [])])
enableSpy.mockRestore()
failureSpy.mockRestore()
}
expect(outcomes.map((outcome) => outcome.status)).toEqual(['fulfilled', 'fulfilled'])
expect({ enableTransactions, failureTransactions }).toEqual({
enableTransactions: 1,
failureTransactions: 1
})
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
generation: 2,
enabled: true
})
expect(await primary.query(
`SELECT consecutive_failures, paused_until FROM relay_region_rehome_worker_state`
)).toEqual([{ consecutive_failures: '1', paused_until: '0' }])
[context.identity.userId]
)
).toEqual([{ count: '1' }])
})
it('increments the disable generation once across competing directors', async () => {
@@ -366,24 +369,6 @@ describePostgres('PostgreSQL regional rehoming', () => {
})
})
it('records one receipt across competing directors', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
const receipts = await Promise.all([
context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted'),
context.competingStore.recordRegionalRehomeDrainReceipt(
attempt!.attemptId,
'accepted'
)
])
expect(receipts.sort()).toEqual([false, true])
expect(await primary.query(
`SELECT drain_outcome FROM relay_region_rehome_attempts WHERE attempt_id = ?`,
[attempt!.attemptId]
)).toEqual([{ drain_outcome: 'accepted' }])
})
it('rechecks a preference changed while the assignment row is locked', async () => {
const context = await fixture()
let unlock!: () => void
@@ -399,9 +384,9 @@ describePostgres('PostgreSQL regional rehoming', () => {
await unlockPromise
})
await lockedPromise
const claim = context.store.claimRegionalRehome()
const claim = cutover(context.store, context.now())
await primary.query(
`UPDATE relay_assignment_region_preferences SET preferred_region = 'us-central1',
`UPDATE relay_region_decisions SET preferred_region = 'us-central1',
observed_at = ? WHERE user_id = ? AND relay_host_id = ?`,
[context.now(), context.identity.userId, context.identity.relayHostId]
)
@@ -409,23 +394,26 @@ describePostgres('PostgreSQL regional rehoming', () => {
await held
await expect(claim).resolves.toBeNull()
expect(await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ count: '0' }])
expect(
await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ count: '0' }])
})
it('rechecks fleet safety under locks before mutating a candidate', async () => {
const context = await fixture()
const [request] = await context.store.selectIdleRegionalRehomeCandidates(safety(context.now()))
expect(request).toBeDefined()
let unlock!: () => void
let locked!: () => void
const lockedPromise = new Promise<void>((resolve) => (locked = resolve))
const unlockPromise = new Promise<void>((resolve) => (unlock = resolve))
const held = secondary.transaction(async (transaction) => {
await transaction.queryLocked(
`SELECT * FROM relay_cell_rehome_safety WHERE cell_id = ?`,
[context.target.id]
)
await transaction.queryLocked(`SELECT * FROM relay_cell_rehome_safety WHERE cell_id = ?`, [
context.target.id
])
await transaction.query(
`UPDATE relay_cell_rehome_safety SET sql_failures = ${REGIONAL_REHOME_SQL_FAILURES_LIMIT + 1} WHERE cell_id = ?`,
[context.target.id]
@@ -434,62 +422,50 @@ describePostgres('PostgreSQL regional rehoming', () => {
await unlockPromise
})
await lockedPromise
const claim = context.store.claimRegionalRehome()
const claim = context.store.commitIdleRegionalRehome(request!, safety(context.now()))
unlock()
await held
await expect(claim).resolves.toBeNull()
await expect(claim).resolves.toEqual({ outcome: 'deferred' })
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
generation: 2,
enabled: false
})
expect(await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ count: '0' }])
expect(
await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ count: '0' }])
})
it('pauses when one required cell exceeds the reconnect limit', async () => {
const context = await fixture()
await primary.query(
`UPDATE relay_cell_rehome_safety SET reconnects = ? WHERE cell_id = ?`,
[REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT + 1, context.source.id]
)
const [request] = await context.store.selectIdleRegionalRehomeCandidates(safety(context.now()))
expect(request).toBeDefined()
await primary.query(`UPDATE relay_cell_rehome_safety SET reconnects = ? WHERE cell_id = ?`, [
REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT + 1,
context.source.id
])
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(
context.store.commitIdleRegionalRehome(request!, safety(context.now()))
).resolves.toEqual({ outcome: 'deferred' })
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 2,
enabled: false
})
expect(await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ count: '0' }])
})
it('does not retry a drain against a replacement source incarnation', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
context.advance(31_000)
await heartbeat(
context.store,
context.source,
'33333333-3333-4333-8333-333333333333',
1,
context.now()
)
await expect(context.competingStore.claimRegionalRehome()).resolves.toBeNull()
expect(await primary.query(
`SELECT send_attempts FROM relay_region_rehome_attempts WHERE attempt_id = ?`,
[attempt!.attemptId]
)).toEqual([{ send_attempts: '1' }])
expect(
await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ count: '0' }])
})
it('makes concurrent completion and expiry cleanup idempotent', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted')
const attempt = await cutover(context.store, context.now())
const targetControl = await context.store.activateControl(context.identity, {
cellId: context.target.id,
assignmentEpoch: attempt!.assignmentEpoch,
@@ -528,17 +504,18 @@ describePostgres('PostgreSQL regional rehoming', () => {
context.competingStore.abortExpiredRegionalRehomes()
])
expect(outcomes).toEqual(expect.arrayContaining([0, 1]))
expect(await primary.query(
`SELECT completed_at IS NOT NULL AS completed, aborted_at IS NOT NULL AS aborted
expect(
await primary.query(
`SELECT completed_at IS NOT NULL AS completed, aborted_at IS NOT NULL AS aborted
FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ completed: true, aborted: false }])
[context.identity.userId]
)
).toEqual([{ completed: true, aborted: false }])
})
it('will not complete against a replacement target incarnation', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted')
const attempt = await cutover(context.store, context.now())
await context.store.activateControl(context.identity, {
cellId: context.target.id,
assignmentEpoch: attempt!.assignmentEpoch,
@@ -559,15 +536,17 @@ describePostgres('PostgreSQL regional rehoming', () => {
)
await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(0)
expect(await primary.query(
`SELECT completed_at, aborted_at FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ completed_at: null, aborted_at: null }])
expect(
await primary.query(
`SELECT completed_at, aborted_at FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ completed_at: null, aborted_at: null }])
})
it('does not roll an unregistered target back to a stale regional source', async () => {
const context = await fixture()
await context.store.claimRegionalRehome()
await cutover(context.store, context.now())
context.advance(6 * 60_000)
await heartbeat(
context.store,
@@ -580,16 +559,17 @@ describePostgres('PostgreSQL regional rehoming', () => {
await expect(context.store.refreshRegionalRehomeLeases()).resolves.toBe(0)
await expect(context.store.abortExpiredEvacuations()).resolves.toBe(0)
expect(await primary.query(
`SELECT cell_id, assignment_epoch FROM relay_assignments WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ cell_id: context.target.id, assignment_epoch: '2' }])
expect(
await primary.query(
`SELECT cell_id, assignment_epoch FROM relay_assignments WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ cell_id: context.target.id, assignment_epoch: '2' }])
})
it('completes after the drained host re-resolves through the director', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted')
const attempt = await cutover(context.store, context.now())
// The drain recovery lands while both controls are still live.
await context.store.assign(context.identity, 'asia-east2')
expect(await controlAccounting(context.identity)).toEqual({
@@ -609,11 +589,13 @@ describePostgres('PostgreSQL regional rehoming', () => {
await context.store.releaseActivity(context.identity, context.sourceControl)
await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(1)
expect(await primary.query(
`SELECT completed_at IS NOT NULL AS completed FROM relay_assignment_migrations
expect(
await primary.query(
`SELECT completed_at IS NOT NULL AS completed FROM relay_assignment_migrations
WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ completed: true }])
[context.identity.userId]
)
).toEqual([{ completed: true }])
expect(await controlAccounting(context.identity)).toEqual({
reservedControls: 1,
controlLeases: 1
@@ -622,7 +604,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
it('repairs a skewed control counter before completing the rehome', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
const attempt = await cutover(context.store, context.now())
await context.store.activateControl(context.identity, {
cellId: context.target.id,
assignmentEpoch: attempt!.assignmentEpoch,
@@ -634,10 +616,9 @@ describePostgres('PostgreSQL regional rehoming', () => {
})
await context.store.releaseActivity(context.identity, context.sourceControl)
// Damage already written by a pre-fix sticky grant.
await primary.query(
`UPDATE relay_assignments SET reserved_controls = 0 WHERE user_id = ?`,
[context.identity.userId]
)
await primary.query(`UPDATE relay_assignments SET reserved_controls = 0 WHERE user_id = ?`, [
context.identity.userId
])
await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(1)
expect(await controlAccounting(context.identity)).toEqual({
@@ -646,6 +627,22 @@ describePostgres('PostgreSQL regional rehoming', () => {
})
})
async function cutover(store: RelayAssignmentStore, now: number) {
const [request] = await store.selectIdleRegionalRehomeCandidates(safety(now))
if (!request) return null
const result = await store.commitIdleRegionalRehome(request, safety(now))
if (result.outcome !== 'committed') return null
const [attempt] = await primary.query(
`SELECT preferred_region, assignment_epoch FROM relay_region_rehome_attempts WHERE attempt_id = ?`,
[request.attemptId]
)
return {
...request,
preferredRegion: String(attempt!.preferred_region),
assignmentEpoch: Number(attempt!.assignment_epoch)
}
}
async function attemptAndMigrationCounts(identity: {
userId: string
relayHostId: string
@@ -711,18 +708,12 @@ describePostgres('PostgreSQL regional rehoming', () => {
drainGraceMs: 60_000
})
await store.reconcileCells([source, target])
await heartbeat(
store,
source,
'11111111-1111-4111-8111-111111111111',
1,
900_000
)
await heartbeat(store, source, '11111111-1111-4111-8111-111111111111', 3, 900_000)
await heartbeat(
store,
target,
'22222222-2222-4222-8222-222222222222',
options.targetProtocol ?? 1,
options.targetProtocol ?? 3,
900_000
)
const identity = {
@@ -733,9 +724,32 @@ describePostgres('PostgreSQL regional rehoming', () => {
const sourceControl = await store.activateControl(identity, {
cellId: source.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 1
generation: 1,
idleRegionalRehome: true,
cellIncarnation: '11111111-1111-4111-8111-111111111111'
})
await store.assign(identity, preferredRegion)
const issued = await store.exchangeRegionCorrection(
identity,
{ v: 1, action: 'issue-window' },
assignment.assignmentEpoch
)
await store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: issued.window!.generation,
assignmentEpoch: assignment.assignmentEpoch,
policyVersion: 1,
outcome: 'conclusive',
measurements: {
'us-central1': preferredRegion === 'us-central1' ? 50 : 150,
'asia-east2': preferredRegion === 'asia-east2' ? 50 : 150
}
},
assignment.assignmentEpoch
)
return {
preferredRegion,
store,
@@ -753,6 +767,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
})
const storeOptions = {
regionalRehomeCohortPercent: 100,
requireLiveCells: true,
heartbeatTtlMs: 45_000
}
@@ -816,3 +831,15 @@ async function heartbeat(
}
})
}
function safety(now: number) {
return {
observedAt: now,
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
}
}
cloud/apps/relay/src/regional-rehome-store.test.ts
+392 -609
View File
File diff suppressed because it is too large Load Diff
cloud/apps/relay/src/regional-rehome-target-selection.test.ts
+53 -15
View File
@@ -25,6 +25,7 @@ async function setup() {
let clock = 1_000_000
const database = await openInMemoryRelayDatabase()
const store = new RelayAssignmentStore(database, () => clock, {
regionalRehomeCohortPercent: 100,
requireLiveCells: true,
heartbeatTtlMs: 45_000
})
@@ -83,11 +84,40 @@ async function setup() {
await store.activateControl(identity, {
cellId: source.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 1
generation: 1,
idleRegionalRehome: true,
cellIncarnation: incarnation(1)
})
await store.assign(identity, 'asia-east2')
const { window } = await store.exchangeRegionCorrection(
identity,
{ v: 1, action: 'issue-window' },
assignment.assignmentEpoch
)
expect(window).toBeDefined()
await store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: window!.generation,
assignmentEpoch: assignment.assignmentEpoch,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 180, 'asia-east2': 40 }
},
assignment.assignmentEpoch
)
}
return { database, store, beat, activatePreferredSource }
const safety = () => ({
observedAt: clock,
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
})
return { database, store, beat, activatePreferredSource, safety }
}
const UNCLEAN = REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT + 1
@@ -95,70 +125,78 @@ const UNCLEAN = REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT + 1
describe('regional rehome target selection', () => {
it('never selects a target without connection headroom, even at lowest load', async () => {
const context = await setup()
await context.beat(source, 1, 1, {
await context.beat(source, 1, 3, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: 0
})
// Lowest load but the connection hard cap is exhausted.
await context.beat(noHeadroom, 2, 1, {
await context.beat(noHeadroom, 2, 3, {
observedRequests: 0,
enforcedConnections: 999,
sqlFailures: 0
})
await context.beat(unclean, 3, 1, {
await context.beat(unclean, 3, 3, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: UNCLEAN
})
await context.beat(highLoad, 4, 1, {
await context.beat(highLoad, 4, 3, {
observedRequests: 50,
enforcedConnections: 0,
sqlFailures: 0
})
await context.beat(lowLoad, 5, 1, {
await context.beat(lowLoad, 5, 3, {
observedRequests: 10,
enforcedConnections: 0,
sqlFailures: 0
})
await context.activatePreferredSource()
const attempt = await context.store.claimRegionalRehome()
const candidates = await context.store.selectIdleRegionalRehomeCandidates(context.safety())
const attempt = candidates[0]
expect(attempt?.targetCellId).toBe(lowLoad.id)
expect(await context.store.commitIdleRegionalRehome(attempt!, context.safety(), 100)).toEqual({
outcome: 'committed'
})
await context.database.close()
})
it('falls to the next clean target when the load winner goes unclean', async () => {
const context = await setup()
await context.beat(source, 1, 1, {
await context.beat(source, 1, 3, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: 0
})
await context.beat(noHeadroom, 2, 1, {
await context.beat(noHeadroom, 2, 3, {
observedRequests: 0,
enforcedConnections: 999,
sqlFailures: 0
})
await context.beat(unclean, 3, 1, {
await context.beat(unclean, 3, 3, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: UNCLEAN
})
await context.beat(highLoad, 4, 1, {
await context.beat(highLoad, 4, 3, {
observedRequests: 50,
enforcedConnections: 0,
sqlFailures: 0
})
await context.beat(lowLoad, 5, 1, {
await context.beat(lowLoad, 5, 3, {
observedRequests: 10,
enforcedConnections: 0,
sqlFailures: UNCLEAN
})
await context.activatePreferredSource()
const attempt = await context.store.claimRegionalRehome()
const candidates = await context.store.selectIdleRegionalRehomeCandidates(context.safety())
const attempt = candidates[0]
expect(attempt?.targetCellId).toBe(highLoad.id)
expect(await context.store.commitIdleRegionalRehome(attempt!, context.safety(), 100)).toEqual({
outcome: 'committed'
})
await context.database.close()
})
})
cloud/apps/relay/src/regional-rehome-worker.test.ts
+30 -168
View File
@@ -11,148 +11,12 @@ import { startRegionalRehomeWorker } from './regional-rehome-worker.js'
describe('regional rehome worker', () => {
afterEach(() => vi.restoreAllMocks())
it('sends an incarnation- and source-epoch-bound drain without exposing identity', async () => {
let now = 0
const attempt = {
attemptId: '11111111-1111-4111-8111-111111111111',
userId: 'private-user',
relayHostId: 'abcdefghijklmnop',
preferredRegion: 'asia-east2',
sourceCellId: 'production-gce-c7',
sourceCellUrl: 'https://c7.relay.example.test',
sourceCellIncarnation: '22222222-2222-4222-8222-222222222222',
targetCellId: 'production-gce-c27',
targetCellIncarnation: '33333333-3333-4333-8333-333333333333',
previousEpoch: 7,
assignmentEpoch: 8,
drainGraceMs: 60_000,
sendAttempts: 1
}
const claimRegionalRehome = vi.fn().mockResolvedValueOnce(null).mockResolvedValue(attempt)
const recordRegionalRehomeDrainReceipt = vi.fn().mockResolvedValue(true)
const assignments = {
claimRegionalRehome,
recordRegionalRehomeDrainReceipt
} as unknown as RelayAssignmentStore
const requests: Array<{ url: string; init?: RequestInit }> = []
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const worker = startRegionalRehomeWorker(config(), assignments, {
now: () => now,
safetySnapshot: () => safety(now),
intervalMs: 60_000,
identityToken: async (audience) => {
expect(audience).toBe('https://relay.example.test/v1/admin/host-drain')
return 'secret-token'
},
fetch: (async (url, init) => {
requests.push({ url: String(url), init })
return Response.json({ v: 1, outcome: 'accepted' })
}) as typeof fetch
})!
await settleWorker()
now = 1_000
await worker.run()
worker.stop()
expect(requests).toHaveLength(1)
expect(requests[0]!.url).toBe('https://c7.relay.example.test/v1/admin/host-drain')
expect(requests[0]!.url).not.toContain('secret-token')
expect(requests[0]!.init?.headers).toMatchObject({
authorization: 'Bearer secret-token'
})
expect(JSON.parse(String(requests[0]!.init?.body))).toEqual({
v: 1,
attemptId: '11111111-1111-4111-8111-111111111111',
userId: 'private-user',
relayHostId: 'abcdefghijklmnop',
sourceCellId: 'production-gce-c7',
sourceCellIncarnation: '22222222-2222-4222-8222-222222222222',
sourceAssignmentEpoch: 7,
graceMs: 60_000
})
expect(recordRegionalRehomeDrainReceipt).toHaveBeenCalledWith(
'11111111-1111-4111-8111-111111111111',
'accepted'
)
const logs = warn.mock.calls.map((call) => String(call[0])).join('\n')
expect(logs).not.toContain('private-user')
expect(logs).not.toContain('abcdefghijklmnop')
})
it('fails closed before the observation gate and records bounded dispatch failures', async () => {
let now = 0
const attempt = {
attemptId: '11111111-1111-4111-8111-111111111111',
userId: 'private-user',
relayHostId: 'abcdefghijklmnop',
sourceCellId: 'source',
sourceCellUrl: 'https://source.example.test',
sourceCellIncarnation: '22222222-2222-4222-8222-222222222222',
targetCellId: 'target',
previousEpoch: 1,
assignmentEpoch: 2,
drainGraceMs: 60_000,
sendAttempts: 1
}
const claimRegionalRehome = vi.fn().mockResolvedValueOnce(null).mockResolvedValue(attempt)
const assignments = {
claimRegionalRehome,
recordRegionalRehomeDispatchFailure: vi.fn().mockResolvedValue(undefined)
} as unknown as RelayAssignmentStore
const worker = startRegionalRehomeWorker(config(), assignments, {
now: () => now,
safetySnapshot: () => safety(now),
intervalMs: 60_000,
identityToken: async () => {
throw new Error('token unavailable')
}
})!
await settleWorker()
claimRegionalRehome.mockClear()
now = 100
await worker.run()
worker.stop()
expect(assignments.recordRegionalRehomeDispatchFailure).toHaveBeenCalledWith(
'11111111-1111-4111-8111-111111111111'
)
})
it('keeps a failed poll out of the durable dispatch-failure budget', async () => {
let now = 0
const claimRegionalRehome = vi
.fn()
.mockResolvedValueOnce(null)
.mockRejectedValue(new Error('Connection terminated due to connection timeout'))
const recordRegionalRehomeDispatchFailure = vi.fn().mockResolvedValue(undefined)
const assignments = {
claimRegionalRehome,
recordRegionalRehomeDispatchFailure
} as unknown as RelayAssignmentStore
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const worker = startRegionalRehomeWorker(config(), assignments, {
now: () => now,
safetySnapshot: () => safety(now),
intervalMs: 60_000
})!
await settleWorker()
now = 1_000
await expect(worker.run()).resolves.toBeUndefined()
worker.stop()
// The poll never claimed an attempt, so nothing was drained and nothing may
// be charged to the budget that latches the durable control off.
expect(recordRegionalRehomeDispatchFailure).not.toHaveBeenCalled()
expect(warn.mock.calls.map((call) => JSON.parse(String(call[0])).event)).toEqual([
'orca_relay_regional_rehome_poll_failed'
])
})
it('passes unsafe process telemetry to the durable claim gate', async () => {
let now = 0
let sqlFailures = 0
const claimRegionalRehome = vi.fn().mockResolvedValue(null)
const selectIdleRegionalRehomeCandidates = vi.fn().mockResolvedValue([])
const assignments = {
claimRegionalRehome
selectIdleRegionalRehomeCandidates
} as unknown as RelayAssignmentStore
const worker = startRegionalRehomeWorker(config(), assignments, {
now: () => now,
@@ -160,46 +24,40 @@ describe('regional rehome worker', () => {
intervalMs: 60_000
})!
await settleWorker()
claimRegionalRehome.mockClear()
selectIdleRegionalRehomeCandidates.mockClear()
now = 100
sqlFailures = 1
await worker.run()
worker.stop()
expect(claimRegionalRehome).toHaveBeenCalledWith(
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledWith(
expect.objectContaining({ observedAt: 100, sqlFailures: 1 })
)
})
it('starts inert on directors so durable control can enable without a restart', async () => {
let now = 0
const claimRegionalRehome = vi.fn().mockResolvedValue(null)
const selectIdleRegionalRehomeCandidates = vi.fn().mockResolvedValue([])
const assignments = {
claimRegionalRehome
selectIdleRegionalRehomeCandidates
} as unknown as RelayAssignmentStore
const worker = startRegionalRehomeWorker(
config(),
assignments,
{
now: () => now,
safetySnapshot: () => safety(now),
intervalMs: 60_000
}
)
const worker = startRegionalRehomeWorker(config(), assignments, {
now: () => now,
safetySnapshot: () => safety(now),
intervalMs: 60_000
})
expect(worker).not.toBeNull()
await settleWorker()
claimRegionalRehome.mockClear()
selectIdleRegionalRehomeCandidates.mockClear()
now = 100
await worker!.run()
worker!.stop()
expect(claimRegionalRehome).toHaveBeenCalledOnce()
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledOnce()
expect(
startRegionalRehomeWorker(
config({ role: 'cell' }),
{} as RelayAssignmentStore,
{ safetySnapshot: () => safety(1) }
)
startRegionalRehomeWorker(config({ role: 'cell' }), {} as RelayAssignmentStore, {
safetySnapshot: () => safety(1)
})
).toBeNull()
})
@@ -208,16 +66,20 @@ describe('regional rehome worker', () => {
const limit = cells * REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT
const processSafety = { ...safety(100), reconnects: limit * 10 }
const fleetSafety = { ...safety(100), reconnects: limit }
expect(regionalRehomeSafetyFailure(
combineRegionalRehomeSafety(processSafety, fleetSafety),
100,
cells
)).toBeNull()
expect(regionalRehomeSafetyFailure(
combineRegionalRehomeSafety(processSafety, { ...fleetSafety, reconnects: limit + 1 }),
100,
cells
)).toBe('elevated_reconnects')
expect(
regionalRehomeSafetyFailure(
combineRegionalRehomeSafety(processSafety, fleetSafety),
100,
cells
)
).toBeNull()
expect(
regionalRehomeSafetyFailure(
combineRegionalRehomeSafety(processSafety, { ...fleetSafety, reconnects: limit + 1 }),
100,
cells
)
).toBe('elevated_reconnects')
})
})
cloud/apps/relay/src/regional-rehome-worker.ts
+39 -61
View File
@@ -1,4 +1,4 @@
import { z } from 'zod'
import { IdleRegionalRehomeResponseSchema } from '@orca-cloud/relay-contract'
import type { RelayAssignmentStore } from './assignment-store.js'
import type { RelayConfig } from './config.js'
import { googleMetadataIdentityToken } from './google-metadata-identity-token.js'
@@ -20,13 +20,6 @@ export type RegionalRehomeWorker = {
stop: () => void
}
const RegionalHostDrainResponseSchema = z
.object({
v: z.literal(1),
outcome: z.enum(['accepted', 'already-accepted', 'host-not-connected'])
})
.strict()
export function startRegionalRehomeWorker(
config: RelayConfig,
assignments: RelayAssignmentStore,
@@ -42,7 +35,6 @@ export function startRegionalRehomeWorker(
}
const audience = config.rehomeAudience
const safetySnapshot = options.safetySnapshot
const now = options.now ?? Date.now
const fetchImpl = options.fetch ?? fetch
const tokenProvider =
options.identityToken ??
@@ -52,64 +44,50 @@ export function startRegionalRehomeWorker(
const run = async (): Promise<void> => {
if (stopped || inFlight) return
inFlight = true
let attemptId: string | null = null
try {
const processSafety = safetySnapshot()
const attempt = await assignments.claimRegionalRehome(processSafety)
if (!attempt) return
attemptId = attempt.attemptId
const candidates = await assignments.selectIdleRegionalRehomeCandidates(safetySnapshot())
if (candidates.length === 0) return
const token = await tokenProvider(audience)
const response = await fetchImpl(
new URL('/v1/admin/host-drain', attempt.sourceCellUrl),
{
method: 'POST',
headers: {
authorization: `Bearer ${token}`,
'content-type': 'application/json'
},
body: JSON.stringify({
v: 1,
attemptId: attempt.attemptId,
userId: attempt.userId,
relayHostId: attempt.relayHostId,
sourceCellId: attempt.sourceCellId,
sourceCellIncarnation: attempt.sourceCellIncarnation,
sourceAssignmentEpoch: attempt.previousEpoch,
graceMs: attempt.drainGraceMs
}),
signal: AbortSignal.timeout(options.requestTimeoutMs ?? 10_000)
for (const candidate of candidates) {
if (stopped) return
const { sourceCellUrl, ...request } = candidate
try {
const response = await fetchImpl(new URL('/v1/admin/host-idle-rehome', sourceCellUrl), {
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify({
...request,
cohortPercent: config.regionCorrectionCohortPercent ?? 0,
directorSafety: safetySnapshot()
}),
signal: AbortSignal.timeout(options.requestTimeoutMs ?? 10_000)
})
if (!response.ok) throw new Error(`regional_rehome_source_${response.status}`)
const body = IdleRegionalRehomeResponseSchema.parse(await response.json())
if (body.outcome === 'committed') {
console.warn(
JSON.stringify({
event: 'orca_relay_idle_rehome_committed',
sourceCellId: candidate.sourceCellId,
targetCellId: candidate.targetCellId
})
)
return
}
} catch (error) {
// The source may have committed; its durable outcome owns recovery.
console.warn(
JSON.stringify({
event: 'orca_relay_idle_rehome_request_failed',
reason: error instanceof Error ? error.message : 'unknown'
})
)
}
)
if (!response.ok) throw new Error(`regional_rehome_source_${response.status}`)
const body = RegionalHostDrainResponseSchema.safeParse(await response.json())
if (!body.success) throw new Error('regional_rehome_source_invalid_response')
await assignments.recordRegionalRehomeDrainReceipt(
attempt.attemptId,
body.data.outcome
)
console.warn(
JSON.stringify({
event: 'orca_relay_regional_rehome_dispatched',
sourceCellId: attempt.sourceCellId,
targetCellId: attempt.targetCellId,
outcome: body.data.outcome,
sendAttempts: attempt.sendAttempts
})
)
} catch (error) {
// Only a claimed attempt was drained. A poll that failed before the claim
// - a pool timeout on the once-a-second control read - dispatched nothing,
// so it must not spend the budget that latches the durable control off.
if (attemptId) {
await assignments
.recordRegionalRehomeDispatchFailure(attemptId)
.catch(() => undefined)
}
} catch (error) {
console.warn(
JSON.stringify({
event: attemptId
? 'orca_relay_regional_rehome_dispatch_failed'
: 'orca_relay_regional_rehome_poll_failed',
event: 'orca_relay_regional_rehome_poll_failed',
reason: error instanceof Error ? error.message : 'unknown'
})
)
cloud/apps/relay/src/relay-region-app.test.ts
+155
View File
@@ -40,6 +40,7 @@ describe('Relay region API', () => {
)
expect(response.status).toBe(200)
expect(await response.clone().json()).not.toHaveProperty('regionCorrection')
expect(assign).toHaveBeenCalledWith(
{ userId: 'user-1', relayHostId: 'asiahost00000001' },
'asia-east2',
@@ -83,6 +84,160 @@ describe('Relay region API', () => {
)
})
it('preserves the cold-start hint and binds a negotiated window after placement', async () => {
const assignment = {
userId: 'user-1',
relayHostId: 'abcdefghijklmnop',
cellId: 'asia-c1',
cellUrl: 'https://asia-c1.relay.example.test',
region: 'asia-east2',
assignmentEpoch: 7,
leaseExpiresAt: Date.now() + 300_000
}
const assign = vi.fn(async () => assignment)
const window = {
generation: 2,
expiresAt: Date.now() + 86_400_000,
assignmentEpoch: 7,
incumbentRegion: 'asia-east2',
policyVersion: 1
}
const exchangeRegionCorrection = vi.fn(async () => ({ v: 1, window }))
const app = createRelayApp(config(), {
store: {} as never,
assignments: { assign, exchangeRegionCorrection } as never,
drain: vi.fn(),
ready: async () => true
})
const regionCorrection = { v: 1, action: 'issue-window' }
const response = await app.request(
'/v1/assign',
assignmentRequest('abcdefghijklmnop', {
preferredRegion: 'asia-east2',
regionCorrection
})
)
expect(response.status).toBe(200)
expect(assign).toHaveBeenCalledWith(
{ userId: 'user-1', relayHostId: 'abcdefghijklmnop' },
'asia-east2',
'asia-east2'
)
expect(exchangeRegionCorrection).toHaveBeenCalledWith(
{ userId: 'user-1', relayHostId: 'abcdefghijklmnop' },
regionCorrection,
7
)
expect(((await response.json()) as { regionCorrection: unknown }).regionCorrection).toEqual({
v: 1,
window
})
})
it('returns successful placement when optional window storage is unavailable', async () => {
const app = createRelayApp(config(), {
store: {} as never,
assignments: {
assign: async () => ({
cellId: 'asia-c1',
region: 'asia-east2',
cellUrl: 'https://asia-c1.relay.example.test',
assignmentEpoch: 7
}),
exchangeRegionCorrection: async () => {
throw new Error('database unavailable')
}
} as never,
drain: vi.fn(),
ready: async () => true
})
const response = await app.request(
'/v1/assign',
assignmentRequest('abcdefghijklmnop', {
preferredRegion: 'asia-east2',
regionCorrection: { v: 1, action: 'issue-window' }
})
)
expect(response.status).toBe(200)
expect(await response.json()).toMatchObject({
cellUrl: 'https://asia-c1.relay.example.test',
assignmentEpoch: 7
})
})
it('does not place or write a legacy hint when reporting migration evidence', async () => {
const current = {
userId: 'user-1',
relayHostId: 'abcdefghijklmnop',
cellId: 'asia-c1',
cellUrl: 'https://asia-c1.relay.example.test',
region: 'asia-east2',
assignmentEpoch: 7,
leaseExpiresAt: Date.now() + 300_000
}
const assign = vi.fn()
const resolve = vi.fn(async () => current)
const exchangeRegionCorrection = vi.fn(async () => ({ v: 1, reportStatus: 'accepted' }))
const app = createRelayApp(config(), {
store: {} as never,
assignments: { assign, resolve, exchangeRegionCorrection } as never,
drain: vi.fn(),
ready: async () => true
})
const regionCorrection = {
v: 1,
action: 'report',
generation: 2,
assignmentEpoch: 7,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 40, 'asia-east2': 180 }
}
const response = await app.request(
'/v1/assign',
assignmentRequest('abcdefghijklmnop', {
preferredRegion: 'us-central1',
regionCorrection
})
)
expect(response.status).toBe(200)
expect(assign).not.toHaveBeenCalled()
expect(exchangeRegionCorrection).toHaveBeenCalledWith(
{ userId: 'user-1', relayHostId: 'abcdefghijklmnop' },
regionCorrection,
7
)
expect(((await response.json()) as { assignmentEpoch: number }).assignmentEpoch).toBe(7)
})
it('does not manufacture an assignment for a report whose assignment disappeared', async () => {
const assign = vi.fn()
const exchangeRegionCorrection = vi.fn()
const app = createRelayApp(config(), {
store: {} as never,
assignments: { assign, resolve: async () => null, exchangeRegionCorrection } as never,
drain: vi.fn(),
ready: async () => true
})
const response = await app.request(
'/v1/assign',
assignmentRequest('abcdefghijklmnop', {
regionCorrection: {
v: 1,
action: 'report',
generation: 2,
assignmentEpoch: 7,
policyVersion: 1,
outcome: 'inconclusive',
reason: 'timeout'
}
})
)
expect(response.status).toBe(409)
expect(assign).not.toHaveBeenCalled()
expect(exchangeRegionCorrection).not.toHaveBeenCalled()
})
it('exposes only the store-provided healthy catalog from directors', async () => {
const regionCatalog = vi.fn(async () => [
{ region: 'us-central1' as const, probeOrigins: ['https://us.relay.example.test'] }
cloud/apps/relay/src/relay-server.ts
+25 -8
View File
@@ -20,14 +20,12 @@ import { createRelayApp } from './app.js'
import { RelayAssignmentStore } from './assignment-store.js'
import type { RelayConfig } from './config.js'
import { RelayCredentialStore } from './credential-store.js'
import type { RelayDatabase } from './database.js'
import { readRelayDatabasePoolPressure, type RelayDatabase } from './database.js'
import { HostSessionRegistry } from './host-session-registry.js'
import { observeRelayDatabase } from './observed-relay-database.js'
import { RelayObservability } from './relay-observability.js'
import {
RelayConnectionLedger,
type RelayConnectionUpgrade
} from './relay-connection-ledger.js'
import { combineRegionalRehomeSafety } from './regional-rehome-safety.js'
import { RelayConnectionLedger, type RelayConnectionUpgrade } from './relay-connection-ledger.js'
import { createRelayReadiness } from './relay-readiness.js'
import { createRelayTokenVerifier, readBearer } from './relay-token-verifier.js'
import { closeRelayWebSocket } from './relay-websocket-close.js'
@@ -65,7 +63,7 @@ function guardSocketErrors(socket: WebSocket, kind: string): void {
function admissionSource(request: IncomingMessage): string {
const forwarded = request.headers['x-forwarded-for']
const chain = (Array.isArray(forwarded) ? forwarded.join(',') : forwarded ?? '')
const chain = (Array.isArray(forwarded) ? forwarded.join(',') : (forwarded ?? ''))
.split(',')
.map((entry) => entry.trim())
.filter(Boolean)
@@ -112,6 +110,7 @@ export function createRelayServer(
const store = new RelayCredentialStore(observedDatabase, options.now)
const assignments = new RelayAssignmentStore(observedDatabase, options.now, {
requireLiveCells: config.role === 'director',
regionalRehomeCohortPercent: config.regionCorrectionCohortPercent ?? 0,
recordControlRenewal: (durationMs, outcome) =>
observability.recordControlRenewal?.(durationMs, outcome)
})
@@ -127,17 +126,35 @@ export function createRelayServer(
queuedBytes,
observability,
options.now,
options.random
options.random,
cellIncarnation
)
const app = createRelayApp(config, {
store,
assignments,
drain: (graceMs) => sessions.drain(graceMs),
drainHost: (input) => sessions.drainHost(input),
idleRehome: (input) => {
const now = (options.now ?? Date.now)()
if (input.directorSafety.observedAt > now || now - input.directorSafety.observedAt > 60_000) {
return Promise.resolve({ outcome: 'deferred' })
}
return sessions.idleRehome(input,
() => assignments.commitIdleRegionalRehome(input, combineRegionalRehomeSafety(
input.directorSafety,
{ ...observability.regionalRehomeRuntimeSafety(), ...readRelayDatabasePoolPressure(database) }
), input.cohortPercent),
() => assignments.reconcileIdleRegionalRehome(input)
)
},
regionalRehomeTrustProbeHostExists: (input) => sessions.get(input) !== null,
cellIncarnation,
isDraining: () => sessions.isDraining(),
runtimeCounts: () => runtimeCounts(),
regionalRehomeSafetySnapshot: () => ({
...observability.regionalRehomeRuntimeSafety(),
...readRelayDatabasePoolPressure(database)
}),
ready,
recordAssignmentAdmission: (outcome) => observability.recordAssignmentAdmission?.(outcome),
recordAssignmentRejectionReason: (lane, reason) =>
@@ -339,7 +356,7 @@ export function createRelayServer(
const identity = invite ? { userId: invite.userId, relayHostId: hostId } : null
// Released combined-service invites gain their first durable cell assignment here.
const assignment = identity
? (await assignments.resolve(identity)) ?? (await assignments.assign(identity))
? ((await assignments.resolve(identity)) ?? (await assignments.assign(identity)))
: null
if (!invite || !assignment) {
phoneAdmission?.hostData.release()
cloud/apps/relay/src/relay-sweep-schedule.test.ts
+1 -1
View File
@@ -35,7 +35,7 @@ describe('sweep schedule jitter', () => {
rehomeAudience: 'https://rehome.example.test',
rehomeDirectorServiceAccount: 'rehome@example.test'
} as never,
{ claimRegionalRehome: async () => null } as never,
{ selectIdleRegionalRehomeCandidates: async () => [] } as never,
{ random: () => 0.5, safetySnapshot: () => ({}) as never }
)
} finally {
cloud/apps/relay/src/test-fixtures/relay-contract-baseline/README.md
+8
View File
@@ -0,0 +1,8 @@
Exact relay contract snapshots from `027acb4efa2e6b226d40df266b86367423946d62`, `cloud/packages/relay-contract/src/`. Used to exercise the pre-correction strict wire parsers. Do not format or edit these baseline sources.
```text
aba94e108a5cd0f1af8b38875429ad8636d24c43a728273e3df60d9a1a1d1b6d director-messages.ts
bd13b5a694a5d683a5b680c14e46ab33f4ef4a5bfedf040d046b09d540cb4c17 wire-scalars.ts
bc89116f884a2f20a6588f9b91219aa596bc2410d28b499a93a78350def109d5 relay-regions.ts
8fcae470a5fc72f2fcdde9d2f09cd20289c256356dd490484ac1cfa53839fbe4 control-messages.ts
```
cloud/apps/relay/src/test-fixtures/relay-contract-baseline/control-messages.ts
+146
View File
@@ -0,0 +1,146 @@
import { z } from 'zod'
import {
Base6432ByteSchema,
Base64Raw24ByteSchema,
Base64Url32ByteSchema,
EpochMsSchema,
GenerationSchema,
OpaqueIdSchema,
PositiveDurationMsSchema,
RelayHostIdSchema
} from './wire-scalars.js'
const AppVersionSchema = z.string().min(1).max(128)
const BoundedCiphertextSchema = z
.string()
.min(1)
.max(16 * 1024)
.regex(/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/)
const ConnectionKindSchema = z.enum(['invite', 'resume'])
export const HostHelloSchema = z
.object({
v: z.literal(1),
relayHostId: RelayHostIdSchema,
assignmentEpoch: GenerationSchema,
hostPublicKeyB64: Base6432ByteSchema,
appVersion: AppVersionSchema,
previousGeneration: GenerationSchema.optional(),
controlResumeSecret: Base64Url32ByteSchema.optional()
})
.strict()
export const HostChallengeSchema = z
.object({
challengeId: OpaqueIdSchema,
relayEphemeralPublicKeyB64: Base6432ByteSchema,
nonceB64: Base64Raw24ByteSchema,
ciphertextB64: BoundedCiphertextSchema,
expiresAt: EpochMsSchema
})
.strict()
export const HostChallengeAckSchema = z
.object({ challengeId: OpaqueIdSchema, proofB64: Base6432ByteSchema })
.strict()
// Advertised on the control upgrade rather than in host-hello: HostHelloSchema
// is strict, so a new hello key is refused by every already-deployed cell.
export const RELAY_HOST_CAPABILITIES_HEADER = 'x-orca-host-capabilities'
// The host accepts kind/relayDeviceId on a pendingConns entry. A host that does
// not advertise this parses those entries strictly and would drop the whole ack.
export const RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS = 'pending-conn-details'
export function parseRelayHostCapabilities(
header: string | string[] | undefined
): ReadonlySet<string> {
const raw = Array.isArray(header) ? header.join(',') : (header ?? '')
return new Set(
raw
.split(',')
.map((token) => token.trim())
.filter((token) => token.length > 0 && token.length <= 64)
.slice(0, 16)
)
}
// kind/relayDeviceId are optional so an entry stays readable by a host that
// predates them; the cell only emits them to a host that advertised support.
const PendingConnectionSchema = z
.object({
connId: OpaqueIdSchema,
connTicket: Base64Url32ByteSchema,
kind: ConnectionKindSchema.optional(),
relayDeviceId: OpaqueIdSchema.optional()
})
.strict()
export const HostHelloAckSchema = z
.object({
v: z.literal(1),
generation: GenerationSchema,
controlResumeSecret: Base64Url32ByteSchema,
leaseExpiresAt: EpochMsSchema,
activeConnIds: z.array(OpaqueIdSchema).max(8),
pendingConns: z.array(PendingConnectionSchema).max(8)
})
.strict()
export const ConnectionOpenSchema = z
.object({
connId: OpaqueIdSchema,
connTicket: Base64Url32ByteSchema,
kind: ConnectionKindSchema,
relayDeviceId: OpaqueIdSchema,
attachDeadlineMs: PositiveDurationMsSchema
})
.strict()
export const HostDataAuthSchema = z
.object({
v: z.literal(1),
connTicket: Base64Url32ByteSchema,
generation: GenerationSchema
})
.strict()
export const InviteCreateSchema = z
.object({ reqId: OpaqueIdSchema, relayDeviceId: OpaqueIdSchema })
.strict()
export const InviteCreatedSchema = z
.object({
reqId: OpaqueIdSchema,
inviteToken: Base64Url32ByteSchema,
expiresAt: EpochMsSchema,
maxAttempts: z.number().int().positive().max(16)
})
.strict()
export const DeviceRevokeSchema = z
.object({ reqId: OpaqueIdSchema, relayDeviceId: OpaqueIdSchema })
.strict()
export const AuthRefreshSchema = z.object({ relayJwt: z.string().min(1).max(8 * 1024) }).strict()
export const DrainSchema = z
.object({
graceMs: z.number().int().nonnegative().max(60 * 60 * 1000),
recovery: z.literal('resolve-director')
})
.strict()
export const HeartbeatSchema = z.object({ t: EpochMsSchema }).strict()
export type HostHello = z.infer<typeof HostHelloSchema>
export type HostChallenge = z.infer<typeof HostChallengeSchema>
export type HostChallengeAck = z.infer<typeof HostChallengeAckSchema>
export type HostHelloAck = z.infer<typeof HostHelloAckSchema>
export type ConnectionOpen = z.infer<typeof ConnectionOpenSchema>
export type HostDataAuth = z.infer<typeof HostDataAuthSchema>
export type InviteCreate = z.infer<typeof InviteCreateSchema>
export type InviteCreated = z.infer<typeof InviteCreatedSchema>
export type DeviceRevoke = z.infer<typeof DeviceRevokeSchema>
export type AuthRefresh = z.infer<typeof AuthRefreshSchema>
export type Drain = z.infer<typeof DrainSchema>
export type Heartbeat = z.infer<typeof HeartbeatSchema>
cloud/apps/relay/src/test-fixtures/relay-contract-baseline/director-messages.ts
+75
View File
@@ -0,0 +1,75 @@
import { z } from 'zod'
import {
Base64Url32ByteSchema,
CanonicalHttpsOriginSchema,
EpochMsSchema,
GenerationSchema,
RelayHostIdSchema
} from './wire-scalars.js'
import { RelayRegionSchema } from './relay-regions.js'
const SignedAssignmentLeaseSchema = z.string().min(1).max(8 * 1024)
export const AssignmentRequestSchema = z
.object({
v: z.literal(1),
relayHostId: RelayHostIdSchema,
// Client-declared reconnection; the director verifies it against the
// durable assignment before granting fast-lane admission.
reconnect: z.boolean().optional(),
preferredRegion: RelayRegionSchema.optional()
})
.strict()
export const AssignmentResponseSchema = z
.object({
v: z.literal(1),
cellUrl: CanonicalHttpsOriginSchema,
assignmentEpoch: GenerationSchema,
lease: SignedAssignmentLeaseSchema
})
.strict()
export const ResolveRequestSchema = z
.object({
v: z.literal(1),
relayHostId: RelayHostIdSchema,
resumeToken: Base64Url32ByteSchema
})
.strict()
export const ResolveResponseSchema = z
.object({
v: z.literal(1),
cellUrl: CanonicalHttpsOriginSchema,
assignmentEpoch: GenerationSchema,
leaseExpiresAt: EpochMsSchema
})
.strict()
export const RelayMovedSchema = z
.object({
v: z.literal(1),
cellUrl: CanonicalHttpsOriginSchema,
assignmentEpoch: GenerationSchema
})
.strict()
export function isTrustedNewerMove(input: {
sourceOrigin: string
configuredDirectorOrigin: string
currentAssignmentEpoch: number
move: z.infer<typeof RelayMovedSchema>
}): boolean {
// Why: cells and stale director responses must never redirect a credential-bearing client.
return (
input.sourceOrigin === input.configuredDirectorOrigin &&
input.move.assignmentEpoch > input.currentAssignmentEpoch
)
}
export type AssignmentRequest = z.infer<typeof AssignmentRequestSchema>
export type AssignmentResponse = z.infer<typeof AssignmentResponseSchema>
export type ResolveRequest = z.infer<typeof ResolveRequestSchema>
export type ResolveResponse = z.infer<typeof ResolveResponseSchema>
export type RelayMoved = z.infer<typeof RelayMovedSchema>
cloud/apps/relay/src/test-fixtures/relay-contract-baseline/relay-regions.ts
+71
View File
@@ -0,0 +1,71 @@
import { z } from 'zod'
export const RELAY_REGIONS = ['us-central1', 'asia-east2'] as const
export const RelayRegionSchema = z.enum(RELAY_REGIONS)
export type RelayRegion = z.infer<typeof RelayRegionSchema>
export const RELAY_DEFAULT_REGION: RelayRegion = 'us-central1'
// Field-name segment for the flat per-region runtime counters, spelled out rather than derived so
// the Terraform side can hold the same literal and a test can compare the two. `satisfies` makes a
// new region a compile error here, which is the point: a region with no segment would silently
// drop out of the region-skew alert's denominators.
export const RELAY_REGION_METRIC_SEGMENTS = {
'us-central1': 'UsCentral1',
'asia-east2': 'AsiaEast2'
} as const satisfies Record<RelayRegion, string>
const RelayProbeOriginSchema = z.string().url().max(2_048).refine(isCanonicalHttpsOrigin)
export const RelayRegionCatalogResponseSchema = z
.object({
v: z.literal(1),
regions: z
.array(
z
.object({
region: RelayRegionSchema,
probeOrigins: z.array(RelayProbeOriginSchema).min(1).max(2)
})
.strict()
)
.max(RELAY_REGIONS.length)
})
.strict()
.superRefine((catalog, context) => {
const regions = new Set<RelayRegion>()
const origins = new Set<string>()
for (const [regionIndex, entry] of catalog.regions.entries()) {
if (regions.has(entry.region)) {
context.addIssue({
code: 'custom',
message: 'duplicate relay region',
path: ['regions', regionIndex, 'region']
})
}
regions.add(entry.region)
for (const [originIndex, origin] of entry.probeOrigins.entries()) {
if (origins.has(origin)) {
context.addIssue({
code: 'custom',
message: 'duplicate relay probe origin',
path: ['regions', regionIndex, 'probeOrigins', originIndex]
})
}
origins.add(origin)
}
}
})
export type RelayRegionCatalogResponse = z.infer<typeof RelayRegionCatalogResponseSchema>
function isCanonicalHttpsOrigin(value: string): boolean {
try {
const url = new URL(value)
return url.protocol === 'https:' && url.origin === value
} catch {
return false
}
}
cloud/apps/relay/src/test-fixtures/relay-contract-baseline/wire-scalars.ts
+20
View File
@@ -0,0 +1,20 @@
import { z } from 'zod'
export const Base64Url32ByteSchema = z.string().regex(/^[A-Za-z0-9_-]{43}$/)
export const Base64Url24ByteSchema = z.string().regex(/^[A-Za-z0-9_-]{32}$/)
export const Base6432ByteSchema = z.string().regex(/^(?:[A-Za-z0-9+/]{4}){10}[A-Za-z0-9+/]{3}=$/)
export const Base64Raw24ByteSchema = z.string().regex(/^(?:[A-Za-z0-9+/]{4}){8}$/)
export const RelayHostIdSchema = z.string().regex(/^[A-Za-z0-9_-]{16}$/)
export const OpaqueIdSchema = z.string().min(1).max(128)
export const EpochMsSchema = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER)
export const GenerationSchema = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER)
export const PositiveDurationMsSchema = z.number().int().positive().max(24 * 60 * 60 * 1000)
export const CanonicalHttpsOriginSchema = z.string().max(2048).refine((value) => {
try {
const url = new URL(value)
return url.protocol === 'https:' && url.origin === value && url.pathname === '/'
} catch {
return false
}
}, 'must be a canonical HTTPS origin')
cloud/apps/relay/tsconfig.build.json
+1 -1
View File
@@ -6,5 +6,5 @@
"outDir": "dist",
"rootDir": "src"
},
"exclude": ["src/**/*.test.ts"]
"exclude": ["src/**/*.test.ts", "src/test-fixtures/**"]
}
cloud/dev/scripts/deploy-relay-blue-green.mjs
+22 -1
View File
@@ -10,6 +10,7 @@ export const DIRECTOR_REGIONAL_PLACEMENT_SECRET =
'orca-cloud-relay-regional-placement-enabled'
export const DIRECTOR_REGIONAL_PLACEMENT_ENV =
'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED'
export const DIRECTOR_CORRECTION_COHORT_ENV = 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT'
export const DIRECTOR_REHOME_IDENTITY_ENV =
'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT'
export const DIRECTOR_REHOME_AUDIENCE_ENV = 'ORCA_RELAY_REHOME_AUDIENCE'
@@ -228,6 +229,13 @@ export function directorCellSetAddition(currentValue, desiredValue) {
return { changed: additions.length > 0, value: JSON.stringify(desired) }
}
export function correctionCohortPercent(value) {
if (!/^(?:[0-9]|[1-9][0-9]|100)$/.test(String(value))) {
throw new Error('region correction cohort must be an integer from 0 to 100')
}
return String(value)
}
export function directorDeploymentEnvironment(config) {
const imageDigest = config.image?.match(/@(sha256:[a-f0-9]{64})$/)?.[1]
if (config.image !== undefined && imageDigest === undefined) {
@@ -238,6 +246,10 @@ export function directorDeploymentEnvironment(config) {
ORCA_RELAY_ADMISSION_SELECTOR_VERSION: SELECTOR_REVISION_MARKER,
...(imageDigest === undefined ? {} : { ORCA_RELAY_IMAGE_DIGEST: imageDigest })
}
if (config['region-correction-cohort-percent'] !== undefined &&
config['region-correction-cohort-percent'] !== 'preserve') {
environment[DIRECTOR_CORRECTION_COHORT_ENV] = correctionCohortPercent(config['region-correction-cohort-percent'])
}
const serviceAccount = projectServiceAccount(config, 'capacity-service-account')
const asiaProofServiceAccount = projectServiceAccount(config, 'asia-proof-service-account')
const rehomeDirectorServiceAccount = projectServiceAccount(
@@ -302,7 +314,8 @@ export function parseArguments(argv) {
values['rehome-director-service-account'] !== undefined ||
values['rehome-audience'] !== undefined ||
values['expected-rehome-generation'] !== undefined ||
values['rehome-control-origin'] !== undefined
values['rehome-control-origin'] !== undefined ||
values['region-correction-cohort-percent'] !== undefined
) {
throw new Error('director configuration arguments require --role director')
}
@@ -785,6 +798,14 @@ export async function deployDirector(config, tag, overrides = {}) {
config['prune-revisions'] === 'true' ? CONNECTION_CAPACITY_PROTOCOL : undefined
const currentEnvironment = revisionEnvironment(servingRevision)
const deploymentEnvironment = directorDeploymentEnvironment(config)
deploymentEnvironment[DIRECTOR_CORRECTION_COHORT_ENV] ??= correctionCohortPercent(
currentEnvironment[DIRECTOR_CORRECTION_COHORT_ENV] ?? '0'
)
if (config['region-correction-cohort-percent'] !== undefined &&
config['region-correction-cohort-percent'] !== 'preserve' &&
config['expected-rehome-generation'] === undefined) {
throw new Error('cohort changes require an exact disabled regional-rehome generation')
}
const mutableEnvironment = {
...deploymentEnvironment,
[DIRECTOR_REGIONAL_PLACEMENT_ENV]: ''
cloud/dev/scripts/deploy-relay-blue-green.test.mjs
+42
View File
@@ -4,6 +4,8 @@ import { test } from 'node:test'
import { fileURLToPath } from 'node:url'
import {
activeRevision,
correctionCohortPercent,
DIRECTOR_CORRECTION_COHORT_ENV,
cloudRunTrafficTag,
DIRECTOR_ADMISSION_ENVIRONMENT,
DIRECTOR_REGIONAL_PLACEMENT_ENV,
@@ -812,3 +814,43 @@ test('waits for authenticated target readiness without hiding other capacity err
/forbidden/
)
})
test('validates bounded correction cohorts and leaves unspecified values to serving inheritance', () => {
for (const value of ['0', '1', '100']) assert.equal(correctionCohortPercent(value), value)
for (const value of ['-1', '101', '1.5', '', '01', 'true', '1\n']) {
assert.throws(() => correctionCohortPercent(value), /integer from 0 to 100/)
}
assert.equal(directorDeploymentEnvironment({})[DIRECTOR_CORRECTION_COHORT_ENV], undefined)
assert.equal(directorDeploymentEnvironment({ 'region-correction-cohort-percent': 'preserve' })[DIRECTOR_CORRECTION_COHORT_ENV], undefined)
assert.equal(directorDeploymentEnvironment({ 'region-correction-cohort-percent': '1' })[DIRECTOR_CORRECTION_COHORT_ENV], '1')
})
test('inherits the cohort on candidate and rollback revisions without resetting an enabled cohort', async () => {
const harness = directorHarness()
harness.state.revisions.get('relay-00001-old').env[DIRECTOR_CORRECTION_COHORT_ENV] = '3'
await deployDirector({}, 'candidate-new', harness.operations)
for (const revision of ['relay-00002-new', 'relay-00003-new']) {
assert.equal(harness.state.revisions.get(revision).env[DIRECTOR_CORRECTION_COHORT_ENV], '3')
}
})
test('starts an unstamped cohort at zero and rejects a cohort change without disabled-control proof', async () => {
const harness = directorHarness()
await assert.rejects(deployDirector({ 'region-correction-cohort-percent': '1' },
'candidate-new', harness.operations), /exact disabled regional-rehome generation/)
assert.equal(harness.state.activeRevision, 'relay-00001-old')
assert.equal(harness.state.nextRevision, 2)
await deployDirector({}, 'candidate-new', harness.operations)
assert.equal(harness.state.revisions.get('relay-00003-new').env[DIRECTOR_CORRECTION_COHORT_ENV], '0')
})
test('sets a reviewed cohort only behind repeated disabled-control verification', async () => {
const harness = directorHarness()
let verified = 0
const config = { 'region-correction-cohort-percent': '1', 'expected-rehome-generation': '7' }
await deployDirector(config, 'candidate-new', { ...harness.operations,
assertRegionalRehomeDisabled: async () => { verified++ } })
assert.ok(verified >= 2)
assert.equal(harness.state.revisions.get('relay-00003-new').env[DIRECTOR_CORRECTION_COHORT_ENV], '1')
})
cloud/dev/scripts/read-relay-serving-regional-placement-version.mjs
+13 -3
View File
@@ -53,7 +53,7 @@ export function readRelayServingRegionalPlacementVersion(input, dependencies = {
try {
service = run(gcloudArguments('services', input))
} catch (error) {
if (error?.code === 'NOT_FOUND') return { version: input.bootstrap_version }
if (error?.code === 'NOT_FOUND') return { version: input.bootstrap_version, cohort_percent: '0' }
throw error
}
const serving = (service.status?.traffic ?? []).filter(
@@ -67,12 +67,22 @@ export function readRelayServingRegionalPlacementVersion(input, dependencies = {
throw new Error('Relay director must have exactly one revision serving 100% traffic')
}
const revision = run(gcloudArguments('revisions', input, serving[0].revisionName))
const cohortSettings = (revision.spec?.containers ?? []).flatMap((container) =>
(container.env ?? []).filter((environment) =>
environment.name === 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT')
)
if (cohortSettings.length > 1 || (cohortSettings.length === 1 &&
(typeof cohortSettings[0].value !== 'string' ||
!/^(?:[0-9]|[1-9][0-9]|100)$/.test(cohortSettings[0].value)))) {
throw new Error('serving region correction cohort is invalid')
}
const cohort_percent = cohortSettings[0]?.value ?? '0'
const references = (revision.spec?.containers ?? []).flatMap((container) =>
(container.env ?? []).filter(
(environment) => environment.name === 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED'
)
)
if (references.length === 0) return { version: input.bootstrap_version }
if (references.length === 0) return { version: input.bootstrap_version, cohort_percent }
const reference = normalizeSecretReference(references[0])
if (
references.length !== 1 ||
@@ -81,7 +91,7 @@ export function readRelayServingRegionalPlacementVersion(input, dependencies = {
) {
throw new Error('serving regional placement secret reference is invalid')
}
return { version: reference.version }
return { version: reference.version, cohort_percent }
}
// Why: the v2 API reports `valueSource.secretKeyRef.{secret,version}`, but
cloud/dev/scripts/read-relay-serving-regional-placement-version.test.mjs
+32 -4
View File
@@ -67,7 +67,7 @@ test('reads the exact version from the sole traffic-serving revision', () => {
}
})
assert.deepEqual(result, { version: '11' })
assert.deepEqual(result, { version: '11', cohort_percent: '0' })
assert.equal(calls[1][3], 'relay-serving')
})
@@ -78,7 +78,7 @@ test('reads the gcloud v1 secret reference shape by bare id and by full resource
]) {
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
run: (args) => args[1] === 'services' ? serving() : v1Revision(name, '1')
}), { version: '1' })
}), { version: '1', cohort_percent: '0' })
}
})
@@ -100,12 +100,12 @@ test('falls back only when the service or setting is absent', () => {
notFound.code = 'NOT_FOUND'
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
run: () => { throw notFound }
}), { version: '7' })
}), { version: '7', cohort_percent: '0' })
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
run: (args) => args[1] === 'services'
? { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } }
: { spec: { containers: [{ env: [] }] } }
}), { version: '7' })
}), { version: '7', cohort_percent: '0' })
})
test('classifies real absent-service stderr without weakening revision failures', () => {
@@ -136,3 +136,31 @@ test('rejects ambiguous traffic, malformed references, and read failures', () =>
run: () => { throw denied }
}), denied)
})
test('preserves the serving cohort including explicit disable across later Terraform plans', () => {
for (const value of ['0', '1', '17', '100']) {
const servingRevision = revision()
servingRevision.spec.containers[0].env.push({ name: 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT', value })
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
run: (args) => args[1] === 'services' ? serving() : servingRevision
}), { version: '11', cohort_percent: value })
}
})
test('fails closed on malformed, secret-backed or duplicate cohorts rather than resetting them', () => {
const name = 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT'
const cases = [
[{ name, value: '101' }], [{ name, value: '-1' }], [{ name, value: '1.5' }],
[{ name, value: '' }], [{ name, value: '01' }], [{ name, value: 1 }],
[{ name, valueFrom: { secretKeyRef: { name: 'unexpected', key: '1' } } }],
[{ name, value: '1' }, { name, value: '2' }]
]
for (const settings of cases) {
const servingRevision = revision()
servingRevision.spec.containers[0].env.push(...settings)
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
run: (args) => args[1] === 'services' ? serving() : servingRevision
}), /cohort is invalid/)
}
})
cloud/docs/orca-relay-operations.md
+61
View File
@@ -491,3 +491,64 @@ Run and record each scenario in staging before launch:
- return a dormant host, overload a cell, kill a cell, evacuate active work, and exercise pre-registration rollback.
The served black-box relay suite validates the protocol/state transitions used by these procedures. The physical-device and real-GFE canaries remain separate launch gates; unit/black-box success cannot replace them.
## Optional measured region correction (deployment gated)
New optimization claims require both the durable regional-rehome control and
`ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT` (integer 0–100, default **0**).
Turning either gate off stops new optional moves; ordinary migration cleanup and
recovery continue. Legacy preferred-region hints do not certify a correction. Both
cells must advertise regional protocol3 and the authenticated desktop control must
advertise idle-regional-rehome-v1. The source must have no actual client sockets or
pending admission/control work; a live control socket alone does not prevent a move.
The monitor/deploy identity can read **GET `/v1/admin/regional-rehome-preview`**.
It returns full-population eligibility/exclusion counts, open-migration capacity,
process-safety gating and aggregate migration outcomes; it never claims a
host or changes the failure budget. This is advisory, with separately read state:
concurrent assignments, capacity changes, rate pauses and control changes can make
the next claim differ. Inspect the durable control separately before enabling.
Do not treat an unavailable/failed preview as zero eligible hosts.
`orca_relay_region_correction_outcomes` reports attempts by source/target,
registration/completion/abort state, oldest open age and target
reservation units every five minutes. `orca_relay_region_comparison` samples a
stable 10% of accepted reports (including unchanged hosts), keyed by host digest,
assignment epoch and decision generation. Existing control RTT and client-accept
logs include assignment epoch, control generation and drain mode; join those for
matched before/after and unchanged-cohort comparisons. Client accept latency is
connection setup, not application command round trip. No application-latency
improvement has been demonstrated by probe differences alone.
Quiet live connections count as work and defer optional correction indefinitely.
A returning client may race with the short admission gate and retry normally. No
optimization timer may close an established client. Investigate failed registration,
ambiguous authority, stuck reservations and reconnect/failure rates against agreed
limits. A database outage can keep the source fenced until locked reconciliation
establishes its authority; timeout alone is not permission to reopen admissions.
All directors must run the reviewed idle worker before enabling. Record the tested
immutable source and rollback revisions, then verify the ordinary migration recovery
path before rollout. There is no retained-source table or renewal protocol. Deploying
supporting cells/desktops and enabling a cohort require separate rollout authorization
and explicit numerical stop criteria; this change enables neither.
### Setting the correction cohort during a reviewed director rollout
The existing **Deploy Relay Production Director** workflow accepts
`region-correction-cohort-percent`: `preserve` (default) or an integer0–100.
It carries the cohort onto both candidate and compatible rollback revisions and
verifies the environment before promotion. If the predecessor has no setting,
`preserve` stamps zero. An explicit change requires the exact disabled durable
rehome generation; configuring a nonzero cohort does not itself enable the sweep.
The usual image, identity, health and traffic checks remain in force. No workflow
was dispatched as part of implementation.
Terraform reads the cohort from the same traffic-serving revision used to preserve
regional placement. A later apply therefore preserves a workflow-set cohort,
including explicit zero; only an absent service/setting bootstraps to0. Malformed
or ambiguous live settings fail the plan instead of silently resetting the cohort.
The audited director workflow owns subsequent changes.
Before the first nonzero cohort, verify compatible protocol2 cells, updated
cleanup workers, preview eligibility, both serving/rollback images and the
explicitly approved observation/stop criteria.
cloud/infra/terraform/relay.tf
+5
View File
@@ -169,6 +169,11 @@ resource "google_cloud_run_v2_service" "relay" {
}
}
env {
name = "ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT"
value = data.external.relay_serving_regional_placement_version.result.cohort_percent
}
ports {
container_port = 8080
}
cloud/packages/push-contract/src/send-messages.ts
+2 -1
View File
@@ -26,7 +26,8 @@ export const PushNotificationSchema = z
agentState: PushAgentStateSchema.nullable(),
title: z.string().min(1).max(PUSH_LIMITS.titleMaxChars),
body: z.string().max(PUSH_LIMITS.bodyMaxChars),
worktreeId: z.string().min(1).max(2048).optional()
worktreeId: z.string().min(1).max(2048).optional(),
paneKey: z.string().min(1).max(2048).optional()
})
.strict()
.refine(
cloud/packages/relay-contract/src/control-messages.ts
+14 -2
View File
@@ -50,6 +50,7 @@ export const RELAY_HOST_CAPABILITIES_HEADER = 'x-orca-host-capabilities'
// The host accepts kind/relayDeviceId on a pendingConns entry. A host that does
// not advertise this parses those entries strictly and would drop the whole ack.
export const RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS = 'pending-conn-details'
export const RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME = 'idle-regional-rehome-v1'
export function parseRelayHostCapabilities(
header: string | string[] | undefined
@@ -121,11 +122,22 @@ export const DeviceRevokeSchema = z
.object({ reqId: OpaqueIdSchema, relayDeviceId: OpaqueIdSchema })
.strict()
export const AuthRefreshSchema = z.object({ relayJwt: z.string().min(1).max(8 * 1024) }).strict()
export const AuthRefreshSchema = z
.object({
relayJwt: z
.string()
.min(1)
.max(8 * 1024)
})
.strict()
export const DrainSchema = z
.object({
graceMs: z.number().int().nonnegative().max(60 * 60 * 1000),
graceMs: z
.number()
.int()
.nonnegative()
.max(60 * 60 * 1000),
recovery: z.literal('resolve-director')
})
.strict()
cloud/packages/relay-contract/src/director-messages.ts
+12 -3
View File
@@ -7,8 +7,15 @@ import {
RelayHostIdSchema
} from './wire-scalars.js'
import { RelayRegionSchema } from './relay-regions.js'
import {
RegionCorrectionRequestSchema,
RegionCorrectionResponseSchema
} from './region-correction.js'
const SignedAssignmentLeaseSchema = z.string().min(1).max(8 * 1024)
const SignedAssignmentLeaseSchema = z
.string()
.min(1)
.max(8 * 1024)
export const AssignmentRequestSchema = z
.object({
@@ -17,7 +24,8 @@ export const AssignmentRequestSchema = z
// Client-declared reconnection; the director verifies it against the
// durable assignment before granting fast-lane admission.
reconnect: z.boolean().optional(),
preferredRegion: RelayRegionSchema.optional()
preferredRegion: RelayRegionSchema.optional(),
regionCorrection: RegionCorrectionRequestSchema.optional()
})
.strict()
@@ -26,7 +34,8 @@ export const AssignmentResponseSchema = z
v: z.literal(1),
cellUrl: CanonicalHttpsOriginSchema,
assignmentEpoch: GenerationSchema,
lease: SignedAssignmentLeaseSchema
lease: SignedAssignmentLeaseSchema,
regionCorrection: RegionCorrectionResponseSchema.optional()
})
.strict()
cloud/packages/relay-contract/src/idle-regional-rehome.ts
+26
View File
@@ -0,0 +1,26 @@
import { z } from 'zod'
import { GenerationSchema, RelayHostIdSchema } from './wire-scalars.js'
export const IdleRegionalRehomeRequestSchema = z
.object({
v: z.literal(1),
attemptId: z.string().uuid(),
userId: z.string().min(1).max(256),
relayHostId: RelayHostIdSchema,
sourceCellId: z.string().min(1).max(128),
sourceCellIncarnation: z.string().uuid(),
sourceAssignmentEpoch: GenerationSchema.refine((value) => value > 0),
sourceGeneration: GenerationSchema.refine((value) => value > 0),
targetCellId: z.string().min(1).max(128)
})
.strict()
export const IdleRegionalRehomeResponseSchema = z
.object({
v: z.literal(1),
outcome: z.enum(['busy', 'committed', 'deferred', 'stale'])
})
.strict()
export type IdleRegionalRehomeRequest = z.infer<typeof IdleRegionalRehomeRequestSchema>
export type IdleRegionalRehomeOutcome = z.infer<typeof IdleRegionalRehomeResponseSchema>['outcome']
cloud/packages/relay-contract/src/index.ts
+2
View File
@@ -13,3 +13,5 @@ export * from './resume-confirmation-contract.js'
export * from './relay-regions.js'
export * from './splice-state-machine.js'
export * from './wire-scalars.js'
export * from './region-correction.js'
export * from './idle-regional-rehome.js'
cloud/packages/relay-contract/src/region-correction.test.ts
+76
View File
@@ -0,0 +1,76 @@
import { describe, expect, it } from 'vitest'
import { AssignmentRequestSchema, AssignmentResponseSchema } from './director-messages.js'
import { DrainSchema } from './control-messages.js'
import { RegionCorrectionRequestSchema } from './region-correction.js'
const report = {
v: 1,
action: 'report',
generation: 3,
assignmentEpoch: 7,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 40, 'asia-east2': 180 }
}
const retention = {
mode: 'finish-existing',
attemptId: '11111111-1111-4111-8111-111111111111',
sourceGeneration: 3,
sourceAssignmentEpoch: 7
}
describe('region correction wire boundaries', () => {
it('keeps legacy assignment shapes readable without negotiated fields', () => {
expect(
AssignmentRequestSchema.parse({ v: 1, relayHostId: 'abcdefghijklmnop' })
).not.toHaveProperty('regionCorrection')
expect(
AssignmentResponseSchema.parse({
v: 1,
cellUrl: 'https://cell.example',
assignmentEpoch: 1,
lease: 'synthetic-lease'
})
).not.toHaveProperty('regionCorrection')
})
it('accepts complete comparison evidence and explicit inconclusive reports', () => {
expect(RegionCorrectionRequestSchema.safeParse(report).success).toBe(true)
const { measurements: _measurements, ...basis } = report
expect(
RegionCorrectionRequestSchema.safeParse({
...basis,
outcome: 'inconclusive',
reason: 'probe-unavailable'
}).success
).toBe(true)
})
it.each([
{ measurements: { 'us-central1': 40 } },
{ measurements: { 'us-central1': -1, 'asia-east2': 10 } },
{ measurements: { 'us-central1': Infinity, 'asia-east2': 10 } },
{ measurements: { 'us-central1': 120_001, 'asia-east2': 10 } },
{ generation: Number.MAX_SAFE_INTEGER + 1 },
{ assignmentEpoch: 1.2 },
{ policyVersion: 2 },
{ outcome: 'inconclusive', reason: 'timeout' }
])('rejects ambiguous or unbounded evidence: %j', (override) => {
expect(RegionCorrectionRequestSchema.safeParse({ ...report, ...override }).success).toBe(false)
})
it('rejects reporting and issuing a window in the same request', () => {
expect(
RegionCorrectionRequestSchema.safeParse({
...report,
action: 'issue-window'
}).success
).toBe(false)
})
it('uses ordinary drain and rejects the superseded retention extension', () => {
const ordinary = { graceMs: 0, recovery: 'resolve-director' }
expect(DrainSchema.parse(ordinary)).toEqual(ordinary)
expect(DrainSchema.safeParse({ ...ordinary, retention }).success).toBe(false)
})
})
cloud/packages/relay-contract/src/region-correction.ts
+60
View File
@@ -0,0 +1,60 @@
import { z } from 'zod'
import { EpochMsSchema, GenerationSchema } from './wire-scalars.js'
import { RelayRegionSchema } from './relay-regions.js'
const RttSchema = z.number().finite().nonnegative().max(120_000)
export const RegionMeasurementsSchema = z
.object({
'us-central1': RttSchema,
'asia-east2': RttSchema
})
.strict()
export const RegionMeasurementWindowSchema = z
.object({
generation: GenerationSchema,
expiresAt: EpochMsSchema,
assignmentEpoch: GenerationSchema,
incumbentRegion: RelayRegionSchema,
policyVersion: z.literal(1)
})
.strict()
const ReportBasis = {
v: z.literal(1),
action: z.literal('report'),
generation: GenerationSchema,
assignmentEpoch: GenerationSchema,
policyVersion: z.literal(1)
}
export const RegionCorrectionRequestSchema = z.union([
z.object({ v: z.literal(1), action: z.literal('issue-window') }).strict(),
z
.object({
...ReportBasis,
outcome: z.literal('conclusive'),
measurements: RegionMeasurementsSchema
})
.strict(),
z
.object({
...ReportBasis,
outcome: z.literal('inconclusive'),
reason: z.string().min(1).max(64)
})
.strict()
])
export const RegionCorrectionResponseSchema = z
.object({
v: z.literal(1),
window: RegionMeasurementWindowSchema.optional(),
reportStatus: z.enum(['accepted', 'duplicate', 'stale', 'expired', 'basis-changed']).optional()
})
.strict()
export type RegionMeasurements = z.infer<typeof RegionMeasurementsSchema>
export type RegionMeasurementWindow = z.infer<typeof RegionMeasurementWindowSchema>
export type RegionCorrectionRequest = z.infer<typeof RegionCorrectionRequestSchema>
export type RegionCorrectionResponse = z.infer<typeof RegionCorrectionResponseSchema>
config/reliability-gates.jsonc
+147
View File
@@ -10,6 +10,81 @@
}
},
"gates": [
{
"id": "runtime.connection-owned-host-status",
"title": "Host status recovers with its owning connection",
"maturity": "experimental",
"protection": "partial",
"owner": "runtime",
"layer": "service-integration-and-e2e",
"surfaces": [
"sidebar host status",
"desktop runtime connection",
"browser primary connection"
],
"platforms": ["macos", "linux", "windows"],
"providers": ["remote-runtime"],
"coveredPlatforms": ["macos"],
"coveredProviders": ["remote-runtime"],
"coverageNotes": "Real authenticated sockets plus isolated desktop and headless hosts with desktop and browser viewers; deterministic lifecycle tests cover stale results and reader deadlines.",
"motivatingLinks": ["https://github.com/stablyai/orca/pull/19163"],
"invariant": "Failed bootstrap and authenticated reconnect converge without UI triggers; one connection owner publishes verified status, with no independent healthy status polling.",
"oracle": "Observe automatic recovery, retained runtime identity on failure, ordered publications, exact request counts, isolated viewer outages, and retirement on disconnect.",
"commands": [
"ORCA_BACKGROUND_LAUNCH=1 pnpm test src/shared/runtime-host-status-owner.test.ts src/main/ipc/runtime-environment-status-recovery.test.ts src/main/ipc/runtime-environment-status-connection.test.ts src/renderer/src/store/slices/runtime-status-snapshot.test.ts src/renderer/src/web/web-runtime-status-owner.test.ts",
"ORCA_BACKGROUND_LAUNCH=1 ORCA_E2E_WEB_CLIENT=1 pnpm exec playwright test tests/e2e/runtime-host-status-recovery.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1"
],
"testFiles": [
"src/shared/runtime-host-status-owner.test.ts",
"src/main/ipc/runtime-environment-status-recovery.test.ts",
"src/main/ipc/runtime-environment-status-connection.test.ts",
"src/renderer/src/store/slices/runtime-status-snapshot.test.ts",
"src/renderer/src/web/web-runtime-status-owner.test.ts",
"tests/e2e/runtime-host-status-recovery.spec.ts"
],
"assertionRefs": [
{
"file": "src/main/ipc/runtime-environment-status-recovery.test.ts",
"assertions": [
"recovers a saved host after its first status check fails, without another UI request"
]
}
],
"evidenceRuns": [
{
"date": "2026-09-10",
"runner": "local",
"platform": "macos",
"command": "ORCA_BACKGROUND_LAUNCH=1 ORCA_E2E_WEB_CLIENT=1 pnpm exec playwright test tests/e2e/runtime-host-status-recovery.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1",
"result": "passed",
"summary": "Desktop-host and headless-host journeys passed with desktop and browser viewers.",
"durationSeconds": 31.3
}
],
"runtimeBudget": {
"p95Seconds": 180,
"scope": "Target excluding builds; measured p95 not established."
},
"flakeHistory": {
"status": "soaking",
"evidence": "Local candidate runs passed; no sustained CI history yet."
},
"redGreenEvidence": {
"status": "partial",
"evidence": "First-status-failure oracle failed on main 58ff95becb40 (one request instead of two) and passes on the candidate. E2E verifies candidate recovery, not a baseline comparison."
},
"performanceBudget": {
"required": false,
"evidence": "Deterministic tests assert one shared request and no healthy owner polling."
},
"promotionCriteria": ["Collect repeated CI runs without unexplained flakes."],
"knownGaps": [
"No live Linux, Windows, SSH, or mixed-version pair validation.",
"TCP interruption exercises reconnect, not a full real host process restart.",
"The outage begins on the first saved-host check, not by relaunching a preseeded desktop profile."
],
"demotionRule": "Keep experimental until repeated runs establish reliability; preserve request-count and lifecycle assertions."
},
{
"id": "mobile-push.headless-startup-and-policy",
"title": "Headless push lifecycle and mobile delivery policy",
@@ -2852,6 +2927,78 @@
],
"demotionRule": "Keep experimental or demote if assignment calls overlap, duplicate drain events bypass backoff, Retry-After is ignored, close resurrects work, or mixed-version request rate exceeds the reviewed director budget."
},
{
"id": "desktop-relay.region-correction-idle-cutover",
"title": "Regional correction moves only an idle relay source",
"maturity": "experimental",
"protection": "partial",
"owner": "desktop-runtime",
"layer": "cell-desktop-real-websocket",
"surfaces": ["regional correction", "idle source cutover", "desktop relay reconnect"],
"platforms": ["macos", "linux", "windows"],
"providers": ["cloud-relay"],
"coveredPlatforms": ["macos"],
"coveredProviders": ["cloud-relay"],
"coverageNotes": "Real local WebSocket/control/proof/splice traffic and production SQLite store run with synthetic clock and synthetic token verification. Separate PostgreSQL16 suites validate SQL concurrency. This does not measure production network latency, physical phones, UI, or the production token issuer.",
"motivatingLinks": [
"docs/relay-region-correction/RELAY-REGION-CORRECTION-IDLE-CUTOVER-PLAN.md"
],
"invariant": "Regional optimization never closes an established relay client. The source gates admissions only after actual work is idle, commits the exact assignment atomically, and releases its empty control. A failed target uses ordinary migration recovery; previously sent mutations are not replayed.",
"oracle": "Two real TCP WebSocket cells, the actual desktop origin pool, SQLite and an independent execution child verify busy phone/iPad deferral, idle movement, racing arrival rejection, definite-abort admission recovery and observed target-registration failure with ordinary rollback.",
"commands": [
"ORCA_BACKGROUND_LAUNCH=1 pnpm test tests/e2e/relay-region-correction.unit.test.ts"
],
"testFiles": ["tests/e2e/relay-region-correction.unit.test.ts"],
"assertionRefs": [
{
"file": "tests/e2e/relay-region-correction.unit.test.ts",
"assertions": [
"releases the empty source and recovers normally when the target never registers",
"rejects an arrival during cutover and restores admissions after a definite failed commit",
"defers for either connected device, then moves after both disconnect without replaying work"
]
}
],
"evidenceRuns": [
{
"date": "2026-09-11",
"runner": "local",
"platform": "macos",
"command": "ORCA_BACKGROUND_LAUNCH=1 pnpm test tests/e2e/relay-region-correction.unit.test.ts",
"result": "passed",
"durationSeconds": 6.44,
"summary": "Three real TCP WebSocket cases passed after removing store retention. Log: .tmp/idle-cutover-review/transport-without-retention.log. Does not validate packaged or physical clients."
}
],
"runtimeBudget": {
"p95Seconds": 180,
"scope": "three local real-WebSocket scenarios with synthetic elapsed time for ordinary recovery"
},
"flakeHistory": {
"status": "unknown",
"evidence": "Local implementation validation; no CI soak history yet."
},
"redGreenEvidence": {
"status": "partial",
"evidence": "Registry admission accounting negative control and five conflicting operation-identity regressions fail before their fixes and pass afterward. Locked database reconciliation has separate red/green evidence. Full cross-layer counterfactual remains unverified."
},
"performanceBudget": {
"required": true,
"evidence": "No retained source lease or new mobile timer. Candidate selection is read-only; busy sources immediately defer. Idle cutover reuses normal desktop reconnect and existing migration recovery."
},
"promotionCriteria": [
"Collect 100 consecutive CI passes or 14 days of soak.",
"Complete mixed-version and packaged-client validation.",
"Validate bounded rollout latency and reliability against reviewed numerical limits."
],
"knownGaps": [
"Production authentication verifier is mocked.",
"Synthetic elapsed time is not a wall-clock soak.",
"Physical phone lifecycle, packaged mixed versions, SSH execution and production network behavior require separate validation."
],
"demotionRule": "Keep experimental or demote if optimization closes an established client, a gate reopens on ambiguous authority, a mutation is replayed, cleanup is lost, or eligible idle hosts starve."
},
{
"id": "git-worktree.refresh-event-semantics",
"title": "Index-only Git metadata cannot trigger structural worktree refresh fanout",
config/scripts/capture-agent-pty-transcript.mjs
+283
View File
@@ -0,0 +1,283 @@
/**
* Records a live agent CLI session through a real PTY into a test fixture, bytes intact.
*
* Why a PTY and not `agy | tee`: a pipe is not a terminal, so the CLI renders its
* non-interactive path — no alternate screen, no caret, no dialogs. The detector under
* test only ever sees the PTY shape, so that is the only shape worth capturing.
*
* Nothing here strips escapes, folds CRs, or rewraps lines: the transcript is written
* exactly as the terminal received it. See docs/reference/agent-pty-transcript-capture.md.
*/
import { createWriteStream, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
import { dirname, join, resolve } from 'node:path'
import { pathToFileURL } from 'node:url'
import {
formatFindings,
redactTranscript,
scanTranscriptForSecrets
} from './pty-transcript-secret-scan.mjs'
const REPO_ROOT = resolve(import.meta.dirname, '..', '..')
const FIXTURE_DIR = join(REPO_ROOT, 'src', 'main', 'runtime', '__fixtures__')
const STOP_KEY = 0x1d // Ctrl-], consumed by the recorder and never forwarded to the agent.
const NAME_RE = /^[a-z0-9][a-z0-9-]*$/
const USAGE = `Capture a raw agent PTY transcript into src/main/runtime/__fixtures__/.
node config/scripts/capture-agent-pty-transcript.mjs --name <fixture-name> [options] -- <command> [args...]
node config/scripts/capture-agent-pty-transcript.mjs --scan <file...> [--redact]
Options
--name <fixture-name> Output fixture name, e.g. antigravity-ready-personal-non-gemini
--out <path> Write somewhere other than the fixture directory
--cols <n> --rows <n> Pin the PTY size (default: this terminal's size, else 120x40)
--duration <seconds> Stop unattended after N seconds
--send "<ms>:<text>" Type <text> into the PTY at <ms> (repeatable; \\r \\n \\t \\e escapes)
--note "<text>" Recorded in the <name>.meta.json sidecar
--scan <file...> Scan existing transcripts for identifiers/credentials and exit
--redact With --scan: rewrite each finding as a same-length placeholder
Press Ctrl-] to end a capture. That key is consumed here, so the agent keeps whatever
dialog it is showing — which is the only way to capture a dialog that owns the screen.`
function parseArgs(argv) {
const options = { cols: null, rows: null, duration: null, scan: [], sends: [], redact: false }
const command = []
let cursor = 0
let afterSeparator = false
while (cursor < argv.length) {
const arg = argv[cursor]
if (afterSeparator) {
command.push(arg)
cursor += 1
continue
}
if (arg === '--') {
afterSeparator = true
} else if (arg === '--redact') {
options.redact = true
} else if (arg === '--help' || arg === '-h') {
options.help = true
} else if (arg === '--scan') {
while (cursor + 1 < argv.length && !argv[cursor + 1].startsWith('--')) {
cursor += 1
options.scan.push(argv[cursor])
}
} else if (arg === '--send') {
cursor += 1
options.sends.push(parseSend(argv[cursor]))
} else if (arg.startsWith('--')) {
const key = arg.slice(2)
cursor += 1
options[key] = argv[cursor]
}
cursor += 1
}
for (const key of ['cols', 'rows', 'duration']) {
options[key] = options[key] == null ? null : Number(options[key])
}
return { options, command }
}
// String.fromCharCode, not a literal: the formatter rewrites an escape sequence into a raw
// control byte in source, which is unreadable and survives badly in diffs.
const ESC = String.fromCharCode(27)
const SEND_ESCAPES = { r: '\r', n: '\n', t: '\t', e: ESC, '\\': '\\' }
/** `"<ms>:<text>"` — a keystroke to deliver at a fixed offset, for an unattended dialog capture. */
function parseSend(value) {
const separator = String(value ?? '').indexOf(':')
if (separator === -1) {
throw new Error(`--send expects "<ms>:<text>", got ${String(value)}`)
}
const atMs = Number(value.slice(0, separator))
if (!Number.isFinite(atMs)) {
throw new Error(
`--send delay must be a number of milliseconds, got ${value.slice(0, separator)}`
)
}
const text = value
.slice(separator + 1)
.replace(/\\(.)/g, (whole, code) => SEND_ESCAPES[code] ?? whole)
return { atMs, text }
}
function runScan(files, redact) {
let failed = false
for (const file of files) {
const path = resolve(file)
const text = readFileSync(path, 'utf8')
if (redact) {
const { text: redacted, redacted: count } = redactTranscript(text)
writeFileSync(path, redacted)
console.log(`${file}: redacted ${count} span(s) in place, same length each.`)
continue
}
const findings = scanTranscriptForSecrets(text)
console.log(formatFindings(file, findings))
failed ||= findings.length > 0
}
return failed ? 1 : 0
}
function resolveSpawn(command) {
// node-pty cannot run a .cmd/.bat shim directly on Windows; those need cmd.exe.
if (process.platform === 'win32' && /\.(cmd|bat)$/i.test(command[0])) {
return { file: 'cmd.exe', args: ['/c', `"${command[0]}"`, ...command.slice(1)] }
}
return { file: command[0], args: command.slice(1) }
}
async function runCapture(options, command) {
const name = options.name
if (typeof name === 'string' && !NAME_RE.test(name)) {
console.error(`--name must be lowercase kebab-case; got ${name}`)
return 2
}
const outPath = options.out ? resolve(options.out) : join(FIXTURE_DIR, `${name}.txt`)
mkdirSync(dirname(outPath), { recursive: true })
const pty = await import('node-pty').catch((error) => {
console.error(
`node-pty failed to load. Build it for plain node first:
node config/scripts/ensure-native-runtime.mjs --runtime=node
${String(error)}`
)
return null
})
if (pty === null) {
return 2
}
const cols = options.cols ?? process.stdout.columns ?? 120
const rows = options.rows ?? process.stdout.rows ?? 40
const { file, args } = resolveSpawn(command)
const term = pty.spawn(file, args, {
name: 'xterm-256color',
cols,
rows,
cwd: process.cwd(),
env: { ...process.env, TERM: 'xterm-256color' },
encoding: null
})
const sink = createWriteStream(outPath)
let recording = true
term.onData((chunk) => {
const bytes = typeof chunk === 'string' ? Buffer.from(chunk, 'utf8') : chunk
// Why recording stops before the kill: an agent repaints an idle frame on its way out, so
// a transcript that keeps writing through shutdown ends on that frame instead of on the
// state you stopped to capture. A mid-turn or dialog capture cannot survive that.
if (recording) {
sink.write(bytes)
}
process.stdout.write(bytes)
})
const wasRaw = process.stdin.isTTY === true && process.stdin.isRaw === true
if (process.stdin.isTTY) {
process.stdin.setRawMode(true)
}
process.stdin.resume()
let stopping = false
const stop = () => {
if (stopping) {
return
}
stopping = true
recording = false
try {
term.kill()
} catch {
// The agent may have exited on its own; the transcript is already on disk.
}
}
process.stdin.on('data', (chunk) => {
if (chunk.includes(STOP_KEY)) {
stop()
return
}
term.write(chunk.toString('binary'))
})
// Why scripted input: a dialog capture has to be driven, and CI (or an agent) has no TTY to
// type into. The keystrokes ride the same PTY a human's would, so the capture is unchanged.
const sendTimers = options.sends.map((send) => setTimeout(() => term.write(send.text), send.atMs))
const durationTimer = options.duration === null ? null : setTimeout(stop, options.duration * 1000)
const exitCode = await new Promise((resolveExit) => {
term.onExit(({ exitCode: code }) => resolveExit(code ?? 0))
})
for (const timer of sendTimers) {
clearTimeout(timer)
}
if (durationTimer !== null) {
clearTimeout(durationTimer)
}
if (process.stdin.isTTY) {
process.stdin.setRawMode(wasRaw)
}
process.stdin.pause()
await new Promise((done) => sink.end(done))
writeMeta(outPath, { command, cols, rows, note: options.note ?? null, exitCode })
const findings = scanTranscriptForSecrets(readFileSync(outPath, 'utf8'))
console.log(`\nTranscript: ${outPath}`)
console.log(formatFindings('scrub check', findings))
if (findings.length > 0) {
console.log(
`Scrub with:
node config/scripts/capture-agent-pty-transcript.mjs --scan ${outPath} --redact`
)
}
return 0
}
function writeMeta(outPath, details) {
const metaPath = outPath.replace(/\.txt$/, '.meta.json')
writeFileSync(
metaPath,
`${JSON.stringify(
{
capturedAt: new Date().toISOString(),
platform: process.platform,
command: details.command,
cols: details.cols,
rows: details.rows,
note: details.note,
exitCode: details.exitCode
},
null,
2
)}\n`
)
}
async function main() {
const { options, command } = parseArgs(process.argv.slice(2))
if (options.help === true) {
console.log(USAGE)
return 0
}
if (options.scan.length > 0) {
return runScan(options.scan, options.redact)
}
if (command.length === 0 || (options.name === undefined && options.out === undefined)) {
console.error(USAGE)
return 2
}
return runCapture(options, command)
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
main().then(
(code) => {
process.exitCode = code
},
(error) => {
console.error(error)
process.exitCode = 1
}
)
}
export { parseArgs, resolveSpawn }
config/scripts/pty-transcript-secret-scan.mjs
+135
View File
@@ -0,0 +1,135 @@
// Finds account identifiers and credentials in a captured PTY transcript before it is committed.
import os from 'node:os'
// Why same-length replacements: a transcript's value is its exact wrapping and column
// alignment. Shortening a redacted span reflows the screen and destroys the evidence.
const EMAIL_DOMAIN = '@example.com'
const PLACEHOLDER_UUID = '00000000-0000-4000-8000-000000000000'
/** Ordered most-specific first; the first pattern to claim a span owns it. */
function buildPatterns() {
const username = os.userInfo().username
const hostname = os.hostname()
const patterns = [
{ kind: 'jwt', re: /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,}/g },
{ kind: 'google-api-key', re: /\bAIza[0-9A-Za-z_-]{20,}/g },
{ kind: 'google-refresh-token', re: /\b1\/\/[0-9A-Za-z_-]{20,}/g },
{ kind: 'vendor-key', re: /\b(?:sk-|ghp_|gho_|github_pat_|xoxb-|xoxp-)[A-Za-z0-9_-]{16,}/g },
{ kind: 'bearer-token', re: /\bBearer\s+[A-Za-z0-9._~+/=-]{16,}/gi },
{ kind: 'email', re: /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g },
// Why a UUID counts: agy prints a resumable conversation id on exit, and installation and
// project ids look the same. They identify the operator's session, not just its shape.
{ kind: 'uuid', re: /\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi },
{ kind: 'opaque-token', re: /\b[A-Za-z0-9_-]{40,}\b/g }
]
if (username.length >= 3) {
patterns.splice(5, 0, { kind: 'local-username', re: literalPattern(username) })
}
if (hostname.length >= 3) {
patterns.splice(5, 0, { kind: 'local-hostname', re: literalPattern(hostname) })
}
return patterns
}
function literalPattern(value) {
return new RegExp(value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g')
}
/**
* @param {string} text raw transcript, escapes intact
* @returns {{kind: string, line: number, column: number, index: number, match: string}[]}
*/
export function scanTranscriptForSecrets(text) {
const claimed = []
const findings = []
for (const { kind, re } of buildPatterns()) {
re.lastIndex = 0
let match = re.exec(text)
while (match !== null) {
const start = match.index
const end = start + match[0].length
if (!claimed.some(([from, to]) => start < to && end > from)) {
claimed.push([start, end])
if (!isAlreadyScrubbed(kind, match[0])) {
findings.push({ kind, index: start, match: match[0], ...locate(text, start) })
}
}
match = re.exec(text)
}
}
return findings.sort((left, right) => left.index - right.index)
}
// Why: a scrubbed fixture must verify clean, so this scanner has to recognise its own
// placeholders — otherwise "prove it's gone" can never pass and the check gets ignored.
const PLACEHOLDER_DOMAIN_RE = /@(?:example\.(?:com|org|net)|localhost)$/i
function isAlreadyScrubbed(kind, match) {
if (kind === 'email') {
return PLACEHOLDER_DOMAIN_RE.test(match)
}
if (kind === 'uuid') {
return match.toLowerCase() === PLACEHOLDER_UUID
}
return /^(.)\1*$/.test(match)
}
function locate(text, index) {
let line = 1
let lineStart = 0
for (let cursor = 0; cursor < index; cursor += 1) {
if (text.charCodeAt(cursor) === 10) {
line += 1
lineStart = cursor + 1
}
}
return { line, column: index - lineStart + 1 }
}
/** Same-length stand-in so redaction cannot reflow the captured screen. */
export function placeholderFor(kind, length) {
if (kind === 'uuid' && length === PLACEHOLDER_UUID.length) {
return PLACEHOLDER_UUID
}
if (kind === 'email' && length > EMAIL_DOMAIN.length) {
return 'u'.repeat(length - EMAIL_DOMAIN.length) + EMAIL_DOMAIN
}
return kind === 'local-username' || kind === 'local-hostname'
? 'x'.repeat(length)
: 'X'.repeat(length)
}
/** @returns {{text: string, redacted: number}} */
export function redactTranscript(text) {
const findings = scanTranscriptForSecrets(text)
let out = ''
let cursor = 0
for (const finding of findings) {
out += text.slice(cursor, finding.index)
out += placeholderFor(finding.kind, finding.match.length)
cursor = finding.index + finding.match.length
}
return { text: out + text.slice(cursor), redacted: findings.length }
}
export function formatFindings(label, findings) {
if (findings.length === 0) {
return `${label}: clean — no account identifier or credential shapes found.`
}
const rows = findings.map(
(finding) => ` ${finding.line}:${finding.column} ${finding.kind} ${preview(finding.match)}`
)
return [`${label}: ${findings.length} finding(s) — scrub before committing.`, ...rows].join('\n')
}
// Why a codepoint test and not a character class: a control-byte range written as an escape is
// folded back into raw 0x00-0x1f bytes by the formatter, which makes this file binary to the VCS
// and leaves the one file gating real PTY data into history unreviewable in a diff.
function preview(value) {
const head = value.length <= 24 ? value : `${value.slice(0, 21)}...`
let printable = ''
for (const char of head) {
printable += (char.codePointAt(0) ?? 0) < 0x20 ? '?' : char
}
return printable
}
config/scripts/pty-transcript-secret-scan.test.mjs
+133
View File
@@ -0,0 +1,133 @@
// The scrub gate is the only thing standing between a live agent transcript and a
// committed account identifier, so it is pinned on the shapes those transcripts carry.
import { readdirSync, readFileSync } from 'node:fs'
import os from 'node:os'
import { join } from 'node:path'
import { describe, expect, it } from 'vitest'
import {
formatFindings,
placeholderFor,
redactTranscript,
scanTranscriptForSecrets
} from './pty-transcript-secret-scan.mjs'
import { parseArgs, resolveSpawn } from './capture-agent-pty-transcript.mjs'
describe('pty transcript secret scan', () => {
it('finds the account row of a ready screen', () => {
const findings = scanTranscriptForSecrets('Antigravity CLI 1.1.17\njin.woo@acme.dev (Business)')
expect(findings).toHaveLength(1)
expect(findings[0]).toMatchObject({ kind: 'email', line: 2, column: 1 })
})
it('finds credentials an agent may echo while signing in', () => {
const kinds = scanTranscriptForSecrets(
[
'token: eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dBjftJeZ4CVP',
'key: AIzaSyA1234567890abcdefghijklmnopqrstu',
'refresh: 1//0gLm34XyZabcdefghijklmnopqrstuvwx',
'Authorization: Bearer abcdefghijklmnopqrstuvwxyz012345'
].join('\n')
).map((finding) => finding.kind)
expect(kinds).toEqual(['jwt', 'google-api-key', 'google-refresh-token', 'bearer-token'])
})
it('flags this machine’s own username, which a prompt line leaks', () => {
const username = os.userInfo().username
const findings = scanTranscriptForSecrets(`~/Users/${username}/orca/repo\n> `)
expect(findings.some((finding) => finding.kind === 'local-username')).toBe(true)
})
it('finds the resumable conversation id agy prints on exit', () => {
const findings = scanTranscriptForSecrets(
'Resume with -c (or command below):\nagy --conversation=26dc1986-9eec-456a-a534-d93e5c1076c2'
)
expect(findings).toHaveLength(1)
expect(findings[0].kind).toBe('uuid')
expect(placeholderFor('uuid', findings[0].match.length)).toMatch(
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[0-9a-f]{4}-[0-9a-f]{12}$/
)
})
it('reports a clean transcript as clean', () => {
const findings = scanTranscriptForSecrets('Antigravity CLI 1.1.17\nSonnet 4.6 (High)\n> ')
expect(findings).toEqual([])
expect(formatFindings('fixture', findings)).toContain('clean')
})
it('claims a span once, so a token inside an email is not double-reported', () => {
const findings = scanTranscriptForSecrets('longlivedaccountname@corp.internal')
expect(findings).toHaveLength(1)
})
it('passes a fixture that is already scrubbed, so "prove it is gone" can succeed', () => {
const scrubbed = `uuuu@example.com\n${'X'.repeat(44)}`
expect(scanTranscriptForSecrets(scrubbed)).toEqual([])
})
})
describe('redaction', () => {
it('replaces every finding with the same number of characters', () => {
// Why length matters: the fixture's value is its exact wrapping. A shorter
// replacement reflows the screen and invalidates the capture.
const text = 'Antigravity CLI 1.1.17\njin.woo@acme.dev (Antigravity Business)\n> '
const { text: redacted, redacted: count } = redactTranscript(text)
expect(count).toBe(1)
expect(redacted).toHaveLength(text.length)
expect(redacted).not.toContain('jin.woo@acme.dev')
expect(scanTranscriptForSecrets(redacted)).toEqual([])
expect(redactTranscript(redacted).redacted).toBe(0)
})
it('keeps a redacted email shaped like an email', () => {
expect(placeholderFor('email', 'a@b.example.com'.length)).toMatch(/^u+@example\.com$/)
})
it('leaves the rest of the screen byte-for-byte untouched', () => {
const text = 'line one\nuser@corp.io\nline three'
expect(redactTranscript(text).text.split('\n')[2]).toBe('line three')
})
})
describe('committed transcripts', () => {
// Why in CI and not just in the recorder: a transcript is committed once and read forever.
// The capture-time warning is skippable; this is not.
const fixtureDir = join(import.meta.dirname, '..', '..', 'src', 'main', 'runtime', '__fixtures__')
const transcripts = readdirSync(fixtureDir).filter((entry) => entry.endsWith('.txt'))
it.each(transcripts)('%s carries no account identifier or credential', (name) => {
const findings = scanTranscriptForSecrets(readFileSync(join(fixtureDir, name), 'utf8'))
expect(formatFindings(name, findings)).toContain('clean')
})
})
describe('capture argv', () => {
it('splits recorder options from the agent command', () => {
const { options, command } = parseArgs([
'--name',
'antigravity-ready-personal-non-gemini',
'--cols',
'120',
'--',
'agy',
'--model',
'sonnet'
])
expect(options.name).toBe('antigravity-ready-personal-non-gemini')
expect(options.cols).toBe(120)
expect(command).toEqual(['agy', '--model', 'sonnet'])
})
it('collects a multi-file scan list', () => {
const { options } = parseArgs(['--scan', 'a.txt', 'b.txt', '--redact'])
expect(options.scan).toEqual(['a.txt', 'b.txt'])
expect(options.redact).toBe(true)
})
it('routes a Windows shim through cmd.exe, which node-pty cannot spawn directly', () => {
expect(resolveSpawn(['agy.cmd', '--model', 'sonnet'])).toEqual(
process.platform === 'win32'
? { file: 'cmd.exe', args: ['/c', '"agy.cmd"', '--model', 'sonnet'] }
: { file: 'agy.cmd', args: ['--model', 'sonnet'] }
)
})
})
config/scripts/windows-cmd-shim-spawn-boundary.test.mjs
+1
View File
@@ -46,6 +46,7 @@ const WINDOWS_SHIM_SPAWN_ALLOWLIST = [
'config/scripts/electron-builder-config.test.mjs',
'config/scripts/ensure-native-runtime.test.mjs',
'config/scripts/live-remote-freeze-rpc.mjs',
'config/scripts/pty-transcript-secret-scan.test.mjs',
'config/scripts/remote-agent-session-authority-repro.mjs',
// Platform-local build paths; the win32 branch is dead code on both.
'config/scripts/build-mac-local.mjs',
docs/assets/readme-downloads.svg
+4 -4
View File
@@ -1,5 +1,5 @@
<svg xmlns="http://www.w3.org/2000/svg" width="106" height="20" role="img" aria-label="downloads: 47m">
<title>downloads: 47m</title>
<svg xmlns="http://www.w3.org/2000/svg" width="106" height="20" role="img" aria-label="downloads: 48m">
<title>downloads: 48m</title>
<linearGradient id="s" x2="0" y2="100%">
<stop offset="0" stop-color="#bbb" stop-opacity=".1"/>
<stop offset="1" stop-opacity=".1"/>
@@ -15,7 +15,7 @@
<g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="11">
<text x="37" y="15" fill="#010101" fill-opacity=".3">downloads</text>
<text x="37" y="14">downloads</text>
<text x="90" y="15" fill="#010101" fill-opacity=".3">47m</text>
<text x="90" y="14">47m</text>
<text x="90" y="15" fill="#010101" fill-opacity=".3">48m</text>
<text x="90" y="14">48m</text>
</g>
</svg>
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 935 B

After

Width:  |  Height:  |  Size: 935 B

docs/reference/agent-pty-transcript-capture.md
+129
View File
@@ -0,0 +1,129 @@
# Capturing an agent PTY transcript
Orca's readiness and blocked-prompt rules are text rules over what an agent CLI paints on a
terminal. They are only as good as the screens they were written against. This is how to record
one, byte for byte, so a rule can be pinned to evidence instead of to a remembered screen.
Related: [`antigravity-readiness-evidence.md`](./antigravity-readiness-evidence.md) names the
specific Antigravity transcripts that are still missing and what each one decides.
## The recorder
```
node config/scripts/capture-agent-pty-transcript.mjs --name <fixture-name> [options] -- <command> [args...]
```
It allocates a real PTY, spawns the agent inside it, mirrors the session to your terminal so you
can drive it by hand, and appends every byte it receives to
`src/main/runtime/__fixtures__/<fixture-name>.txt`. It does not strip escapes, fold `\r`, rewrap
lines, or normalise anything — the file is what the terminal received.
- **Ending a capture:** press <kbd>Ctrl</kbd>+<kbd>]</kbd>. The recorder consumes that key and
never forwards it, which is the only way to end a capture _while a dialog still owns the
screen_. Quitting the agent instead would first dismiss the dialog you came to record.
- `--cols N --rows M` pin the PTY size (default: your terminal's). Wrapping is part of the
evidence, so record the size — the sidecar does it for you.
- `--duration S` stops unattended after S seconds, for a screen that needs no interaction.
- `--send "<ms>:<text>"` types into the PTY at a fixed offset, repeatable, with `\r` `\n` `\t` `\e`
escapes. A dialog capture has to be driven, and an unattended run (CI, or an agent) has no TTY to
type into; the keystrokes ride the same PTY a human's would. For example, the committed
`antigravity-dialog-model-picker.txt` was recorded with
`--duration 24 --send "14000:/model" --send "16000:\r"`, which leaves the picker owning the
screen when the capture stops.
- `--note "<text>"` records the account type, plan, model and CLI version in the sidecar.
- `--out <path>` writes outside the fixture directory (use it for a first dry run).
Each capture also writes `<fixture-name>.meta.json` with the timestamp, platform, command,
PTY size, note and exit code. Commit it with the transcript; the version and account type behind
a screen are not recoverable from the bytes.
**Prerequisite:** `node-pty` must be built for plain Node:
```
node config/scripts/ensure-native-runtime.mjs --runtime=node
```
Orca itself does not need to be running, and the recorder never touches Orca state.
### Platform notes
- **macOS / Linux:** nothing special. `TERM=xterm-256color` is set for the child.
- **Windows:** run it from Windows Terminal / PowerShell, not a Git Bash (MSYS) pane — MSYS
rewrites arguments that start with `/`, which mangles the `cmd.exe /c` hand-off. A `.cmd` or
`.bat` agent shim cannot be spawned by node-pty directly, so the recorder routes those through
`cmd.exe` for you.
- **WSL:** capture _inside_ the distro (run the recorder from the distro's checkout). Recording
`wsl.exe` from the Windows side adds the login-shell banner to the transcript.
- **SSH:** record on the execution host. A transcript recorded locally is not evidence about what
a remote agent prints.
## Privacy: scrub before committing
A live agent screen routinely contains things that must not enter git history:
| Scrub | Why |
| ---------------------------------------------------------------------- | ---------------------------------------------------- |
| Account email / sign-in identifier | The account row on a ready screen prints it verbatim |
| Org, tenant or team name | Identifies a customer |
| Machine hostname and OS username | Appear in prompts, paths and the OSC title |
| Absolute home paths (`/Users/<you>`, `C:\Users\<you>`) | Contain the username |
| JWTs, `AIza…` keys, `1//…` refresh tokens, `Bearer …`, `sk-…`, `ghp_…` | Live credentials; a sign-in screen can echo one |
| Private repo, branch and ticket names | Leak roadmap detail |
| Anything you pasted into the agent during the capture | You typed it; it is in the transcript |
The recorder scans the file as soon as the capture ends and prints every hit with a line and
column. To scrub:
```
node config/scripts/capture-agent-pty-transcript.mjs --scan src/main/runtime/__fixtures__/<name>.txt --redact
```
Redaction replaces each finding with a **same-length** placeholder (`u…u@example.com`, `XXXX…`).
Length matters: a transcript's value is its exact wrapping and column alignment, and a shorter
replacement reflows the screen and destroys the evidence.
### Verify it is gone
1. `node config/scripts/capture-agent-pty-transcript.mjs --scan src/main/runtime/__fixtures__/<name>.txt`
must print `clean` and exit `0`. It recognises its own placeholders, so a scrubbed file passes.
2. Grep for the specifics the scanner cannot know:
`rg -n -i -- "$(whoami)|<your-email>|<your-org>|<your-hostname>" src/main/runtime/__fixtures__/<name>.txt`
3. Read it once with escapes visible: `LC_ALL=C cat -v src/main/runtime/__fixtures__/<name>.txt`.
The scanner matches shapes; only a human catches a project name.
4. Check the sidecar too — `--note` text is free-form and is committed.
`config/scripts/pty-transcript-secret-scan.test.mjs` re-scans every committed
`__fixtures__/*.txt`, so a transcript that skips step 1 fails the suite.
## Consuming a transcript in a test
Feed the raw bytes through the runtime rather than into a matcher directly: escape handling,
tail retention and title tracking all live in `onPtyData`, and a rule tested on pre-normalised
text is tested on something no pane ever sees.
`src/main/runtime/agent-transcript-pane-test-harness.ts` builds the pane;
`src/main/runtime/terminal-interactive-wait-visibility.test.ts` (cursor-agent) and
`src/main/runtime/antigravity-readiness-transcripts.test.ts` (Antigravity) are the two consumers.
## Worked example: the Antigravity captures
The six committed `antigravity-*.txt` fixtures were recorded this way on macOS against
`agy` 1.1.25. Two points generalise:
- **Reach a state without mutating the operator's config.** The ready-screen captures ran in a
directory the CLI already trusted, so no trust answer was written. Where a dialog could only be
reached by signing the operator out or deleting their settings, it was left uncaptured and
recorded as such rather than forced.
- **An environment variable is a legitimate capture knob** where a setting is not.
`AGY_CLI_HIDE_ACCOUNT_INFO=1` produced a second ready screen with no account row, which is
evidence no amount of reasoning about the first screen could have supplied. It changes nothing
on disk.
## Known gap in the existing captures
The three `cursor-agent-*.txt` fixtures contain **no escape bytes and no carriage returns**.
Whatever produced them went through a renderer and a clipboard, so they preserve wording and
box-drawing glyphs but not the caret, the cursor moves, the repaints, or whether the CLI uses the
alternate screen buffer. They are good enough for the wording-based rules built on them and are
not evidence for anything else. New captures made with this recorder keep those bytes; the
Antigravity scaffold asserts their presence so a pasted screen cannot pass as a capture.
docs/reference/antigravity-readiness-evidence.md
+263
View File
@@ -0,0 +1,263 @@
# Antigravity readiness: what the transcripts show
`findAntigravityReadyPromptIndex` in `src/main/runtime/terminal-wait-detection.ts` decides whether
an Antigravity pane is ready for a prompt. It has been written five times, each version tuned
against a five-line screen typed from memory into a `.spec.ts` fixture. Three of the first four
were found worse than the bug they replaced, and the fifth was reverted.
Real transcripts now exist. They were recorded from a live `agy` on macOS with
[`agent-pty-transcript-capture.md`](./agent-pty-transcript-capture.md) and are committed under
`src/main/runtime/__fixtures__/`. `src/main/runtime/antigravity-readiness-transcripts.test.ts`
replays them through the runtime.
**Headline: on real output the current detector is inverted.** It refuses a genuinely ready screen
and accepts a live model picker. The five attempts argued about which extra condition to add; none
of them had noticed that the condition they all shared — a line beginning with the model name —
never matches a real Antigravity ready screen at all.
## Versions
| Thing | Value |
| ------------------------- | ----------------------------- |
| `agy --version` | `1.1.25` |
| Banner printed by the TUI | `Antigravity CLI 1.2.0` |
| Captured | 2026-09-10, macOS, 120x40 PTY |
The binary and its own banner disagree. Any rule keyed to a version string must read the banner,
not `--version`, and must tolerate the two disagreeing.
## What the captures are
| Fixture | What it is |
| -------------------------------------------- | --------------------------------------------------------- |
| `antigravity-ready-api-key-gemini-model.txt` | Ready screen, API-key identity, Gemini 3.7 Flash (Low) |
| `antigravity-ready-account-info-hidden.txt` | The same ready screen with `AGY_CLI_HIDE_ACCOUNT_INFO=1` |
| `antigravity-dialog-trust-workspace.txt` | Workspace trust dialog, live and unanswered |
| `antigravity-dialog-model-picker.txt` | `/model` picker, live and unanswered |
| `antigravity-dialog-command-palette.txt` | Slash-command palette, live and unanswered |
| `antigravity-dialog-dismissed.txt` | `/model` picker dismissed with esc, then settled |
| `antigravity-busy-mid-turn.txt` | A real turn, recording stopped while the spinner was live |
| `antigravity-busy-turn-ended.txt` | The same turn after it ended and the composer returned |
## What could not be captured, and why
Nothing below was faked. Each is a case the recorder could not reach without changing the
operator's account state or configuration, which is out of bounds.
| Missing | Why |
| ------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| `antigravity-ready-business-non-gemini.txt` | This machine has no OAuth session — the CLI prints _"You are currently not signed in"_ and authenticates from `GEMINI_API_KEY`. Reaching a Business ready screen means signing someone in. |
| A non-Gemini model on any ready screen | `agy models` offers 11 models, all Gemini, and `settings.json` pins `modelProvider: gemini`. A non-Gemini row is not reachable from this account. |
| `antigravity-dialog-sign-in.txt` | Unsetting `GEMINI_API_KEY` does not reach the sign-in dialog; the CLI refuses to start because `modelProvider` is pinned. Reaching it means editing the operator's `settings.json`. |
| `antigravity-dialog-theme-picker.txt` | There is no `/theme` command in 1.2.0 (`Unknown command: /theme`). The picker appears only in first-run onboarding, which means deleting the operator's config. |
| `antigravity-dialog-privacy-notice.txt` | First-run onboarding, as above. |
| `antigravity-dialog-update-banner.txt` | Cannot be forced; no update was pending during the session. |
Each remains as a named, skipping case in the suite so it is visible rather than forgotten.
## What the transcripts show
### 1. The ready screen's model row is not at the start of a line
The ready screen prints a block-glyph logo down the left, and the identity, model and path rows are
painted **on the same physical lines as the logo**. What Orca derives is:
```
▀▀▀▀▀▀ Gemini API key
▀▀▀▀▀▀▀▀ Gemini 3.7 Flash (Low)
▄▀▀ ▀▀▄ ~
```
The detector requires `normalized.startsWith('gemini', trimmedStart)` on a trimmed line. The
trimmed line starts with `▀`. It never matches. Measured three ways on the real screen:
| Input | `isKnownReadyPromptPreview` |
| ------------------------------------------------------ | --------------------------- |
| Real ready screen | `false` |
| The same screen with the logo glyphs stripped | `true` |
| Real ready screen followed by the live `/model` picker | `true` |
So the logo — decoration, and suppressible with `AGY_CLI_HIDE_LOGO` — is what decides readiness
today, and the live dialog is what supplies the model line the ready screen could not.
### 2. The dialog is what satisfies the model rule
`/model` prints its options one per line:
```
Gemini 3.8 Flash
> Gemini 3.7 Flash (current)
Gemini 3.1 Pro
```
Those lines _do_ begin with `Gemini`, and a bare `>` composer line sits earlier in the same tail
from before the picker opened. Both halves of the rule are satisfied **while a dialog owns the
screen**, and the pane reads ready. This is the false-ready hazard the last three attempts were
each trying to close, reproduced from a real capture.
### 3. `>` is the dialog selection marker, not only the composer caret
Every dialog uses `>` to mark the highlighted row: `> Yes, I trust this folder`,
`> Gemini 3.7 Flash (current)`, `> /add-dir`. The idle composer is a line whose whole trimmed
content is `>`. That distinction is the only thing separating them, which means the relaxation
proposed in PRs #15840 and #15852 — accept any line _beginning_ with `>` — would make the trust
dialog and the model picker read as ready. On 1.2.0 the idle composer is a bare `>`; those PRs'
1.1.17 mode-banner claim could not be reproduced here and may be mode-specific.
### 4. There is no email account row, and the row can be switched off entirely
For an API-key user the identity row reads literally `Gemini API key`. There is no `@`, no
domain, nothing an account-row rule can key on. Separately, `AGY_CLI_HIDE_ACCOUNT_INFO=1` — a
supported environment variable in the binary — removes the row from a fully ready screen, which
`antigravity-ready-account-info-hidden.txt` captures.
### 5. Dialogs are drawn two different ways, and the banner is never reprinted
The trust dialog and the sign-in splash take the **alternate screen** (`ESC[?1049h` … `ESC[?1049l`).
The model picker and command palette are drawn **in place on the main screen** with erase-to-EOL.
After dismissal the CLI prints `⎿ Exited /model command` and redraws the composer — it does **not**
reprint the banner. The header stays where it was at startup.
### 6. Rows are positioned with cursor addressing, not newlines
The status row is written with absolute and relative moves (`ESC[13;99H`, `ESC[83X ESC[83C`), so
`? for shortcuts` and `Gemini 3.7 Flash · low` end up on one derived line. Any rule that assumes
one screen row equals one `\n`-delimited line is reading a different document than the user sees.
## 8. Busy frames park the caret exactly like idle frames — the spinner is what differs
The frame that ends a turn-in-progress and the frame that ends an idle screen park the cursor with
the **same bytes**. Only the hint row differs, and the park erases it:
```
idle: ? for shortcuts ESC[83X ESC[83C Gemini 3.7 Flash · low CR ESC[2A ESC[2C ESC[?25h
busy: esc to cancel ESC[85X ESC[85C Gemini 3.7 Flash · low CR ESC[2A ESC[2C ESC[?25h
```
So a rule that keys on "the caret is the last thing in the tail" cannot tell busy from idle **on the
frame alone**. What saves it is what comes next. Each spinner tick is its own repaint with its own
park, two rows higher than the frame's:
```
ESC[?25l CR ESC[2A ⣯ Generating ESC[11D ESC[?25h
ESC[?25l CR ESC[2A ⣟ Generating. ESC[12D ESC[?25h
```
That second `CR ESC[2A` splices the composer row away, so the retained tail during a live turn ends
on the spinner row, not on the caret. Measured on `antigravity-busy-mid-turn.txt`:
| Capture | last retained line | bare `>` line present |
| -------------------------------------------- | ------------------ | --------------------- |
| `antigravity-ready-api-key-gemini-model.txt` | `>` | **yes** |
| `antigravity-busy-mid-turn.txt` | `⣟ Generating...` | **no** |
**Consequence for a caret-based rule:** it already answers "not ready" for a real mid-turn capture,
because there is no bare caret in the tail to match. A constructed input that keeps the park bytes
and only edits the status text is not faithful to a live turn — a live turn has a spinner row
repainting _below_ the composer.
**The residual window, and the clause it implies.** Between a frame park and the next spinner tick
the tail does end on the bare caret and is indistinguishable from idle. The gap is one tick
interval. Any readiness path gated on sustained quiescence is safe, because ticks keep arriving and
the pane is never quiet; a path that only inspects retained text is not. For those paths the
evidence supports one clause, and only one:
> **A braille glyph (U+2800–U+28FF) on the last visible line of the retained tail means working.**
That predicate already exists in this file for cursor-agent (`CURSOR_BUSY_SPINNER_RE`) and should be
reused rather than reinvented. It must be scoped to the **last visible line**, not the whole tail:
a first-run transcript prints `⠾ Signing in...` during startup, which would otherwise pin a ready
screen as busy forever.
Nothing else in the capture distinguishes the two states. The hint row (`esc to cancel` versus
`? for shortcuts`) is erased by the park in both cases, the park offsets are identical, and
`ESC[?25l`/`ESC[?25h` fencing appears around every repaint, idle or busy.
## Confirmed / refuted, by attempt
Evidence column names the fixture; all quoted text is from the committed transcripts.
### Attempt 1 — the rule at HEAD
| # | Claim | Verdict | Evidence |
| ---- | -------------------------------------------------------- | --------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- |
| 1.1 | A ready screen prints the banner `Antigravity CLI` | **Confirmed** | `Antigravity CLI 1.2.0` in both ready fixtures |
| 1.1b | …and its last occurrence in the tail is the live one | **Refuted** | The trust dialog's own body says _"Antigravity CLI requires permission to read, edit, and execute files here"_, so `lastIndexOf` lands inside the dialog |
| 1.2 | The model row begins with the vendor word `Gemini` | **Refuted** | `▀▀▀▀▀▀▀▀ Gemini 3.7 Flash (Low)` — the logo precedes it; never at line start |
| 1.3 | The caret line's whole trimmed content is `>` | **Confirmed** on 1.2.0 idle | bare `>` in both ready fixtures |
| 1.3b | …and only the composer prints `>` | **Refuted** | `> Yes, I trust this folder`, `> Gemini 3.7 Flash (current)`, `> /add-dir` |
| 1.4 | A ready screen prints the workspace path on its own line | **Refuted** | the path shares its line with logo glyphs (`▄▀▀ ▀▀▄ ~`) |
### Attempt 2 (loop 1) — blacklist the model line
| # | Claim | Verdict | Evidence |
| --- | ------------------------------------------ | ----------- | ---------------------------------------------------------------------------------------------------------------- |
| 2.1 | Dialog model-row wording is enumerable | **Refuted** | the palette lists 50+ commands with free-form descriptions; the picker prints whatever models the account offers |
| 2.2 | A dialog never reproduces a real model row | **Refuted** | the `/model` picker prints four real model rows, one per line, at line start |
### Attempt 3 (loop 2) — structural ordering on `headerIndex`
| # | Claim | Verdict | Evidence |
| --- | -------------------------------------------------- | ---------------------------------- | ---------------------------------------------------------------------------------------------------- |
| 3.1 | A live dialog is printed below the ready chrome | **Confirmed** for in-place dialogs | picker and palette append below the composer |
| 3.2 | The banner is reprinted when a dialog is dismissed | **Refuted** | `antigravity-dialog-dismissed.txt` shows `⎿ Exited /model command` and a redrawn composer, no banner |
| 3.3 | Antigravity does not use the alternate screen | **Refuted** | `ESC[?1049h` opens the trust dialog and the sign-in splash |
| 3.4 | No full repaint per keystroke | **Partly refuted** | typing `/mod` repaints the palette region on each keystroke with `ESC[K` |
Because of 3.2, `headerIndex` cannot be the anchor: it never advances. Ordering can only be
expressed against the model/caret positions, which is what 1.2 and 1.3b just invalidated.
### Attempt 4 (loop 3) — require a positive account row
| # | Claim | Verdict | Evidence |
| --- | ---------------------------------------------------- | ---------------------- | --------------------------------------------------------------------------------------------------------------------------- |
| 4.1 | Every ready screen prints an account row | **Refuted, twice** | API-key identity prints `Gemini API key` (no `@`); `AGY_CLI_HIDE_ACCOUNT_INFO=1` removes the row entirely |
| 4.2 | A startup dialog never contains an `@`-and-`.` token | **Not reachable here** | none of the captured dialogs contains one, but the palette shows free-form skill descriptions, which are user-authored text |
| 4.3 | The account row is distinguishable from prose | **Refuted** | the row is not a distinct line; it shares one with the logo |
### Attempt 5 (PR #19749, reverted) — ordering + account row
| # | Claim | Verdict | Evidence |
| --- | -------------------------------------------------------- | ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| 5.1 | Ordering plus an account row separates ready from dialog | **Refuted** | the account row is optional (4.1) and the ordering anchor never moves (3.2) |
| 5.2 | Executing both builds was sufficient verification | **Refuted** | the executed input was the hand-written fixture, so the check reproduced the fixture's assumptions. The real screen disagrees with that fixture on the model row, the path row and the account row |
| 5.3 | The wedge is a model-name problem | **Refuted** | it is a line-start problem. Even `Gemini 3.7 Flash (Low)` — a Gemini model — fails, because a logo glyph precedes it |
### Cross-cutting
| # | Question | Answer |
| --- | ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- |
| X1 | Does `agy` set an OSC title distinguishing busy from idle? | **No.** Not one OSC title sequence appears in any capture. Title-based readiness is unavailable for this agent |
| X2 | Does it repaint with bare `\r`? | **Yes**, constantly, plus `ESC[K` and absolute cursor moves |
| X3 | Does the caret survive in the tail? | **Yes** — a bare `>` line is present in every ready capture |
| X4 | Banner-to-caret distance | ~8 derived lines on a 120x40 PTY; the banner falls outside the 6-line preview window, so only the full retained tail can see it |
| X5 | Pane title on the trust screen versus ready | Identical: none |
## Can attempt six be written?
Yes — but not as a variation on any of the five. Every one of them refined a predicate over
`\n`-delimited lines, and that is the layer where the evidence says the information is not.
What the captures support:
- **The one stable, dialog-free ready marker is a line whose entire trimmed content is `>`.** It is
present in every ready capture and absent from every dialog capture, because a dialog's `>` always
carries its selected row's label. This is a much narrower rule than any attempt used, and it is
the only one that survived contact with the transcripts.
- **Drop the model-row requirement.** It matches dialogs and not ready screens. Keeping it inverted
the detector.
- **Do not require an account row.** It is optional by environment variable and carries no email for
API-key users.
- **Do not anchor on `headerIndex`.** The banner is printed once and never reprinted.
- **The blocked-signal path already works** for the trust dialog: `antigravity-dialog-trust-workspace.txt`
is correctly refused today, by wording, not by structure.
What is still unknown and should be captured before shipping: the sign-in, theme, privacy and
update dialogs, and any ready screen where the composer is not idle (accept-edits and plan mode,
which PRs #15840 and #15852 describe from a screenshot). A bare-`>` rule is only as good as the
claim that those modes still end on a bare `>`; that claim is untested.
The honest summary is that this is a screen-shaped problem being solved with line-shaped tools. A
rule over the derived tail can be made much better than what ships today, but the durable fix is to
ask the terminal emulator what the bottom row of the screen actually is, rather than inferring it
from a byte stream that was written with cursor addressing.
docs/relay-region-correction/RELAY-REGION-CORRECTION-ACCEPTANCE.md
+114
View File
@@ -0,0 +1,114 @@
# Idle regional correction acceptance
Updated 2026-09-11. Scope: [idle-cutover plan](RELAY-REGION-CORRECTION-IDLE-CUTOVER-PLAN.md).
Local implementation is based on main `74cc9b50390b481009b34823a35eee01a5b90e40`,
with uncommitted changes atop `08c0802089a9186cf48ce0d168cd87565bd92d65`.
**Local validation is green; the published PRs are not updated or merge-ready.**
Cloud draft20037 still points to `0462a87011d8783f91f9a9e977ea7a68437a0d5b`;
desktop draft20031 still points to `f9e5b0193448e6c524603836b9278878b96cf51c`.
Their older CI is not evidence for this implementation. No commit, push, merge,
deployment, workflow dispatch or production mutation occurred in this continuation.
## What the implementation proves
The source permits optimization only when actual client sockets, splices, pending
connections and in-flight admission/control work are absent. It installs the local
gate before awaiting a constrained assignment transaction. Duplicate requests bind
exact authority; ambiguous database outcomes keep the source fenced until locked
reconciliation establishes the result. After commit, it releases the empty control
and uses ordinary reconnect and migration recovery. Emergency drains retain their
existing behavior. No retained-source protocol or restoration loop remains.
Three real TCP WebSocket scenarios join authenticated local HTTP dispatch, two
cells, the actual desktop origin pool, SQLite and an independent execution child:
1. Either connected device prevents movement; after both leave, target reconnect
succeeds and the durable append-once mutation oracle shows no replay.
2. A racing arrival receives4409; a definite failed commit restores source admission.
3. The target control connection is actually attempted and fails; source activity
is released and ordinary expiry recovery allows source epoch3 reconnect.
These use synthetic time and token verification. They do not prove physical mobile
background scheduling, production authentication/network behavior or user latency.
## Commands and results
Every test/app command uses `ORCA_BACKGROUND_LAUNCH=1`. Cloud commands run from
`cloud/apps/relay`; root commands run from this worktree. All logs below are under
`.tmp/idle-cutover-review/`.
| Scope | Command | Result / log |
| --- | --- | --- |
| Cloud | `ORCA_RELAY_TEST_POSTGRES_URL='postgresql://postgres@127.0.0.1:55440/postgres?options=-csearch_path%3Didle_full_root_20260911' ORCA_IDLE_REHOME_POSTGRES_URL='postgresql://postgres@127.0.0.1:55440/postgres' ORCA_REGION_CORRECTION_POSTGRES=1 pnpm exec vitest run --no-file-parallelism` |77 files /682 passed /zero skips; `cloud-full-postgres-idle.log` |
| Cloud | `pnpm run typecheck` | Passed; `cloud-typecheck-after-preview.log` |
| Cloud | `pnpm build` | Passed; `cloud-release-build-idle.log` |
| Root | `pnpm test src/main/runtime/relay` |20 files /177 passed; `desktop-relay-full-idle.log` |
| Root | `pnpm test tests/e2e/relay-region-correction.unit.test.ts tests/e2e/relay-region-compatibility.unit.test.ts` |16 passed; `transport-contract-cleanup.log` |
| Root | `pnpm test src/main/global-fetch-call-site-audit.test.ts tests/e2e/relay-region-correction.unit.test.ts` |4 passed after CI fixes; `ci-gaps-green.log` |
| Root | `pnpm tc:node` | Passed; `node-final-idle.log` |
| Root | `pnpm exec oxlint src/main/runtime/relay tests/e2e/relay-region-correction.unit.test.ts tests/e2e/relay-region-compatibility.unit.test.ts` | Passed; `desktop-lint-idle.log` |
| Root | `pnpm run check:code-quality:changed` | Passed,0 new findings across30 changed files; `code-quality-changed-idle.log` |
| Root | `pnpm run check:reliability-gates` |121 manifest gates passed; `reliability-idle.log` |
| Root | `ORCA_E2E_SSH_DOCKER=1 pnpm exec playwright test tests/e2e/ssh-docker-transport-drop-recovery.spec.ts tests/e2e/paired-remote-terminal-serve-restart-binding.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1` |7 passed after fresh build; `ssh-folder-idle.log` |
| Root | `node --test cloud/dev/scripts/deploy-relay-blue-green.test.mjs cloud/dev/scripts/read-relay-serving-regional-placement-version.test.mjs cloud/dev/scripts/relay-regional-rehome-workflow.test.mjs` |47 passed; `deployment-guards-idle.log` |
PostgreSQL16.15 reused the existing `orca-region-release-pg16` container on55440.
The suite used an isolated schema, dropped afterward (`cloud-full-postgres-cleanup.log`);
new concurrency tests create and clean independent schemas. No other PostgreSQL
port was used. Root oxlint ignores cloud; the configured cloud lint is TypeScript.
The seven background Electron checks cover paired folder-capable binding across
serve restart and six real Docker SSH recovery journeys: live pane, output bounds,
host-proven exit, repeated restarts, frozen-host silence and resumed input. They do
not exercise a physical phone-to-SSH regional cutover or packaged upgrade.
## Regression and review evidence
- Reconciliation: four cases fail against constant not-committed, then pass with
locked authority checks (`reconciliation-{red,green}.log`).
- Registry: disabling pre-await admission accounting makes the arrival race fail;
restoring it passes. Five conflicting operation-ID authority tuples fail before
the identity fix, then all48 registry tests pass (`operation-tuple-{red,green}.log`).
- SQLite startup capability upgrade: red before upgrade logic;8 database tests pass
afterward. Legacy controls default to not idle-capable.
- The commit placeholder negative control was run after implementation; it is
counterfactual evidence, not a claim of chronological test-first development.
- Full PostgreSQL verification supersedes intermediate preview/legacy-test failures.
An agent's earlier PostgreSQL safety-latch discrepancy was disproven in an
isolated schema and explicitly withdrawn.
- Fifth GPT-6-astra low audit: **APPROVE within implementation scope**, no new blocker.
Reviewed source barriers, exact duplicates, locked ambiguity and worker progress.
Reviewer had migrated PostgreSQL tests, but did not author the core implementation.
Ledger: `.tmp/idle-cutover-review/review-ledger.md`. No sixth cycle started.
## CI preparation and review artifacts
Old desktop CI failed on a stale global-fetch inventory count and missing `pg` for
the transport test. The count reproduces locally; the downstream catalog/probe
consumers already consume/cancel bodies, so the audited count is corrected.
The unit workflow installs locked cloud relay dependencies and builds their contracts.
From `cloud/`, both commands pass (`ci-relay-dependencies.log`):
- `npx --yes pnpm@10.24.0 --filter '@orca-cloud/relay...' install --frozen-lockfile --ignore-scripts`
- `npx --yes pnpm@10.24.0 --filter '@orca-cloud/relay^...' build`
Local split patches and draft bodies are in `.tmp/idle-cutover-review/`:
`cloud-idle.patch`, `desktop-idle.patch`, `cloud-pr-body.md`, `desktop-pr-body.md`.
`prepare-split.py` verifies ordered application against the stated main baseline;
`split-manifest.json` identifies the combined tree. The actual index/branches remain
unchanged. The HTML explainer is `.tmp/relay-region-explainer.html`; four stages,
failure toggle, light/dark mobile/desktop layout and browser-error checks pass.
## Remaining acceptance gaps
- Update the two draft PRs and verify fresh CI for their exact heads. The original
handoff explicitly prohibited pushes; publication needs authorization.
- Packaged mixed-version desktop/mobile, physical-device lifecycle and platform
transport remain unverified. Pinned wire tests are narrower evidence.
- CI soak and production RTT/interaction benefit remain unmeasured. Correction
defaults off; deployment/enablement require the reviewed rollout procedure.
- Archive of intermediate/superseded evidence:
`.tmp/idle-cutover-review/acceptance-history-before-final.md` and pre-rescope branch
`relay-region-before-idle-implementation`. Earlier retention results do not prove
the idle design or repair the superseded live-retention transport case.
docs/relay-region-correction/RELAY-REGION-CORRECTION-API.md
+61
View File
@@ -0,0 +1,61 @@
# Idle regional correction contracts
Updated 2026-09-11. The [idle-cutover plan](RELAY-REGION-CORRECTION-IDLE-CUTOVER-PLAN.md)
is authoritative. This replaces the former retention/restoration protocol.
## Desktop measurement and capability
The optional assignment `regionCorrection` namespace supports `issue-window` and
`report`. The server issues a generation, fixed expiry, assignment epoch, incumbent
region and policy version1. A report supplies measurements for both regions or an
inconclusive reason. The first accepted report remains immutable; stale or changed
assignment evidence cannot authorize a move. Legacy placement hints remain separate.
Desktop advertises `idle-regional-rehome-v1` in its control capability header.
There is no new mobile message, retained-control lease, or restoration notification.
The cell persists the exact activity, generation, assignment and incarnation with
`idle_regional_rehome`. The obsolete `finish_existing` database column is always
written0 for schema compatibility and is not an eligibility signal.
## Director to source cell
`POST /v1/admin/host-idle-rehome` requires the configured director identity and
matching source cell/incarnation. The strict request contains:
- `v:1`, stable UUID `attemptId`, `userId`, `relayHostId`;
- `sourceCellId`, `sourceCellIncarnation`, `sourceAssignmentEpoch`, `sourceGeneration`;
- `targetCellId`, authenticated `cohortPercent`, and fresh `directorSafety`.
The strict response is `{v:1,outcome}` with `busy`, `committed`, `deferred`, or
`stale`. A lost HTTP reply is ambiguous and does not consume a dispatch-failure
budget. Repeat delivery retains the same operation identity. Cell status advertises
regional protocol3; new selection requires both cells at protocol3 or newer.
## Store and source ownership
`selectIdleRegionalRehomeCandidates` is read-only. It checks fresh evidence,
capability, policy, cooldown, telemetry and target capacity; bounded rotating pages
allow progress past busy hosts. Selection does not prove physical idleness.
`HostSessionRegistry.idleRehome` accounts for accepts, attaches, control commands,
and activation before their first await. Only an idle source installs its admission
gate. New arrivals receive normal retryable routing failure. Conflicting reuse of
an operation ID cannot share another authority tuple's result.
`commitIdleRegionalRehome` rechecks the exact request under existing locks, including
policy, cohort, capacity, global rate and concurrency. It atomically reserves the
target, advances assignment and records the attempt/source generation. It does not
choose a different host or destination. Completion is recorded at the source;
ordinary migration refresh, completion and expiry recovery remain responsible for
the target-registration lifecycle.
`reconcileIdleRegionalRehome` locks the assignment before reading the operation.
It distinguishes committed, not-committed with unchanged live source authority,
and stale authority. Missing data after an unlocked read is never rollback proof.
A database error leaves admissions fenced while reconciliation retries. Successful
cutover closes/releases the empty source control; the desktop resolves its normal
assignment and reconnects. Definite rollback reopens only the same source authority.
Outcome reporting aggregates actual attempts and migrations by cell/state, without
host identities or a retained-source table. A completed idle move does not authorize
closing future clients attached to the target.
docs/relay-region-correction/RELAY-REGION-CORRECTION-CHECKLIST.md
+47
View File
@@ -0,0 +1,47 @@
# Relay region correction — implementation checklist
> **Scope: idle-only correction.** Superseded retention history is preserved in
> `.tmp/idle-cutover-review/checklist-history-before-final.md` and the backup branch.
> Follow [idle-cutover plan](RELAY-REGION-CORRECTION-IDLE-CUTOVER-PLAN.md).
## Idle-only implementation tracker
- [x] Resolve plan review findings and obtain Astra approval (revision 2).
- [x] Preserve pre-rescope revision on `relay-region-before-idle-implementation`.
- [x] Endpoint authentication and incarnation check: deterministic red/green.
- [x] Worker selects before cutover; busy deferral and lost reply tests: red/green.
- [x] Source barrier covers accepts, attaches, commands and control replacement (48 registry tests, including conflicting operation-ID authority tuples; red/green evidence in acceptance).
- [x] Constrained assignment commit and locked ambiguous-outcome reconciliation (11 PostgreSQL 16 tests on 55440, including both replacement orders and a lost commit reply).
- [x] Desktop removes live-retention handling, retains fresh decisions and fallback (53 focused desktop/compatibility tests; node typecheck passes).
- [x] Remove superseded cloud retention protocol/store/cleanup and obsolete tests. The legacy database capability column remains written as0 for existing schema compatibility; it enables no behavior.
- [x] Real two-cell transport: two clients, quiet connection, idle move, arrival race, and observed target-registration failure with ordinary recovery (3 tests pass).
- [x] Focused PostgreSQL transaction/concurrency checks on 55440 (11 passed).
- [x] Full local relevant cloud/desktop suites: 682 cloud tests with PostgreSQL,177 desktop relay tests,16 transport/compatibility tests.
- [x] Local pinned-wire compatibility, Docker SSH/folder continuity (7), types, lint and reliability manifest. Packaged/device/platform gates remain open.
- [x] Fifth Astra low implementation audit: APPROVE within the documented scope.
- [ ] Rewrite and validate cloud/desktop PRs, CI and final acceptance evidence.
These are merge-readiness tasks. Device/package/platform and production rollout
requirements remain explicit gaps until independently evidenced.
Current transport disposition: **the replacement idle-only transport suite is green
(3 tests), but the PRs are not ready**. Local cloud/desktop verification, final implementation audit, SSH/folder evidence
and reliability docs are complete. PR updates, fresh CI and release evidence remain.
The full cloud suite passes682 tests with PostgreSQL16 and no skips; cloud typecheck passes. Final audit and remaining end-to-end/PR tasks are still open. Exact commands/results are at the top of the acceptance document. The original
live-retention rollback assertion is superseded by the approved product rescope;
these results do not claim that old design was repaired.
## Publication and release gates
- [x] Prepare separate cloud/desktop patches and concrete PR descriptions locally.
- [x] Reproduce and address old CI failures: fetch audit count and cloud test dependencies.
- [ ] Obtain authorization to publish under the original no-push handoff constraint.
- [ ] Update cloud draft20037 and desktop draft20031; verify exact-head CI.
- [ ] Packaged mixed-version desktop/mobile and physical-device lifecycle.
- [ ] Linux/Windows transport evidence, CI soak, bounded rollout and measured benefit.
No production mutation or deployment occurred. Local passing tests and audit approval
are not a claim that the current published PRs are ready or the feature is deployed.
The acceptance document lists commands, evidence scope and remaining gaps.
docs/relay-region-correction/RELAY-REGION-CORRECTION-IDLE-CUTOVER-PLAN.md
+213
View File
@@ -0,0 +1,213 @@
# Relay region correction — idle cutover plan
Status: **REVISION 2 — DESIGN APPROVED; implementation and verification pending.**
Independent GPT-6-astra low review found no remaining design blocker. Approval is
conditional on the implementation gates below: pre-await ownership accounting,
shared assignment-row serialization, locked ambiguous-outcome reconciliation,
operation identity deduplication and late-callback fencing must be proven in tests.
Four independent idle-scope audits have been counted conservatively (scope, plan v1,
simplification, revision 2). If another plan revision fails review and a sixth cycle
would be needed, stop for user re-evaluation. Approval does not mean the current
live-retention PRs implement this design or are ready to merge.
## Requirement and deliberate tradeoffs
Automatically improve the host's relay region when no relay clients are using it.
A phone and iPad both disconnecting can create an opportunity. Continuous clients
can postpone optimization indefinitely. Returning during a move may incur ordinary
reconnect delay; zero-delay reconnect is NOT a requirement. Do not deliberately
close an established client connection to optimize latency. Direct LAN sessions
and running terminal processes do not themselves count as relay clients.
The mobile 30-second background grace is not a reliable scheduling event: the OS
can delay it. Use actual source-cell connection state. Desktop control remains
open without phones; do not wait for it to disappear naturally.
Keep fresh desktop measurements, incumbent-relative 25ms AND 20% improvement,
cohort/enable controls, capacity checks and cooldown. Probe improvement is not
proof of improved application latency. Cloud selects authoritative assignments;
mobile reconnect reads an assignment, it does not choose a faster region.
## Simplicity constraints
Reuse the regional worker, assignment transactions, reservations, request identity,
cell admission registry and normal desktop reconnect. No live-source retention,
splice transfer, recurring retained-control lease, new mobile protocol, global idle
poller, or desktop background-disconnect timer. Existing data sockets are never
copied or transferred (the former design did not transfer them either).
Try an idle opportunity, immediately defer if busy; do not keep a gate waiting for
users to leave. The gate exists only for a short attempted cutover. A retryable
arrival may lose this race and reconnect normally. Do not count that as evidence
of lost execution or replay a previously sent mutation.
Source cells already share the assignment database. Use one source-owned local
barrier around a constrained assignment transaction; do NOT introduce a distributed
prepare/commit protocol, a durable preparation table, or a second admission service.
The director selects candidates; the source's transaction rechecks director policy.
## What is idle?
The source owns one per-host/generation admission barrier covering every accept,
attach and control replacement path. Inventory these paths before changing them.
The quiescence predicate must require:
- `activeConnIds.size === 0` (includes attaches while persistence awaits).
- `activeSplices.size === 0` and `pendingConns.size === 0`.
- Zero accepts in flight before insertion in those maps. Track ownership before
the first asynchronous operation that can admit this host; every exit releases it.
- Zero in-flight control operations that install credentials, mutate connection
basis or create admissions. Track actual server handlers; do not invent an
approximate count from RPC traffic or old database leases.
An authenticated open desktop control socket, its ping/pong and activity renewal
are NOT client work and NOT a reason to defer. They can remain until cutover closes
the empty session. A desktop request arriving after the barrier gets a normal
retryable transport failure; finish previously accepted state mutations before
claiming idle. Reconnection must preserve pairing and not blindly replay mutations.
Missing/expired database activity leases are never proof of idle. Outstanding
reservation cleanup remains owned and must be completed or safely fenced, but
waiting for every lease to expire would wrongly wait on the healthy control lease.
## Source-owned try-cutover
1. The director selects candidates read-only, using existing eligibility and pacing.
It sends an authenticated idle-cutover request with stable operation ID, expected
source assignment epoch/incarnation/generation and candidate target. Selection
does not reserve capacity or change assignment. Repeat delivery uses the same ID.
2. Source validates the request and current session. In one synchronous segment,
check all counters and install a per-host barrier. If busy, return `busy` without
closing sockets or holding a barrier. Busy is a deferral, not a dispatch failure.
The next worker pass must progress past busy candidates rather than starve others.
3. While barred, reject new clients and any new source control activation/rebind.
Late asynchronous continuations must recheck barrier/session after awaits and
release abandoned reservations. Install this fence before the first await of
the cutover. No barrier timer may reopen admissions on its own.
4. Source calls a constrained version of the existing assignment transaction. Reuse
its lock order, global rate/concurrency controls, cooldown, capacity reservation,
freshness and safety checks. Recheck exact source epoch/incarnation/generation,
target and operation identity. Reserve target, update assignment/epoch and record
the existing durable migration/attempt atomically. Do not call today's unrestricted
`claimRegionalRehome` and let it choose a different host. No network calls inside
DB locks. Zero sockets is established locally, not inferred from activity leases.
5. If committed, retire the still-empty source session/control via the existing
resolve-director closure path and release its activity. Desktop uses normal
reconnect and registers on target. Reply with the durable operation outcome.
6. If definitively not committed and source authority remains unchanged, remove the
barrier and continue on the original control. No target reservation survives a
rolled-back transaction. If authority changed, retire the obsolete source instead.
A timeout or transport error is NOT definitive rollback.
## Ambiguous transactions, restarts and failures
The request operation ID is known BEFORE the transaction and recorded as the
existing attempt ID on commit. Duplicate calls return its outcome and never start
another migration. A concurrent retry, cancellation or definitive-abort check must
serialize under the same host lock as commit; an unlocked absent-row lookup is
insufficient because the original transaction could still commit later.
Keep a barrier until the database transaction is known terminal and a locked
reconciliation establishes the outcome. If the driver result is ambiguous, retry
status through a per-attempt backoff callback, using the same identity. If durable
access is unavailable, remain fenced; bounded availability cannot be promised
while the assignment's authority is unknown. No timeout-only reopen. Use the
existing DB transaction timeout and request timeout, not a new renewable gate lease.
A lost director HTTP reply does not interrupt source-owned completion: source
finishes its transaction, reads durable outcome and closes/reopens locally. A
retry from any director sees the same operation. Director crashes do not strand
preparations because no separate preparation exists.
A source restart/replacement control is fenced by existing authoritative registration
and activity validation. It must serialize against the cutover transaction under
host locks, validate the current assignment and reject old source ownership if the
commit won. If replacement won, the old transaction must fail its generation/
incarnation recheck. This requires tracing current registration persistence and
proving the shared serialization point, not relying solely on the in-memory fence.
Late callbacks from a closed session cannot reopen admissions for its replacement.
Emergency drain invalidates local authority and participates in this serialization;
a cutover already committed follows its outcome, never reopens the emergency source.
After commit, target registration failure uses ordinary bounded migration recovery.
The old empty control must be released even if outcome delivery failed; otherwise
current rollback refuses while source activity remains (`assignment-store.ts:6923`).
Source process death is handled by normal activity expiry and cell incarnation
fencing. No live clients were discarded, but a returning client may wait for recovery.
Once clients attach at target, preserve them under ordinary assignment rules:
initial source idleness never authorizes closing future target clients.
## Compatibility and authority
Mobile already re-resolves on `WRONG_CELL` (4409) through
`dialRelayThroughDirectorFallback`; use that existing close code for arrivals at a
gated source. Before commit, resolution can still return the old address: existing
backoff must prevent tight retry loops. After commit it returns the target. Pin
old parser/client fixtures and test the actual codes; do not assume every error is
retryable. Do not introduce a new mobile close code or protocol message.
Empty host control closure uses the existing `DRAINING` / resolve-director path;
verify its reconnect behavior against the baseline desktop implementation.
Negotiate a distinct idle-cutover capability for participating cells and updated
desktops; do not reuse finish-existing capability to imply this new behavior.
Unsupported participants skip optional correction. Preserve old-server HTTP-400
fallback for measurement fields. Emergency drain/auth enforcement can invalidate
any in-flight cutover; it must fence late commit and preserve existing hard deadlines.
Disabling correction stops new attempts; existing ones still reconcile.
## Review and implementation gates
Review must verify the shared-store serialization and identify every admission and
control mutation path before approving implementation. Minimum tests (red before
green for new guarantees):
1. Phone remains while iPad disconnects: no move. Both disconnect: move possible.
A quiet established socket or expired DB splice lease still prevents a move.
2. Accept before/after barrier, accept awaiting activity persistence, attach awaiting
basis persistence, and credential mutation crossing the barrier. No late attach,
leaked reservation or interrupted established client.
3. Busy attempt leaves source admissions usable; repeated busy hosts do not starve
idle candidates or consume dispatch-failure budget.
4. Commit/replacement race; lost database/HTTP replies; timeout during transaction;
duplicate workers; director crash; source restart; replacement control; stale
epoch/incarnation; database outage and recovery. No timeout-only reopening.
5. Target failure before registration and after new client attachment; source-control
release; ordinary recovery completes without retained-source restoration.
6. Current/old mobile reconnect before and after commit, same-address retry pacing,
pairing preservation and no mutation replay. Old/new desktop/cloud combinations.
7. Emergency drain/auth denial during the cutover; no altered hard-drain behavior.
8. Real TCP WebSockets for two clients, admission race and failed cutover, independent
execution-process identity and append-once mutation evidence. Docker SSH and
folder workspace continuity. Tests use `ORCA_BACKGROUND_LAUNCH=1`.
Keep tests proportional: deterministic component races first, then real transport,
relevant cloud/desktop suites, types/lint and PR CI. PostgreSQL only on 55440 when
validating authoritative transactions. Do not replace a failing oracle with a weaker
assertion or accumulate tests mirroring implementation.
After clean review, replace superseded feature code/tests/docs on the two draft PRs,
preserving an immutable backup. Freshness and appropriate compatibility tests stay;
retention-only mechanisms and release requirements must be removed if irrelevant.
Do not claim readiness from tests of the superseded design.
Release separately from merge readiness: packaged mixed versions, physical device
background timing, Linux/Windows, bounded rollout and measured user benefit remain
explicit evidence requirements. No deployment or enable is authorized by this plan.
## Implementation notes — 2026-09-11
The authenticated director command carries its configured cohort percentage and
fresh process safety snapshot. A cell cannot use its own default-zero director
cohort setting to authorize or reject a selected host; it validates the authenticated
command, combines director/source safety, and rechecks durable policy, the host's
cohort bucket, and fleet/target safety inside the existing transaction. These fields
are on the internal admin endpoint, not the mobile or desktop protocol.
Candidate selection is read-only and uses a rotating page offset to progress past
busy hosts. A deterministic UUIDv5 derived from the exact source authority and target
keeps operation identity stable across director retries/restarts. Both details still
need final implementation audit and an explicit page-boundary fairness test.
Current focused and real-transport results are recorded at the top of the acceptance
document. They do not complete the remaining compatibility, cleanup and PR gates.
docs/relay-region-correction/RELAY-REGION-CORRECTION-PLAN.md
+177
View File
@@ -0,0 +1,177 @@
# Relay automatic region correction — implementation plan v2
Status: **IMPLEMENTED AND LOCALLY VALIDATED; RELEASE GATES REMAIN**. Section 5a is normative for first-grant adoption, retained-source rollback, renewal and cleanup. This replaces the idle-only proposal. Historical design reviews are preserved in the combined backup revision; current verification and remaining gaps are recorded in the acceptance document.
Implementation tracking: [live checklist](RELAY-REGION-CORRECTION-CHECKLIST.md). Completed work and validation evidence are recorded there; this document defines the behavior.
## Outcome and precise terms
An updated desktop obtains a reliable region preference. If its assigned region is materially worse, the existing director worker makes the better region the destination for new connections. Existing physical relay data connections continue through the old cell until they close or fail naturally. The desktop then releases the old origin and the durable migration completes.
“Existing connection” means a physical client-to-desktop relay data connection, represented by a cell splice and desktop connection ownership. It can carry multiple commands and subscriptions; it is not a terminal process, agent run, saved pairing, or recent typing. Tracked pending attachments and basis-bound control requests also protect source retirement. Quiet live connections are retained. No fixed maximum connection duration has been established or proposed for optional optimization.
For mobile: active use remains on the old cell; after the app actually suspends/closes that connection, the next successful connection resolves to the target. Current mobile schedules background suspension after 30 seconds, and checks an overdue deadline on foreground if the timer did not run. Putting the phone down while the app remains foreground is not a disconnect. No re-pairing is required solely for relocation.
Promise: optional region optimization does not deliberately terminate pre-existing connections merely because its grace timer expired. This is not a guarantee against network failure, process exit, revoked/expired auth, or emergency maintenance. New connection attempts during target startup may need normal recovery; do not promise zero delay on those attempts.
## Evidence and reuse
Source experiments used `origin/main` at `721a2692893ab29f8daee3149965bf5e9adf99a0`. Review must fetch current main and record its SHA, distinguishing changed source from these dated results.
- Bidirectional worker already exists in #19241 and predates deployed #19915. Do not implement a second placement/migration engine.
- Desktop `relay-origin-pool.ts` already opens a target, retains source connection ownership, and closes the source after final release. Auth refresh already visits every origin.
- Cell `host-session-registry.ts` has drain-only state, live/pending connection maps, and existing control renewal. Director store already retains migration state until source activity releases.
- Two unconditional deadline sites force interruption: desktop origin-pool and cell regional host drain. Removing those scheduling sites in a diagnostic snapshot made both preservation tests pass; restoring source made the identical tests fail again.
- A SQLite store test sustained a registered migration for a simulated hour with both controls renewed, then completed on source release.
- Mobile harness restored credentials/assignment/subscriptions after simulated drain. Sent mutations can become delivery-unknown and are not blindly replayed. The 251ms fake-clock recovery result is not measured real-world downtime.
Full evidence: [interruption findings](RELAY-INTERRUPTION-FINDINGS.md), [harness and patches](https://github.com/stablyai/orca/blob/0db9fdc486366f7451289f0c0599eed9ae1d94be/tests/tools/relay-rehome-interruption/README.md). The counterfactual patch is NOT production code: it has no negotiation, incorrectly changes normal deadline semantics, and does not validate failure/replay paths.
## 1. Ordered, expiring region decisions
Reuse desktop sampling and the existing assignment exchange. Separate legacy placement hints from eligibility to move an existing assignment.
Preserve cold-start placement: run the existing pre-placement probe and send its placement hint with the first assignment request. Do not create a default-region assignment merely to obtain a measurement window. Obtain the epoch-bound window with or after that first assignment; only a subsequent post-window measurement can certify migration eligibility. Test first placement with and without conclusive probes, plus old-server fallback, so the new migration protocol does not regress initial placement.
Proposed concrete protocol: an opt-in server-issued measurement window. A supporting desktop requests a window; director returns a per-host monotonic generation, fixed server expiry, and incumbent assignment epoch/region. Issuing a successor invalidates the predecessor for migration. The desktop probes after receiving the window, then reports a conclusive or inconclusive decision for that generation. Repeated delivery cannot extend expiry. An inconclusive outcome is retained as a tombstone; delayed older conclusive reports cannot resurrect eligibility. A restart that cannot reuse a valid cached decision obtains a successor window. Server time is authoritative for expiry.
Persist window generation/expiry, incumbent basis, supported probe-policy version, and outcome in the existing preference state. Keep the latest supported evidence only; no history of every probe. Duplicate same-generation decisions are idempotent; conflicting same-generation outcomes must not upgrade inconclusive to conclusive. Serialize window issuance/reporting per broker and reject stale assignment basis under claim lock. The exact schema and transaction ordering must be reviewed before implementation.
Legacy enum-only requests retain placement/reconnect behavior but cannot inherit or overwrite verified migration eligibility. Explicit new inconclusive decisions and missing legacy fields are different operations. Diagnostic region overrides are not measured proof. New request and response fields require explicit opt-in because both schemas are strict. Deploy server support before clients; implement old-server 400 fallback for both request shape and response negotiation without changing a healthy assignment.
## 2. Compare with the actual assigned region
Keep the current warm-up, sample count, spread rejection, and requirement that both regions be measurable. Initial placement may pick a best measured region. Moving an existing assignment additionally requires target latency at least **25ms lower AND 20% lower** than the measured incumbent region. These are the existing hysteresis thresholds applied to the correct basis, not a production-validated optimum.
Report compact comparison evidence (incumbent/target region timing, policy and reason), tied to the window and assignment epoch. Director validates bounds, eligibility and margin. A missing incumbent, incomplete catalog, rejected measurement, nearly tied regions, unsupported policy or stale assignment basis means no move. Clear legacy probe caches on upgrade without treating an unopposed or 1ms winner as migration evidence.
This optimizes the desktop-to-region path. Sample actual assigned-cell and mobile application behavior during rollout before claiming end-to-end user benefit.
## 3. Refresh ownership and cadence
The broker owns one decision deadline and one in-flight refresh/report task. Cancel them on broker close. Reuse the successful 24-hour cache interval and one-hour inconclusive retry interval, with jitter/backoff; eligibility expiry is distinct from retry scheduling. Do not wake offline/sleeping desktops for probes. On resume, check deadlines before using expired eligibility. Debounce network-change refresh only where the existing lifecycle provides a reliable signal.
Probe/report failure must not fail successful auth renewal or intentionally reconnect healthy controls. Serialize assignment response application with drain/recovery: same cell/epoch updates metadata; a newer assignment follows the existing target activation flow; stale responses are discarded. Report retries reuse the same decision/window rather than re-probing or extending expiry. During an open migration, do not claim another move or replace the retained source; refresh decisions may be stored for later reevaluation only.
## 4. Extend graceful migration to finish existing connections
Introduce an explicit opt-in drain mode for optional regional optimization. Persist it on the attempt and propagate it through the director-to-cell host-drain command and cell-to-desktop drain message. The source must learn mode from durable protocol state, not infer it from grace=0, a hostname or a retry count. Maintenance/emergency drains keep their current hard deadlines.
Eligibility requires supporting desktop and cell capabilities, fresh decision and matching source epoch/incarnation. New protocol/capability values and strict-parser fallbacks need cross-version tests. An unsupported participant defers optional correction rather than falling back to forced close. The reviewer should challenge how host capability remains bound to the currently active source generation, not merely a stale version-bearing report.
Nominal transition:
1. Existing claim reserves target capacity and commits target assignment plus migration/attempt state.
2. Source receives the mode-bearing drain and establishes the first valid authorized grant specified in section 5a before acknowledgment/cutover; desktop resolves/registers the target through existing code.
3. Target becomes the desktop's active origin. Existing source connections retain their source ownership; new connections resolve the current assignment. Already-admitted pending source connections may finish attaching and remain there.
4. Optional regional mode installs no forced old-origin/session close deadline. Existing event callbacks retire the source after its last owned connection and pending control operation end. Retain normal auth enforcement and operational emergency-close behavior.
5. Source control release/orphan cleanup removes its remaining activity. Existing completion logic finalizes the migration after source activity is gone and target is live.
Do not use DB splice lease absence to infer idle: those leases can expire with a live socket. This design instead allows source work to exist. Do not build the proposed pre-move globally atomic idle gate.
Required integration checks: pending control-RPC completion must trigger retirement when it was the final outstanding item; attach timeout/rejection and late async admission must not leak an origin or attach after retirement. Preserve existing source/target identity and request ownership. Cover two simultaneous mobile clients and a quiet connection; terminal counts and workspace type do not decide transport lifetime.
## 5. Failure, replay and bounded resource use
This section is a required implementation contract, not evidence that the current implementation already satisfies it.
| Condition | Required behavior |
| --- | --- |
| Target registration fails | Retain still-live source work; retry or reconcile using existing migration recovery. Restore source new-admission authority only through section 5a's retained-generation rollback; do not discard a migration that still owns connections. |
| Lost receipt / duplicate drain / zero-grace re-drain | Read durable mode and preserve existing work. Replay must never turn optional mode into forced closure. |
| Desktop restart / source generation replacement | Old process connections may already be gone; reconcile exact generations before retaining or clearing state. Unsupported replacement must not silently hard-drain existing work. |
| Source network/cell failure | Use existing failure recovery; connectivity loss is not proof remote execution exited. This is outside the no-deliberate-interruption promise. |
| Target fails after becoming active | Keep source connections that remain live; reconcile assignment/new connections through existing recovery without starting a third overlapping rehome. |
| Auth expiry/revocation or emergency cell drain | Preserve existing enforcement; optimization does not exempt sessions from security/maintenance lifecycle. |
| Last source data connection closes while control RPC pending | Wait for bounded RPC completion/timeout, then run cleanup without a polling loop. |
| Source remains busy for hours | Keep the migration open while healthy. Long duration alone does not force-close users or spend dispatch-failure budget. |
Bound **concurrent open migrations** in addition to starts/minute. Count pre-existing attempts; one migration per host remains enforced. Retain both controls' auth/lease renewals and account for source use plus target reservations. Data is not duplicated; extra controls/reservations still consume capacity.
Ensure fair progress in the existing 100-row lease-refresh and 10-row candidate/sweep pages; waiting old migrations must not starve registration, renewal or cleanup for newer ones. Initially enforce a conservative concurrency bound below the smallest relevant page capacity, counting existing open work, until fair traversal is verified. Filters for cohort/policy/capability belong before LIMIT and are rechecked under locks.
No user-visible maximum drain duration is claimed. If operations later require one for optimization, forcing closure would change the product promise and needs an explicit decision; a larger timer is not equivalent to finish-existing behavior.
## 5a. Review corrections: retained-source authority and lifetime
The independent review found three required contracts. This section supersedes any suggestion above that unchanged auth renewal or generic rollback suffices. Current source-control lifetime is six hours with thirty-minute jitter; auth-refresh does not extend it, and ordinary rotation only renews the active target. Current durable rehome refresh also has a 24-hour age ceiling. The one-hour SQLite experiment proved neither of those paths safe for indefinite live retention.
### Reuse the cell's existing activity renewal (supersedes the extra wire exchange)
Follow-up investigation found a smaller mechanism: a successfully validated `renewControlActivity` result can extend the same retained control's in-memory lease to `max(existingExpiry, requestedActivityExpiry)`. The cell already runs this renewal; no new desktop renewal timer, old-source rebind or recurring WebSocket exchange is needed. See [prototype evidence](https://github.com/stablyai/orca/blob/0db9fdc486366f7451289f0c0599eed9ae1d94be/tests/tools/relay-rehome-interruption/RETAINED-CONTROL-LEASE.md).
Before acknowledging the optional drain and telling desktop to cut over, establish the first short authorized renewal for the exact mode/attempt/source generation/incarnation. Subsequent grants reuse normal heartbeat renewal. Failure or stale state during adoption does not authorize retaining the source; reconcile the provisional target through the rollback contract below. A mode flag, pending database request or activity reacquisition alone is not a grant.
Narrow the existing atomic database renewal predicate with the retained attempt/mode/source basis, using immutable fields and preserving existing lock order; avoid a new precheck/query that can race mutation. The success callback must still match captured attempt, live session/socket/generation and mode; retirement/rollback/replacement/emergency drain invalidates it. Use the deadline sent at request start (currently 105s ahead), not response time, and do not add six hours on every heartbeat. Keep normal JWT/silence/watchdog enforcement and ordinary control rotation unchanged. Normal mode retains its existing lease; only extra retention needs these short successful grants.
Prototype evidence: 43 cell-registry tests and package typecheck pass, including >12h simulated retention without extra recurring renewal calls; 6 real Postgres renewal tests execute with zero skips on local 55440. Baseline/revert fails the three extension oracles. The prototype injects the future mode marker and is NOT production-ready: initial adoption ordering, durable mode predicate, wire negotiation and rollback remain integration work. This revision addresses source renewal only, not the whole retention feature.
### Final renewal-review correction: fence failures as well as success
Fresh GPT-6-astra / low review: [retained-control review](https://github.com/stablyai/orca/blob/0db9fdc486366f7451289f0c0599eed9ae1d94be/docs/relay-region-correction/RELAY-RETAINED-CONTROL-REVIEW.md), **REVISE one completion-fencing detail; heartbeat reuse supported**. The following correction is incorporated after review, not independently approved or implemented.
Every renewal completion and awaited recovery continuation must validate its captured socket/session, activity ID, current authority/mode transition and applicable ordering before altering scheduling, extending expiry, closing a socket, or reacquiring activity. In particular, a denial from an aborted retained attempt arriving after same-generation rollback must not close the restored source. A current applicable denial must still enforce closure. An obsolete missing-activity result must not initiate recovery; after awaited recovery, recheck authority and clean up abandoned acquisition as required. Do not use a blanket success-only fence or suppress all failures.
Add controlled-promise tests for late denial after rollback, after a newer valid authority transition, obsolete missing-activity recovery, and applicable denial. Verify both preserved socket/splice identity and correct rejection, not just expiry values. First-grant adoption must also reject a success whose requested expiry is already past.
In retained-mode SQL, the ordinary current-assignment authorization alternative must not bypass an aborted attempt check after rollback. If locking attempt rows, preserve assignment -> attempt -> migration -> activity dependency order used by existing rehome operations, rather than appending a late attempt lock. Use the prescribed PostgreSQL 16 environment for implementation concurrency validation on 55440; the earlier six-test PostgreSQL 17 run remains accurately labeled as narrower evidence.
### Retained-generation rollback
Generic source reassignment alone is insufficient: the source cell otherwise remains drain-only, and a fresh generation closes old splices. Add a durable, idempotent rollback transition for the exact optional attempt. In one authoritative store transaction, assign a newer source epoch and record that attempt's rollback outcome and retained source generation. Reconcile the source cell to that state: clear only that attempt's optional drain, preserve its socket/splices, update assignment metadata, and restore new admission only after validating current authority. Keep an aborted-attempt tombstone so a late drain cannot reverse rollback.
Desktop recovery must find/reuse the retained source origin and update its assignment metadata without replacing its control generation or transports. This needs an explicit supported transition, not the current rebind-failed -> fresh-generation fallback. Late target registration cannot override the newer source epoch. Release target reservations when reconciled; keep one open migration per host until cleanup completes. If the source process/generation is gone, use ordinary failure recovery and report that preservation is unavailable; do not infer remote execution exited.
### Transition table
| State/event | Authority and action | Existing source work |
| --- | --- | --- |
| Claim optional move | Durable attempt binds mode, source generation/incarnation, target, epoch and supported participants | Retained |
| Target registering | Source receives optional drain; existing target retry/reconciliation proceeds | Retained; do not convert age into forced closure |
| Target registered | Director target assignment is authoritative; desktop activates target | Existing source connections keep their origin |
| Retained source needs renewal | Existing cell activity renewal + exact optional migration authorize a short same-generation lease extension | Retained, source remains drain-only |
| Target failure / rollback | New durable source epoch plus rollback tombstone; source cell and desktop reuse exact retained generation | Retained if that generation still exists |
| Final source work ends | Connection and pending-work callbacks retire source; cancel renewals; release activity and complete | No source work left to preserve |
| Delayed drain/renew/register | Compare durable attempt outcome and epochs; ignore/reject obsolete transition | Must not resurrect draining or replace generation |
| Source failed / emergency drain | Existing authenticated operational/failure semantics apply | Preservation not promised under those failures |
### Mode-specific durable lifetime and compatibility floor
For healthy registered optional retained-source attempts, remove the current 24-hour age-only lease-refresh ceiling and exclude them from the age-only zero-grace re-drain lane. Keep bounded target-registration failure/reconciliation; do not extend an unreachable unregistered target forever. Duration alone is not dispatch failure, and active source data must not be dropped to reclaim an optimization slot. Current generic and regional cleanup paths must both understand the optional mode.
Before enabling optional retention, deploy a director/worker compatibility floor that understands all durable mode and rollback states even when new claims are disabled. Operational rollback must not go below that floor while such attempts exist. A disabled enable flag does not stop older cleanup/redrain code from misinterpreting new rows. Prove safe restart/rollback with existing open attempts and multi-day retention. The minimum revision will be recorded only after the compatible implementation is merged and validated.
Additional accepted obligations: notify retirement on every final pending-control transition (response, rejection, timeout, close); perform a final local session/generation check after awaited admission work and release abandoned reservations; bind negotiated support to current authenticated source generation rather than a stale measurement report; enforce the concurrent-migration cap in locked shared state, including pre-existing work. None of these requires rebuilding a global idle detector.
## 6. Rollout and observability
Keep rehome disabled while implementing/testing. Deploy compatible director/database support, then supporting cells and desktops with feature gated off. Verify readiness and actual capabilities before enabling an authorized bounded cohort. Do not dispatch workflows as part of this review.
Reuse current rate, cooldown, safety, capacity, durable attempts, and failure-budget controls. Preview must be read-only and share eligibility predicates, report full aggregate counts rather than a capped candidate page, and never claim attempts or consume budget.
Track eligibility/exclusions by direction, target registration, migration completion/abort, number/age of retained sources, concurrent reservations, deliberate forced-close count by drain mode, and reconnect/error rates. Retain compact sampled comparisons and matched before/after assigned-cell/application latency where available; a registered target alone is not evidence of user benefit. Use an unchanged cohort to detect unrelated network variation. Never log credentials, pairing data, or raw host IDs.
Acceptance: supported eligible hosts move new connections to their chosen target; old data connections survive optional-drain deadlines and retire on actual release; no forced source close solely due to optimization age; resources clean up; failures remain recoverable; sampled latency/reliability shows benefit without material regression. Specify sample sizes and numerical regression limits before production enable, using available traffic rather than inventing measured thresholds here.
## 7. Required validation and implementation order
1. Land freshness/ordering and incumbent-relative eligibility support under disabled control, with strict request/response compatibility tests. Cases: first-ever placement with correction disabled, placement-hint/actual-assignment mismatch, cold-start inconclusive probes and old-server fallback; delayed old reports, duplicates, inconclusive tombstones, clock changes, restarts, legacy writes, overrides, policy upgrades, stale epochs and cache clearing.
2. Implement broker refresh and event-driven origin retirement, test no auth coupling, no reconnect on unchanged assignment, pending-operation completion and sleep/resume.
3. Implement negotiated optional drain mode end-to-end in existing worker/cell/desktop paths, including normal emergency deadlines and replay. Extend the diagnostic oracles into real feature tests; do not merge the timer-removal experiment.
4. Validate real WebSocket traffic across source/target while sending unique stream markers and a mutation with delayed acknowledgment; check no duplicate/replayed mutation and independent host-side execution/output. Then validate mobile background/foreground reconnection and pairing preservation. Mock tests are not an end-to-end substitute.
5. Run actual Postgres integration/concurrency tests on **55440 only**. Require configured database, executed test counts and no conditional skip. Cover registration failure, target failure, concurrent admissions, cleanup, pagination and capacity accounting with long-lived sources.
6. Validate supported/unsupported desktop and cell combinations and old/new director rollback. SSH-hosted execution and folder workspaces remain governed by transport/owning host, not local process assumptions. No visible app tests on the user's desktop; background launch and isolated profiles are required.
7. Run a fresh operational safety gate and capability check only when a reviewed rollout is authorized. Enable a bounded cohort, observe retained-source/resource/benefit evidence, then expand. Disable stops new optional moves while safely reconciling existing ones.
### Implementation correction: restoration confirmation can retry
A rollback response can reach the cell while desktop director corroboration fails.
The cell therefore retains an exact pending-restoration authority (aborted attempt,
newer source epoch, original generation/incarnation/activity) until ordinary
same-generation rebind confirms restoration. Each successful existing heartbeat
renews only its short request-start deadline and replays `region-restored`.
Retained and restored authority are mutually exclusive; the old retained authority
remains rejected after rollback. Rebind, replacement, emergency drain, and applicable
denial fence outstanding callbacks. No extra timer or six-hour heartbeat grant is
introduced. This avoids losing long-lived source connections merely because the
first director confirmation failed.
docs/relay-region-correction/RELAY-REGION-CORRECTION-ROLLOUT.md
+60
View File
@@ -0,0 +1,60 @@
# Remaining release work and proposed rollout
Status: implementation and local validation; no deployment authorization used.
## Ordered release actions
1. Review the local changes/PR and CI results. Record the immutable merged commit
and image for deployment and rollback. All directors must run the reviewed idle
worker before enabling correction; verify mixed-version deployment behavior
while correction remains disabled.
2. Publish the relay image and deploy the director with correction cohort0 and
durable rehome disabled. Verify image, health, preview route, migration schema,
pool pressure and cleanup. This requires the authorized deployment workflow.
3. Roll supporting cell images using their existing cell workflow; verify protocol3,
correct incarnation and telemetry. Release the updated desktop normally.
The phone protocol is unchanged; no mobile update is required for correction.
4. Validate a packaged desktop with a physical phone (foreground, actual background
suspension, resume, quiet connection, and target failure), and existing clients
against new cells. Run the same transport gate on Linux/Windows in CI; validate
an actual SSH-owned terminal survives client disconnection and normal reconnect.
5. Read the authenticated aggregate preview. Estimate eligible population by
direction; use a matching unchanged comparison cohort. Choose the initial
cohort and record approvals before changing settings or enabling.
6. Configure the approved cohort through the existing director workflow input,
keeping the durable control disabled during deployment. Verify the serving revision and tagged rollback
revision, then enable through the existing regional-rehome control workflow.
7. Stop new claims on a regression while ordinary migration cleanup and recovery continue.
Investigate existing work rather than forcing a timer-based source closure.
## Proposed numerical acceptance criteria (must be approved before enable)
These are rollout proposals, not measurements of production baseline or authorization.
- First phase:1% deterministic host cohort; global cap remains8 open migrations.
Observe at least24h and30 completed moves. If traffic cannot supply30, extend
observation; do not treat a small sample as success.
- Immediate stop: any optimization-induced close of an established client, admission
reopening on ambiguous source authority, duplicated mutation, authorization bypass, or more than8
optimization migrations admitted (pre-existing work also consumes cap).
- Reliability stop: compared with an unchanged cohort over matching15-minute
windows, assignment/connect failure rate rises by>=1 percentage point or2x
(require>=100 attempts in each comparison group); investigate lower-count failures
individually. Existing production incident limits always take precedence.
- Performance acceptance: matched post-move assigned-cell control RTT improves by
>=25ms AND>=20% median per host for at least80% of evaluable moved hosts; require
two independent samples before and after. Identify samples using
assignment epoch/cell identity. Log sample insufficiency as unevaluable.
- Client connection setup p95 must not regress by>10% versus its matched baseline
after accounting for the unchanged cohort. Setup is not application command
latency; separately record physical-phone interaction timings on validation runs.
- Expansion requires healthy registration/completion, stable reservation usage, no growing stuck-recovery backlog, and numerical criteria
above. Continuously connected clients may postpone optimization indefinitely.
## Evidence boundaries
Local tests use real socket traffic and independent host execution, but synthetic
clock/authentication. Neither the build nor schema fixtures prove distribution,
production latency, a physical phone or a signed desktop upgrade. Separate Docker SSH tests validate
SSH-provider recovery; they are not physical mobile-to-SSH cutover evidence.
The checklist leaves these release gates open deliberately.
mobile/src/components/NewWorktreeModalController.tsx
+1 -1
View File
@@ -16,7 +16,7 @@ type Props = {
openExternalUrl: (url: string) => Promise<unknown>
onVisibleChange?: (visible: boolean) => void
onRouteVisibleChange: (visible: boolean) => void
onCreated: (worktreeId: string, name: string) => void
onCreated: (worktreeId: string, name: string, warning?: string) => void
}
export const NewWorktreeModalController = forwardRef<NewWorktreeModalControllerHandle, Props>(
mobile/src/components/new-worktree-modal-types.ts
+1 -1
View File
@@ -24,7 +24,7 @@ export type NewWorktreeModalProps = {
existingWorktreePaths?: readonly string[]
existingWorktrees?: readonly { repoId: string; branch: string }[]
openExternalUrl: (url: string) => Promise<unknown>
onCreated: (worktreeId: string, name: string) => void
onCreated: (worktreeId: string, name: string, warning?: string) => void
onClose: () => void
}
mobile/src/components/use-new-workspace-create-submit.ts
+2 -2
View File
@@ -58,7 +58,7 @@ export function useNewWorkspaceCreateSubmit(args: {
getWorktreeCreateCutoverSupport: () => Promise<WorktreeCreateIdempotencySupport | false>
transitionDrawer: (view: Exclude<NewWorktreeDrawerView, 'transition'>) => void
setError: Dispatch<SetStateAction<string>>
onCreated: (worktreeId: string, name: string) => void
onCreated: (worktreeId: string, name: string, warning?: string) => void
onClose: () => void
}): {
creating: boolean
@@ -181,7 +181,7 @@ export function useNewWorkspaceCreateSubmit(args: {
return
}
args.onClose()
args.onCreated(result.worktreeId, result.name)
args.onCreated(result.worktreeId, result.name, result.warning)
} catch (error) {
args.setError(error instanceof Error ? error.message : 'Failed to create workspace')
} finally {
mobile/src/host-route-action-state.test.ts
+24
View File
@@ -19,6 +19,30 @@ describe('host route action state', () => {
)
})
// Why: the host reports a create that succeeded with a failed startup terminal via `warning`;
// dropping it here is what lands the phone on an unexplained empty session.
it('carries a host create warning into the session route', () => {
expect(
hostNewWorktreeSessionRoute(
'local',
'wt-1',
'Hammerhead',
'Failed to create the startup terminal'
)
).toBe(
'/h/local/session/wt-1?name=Hammerhead&created=1&warning=Failed+to+create+the+startup+terminal'
)
})
it('omits an absent or blank create warning', () => {
expect(hostNewWorktreeSessionRoute('local', 'wt-1', 'Hammerhead', ' ')).toBe(
'/h/local/session/wt-1?name=Hammerhead&created=1'
)
expect(hostNewWorktreeSessionRoute('local', 'wt-1', 'Hammerhead')).toBe(
'/h/local/session/wt-1?name=Hammerhead&created=1'
)
})
it('opens new worktree modal on an initial newWorktree action', () => {
expect(createInitialHostRouteActionState('newWorktree')).toEqual({
routeAction: 'newWorktree',
mobile/src/host-route-action-state.ts
+6 -1
View File
@@ -10,9 +10,14 @@ export function hostNewWorktreeRoute(hostId: string): `/h/${string}?action=newWo
export function hostNewWorktreeSessionRoute(
hostId: string,
worktreeId: string,
worktreeName: string
worktreeName: string,
/** Host-reported create warning (e.g. the startup terminal failed to spawn). */
warning?: string
): `/h/${string}/session/${string}?${string}` {
const params = new URLSearchParams({ name: worktreeName, created: '1' })
if (warning?.trim()) {
params.set('warning', warning)
}
return `/h/${encodeURIComponent(hostId)}/session/${encodeURIComponent(worktreeId)}?${params}`
}
mobile/src/host-screen/host-screen-overlays.tsx
+2 -2
View File
@@ -219,10 +219,10 @@ export function HostScreenOverlays({ controller }: { controller: HostScreenContr
onVisibleChange={(visible) => {
state.newWorktreeModalVisibleRef.current = visible
}}
onCreated={(worktreeId, worktreeName) => {
onCreated={(worktreeId, worktreeName, warning) => {
void catalog.fetchWorktrees({ allowDuringModal: true })
actions.navigateFromHostList(
hostNewWorktreeSessionRoute(hostId, worktreeId, worktreeName)
hostNewWorktreeSessionRoute(hostId, worktreeId, worktreeName, warning)
)
}}
onRouteVisibleChange={actions.setShowNewWorktreeVisible}
mobile/src/session/MobileNativeChatMessage.test.ts
+4 -3
View File
@@ -9,6 +9,7 @@ vi.mock('react-native', async () => {
const Text = ({ children, ...props }: { children?: unknown }): unknown =>
React.createElement('Text', props, children)
return {
ActivityIndicator: 'ActivityIndicator',
Animated: {
Text,
Value: class {
@@ -268,12 +269,12 @@ describe('MobileNativeChatMessage', () => {
expect(tree.root.findAllByType('Wrench' as never)).toHaveLength(0)
})
it('renders the turn status row under a user message', () => {
it('renders the settled turn status row under a user message', () => {
const tree = render(userMessage([{ type: 'text', text: 'go' }]), {
structuredActivityUi: true,
turnStatus: { startedAt: Date.now(), thinking: true, workedSeconds: null }
turnStatus: { startedAt: Date.now() - 3_000, thinking: false, workedSeconds: 3 }
})
expect(textIn(tree.root)).toContain('Thinking')
expect(textIn(tree.root)).toContain('Worked for 3s')
})
it('does not render a turn status row without one', () => {
mobile/src/session/MobileNativeChatMessage.tsx
+1 -1
View File
@@ -82,7 +82,7 @@ function MobileNativeChatMessageImpl({
/** Multiplies all chat text sizes for pinch-to-zoom (1 = no change). */
fontScale?: number
onOpenFile?: (relativePath: string) => void
/** This turn's status row, rendered under a user message (desktop parity). */
/** This settled turn's status row, rendered under its user message. */
turnStatus?: NativeChatTurnStatus | null
/** Whether the turn caret has disclosed this turn's activity. */
turnExpanded?: boolean
mobile/src/session/MobileNativeChatOverlay.tsx
+1
View File
@@ -73,6 +73,7 @@ export function MobileNativeChatOverlay({
agentWorking={controller.nativeChatAgentWorking}
canStop={controller.nativeChatCanStop}
structuredActivityUi={controller.nativeChatStructured}
turnIndicator={controller.nativeChatTurnIndicator}
workingStartedAt={controller.nativeChatWorkingStartedAt}
settledTurns={controller.nativeChatSettledTurns}
streaming={streaming}
mobile/src/session/MobileNativeChatTurnStatus.test.ts
+25 -16
View File
@@ -7,18 +7,8 @@ vi.mock('react-native', async () => {
const Text = ({ children, ...props }: { children?: unknown }): unknown =>
React.createElement('Text', props, children)
return {
Animated: {
Text,
Value: class {
constructor(private value: number) {}
setValue(next: number): void {
this.value = next
}
},
loop: (animation: unknown) => animation,
sequence: () => ({ start: vi.fn(), stop: vi.fn() }),
timing: () => ({ start: vi.fn(), stop: vi.fn() })
},
ActivityIndicator: (props: Record<string, unknown>) =>
React.createElement('ActivityIndicator', props),
Pressable: ({ children, ...props }: { children?: unknown }) =>
React.createElement('Pressable', props, children),
Text,
@@ -49,6 +39,7 @@ describe('MobileNativeChatTurnStatus', () => {
startedAt: number | null
thinking: boolean
workedSeconds?: number | null
activityText?: string | null
expanded?: boolean
onToggleExpanded?: () => void
}): ReactTestRenderer {
@@ -61,12 +52,16 @@ describe('MobileNativeChatTurnStatus', () => {
const labels = (node: ReactTestInstance): string[] =>
node.findAllByType('Text' as never).map((text) => String(text.children.join('')))
it('reads "Thinking" before the turn produces output', () => {
const spinners = (node: ReactTestInstance): ReactTestInstance[] =>
node.findAllByType('ActivityIndicator' as never)
it('reads "Thinking" beside one spinner while the turn reasons', () => {
const tree = render({ startedAt: Date.now(), thinking: true })
expect(labels(tree.root)).toEqual(['Thinking'])
expect(spinners(tree.root)).toHaveLength(1)
})
it('counts up once the turn is producing output', () => {
it('counts up on that same single row when the turn is not reasoning', () => {
const startedAt = Date.now()
const tree = render({ startedAt, thinking: false })
expect(labels(tree.root)).toEqual(['Working for 0s'])
@@ -74,6 +69,19 @@ describe('MobileNativeChatTurnStatus', () => {
vi.advanceTimersByTime(12_000)
})
expect(labels(tree.root)).toEqual(['Working for 12s'])
expect(spinners(tree.root)).toHaveLength(1)
})
it('lets provider activity text beat both fallbacks and hold the clock', () => {
const tree = render({
startedAt: Date.now(),
thinking: true,
activityText: 'Running pnpm test'
})
expect(labels(tree.root)).toEqual(['Running pnpm test'])
expect(spinners(tree.root)).toHaveLength(1)
// No label consumes the duration, so nothing schedules a tick for it.
expect(vi.getTimerCount()).toBe(0)
})
it('settles to a tappable "Worked for" row that toggles the turn', () => {
@@ -98,9 +106,10 @@ describe('MobileNativeChatTurnStatus', () => {
expect(labels(tree.root)).toEqual(['Worked for 5s'])
})
it('holds no interval once the turn has settled', () => {
render({ startedAt: Date.now(), thinking: false, workedSeconds: 5 })
it('holds no interval, and no spinner, once the turn has settled', () => {
const tree = render({ startedAt: Date.now(), thinking: false, workedSeconds: 5 })
expect(vi.getTimerCount()).toBe(0)
expect(spinners(tree.root)).toHaveLength(0)
})
it('announces the live row to assistive tech', () => {
mobile/src/session/MobileNativeChatTurnStatus.tsx
+25 -30
View File
@@ -1,7 +1,8 @@
import { useEffect, useRef, useState } from 'react'
import { Animated, Pressable, StyleSheet, Text, View } from 'react-native'
import { useEffect, useState } from 'react'
import { ActivityIndicator, Pressable, StyleSheet, Text, View } from 'react-native'
import { ChevronRight } from 'lucide-react-native'
import {
formatNativeChatActiveTurnLabel,
formatNativeChatTurnStatusLabel,
NATIVE_CHAT_TURN_STATUS_COPY,
nativeChatElapsedSeconds
@@ -25,48 +26,38 @@ function useElapsedSeconds(startedAt: number | null, counting: boolean): number
return counting ? nativeChatElapsedSeconds(startedAt, mountedAt, now) : 0
}
/** The per-turn status row — "Thinking", then "Working for 12s" while the turn
* runs, settling to a tappable "Worked for 3m 4s" that discloses the turn's
* tool activity. Desktop parity: `NativeChatWorkingStatus`. */
/** The per-turn status row. While the turn runs it is the one live indicator — a
* spinner beside what the provider says it is doing, else "Thinking", else
* "Working for 12s". It settles to a tappable "Worked for 3m 4s" that discloses
* the turn's tool activity. Desktop parity: `NativeChatTurnActivityLine` for the
* live row, `NativeChatWorkingStatus` for the settled one. */
export function MobileNativeChatTurnStatus({
startedAt,
thinking,
workedSeconds,
activityText,
expanded = false,
onToggleExpanded
}: {
startedAt: number | null
thinking: boolean
workedSeconds?: number | null
/** Provider activity copy for a live turn; outranks the other two labels. */
activityText?: string | null
expanded?: boolean
onToggleExpanded?: () => void
}): React.JSX.Element {
const counting = !thinking && workedSeconds == null
const settled = workedSeconds != null
const counting = !settled && !thinking && !activityText?.trim()
const elapsedSeconds = useElapsedSeconds(startedAt, counting)
const label = formatNativeChatTurnStatusLabel({ thinking, workedSeconds, elapsedSeconds })
const label = settled
? formatNativeChatTurnStatusLabel({ thinking, workedSeconds, elapsedSeconds })
: formatNativeChatActiveTurnLabel({ activityText, thinking, elapsedSeconds })
const pulse = useRef(new Animated.Value(1)).current
useEffect(() => {
if (!thinking) {
pulse.setValue(1)
return
}
const animation = Animated.loop(
Animated.sequence([
Animated.timing(pulse, { toValue: 0.45, duration: 700, useNativeDriver: true }),
Animated.timing(pulse, { toValue: 1, duration: 700, useNativeDriver: true })
])
)
animation.start()
return () => animation.stop()
}, [pulse, thinking])
const rowStyle = [styles.row, thinking ? null : styles.rowSettled]
if (workedSeconds != null && onToggleExpanded) {
if (settled && onToggleExpanded) {
return (
<Pressable
style={({ pressed }) => [...rowStyle, pressed && styles.pressed]}
style={({ pressed }) => [styles.row, styles.rowSettled, pressed && styles.pressed]}
onPress={onToggleExpanded}
hitSlop={6}
accessibilityRole="button"
@@ -83,11 +74,14 @@ export function MobileNativeChatTurnStatus({
return (
<View
style={rowStyle}
style={[styles.row, settled ? styles.rowSettled : null]}
accessibilityLiveRegion="polite"
accessibilityLabel={NATIVE_CHAT_TURN_STATUS_COPY.responding}
>
<Animated.Text style={[styles.label, thinking && { opacity: pulse }]}>{label}</Animated.Text>
{settled ? null : <ActivityIndicator size="small" color={colors.textMuted} />}
<Text style={styles.label} numberOfLines={1}>
{label}
</Text>
</View>
)
}
@@ -109,7 +103,8 @@ const styles = StyleSheet.create({
},
label: {
color: colors.textMuted,
fontSize: typography.bodySize
fontSize: typography.bodySize,
flexShrink: 1
},
caretOpen: {
transform: [{ rotate: '90deg' }]
mobile/src/session/MobileNativeChatView.test.ts
+65 -6
View File
@@ -73,6 +73,7 @@ type Overrides = {
onSend?: (text: string) => Promise<boolean>
pending?: Parameters<typeof MobileNativeChatView>[0]['pending']
structuredActivityUi?: boolean
turnIndicator?: Parameters<typeof MobileNativeChatView>[0]['turnIndicator']
agentWorking?: boolean
canStop?: boolean
sendSurfaceId?: string
@@ -273,20 +274,74 @@ describe('MobileNativeChatView', () => {
return (renderedRow(id) as { props: Record<string, unknown> }).props
}
function footerProps(): Record<string, unknown> | null {
const list = renderer!.root.find((node) => node.type === 'FlatList')
const footer = list.props.ListFooterComponent as
| { props: Record<string, unknown> }
| null
| undefined
return footer?.props ?? null
}
function workingIndicators(): ReactTestInstance[] {
return renderer!.root.findAll((node) => node.type === 'WorkingIndicator')
}
it('gives the live user turn a status row and drops the three-dot indicator', async () => {
const folded = [userTurn('u1', 'go')]
it('puts the live status at the turn tail and drops the three-dot indicator', async () => {
const folded = [userTurn('u1', 'go'), assistantTurn('a1', 'still working')]
await render({ messages: folded, folded, structuredActivityUi: true, agentWorking: true })
const props = rowProps('u1')
expect(props.structuredActivityUi).toBe(true)
expect(props.turnStatus).toMatchObject({ thinking: true, workedSeconds: null })
expect(props.turnStatus).toBeNull()
// Nothing reports reasoning, so the one live footer counts instead of guessing.
expect(footerProps()).toMatchObject({ thinking: false, workedSeconds: null })
expect(listIds().at(-1)).toBe('a1')
expect(props.activeTurnIsWorking).toBe(true)
expect(workingIndicators()).toHaveLength(0)
})
it('reports the live turn as thinking only when its journal says it is reasoning', async () => {
const folded = [userTurn('u1', 'go')]
await render({
messages: folded,
folded,
structuredActivityUi: true,
agentWorking: true,
turnIndicator: { thinking: true, activityText: null }
})
expect(rowProps('u1').turnStatus).toBeNull()
expect(footerProps()).toMatchObject({ thinking: true, workedSeconds: null })
})
it('hands the live row the provider activity copy that outranks its fallbacks', async () => {
const folded = [userTurn('u1', 'go')]
await render({
messages: folded,
folded,
structuredActivityUi: true,
agentWorking: true,
turnIndicator: { thinking: true, activityText: 'Running pnpm test' }
})
expect(footerProps()).toMatchObject({
thinking: true,
activityText: 'Running pnpm test'
})
})
it('keeps the activity copy on the live footer instead of a historical row', async () => {
const folded = [userTurn('u1', 'go'), userTurn('u2', 'again')]
await render({
messages: folded,
folded,
structuredActivityUi: true,
agentWorking: true,
turnIndicator: { thinking: false, activityText: 'Running pnpm test' }
})
expect(rowProps('u1')).not.toHaveProperty('turnActivityText')
expect(rowProps('u2')).not.toHaveProperty('turnActivityText')
expect(footerProps()).toMatchObject({ activityText: 'Running pnpm test' })
})
it('keeps the bridge lane on the three-dot indicator with no turn status', async () => {
const folded = [userTurn('u1', 'go')]
await render({ messages: folded, folded, agentWorking: true })
@@ -294,13 +349,15 @@ describe('MobileNativeChatView', () => {
expect(props.structuredActivityUi).toBe(false)
expect(props.turnStatus).toBeNull()
expect(props.activeTurnIsWorking).toBe(false)
expect(footerProps()).toBeNull()
expect(workingIndicators()).toHaveLength(1)
})
it('settles the finished turn to a tappable duration', async () => {
const folded = [userTurn('u1', 'go'), assistantTurn('a1', 'done')]
await render({ messages: folded, folded, structuredActivityUi: true, agentWorking: true })
expect(rowProps('u1').turnStatus).toMatchObject({ thinking: false, workedSeconds: null })
expect(rowProps('u1').turnStatus).toBeNull()
expect(footerProps()).toMatchObject({ thinking: false, workedSeconds: null })
await update({ messages: folded, folded, structuredActivityUi: true, agentWorking: false })
const settled = rowProps('u1')
expect(settled.turnStatus).toMatchObject({ thinking: false })
@@ -309,6 +366,7 @@ describe('MobileNativeChatView', () => {
)
expect(settled.onToggleTurn).toBeTypeOf('function')
expect(settled.activeTurnIsWorking).toBe(false)
expect(footerProps()).toBeNull()
})
it('hangs no status row on an assistant row', async () => {
@@ -317,6 +375,7 @@ describe('MobileNativeChatView', () => {
expect(rowProps('a1').turnStatus).toBeNull()
// The assistant row still belongs to the live turn, so its tool row stays visible.
expect(rowProps('a1').activeTurnIsWorking).toBe(true)
expect(footerProps()).toMatchObject({ workedSeconds: null })
})
it('does not carry a running turn clock across chat surfaces', async () => {
@@ -331,7 +390,7 @@ describe('MobileNativeChatView', () => {
agentWorking: true,
sendSurfaceId: 'host\0worktree\0tab-a'
})
expect(rowProps('u1').turnStatus).toMatchObject({ startedAt: 1_000 })
expect(footerProps()).toMatchObject({ startedAt: 1_000 })
vi.setSystemTime(12_000)
const secondTab = [userTurn('u2', 'second')]
@@ -343,7 +402,7 @@ describe('MobileNativeChatView', () => {
sendSurfaceId: 'host\0worktree\0tab-b'
})
expect(rowProps('u2').turnStatus).toMatchObject({ startedAt: 12_000 })
expect(footerProps()).toMatchObject({ startedAt: 12_000 })
} finally {
vi.useRealTimers()
}
mobile/src/session/MobileNativeChatView.tsx
+14 -4
View File
@@ -13,7 +13,10 @@ import { GestureDetector, GestureHandlerRootView } from 'react-native-gesture-ha
import { ArrowDown, ChevronsDownUp, ChevronsUpDown, Square } from 'lucide-react-native'
import type { AskAnswerSelection, AskPrompt } from '../../../src/shared/native-chat-ask'
import type { NativeChatMessage } from '../../../src/shared/native-chat-types'
import type { NativeChatSettledTurns } from '../../../src/shared/native-chat-turn-status'
import type {
NativeChatLiveTurnIndicator,
NativeChatSettledTurns
} from '../../../src/shared/native-chat-turn-status'
import { colors } from '../theme/mobile-theme'
import { styles } from './mobile-native-chat-view-styles'
import {
@@ -53,6 +56,8 @@ type Props = {
/** Structured lane: per-turn "Working for N" status plus live tool progress,
* replacing the bridge lane's static three-dot working row (desktop parity). */
structuredActivityUi?: boolean
/** What labels the live turn's one indicator row (structured lane only). */
turnIndicator?: NativeChatLiveTurnIndicator | null
/** Structured lane: host-recorded turn timing feeding the per-turn status rows. */
workingStartedAt?: number | null
settledTurns?: NativeChatSettledTurns | null
@@ -136,6 +141,7 @@ export function MobileNativeChatView({
agentWorking,
canStop = agentWorking,
structuredActivityUi = false,
turnIndicator = null,
workingStartedAt,
settledTurns,
onStop,
@@ -259,14 +265,17 @@ export function MobileNativeChatView({
[hasMore, loadingEarlier, onLoadEarlier]
)
// Per-turn "Thinking / Working for N / Worked for N" rows. The structured lane
// owns them; the bridge lane keeps its three-dot indicator.
// Per-turn status rows: one live indicator while the turn runs, then a settled
// "Worked for N" row. The structured lane owns them; the bridge lane keeps its
// three-dot indicator.
const turns = useMobileNativeChatTurnDisclosure({
messages: data,
enabled: structuredActivityUi,
isWorking: agentWorking === true,
workingStartedAt,
settledTurns,
thinking: turnIndicator?.thinking === true,
activityText: turnIndicator?.activityText ?? null,
scopeKey: sendSurfaceId
})
@@ -331,11 +340,12 @@ export function MobileNativeChatView({
) : null
}
ListFooterComponent={
turns.activeTurnIsUnanchored && turns.active ? (
structuredActivityUi && agentWorking && turns.active ? (
<MobileNativeChatTurnStatus
startedAt={turns.active.startedAt}
thinking={turns.active.thinking}
workedSeconds={turns.active.workedSeconds}
activityText={turns.activeActivityText}
/>
) : null
}
mobile/src/session/mobile-native-chat-controller-contract.ts
+6 -1
View File
@@ -6,7 +6,10 @@ import type {
} from '../../../src/shared/native-chat-ask'
import type { detectAgentPermission } from './mobile-native-chat-permission'
import type { parseAgentQuestion } from './mobile-native-chat-question'
import type { NativeChatSettledTurns } from '../../../src/shared/native-chat-turn-status'
import type {
NativeChatLiveTurnIndicator,
NativeChatSettledTurns
} from '../../../src/shared/native-chat-turn-status'
import type { MobileNativeChatSendOutcome } from './mobile-native-chat-send'
import type { MobileNativeChatPendingMessage } from './use-mobile-native-chat-drafts'
import type { useMobileNativeChatSession } from './use-mobile-native-chat-session'
@@ -29,6 +32,8 @@ export type MobileNativeChatController = {
/** Structured lane: drives the per-turn status row and live tool progress. */
nativeChatStructured: boolean
nativeChatAgentWorking: boolean
/** What labels the live turn's one indicator row; null off the structured lane. */
nativeChatTurnIndicator: NativeChatLiveTurnIndicator | null
/** Structured lane: host-recorded turn timing for the per-turn status rows. */
nativeChatWorkingStartedAt: number | null
nativeChatSettledTurns: NativeChatSettledTurns | null
mobile/src/session/mobile-session-route-parity.test.ts
+2 -2
View File
@@ -73,7 +73,7 @@ const HEAD_CONTENT_HOOK_SHA256 = '9c3b612fef3f370d66873aefdbe1d701f20cb64ded31fe
// after the anchor it sent the host instead of appending. Only that body changed; the other 11
// extracted functions are byte-identical.
const HEAD_NESTED_FUNCTION_SHA256 =
'e54db6971e298dc27b59da9cbb2c455b85f0e1224cdf00482410ac43d179b02d'
'5999a4118d60b40935e6fd61a5457a12c110c1108479d56439b8e186b863f8d8'
const HEAD_NATIVE_REGISTRATION_SHA256 =
'cab85e4e4a3f43289ba93ddea9ccce57aea83e0bf14fd1620a965aad0c1cb49e'
const HEAD_NATIVE_REMOVAL_SHA256 =
@@ -82,7 +82,7 @@ const HEAD_TIMER_CREATION_SHA256 =
'1a31b625e2174c3db77272249843196d2b6b06ab1e654a96d8f7858e3082e66b'
const HEAD_TIMER_CLEANUP_SHA256 = 'c73f1d1c2cc89642f3d727d6f3b6b81860a9d6f34234541a2065ec3d1a8cd116'
const HEAD_RUNTIME_STRING_SHA256 =
'31951b0b83be01ebfa659c4b94df9ad7eaff6404df5338fbade89eb7473a3cb4'
'57ef354b97fb4fd3776fd1b09a34305d84022c04c43c6391bd130517bf6e37af'
const HEAD_HOST_JSX_SHA256 = '390405926b1695fa3a33686f0bc192b432f5468d8576499d7cafbb4922defbb5'
const HEAD_LEAF_JSX_SHA256 = '21dba981875e173f692590bf910d60964660c5f4cbb79f3a377c7e54f6a1f016'
const HEAD_STYLE_REFERENCE_SHA256 =
mobile/src/session/use-mobile-bridge-chat-prompt-writes.ts
+65
View File
@@ -0,0 +1,65 @@
import type { MutableRefObject } from 'react'
import type { RpcClient } from '../transport/rpc-client'
import { useMobileNativeChatPermissionSend } from './mobile-native-chat-permission-send'
import { useMobileNativeChatAnswerSend } from './use-mobile-native-chat-answer-send'
import { useMobileNativeChatCancelAsk } from './use-mobile-native-chat-cancel-ask'
import { useMobileNativeChatStop } from './use-mobile-native-chat-stop'
import type { MobileNativeChatAnswerSend } from './use-mobile-native-chat-answer-send'
/** The bridge lane's four prompt/interrupt write seams. They share one enable
* gate and chain through the answer seam's `cancelPending`, so a caller cannot
* wire one of them to a different lane or forget to drop in-flight answer
* writes before an Escape. The structured lane answers over RPC instead. */
export function useMobileBridgeChatPromptWrites(args: {
client: RpcClient | null
enabled: boolean
handleRef: MutableRefObject<string | null>
deviceTokenRef: MutableRefObject<string | null>
agentRef: MutableRefObject<string | null>
/** Changes on chat session swap; cancels pending writes when it does. */
sessionId: string | null
streamIdentity: string
onSendError: (message: string) => void
}): {
answerAsk: MobileNativeChatAnswerSend['answerAsk']
cancelAsk: () => Promise<boolean>
respondPermission: (send: string) => Promise<boolean>
stop: () => void
} {
const { client, enabled, handleRef, deviceTokenRef, streamIdentity, onSendError } = args
const { answerAsk, cancelPending } = useMobileNativeChatAnswerSend({
client,
enabled,
handleRef,
deviceTokenRef,
agentRef: args.agentRef,
sessionId: args.sessionId,
streamIdentity,
onSendError
})
const cancelAsk = useMobileNativeChatCancelAsk({
client,
enabled,
handleRef,
deviceTokenRef,
cancelPending,
onSendError
})
const respondPermission = useMobileNativeChatPermissionSend({
client,
enabled,
handleRef,
deviceTokenRef,
onSendError
})
const stop = useMobileNativeChatStop({
client,
enabled,
handleRef,
deviceTokenRef,
streamIdentity,
cancelPending,
onSendError
})
return { answerAsk, cancelAsk, respondPermission, stop }
}
mobile/src/session/use-mobile-native-chat-controller.ts
+10 -35
View File
@@ -2,10 +2,7 @@ import { useLayoutEffect, useRef, type MutableRefObject } from 'react'
import type { RpcClient } from '../transport/rpc-client'
import type { ConnectionState } from '../transport/types'
import type { MobileNativeChatTab } from './mobile-native-chat-eligibility'
import { useMobileNativeChatPermissionSend } from './mobile-native-chat-permission-send'
import { useMobileNativeChatAnswerSend } from './use-mobile-native-chat-answer-send'
import { useMobileNativeChatAskDismiss } from './use-mobile-native-chat-ask-dismiss'
import { useMobileNativeChatCancelAsk } from './use-mobile-native-chat-cancel-ask'
import { useMobileNativeChatDrafts } from './use-mobile-native-chat-drafts'
import { useMobileNativeChatFileSearch } from './use-mobile-native-chat-file-search'
import { useMobileNativeChatMessageSend } from './use-mobile-native-chat-message-send'
@@ -14,10 +11,10 @@ import { useMobileNativeChatSessionOptionController } from './use-mobile-native-
import { useMobileNativeChatSessionLane } from './use-mobile-native-chat-session-lane'
import { useMobileStructuredNativeChatSendBridge } from './use-mobile-structured-native-chat-send-bridge'
import { useMobileNativeChatPrompts } from './use-mobile-native-chat-prompts'
import { useMobileNativeChatStop } from './use-mobile-native-chat-stop'
import { useNativeChatAcceptedAction } from './use-native-chat-action-outcomes'
import { useThrottledLatestValue } from './use-throttled-latest-value'
import type { MobileNativeChatController } from './mobile-native-chat-controller-contract'
import { useMobileBridgeChatPromptWrites } from './use-mobile-bridge-chat-prompt-writes'
import { useMobileNativeChatActiveResolution } from './use-mobile-native-chat-active-resolution'
export type { MobileNativeChatController } from './mobile-native-chat-controller-contract'
@@ -171,42 +168,19 @@ export function useMobileNativeChatController(args: {
? client != null && activeChatSessionId != null && connState === 'connected'
: nativeChatInputLeaseReady && connState === 'connected'
const { answerAsk: handleNativeChatAnswerAsk, cancelPending: cancelNativeChatAnswer } =
useMobileNativeChatAnswerSend({
client,
enabled: inputSendable && !activeChatStructured,
handleRef: activeHandleRef,
deviceTokenRef,
agentRef: activeChatAgentRef,
sessionId: activeChatSessionId,
streamIdentity,
onSendError
})
const handleNativeChatCancelAsk = useMobileNativeChatCancelAsk({
client,
enabled: inputSendable && !activeChatStructured,
handleRef: activeHandleRef,
deviceTokenRef,
cancelPending: cancelNativeChatAnswer,
onSendError
})
const legacyHandleNativeChatRespondPermission = useMobileNativeChatPermissionSend({
client,
enabled: inputSendable && !activeChatStructured,
handleRef: activeHandleRef,
deviceTokenRef,
onSendError
})
const handleNativeChatStop = useMobileNativeChatStop({
const {
answerAsk: handleNativeChatAnswerAsk,
cancelAsk: handleNativeChatCancelAsk,
respondPermission: legacyHandleNativeChatRespondPermission,
stop: handleNativeChatStop
} = useMobileBridgeChatPromptWrites({
client,
enabled: inputSendable && !activeChatStructured,
handleRef: activeHandleRef,
deviceTokenRef,
agentRef: activeChatAgentRef,
sessionId: activeChatSessionId,
streamIdentity,
cancelPending: cancelNativeChatAnswer,
onSendError
})
@@ -299,6 +273,7 @@ export function useMobileNativeChatController(args: {
/** Structured lane: drives the per-turn status row and live tool progress. */
nativeChatStructured: activeChatStructured,
nativeChatAgentWorking,
nativeChatTurnIndicator: activeChatStructured ? structuredNativeChat.turnIndicator : null,
nativeChatWorkingStartedAt: activeChatStructured ? structuredNativeChat.workingStartedAt : null,
nativeChatSettledTurns: activeChatStructured ? structuredNativeChat.settledTurns : null,
nativeChatCanStop: activeChatStructured
mobile/src/session/use-mobile-native-chat-turn-disclosure.ts
+15 -10
View File
@@ -28,6 +28,8 @@ export function useMobileNativeChatTurnDisclosure({
isWorking,
workingStartedAt,
settledTurns,
thinking = false,
activityText = null,
scopeKey
}: {
messages: readonly NativeChatMessage[]
@@ -36,12 +38,16 @@ export function useMobileNativeChatTurnDisclosure({
workingStartedAt?: number | null
/** Host-recorded durations; they outrank whatever this client observed. */
settledTurns?: NativeChatSettledTurns | null
/** Whether the turn is reasoning right now, derived from its journal content. */
thinking?: boolean
/** What the provider says the live turn is doing; outranks the other labels. */
activityText?: string | null
/** Host/worktree/tab identity for timing and disclosure isolation. */
scopeKey: string
}): {
active: NativeChatTurnStatus | null
/** True when the live turn has no user message to hang its status row under. */
activeTurnIsUnanchored: boolean
/** The live turn's provider activity copy, for the footer row. */
activeActivityText: string | null
onToggleTurn: (turnKey: string) => void
resolveRow: (index: number, message: NativeChatMessage) => MobileNativeChatTurnRow
} {
@@ -51,6 +57,7 @@ export function useMobileNativeChatTurnDisclosure({
isWorking,
workingStartedAt,
settledTurns,
thinking,
scopeKey
})
const [expandedTurns, setExpandedTurns] = useState<{
@@ -93,17 +100,16 @@ export function useMobileNativeChatTurnDisclosure({
}, [enabled, messages])
const { active, activeTurnKey, completedByTurn } = turnStatuses
const activeActivityText = enabled && isWorking ? (activityText ?? null) : null
const resolveRow = useCallback(
(index: number, message: NativeChatMessage): MobileNativeChatTurnRow => {
const turnKey = turnKeys[index]
const turnStatus =
!enabled || message.role !== 'user'
? null
: turnKey === activeTurnKey
? active
: turnKey
? (completedByTurn[turnKey] ?? null)
: null
: turnKey
? (completedByTurn[turnKey] ?? null)
: null
return {
turnStatus,
turnExpanded: turnKey ? expandedTurnIds.has(turnKey) : false,
@@ -120,15 +126,14 @@ export function useMobileNativeChatTurnDisclosure({
(turnKey === undefined && activeTurnKey === MOBILE_UNANCHORED_TURN_KEY))
}
},
[turnKeys, enabled, activeTurnKey, active, completedByTurn, expandedTurnIds, isWorking]
[turnKeys, enabled, activeTurnKey, completedByTurn, expandedTurnIds, isWorking]
)
return {
active,
activeActivityText,
/** Stable for a given chat scope, so it never disturbs a row's memo. */
onToggleTurn: toggleExpandedTurn,
activeTurnIsUnanchored:
enabled && active != null && activeTurnKey === MOBILE_UNANCHORED_TURN_KEY,
resolveRow
}
}
mobile/src/session/use-mobile-native-chat-turn-status.ts
+6 -11
View File
@@ -1,7 +1,6 @@
import { useEffect, useMemo, useRef, useState } from 'react'
import type { NativeChatMessage } from '../../../src/shared/native-chat-types'
import {
nativeChatTurnHasResponse,
reduceNativeChatTurnTiming,
selectNativeChatTurnStatuses,
type NativeChatSettledTurns,
@@ -27,6 +26,7 @@ export function useMobileNativeChatTurnStatus({
isWorking,
workingStartedAt,
settledTurns,
thinking = false,
scopeKey
}: {
messages: readonly NativeChatMessage[]
@@ -35,6 +35,8 @@ export function useMobileNativeChatTurnStatus({
workingStartedAt?: number | null
/** Host-recorded durations; they outrank whatever this client observed. */
settledTurns?: NativeChatSettledTurns | null
/** Whether the turn is reasoning right now, derived from its journal content. */
thinking?: boolean
/** Host/worktree/tab identity. Timings never carry across chat surfaces. */
scopeKey: string
}): {
@@ -45,7 +47,6 @@ export function useMobileNativeChatTurnStatus({
const latestUserIndex = enabled
? messages.findLastIndex((message) => message.role === 'user')
: -1
const hasCurrentTurnResponse = enabled && nativeChatTurnHasResponse(messages, latestUserIndex)
const latestUserId = latestUserIndex !== -1 ? (messages[latestUserIndex]?.id ?? null) : null
const activeTurnKey = latestUserId ?? MOBILE_UNANCHORED_TURN_KEY
const [scopedTiming, setScopedTiming] = useState<ScopedTurnTiming>(() => ({
@@ -95,6 +96,7 @@ export function useMobileNativeChatTurnStatus({
// turn re-renders ~20x/s. Without this, every settled turn's row gets fresh
// props each tick and the memoized message rows all re-render.
const turnIsWorking = enabled && isWorking
const turnIsThinking = enabled && thinking
const settledByTurn = enabled ? (settledTurns ?? undefined) : undefined
const statuses = useMemo(
() =>
@@ -102,17 +104,10 @@ export function useMobileNativeChatTurnStatus({
activeTurnKey,
isWorking: turnIsWorking,
workingStartedAt,
hasCurrentTurnResponse,
thinking: turnIsThinking,
settledByTurn
}),
[
timingByTurn,
activeTurnKey,
turnIsWorking,
workingStartedAt,
hasCurrentTurnResponse,
settledByTurn
]
[timingByTurn, activeTurnKey, turnIsWorking, workingStartedAt, turnIsThinking, settledByTurn]
)
return { ...statuses, activeTurnKey }
}
mobile/src/session/use-mobile-session-terminal-create-actions.test.ts
+45
View File
@@ -263,6 +263,51 @@ describe('mobile + Codex tab creation routing', () => {
expect(scope.showToast).toHaveBeenCalledWith('create outcome ambiguous', 1800)
}
)
// Why: pty exhaustion, a disabled agent and an unresolved worktree owner all arrived as the
// same 'Failed to create terminal', leaving the empty session with nothing to act on.
it('surfaces the host reason instead of a generic terminal-create error', async () => {
const client = clientReturning({
ok: false,
error: {
code: 'runtime_error',
message: 'Your system cannot allocate any more pty devices.'
}
})
const scope = createScope(client)
let actions: ReturnType<typeof useMobileSessionTerminalCreateActions> | undefined
function Harness() {
actions = useMobileSessionTerminalCreateActions(scope as never)
return null
}
await act(async () => {
renderer = create(createElement(Harness))
})
await act(async () => {
await actions?.handleCreateTerminal()
})
expect(scope.setCreateError).toHaveBeenCalledWith(
'Your system cannot allocate any more pty devices.'
)
})
it('falls back to the generic message when the host gives no reason', async () => {
const client = clientReturning({ ok: false, error: { code: 'runtime_error', message: '' } })
const scope = createScope(client)
let actions: ReturnType<typeof useMobileSessionTerminalCreateActions> | undefined
function Harness() {
actions = useMobileSessionTerminalCreateActions(scope as never)
return null
}
await act(async () => {
renderer = create(createElement(Harness))
})
await act(async () => {
await actions?.handleCreateTerminal()
})
expect(scope.setCreateError).toHaveBeenCalledWith('Failed to create terminal')
})
})
describe('optimistic placement of a created tab', () => {

Some files were not shown because too many files have changed in this diff Show More