Merge origin/main into nwparker/piere-diffs

Only conflict was pnpm-workspace.yaml: both sides appended to minimumReleaseAgeExclude. Kept both
entries -- ours for @pierre/diffs@1.4.1 and main's for electron@43.7.0 -- and our
patchedDependencies line is untouched.
This commit is contained in:
Neil
2026-09-12 00:59:33 -07:00
1335 changed files with 77225 additions and 19830 deletions
+7
View File
@@ -31,6 +31,13 @@
# the reviewable change, and pin LF because they are compared byte-for-byte.
# Not -diff: the shell diff is the review surface when a wrapper does change.
/src/main/__fixtures__/shell-wrapper-snapshots/*.txt linguist-generated=true text eol=lf
# Captured agent PTY transcripts. -text, not `text eol=lf` like the wrapper snapshots above:
# these carry real CR and CRLF bytes as the terminal emitted them, and line-ending
# normalisation on a Windows checkout would rewrite the evidence the fixture exists to be.
/src/main/runtime/__fixtures__/*.txt -text
# Generated runtime English subset: compared byte-for-byte by
# verify:localization-runtime-catalog, so a CRLF checkout would fail the gate.
/src/renderer/src/i18n/en-runtime-required.json linguist-generated=true text eol=lf
# Generated method->params catalog: compared byte-for-byte by
# verify:rpc-params-catalog, so a CRLF checkout would fail the gate.
/src/shared/rpc-contract/rpc-params-catalog.generated.ts linguist-generated=true text eol=lf
@@ -13,6 +13,11 @@ on:
default: preserve
type: choice
options: [preserve, enable, disable]
region-correction-cohort-percent:
description: 'Preserve the measured-correction cohort, or set an integer 0–100; durable rehome stays disabled'
required: true
default: preserve
type: string
prune-incompatible-revisions:
description: Retain only the newly verified serving and rollback revisions
required: true
@@ -62,6 +67,7 @@ jobs:
REGIONAL_PLACEMENT_SECRET: orca-cloud-relay-regional-placement-enabled
IMAGE_DIGEST: ${{ inputs.image-digest }}
REGIONAL_PLACEMENT_MODE: ${{ inputs.regional-placement-mode }}
REGION_CORRECTION_COHORT_PERCENT: ${{ inputs.region-correction-cohort-percent }}
PRUNE_INCOMPATIBLE_REVISIONS: ${{ inputs.prune-incompatible-revisions }}
# Floor the served revision must keep, matching relay_min_instances in
# environments/production.tfvars. This gate only fails a bad deploy; Terraform
@@ -106,8 +112,11 @@ jobs:
echo "image-digest must be an immutable lowercase sha256 digest" >&2
exit 1
fi
if test "${REGION_CORRECTION_COHORT_PERCENT}" != preserve; then
[[ "${REGION_CORRECTION_COHORT_PERCENT}" =~ ^([0-9]|[1-9][0-9]|100)$ ]]
fi
IMAGE="${IMAGE_REPOSITORY}@${IMAGE_DIGEST}"
SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" --format='value(image_summary.digest)')"
SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
test "${SERVED_DIGEST}" = "${IMAGE_DIGEST}"
[[ "${PRUNE_INCOMPATIBLE_REVISIONS}" =~ ^(true|false)$ ]]
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
@@ -218,7 +227,8 @@ jobs:
--max-instances "${DIRECTOR_MAX_INSTANCES}" \
--prune-revisions "${PRUNE_INCOMPATIBLE_REVISIONS}" \
--release-id "${RELEASE_ID}" \
--regional-placement-secret-version "${target_version}"
--regional-placement-secret-version "${target_version}" \
--region-correction-cohort-percent "${REGION_CORRECTION_COHORT_PERCENT}"
echo "REGIONAL_PLACEMENT_ENABLED=${desired}" >> "${GITHUB_ENV}"
echo "REGIONAL_PLACEMENT_VERSION=${target_version}" >> "${GITHUB_ENV}"
@@ -67,8 +67,8 @@ jobs:
[[ "${TARGET_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
[[ "${ROLLBACK_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
test "${TARGET_IMAGE_DIGEST}" != "${ROLLBACK_IMAGE_DIGEST}"
[[ "${TARGET_REHOME_PROTOCOL}" =~ ^[01]$ ]]
[[ "${ROLLBACK_REHOME_PROTOCOL}" =~ ^[01]$ ]]
[[ "${TARGET_REHOME_PROTOCOL}" =~ ^(0|1|3)$ ]]
[[ "${ROLLBACK_REHOME_PROTOCOL}" =~ ^(0|1|3)$ ]]
[[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
[[ "${WAVE_INDEX}" =~ ^[0-3]$ ]]
@@ -599,7 +599,7 @@ jobs:
| jq -e '.control.enabled == false' >/dev/null
- name: Prove exact per-host trust and idempotent no-neighbor behavior
if: ${{ inputs.mode != 'verify' && ((inputs.mode == 'rollback' && inputs.rollback-rehome-protocol == '1') || (inputs.mode != 'rollback' && inputs.target-rehome-protocol == '1')) }}
if: ${{ inputs.mode != 'verify' && ((inputs.mode == 'rollback' && inputs.rollback-rehome-protocol != '0') || (inputs.mode != 'rollback' && inputs.target-rehome-protocol != '0')) }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
run: |
@@ -26,13 +26,13 @@ on:
required: true
default: '1'
type: choice
options: ['0', '1']
options: ['0', '1', '3']
rollback-rehome-protocol:
description: Exact rollback regional-rehome protocol
required: true
default: '0'
type: choice
options: ['0', '1']
options: ['0', '1', '3']
expected-selector-generation:
description: Exact selector generation before the first cell
required: true
@@ -62,7 +62,7 @@ on:
required: false
type: string
canary-run-id:
description: Successful same-commit canary run required for batch-apply
description: Successful same-code canary in this rehome control generation; reusable across batches
required: false
type: string
confirmation:
+5
View File
@@ -173,6 +173,11 @@ jobs:
- name: Run E2E tests (${{ matrix.shard_name }})
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 ORCA_E2E_WEB_CLIENT=1 ORCA_RELAY_PATH="$GITHUB_WORKSPACE/out/relay" pnpm run test:e2e --shard=${{ matrix.shard }}
# The frame benchmark needs a mapped window, which the headless shards exclude.
- name: Run worktree first-paint benchmark
if: matrix.shard == '1/14'
run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm exec playwright test tests/e2e/worktree-switch-first-paint.spec.ts --config tests/playwright.config.ts --project=electron-headful --workers=1
# Why: Playwright retains traces/screenshots only on failure. Uploading
# them as an artifact makes post-mortem debugging on CI possible without
# re-running locally.
+2
View File
@@ -94,6 +94,8 @@ jobs:
run: node -e 'const fs = require("node:fs"); const { expo } = require("./app.json"); fs.appendFileSync(process.env.GITHUB_OUTPUT, `version=${expo.version}\nbuild_number=${expo.ios.buildNumber}\n`)'
- name: Expo prebuild
env:
ORCA_IOS_APS_ENVIRONMENT: production
run: npx expo prebuild --platform ios --no-install
- name: Install CocoaPods
+29
View File
@@ -0,0 +1,29 @@
name: Pi owner runtime verification
on:
pull_request:
paths:
- 'src/main/pi/agent-status-handler-source.ts'
- 'tests/tools/pi-owner-runtime-smoke.mjs'
- '.github/workflows/pi-owner-runtime.yml'
workflow_dispatch:
permissions:
contents: read
jobs:
runtime:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
timeout-minutes: 20
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
- name: Install pinned extension loader
run: npm install --prefix .cache/pi-owner --ignore-scripts --no-audit --no-fund @earendil-works/pi-coding-agent@0.83.0
- name: Verify real owner exit and hook delivery
run: node tests/tools/pi-owner-runtime-smoke.mjs .cache/pi-owner/node_modules/@earendil-works/pi-coding-agent
+28
View File
@@ -0,0 +1,28 @@
name: Pi extension provider verification
on:
pull_request:
paths:
- 'src/shared/commit-message-agent-specs-primary.ts'
- 'tests/tools/pi-provider-runtime-smoke.mjs'
- '.github/workflows/pi-provider-runtime.yml'
permissions:
contents: read
jobs:
runtime:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
timeout-minutes: 20
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
- name: Install pinned Pi runtime
run: npm install --prefix .cache/pi-provider --ignore-scripts --no-audit --no-fund @earendil-works/pi-coding-agent@0.84.2
- name: Verify extension model generation before and after
run: node tests/tools/pi-provider-runtime-smoke.mjs .cache/pi-provider/node_modules/@earendil-works/pi-coding-agent/dist/cli.js
+3
View File
@@ -203,6 +203,9 @@ jobs:
- name: Boot orcad and round-trip a terminal
run: pnpm run smoke:orcad-terminal
- name: Verify the generated RPC params catalog
run: pnpm run verify:rpc-params-catalog
- name: Verify bundled skill guides
run: pnpm run verify:bundled-skill-guides
+7
View File
@@ -37,6 +37,13 @@ jobs:
- name: Install Electron package binary for tests
run: node config/scripts/install-electron-package-binary.mjs
# The real two-cell transport test imports cloud relay source and its contracts.
- name: Install relay integration dependencies
working-directory: cloud
run: |
npx --yes pnpm@10.24.0 --filter '@orca-cloud/relay...' install --frozen-lockfile --ignore-scripts
npx --yes pnpm@10.24.0 --filter '@orca-cloud/relay^...' build
- name: Test shard
run: |
pnpm exec vitest run --config config/vitest.config.ts \
+3
View File
@@ -103,7 +103,10 @@ docs/**
!docs/agent-skill-sharing-implementation-checklist.md
!docs/mobile-terminal-shortcut-bar.md
!docs/reference/
!docs/reference/agent-pty-transcript-capture.md
!docs/reference/agent-session-search-query-tuning.md
!docs/reference/agent-status-store.md
!docs/reference/antigravity-readiness-evidence.md
!docs/reference/git-compatibility.md
!docs/reference/headless-linux-server.md
!docs/reference/ime-regression-checklist.md
+1
View File
@@ -180,6 +180,7 @@
}
],
"ignorePatterns": [
"src/shared/rpc-contract/rpc-params-catalog.generated.ts",
"**/node_modules",
"**/dist",
"**/out",
+4
View File
@@ -72,6 +72,10 @@ All changes must consider folder workspaces as well as git worktrees. Don't assu
The execution host owns agent status in one store, the hook server's, and every reader (sidebar, `worktree ps`, mobile, dashboard) subscribes to it. Before adding a producer, a cache, or a reader-side precedence rule, read [`docs/reference/agent-status-store.md`](./docs/reference/agent-status-store.md): new producers write into that store, and readers keep only presentation policy.
## Agent Terminal Screens
A rule that reads what an agent CLI paints on a terminal — readiness, blocked prompts, idle — must be written against a captured transcript, not a remembered screen. Record one with [`docs/reference/agent-pty-transcript-capture.md`](./docs/reference/agent-pty-transcript-capture.md), which keeps escapes and wrapping intact and scrubs account identifiers before they reach git. Antigravity readiness has no transcript yet and five failed attempts without one; before touching it, read [`docs/reference/antigravity-readiness-evidence.md`](./docs/reference/antigravity-readiness-evidence.md).
## Remote Wire Compatibility
Clients and remote Orca servers update independently, so mixed versions are the normal state. Before changing anything a paired client and host exchange — RPC params, stream frames, or the content either side publishes over them — follow [`docs/reference/remote-wire-compatibility.md`](./docs/reference/remote-wire-compatibility.md). A new optional field is safe; a new stream opcode must be capability-negotiated because decoders drop unknown opcodes silently; and changing what the host publishes reaches old clients even with no wire change.
+13 -24
View File
@@ -1,6 +1,6 @@
import { createHash } from 'node:crypto'
import { describe, expect, it } from 'vitest'
import { fcmCollapseKey, FcmClient, type FcmRequest, type FcmResponse } from './fcm-client.js'
import { FcmClient, type FcmRequest, type FcmResponse } from './fcm-client.js'
import { buildPushDelivery } from './push-delivery-message.js'
const NOW = 1_700_000_000_000
@@ -20,6 +20,7 @@ function delivery(agentState: 'needs-input' | null = 'needs-input') {
agentState,
title: 'Agent needs input',
body: 'Waiting on your answer',
paneKey: 'tab-b:pane-1',
worktreeId: 'wt-1'
}
})
@@ -59,27 +60,14 @@ describe('fcm client', () => {
expect(JSON.parse(request.body)).toEqual({
message: {
token: TOKEN,
notification: { title: 'Agent needs input', body: 'Waiting on your answer' },
android: {
priority: 'HIGH',
ttl: '300s',
collapse_key: createHash('sha256')
.update(
createHash('sha256')
.update(JSON.stringify([HOST, 'note-1']))
.digest('hex')
)
.digest('hex')
.slice(0, 32),
notification: {
channel_id: 'orca-desktop',
tag: createHash('sha256')
.update(JSON.stringify([HOST, 'note-1']))
.digest('hex')
}
},
android: { priority: 'HIGH', ttl: '300s' },
data: {
title: 'Agent needs input',
message: 'Waiting on your answer',
tag: delivery().collapseId,
channelId: 'orca-desktop',
hostFingerprint: HOST,
paneKey: 'tab-b:pane-1',
worktreeId: 'wt-1',
notificationId: 'note-1',
notificationSeq: '7',
@@ -96,7 +84,7 @@ describe('fcm client', () => {
await fcm.send(delivery(null), { token: TOKEN })
const message = JSON.parse(fake.requests[0]!.body) as {
message: {
android: { collapse_key: string; notification: { tag: string } }
android: Record<string, unknown>
data: Record<string, string>
}
}
@@ -108,9 +96,10 @@ describe('fcm client', () => {
.update(JSON.stringify([HOST, 'note-1']))
.digest('hex')
expect(message.message.data.coalescedCount).toBeUndefined()
expect(message.message.android.notification.tag).toBe(tag)
expect(message.message.android.collapse_key).toBe(fcmCollapseKey(tag))
expect(message.message.android.collapse_key).toHaveLength(32)
expect(message.message.data.tag).toBe(tag)
expect(message.message.android).not.toHaveProperty('collapse_key')
expect(message.message).not.toHaveProperty('notification')
expect(message.message.data).not.toHaveProperty('body')
})
it('marks an unregistered token dead from the status or the error detail', async () => {
+11 -19
View File
@@ -1,5 +1,4 @@
import { providerRetryAfter } from './provider-retry-delay.js'
import { createHash } from 'node:crypto'
import { PUSH_DEFAULTS } from '@orca-cloud/push-contract'
import { orcaDataStrings, type PushDelivery } from './push-delivery-message.js'
import type { PushProviderOutcome } from './push-provider-outcome.js'
@@ -22,12 +21,6 @@ type FcmErrorBody = {
error?: { status?: unknown; message?: unknown; details?: { errorCode?: unknown }[] }
}
// FCM collapse_key is a short opaque string, so the collapse id is hashed
// rather than truncated: truncation would merge unrelated notifications.
export function fcmCollapseKey(collapseId: string): string {
return createHash('sha256').update(collapseId).digest('hex').slice(0, 32)
}
export function fcmMessageBody(input: {
delivery: PushDelivery
token: string
@@ -39,24 +32,23 @@ export function fcmMessageBody(input: {
return JSON.stringify({
message: {
token: input.token,
...(delivery.orca.kind === 'dismiss'
? {}
: { notification: { title: delivery.title, body: delivery.body } }),
android: {
priority: 'HIGH',
ttl: `${Math.max(0, Math.ceil((delivery.expiresAt - now) / 1000))}s`,
collapse_key: fcmCollapseKey(delivery.collapseId),
ttl: `${Math.max(0, Math.ceil((delivery.expiresAt - now) / 1000))}s`
},
// Notification payloads collapse offline; Expo renders these data messages natively.
data: {
...orcaDataStrings(delivery.orca),
...(delivery.orca.kind === 'dismiss'
? {}
: {
notification: {
channel_id:
delivery.sound === false ? `${input.channelId}-silent` : input.channelId,
tag: delivery.collapseId
}
title: delivery.title,
message: delivery.body,
tag: delivery.collapseId,
channelId: delivery.sound === false ? `${input.channelId}-silent` : input.channelId,
...(delivery.sound === false ? { sound: '' } : {})
})
},
data: orcaDataStrings(delivery.orca)
}
}
})
}
@@ -5,6 +5,7 @@ export type PushOrcaData = {
kind?: 'alert' | 'dismiss'
hostFingerprint: string
worktreeId?: string
paneKey?: string
notificationId?: string
notificationSeq: number
notificationEpoch: string
@@ -51,6 +52,7 @@ export function buildPushDelivery(input: {
orca: {
...(notification.kind ? { kind: notification.kind } : {}),
hostFingerprint,
...(notification.paneKey === undefined ? {} : { paneKey: notification.paneKey }),
...(notification.worktreeId === undefined ? {} : { worktreeId: notification.worktreeId }),
...(notification.notificationId === undefined
? {}
@@ -23,5 +23,9 @@ it('dismissal provider payloads cannot display a new alert or play a sound', ()
const android = JSON.parse(fcmMessageBody({ delivery, token: 'test', channelId: 'test' })).message
expect(android).not.toHaveProperty('notification')
expect(android.android).not.toHaveProperty('notification')
expect(android.android).not.toHaveProperty('collapse_key')
expect(android.data).not.toHaveProperty('title')
expect(android.data).not.toHaveProperty('message')
expect(android.data).not.toHaveProperty('sound')
expect(android.data.kind).toBe('dismiss')
})
@@ -23,7 +23,11 @@ it('carries a silent preference through validation to APNs and Android payloads'
expect(JSON.parse(apnsBody(delivery)).aps).not.toHaveProperty('sound')
expect(
JSON.parse(fcmMessageBody({ delivery, token: 'test-token', channelId: 'orca-desktop' })).message
.android.notification.channel_id
.data.channelId
).toBe('orca-desktop-silent')
expect(
JSON.parse(fcmMessageBody({ delivery, token: 'test-token', channelId: 'orca-desktop' })).message
.data.sound
).toBe('')
expect(JSON.parse(apnsBody({ ...delivery, sound: undefined })).aps.sound).toBe('default')
})
@@ -0,0 +1,27 @@
import { expect, it } from 'vitest'
import { PushNotificationSchema } from '@orca-cloud/push-contract'
import { buildPushDelivery, orcaDataStrings } from './push-delivery-message.js'
it('preserves pane identity for both APNs and FCM, and accepts older workspace-only messages', () => {
const base = {
notificationSeq: 1,
notificationEpoch: 'epoch',
source: 'agent-task-complete',
agentState: 'finished',
title: 'Done',
body: '',
worktreeId: 'folder:/work'
}
const paneKey = 'tab-b:11111111-1111-4111-8111-111111111111'
for (const extra of [{}, { paneKey }]) {
const notification = PushNotificationSchema.parse({ ...base, ...extra })
const delivery = buildPushDelivery({
notification,
hostFingerprint: 'host',
registrationId: 'phone',
expiresAt: Date.now() + 300000
})
expect(delivery.orca.paneKey).toBe('paneKey' in extra ? paneKey : undefined)
expect(orcaDataStrings(delivery.orca).paneKey).toBe('paneKey' in extra ? paneKey : undefined)
}
})
+2 -2
View File
@@ -56,7 +56,7 @@ describe('push gateway send route', () => {
await harness.flushDeliveries()
expect(harness.fcmRequests).toHaveLength(1)
expect(JSON.parse(harness.fcmRequests[0]!.body)).toMatchObject({
message: { token: FCM_TOKEN, notification: { title: 'Agent needs input' } }
message: { token: FCM_TOKEN, data: { title: 'Agent needs input' } }
})
const afterDeath = await harness.post(
@@ -179,7 +179,7 @@ describe('push gateway send route', () => {
const message = JSON.parse(harness.fcmRequests[0]!.body) as {
message: { android: { notification: { tag: string } }; data: Record<string, string> }
}
expect(message.message.android.notification.tag).toMatch(/^[a-f0-9]{64}$/)
expect(message.message.data.tag).toMatch(/^[a-f0-9]{64}$/)
expect(message.message.data.coalescedCount).toBeUndefined()
})
@@ -273,6 +273,26 @@ describe('incident monitor evaluator', () => {
)
})
it('allows at most three unexpected director errors per five minutes without relaxing other gates', () => {
for (const errors of [1, 2, 3]) {
const sample = healthySample()
sample.sources['cloud-monitoring']!.signals['director.errors'] = signal(errors)
expect(evaluateIncidentSample(sample, startedAt).status).toBe('green')
}
const excess = healthySample()
excess.sources['cloud-monitoring']!.signals['director.errors'] = signal(4)
expect(evaluateIncidentSample(excess, startedAt).failures).toContainEqual(
expect.objectContaining({ signal: 'director.errors', observed: 4, threshold: 3 })
)
const auth = healthySample()
auth.sources['cloud-monitoring']!.signals['auth.errors'] = signal(1)
expect(evaluateIncidentSample(auth, startedAt).status).toBe('freeze')
const pressure = healthySample()
pressure.sources['cloud-monitoring']!.signals['director.errors'] = signal(1)
pressure.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.81)
expect(evaluateIncidentSample(pressure, startedAt).status).toBe('freeze')
})
it('freezes on SQL, director, relay pool, heartbeat, and migration breaches', () => {
const sample = healthySample()
sample.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.81)
+2 -1
View File
@@ -101,7 +101,8 @@ export const INCIDENT_MONITOR_THRESHOLDS = {
directorCpuUtilization: 0.8,
directorMemoryUtilization: 0.8,
directorConcurrency: 64,
directorErrors: 0,
// Sparse connection timeouts must not block a healthy rollout; four/5min still freezes.
directorErrors: 3,
authErrors: 0,
// Why: 800 exceeded the 600 hard cap, so this could never trigger on a capped cell. 500 is
// the ordinary admission limit a cell actually stops at (600 cap - 100 control-rebind reserve).
@@ -6,6 +6,7 @@ export const RELAY_MONITOR_ADMIN_ROUTES = [
'/v1/admin/cell-status',
'/v1/admin/evacuation-status',
'/v1/admin/regional-rehome-control',
'/v1/admin/regional-rehome-preview',
'/v1/admin/runtime-status'
] as const
+137 -51
View File
@@ -1,5 +1,9 @@
import {
AssignmentRequestSchema,
IdleRegionalRehomeRequestSchema,
type IdleRegionalRehomeRequest,
type IdleRegionalRehomeOutcome,
type RegionCorrectionResponse,
isRelayCellConnectionHardCap,
RELAY_ADMISSION_BUDGETS,
RELAY_DEFAULT_REGION,
@@ -39,7 +43,7 @@ import {
type AssignmentAdmissionRejection
} from './public-assignment-admission.js'
import { relayHostLogDigest } from './relay-host-log-digest.js'
import type { RelayRuntimeCounts } from './relay-observability.js'
import type { RegionalRehomeSafetySnapshot, RelayRuntimeCounts } from './relay-observability.js'
import {
isRegionalRehomeTrustProbe,
probeRegionalRehomeTrust
@@ -68,21 +72,28 @@ export function createRelayApp(
store: RelayCredentialStore
assignments: RelayAssignmentStore
drain: (graceMs: number) => void
idleRehome?: (input: IdleRegionalRehomeRequest & {
cohortPercent: number
directorSafety: RegionalRehomeSafetySnapshot
}) => Promise<{ outcome: IdleRegionalRehomeOutcome }>
drainHost?: (input: {
attemptId: string
userId: string
relayHostId: string
sourceAssignmentEpoch: number
sourceCellIncarnation: string
graceMs: number
}) => 'accepted' | 'already-accepted' | 'host-not-connected'
}) =>
| 'accepted'
| 'already-accepted'
| 'host-not-connected'
| Promise<'accepted' | 'already-accepted' | 'host-not-connected'>
regionalRehomeIdentityToken?: (audience: string) => Promise<string>
regionalRehomeFetch?: typeof fetch
regionalRehomeTrustProbeHostExists?: (input: {
userId: string
relayHostId: string
}) => boolean
regionalRehomeTrustProbeHostExists?: (input: { userId: string; relayHostId: string }) => boolean
cellIncarnation?: string
isDraining?: () => boolean
regionalRehomeSafetySnapshot?: () => RegionalRehomeSafetySnapshot
runtimeCounts?: () => RelayRuntimeCounts
ready: () => Promise<boolean>
recordAssignmentAdmission?: (
@@ -226,7 +237,8 @@ export function createRelayApp(
return context.json({ error: 'host_identity_mismatch' }, 403)
}
const identity = { userId: claims.sub, relayHostId: claims.relayHostId }
const requestedRegion = body.data.preferredRegion
const requestedRegion =
body.data.regionCorrection?.action === 'report' ? undefined : body.data.preferredRegion
const targetRegion =
config.regionalPlacementEnabled !== false && requestedRegion
? requestedRegion
@@ -295,10 +307,30 @@ export function createRelayApp(
}
}
let assignment: RelayAssignment
let regionCorrection: RegionCorrectionResponse | undefined
try {
assignment = requestedRegion
? await operations.assignments.assign(identity, requestedRegion, targetRegion)
: await operations.assignments.assign(identity)
if (body.data.regionCorrection?.action === 'report') {
const current = await operations.assignments.resolve(identity)
if (!current) return context.json({ error: 'assignment_not_found' }, 409)
assignment = current
} else {
assignment = requestedRegion
? await operations.assignments.assign(identity, requestedRegion, targetRegion)
: await operations.assignments.assign(identity)
}
if (body.data.regionCorrection) {
try {
regionCorrection = await operations.assignments.exchangeRegionCorrection(
identity,
body.data.regionCorrection,
assignment.assignmentEpoch
)
} catch (error) {
if (body.data.regionCorrection.action === 'report') throw error
// Optional measurement setup must not discard an otherwise valid placement.
console.warn(JSON.stringify({ event: 'orca_relay_region_window_unavailable' }))
}
}
} catch (error) {
if (isRelayAssignmentCapacityError(error) || isRelayDatabaseTransientError(error)) {
logAssignmentRejection({
@@ -353,13 +385,16 @@ export function createRelayApp(
v: 1,
cellUrl: assignment.cellUrl,
assignmentEpoch: assignment.assignmentEpoch,
lease
lease,
...(regionCorrection ? { regionCorrection } : {})
})
})
app.post('/v1/resolve', async (context) => {
if (config.role === 'cell') return context.json({ error: 'director_only' }, 404)
if (!config.publicAssignmentsEnabled) return rejectPublicAssignment(context)
if (Number(context.req.header('content-length') ?? 0) > RELAY_PROTOCOL_LIMITS.maxHttpBodyBytes) {
if (
Number(context.req.header('content-length') ?? 0) > RELAY_PROTOCOL_LIMITS.maxHttpBodyBytes
) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = ResolveRequestSchema.safeParse(await context.req.json().catch(() => null))
@@ -433,6 +468,34 @@ export function createRelayApp(
operations.drain(body.data.graceMs)
return context.json({ ok: true })
})
app.post('/v1/admin/host-idle-rehome', async (context) => {
if (config.role !== 'cell' || !operations.idleRehome) {
return context.json({ error: 'cell_only' }, 404)
}
const bearer = readBearer(context.req.header('authorization'))
if (!bearer || !(await verifyRegionalRehomeToken(bearer))) {
return context.json({ error: 'invalid_token' }, 401)
}
if (requestTooLarge(context.req.header('content-length'))) {
return context.json({ error: 'request_too_large' }, 413)
}
const body = IdleRegionalRehomeCommandSchema.safeParse(
await context.req.json().catch(() => null)
)
if (!body.success) return context.json({ error: 'invalid_request' }, 400)
if (
body.data.sourceCellId !== config.cellId ||
!operations.cellIncarnation ||
body.data.sourceCellIncarnation !== operations.cellIncarnation
) {
return context.json({ error: 'regional_rehome_source_generation_mismatch' }, 409)
}
try {
return context.json({ v: 1, ...(await operations.idleRehome(body.data)) })
} catch (error) {
return context.json({ error: operationError(error) }, 409)
}
})
app.post('/v1/admin/host-drain', async (context) => {
if (config.role !== 'cell' || !operations.drainHost) {
return context.json({ error: 'cell_only' }, 404)
@@ -474,7 +537,7 @@ export function createRelayApp(
}
sharedRuntimeIdentityRejected = true
}
const outcome = operations.drainHost(body.data)
const outcome = await operations.drainHost(body.data)
return context.json({
v: 1,
outcome,
@@ -502,8 +565,7 @@ export function createRelayApp(
region: config.region ?? RELAY_DEFAULT_REGION,
imageDigest: config.imageDigest ?? null,
draining: operations.isDraining?.() ?? false,
regionalRehomeProtocol:
config.rehomeAudience && config.rehomeDirectorServiceAccount ? 1 : 0,
regionalRehomeProtocol: config.rehomeAudience && config.rehomeDirectorServiceAccount ? 3 : 0,
connectionCapacity:
config.connectionHardCap === undefined
? null
@@ -559,6 +621,18 @@ export function createRelayApp(
return context.json({ error: operationError(error) }, 409)
}
})
app.get('/v1/admin/regional-rehome-preview', async (context) => {
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
const bearer = readBearer(context.req.header('authorization'))
if (!bearer || !(await verifyAdminToken(bearer, context.req.path))) {
return context.json({ error: 'invalid_token' }, 401)
}
const preview = await operations.assignments.previewRegionalRehomeEligibility(
operations.regionalRehomeSafetySnapshot?.()
)
const outcomes = await operations.assignments.regionCorrectionOutcomes()
return context.json({ v: 1, preview, outcomes })
})
app.post('/v1/admin/regional-rehome-control', async (context) => {
if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
const bearer = readBearer(context.req.header('authorization'))
@@ -1295,6 +1369,11 @@ const RegionalRehomeSafetySchema = z
})
.strict()
const IdleRegionalRehomeCommandSchema = IdleRegionalRehomeRequestSchema.extend({
cohortPercent: z.number().int().min(0).max(100),
directorSafety: RegionalRehomeSafetySchema
})
const CellHeartbeatSchema = z
.object({
v: z.literal(1),
@@ -1394,45 +1473,48 @@ const CellRegionalRehomeStatusSchema = z
v: z.literal(1),
cellId: z.string().min(1).max(128),
cellIncarnation: z.string().uuid(),
regionalRehomeProtocol: z.number().int().min(0).max(1),
regionalRehomeProtocol: z.number().int().min(0).max(3),
safety: RegionalRehomeSafetySchema
})
.strict()
const RegionalRehomeControlSchema = z.discriminatedUnion('action', [
z.object({ v: z.literal(1), action: z.literal('inspect') }).strict(),
z.object({
v: z.literal(1),
action: z.literal('apply'),
expectedGeneration: z.number().int().nonnegative(),
enabled: z.boolean(),
notBefore: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
ratePerMinute: z.number().int().min(1).max(120),
preferenceMaxAgeMs: z
.number()
.int()
.min(60_000)
.max(30 * 24 * 60 * 60_000),
hostCooldownMs: z
.number()
.int()
.min(60_000)
.max(30 * 24 * 60 * 60_000),
drainGraceMs: z.number().int().min(60_000).max(60 * 60_000),
confirmation: z.enum([
'ENABLE_REGIONAL_REHOMING',
'DISABLE_REGIONAL_REHOMING'
])
}).strict()
]).superRefine((value, context) => {
if (value.action !== 'apply') return
const expected = value.enabled
? 'ENABLE_REGIONAL_REHOMING'
: 'DISABLE_REGIONAL_REHOMING'
if (value.confirmation !== expected) {
context.addIssue({ code: 'custom', message: 'confirmation does not match state' })
}
})
const RegionalRehomeControlSchema = z
.discriminatedUnion('action', [
z.object({ v: z.literal(1), action: z.literal('inspect') }).strict(),
z
.object({
v: z.literal(1),
action: z.literal('apply'),
expectedGeneration: z.number().int().nonnegative(),
enabled: z.boolean(),
notBefore: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
ratePerMinute: z.number().int().min(1).max(120),
preferenceMaxAgeMs: z
.number()
.int()
.min(60_000)
.max(30 * 24 * 60 * 60_000),
hostCooldownMs: z
.number()
.int()
.min(60_000)
.max(30 * 24 * 60 * 60_000),
drainGraceMs: z
.number()
.int()
.min(60_000)
.max(60 * 60_000),
confirmation: z.enum(['ENABLE_REGIONAL_REHOMING', 'DISABLE_REGIONAL_REHOMING'])
})
.strict()
])
.superRefine((value, context) => {
if (value.action !== 'apply') return
const expected = value.enabled ? 'ENABLE_REGIONAL_REHOMING' : 'DISABLE_REGIONAL_REHOMING'
if (value.confirmation !== expected) {
context.addIssue({ code: 'custom', message: 'confirmation does not match state' })
}
})
const RegionalRehomeTrustProbeSchema = z
.object({
@@ -1776,7 +1858,11 @@ const RegionalHostDrainSchema = z
sourceCellId: z.string().min(1).max(128),
sourceCellIncarnation: z.string().uuid(),
sourceAssignmentEpoch: z.number().int().positive(),
graceMs: z.number().int().nonnegative().max(60 * 60 * 1000)
graceMs: z
.number()
.int()
.nonnegative()
.max(60 * 60 * 1000)
})
.strict()
File diff suppressed because it is too large Load Diff
@@ -92,7 +92,7 @@ describe('cell heartbeat client', () => {
client.stop()
expect(JSON.parse(String(requests[1]!.body))).toMatchObject({
regionalRehomeProtocol: 1,
regionalRehomeProtocol: 3,
safety: {
observedAt: 120,
sqlFailures: 0,
@@ -149,28 +149,34 @@ describe('cell heartbeat client', () => {
it('does not start outside an explicitly configured cell role', () => {
expect(
startCellHeartbeat({ ...CONFIG, role: 'director' }, {
ready: async () => true,
observedRequests: () => 0,
connectionCounts: () => ({
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0
})
})
startCellHeartbeat(
{ ...CONFIG, role: 'director' },
{
ready: async () => true,
observedRequests: () => 0,
connectionCounts: () => ({
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0
})
}
)
).toBeNull()
expect(
startCellHeartbeat({ ...CONFIG, directorUrl: undefined }, {
ready: async () => true,
observedRequests: () => 0,
connectionCounts: () => ({
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0
})
})
startCellHeartbeat(
{ ...CONFIG, directorUrl: undefined },
{
ready: async () => true,
observedRequests: () => 0,
connectionCounts: () => ({
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0
})
}
)
).toBeNull()
})
})
@@ -70,8 +70,7 @@ export function startCellHeartbeat(
inFlightConnections: connectionCounts!.inFlightConnections,
reservedConnectionUnits: connectionCounts!.reservedConnectionUnits,
enforcedConnectionUnits: connectionCounts!.enforcedConnectionUnits,
connectionInclusionWatermark:
connectionCounts!.inclusionWatermark,
connectionInclusionWatermark: connectionCounts!.inclusionWatermark,
connectionHardCap: config.connectionHardCap,
connectionUnobservedBound: config.connectionUnobservedBound
})
@@ -94,7 +93,7 @@ export function startCellHeartbeat(
cellId: config.cellId,
cellIncarnation,
regionalRehomeProtocol:
config.rehomeAudience && config.rehomeDirectorServiceAccount ? 1 : 0,
config.rehomeAudience && config.rehomeDirectorServiceAccount ? 3 : 0,
safety: options.regionalRehomeSafety()
}),
signal: AbortSignal.timeout(10_000)
@@ -106,7 +105,10 @@ export function startCellHeartbeat(
}
} catch (error) {
// A heartbeat must fail closed without ever logging its bearer token.
console.warn('[orca-relay] cell heartbeat failed', error instanceof Error ? error.message : '')
console.warn(
'[orca-relay] cell heartbeat failed',
error instanceof Error ? error.message : ''
)
} finally {
inFlight = false
}
@@ -43,14 +43,13 @@ const CENSUS: CensusEntry[] = [
{ method: 'completeEvacuation', mode: 'nowait', reach: 'both' },
{ method: 'completeEvacuation', mode: 'pool-default', reach: 'both' },
{ method: 'rebalanceDormant', mode: 'request', reach: 'request' },
{ method: 'startRegionalRehomeCandidate', mode: 'nowait', reach: 'sweep' },
{ method: 'lockedRegionalRehomeFleetSafety', mode: 'nowait', reach: 'sweep' },
{ method: 'startRegionalRehomeCandidate', mode: 'nowait', reach: 'request' },
{ method: 'completeRegionalRehomeCandidate', mode: 'nowait', reach: 'sweep' },
{ method: 'abortExpiredRegionalRehomes', mode: 'nowait', reach: 'sweep' },
{ method: 'abortExpiredEvacuations', mode: 'nowait', reach: 'sweep' },
{ method: 'abortExpiredEvacuations', mode: 'nowait', reach: 'sweep' },
{ method: 'releaseExpiredActivityLeases', mode: 'nowait', reach: 'sweep' },
{ method: 'releaseExpiredActivity', mode: 'nowait', reach: 'sweep' },
{ method: 'releaseExpiredActivity', mode: 'nowait', reach: 'sweep' }
// reconcileReservationAccounting and leastLoadedCell are gone too: the first
// repairs exactly two cells' counters and now holds only those rows, and the
// second selects from the inventory its single caller has already locked.
@@ -119,9 +118,10 @@ function storeCallGraph(lines: string[]): Map<string, Set<string>> {
bounds.forEach((method, index) => {
const end = bounds[index + 1]?.start ?? lines.length
const names = callees.get(method.name) ?? new Set<string>()
for (const call of lines.slice(method.start, end).join('\n').matchAll(
/this\.([A-Za-z_][\w]*)\s*\(/g
)) {
for (const call of lines
.slice(method.start, end)
.join('\n')
.matchAll(/this\.([A-Za-z_][\w]*)\s*\(/g)) {
names.add(call[1]!)
}
callees.set(method.name, names)
@@ -174,9 +174,7 @@ function readCallSites(): { method: string; mode: CensusMode }[] {
describe('cell inventory lock call-site census', () => {
it('classifies every call site exactly as recorded', () => {
expect(readCallSites()).toEqual(
CENSUS.map(({ method, mode }) => ({ method, mode }))
)
expect(readCallSites()).toEqual(CENSUS.map(({ method, mode }) => ({ method, mode })))
})
// Why: the census only sees lockCellInventory calls, so a hand-written
+11
View File
@@ -25,6 +25,17 @@ function cellEnvironment(capacity: number): NodeJS.ProcessEnv {
}
describe('GCE relay capacity configuration', () => {
it('defaults optional region correction off and bounds the cohort', () => {
const env = cellEnvironment(4_000)
expect(loadRelayConfig(env).regionCorrectionCohortPercent).toBe(0)
env.ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT = '5'
expect(loadRelayConfig(env).regionCorrectionCohortPercent).toBe(5)
for (const invalid of ['-1', '101', '1.5', 'not-a-number']) {
env.ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT = invalid
expect(() => loadRelayConfig(env)).toThrow()
}
})
it('requires distinct dedicated admin identities and accepts omitted values', () => {
const env = cellEnvironment(4_000)
expect(loadRelayConfig(env)).toMatchObject({
+7 -1
View File
@@ -75,11 +75,15 @@ const EnvSchema = z.object({
ORCA_RELAY_RUNTIME_SERVICE_ACCOUNT: z.string().email().optional(),
ORCA_RELAY_DIRECTOR_URL: z.string().url().optional(),
ORCA_RELAY_HEARTBEAT_AUDIENCE: z.string().url().optional(),
ORCA_RELAY_IMAGE_DIGEST: z.string().regex(/^sha256:[a-f0-9]{64}$/).optional(),
ORCA_RELAY_IMAGE_DIGEST: z
.string()
.regex(/^sha256:[a-f0-9]{64}$/)
.optional(),
ORCA_RELAY_ADMIN_JWKS_URL: z.string().url().default('https://www.googleapis.com/oauth2/v3/certs'),
ORCA_RELAY_DATABASE_POOL_MAX: z.coerce.number().int().positive().max(100).optional(),
ORCA_RELAY_PUBLIC_ASSIGNMENTS_ENABLED: EnvironmentBooleanSchema,
ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED: EnvironmentBooleanSchema,
ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT: z.coerce.number().int().min(0).max(100).default(0),
ORCA_RELAY_PUBLIC_ASSIGNMENT_CONCURRENCY: z.coerce.number().int().positive().max(100).default(2),
ORCA_RELAY_PUBLIC_STICKY_CONCURRENCY: z.coerce.number().int().positive().max(100).default(1),
ORCA_RELAY_PUBLIC_STICKY_QUEUE_MAX: z.coerce.number().int().positive().max(4_096).default(64),
@@ -185,6 +189,7 @@ export type RelayConfig = {
databasePoolMax: number
publicAssignmentsEnabled: boolean
regionalPlacementEnabled?: boolean
regionCorrectionCohortPercent?: number
publicAssignmentConcurrency: number
publicAssignmentQueueMax: number
publicAssignmentWaitMs: number
@@ -332,6 +337,7 @@ export function loadRelayConfig(env: NodeJS.ProcessEnv = process.env): RelayConf
databasePoolMax,
publicAssignmentsEnabled: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENTS_ENABLED,
regionalPlacementEnabled: parsed.ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED,
regionCorrectionCohortPercent: parsed.ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT,
publicAssignmentConcurrency: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENT_CONCURRENCY,
publicAssignmentQueueMax: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENT_QUEUE_MAX,
publicAssignmentWaitMs: parsed.ORCA_RELAY_PUBLIC_ASSIGNMENT_WAIT_MS,
+32 -6
View File
@@ -18,6 +18,34 @@ afterEach(() => {
})
describe('relay database', () => {
it('upgrades an existing SQLite relay without treating legacy controls as idle-capable', async () => {
const dataDir = mkdtempSync(join(tmpdir(), 'orca-idle-schema-'))
temporaryDirectories.push(dataDir)
const legacy = await openRelayDatabase({ dataDir })
await legacy.query('ALTER TABLE relay_control_capabilities DROP COLUMN idle_regional_rehome')
await legacy.query('ALTER TABLE relay_region_rehome_attempts DROP COLUMN source_generation')
await legacy.query(
`INSERT INTO relay_control_capabilities
(user_id, relay_host_id, activity_id, cell_id, cell_incarnation, assignment_epoch, generation, finish_existing)
VALUES ('legacy-user', 'abcdefghijklmnop', 'control:source:1', 'source', 'legacy-incarnation', 1, 1, 1)`
)
await legacy.close()
const upgraded = await openRelayDatabase({ dataDir })
try {
expect(
await upgraded.query('SELECT idle_regional_rehome FROM relay_control_capabilities')
).toEqual([{ idle_regional_rehome: 0 }])
const columns = await upgraded.query(
"SELECT * FROM pragma_table_info('relay_region_rehome_attempts')"
)
expect(columns.find((column) => column.name === 'source_generation')).toMatchObject({
dflt_value: '0'
})
} finally {
await upgraded.close()
}
})
it('creates every durable relay state table', async () => {
const database = await openInMemoryRelayDatabase()
const rows = await database.query(
@@ -54,6 +82,7 @@ describe('relay database', () => {
'relay_confirm_results',
'relay_confirmable_splices',
'relay_connection_bases',
'relay_control_capabilities',
'relay_control_connection_reservations',
'relay_devices',
'relay_direct_authorizations',
@@ -62,6 +91,7 @@ describe('relay database', () => {
'relay_migration_leases',
'relay_post_drain_migration_pins',
'relay_rate_windows',
'relay_region_decisions',
'relay_region_rehome_attempts',
'relay_region_rehome_control',
'relay_region_rehome_worker_state'
@@ -140,9 +170,7 @@ describe('relay database', () => {
const second = await openRelayDatabase({ dataDir })
expect(
await second.query(`SELECT region FROM relay_cell_regions WHERE cell_id = ?`, [
'legacy-cell'
])
await second.query(`SELECT region FROM relay_cell_regions WHERE cell_id = ?`, ['legacy-cell'])
).toEqual([{ region: 'us-central1' }])
await second.close()
})
@@ -165,9 +193,7 @@ describe('relay database', () => {
'relay_region_rehome_attempts'
])
expect(checked.every((row) => String(row.sql).includes(list))).toBe(true)
expect(
POSTGRES_SCHEMA_MIGRATIONS.some((statement) => statement.includes(list))
).toBe(true)
expect(POSTGRES_SCHEMA_MIGRATIONS.some((statement) => statement.includes(list))).toBe(true)
await database.close()
})
+33 -1
View File
@@ -197,6 +197,24 @@ CREATE TABLE IF NOT EXISTS relay_assignment_region_preferences (
CREATE INDEX IF NOT EXISTS relay_assignment_region_preferences_observed
ON relay_assignment_region_preferences(observed_at);
CREATE TABLE IF NOT EXISTS relay_region_decisions (
user_id TEXT NOT NULL, relay_host_id TEXT NOT NULL,
generation BIGINT NOT NULL, expires_at BIGINT NOT NULL,
assignment_epoch BIGINT NOT NULL, incumbent_region TEXT NOT NULL,
policy_version BIGINT NOT NULL, outcome TEXT NOT NULL,
cohort_bucket BIGINT NOT NULL DEFAULT 0,
last_considered_at BIGINT NOT NULL DEFAULT 0,
preferred_region TEXT, observed_at BIGINT NOT NULL, report_json TEXT,
PRIMARY KEY (user_id, relay_host_id)
);
CREATE TABLE IF NOT EXISTS relay_control_capabilities (
user_id TEXT NOT NULL, relay_host_id TEXT NOT NULL, activity_id TEXT NOT NULL,
cell_id TEXT NOT NULL, cell_incarnation TEXT NOT NULL,
assignment_epoch BIGINT NOT NULL, generation BIGINT NOT NULL,
finish_existing BIGINT NOT NULL,
idle_regional_rehome BIGINT NOT NULL DEFAULT 0,
PRIMARY KEY (user_id, relay_host_id, activity_id)
);
CREATE TABLE IF NOT EXISTS relay_region_rehome_worker_state (
worker_id TEXT PRIMARY KEY,
next_dispatch_at BIGINT NOT NULL,
@@ -228,6 +246,7 @@ CREATE TABLE IF NOT EXISTS relay_region_rehome_attempts (
CHECK (preferred_region IN (${REGION_LIST})),
source_cell_id TEXT NOT NULL,
source_cell_incarnation TEXT NOT NULL,
source_generation BIGINT NOT NULL DEFAULT 0,
target_cell_id TEXT NOT NULL,
target_cell_incarnation TEXT NOT NULL,
previous_epoch BIGINT NOT NULL,
@@ -600,6 +619,8 @@ CREATE INDEX IF NOT EXISTS relay_audit_events_at ON relay_audit_events(at);
// auto-named; the replacement is named, so both statements are no-ops on a
// database the current schema created and neither can drop the other.
export const POSTGRES_SCHEMA_MIGRATIONS = [
`ALTER TABLE relay_region_decisions ADD COLUMN IF NOT EXISTS last_considered_at BIGINT NOT NULL DEFAULT 0`,
`ALTER TABLE relay_region_decisions ADD COLUMN IF NOT EXISTS cohort_bucket BIGINT NOT NULL DEFAULT 0`,
`ALTER TABLE relay_region_rehome_attempts
DROP CONSTRAINT IF EXISTS relay_region_rehome_attempts_preferred_region_check`,
`ALTER TABLE relay_region_rehome_attempts
@@ -607,7 +628,9 @@ export const POSTGRES_SCHEMA_MIGRATIONS = [
CHECK (preferred_region IN (${REGION_LIST}))`,
`ALTER TABLE relay_region_rehome_control
ADD COLUMN IF NOT EXISTS host_cooldown_ms BIGINT NOT NULL
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS}`
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS}`,
`ALTER TABLE relay_control_capabilities ADD COLUMN IF NOT EXISTS idle_regional_rehome BIGINT NOT NULL DEFAULT 0`,
`ALTER TABLE relay_region_rehome_attempts ADD COLUMN IF NOT EXISTS source_generation BIGINT NOT NULL DEFAULT 0`
]
function postgresSql(sql: string): string {
@@ -1013,6 +1036,15 @@ async function applySchema(database: RelayDatabase): Promise<void> {
for (const statement of SCHEMA.split(';')) {
if (statement.trim()) await database.query(statement)
}
for (const [table, column] of [
['relay_control_capabilities', 'idle_regional_rehome'],
['relay_region_rehome_attempts', 'source_generation']
]) {
const columns = await database.query('SELECT name FROM pragma_table_info(?)', [table])
if (!columns.some((existing) => existing.name === column)) {
await database.query(`ALTER TABLE ${table} ADD COLUMN ${column} BIGINT NOT NULL DEFAULT 0`)
}
}
}
// Why: DDL is not a request. A CREATE INDEX on a grown table legitimately runs
@@ -155,6 +155,66 @@ describe('client accept abandoned mid-DB-phase', () => {
vi.useRealTimers()
})
it('does not admit new source work after a drain crosses activity acquisition', async () => {
const h = harness()
const control = await activeHost(h)
const slow = deferred<void>()
h.acquireActivity.mockReturnValueOnce(slow.promise)
const client = new FakeSocket()
const capacity = { bind: vi.fn(), release: vi.fn() }
const accepting = h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential',
capacity
)
await vi.advanceTimersByTimeAsync(0)
h.registry.drainHost({
attemptId: 'attempt',
userId: identity.sub,
relayHostId: identity.relayHostId,
sourceAssignmentEpoch: 1,
graceMs: 60_000
})
slow.resolve()
await accepting
expect(control.send).not.toHaveBeenCalledWith(expect.stringContaining('conn-open'))
expect(capacity.bind).not.toHaveBeenCalled()
expect(client.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, expect.any(String))
expect(h.releaseActivity).toHaveBeenCalled()
})
it('does not splice an attachment whose generation retired during basis persistence', async () => {
const h = harness()
await activeHost(h)
const client = new FakeSocket()
await h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential'
)
const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
const pending = [...session.pendingConns.values()][0]!
const slow = deferred<void>()
h.store.recordConnectionBasis.mockReturnValueOnce(slow.promise)
const host = new FakeSocket()
const attaching = h.registry.acceptHostData(
host as unknown as WebSocket,
pending.connId,
pending.connTicket,
1
)
await vi.advanceTimersByTimeAsync(0)
h.registry.drain(0)
await vi.advanceTimersByTimeAsync(0)
slow.resolve()
expect(await attaching).toBe(false)
expect(session.activeSplices.size).toBe(0)
expect(h.store.deactivateBasis).toHaveBeenCalledWith(pending.connId)
expect(client.send).not.toHaveBeenCalledWith(expect.stringContaining('\"ok\":true'))
expect(host.close).toHaveBeenCalled()
})
it('stops after a slow activity acquire when the phone already hung up', async () => {
const h = harness()
const control = await activeHost(h)
@@ -390,6 +450,7 @@ describe('successful client accept timing', () => {
relayHostIdDigest: string
}
expect(event.credentialKind).toBe('resume')
expect(event).toMatchObject({ assignmentEpoch: 1, controlGeneration: 1, drainMode: 'none' })
// Joins the line back to the emitting process, like the runtime metrics event.
expect(event).toMatchObject({ role: 'cell', cellId: config.cellId, region: 'us-central1' })
expect(Object.keys(event.stageMs).sort()).toEqual([
@@ -460,6 +521,9 @@ describe('control round-trip sampling', () => {
cellId: config.cellId,
region: 'us-central1',
rttMsMedian: 40,
assignmentEpoch: 1,
controlGeneration: 1,
drainMode: 'none',
sampleCount: 4
})
expect(rttLines()[0]).not.toContain(identity.relayHostId)
@@ -4,6 +4,7 @@ import {
CONTROL_CONTINUITY_LIMITS,
RELAY_CLOSE_CODE,
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS,
RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME,
RELAY_PROTOCOL_LIMITS
} from '@orca-cloud/relay-contract'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
@@ -140,7 +141,10 @@ function createRegistry(
store as RelayCredentialStore,
assignments,
new ProcessQueuedByteBudget(),
observer
observer,
Date.now,
Math.random,
'incarnation-1'
)
// Mirrors the production signature exactly so a future positional shift fails to compile.
const bound = (
@@ -166,7 +170,14 @@ function createRegistry(
assignmentEpoch,
appVersion = '1.4.173'
) => bound(socket, identity, existing, generation, rebind, assignmentEpoch, appVersion)
return { registry, activate, acquireActivity, renewControlActivity, releaseActivity, observer }
return {
registry,
activate,
acquireActivity,
renewControlActivity,
releaseActivity,
observer
}
}
describe('host session cleanup races', () => {
@@ -398,26 +409,20 @@ describe('host session cleanup races', () => {
attemptId: '22222222-2222-4222-8222-222222222222'
})
).toThrow('regional_rehome_attempt_conflict')
expect(() =>
registry.drainHost({ ...request, sourceAssignmentEpoch: 8 })
).toThrow('regional_rehome_assignment_epoch_mismatch')
expect(() => registry.drainHost({ ...request, sourceAssignmentEpoch: 8 })).toThrow(
'regional_rehome_assignment_epoch_mismatch'
)
const rebound = new FakeSocket()
await activate(
rebound as unknown as WebSocket,
identity,
registry.get(request),
1,
true,
7
)
await activate(rebound as unknown as WebSocket, identity, registry.get(request), 1, true, 7)
expect(registry.get(request)?.state).toBe('drain-only')
expect(rebound.send).toHaveBeenCalledWith(expect.stringContaining('"type":"drain"'))
await vi.advanceTimersByTimeAsync(30_000)
expect(registry.get(request)).toBeNull()
expect(registry.get({ userId: secondIdentity.sub, relayHostId: secondIdentity.relayHostId }))
.not.toBeNull()
expect(
registry.get({ userId: secondIdentity.sub, relayHostId: secondIdentity.relayHostId })
).not.toBeNull()
expect(secondSocket.close).not.toHaveBeenCalled()
})
@@ -513,14 +518,7 @@ describe('host session cleanup races', () => {
expect(original).not.toBeNull()
const rebindSocket = new FakeSocket()
const rebinding = activate(
rebindSocket as unknown as WebSocket,
identity,
original,
1,
true,
1
)
const rebinding = activate(rebindSocket as unknown as WebSocket, identity, original, 1, true, 1)
rebindSocket.close()
blocked.resolve('control:production-gce-c3:1')
await rebinding
@@ -659,14 +657,7 @@ describe('host session cleanup races', () => {
originalSocket.close()
const replacementSocket = new FakeSocket()
await activate(
replacementSocket as unknown as WebSocket,
identity,
original,
2,
false,
1
)
await activate(replacementSocket as unknown as WebSocket, identity, original, 2, false, 1)
const replacement = registry.get({
userId: identity.sub,
relayHostId: identity.relayHostId
@@ -696,14 +687,7 @@ describe('host session cleanup races', () => {
})
expect(original).not.toBeNull()
await activate(
new FakeSocket() as unknown as WebSocket,
identity,
original,
2,
false,
1
)
await activate(new FakeSocket() as unknown as WebSocket, identity, original, 2, false, 1)
vi.advanceTimersByTime(15_000)
expect(renewControlActivity).toHaveBeenCalledOnce()
@@ -718,6 +702,53 @@ describe('host session cleanup races', () => {
vi.advanceTimersByTime(0)
})
it('ignores a denial belonging to the socket before a same-generation rebind', async () => {
const h = createRegistry(vi.fn().mockResolvedValue('control:production-gce-c3:1'))
const oldSocket = new FakeSocket()
await h.activate(oldSocket as unknown as WebSocket, identity, null, 1, false, 1)
const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
let reject!: (error: Error) => void
h.renewControlActivity.mockReturnValueOnce(
new Promise<void>((_, fail) => {
reject = fail
})
)
await vi.advanceTimersByTimeAsync(15_000)
const replacement = new FakeSocket()
await h.activate(replacement as unknown as WebSocket, identity, session, 1, true, 1)
reject(new Error('activity_cell_not_authoritative'))
await vi.advanceTimersByTimeAsync(0)
expect(replacement.close).not.toHaveBeenCalled()
expect(session.socket).toBe(replacement)
expect(session.generation).toBe(1)
})
it('ignores missing-activity recovery denial after an authority transition', async () => {
const h = createRegistry(vi.fn().mockResolvedValue('control:production-gce-c3:1'))
const socket = new FakeSocket()
await h.activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
h.renewControlActivity.mockRejectedValueOnce(new Error('control_activity_not_found'))
let reject!: (error: Error) => void
h.acquireActivity.mockReturnValueOnce(
new Promise<void>((_, fail) => {
reject = fail
})
)
await vi.advanceTimersByTimeAsync(15_000)
h.registry.drainHost({
attemptId: 'attempt',
userId: identity.sub,
relayHostId: identity.relayHostId,
sourceAssignmentEpoch: 1,
graceMs: 60_000
})
reject(new Error('activity_cell_not_authoritative'))
await vi.advanceTimersByTimeAsync(0)
expect(socket.close).not.toHaveBeenCalled()
expect(session.state).toBe('drain-only')
})
it('keeps 15s pings while halving steady-state control renewals', async () => {
const activateControl = vi
.fn<RelayAssignmentStore['activateControl']>()
@@ -732,9 +763,7 @@ describe('host session cleanup races', () => {
socket.emit('message', Buffer.from(JSON.stringify({ type: 'pong' })), false)
}
const pings = socket.send.mock.calls.filter((call) =>
String(call[0]).includes('"ping"')
)
const pings = socket.send.mock.calls.filter((call) => String(call[0]).includes('"ping"'))
expect(pings).toHaveLength(4)
expect(renewControlActivity).toHaveBeenCalledTimes(2)
const firstExpiry = Number(renewControlActivity.mock.calls[0]![1].expiresAt)
@@ -1118,3 +1147,277 @@ describe('host hello ack pending connections', () => {
expect(rebound.pendingConns).toEqual([DETAILED_ENTRY])
})
})
describe('source-owned idle cutover', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
const request = {
attemptId: 'idle-1',
userId: identity.sub,
relayHostId: identity.relayHostId,
sourceAssignmentEpoch: 1,
sourceGeneration: 1,
sourceCellIncarnation: 'incarnation-1',
targetCellId: 'target'
}
async function source(store: Partial<RelayCredentialStore> = {}) {
const h = createRegistry(vi.fn().mockResolvedValue('control:1'), store)
const socket = new FakeSocket()
h.registry.acceptControl(
socket as unknown as WebSocket,
identity,
undefined,
new Set([RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME])
)
socket.removeAllListeners('message')
await h.activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
return { ...h, socket, session: h.registry.get(request)! }
}
it('keeps either established client busy until both actually leave', async () => {
const h = await source()
h.session.activeConnIds.add('phone')
h.session.activeConnIds.add('ipad')
const commit = vi.fn().mockResolvedValue({ outcome: 'committed' })
h.session.activeConnIds.delete('ipad')
expect(
await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('not-committed'))
).toEqual({ outcome: 'busy' })
expect(commit).not.toHaveBeenCalled()
h.session.activeConnIds.delete('phone')
expect(
await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('not-committed'))
).toEqual({ outcome: 'committed' })
expect(h.socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.DRAINING, expect.any(String))
expect(h.releaseActivity).toHaveBeenCalled()
})
it.each([
{ userId: 'other-user' },
{ sourceAssignmentEpoch: 2 },
{ sourceGeneration: 2 },
{ sourceCellIncarnation: 'other-incarnation' },
{ targetCellId: 'other-target' }
])('rejects a reused operation ID with changed authority %j', async (change) => {
const h = await source()
const result = deferred<{ outcome: 'deferred' }>()
const commit = vi.fn().mockReturnValue(result.promise)
const reconcile = vi.fn().mockResolvedValue('not-committed')
const moving = h.registry.idleRehome(request, commit, reconcile)
const conflicting = h.registry.idleRehome({ ...request, ...change }, commit, reconcile)
result.resolve({ outcome: 'deferred' })
expect(await conflicting).toEqual({ outcome: 'stale' })
expect(await moving).toEqual({ outcome: 'deferred' })
expect(commit).toHaveBeenCalledOnce()
expect(h.socket.close).not.toHaveBeenCalled()
})
it('accounts for accepts before credential identity resolves', async () => {
const lookup = deferred<null>()
const h = await source({
resolveResume: vi.fn().mockReturnValue(lookup.promise),
resolveInviteForMove: vi.fn().mockResolvedValue(null)
})
const client = new FakeSocket()
const accept = h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential'
)
expect(
await h.registry.idleRehome(request, vi.fn(), vi.fn().mockResolvedValue('not-committed'))
).toEqual({ outcome: 'busy' })
lookup.resolve(null)
await accept
expect(h.socket.close).not.toHaveBeenCalled()
})
it('rejects new accepts and replacements synchronously while a commit awaits', async () => {
const h = await source()
const result = deferred<{ outcome: 'deferred' }>()
const commit = vi.fn().mockReturnValue(result.promise)
const moving = h.registry.idleRehome(
request,
commit,
vi.fn().mockResolvedValue('not-committed')
)
const duplicate = h.registry.idleRehome(
request,
commit,
vi.fn().mockResolvedValue('not-committed')
)
const client = new FakeSocket()
const release = vi.fn()
await h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential',
{ release } as never
)
expect(client.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, expect.any(String))
expect(release).toHaveBeenCalledOnce()
const replacement = new FakeSocket()
await h.activate(replacement as unknown as WebSocket, identity, h.session, 2, false, 1)
expect(replacement.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, expect.any(String))
result.resolve({ outcome: 'deferred' })
await moving
await duplicate
expect(commit).toHaveBeenCalledOnce()
expect(h.socket.close).not.toHaveBeenCalled()
expect(
await h.registry.idleRehome(
{ ...request, attemptId: 'next' },
vi.fn().mockResolvedValue({ outcome: 'committed' }),
vi.fn().mockResolvedValue('not-committed')
)
).toEqual({ outcome: 'committed' })
})
it.each(['ambiguous', 'deferred'])(
'keeps %s outcomes fenced until locked reconciliation succeeds',
async (claim) => {
const h = await source()
const reconcile = vi
.fn()
.mockRejectedValueOnce(new Error('database unavailable'))
.mockRejectedValueOnce(new Error('database unavailable'))
.mockResolvedValue('not-committed')
const moving = h.registry.idleRehome(
request,
claim === 'ambiguous'
? vi.fn().mockRejectedValue(new Error('lost commit reply'))
: vi.fn().mockResolvedValue({ outcome: 'deferred' }),
reconcile
)
await vi.advanceTimersByTimeAsync(50)
expect(
await h.registry.idleRehome(
{ ...request, attemptId: 'other' },
vi.fn(),
vi.fn().mockResolvedValue('not-committed')
)
).toEqual({ outcome: 'busy' })
expect(h.socket.close).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(300)
expect(await moving).toEqual({ outcome: 'deferred' })
expect(reconcile).toHaveBeenCalledTimes(3)
expect(h.socket.close).not.toHaveBeenCalled()
}
)
it('owns accepted control mutations before the handler first awaits', async () => {
const mutation = deferred<RelayTokenClaims | null>()
const h = await source()
;(h.registry as unknown as { verifyRelayToken: unknown }).verifyRelayToken = vi
.fn()
.mockReturnValue(mutation.promise)
h.socket.emit(
'message',
Buffer.from(JSON.stringify({ type: 'auth-refresh', relayJwt: 'token' })),
false
)
const commit = vi.fn().mockResolvedValue({ outcome: 'deferred' })
expect(
await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('not-committed'))
).toEqual({ outcome: 'busy' })
mutation.resolve(identity)
await vi.advanceTimersByTimeAsync(0)
expect(
await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('not-committed'))
).toEqual({ outcome: 'deferred' })
})
it('owns queued replacement activation before its first persistence await', async () => {
const h = await source()
const activation = deferred<string>()
const assignments = (h.registry as unknown as { assignments: { activateControl: unknown } })
.assignments
assignments.activateControl = vi.fn().mockReturnValue(activation.promise)
const replacement = new FakeSocket()
const activating = h.activate(
replacement as unknown as WebSocket,
identity,
h.session,
2,
false,
1
)
expect(
await h.registry.idleRehome(request, vi.fn(), vi.fn().mockResolvedValue('not-committed'))
).toEqual({ outcome: 'busy' })
activation.resolve('control:2')
await activating
})
it('retires changed authority even when the claim definitively deferred', async () => {
const h = await source()
expect(
await h.registry.idleRehome(
request,
vi.fn().mockResolvedValue({ outcome: 'deferred' }),
vi.fn().mockResolvedValue('stale')
)
).toEqual({ outcome: 'stale' })
expect(h.session.state).toBe('closed')
expect(h.releaseActivity).toHaveBeenCalled()
})
it('holds attach ownership through basis failure reservation cleanup', async () => {
const basis = deferred<void>()
const cleanup = deferred<void>()
const h = await source({
recordConnectionBasis: vi.fn().mockImplementation(async () => {
await basis.promise
throw new Error('basis failed')
}),
failReservation: vi.fn().mockReturnValue(cleanup.promise)
})
const client = new FakeSocket()
h.session.pendingConns.set('conn', {
connId: 'conn',
connTicket: 'ticket',
client: client as unknown as WebSocket,
reservation: {
userId: identity.sub,
relayHostId: identity.relayHostId,
credentialKind: 'invite',
leaseExpiresAt: Date.now() + 1000
},
attachTimer: setTimeout(() => {}, 1000),
credentialActivityId: null
} as never)
const attached = h.registry.acceptHostData(
new FakeSocket() as unknown as WebSocket,
'conn',
'ticket',
1
)
const commit = vi.fn().mockResolvedValue({ outcome: 'deferred' })
expect(await h.registry.idleRehome(request, commit, vi.fn())).toEqual({ outcome: 'busy' })
basis.resolve()
await vi.advanceTimersByTimeAsync(0)
expect(h.session.activeConnIds.size).toBe(0)
expect(await h.registry.idleRehome(request, commit, vi.fn())).toEqual({ outcome: 'busy' })
expect(commit).not.toHaveBeenCalled()
cleanup.resolve()
await attached
})
it('returns the durable operation outcome after source retirement', async () => {
const h = await source()
const commit = vi.fn().mockResolvedValue({ outcome: 'committed' })
await h.registry.idleRehome(request, commit, vi.fn())
expect(
await h.registry.idleRehome(request, commit, vi.fn().mockResolvedValue('committed'))
).toEqual({ outcome: 'committed' })
expect(commit).toHaveBeenCalledOnce()
})
it('does not reopen a source overtaken by emergency drain', async () => {
const h = await source()
const result = deferred<{ outcome: 'deferred' }>()
const moving = h.registry.idleRehome(
request,
() => result.promise,
vi.fn().mockResolvedValue('not-committed')
)
h.registry.drain(0)
await vi.advanceTimersByTimeAsync(0)
result.resolve({ outcome: 'deferred' })
await moving
expect(h.session.state).toBe('closed')
expect(h.registry.get(request)).toBeNull()
})
})
+331 -53
View File
@@ -15,6 +15,7 @@ import {
HostHelloSchema,
InviteCreateSchema,
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS,
RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME,
RELAY_PROTOCOL_LIMITS,
RELAY_CLOSE_CODE,
type RelayHostCloseReason,
@@ -25,10 +26,7 @@ import type WebSocket from 'ws'
import type { RawData } from 'ws'
import type { RelayConfig } from './config.js'
import type { RelayAssignmentStore } from './assignment-store.js'
import {
RelayCredentialStore,
type CredentialReservation
} from './credential-store.js'
import { RelayCredentialStore, type CredentialReservation } from './credential-store.js'
import { HostCloseReasonMemory } from './host-close-reason-memory.js'
import { relayHostLogDigest } from './relay-host-log-digest.js'
import type { RelayTokenClaims } from './relay-token-verifier.js'
@@ -78,7 +76,7 @@ export type HostSession = {
identity: RelayTokenClaims
readonly relayHostId: string
readonly generation: number
readonly assignmentEpoch: number
assignmentEpoch: number
readonly controlActivityId: string | null
readonly controlResumeSecret: string
// Why: reconnect churn is only actionable once it can be pinned to a client build.
@@ -94,6 +92,7 @@ export type HostSession = {
pendingPingAt: number | null
controlRttSamplesMs: number[]
controlRttLoggedAt: number | null
authorityRevision: number
activityRenewalDueAt: number
activityRenewalAttempt: number
activityRenewalCompletedAttempt: number
@@ -108,10 +107,7 @@ export type HostSession = {
regionalDrainExpiresAt: number | null
}
export type RegionalHostDrainOutcome =
| 'accepted'
| 'already-accepted'
| 'host-not-connected'
export type RegionalHostDrainOutcome = 'accepted' | 'already-accepted' | 'host-not-connected'
type PendingConnection = {
connId: string
@@ -184,6 +180,113 @@ export class HostSessionRegistry {
private readonly hostCapabilities = new WeakMap<WebSocket, ReadonlySet<string>>()
private draining = false
private readonly idleWork = new Map<string, number>()
private readonly idleAttempts = new Map<
string,
{
attemptId: string
authorityKey: string
promise: Promise<{ outcome: 'committed' | 'deferred' | 'stale' }>
}
>()
async idleRehome(
input: {
attemptId: string
userId: string
relayHostId: string
sourceAssignmentEpoch: number
sourceGeneration: number
sourceCellIncarnation: string
targetCellId: string
},
commit: () => Promise<{ outcome: 'committed' | 'deferred' | 'stale' }>,
reconcile: () => Promise<'committed' | 'not-committed' | 'stale'>
): Promise<{ outcome: 'busy' | 'committed' | 'deferred' | 'stale' }> {
const authorityKey = JSON.stringify([
input.userId,
input.sourceAssignmentEpoch,
input.sourceGeneration,
input.sourceCellIncarnation,
input.targetCellId
])
const prior = this.idleAttempts.get(input.relayHostId)
if (prior) {
if (prior.attemptId !== input.attemptId) return { outcome: 'busy' }
return prior.authorityKey === authorityKey ? prior.promise : { outcome: 'stale' }
}
const session = this.get(input)
if (
this.draining ||
!session ||
session.state !== 'active' ||
session.generation !== input.sourceGeneration ||
session.assignmentEpoch !== input.sourceAssignmentEpoch ||
this.cellIncarnation !== input.sourceCellIncarnation
) {
const durable = await reconcile()
return { outcome: durable === 'committed' ? 'committed' : 'stale' }
}
if (
!session.socket ||
!this.hostCapabilities.get(session.socket)?.has(RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME)
)
return { outcome: 'deferred' }
if (
(this.idleWork.get(input.relayHostId) ?? 0) !== 0 ||
session.activeConnIds.size !== 0 ||
session.activeSplices.size !== 0 ||
session.pendingConns.size !== 0
)
return { outcome: 'busy' }
const revision = session.authorityRevision
const promise = Promise.resolve().then(async () => {
let outcome: 'committed' | 'deferred' | 'stale'
try {
outcome = (await commit()).outcome
if (outcome === 'deferred') {
const durable = await reconcile()
outcome = durable === 'not-committed' ? 'deferred' : durable
}
} catch {
let delay = 100
for (;;) {
try {
const durable = await reconcile()
outcome = durable === 'not-committed' ? 'deferred' : durable
break
} catch {
await new Promise<void>((resolve) => {
const timer = setTimeout(resolve, delay)
timer.unref?.()
})
delay = Math.min(delay * 2, 5000)
}
}
}
if (this.get(input) === session) {
if (outcome !== 'deferred' || this.draining || session.authorityRevision !== revision) {
this.closeDrainedSession(session)
}
}
if (this.idleAttempts.get(input.relayHostId)?.promise === promise)
this.idleAttempts.delete(input.relayHostId)
return { outcome }
})
this.idleAttempts.set(input.relayHostId, { attemptId: input.attemptId, authorityKey, promise })
return promise
}
private beginIdleWork(hostId: string): (() => void) | null {
if (this.idleAttempts.has(hostId)) return null
this.idleWork.set(hostId, (this.idleWork.get(hostId) ?? 0) + 1)
return () => {
const remaining = (this.idleWork.get(hostId) ?? 1) - 1
if (remaining === 0) this.idleWork.delete(hostId)
else this.idleWork.set(hostId, remaining)
}
}
constructor(
private readonly config: RelayConfig,
private readonly verifyRelayToken: VerifyRelayToken,
@@ -192,7 +295,8 @@ export class HostSessionRegistry {
private readonly queuedByteBudget: ProcessQueuedByteBudget,
private readonly observer: RelayRuntimeObserver,
private readonly now: () => number = Date.now,
private readonly random: () => number = Math.random
private readonly random: () => number = Math.random,
private readonly cellIncarnation?: string
) {}
// Uniform over [CONTROL_LEASE_MS - jitter, CONTROL_LEASE_MS + jitter).
@@ -206,6 +310,25 @@ export class HostSessionRegistry {
hostId: string,
credential: string,
capacityReservation?: PendingHostDataReservation
): Promise<void> {
const release = this.beginIdleWork(hostId)
if (!release) {
capacityReservation?.release()
this.rejectClient(socket, RELAY_CLOSE_CODE.WRONG_CELL)
return
}
try {
await this.acceptClientUnfenced(socket, hostId, credential, capacityReservation)
} finally {
release()
}
}
private async acceptClientUnfenced(
socket: WebSocket,
hostId: string,
credential: string,
capacityReservation?: PendingHostDataReservation
): Promise<void> {
if (this.draining) {
capacityReservation?.release()
@@ -295,6 +418,7 @@ export class HostSessionRegistry {
this.rejectClient(socket, RELAY_CLOSE_CODE.LIMIT_EXCEEDED)
return
}
const admittingSocket = session.socket
const connId = randomUUID()
const connTicket = randomBytes(32).toString('base64url')
const identity = { userId: reservation.userId, relayHostId: hostId }
@@ -324,6 +448,20 @@ export class HostSessionRegistry {
) {
return
}
// Admission may have crossed a drain or control replacement while persisting activity.
if (
this.draining ||
this.sessions.get(sessionKey) !== session ||
session.state !== 'active' ||
session.socket !== admittingSocket ||
admittingSocket.readyState !== admittingSocket.OPEN
) {
capacityReservation?.release()
this.failReservationBestEffort(reservation)
if (credentialActivityId) this.releaseActivityBestEffort(identity, credentialActivityId)
this.rejectClient(socket, RELAY_CLOSE_CODE.WRONG_CELL)
return
}
markStage('activity')
const attachTimer = setTimeout(() => {
session.pendingConns.delete(connId)
@@ -372,6 +510,27 @@ export class HostSessionRegistry {
connId: string,
connTicket: string,
generation: number
): Promise<boolean> {
const owner = [...this.sessions.values()].find((candidate) =>
candidate.pendingConns.has(connId)
)
const release = owner ? this.beginIdleWork(owner.relayHostId) : () => {}
if (!release) {
socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress')
return false
}
try {
return await this.acceptHostDataUnfenced(socket, connId, connTicket, generation)
} finally {
release()
}
}
private async acceptHostDataUnfenced(
socket: WebSocket,
connId: string,
connTicket: string,
generation: number
): Promise<boolean> {
const session = [...this.sessions.values()].find((candidate) =>
candidate.pendingConns.has(connId)
@@ -427,6 +586,27 @@ export class HostSessionRegistry {
socket.close(RELAY_CLOSE_CODE.LIMIT_EXCEEDED, 'basis persistence failed')
return false
}
// Already admitted attachments may finish a regional drain, but never a retired generation.
if (
this.draining ||
this.sessions.get(this.key(identity.userId, identity.relayHostId)) !== session ||
this.get(identity)?.state === 'closed' ||
!session.activeConnIds.has(connId) ||
socket.readyState !== socket.OPEN ||
pending.client.readyState !== pending.client.OPEN
) {
session.activeConnIds.delete(connId)
pending.capacityReservation?.release()
this.deactivateBasisBestEffort(connId)
this.failReservationBestEffort(pending.reservation)
if (spliceActivityId) this.releaseActivityBestEffort(identity, spliceActivityId)
if (pending.credentialActivityId) {
this.releaseActivityBestEffort(identity, pending.credentialActivityId)
}
this.rejectClient(pending.client, RELAY_CLOSE_CODE.DRAINING)
socket.close(RELAY_CLOSE_CODE.DRAINING, 'host retired during attachment')
return false
}
const close = wireSplice({
client: pending.client,
host: socket,
@@ -505,6 +685,7 @@ export class HostSessionRegistry {
JSON.stringify({
event: 'orca_relay_client_accept_completed',
...this.logIdentity(),
...this.sessionPlacementLogFields(session),
credentialKind: pending.reservation.credentialKind,
stageMs,
totalMs,
@@ -513,6 +694,14 @@ export class HostSessionRegistry {
)
}
private sessionPlacementLogFields(session: HostSession) {
return {
assignmentEpoch: session.assignmentEpoch,
controlGeneration: session.generation,
drainMode: session.regionalDrainAttemptId ? 'deadline' : 'none'
}
}
// Matches the runtime metrics event so a log line and a metric point can be
// joined back to the process that emitted them.
private logIdentity(): { role: string; cellId: string; region: RelayRegion } {
@@ -549,6 +738,7 @@ export class HostSessionRegistry {
JSON.stringify({
event: 'orca_relay_host_control_rtt',
...this.logIdentity(),
...this.sessionPlacementLogFields(session),
relayHostIdDigest: relayHostLogDigest(session.relayHostId),
rttMsMedian: percentile(samples, 0.5),
sampleCount: samples.length
@@ -562,6 +752,10 @@ export class HostSessionRegistry {
connectionInclusionWatermark?: number,
hostCapabilities?: ReadonlySet<string>
): void {
if (this.idleAttempts.has(identity.relayHostId)) {
socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress')
return
}
// Keyed by socket, not session: a rebind swaps the session's socket, and the
// successor's own advertisement is the only one that describes its decoder.
if (hostCapabilities?.size) this.hostCapabilities.set(socket, hostCapabilities)
@@ -602,8 +796,7 @@ export class HostSessionRegistry {
socket: WebSocket | null,
context: string
): void {
void Promise.resolve()
.then(task)
void (async () => task())()
.catch((error: unknown) => {
const message = (error instanceof Error ? error.message : 'unknown')
// Untruncated, unlike peer-supplied close reasons: this is the
@@ -653,6 +846,7 @@ export class HostSessionRegistry {
this.draining = true
for (const session of this.sessions.values()) {
if (session.state === 'closed') continue
session.authorityRevision += 1
session.state = 'drain-only'
if (session.socket) send(session.socket, 'drain', { graceMs, recovery: 'resolve-director' })
setTimeout(() => this.closeDrainedSession(session), graceMs)
@@ -665,7 +859,8 @@ export class HostSessionRegistry {
relayHostId: string
sourceAssignmentEpoch: number
graceMs: number
}): RegionalHostDrainOutcome {
sourceCellIncarnation?: string
}): RegionalHostDrainOutcome | Promise<RegionalHostDrainOutcome> {
const session = this.get(input)
if (!session || session.state === 'closed') return 'host-not-connected'
if (session.assignmentEpoch !== input.sourceAssignmentEpoch) {
@@ -678,13 +873,11 @@ export class HostSessionRegistry {
this.reassertRegionalDrain(session)
return 'already-accepted'
}
session.authorityRevision += 1
session.regionalDrainAttemptId = input.attemptId
session.regionalDrainExpiresAt = this.now() + input.graceMs
this.reassertRegionalDrain(session)
session.regionalDrainTimer = setTimeout(
() => this.closeDrainedSession(session),
input.graceMs
)
session.regionalDrainTimer = setTimeout(() => this.closeDrainedSession(session), input.graceMs)
return 'accepted'
}
@@ -740,9 +933,9 @@ export class HostSessionRegistry {
const existing = this.sessions.get(key)
const rebind = Boolean(
existing &&
hello.data.controlResumeSecret &&
hello.data.controlResumeSecret === existing.controlResumeSecret &&
(existing.state === 'orphaned' || existing.state === 'active')
hello.data.controlResumeSecret &&
hello.data.controlResumeSecret === existing.controlResumeSecret &&
(existing.state === 'orphaned' || existing.state === 'active')
)
const generation = rebind ? existing!.generation : (existing?.generation ?? 0) + 1
const ephemeral = nacl.box.keyPair()
@@ -784,7 +977,9 @@ export class HostSessionRegistry {
}, 10_000)
socket.once('message', (raw, isBinary) => {
clearTimeout(proofTimer)
const ack = isBinary ? null : HostChallengeAckSchema.safeParse(payload(raw, 'host-challenge-ack'))
const ack = isBinary
? null
: HostChallengeAckSchema.safeParse(payload(raw, 'host-challenge-ack'))
const proof = ack?.success ? decodeCanonicalBase64(ack.data.proofB64, 32) : null
if (
!ack?.success ||
@@ -826,6 +1021,11 @@ export class HostSessionRegistry {
appVersion: string,
connectionInclusionWatermark?: number
): Promise<void> {
const release = this.beginIdleWork(identity.relayHostId)
if (!release) {
socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress')
return Promise.resolve()
}
const key = this.key(identity.sub, identity.relayHostId)
const previous = this.activationQueues.get(key) ?? Promise.resolve()
// The timeout only fails this waiting socket; the queue entry still chains
@@ -836,26 +1036,29 @@ export class HostSessionRegistry {
socket.close(RELAY_CLOSE_CODE.LIMIT_EXCEEDED, 'control activation queue stalled')
}, ACTIVATION_QUEUE_WAIT_MS)
queueWaitTimer.unref?.()
const activation = previous.catch(() => undefined).then(async () => {
clearTimeout(queueWaitTimer)
if (queueWaitExpired) return
if ((this.sessions.get(key) ?? null) !== existing) {
socket.close(RELAY_CLOSE_CODE.PEER_DROPPED, 'control activation superseded')
return
}
await this.activateCurrent(
socket,
identity,
existing,
generation,
rebind,
assignmentEpoch,
appVersion,
connectionInclusionWatermark
)
})
const activation = previous
.catch(() => undefined)
.then(async () => {
clearTimeout(queueWaitTimer)
if (queueWaitExpired) return
if ((this.sessions.get(key) ?? null) !== existing) {
socket.close(RELAY_CLOSE_CODE.PEER_DROPPED, 'control activation superseded')
return
}
await this.activateCurrent(
socket,
identity,
existing,
generation,
rebind,
assignmentEpoch,
appVersion,
connectionInclusionWatermark
)
})
this.activationQueues.set(key, activation)
const cleanup = (): void => {
release()
if (this.activationQueues.get(key) === activation) this.activationQueues.delete(key)
}
void activation.then(cleanup, cleanup)
@@ -881,7 +1084,11 @@ export class HostSessionRegistry {
cellId: this.config.cellId,
assignmentEpoch,
generation,
connectionInclusionWatermark
connectionInclusionWatermark,
idleRegionalRehome:
this.hostCapabilities.get(socket)?.has(RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME) ??
false,
cellIncarnation: this.cellIncarnation
}
)
await this.assignments.markMigrationTargetRegistered(
@@ -918,14 +1125,15 @@ export class HostSessionRegistry {
const previousSocket = existing.socket
if (existing.orphanTimer) clearTimeout(existing.orphanTimer)
existing.orphanTimer = null
existing.authorityRevision += 1
existing.assignmentEpoch = assignmentEpoch
existing.socket = socket
existing.state = existing.regionalDrainAttemptId ? 'drain-only' : 'active'
existing.appVersion = appVersion
existing.leaseExpiresAt = this.controlLeaseExpiresAt()
existing.lastPongAt = this.now()
existing.pendingPingAt = null
existing.activityRenewalDueAt =
this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs
existing.activityRenewalDueAt = this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs
this.wireActiveControl(existing)
this.sendHelloAck(existing)
if (existing.regionalDrainAttemptId) this.reassertRegionalDrain(existing)
@@ -980,6 +1188,7 @@ export class HostSessionRegistry {
pendingPingAt: null,
controlRttSamplesMs: [],
controlRttLoggedAt: null,
authorityRevision: 0,
activityRenewalDueAt: this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs,
activityRenewalAttempt: 0,
activityRenewalCompletedAttempt: 0,
@@ -1028,7 +1237,9 @@ export class HostSessionRegistry {
` splices=${session.closingCounts?.splices ?? session.activeSplices.size}` +
` pending=${session.closingCounts?.pending ?? session.pendingConns.size}` +
` code=${code} reason=${JSON.stringify(printableCloseReason(reason))}` +
(socketError === null ? '' : ` error=${JSON.stringify(printableCloseReason(socketError))}`)
(socketError === null
? ''
: ` error=${JSON.stringify(printableCloseReason(socketError))}`)
)
})
socket.on('message', (raw, isBinary) => {
@@ -1077,6 +1288,19 @@ export class HostSessionRegistry {
}
private async acceptRefresh(session: HostSession, raw: RawData): Promise<void> {
const release = this.beginIdleWork(session.relayHostId)
if (!release) {
session.socket?.close(RELAY_CLOSE_CODE.DRAINING, 'resolve-director')
return
}
try {
await this.acceptRefreshUnfenced(session, raw)
} finally {
release()
}
}
private async acceptRefreshUnfenced(session: HostSession, raw: RawData): Promise<void> {
const parsed = AuthRefreshSchema.safeParse(payload(raw, 'auth-refresh'))
if (!parsed.success) return
const refreshed = await this.verifyRelayToken(parsed.data.relayJwt)
@@ -1107,6 +1331,16 @@ export class HostSessionRegistry {
if (controlActivityId && now >= session.activityRenewalDueAt) {
const attempt = ++session.activityRenewalAttempt
const startedAt = now
const socket = session.socket
const authorityRevision = session.authorityRevision
const current = (): boolean =>
this.sessions.get(key) === session &&
session.state !== 'closed' &&
session.socket === socket &&
socket.readyState === socket.OPEN &&
session.controlActivityId === controlActivityId &&
session.authorityRevision === authorityRevision &&
attempt > session.activityRenewalCompletedAttempt
void this.assignments
.renewControlActivity(
{ userId: session.identity.sub, relayHostId: session.relayHostId },
@@ -1117,11 +1351,16 @@ export class HostSessionRegistry {
}
)
.then(() => {
if (attempt <= session.activityRenewalCompletedAttempt) return
if (!current()) return
session.activityRenewalCompletedAttempt = attempt
session.activityRenewalDueAt = startedAt + CONTROL_ACTIVITY_RENEWAL_INTERVAL_MS
})
.catch(async (error: unknown) => {
if (!current()) return
if (error instanceof Error && error.message === 'assignment_not_found') {
socket.close(RELAY_CLOSE_CODE.DRAINING, 'control assignment missing')
return
}
if (error instanceof Error && error.message === 'activity_cell_not_authoritative') {
// Completion fences a late drain-only heartbeat after all source work is gone.
session.socket?.close(RELAY_CLOSE_CODE.DRAINING, 'control migration completed')
@@ -1144,8 +1383,22 @@ export class HostSessionRegistry {
cellId: this.config.cellId
}
)
if (!current()) {
// A replaced activity must not remain leased after its owner disappears.
if (
!this.sessions.get(key) ||
this.sessions.get(key)?.controlActivityId !== controlActivityId
) {
this.releaseActivityBestEffort(
{ userId: session.identity.sub, relayHostId: session.relayHostId },
controlActivityId
)
}
return
}
this.observer.recordControlActivityRecovery?.(true)
} catch (acquireError: unknown) {
if (!current()) return
this.observer.recordControlActivityRecovery?.(false)
if (
acquireError instanceof Error &&
@@ -1218,6 +1471,21 @@ export class HostSessionRegistry {
private closeDrainedSession(session: HostSession): void {
if (session.state === 'closed') return
const forcedConnections = session.activeConnIds.size + session.pendingConns.size
if (forcedConnections > 0) {
console.warn(
JSON.stringify({
event: 'orca_relay_host_drain_forced_close',
...this.logIdentity(),
...this.sessionPlacementLogFields(session),
relayHostIdDigest: relayHostLogDigest(session.relayHostId),
reason: this.draining ? 'emergency' : 'regional-deadline',
forcedConnections,
splices: session.activeSplices.size,
pending: session.pendingConns.size
})
)
}
if (session.heartbeatTimer) clearInterval(session.heartbeatTimer)
if (session.orphanTimer) clearTimeout(session.orphanTimer)
if (session.regionalDrainTimer) clearTimeout(session.regionalDrainTimer)
@@ -1250,11 +1518,7 @@ export class HostSessionRegistry {
session.pendingConns.clear()
session.state = 'closed'
if (session.socket) {
closeRelayWebSocket(
session.socket,
RELAY_CLOSE_CODE.DRAINING,
'resolve configured director'
)
closeRelayWebSocket(session.socket, RELAY_CLOSE_CODE.DRAINING, 'resolve configured director')
}
const key = this.key(session.identity.sub, session.relayHostId)
if (this.sessions.get(key) === session) this.sessions.delete(key)
@@ -1278,6 +1542,23 @@ export class HostSessionRegistry {
session: HostSession,
type: unknown,
raw: RawData
): Promise<void> {
const release = this.beginIdleWork(session.relayHostId)
if (!release) {
session.socket?.close(RELAY_CLOSE_CODE.DRAINING, 'resolve-director')
return
}
try {
await this.acceptControlCommandUnfenced(session, type, raw)
} finally {
release()
}
}
private async acceptControlCommandUnfenced(
session: HostSession,
type: unknown,
raw: RawData
): Promise<void> {
if (typeof type !== 'string' || !session.socket) return
try {
@@ -1315,10 +1596,7 @@ export class HostSessionRegistry {
}
if (type === 'device-credential-install') {
const request = DeviceCredentialInstallSchema.parse(payload(raw, type))
if (
session.state !== 'active' &&
request.authorization.mode === 'authenticated-direct'
) {
if (session.state !== 'active' && request.authorization.mode === 'authenticated-direct') {
throw new Error('authorization_expired')
}
const installActivityId = `install:${request.reqId}`
@@ -0,0 +1,103 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import type { RelayDatabase } from './database.js'
import { openIdleRehomeTestDatabase } from './idle-regional-rehome-test-database.js'
const request = {
v: 1 as const,
attemptId: '33333333-3333-4333-8333-333333333333',
userId: 'idle-reconciliation-test',
relayHostId: 'abcdefghijklmnop',
sourceCellId: 'source',
sourceCellIncarnation: '11111111-1111-4111-8111-111111111111',
sourceAssignmentEpoch: 1,
sourceGeneration: 7,
targetCellId: 'target'
}
const databases: RelayDatabase[] = []
afterEach(async () => {
vi.restoreAllMocks()
for (const database of databases.splice(0)) await database.close()
})
async function setup() {
const database = await openIdleRehomeTestDatabase()
databases.push(database)
const store = new RelayAssignmentStore(database, () => 100_000_000)
await store.reconcileCells([
{ id: 'source', url: 'https://source.example.test', capacityRequests: 100 },
{ id: 'target', url: 'https://target.example.test', capacityRequests: 100 }
])
await store.assign(request)
await store.activateControl(request, {
cellId: request.sourceCellId,
assignmentEpoch: request.sourceAssignmentEpoch,
generation: request.sourceGeneration,
cellIncarnation: request.sourceCellIncarnation
})
return { database, store }
}
describe('idle cutover durable reconciliation', () => {
it('only permits reopening when the exact source still owns the assignment', async () => {
const { store } = await setup()
expect(await store.reconcileIdleRegionalRehome(request)).toBe('not-committed')
await store.activateControl(request, {
cellId: request.sourceCellId,
assignmentEpoch: request.sourceAssignmentEpoch,
generation: request.sourceGeneration + 1,
cellIncarnation: request.sourceCellIncarnation
})
expect(await store.reconcileIdleRegionalRehome(request)).toBe('stale')
})
it('does not reopen an obsolete source after an assignment change', async () => {
const { store } = await setup()
await store.startEvacuation(request, request.targetCellId)
expect(await store.reconcileIdleRegionalRehome(request)).toBe('stale')
})
it('propagates unavailable durable state instead of declaring rollback', async () => {
const { database, store } = await setup()
vi.spyOn(database, 'transaction').mockRejectedValue(new Error('database_unavailable'))
await expect(store.reconcileIdleRegionalRehome(request)).rejects.toThrow('database_unavailable')
})
it('waits for an outstanding assignment transaction before deciding authority', async () => {
const { database, store } = await setup()
let release!: () => void
let entered!: () => void
const locked = new Promise<void>((resolve) => {
entered = resolve
})
const gate = new Promise<void>((resolve) => {
release = resolve
})
const commit = database.transaction(async (transaction) => {
await transaction.queryLocked(
'SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?',
[request.userId, request.relayHostId]
)
entered()
await gate
await transaction.query(
'UPDATE relay_assignments SET assignment_epoch = assignment_epoch + 1 WHERE user_id = ? AND relay_host_id = ?',
[request.userId, request.relayHostId]
)
})
await locked
let settled = false
const reconciliation = store.reconcileIdleRegionalRehome(request).finally(() => {
settled = true
})
try {
await new Promise<void>((resolve) => setImmediate(resolve))
expect(settled).toBe(false)
} finally {
release()
await commit
await reconciliation
}
expect(await reconciliation).toBe('stale')
})
})
@@ -0,0 +1,117 @@
import { createHash } from 'node:crypto'
import type { IdleRegionalRehomeRequest } from '@orca-cloud/relay-contract'
import type { RelayDatabase, SqlRow } from './database.js'
export const IDLE_REHOME_PAGE_SIZE = 100
export async function selectIdleRegionalRehomes(input: {
database: RelayDatabase
now: number
heartbeatTtlMs: number
cohortPercent: number
offset: number
connectionHeadroom: Map<string, boolean>
cellIsClean: (safety: SqlRow | undefined, runtime: SqlRow, now: number) => boolean
}): Promise<Array<IdleRegionalRehomeRequest & { sourceCellUrl: string }>> {
const [runtimes, safetyRows] = await Promise.all([
input.database.query('SELECT * FROM relay_cell_runtime'),
input.database.query('SELECT * FROM relay_cell_rehome_safety')
])
const cleanCells = runtimes
.filter((runtime) =>
input.cellIsClean(
safetyRows.find((safety) => safety.cell_id === runtime.cell_id),
runtime,
input.now
)
)
.map((runtime) => String(runtime.cell_id))
const targetCells = cleanCells.filter((id) => input.connectionHeadroom.get(id) !== false)
if (!cleanCells.length || !targetCells.length) return []
const rows = await input.database.query(
`SELECT a.user_id, a.relay_host_id, a.cell_id AS source_cell_id,
a.assignment_epoch, host.generation, r.cell_incarnation,
s.cell_url, target.cell_id AS target_cell_id
FROM relay_region_rehome_control policy
JOIN relay_region_decisions d ON d.outcome = 'conclusive'
JOIN relay_assignments a ON a.user_id = d.user_id AND a.relay_host_id = d.relay_host_id
JOIN relay_cells s ON s.cell_id = a.cell_id AND s.enabled = 1
JOIN relay_cell_regions sr ON sr.cell_id = a.cell_id
JOIN relay_cell_admission sa ON sa.cell_id = a.cell_id AND sa.admission_state = 'general'
JOIN relay_cell_runtime r ON r.cell_id = a.cell_id AND r.ready = 1
JOIN relay_cell_capabilities c ON c.cell_id = r.cell_id AND c.cell_incarnation = r.cell_incarnation
JOIN relay_control_capabilities host ON host.user_id = a.user_id AND host.relay_host_id = a.relay_host_id
AND host.cell_id = a.cell_id AND host.assignment_epoch = a.assignment_epoch
AND host.cell_incarnation = r.cell_incarnation AND host.idle_regional_rehome = 1
JOIN relay_assignment_activity_leases lease ON lease.user_id = host.user_id
AND lease.relay_host_id = host.relay_host_id AND lease.activity_id = host.activity_id
AND lease.cell_id = a.cell_id AND lease.activity_kind = 'control'
JOIN relay_cell_regions tr ON tr.region = d.preferred_region
JOIN relay_cells target ON target.cell_id = tr.cell_id AND target.enabled = 1
JOIN relay_cell_admission ta ON ta.cell_id = target.cell_id AND ta.admission_state = 'general'
JOIN relay_cell_runtime rt ON rt.cell_id = target.cell_id AND rt.ready = 1
JOIN relay_cell_capabilities ct ON ct.cell_id = rt.cell_id AND ct.cell_incarnation = rt.cell_incarnation
WHERE policy.control_id = 'global' AND policy.enabled = 1 AND policy.not_before <= ?
AND d.preferred_region <> sr.region AND d.incumbent_region = sr.region
AND d.assignment_epoch = a.assignment_epoch AND d.policy_version = 1
AND d.expires_at > ? AND d.observed_at >= ? - policy.preference_max_age_ms
AND d.cohort_bucket < ? AND lease.expires_at > ? AND lease.updated_at >= r.started_at
AND r.last_heartbeat_at > ? AND rt.last_heartbeat_at > ?
AND s.cell_id IN (${cleanCells.map(() => '?').join(',')})
AND target.cell_id IN (${targetCells.map(() => '?').join(',')})
-- Reserve the moving host's source activity plus its assignment on the target.
AND target.reserved_requests + 1 + (
SELECT COALESCE(SUM(activity.request_units), 0)
FROM relay_assignment_activity_leases activity
WHERE activity.user_id = a.user_id AND activity.relay_host_id = a.relay_host_id
AND activity.cell_id = a.cell_id
) <= target.capacity_requests
AND c.regional_rehome_protocol >= 3 AND ct.regional_rehome_protocol >= 3
AND NOT EXISTS (SELECT 1 FROM relay_assignment_migrations migration
WHERE migration.user_id = a.user_id AND migration.relay_host_id = a.relay_host_id
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL)
AND NOT EXISTS (SELECT 1 FROM relay_region_rehome_attempts attempt
WHERE attempt.user_id = a.user_id AND attempt.relay_host_id = a.relay_host_id
AND attempt.created_at > ? - policy.host_cooldown_ms)
ORDER BY a.user_id, a.relay_host_id, host.generation DESC,
(target.reserved_requests + rt.observed_requests) * 1.0 / target.capacity_requests,
target.cell_id
LIMIT ? OFFSET ?`,
[
input.now,
input.now,
input.now,
input.cohortPercent,
input.now,
input.now - input.heartbeatTtlMs,
input.now - input.heartbeatTtlMs,
...cleanCells,
...targetCells,
input.now,
IDLE_REHOME_PAGE_SIZE,
input.offset
]
)
return rows.map((row) => {
const request = {
v: 1 as const,
userId: String(row.user_id),
relayHostId: String(row.relay_host_id),
sourceCellId: String(row.source_cell_id),
sourceCellIncarnation: String(row.cell_incarnation),
sourceAssignmentEpoch: Number(row.assignment_epoch),
sourceGeneration: Number(row.generation),
targetCellId: String(row.target_cell_id)
}
// UUIDv5 keeps retries on every director bound to the same source authority and target.
const digest = createHash('sha1')
.update(Buffer.from('0a1c5a9b197b4ea8b6f1f3bcaa3d712c', 'hex'))
.update(JSON.stringify(request))
.digest()
digest[6] = (digest[6]! & 0x0f) | 0x50
digest[8] = (digest[8]! & 0x3f) | 0x80
const hex = digest.subarray(0, 16).toString('hex')
const attemptId = `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`
return { ...request, attemptId, sourceCellUrl: String(row.cell_url) }
})
}
@@ -0,0 +1,396 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import type { RelayDatabase, RelayLockOptions } from './database.js'
import { openIdleRehomeTestDatabase } from './idle-regional-rehome-test-database.js'
const identity = { userId: 'idle-store-test', relayHostId: 'abcdefghijklmnop' }
const incarnations = [
'11111111-1111-4111-8111-111111111111',
'22222222-2222-4222-8222-222222222222'
]
const cells = [
{
id: 'source',
url: 'https://source.example.test',
region: 'us-central1' as const,
capacityRequests: 100
},
{
id: 'target',
url: 'https://target.example.test',
region: 'asia-east2' as const,
capacityRequests: 100
}
]
const databases: RelayDatabase[] = []
afterEach(async () => {
vi.restoreAllMocks()
for (const database of databases.splice(0)) await database.close()
})
async function setup() {
const database = await openIdleRehomeTestDatabase()
databases.push(database)
let now = 100_000_000
const store = new RelayAssignmentStore(database, () => now, { regionalRehomeCohortPercent: 100 })
await store.inspectRegionalRehomeControl()
now += 86_400_000
await store.applyRegionalRehomeControl({
expectedGeneration: 0,
enabled: true,
notBefore: now,
ratePerMinute: 10,
preferenceMaxAgeMs: 86_400_000,
hostCooldownMs: 604_800_000,
drainGraceMs: 60_000
})
await store.reconcileCells(cells)
const safety = {
observedAt: now,
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
}
for (const [index, cell] of cells.entries()) {
await store.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
region: cell.region,
cellIncarnation: incarnations[index]!,
startedAt: now - 1_000,
ready: true,
observedRequests: 0
})
await store.recordCellRegionalRehomeStatus({
cellId: cell.id,
cellIncarnation: incarnations[index]!,
regionalRehomeProtocol: 3,
safety
})
}
const assignment = await store.assign(identity, undefined, 'us-central1')
await store.activateControl(identity, {
cellId: cells[0]!.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 7,
cellIncarnation: incarnations[0],
idleRegionalRehome: true
})
const issued = await store.exchangeRegionCorrection(
identity,
{ v: 1, action: 'issue-window' },
assignment.assignmentEpoch
)
await store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: issued.window!.generation,
assignmentEpoch: assignment.assignmentEpoch,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 180, 'asia-east2': 40 }
},
assignment.assignmentEpoch
)
const request = {
v: 1 as const,
...identity,
attemptId: '33333333-3333-4333-8333-333333333333',
sourceCellId: cells[0]!.id,
sourceCellIncarnation: incarnations[0]!,
sourceAssignmentEpoch: assignment.assignmentEpoch,
sourceGeneration: 7,
targetCellId: cells[1]!.id
}
return { store, database, safety, request }
}
describe('constrained idle regional assignment transaction', () => {
it.each(['missing', 'disabled', 'future'] as const)(
'does only one read per tick with %s durable control and sees later enablement',
async (state) => {
const { store, database, safety } = await setup()
const control = (await database.query(
"SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'"
))[0]!
if (state === 'missing') {
await database.query('DELETE FROM relay_region_rehome_control')
} else {
await database.query(
"UPDATE relay_region_rehome_control SET enabled = ?, not_before = ? WHERE control_id = 'global'",
[state === 'disabled' ? 0 : 1, safety.observedAt + (state === 'future' ? 1 : 0)]
)
}
const query = vi.spyOn(database, 'query')
const transaction = vi.spyOn(database, 'transaction')
for (let tick = 0; tick < 3; tick++) {
query.mockClear()
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
expect(query).toHaveBeenCalledTimes(1)
expect(query.mock.calls[0]![0]).toMatch(/^SELECT .*FROM relay_region_rehome_control/s)
expect(transaction).not.toHaveBeenCalled()
}
if (state === 'missing') {
const columns = Object.keys(control)
await database.query(
`INSERT INTO relay_region_rehome_control (${columns.join(',')}) VALUES (${columns.map(() => '?').join(',')})`,
Object.values(control)
)
} else {
await database.query(
"UPDATE relay_region_rehome_control SET enabled = 1, not_before = ? WHERE control_id = 'global'",
[safety.observedAt]
)
}
query.mockClear()
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(1)
expect(query.mock.calls.length).toBeGreaterThan(1)
await database.query("UPDATE relay_region_rehome_control SET enabled = 0 WHERE control_id = 'global'")
query.mockClear()
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
expect(query).toHaveBeenCalledTimes(1)
}
)
it.each([10, 11])('reserves source activity plus assignment at target capacity %i', async (capacity) => {
const { store, database, safety, request } = await setup()
// Model three source activity units and seven units already reserved at the target.
await database.query(
'UPDATE relay_assignment_activity_leases SET request_units = 3 WHERE user_id = ? AND relay_host_id = ?',
[identity.userId, identity.relayHostId]
)
await database.query("UPDATE relay_cells SET reserved_requests = 4 WHERE cell_id = 'source'")
await database.query(
"UPDATE relay_cells SET reserved_requests = 7, capacity_requests = ? WHERE cell_id = 'target'",
[capacity]
)
const candidates = await store.selectIdleRegionalRehomeCandidates(safety)
expect(candidates).toHaveLength(capacity === 11 ? 1 : 0)
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({
outcome: capacity === 11 ? 'committed' : 'deferred'
})
const [target] = await database.query("SELECT reserved_requests FROM relay_cells WHERE cell_id = 'target'")
expect(Number(target!.reserved_requests)).toBe(capacity === 11 ? 11 : 7)
expect(await store.resolve(identity)).toMatchObject({
cellId: capacity === 11 ? 'target' : 'source',
assignmentEpoch: capacity === 11 ? 2 : 1
})
})
it('progresses past a full page of busy candidates without writing eligibility state', async () => {
const { store, database, safety } = await setup()
for (const table of [
'relay_assignments',
'relay_assignment_activity_leases',
'relay_control_capabilities',
'relay_region_decisions'
]) {
const template = (
await database.query(`SELECT * FROM ${table} WHERE user_id = ? AND relay_host_id = ?`, [
identity.userId,
identity.relayHostId
])
)[0]!
const columns = Object.keys(template)
for (let index = 0; index < 100; index++) {
const values = columns.map((column) =>
column === 'user_id' || column === 'relay_host_id' ? '?' : column
)
await database.query(
`INSERT INTO ${table} (${columns.join(', ')}) SELECT ${values.join(', ')} FROM ${table}
WHERE user_id = ? AND relay_host_id = ?`,
[
`idle-store-test-${String(index).padStart(3, '0')}`,
`pagehost${String(index).padStart(8, '0')}`,
identity.userId,
identity.relayHostId
]
)
}
}
const first = await store.selectIdleRegionalRehomeCandidates(safety)
const next = await store.selectIdleRegionalRehomeCandidates(safety)
expect(first).toHaveLength(100)
expect(next).toHaveLength(1)
expect(next[0]!.relayHostId).toBe('pagehost00000099')
const restarted = new RelayAssignmentStore(database, () => safety.observedAt, {
regionalRehomeCohortPercent: 100
})
expect(await restarted.selectIdleRegionalRehomeCandidates(safety)).toEqual(first)
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
const decisions = await database.query('SELECT last_considered_at FROM relay_region_decisions')
expect(decisions.every((decision) => Number(decision.last_considered_at) === 0)).toBe(true)
})
it.runIf(Boolean(process.env.ORCA_IDLE_REHOME_POSTGRES_URL))(
'rechecks generation when replacement wins after the initial authority lookup',
async () => {
const { store, safety, request, database } = await setup()
const held = holdStatement(database, 'SELECT * FROM relay_region_rehome_control')
const commit = store.commitIdleRegionalRehome(request, safety)
await held.entered
try {
await store.activateControl(identity, {
cellId: 'source',
assignmentEpoch: 1,
generation: 8,
cellIncarnation: incarnations[0],
idleRegionalRehome: true
})
} finally {
held.release()
}
expect(await commit).toEqual({ outcome: 'deferred' })
expect(await store.reconcileIdleRegionalRehome(request)).toBe('stale')
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
}
)
it('rejects source replacement when the cutover already holds assignment authority', async () => {
const { store, safety, request, database } = await setup()
const held = holdStatement(database, 'UPDATE relay_assignments SET cell_id')
const commit = store.commitIdleRegionalRehome(request, safety)
await held.entered
const replacement = store.activateControl(identity, {
cellId: 'source',
assignmentEpoch: 1,
generation: 8,
cellIncarnation: incarnations[0],
idleRegionalRehome: true
})
const rejected = expect(replacement).rejects.toThrow('wrong_assignment')
held.release()
expect(await commit).toEqual({ outcome: 'committed' })
await rejected
expect(await store.resolve(identity)).toMatchObject({ cellId: 'target', assignmentEpoch: 2 })
})
it('finds the committed attempt after its database reply is lost', async () => {
const { store, safety, request, database } = await setup()
const transaction = database.transaction.bind(database)
const intercepted = vi
.spyOn(database, 'transaction')
.mockImplementation(async (operation, options) => {
let changed = false
const result = await transaction(
async (tx) =>
operation(
new Proxy(tx, {
get(target, key) {
if (key === 'query')
return async (sql: string, params?: unknown[]) => {
if (sql.includes('INSERT INTO relay_region_rehome_attempts')) changed = true
return target.query(sql, params)
}
const value = Reflect.get(target, key)
return typeof value === 'function' ? value.bind(target) : value
}
})
),
options
)
if (changed) throw new Error('simulated_commit_reply_lost')
return result
})
await expect(store.commitIdleRegionalRehome(request, safety)).rejects.toThrow(
'simulated_commit_reply_lost'
)
intercepted.mockRestore()
expect(await store.reconcileIdleRegionalRehome(request)).toBe('committed')
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({ outcome: 'committed' })
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toHaveLength(1)
})
it('commits the requested move once and records its outcome without source retention', async () => {
const { store, database, safety, request } = await setup()
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({ outcome: 'committed' })
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({ outcome: 'committed' })
expect(await store.resolve(identity)).toMatchObject({ cellId: 'target', assignmentEpoch: 2 })
const attempts = await database.query('SELECT * FROM relay_region_rehome_attempts')
expect(attempts).toHaveLength(1)
expect(attempts[0]!.attempt_id).toBe(request.attemptId)
expect(Number(attempts[0]!.source_generation)).toBe(7)
})
it('rejects a replaced control and never substitutes a different target', async () => {
const { store, safety, request } = await setup()
expect(
await store.commitIdleRegionalRehome({ ...request, targetCellId: 'missing' }, safety)
).toEqual({ outcome: 'deferred' })
await store.activateControl(identity, {
cellId: 'source',
assignmentEpoch: 1,
generation: 8,
cellIncarnation: incarnations[0],
idleRegionalRehome: true
})
expect(await store.commitIdleRegionalRehome(request, safety)).toEqual({ outcome: 'stale' })
expect(await store.resolve(identity)).toMatchObject({ cellId: 'source', assignmentEpoch: 1 })
})
it('does not commit without process safety or cohort authorization', async () => {
const { store, safety, request, database } = await setup()
expect(await store.commitIdleRegionalRehome(request)).toEqual({ outcome: 'deferred' })
expect(await store.commitIdleRegionalRehome(request, safety, 0)).toEqual({
outcome: 'deferred'
})
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
})
it('selects read-only with stable identity and the control generation, not probe generation', async () => {
const { store, safety, database } = await setup()
const before = await database.query('SELECT * FROM relay_assignments')
const candidates = await store.selectIdleRegionalRehomeCandidates(safety)
expect(candidates).toHaveLength(1)
expect(candidates[0]).toMatchObject({
sourceGeneration: 7,
sourceCellId: 'source',
targetCellId: 'target'
})
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual(candidates)
expect(await database.query('SELECT * FROM relay_assignments')).toEqual(before)
expect(await database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual([])
})
})
function holdStatement(database: RelayDatabase, fragment: string) {
let entered!: () => void
let release!: () => void
const arrival = new Promise<void>((resolve) => {
entered = resolve
})
const gate = new Promise<void>((resolve) => {
release = resolve
})
let held = false
const transaction = database.transaction.bind(database)
vi.spyOn(database, 'transaction').mockImplementation((operation, options) =>
transaction(async (tx) => {
return operation(
new Proxy(tx, {
get(target, key) {
if (key === 'query' || key === 'queryLocked')
return async (sql: string, params?: unknown[], lockOptions?: RelayLockOptions) => {
if (!held && sql.includes(fragment)) {
held = true
entered()
await gate
}
return key === 'queryLocked'
? target.queryLocked(sql, params, lockOptions)
: target.query(sql, params)
}
const value = Reflect.get(target, key)
return typeof value === 'function' ? value.bind(target) : value
}
})
)
}, options)
)
return { entered: arrival, release }
}
@@ -0,0 +1,40 @@
import { randomUUID } from 'node:crypto'
import pg from 'pg'
import { openInMemoryRelayDatabase, openRelayDatabase, type RelayDatabase } from './database.js'
export async function openIdleRehomeTestDatabase(): Promise<RelayDatabase> {
const configured = process.env.ORCA_IDLE_REHOME_POSTGRES_URL
if (!configured) return openInMemoryRelayDatabase()
const url = new URL(configured)
if (url.port !== '55440' || !['localhost', '127.0.0.1', '[::1]'].includes(url.hostname)) {
throw new Error('idle_rehome_tests_require_local_postgres_55440')
}
const schema = `idle_rehome_${randomUUID().replaceAll('-', '')}`
const admin = new pg.Client({ connectionString: configured })
await admin.connect()
try {
await admin.query(`CREATE SCHEMA ${schema}`)
url.searchParams.set('options', `-c search_path=${schema}`)
const database = await openRelayDatabase({ databaseUrl: url.toString(), dataDir: '' })
const close = database.close.bind(database)
database.close = async () => {
try {
await close()
} finally {
try {
await admin.query(`DROP SCHEMA ${schema} CASCADE`)
} finally {
await admin.end()
}
}
}
return database
} catch (error) {
try {
await admin.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
} finally {
await admin.end()
}
throw error
}
}
@@ -0,0 +1,105 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { RelayAssignmentStore } from './assignment-store.js'
import type { RelayConfig } from './config.js'
import { startRegionalRehomeWorker } from './regional-rehome-worker.js'
const candidate = {
v: 1,
attemptId: '11111111-1111-4111-8111-111111111111',
userId: 'private-user',
relayHostId: 'abcdefghijklmnop',
sourceCellId: 'source',
sourceCellUrl: 'https://source.example.test',
sourceCellIncarnation: '22222222-2222-4222-8222-222222222222',
sourceAssignmentEpoch: 7,
sourceGeneration: 3,
targetCellId: 'target'
}
const config = {
role: 'director',
regionCorrectionCohortPercent: 100,
rehomeAudience: 'https://relay.example.test/v1/admin/host-drain',
rehomeDirectorServiceAccount: 'director@example.test'
} as RelayConfig
function setup(fetch: typeof globalThis.fetch) {
const selectIdleRegionalRehomeCandidates = vi
.fn()
.mockResolvedValueOnce([])
.mockResolvedValue([candidate])
const claimRegionalRehome = vi.fn()
const recordRegionalRehomeDispatchFailure = vi.fn()
const worker = startRegionalRehomeWorker(
config,
{
selectIdleRegionalRehomeCandidates,
claimRegionalRehome,
recordRegionalRehomeDispatchFailure
} as unknown as RelayAssignmentStore,
{
safetySnapshot: () => ({ observedAt: 100 }) as never,
intervalMs: 60_000,
identityToken: async () => 'private-token',
fetch
}
)!
return {
worker,
selectIdleRegionalRehomeCandidates,
claimRegionalRehome,
recordRegionalRehomeDispatchFailure
}
}
describe('idle regional worker dispatch', () => {
afterEach(() => vi.restoreAllMocks())
it('sends an idle request without claiming an assignment first', async () => {
const fetch = vi.fn<typeof globalThis.fetch>(async () =>
Response.json({ v: 1, outcome: 'committed' })
)
const c = setup(fetch)
await vi.waitFor(() => expect(c.selectIdleRegionalRehomeCandidates).toHaveBeenCalledOnce())
await c.worker.run()
c.worker.stop()
expect(c.claimRegionalRehome).not.toHaveBeenCalled()
expect(fetch).toHaveBeenCalledOnce()
const [url, init] = fetch.mock.calls[0]!
expect(String(url)).toBe('https://source.example.test/v1/admin/host-idle-rehome')
const { sourceCellUrl: _, ...request } = candidate
expect(JSON.parse(String(init?.body))).toEqual({
...request,
cohortPercent: 100,
directorSafety: { observedAt: 100 }
})
})
it('progresses past busy hosts without charging a dispatch failure', async () => {
const fetch = vi
.fn<typeof globalThis.fetch>()
.mockResolvedValueOnce(Response.json({ v: 1, outcome: 'busy' }))
.mockResolvedValueOnce(Response.json({ v: 1, outcome: 'committed' }))
const c = setup(fetch)
await vi.waitFor(() => expect(c.selectIdleRegionalRehomeCandidates).toHaveBeenCalledOnce())
c.selectIdleRegionalRehomeCandidates.mockResolvedValue([
candidate,
{
...candidate,
relayHostId: 'ponmlkjihgfedcba',
attemptId: '33333333-3333-4333-8333-333333333333'
}
])
await c.worker.run()
c.worker.stop()
expect(fetch).toHaveBeenCalledTimes(2)
expect(c.recordRegionalRehomeDispatchFailure).not.toHaveBeenCalled()
})
it('does not charge a lost response as a claimed migration failure', async () => {
const fetch = vi.fn<typeof globalThis.fetch>(async () => {
throw new Error('response lost')
})
const c = setup(fetch)
await vi.waitFor(() => expect(c.selectIdleRegionalRehomeCandidates).toHaveBeenCalledOnce())
await c.worker.run()
c.worker.stop()
expect(c.recordRegionalRehomeDispatchFailure).not.toHaveBeenCalled()
})
})
+8
View File
@@ -2,6 +2,7 @@ import {
formatAssignmentInventorySnapshot,
readAssignmentInventorySnapshot
} from './assignment-inventory-snapshot.js'
import { readRegionCorrectionOutcomes } from './region-correction-outcomes.js'
import { RelayAssignmentStore } from './assignment-store.js'
import { loadRelayConfig } from './config.js'
import { startCellHeartbeat } from './cell-heartbeat-client.js'
@@ -71,6 +72,13 @@ const migrationInventoryTimer = roleOwnsAssignmentMaintenance(config.role)
void runRelayBackgroundOperation(async () => {
const inventory = await readRegisteredMigrationInventory(database, Date.now())
for (const line of formatRegisteredMigrationInventory(inventory)) console.warn(line)
console.log(
JSON.stringify({
event: 'orca_relay_region_correction_outcomes',
observedAt: Date.now(),
outcomes: await readRegionCorrectionOutcomes(database, Date.now())
})
)
}, '[orca-relay] migration inventory failed')
}, 5 * 60_000)
: null
@@ -0,0 +1,29 @@
import type { RelayDatabase } from './database.js'
export async function readRegionCorrectionOutcomes(database: RelayDatabase, now: number) {
const rows = await database.query(
`SELECT attempt.source_cell_id, attempt.target_cell_id,
CASE WHEN attempt.aborted_at IS NOT NULL THEN 'aborted'
WHEN attempt.completed_at IS NOT NULL THEN 'completed'
WHEN migration.target_registered_at IS NOT NULL THEN 'registered' ELSE 'registering' END AS state,
COUNT(*) AS count,
COALESCE(MAX(CASE WHEN attempt.completed_at IS NULL AND attempt.aborted_at IS NULL
THEN ? - attempt.created_at ELSE 0 END), 0) AS oldest_open_ms,
COALESCE(SUM(CASE WHEN attempt.completed_at IS NULL AND attempt.aborted_at IS NULL
THEN migration.target_reserved_units ELSE 0 END), 0) AS target_reserved_units
FROM relay_region_rehome_attempts attempt
JOIN relay_assignment_migrations migration ON migration.user_id = attempt.user_id
AND migration.relay_host_id = attempt.relay_host_id AND migration.assignment_epoch = attempt.assignment_epoch
GROUP BY attempt.source_cell_id, attempt.target_cell_id, state
ORDER BY attempt.source_cell_id, attempt.target_cell_id, state`,
[now]
)
return rows.map((row) => ({
sourceCellId: String(row.source_cell_id),
targetCellId: String(row.target_cell_id),
state: String(row.state),
count: Number(row.count),
oldestOpenMs: Number(row.oldest_open_ms),
targetReservedUnits: Number(row.target_reserved_units)
}))
}
@@ -0,0 +1,157 @@
import type { RelayDatabase, SqlRow } from './database.js'
import { REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS } from './database.js'
import {
REGIONAL_REHOME_CONCURRENT_LIMIT,
REGION_DECISION_TTL_MS
} from './region-correction-state.js'
export type RegionCorrectionPreview = {
observedAt: number
newClaimsEnabled: boolean
cohortPercent: number
openMigrations: number
availableMigrationSlots: number
globalSafetyFailure: string | null
counts: Record<string, number>
}
export async function previewRegionalRehomeEligibility(input: {
database: RelayDatabase
now: number
heartbeatTtlMs: number
cohortPercent: number
globalSafetyFailure: string | null
connectionHeadroom: ReadonlyMap<string, boolean>
cellIsClean: (safety: SqlRow | undefined, runtime: SqlRow, now: number) => boolean
}): Promise<RegionCorrectionPreview> {
const { database, now } = input
const [hosts, cells, runtimeRows, capabilityRows, safetyRows, controls, migrations] =
await Promise.all([
database.query(
`SELECT assignment.cell_id, assignment.assignment_epoch,
decision.generation, decision.assignment_epoch AS decision_epoch, decision.expires_at,
decision.incumbent_region, decision.preferred_region, decision.outcome, decision.policy_version,
decision.observed_at, decision.cohort_bucket,
(SELECT MAX(attempt.created_at) FROM relay_region_rehome_attempts attempt
WHERE attempt.user_id = assignment.user_id AND attempt.relay_host_id = assignment.relay_host_id) AS last_attempt_at,
(SELECT COUNT(*) FROM relay_assignment_migrations migration
WHERE migration.user_id = assignment.user_id AND migration.relay_host_id = assignment.relay_host_id
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL) AS open_migrations,
(SELECT COALESCE(SUM(lease.request_units),0) FROM relay_assignment_activity_leases lease
WHERE lease.user_id = assignment.user_id AND lease.relay_host_id = assignment.relay_host_id
AND lease.cell_id = assignment.cell_id) AS source_units,
(SELECT COUNT(*) FROM relay_control_capabilities host_capability
JOIN relay_assignment_activity_leases lease ON lease.user_id = host_capability.user_id
AND lease.relay_host_id = host_capability.relay_host_id AND lease.activity_id = host_capability.activity_id
JOIN relay_cell_runtime runtime ON runtime.cell_id = host_capability.cell_id
AND runtime.cell_incarnation = host_capability.cell_incarnation
WHERE host_capability.user_id = assignment.user_id AND host_capability.relay_host_id = assignment.relay_host_id
AND host_capability.cell_id = assignment.cell_id AND host_capability.assignment_epoch = assignment.assignment_epoch
AND host_capability.idle_regional_rehome = 1 AND lease.activity_kind = 'control'
AND lease.activity_id NOT LIKE 'control-pending:%' AND lease.expires_at > ?
AND lease.updated_at >= runtime.started_at) AS capable_controls
FROM relay_assignments assignment LEFT JOIN relay_region_decisions decision
ON decision.user_id = assignment.user_id AND decision.relay_host_id = assignment.relay_host_id`,
[now]
),
database.query(`SELECT cell.*, region.region, admission.admission_state FROM relay_cells cell
LEFT JOIN relay_cell_regions region ON region.cell_id = cell.cell_id
LEFT JOIN relay_cell_admission admission ON admission.cell_id = cell.cell_id`),
database.query(`SELECT * FROM relay_cell_runtime`),
database.query(`SELECT * FROM relay_cell_capabilities`),
database.query(`SELECT * FROM relay_cell_rehome_safety`),
database.query(`SELECT * FROM relay_region_rehome_control WHERE control_id = 'global'`),
database.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE completed_at IS NULL AND aborted_at IS NULL`
)
])
const byCell = (rows: SqlRow[]) => new Map(rows.map((row) => [String(row.cell_id), row]))
const runtimes = byCell(runtimeRows)
const capabilities = byCell(capabilityRows)
const safety = byCell(safetyRows)
const inventory = byCell(cells)
const control = controls[0]
const cooldown = Number(control?.host_cooldown_ms ?? REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS)
const maxAge = Number(control?.preference_max_age_ms ?? REGION_DECISION_TTL_MS)
const openMigrations = Number(migrations[0]?.count ?? 0)
const counts: Record<string, number> = {}
const count = (reason: string) => {
counts[reason] = (counts[reason] ?? 0) + 1
}
const available = (cell: SqlRow): boolean => {
const id = String(cell.cell_id)
const runtime = runtimes.get(id)
const capability = capabilities.get(id)
return (
Number(cell.enabled) === 1 &&
cell.admission_state === 'general' &&
cell.region != null &&
runtime !== undefined &&
Number(runtime.ready) === 1 &&
Number(runtime.last_heartbeat_at) > now - input.heartbeatTtlMs &&
capability !== undefined &&
capability.cell_incarnation === runtime.cell_incarnation &&
Number(capability.regional_rehome_protocol) >= 3
)
}
for (const host of hosts) {
let reason: string | null = null
const source = inventory.get(String(host.cell_id))
if (host.generation == null) reason = 'no-verified-decision'
else if (Number(host.expires_at) <= now || Number(host.observed_at) < now - maxAge)
reason = 'expired'
else if (
Number(host.decision_epoch) !== Number(host.assignment_epoch) ||
host.incumbent_region !== source?.region
)
reason = 'basis-changed'
else if (
host.outcome !== 'conclusive' ||
Number(host.policy_version) !== 1 ||
host.preferred_region == null
)
reason = 'inconclusive-or-insufficient-improvement'
else if (Number(host.cohort_bucket) >= input.cohortPercent) reason = 'outside-cohort'
else if (Number(host.open_migrations) > 0) reason = 'migration-open'
else if (host.last_attempt_at != null && Number(host.last_attempt_at) > now - cooldown)
reason = 'host-cooldown'
else if (!source || !available(source)) reason = 'source-ineligible'
else if (Number(host.capable_controls) === 0) reason = 'source-control-unsupported-or-inactive'
else if (
!input.cellIsClean(safety.get(String(host.cell_id)), runtimes.get(String(host.cell_id))!, now)
)
reason = 'source-unclean'
if (reason) {
count(reason)
continue
}
const targets = cells.filter(
(cell) =>
cell.cell_id !== host.cell_id && cell.region === host.preferred_region && available(cell)
)
const clean = targets.filter((cell) =>
input.cellIsClean(safety.get(String(cell.cell_id)), runtimes.get(String(cell.cell_id))!, now)
)
const capacity = clean.filter(
(cell) =>
input.connectionHeadroom.get(String(cell.cell_id)) !== false &&
Number(cell.reserved_requests) + Number(host.source_units) + 1 <=
Number(cell.capacity_requests)
)
if (targets.length === 0) count('no-eligible-target')
else if (clean.length === 0) count('target-unclean')
else if (capacity.length === 0) count('no-target-headroom')
else if (input.globalSafetyFailure) count('global-safety-blocked')
else if (openMigrations >= REGIONAL_REHOME_CONCURRENT_LIMIT) count('concurrent-migration-cap')
else count(`eligible:${host.incumbent_region}-to-${host.preferred_region}`)
}
return {
observedAt: now,
newClaimsEnabled: Number(control?.enabled ?? 0) === 1,
cohortPercent: input.cohortPercent,
openMigrations,
availableMigrationSlots: Math.max(0, REGIONAL_REHOME_CONCURRENT_LIMIT - openMigrations),
globalSafetyFailure: input.globalSafetyFailure,
counts
}
}
@@ -0,0 +1,119 @@
import { mkdtemp, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { openRelayDatabase, type RelayDatabase } from './database.js'
const identity = { userId: 'restart-test-user', relayHostId: 'abcdefghijklmnop' }
const paths: string[] = []
const databases = new Set<RelayDatabase>()
afterEach(async () => {
for (const database of databases) await database.close()
databases.clear()
for (const path of paths.splice(0)) await rm(path, { recursive: true, force: true })
})
async function setup() {
const dataDir = await mkdtemp(join(tmpdir(), 'relay-region-restart-'))
paths.push(dataDir)
let now = 1_000_000_000
const open = async () => {
const database = await openRelayDatabase({ dataDir })
databases.add(database)
return { database, store: new RelayAssignmentStore(database, () => now) }
}
const first = await open()
const cell = {
id: 'restart-us',
url: 'https://restart-us.example.test',
region: 'us-central1' as const,
capacityRequests: 100
}
await first.store.reconcileCells([cell])
await first.store.setCellEnabled(cell.id, true)
await first.store.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
cellIncarnation: '11111111-1111-4111-8111-111111111111',
region: cell.region,
startedAt: now - 1_000,
ready: true,
observedRequests: 0
})
const assignment = await first.store.assign(identity)
const issue = (store: RelayAssignmentStore) =>
store.exchangeRegionCorrection(identity, { v: 1, action: 'issue-window' }, assignment.assignmentEpoch)
const window = (await issue(first.store)).window!
const report = {
v: 1 as const,
action: 'report' as const,
generation: window.generation,
assignmentEpoch: window.assignmentEpoch,
policyVersion: 1 as const,
outcome: 'conclusive' as const,
measurements: { 'us-central1': 200, 'asia-east2': 40 }
}
const restart = async () => {
await first.database.close()
databases.delete(first.database)
return open()
}
return {
...first, window, report, issue, restart,
setNow: (value: number) => { now = value }
}
}
describe('persisted region decisions across director restart', () => {
it('keeps tombstones and fixed expiry, then invalidates the prior generation after restart', async () => {
const context = await setup()
const epoch = context.window.assignmentEpoch
await context.store.exchangeRegionCorrection(identity, {
v: 1, action: 'report', generation: context.window.generation,
assignmentEpoch: epoch, policyVersion: 1, outcome: 'inconclusive', reason: 'jitter'
}, epoch)
const restarted = await context.restart()
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, epoch))
.toMatchObject({ reportStatus: 'duplicate' })
const row = (await restarted.database.query('SELECT * FROM relay_region_decisions'))[0]!
expect(row.outcome).toBe('inconclusive')
expect(Number(row.expires_at)).toBe(context.window.expiresAt)
const successor = (await context.issue(restarted.store)).window!
expect(successor.generation).toBe(context.window.generation + 1)
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, epoch))
.toMatchObject({ reportStatus: 'stale' })
})
it('uses server expiry after a restart regardless of an old client report', async () => {
const context = await setup()
context.setNow(context.window.expiresAt)
const restarted = await context.restart()
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, context.window.assignmentEpoch))
.toMatchObject({ reportStatus: 'expired' })
expect(await restarted.store.previewRegionCorrection()).toEqual({ expired: 1 })
})
it('does not interpret a persisted future-policy window using the old policy after rollback', async () => {
const context = await setup()
await context.database.query('UPDATE relay_region_decisions SET policy_version = 2')
const restarted = await context.restart()
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, context.window.assignmentEpoch))
.toMatchObject({ reportStatus: 'stale' })
const row = (await restarted.database.query('SELECT * FROM relay_region_decisions'))[0]!
expect(row.outcome).toBe('pending')
expect(row.preferred_region).toBeNull()
expect(row.report_json).toBeNull()
})
it('keeps generation ordering when the server clock moves backwards across restart', async () => {
const context = await setup()
context.setNow(1_000_000_000 - 60_000)
const restarted = await context.restart()
const successor = (await context.issue(restarted.store)).window!
expect(successor.generation).toBe(context.window.generation + 1)
expect(successor.expiresAt).toBe(context.window.expiresAt - 60_000)
expect(await restarted.store.exchangeRegionCorrection(identity, context.report, context.window.assignmentEpoch))
.toMatchObject({ reportStatus: 'stale' })
})
})
@@ -0,0 +1,158 @@
import { relayHostLogDigest } from './relay-host-log-digest.js'
import { createHash } from 'node:crypto'
import type {
RegionCorrectionRequest,
RegionCorrectionResponse,
RelayRegion
} from '@orca-cloud/relay-contract'
import type { RelayDatabase } from './database.js'
type Identity = { userId: string; relayHostId: string }
export const REGION_DECISION_TTL_MS = 24 * 60 * 60_000
export const REGIONAL_REHOME_CONCURRENT_LIMIT = 8
export async function exchangeRegionCorrection(
database: RelayDatabase,
identity: Identity,
request: RegionCorrectionRequest,
assignmentEpoch: number,
now: number
): Promise<RegionCorrectionResponse> {
const result: RegionCorrectionResponse = await database.transaction(async (transaction) => {
const assignment = (
await transaction.queryLocked(
`SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
)[0]
const region =
assignment &&
(
await transaction.query(`SELECT region FROM relay_cell_regions WHERE cell_id = ?`, [
assignment.cell_id
])
)[0]
if (!assignment || !region || Number(assignment.assignment_epoch) !== assignmentEpoch) {
return { v: 1, reportStatus: 'basis-changed' }
}
const prior = (
await transaction.queryLocked(
`SELECT * FROM relay_region_decisions WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
)[0]
if (request.action === 'issue-window') {
const generation = Number(prior?.generation ?? 0) + 1
if (!Number.isSafeInteger(generation)) throw new Error('region_generation_exhausted')
const expiresAt = now + REGION_DECISION_TTL_MS
const cohortBucket =
createHash('sha256')
.update(JSON.stringify([identity.userId, identity.relayHostId]))
.digest()
.readUInt32BE(0) % 100
await transaction.query(
`INSERT INTO relay_region_decisions
(user_id, relay_host_id, generation, expires_at, assignment_epoch, incumbent_region,
policy_version, outcome, preferred_region, observed_at, report_json, cohort_bucket)
VALUES (?, ?, ?, ?, ?, ?, 1, 'pending', NULL, ?, NULL, ?)
ON CONFLICT (user_id, relay_host_id) DO UPDATE SET
generation = excluded.generation, expires_at = excluded.expires_at,
assignment_epoch = excluded.assignment_epoch, incumbent_region = excluded.incumbent_region,
policy_version = 1, outcome = 'pending', preferred_region = NULL,
observed_at = excluded.observed_at, report_json = NULL, cohort_bucket = excluded.cohort_bucket`,
[
identity.userId,
identity.relayHostId,
generation,
expiresAt,
assignmentEpoch,
region.region,
now,
cohortBucket
]
)
return {
v: 1,
window: {
generation,
expiresAt,
assignmentEpoch,
incumbentRegion: region.region as RelayRegion,
policyVersion: 1
}
}
}
if (!prior || Number(prior.generation) !== request.generation)
return { v: 1, reportStatus: 'stale' }
if (Number(prior.policy_version) !== request.policyVersion)
return { v: 1, reportStatus: 'stale' }
if (Number(prior.expires_at) <= now) return { v: 1, reportStatus: 'expired' }
if (
request.assignmentEpoch !== assignmentEpoch ||
Number(prior.assignment_epoch) !== assignmentEpoch ||
prior.incumbent_region !== region.region
) {
return { v: 1, reportStatus: 'basis-changed' }
}
// The first report wins, including an inconclusive tombstone.
if (prior.outcome !== 'pending') return { v: 1, reportStatus: 'duplicate' }
let preferredRegion: RelayRegion | null = null
if (request.outcome === 'conclusive') {
const incumbent = request.measurements[region.region as RelayRegion]
const target: RelayRegion = region.region === 'us-central1' ? 'asia-east2' : 'us-central1'
const targetRtt = request.measurements[target]
if (incumbent - targetRtt >= 25 && targetRtt <= incumbent * 0.8) preferredRegion = target
}
await transaction.query(
`UPDATE relay_region_decisions SET outcome = ?, preferred_region = ?, report_json = ?
WHERE user_id = ? AND relay_host_id = ? AND generation = ?`,
[
request.outcome,
preferredRegion,
JSON.stringify(request),
identity.userId,
identity.relayHostId,
request.generation
]
)
return { v: 1, reportStatus: 'accepted' }
})
if (request.action === 'report' && result.reportStatus === 'accepted') {
const digest = relayHostLogDigest(identity.relayHostId)
// Stable sampling includes unchanged hosts for before/after comparisons.
if (Number.parseInt(digest.slice(0, 8), 16) % 10 === 0) {
console.log(
JSON.stringify({
event: 'orca_relay_region_comparison',
relayHostIdDigest: digest,
assignmentEpoch,
generation: request.generation,
policyVersion: request.policyVersion,
outcome: request.outcome,
...(request.outcome === 'conclusive' ? { measurements: request.measurements } : {})
})
)
}
}
return result
}
export async function previewRegionCorrection(
database: RelayDatabase,
now: number
): Promise<Record<string, number>> {
const rows = await database.query(
`SELECT CASE WHEN decision.expires_at <= ? THEN 'expired'
WHEN decision.assignment_epoch <> assignment.assignment_epoch THEN 'basis-changed'
WHEN decision.outcome = 'pending' THEN 'pending'
WHEN decision.preferred_region IS NULL THEN 'ineligible'
ELSE decision.incumbent_region || '-to-' || decision.preferred_region END AS reason,
COUNT(*) AS count
FROM relay_region_decisions decision
JOIN relay_assignments assignment ON assignment.user_id = decision.user_id
AND assignment.relay_host_id = decision.relay_host_id
GROUP BY reason`,
[now]
)
return Object.fromEntries(rows.map((row) => [String(row.reason), Number(row.count)]))
}
@@ -0,0 +1,359 @@
import { afterEach, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { openInMemoryRelayDatabase, openRelayDatabase, type RelayDatabase } from './database.js'
const identity = { userId: 'region-correction-test-user', relayHostId: 'abcdefghijklmnop' }
const cells = [
{
id: 'decision-us',
url: 'https://decision-us.example.test',
region: 'us-central1' as const,
capacityRequests: 100
},
{
id: 'decision-asia',
url: 'https://decision-asia.example.test',
region: 'asia-east2' as const,
capacityRequests: 100
}
]
const incarnations = [
'11111111-1111-4111-8111-111111111111',
'22222222-2222-4222-8222-222222222222'
]
const opened: RelayDatabase[] = []
afterEach(async () => {
for (const database of opened.splice(0)) {
if (database.dialect === 'postgres') await cleanupPostgres(database)
await database.close()
}
})
async function cleanupPostgres(database: RelayDatabase) {
for (const table of [
'relay_control_connection_reservations',
'relay_region_decisions',
'relay_control_capabilities',
'relay_assignment_activity_leases',
'relay_assignment_migrations',
'relay_assignment_migration_incarnations',
'relay_assignment_region_preferences',
'relay_region_rehome_attempts',
'relay_assignments'
]) {
await database.query(`DELETE FROM ${table} WHERE user_id = ?`, [identity.userId])
}
for (const table of [
'relay_cell_rehome_safety',
'relay_cell_capabilities',
'relay_cell_connection_snapshots',
'relay_cell_connection_runtime',
'relay_cell_runtime',
'relay_cell_connection_limits',
'relay_cell_admission',
'relay_cell_regions',
'relay_cells'
]) {
await database.query(
`DELETE FROM ${table} WHERE cell_id IN (?, ?)`,
cells.map((cell) => cell.id)
)
}
}
async function setup() {
const database =
process.env.ORCA_REGION_CORRECTION_POSTGRES === '1'
? await openRelayDatabase({
databaseUrl: requiredPostgresUrl(),
dataDir: '/tmp/orca-region-correction-unused'
})
: await openInMemoryRelayDatabase()
opened.push(database)
if (database.dialect === 'postgres') await cleanupPostgres(database)
let clock = 100_000_000
const store = new RelayAssignmentStore(database, () => clock, {
regionalRehomeCohortPercent: 100
})
await store.reconcileCells(cells)
for (const cell of cells) await store.setCellEnabled(cell.id, true)
for (const [index, cell] of cells.entries()) {
await store.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
region: cell.region,
cellIncarnation: incarnations[index]!,
startedAt: clock - 1_000,
ready: true,
observedRequests: 0
})
}
const assignment = await store.assign(identity, undefined, 'us-central1')
const activityId = await store.activateControl(identity, {
cellId: cells[0]!.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 7,
cellIncarnation: incarnations[0],
idleRegionalRehome: true
})
return {
database,
store,
assignment,
activityId,
now: () => clock,
advance: (ms: number) => {
clock += ms
}
}
}
function requiredPostgresUrl(): string {
const url = process.env.ORCA_RELAY_TEST_POSTGRES_URL
if (!url || new URL(url).port !== '55440')
throw new Error('PostgreSQL tests require configured port 55440')
return url
}
async function window(context: Awaited<ReturnType<typeof setup>>) {
const result = await context.store.exchangeRegionCorrection(
identity,
{ v: 1, action: 'issue-window' },
context.assignment.assignmentEpoch
)
return result.window!
}
async function regionalMigration(context: Awaited<ReturnType<typeof setup>>) {
const migration = await context.store.startEvacuation(identity, cells[1]!.id)
const attemptId = '33333333-3333-4333-8333-333333333333'
await context.database.query(
`INSERT INTO relay_region_rehome_attempts
(attempt_id,user_id,relay_host_id,preferred_region,source_cell_id,source_cell_incarnation,
target_cell_id,target_cell_incarnation,previous_epoch,assignment_epoch,drain_grace_ms,send_attempts,created_at,updated_at)
VALUES (?,?,?,'asia-east2',?,?,?,?,?,?,60000,1,?,?)`,
[
attemptId,
identity.userId,
identity.relayHostId,
cells[0]!.id,
incarnations[0],
cells[1]!.id,
incarnations[1],
migration.previousEpoch,
migration.assignmentEpoch,
context.now(),
context.now()
]
)
return { migration }
}
describe('ordered region decisions and migration outcomes', () => {
it('reports aggregate migration lifecycle and reservations without identity disclosure or writes', async () => {
const context = await setup()
const { migration } = await regionalMigration(context)
context.advance(1_000)
const before = await context.database.query('SELECT * FROM relay_region_rehome_attempts')
const outcomes = await context.store.regionCorrectionOutcomes()
expect(outcomes).toEqual([
expect.objectContaining({
sourceCellId: cells[0]!.id,
targetCellId: cells[1]!.id,
state: 'registering',
count: 1,
oldestOpenMs: 1_000
})
])
expect(outcomes[0]!.targetReservedUnits).toBeGreaterThan(0)
expect(JSON.stringify(outcomes)).not.toContain(identity.relayHostId)
expect(JSON.stringify(outcomes)).not.toContain(identity.userId)
expect(await context.database.query('SELECT * FROM relay_region_rehome_attempts')).toEqual(
before
)
await context.store.activateControl(identity, {
cellId: cells[1]!.id,
assignmentEpoch: migration.assignmentEpoch,
generation: 1
})
await context.store.markMigrationTargetRegistered(identity, {
cellId: cells[1]!.id,
assignmentEpoch: migration.assignmentEpoch
})
expect(await context.store.regionCorrectionOutcomes()).toEqual([
expect.objectContaining({ state: 'registered' })
])
await context.store.releaseActivity(identity, context.activityId)
expect(await context.store.completeReadyRegionalRehomes()).toBe(1)
expect(await context.store.regionCorrectionOutcomes()).toEqual([
expect.objectContaining({
state: 'completed',
targetReservedUnits: 0,
oldestOpenMs: 0
})
])
})
it('supersedes prior windows and keeps an inconclusive tombstone immutable', async () => {
const context = await setup()
const first = await window(context)
const second = await window(context)
expect(second.generation).toBe(first.generation + 1)
const report = {
v: 1 as const,
action: 'report' as const,
assignmentEpoch: first.assignmentEpoch,
policyVersion: 1 as const,
outcome: 'conclusive' as const,
measurements: { 'us-central1': 200, 'asia-east2': 40 }
}
expect(
await context.store.exchangeRegionCorrection(
identity,
{ ...report, generation: first.generation },
first.assignmentEpoch
)
).toMatchObject({ reportStatus: 'stale' })
expect(
await context.store.exchangeRegionCorrection(
identity,
{ ...report, generation: second.generation, outcome: 'inconclusive', reason: 'jitter' },
second.assignmentEpoch
)
).toMatchObject({ reportStatus: 'accepted' })
expect(
await context.store.exchangeRegionCorrection(
identity,
{ ...report, generation: second.generation },
second.assignmentEpoch
)
).toMatchObject({ reportStatus: 'duplicate' })
expect(await context.store.previewRegionCorrection()).toEqual({ ineligible: 1 })
})
it('previews the uncapped fleet without writes, claims, or locked reads', async () => {
const context = await setup()
const query = context.database.query.bind(context.database)
const transaction = context.database.transaction.bind(context.database)
const queryLocked = context.database.queryLocked.bind(context.database)
context.database.query = async (sql, params) => {
expect(sql.trim()).toMatch(/^(SELECT|WITH)/i)
return query(sql, params)
}
context.database.transaction = async () => {
throw new Error('preview_must_not_open_mutating_transaction')
}
context.database.queryLocked = async () => {
throw new Error('preview_must_not_lock')
}
try {
const preview = await context.store.previewRegionalRehomeEligibility()
expect(preview.counts['no-verified-decision']).toBeGreaterThanOrEqual(1)
expect(preview.globalSafetyFailure).toBe('process-safety-unavailable')
expect(JSON.stringify(preview)).not.toContain(identity.relayHostId)
expect(JSON.stringify(preview)).not.toContain(identity.userId)
} finally {
context.database.query = query
context.database.transaction = transaction
context.database.queryLocked = queryLocked
}
})
it('allocates distinct ordered generations for concurrent window issuers', async () => {
const context = await setup()
const replies = await Promise.all([window(context), window(context), window(context)])
expect(replies.map((reply) => reply.generation).sort((a, b) => a - b)).toEqual([1, 2, 3])
const older = replies.find((reply) => reply.generation === 2)!
expect(
await context.store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: older.generation,
assignmentEpoch: older.assignmentEpoch,
policyVersion: 1,
outcome: 'inconclusive',
reason: 'delayed'
},
older.assignmentEpoch
)
).toMatchObject({ reportStatus: 'stale' })
})
it('compares with assigned region, preserves hints, and never extends a window on report', async () => {
const context = await setup()
await context.store.assign(identity, 'asia-east2')
const issued = await window(context)
expect(issued.incumbentRegion).toBe('us-central1')
context.advance(50)
await context.store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: issued.generation,
assignmentEpoch: issued.assignmentEpoch,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 110, 'asia-east2': 90 }
},
issued.assignmentEpoch
)
expect(await context.store.previewRegionCorrection()).toEqual({ ineligible: 1 })
const row = (await context.database.query(`SELECT * FROM relay_region_decisions`))[0]!
expect(Number(row.expires_at)).toBe(issued.expiresAt)
const hint = (
await context.database.query(
`SELECT preferred_region FROM relay_assignment_region_preferences WHERE user_id = ?`,
[identity.userId]
)
)[0]
expect(hint?.preferred_region).toBe('asia-east2')
context.advance(24 * 60 * 60_000)
expect(
await context.store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: issued.generation,
assignmentEpoch: issued.assignmentEpoch,
policyVersion: 1,
outcome: 'inconclusive',
reason: 'late'
},
issued.assignmentEpoch
)
).toMatchObject({ reportStatus: 'expired' })
})
it('rejects stale assignment basis and requires both thresholds', async () => {
const context = await setup()
const issued = await window(context)
await context.store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: issued.generation,
assignmentEpoch: issued.assignmentEpoch,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 150, 'asia-east2': 100 }
},
issued.assignmentEpoch
)
expect(await context.store.previewRegionCorrection()).toEqual({
'us-central1-to-asia-east2': 1
})
await context.store.startEvacuation(identity, cells[1]!.id)
expect(
await context.store.exchangeRegionCorrection(
identity,
{ v: 1, action: 'issue-window' },
issued.assignmentEpoch
)
).toMatchObject({ reportStatus: 'basis-changed' })
})
})
@@ -5,7 +5,9 @@ vi.mock('./admin-token-verifier.js', () => ({
createAdminTokenVerifier: () => async (token: string, route?: string) =>
token === 'deploy-token' ||
(token === 'monitor-token' &&
(!route || route === '/v1/admin/regional-rehome-control')),
(!route ||
route === '/v1/admin/regional-rehome-control' ||
route === '/v1/admin/regional-rehome-preview')),
createReadOnlyAdminTokenVerifier: () => async () => false,
createRegionalRehomeControlApplyTokenVerifier: () => async (token: string) =>
token === 'deploy-token',
@@ -41,7 +43,83 @@ const request = {
graceMs: 60_000
}
describe('idle regional cutover endpoint', () => {
it('authenticates and fences the source before invoking a cutover', async () => {
const idleRehome = vi.fn(async () => ({ outcome: 'busy' }))
const app = createRelayApp(config(), {
store: {} as never,
assignments: {} as never,
drain: vi.fn(),
idleRehome,
cellIncarnation,
ready: vi.fn(async () => true)
} as Parameters<typeof createRelayApp>[1])
const input = {
v: 1,
attemptId: request.attemptId,
userId: request.userId,
relayHostId: request.relayHostId,
sourceCellId: request.sourceCellId,
sourceCellIncarnation: cellIncarnation,
sourceAssignmentEpoch: 7,
sourceGeneration: 1,
targetCellId: 'target-cell',
cohortPercent: 100,
directorSafety: {
observedAt: 100, sqlFailures: 0, reconnects: 0, controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0, databasePoolWaitersMax: 0, databasePoolWaitMsMax: 0
}
}
const path = '/v1/admin/host-idle-rehome'
expect((await postPath(app, path, 'runtime-token', input)).status).toBe(401)
expect(
(
await postPath(app, path, 'rehome-token', {
...input,
sourceCellIncarnation: '33333333-3333-4333-8333-333333333333'
})
).status
).toBe(409)
expect(idleRehome).not.toHaveBeenCalled()
const response = await postPath(app, path, 'rehome-token', input)
expect(response.status).toBe(200)
expect(await response.json()).toEqual({ v: 1, outcome: 'busy' })
expect(idleRehome).toHaveBeenCalledExactlyOnceWith(input)
})
})
describe('regional host drain endpoint', () => {
it('exposes aggregate preview to monitors without a mutation path', async () => {
const preview = { counts: { 'eligible:asia-east2-to-us-central1': 2 } }
const safety = { observedAt: 100 }
const previewRegionalRehomeEligibility = vi.fn(async () => preview)
const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
store: {} as never,
assignments: {
previewRegionalRehomeEligibility,
regionCorrectionOutcomes: async () => []
} as never,
regionalRehomeSafetySnapshot: () => safety as never,
drain: vi.fn(),
ready: vi.fn(async () => true)
})
const path = '/v1/admin/regional-rehome-preview'
expect((await app.request(path)).status).toBe(401)
expect(previewRegionalRehomeEligibility).not.toHaveBeenCalled()
const response = await app.request(path, { headers: { authorization: 'Bearer monitor-token' } })
expect(response.status).toBe(200)
expect(await response.json()).toEqual({ v: 1, preview, outcomes: [] })
expect(previewRegionalRehomeEligibility).toHaveBeenCalledExactlyOnceWith(safety)
expect(
(
await app.request(path, {
method: 'POST',
headers: { authorization: 'Bearer deploy-token' }
})
).status
).toBe(404)
})
it('accepts only the dedicated identity and exact cell generation', async () => {
const drainHost = vi.fn(() => 'accepted' as const)
const app = createRelayApp(config(), {
@@ -60,14 +138,66 @@ describe('regional host drain endpoint', () => {
expect((await post(app, 'deploy-token', request)).status).toBe(401)
expect(
(await post(app, 'rehome-token', {
...request,
sourceCellIncarnation: '33333333-3333-4333-8333-333333333333'
})).status
(
await post(app, 'rehome-token', {
...request,
sourceCellIncarnation: '33333333-3333-4333-8333-333333333333'
})
).status
).toBe(409)
expect(drainHost).toHaveBeenCalledOnce()
})
it('waits for an asynchronous drain operation before acknowledging', async () => {
let grant!: (value: 'accepted') => void
let entered!: () => void
const started = new Promise<void>((resolve) => {
entered = resolve
})
const drainHost = vi.fn(() => {
entered()
return new Promise<'accepted'>((resolve) => {
grant = resolve
})
})
const app = createRelayApp(config(), {
store: {} as never,
assignments: {} as never,
drain: vi.fn(),
drainHost,
cellIncarnation,
ready: vi.fn(async () => true)
})
const pending = post(app, 'rehome-token', request)
let acknowledged = false
void pending.then(() => {
acknowledged = true
})
await started
await Promise.resolve()
expect(acknowledged).toBe(false)
grant('accepted')
const response = await pending
expect(response.status).toBe(200)
expect(await response.json()).toEqual({ v: 1, outcome: 'accepted' })
})
it('rejects a failed asynchronous drain instead of acknowledging it', async () => {
const app = createRelayApp(config(), {
store: {} as never,
assignments: {} as never,
drain: vi.fn(),
drainHost: async () => {
throw new Error('activity_cell_not_authoritative')
},
cellIncarnation,
ready: vi.fn(async () => true)
})
const response = await post(app, 'rehome-token', request)
expect(response.status).toBe(409)
expect(await response.json()).toEqual({ error: 'activity_cell_not_authoritative' })
})
it('rejects malformed identities before touching the session registry', async () => {
const drainHost = vi.fn(() => 'accepted' as const)
const app = createRelayApp(config(), {
@@ -224,7 +354,7 @@ describe('regional rehome director controls', () => {
v: 1,
cellId: 'production-gce-c7',
cellIncarnation,
regionalRehomeProtocol: 1,
regionalRehomeProtocol: 2,
safety: {
observedAt: 100,
sqlFailures: 0,
@@ -235,12 +365,7 @@ describe('regional rehome director controls', () => {
databasePoolWaitMsMax: 0
}
}
const response = await postPath(
app,
'/v1/admin/cell-rehome-status',
'runtime-token',
body
)
const response = await postPath(app, '/v1/admin/cell-rehome-status', 'runtime-token', body)
expect(response.status).toBe(200)
expect(recordCellRegionalRehomeStatus).toHaveBeenCalledWith(body)
@@ -266,18 +391,13 @@ describe('regional rehome director controls', () => {
v: 1,
cellId: 'production-gce-c7',
cellIncarnation,
regionalRehomeProtocol: 1,
regionalRehomeProtocol: 2,
safety: {
...observability.regionalRehomeRuntimeSafety(),
...emptyPostgresPoolPressureCounts()
}
}
const response = await postPath(
app,
'/v1/admin/cell-rehome-status',
'runtime-token',
body
)
const response = await postPath(app, '/v1/admin/cell-rehome-status', 'runtime-token', body)
expect(response.status).toBe(200)
expect(recordCellRegionalRehomeStatus).toHaveBeenCalledWith(body)
@@ -301,12 +421,14 @@ describe('regional rehome director controls', () => {
drain: vi.fn(),
ready: vi.fn(async () => true)
})
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'deploy-token',
{ v: 1, action: 'inspect' }
)).status).toBe(200)
expect(
(
await postPath(app, '/v1/admin/regional-rehome-control', 'deploy-token', {
v: 1,
action: 'inspect'
})
).status
).toBe(200)
const apply = {
v: 1,
action: 'apply',
@@ -319,39 +441,35 @@ describe('regional rehome director controls', () => {
drainGraceMs: 60_000,
confirmation: 'ENABLE_REGIONAL_REHOMING'
}
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'deploy-token',
apply
)).status).toBe(200)
expect(
(await postPath(app, '/v1/admin/regional-rehome-control', 'deploy-token', apply)).status
).toBe(200)
expect(applyRegionalRehomeControl).toHaveBeenCalledOnce()
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'monitor-token',
{ v: 1, action: 'inspect' }
)).status).toBe(200)
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'monitor-token',
apply
)).status).toBe(403)
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'deploy-token',
{ ...apply, confirmation: 'DISABLE_REGIONAL_REHOMING' }
)).status).toBe(400)
expect(
(
await postPath(app, '/v1/admin/regional-rehome-control', 'monitor-token', {
v: 1,
action: 'inspect'
})
).status
).toBe(200)
expect(
(await postPath(app, '/v1/admin/regional-rehome-control', 'monitor-token', apply)).status
).toBe(403)
expect(
(
await postPath(app, '/v1/admin/regional-rehome-control', 'deploy-token', {
...apply,
confirmation: 'DISABLE_REGIONAL_REHOMING'
})
).status
).toBe(400)
// The per-host cooldown is part of the durable shape an operator must state.
const { hostCooldownMs: _omitted, ...withoutCooldown } = apply
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'deploy-token',
withoutCooldown
)).status).toBe(400)
expect(
(await postPath(app, '/v1/admin/regional-rehome-control', 'deploy-token', withoutCooldown))
.status
).toBe(400)
})
it('probes dedicated trust twice and returns only aggregate proof', async () => {
@@ -382,12 +500,11 @@ describe('regional rehome director controls', () => {
}) as typeof fetch,
ready: vi.fn(async () => true)
})
const response = await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ v: 1, sourceCellId: 'production-gce-c7', sourceCellIncarnation: cellIncarnation }
)
const response = await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', {
v: 1,
sourceCellId: 'production-gce-c7',
sourceCellIncarnation: cellIncarnation
})
expect(response.status).toBe(200)
const responseBody = await response.json()
@@ -448,12 +565,11 @@ describe('regional rehome director controls', () => {
ready: vi.fn(async () => true)
})
const response = await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ v: 1, sourceCellId: 'production-gce-c27', sourceCellIncarnation: cellIncarnation }
)
const response = await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', {
v: 1,
sourceCellId: 'production-gce-c27',
sourceCellIncarnation: cellIncarnation
})
expect(response.status).toBe(200)
expect(await response.json()).toMatchObject({ proven: true })
@@ -481,12 +597,11 @@ describe('regional rehome director controls', () => {
ready: vi.fn(async () => true)
})
const response = await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ v: 1, sourceCellId: 'production-gce-c27', sourceCellIncarnation: cellIncarnation }
)
const response = await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', {
v: 1,
sourceCellId: 'production-gce-c27',
sourceCellIncarnation: cellIncarnation
})
expect(response.status).toBe(409)
expect(sourceFetch).not.toHaveBeenCalled()
@@ -504,18 +619,17 @@ describe('regional rehome director controls', () => {
sourceCellId: 'production-gce-c7',
sourceCellIncarnation: cellIncarnation
}
expect((await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'monitor-token',
body
)).status).toBe(401)
expect((await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ ...body, unexpected: true }
)).status).toBe(400)
expect(
(await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'monitor-token', body)).status
).toBe(401)
expect(
(
await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', {
...body,
unexpected: true
})
).status
).toBe(400)
})
it('fails closed when the source rejects the dedicated identity', async () => {
@@ -529,9 +643,9 @@ describe('regional rehome director controls', () => {
regionalRehomeProtocol: 1
}
})
const sourceFetch = vi.fn<typeof fetch>().mockResolvedValue(
Response.json({ error: 'invalid_token' }, { status: 401 })
)
const sourceFetch = vi
.fn<typeof fetch>()
.mockResolvedValue(Response.json({ error: 'invalid_token' }, { status: 401 }))
const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
store: {} as never,
assignments: { cellDeploymentStatus } as never,
@@ -540,12 +654,11 @@ describe('regional rehome director controls', () => {
regionalRehomeFetch: sourceFetch,
ready: vi.fn(async () => true)
})
const response = await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ v: 1, sourceCellId: 'production-gce-c7', sourceCellIncarnation: cellIncarnation }
)
const response = await postPath(app, '/v1/admin/regional-rehome-trust-probe', 'deploy-token', {
v: 1,
sourceCellId: 'production-gce-c7',
sourceCellIncarnation: cellIncarnation
})
expect(response.status).toBe(409)
expect(sourceFetch).toHaveBeenCalledOnce()
@@ -29,6 +29,9 @@ describePostgres('PostgreSQL regional rehoming', () => {
})
async function cleanup(): Promise<void> {
for (const table of ['relay_region_decisions', 'relay_control_capabilities']) {
await primary.query(`DELETE FROM ${table} WHERE user_id LIKE 'pg-rehome-user-%'`)
}
await primary.query(
`DELETE FROM relay_region_rehome_attempts WHERE user_id LIKE 'pg-rehome-user-%'`
)
@@ -69,6 +72,81 @@ describePostgres('PostgreSQL regional rehoming', () => {
}
}
it('defaults to a closed correction cohort even with enabled durable control', async () => {
const context = await fixture()
const closed = new RelayAssignmentStore(primary, context.now, {
requireLiveCells: true,
heartbeatTtlMs: 45_000
})
expect(await cutover(closed, context.now())).toBeNull()
const preview = await closed.previewRegionalRehomeEligibility({
observedAt: context.now(),
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
})
expect(preview.cohortPercent).toBe(0)
expect(preview.counts['outside-cohort']).toBe(1)
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('counts existing generic migrations against the optimization cap and preview', async () => {
const context = await fixture()
const safety = {
observedAt: context.now(),
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
}
const before = await context.store.previewRegionalRehomeEligibility(safety)
expect(before.counts['eligible:us-central1-to-asia-east2']).toBe(1)
for (let index = 0; index < 8; index++) {
const identity = {
userId: `pg-rehome-user-budget-${sequence}-${index}`,
relayHostId: `budgethost${String(index).padStart(6, '0')}`
}
await context.store.assign(identity, undefined, 'us-central1')
await context.store.startEvacuation(identity, context.target.id)
}
const preview = await context.store.previewRegionalRehomeEligibility(safety)
expect(preview.openMigrations).toBe(8)
expect(preview.availableMigrationSlots).toBe(0)
expect(preview.counts['concurrent-migration-cap']).toBe(1)
expect(await cutover(context.store, context.now())).toBeNull()
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('preview excludes request capacity exhaustion before a claim', async () => {
const context = await fixture()
await primary.query(
`UPDATE relay_cells SET capacity_requests = reserved_requests + 1 WHERE cell_id = ?`,
[context.target.id]
)
const preview = await context.store.previewRegionalRehomeEligibility({
observedAt: context.now(),
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
})
expect(preview.counts['no-target-headroom']).toBe(1)
expect(await cutover(context.store, context.now())).toBeNull()
})
it('claims through ambient per-cell sql retry noise', async () => {
const context = await fixture()
await primary.query(
@@ -78,27 +156,31 @@ describePostgres('PostgreSQL regional rehoming', () => {
[context.source.id, context.target.id]
)
expect(await context.store.claimRegionalRehome()).not.toBeNull()
expect(await cutover(context.store, context.now())).not.toBeNull()
})
it('moves a us-central1 host onto a cell in its preferred asia-east2 region', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
const attempt = await cutover(context.store, context.now())
expect(attempt).toMatchObject({
preferredRegion: 'asia-east2',
sourceCellId: context.source.id,
targetCellId: context.target.id
})
expect(await primary.query(
`SELECT preferred_region, source_cell_id, target_cell_id
expect(
await primary.query(
`SELECT preferred_region, source_cell_id, target_cell_id
FROM relay_region_rehome_attempts WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{
preferred_region: 'asia-east2',
source_cell_id: context.source.id,
target_cell_id: context.target.id
}])
[context.identity.userId]
)
).toEqual([
{
preferred_region: 'asia-east2',
source_cell_id: context.source.id,
target_cell_id: context.target.id
}
])
})
it('moves an asia-east2 host back onto a cell in its preferred us-central1 region', async () => {
@@ -107,32 +189,37 @@ describePostgres('PostgreSQL regional rehoming', () => {
targetRegion: 'us-central1'
})
const attempt = await context.store.claimRegionalRehome()
const attempt = await cutover(context.store, context.now())
expect(attempt).toMatchObject({
preferredRegion: 'us-central1',
sourceCellId: context.source.id,
targetCellId: context.target.id
})
// The durable attempt row must accept the reverse direction too.
expect(await primary.query(
`SELECT preferred_region, source_cell_id, target_cell_id
expect(
await primary.query(
`SELECT preferred_region, source_cell_id, target_cell_id
FROM relay_region_rehome_attempts WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{
preferred_region: 'us-central1',
source_cell_id: context.source.id,
target_cell_id: context.target.id
}])
expect(await primary.query(
`SELECT cell_id FROM relay_assignments WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ cell_id: context.target.id }])
[context.identity.userId]
)
).toEqual([
{
preferred_region: 'us-central1',
source_cell_id: context.source.id,
target_cell_id: context.target.id
}
])
expect(
await primary.query(`SELECT cell_id FROM relay_assignments WHERE user_id = ?`, [
context.identity.userId
])
).toEqual([{ cell_id: context.target.id }])
})
it('leaves a host whose preference already matches its own region', async () => {
const context = await fixture({ preferredRegion: 'us-central1' })
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(cutover(context.store, context.now())).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true
@@ -146,16 +233,12 @@ describePostgres('PostgreSQL regional rehoming', () => {
it('leaves a host whose preference is older than the configured max age', async () => {
const context = await fixture()
await primary.query(
`UPDATE relay_assignment_region_preferences SET observed_at = ?
`UPDATE relay_region_decisions SET observed_at = ?
WHERE user_id = ? AND relay_host_id = ?`,
[
context.now() - 24 * 60 * 60_000 - 1,
context.identity.userId,
context.identity.relayHostId
]
[context.now() - 24 * 60 * 60_000 - 1, context.identity.userId, context.identity.relayHostId]
)
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(cutover(context.store, context.now())).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true
@@ -190,7 +273,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
]
)
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(cutover(context.store, context.now())).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true,
@@ -206,7 +289,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
`UPDATE relay_region_rehome_attempts SET created_at = ? WHERE user_id = ?`,
[context.now() - 3 * 24 * 60 * 60_000, context.identity.userId]
)
await expect(context.store.claimRegionalRehome()).resolves.toMatchObject({
await expect(cutover(context.store, context.now())).resolves.toMatchObject({
sourceCellId: context.source.id,
targetCellId: context.target.id
})
@@ -217,7 +300,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
// where no later rehome could move it out again.
const context = await fixture({ targetProtocol: 0 })
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(cutover(context.store, context.now())).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true
@@ -237,119 +320,39 @@ describePostgres('PostgreSQL regional rehoming', () => {
[context.target.id]
)
expect(await context.store.claimRegionalRehome()).toBeNull()
expect(await cutover(context.store, context.now())).toBeNull()
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
generation: 1,
enabled: true
})
expect(await primary.query(
`SELECT next_dispatch_at FROM relay_region_rehome_worker_state`
)).toEqual([{ next_dispatch_at: String(context.now() + 6_000) }])
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('lets only one director claim a host', async () => {
const context = await fixture()
const claims = await Promise.all([
context.store.claimRegionalRehome(),
context.competingStore.claimRegionalRehome()
cutover(context.store, context.now()),
cutover(context.competingStore, context.now())
])
expect(claims.filter(Boolean)).toHaveLength(1)
expect(await primary.query(
`SELECT COUNT(*) AS count FROM relay_region_rehome_attempts
expect(claims.filter(Boolean).length).toBeGreaterThanOrEqual(1)
expect(
await primary.query(
`SELECT COUNT(*) AS count FROM relay_region_rehome_attempts
WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ count: '1' }])
expect(await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations
[context.identity.userId]
)
).toEqual([{ count: '1' }])
expect(
await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations
WHERE user_id = ? AND completed_at IS NULL AND aborted_at IS NULL`,
[context.identity.userId]
)).toEqual([{ count: '1' }])
})
it('serializes an enable with a budget-exhausting failure without retries', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
await context.store.recordRegionalRehomeDispatchFailure(attempt!.attemptId)
await context.store.recordRegionalRehomeDispatchFailure(attempt!.attemptId)
const locked = Promise.withResolvers<void>()
const release = Promise.withResolvers<void>()
const primaryTransaction = primary.transaction.bind(primary)
const secondaryTransaction = secondary.transaction.bind(secondary)
let enableTransactions = 0
let failureTransactions = 0
let enablePid = 0
let failurePid = 0
const enableSpy = vi.spyOn(primary, 'transaction').mockImplementation((operation, options) =>
primaryTransaction(async (transaction) => {
enableTransactions++
enablePid = Number((await transaction.query('SELECT pg_backend_pid() AS pid'))[0]!.pid)
return await operation({
dialect: 'postgres',
query: transaction.query.bind(transaction),
queryLocked: async (sql, params, lockOptions) => {
const rows = await transaction.queryLocked(sql, params, lockOptions)
if (sql.includes('FROM relay_region_rehome_control')) {
locked.resolve()
await release.promise
}
return rows
},
transaction: transaction.transaction.bind(transaction),
close: transaction.close.bind(transaction)
})
}, options)
)
const failureSpy = vi.spyOn(secondary, 'transaction').mockImplementation((operation, options) =>
secondaryTransaction(async (transaction) => {
failureTransactions++
failurePid = Number((await transaction.query('SELECT pg_backend_pid() AS pid'))[0]!.pid)
return await operation(transaction)
}, options)
)
const enable = context.store.applyRegionalRehomeControl({
expectedGeneration: 1,
enabled: true,
notBefore: context.now(),
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: 7 * 24 * 60 * 60_000,
drainGraceMs: 60_000
})
let failure: Promise<void> | undefined
let outcomes: PromiseSettledResult<unknown>[] = []
try {
await Promise.race([
locked.promise,
enable.then(() => {
throw new Error('enable completed before the control lock')
})
])
failure = context.competingStore.recordRegionalRehomeDispatchFailure(attempt!.attemptId)
// Observe the actual PostgreSQL wait before letting enable acquire the worker row.
await vi.waitFor(async () => {
expect(failurePid).not.toBe(0)
const rows = await primary.query('SELECT pg_blocking_pids(?) AS blockers', [failurePid])
expect(rows[0]!.blockers).toContain(enablePid)
}, { interval: 10, timeout: 800 })
} finally {
release.resolve()
outcomes = await Promise.allSettled([enable, ...(failure ? [failure] : [])])
enableSpy.mockRestore()
failureSpy.mockRestore()
}
expect(outcomes.map((outcome) => outcome.status)).toEqual(['fulfilled', 'fulfilled'])
expect({ enableTransactions, failureTransactions }).toEqual({
enableTransactions: 1,
failureTransactions: 1
})
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
generation: 2,
enabled: true
})
expect(await primary.query(
`SELECT consecutive_failures, paused_until FROM relay_region_rehome_worker_state`
)).toEqual([{ consecutive_failures: '1', paused_until: '0' }])
[context.identity.userId]
)
).toEqual([{ count: '1' }])
})
it('increments the disable generation once across competing directors', async () => {
@@ -366,24 +369,6 @@ describePostgres('PostgreSQL regional rehoming', () => {
})
})
it('records one receipt across competing directors', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
const receipts = await Promise.all([
context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted'),
context.competingStore.recordRegionalRehomeDrainReceipt(
attempt!.attemptId,
'accepted'
)
])
expect(receipts.sort()).toEqual([false, true])
expect(await primary.query(
`SELECT drain_outcome FROM relay_region_rehome_attempts WHERE attempt_id = ?`,
[attempt!.attemptId]
)).toEqual([{ drain_outcome: 'accepted' }])
})
it('rechecks a preference changed while the assignment row is locked', async () => {
const context = await fixture()
let unlock!: () => void
@@ -399,9 +384,9 @@ describePostgres('PostgreSQL regional rehoming', () => {
await unlockPromise
})
await lockedPromise
const claim = context.store.claimRegionalRehome()
const claim = cutover(context.store, context.now())
await primary.query(
`UPDATE relay_assignment_region_preferences SET preferred_region = 'us-central1',
`UPDATE relay_region_decisions SET preferred_region = 'us-central1',
observed_at = ? WHERE user_id = ? AND relay_host_id = ?`,
[context.now(), context.identity.userId, context.identity.relayHostId]
)
@@ -409,23 +394,26 @@ describePostgres('PostgreSQL regional rehoming', () => {
await held
await expect(claim).resolves.toBeNull()
expect(await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ count: '0' }])
expect(
await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ count: '0' }])
})
it('rechecks fleet safety under locks before mutating a candidate', async () => {
const context = await fixture()
const [request] = await context.store.selectIdleRegionalRehomeCandidates(safety(context.now()))
expect(request).toBeDefined()
let unlock!: () => void
let locked!: () => void
const lockedPromise = new Promise<void>((resolve) => (locked = resolve))
const unlockPromise = new Promise<void>((resolve) => (unlock = resolve))
const held = secondary.transaction(async (transaction) => {
await transaction.queryLocked(
`SELECT * FROM relay_cell_rehome_safety WHERE cell_id = ?`,
[context.target.id]
)
await transaction.queryLocked(`SELECT * FROM relay_cell_rehome_safety WHERE cell_id = ?`, [
context.target.id
])
await transaction.query(
`UPDATE relay_cell_rehome_safety SET sql_failures = ${REGIONAL_REHOME_SQL_FAILURES_LIMIT + 1} WHERE cell_id = ?`,
[context.target.id]
@@ -434,62 +422,50 @@ describePostgres('PostgreSQL regional rehoming', () => {
await unlockPromise
})
await lockedPromise
const claim = context.store.claimRegionalRehome()
const claim = context.store.commitIdleRegionalRehome(request!, safety(context.now()))
unlock()
await held
await expect(claim).resolves.toBeNull()
await expect(claim).resolves.toEqual({ outcome: 'deferred' })
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
generation: 2,
enabled: false
})
expect(await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ count: '0' }])
expect(
await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ count: '0' }])
})
it('pauses when one required cell exceeds the reconnect limit', async () => {
const context = await fixture()
await primary.query(
`UPDATE relay_cell_rehome_safety SET reconnects = ? WHERE cell_id = ?`,
[REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT + 1, context.source.id]
)
const [request] = await context.store.selectIdleRegionalRehomeCandidates(safety(context.now()))
expect(request).toBeDefined()
await primary.query(`UPDATE relay_cell_rehome_safety SET reconnects = ? WHERE cell_id = ?`, [
REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT + 1,
context.source.id
])
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(
context.store.commitIdleRegionalRehome(request!, safety(context.now()))
).resolves.toEqual({ outcome: 'deferred' })
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 2,
enabled: false
})
expect(await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ count: '0' }])
})
it('does not retry a drain against a replacement source incarnation', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
context.advance(31_000)
await heartbeat(
context.store,
context.source,
'33333333-3333-4333-8333-333333333333',
1,
context.now()
)
await expect(context.competingStore.claimRegionalRehome()).resolves.toBeNull()
expect(await primary.query(
`SELECT send_attempts FROM relay_region_rehome_attempts WHERE attempt_id = ?`,
[attempt!.attemptId]
)).toEqual([{ send_attempts: '1' }])
expect(
await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ count: '0' }])
})
it('makes concurrent completion and expiry cleanup idempotent', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted')
const attempt = await cutover(context.store, context.now())
const targetControl = await context.store.activateControl(context.identity, {
cellId: context.target.id,
assignmentEpoch: attempt!.assignmentEpoch,
@@ -528,17 +504,18 @@ describePostgres('PostgreSQL regional rehoming', () => {
context.competingStore.abortExpiredRegionalRehomes()
])
expect(outcomes).toEqual(expect.arrayContaining([0, 1]))
expect(await primary.query(
`SELECT completed_at IS NOT NULL AS completed, aborted_at IS NOT NULL AS aborted
expect(
await primary.query(
`SELECT completed_at IS NOT NULL AS completed, aborted_at IS NOT NULL AS aborted
FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ completed: true, aborted: false }])
[context.identity.userId]
)
).toEqual([{ completed: true, aborted: false }])
})
it('will not complete against a replacement target incarnation', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted')
const attempt = await cutover(context.store, context.now())
await context.store.activateControl(context.identity, {
cellId: context.target.id,
assignmentEpoch: attempt!.assignmentEpoch,
@@ -559,15 +536,17 @@ describePostgres('PostgreSQL regional rehoming', () => {
)
await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(0)
expect(await primary.query(
`SELECT completed_at, aborted_at FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ completed_at: null, aborted_at: null }])
expect(
await primary.query(
`SELECT completed_at, aborted_at FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ completed_at: null, aborted_at: null }])
})
it('does not roll an unregistered target back to a stale regional source', async () => {
const context = await fixture()
await context.store.claimRegionalRehome()
await cutover(context.store, context.now())
context.advance(6 * 60_000)
await heartbeat(
context.store,
@@ -580,16 +559,17 @@ describePostgres('PostgreSQL regional rehoming', () => {
await expect(context.store.refreshRegionalRehomeLeases()).resolves.toBe(0)
await expect(context.store.abortExpiredEvacuations()).resolves.toBe(0)
expect(await primary.query(
`SELECT cell_id, assignment_epoch FROM relay_assignments WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ cell_id: context.target.id, assignment_epoch: '2' }])
expect(
await primary.query(
`SELECT cell_id, assignment_epoch FROM relay_assignments WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ cell_id: context.target.id, assignment_epoch: '2' }])
})
it('completes after the drained host re-resolves through the director', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted')
const attempt = await cutover(context.store, context.now())
// The drain recovery lands while both controls are still live.
await context.store.assign(context.identity, 'asia-east2')
expect(await controlAccounting(context.identity)).toEqual({
@@ -609,11 +589,13 @@ describePostgres('PostgreSQL regional rehoming', () => {
await context.store.releaseActivity(context.identity, context.sourceControl)
await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(1)
expect(await primary.query(
`SELECT completed_at IS NOT NULL AS completed FROM relay_assignment_migrations
expect(
await primary.query(
`SELECT completed_at IS NOT NULL AS completed FROM relay_assignment_migrations
WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ completed: true }])
[context.identity.userId]
)
).toEqual([{ completed: true }])
expect(await controlAccounting(context.identity)).toEqual({
reservedControls: 1,
controlLeases: 1
@@ -622,7 +604,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
it('repairs a skewed control counter before completing the rehome', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
const attempt = await cutover(context.store, context.now())
await context.store.activateControl(context.identity, {
cellId: context.target.id,
assignmentEpoch: attempt!.assignmentEpoch,
@@ -634,10 +616,9 @@ describePostgres('PostgreSQL regional rehoming', () => {
})
await context.store.releaseActivity(context.identity, context.sourceControl)
// Damage already written by a pre-fix sticky grant.
await primary.query(
`UPDATE relay_assignments SET reserved_controls = 0 WHERE user_id = ?`,
[context.identity.userId]
)
await primary.query(`UPDATE relay_assignments SET reserved_controls = 0 WHERE user_id = ?`, [
context.identity.userId
])
await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(1)
expect(await controlAccounting(context.identity)).toEqual({
@@ -646,6 +627,22 @@ describePostgres('PostgreSQL regional rehoming', () => {
})
})
async function cutover(store: RelayAssignmentStore, now: number) {
const [request] = await store.selectIdleRegionalRehomeCandidates(safety(now))
if (!request) return null
const result = await store.commitIdleRegionalRehome(request, safety(now))
if (result.outcome !== 'committed') return null
const [attempt] = await primary.query(
`SELECT preferred_region, assignment_epoch FROM relay_region_rehome_attempts WHERE attempt_id = ?`,
[request.attemptId]
)
return {
...request,
preferredRegion: String(attempt!.preferred_region),
assignmentEpoch: Number(attempt!.assignment_epoch)
}
}
async function attemptAndMigrationCounts(identity: {
userId: string
relayHostId: string
@@ -711,18 +708,12 @@ describePostgres('PostgreSQL regional rehoming', () => {
drainGraceMs: 60_000
})
await store.reconcileCells([source, target])
await heartbeat(
store,
source,
'11111111-1111-4111-8111-111111111111',
1,
900_000
)
await heartbeat(store, source, '11111111-1111-4111-8111-111111111111', 3, 900_000)
await heartbeat(
store,
target,
'22222222-2222-4222-8222-222222222222',
options.targetProtocol ?? 1,
options.targetProtocol ?? 3,
900_000
)
const identity = {
@@ -733,9 +724,32 @@ describePostgres('PostgreSQL regional rehoming', () => {
const sourceControl = await store.activateControl(identity, {
cellId: source.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 1
generation: 1,
idleRegionalRehome: true,
cellIncarnation: '11111111-1111-4111-8111-111111111111'
})
await store.assign(identity, preferredRegion)
const issued = await store.exchangeRegionCorrection(
identity,
{ v: 1, action: 'issue-window' },
assignment.assignmentEpoch
)
await store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: issued.window!.generation,
assignmentEpoch: assignment.assignmentEpoch,
policyVersion: 1,
outcome: 'conclusive',
measurements: {
'us-central1': preferredRegion === 'us-central1' ? 50 : 150,
'asia-east2': preferredRegion === 'asia-east2' ? 50 : 150
}
},
assignment.assignmentEpoch
)
return {
preferredRegion,
store,
@@ -753,6 +767,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
})
const storeOptions = {
regionalRehomeCohortPercent: 100,
requireLiveCells: true,
heartbeatTtlMs: 45_000
}
@@ -816,3 +831,15 @@ async function heartbeat(
}
})
}
function safety(now: number) {
return {
observedAt: now,
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
}
}
File diff suppressed because it is too large Load Diff
@@ -25,6 +25,7 @@ async function setup() {
let clock = 1_000_000
const database = await openInMemoryRelayDatabase()
const store = new RelayAssignmentStore(database, () => clock, {
regionalRehomeCohortPercent: 100,
requireLiveCells: true,
heartbeatTtlMs: 45_000
})
@@ -83,11 +84,40 @@ async function setup() {
await store.activateControl(identity, {
cellId: source.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 1
generation: 1,
idleRegionalRehome: true,
cellIncarnation: incarnation(1)
})
await store.assign(identity, 'asia-east2')
const { window } = await store.exchangeRegionCorrection(
identity,
{ v: 1, action: 'issue-window' },
assignment.assignmentEpoch
)
expect(window).toBeDefined()
await store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: window!.generation,
assignmentEpoch: assignment.assignmentEpoch,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 180, 'asia-east2': 40 }
},
assignment.assignmentEpoch
)
}
return { database, store, beat, activatePreferredSource }
const safety = () => ({
observedAt: clock,
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
})
return { database, store, beat, activatePreferredSource, safety }
}
const UNCLEAN = REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT + 1
@@ -95,70 +125,78 @@ const UNCLEAN = REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT + 1
describe('regional rehome target selection', () => {
it('never selects a target without connection headroom, even at lowest load', async () => {
const context = await setup()
await context.beat(source, 1, 1, {
await context.beat(source, 1, 3, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: 0
})
// Lowest load but the connection hard cap is exhausted.
await context.beat(noHeadroom, 2, 1, {
await context.beat(noHeadroom, 2, 3, {
observedRequests: 0,
enforcedConnections: 999,
sqlFailures: 0
})
await context.beat(unclean, 3, 1, {
await context.beat(unclean, 3, 3, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: UNCLEAN
})
await context.beat(highLoad, 4, 1, {
await context.beat(highLoad, 4, 3, {
observedRequests: 50,
enforcedConnections: 0,
sqlFailures: 0
})
await context.beat(lowLoad, 5, 1, {
await context.beat(lowLoad, 5, 3, {
observedRequests: 10,
enforcedConnections: 0,
sqlFailures: 0
})
await context.activatePreferredSource()
const attempt = await context.store.claimRegionalRehome()
const candidates = await context.store.selectIdleRegionalRehomeCandidates(context.safety())
const attempt = candidates[0]
expect(attempt?.targetCellId).toBe(lowLoad.id)
expect(await context.store.commitIdleRegionalRehome(attempt!, context.safety(), 100)).toEqual({
outcome: 'committed'
})
await context.database.close()
})
it('falls to the next clean target when the load winner goes unclean', async () => {
const context = await setup()
await context.beat(source, 1, 1, {
await context.beat(source, 1, 3, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: 0
})
await context.beat(noHeadroom, 2, 1, {
await context.beat(noHeadroom, 2, 3, {
observedRequests: 0,
enforcedConnections: 999,
sqlFailures: 0
})
await context.beat(unclean, 3, 1, {
await context.beat(unclean, 3, 3, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: UNCLEAN
})
await context.beat(highLoad, 4, 1, {
await context.beat(highLoad, 4, 3, {
observedRequests: 50,
enforcedConnections: 0,
sqlFailures: 0
})
await context.beat(lowLoad, 5, 1, {
await context.beat(lowLoad, 5, 3, {
observedRequests: 10,
enforcedConnections: 0,
sqlFailures: UNCLEAN
})
await context.activatePreferredSource()
const attempt = await context.store.claimRegionalRehome()
const candidates = await context.store.selectIdleRegionalRehomeCandidates(context.safety())
const attempt = candidates[0]
expect(attempt?.targetCellId).toBe(highLoad.id)
expect(await context.store.commitIdleRegionalRehome(attempt!, context.safety(), 100)).toEqual({
outcome: 'committed'
})
await context.database.close()
})
})
@@ -11,148 +11,37 @@ import { startRegionalRehomeWorker } from './regional-rehome-worker.js'
describe('regional rehome worker', () => {
afterEach(() => vi.restoreAllMocks())
it('sends an incarnation- and source-epoch-bound drain without exposing identity', async () => {
let now = 0
const attempt = {
attemptId: '11111111-1111-4111-8111-111111111111',
userId: 'private-user',
relayHostId: 'abcdefghijklmnop',
preferredRegion: 'asia-east2',
sourceCellId: 'production-gce-c7',
sourceCellUrl: 'https://c7.relay.example.test',
sourceCellIncarnation: '22222222-2222-4222-8222-222222222222',
targetCellId: 'production-gce-c27',
targetCellIncarnation: '33333333-3333-4333-8333-333333333333',
previousEpoch: 7,
assignmentEpoch: 8,
drainGraceMs: 60_000,
sendAttempts: 1
it('bounds empty polling to the six-second cadence and stops its timer', async () => {
vi.useFakeTimers()
const selectIdleRegionalRehomeCandidates = vi.fn().mockResolvedValue([])
const worker = startRegionalRehomeWorker(config(), {
selectIdleRegionalRehomeCandidates
} as unknown as RelayAssignmentStore, {
safetySnapshot: () => safety(Date.now()),
random: () => 0
})!
try {
await vi.advanceTimersByTimeAsync(0)
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledTimes(1)
await vi.advanceTimersByTimeAsync(5_999)
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledTimes(1)
await vi.advanceTimersByTimeAsync(1)
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledTimes(2)
worker.stop()
await vi.advanceTimersByTimeAsync(60_000)
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledTimes(2)
} finally {
worker.stop()
vi.useRealTimers()
}
const claimRegionalRehome = vi.fn().mockResolvedValueOnce(null).mockResolvedValue(attempt)
const recordRegionalRehomeDrainReceipt = vi.fn().mockResolvedValue(true)
const assignments = {
claimRegionalRehome,
recordRegionalRehomeDrainReceipt
} as unknown as RelayAssignmentStore
const requests: Array<{ url: string; init?: RequestInit }> = []
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const worker = startRegionalRehomeWorker(config(), assignments, {
now: () => now,
safetySnapshot: () => safety(now),
intervalMs: 60_000,
identityToken: async (audience) => {
expect(audience).toBe('https://relay.example.test/v1/admin/host-drain')
return 'secret-token'
},
fetch: (async (url, init) => {
requests.push({ url: String(url), init })
return Response.json({ v: 1, outcome: 'accepted' })
}) as typeof fetch
})!
await settleWorker()
now = 1_000
await worker.run()
worker.stop()
expect(requests).toHaveLength(1)
expect(requests[0]!.url).toBe('https://c7.relay.example.test/v1/admin/host-drain')
expect(requests[0]!.url).not.toContain('secret-token')
expect(requests[0]!.init?.headers).toMatchObject({
authorization: 'Bearer secret-token'
})
expect(JSON.parse(String(requests[0]!.init?.body))).toEqual({
v: 1,
attemptId: '11111111-1111-4111-8111-111111111111',
userId: 'private-user',
relayHostId: 'abcdefghijklmnop',
sourceCellId: 'production-gce-c7',
sourceCellIncarnation: '22222222-2222-4222-8222-222222222222',
sourceAssignmentEpoch: 7,
graceMs: 60_000
})
expect(recordRegionalRehomeDrainReceipt).toHaveBeenCalledWith(
'11111111-1111-4111-8111-111111111111',
'accepted'
)
const logs = warn.mock.calls.map((call) => String(call[0])).join('\n')
expect(logs).not.toContain('private-user')
expect(logs).not.toContain('abcdefghijklmnop')
})
it('fails closed before the observation gate and records bounded dispatch failures', async () => {
let now = 0
const attempt = {
attemptId: '11111111-1111-4111-8111-111111111111',
userId: 'private-user',
relayHostId: 'abcdefghijklmnop',
sourceCellId: 'source',
sourceCellUrl: 'https://source.example.test',
sourceCellIncarnation: '22222222-2222-4222-8222-222222222222',
targetCellId: 'target',
previousEpoch: 1,
assignmentEpoch: 2,
drainGraceMs: 60_000,
sendAttempts: 1
}
const claimRegionalRehome = vi.fn().mockResolvedValueOnce(null).mockResolvedValue(attempt)
const assignments = {
claimRegionalRehome,
recordRegionalRehomeDispatchFailure: vi.fn().mockResolvedValue(undefined)
} as unknown as RelayAssignmentStore
const worker = startRegionalRehomeWorker(config(), assignments, {
now: () => now,
safetySnapshot: () => safety(now),
intervalMs: 60_000,
identityToken: async () => {
throw new Error('token unavailable')
}
})!
await settleWorker()
claimRegionalRehome.mockClear()
now = 100
await worker.run()
worker.stop()
expect(assignments.recordRegionalRehomeDispatchFailure).toHaveBeenCalledWith(
'11111111-1111-4111-8111-111111111111'
)
})
it('keeps a failed poll out of the durable dispatch-failure budget', async () => {
let now = 0
const claimRegionalRehome = vi
.fn()
.mockResolvedValueOnce(null)
.mockRejectedValue(new Error('Connection terminated due to connection timeout'))
const recordRegionalRehomeDispatchFailure = vi.fn().mockResolvedValue(undefined)
const assignments = {
claimRegionalRehome,
recordRegionalRehomeDispatchFailure
} as unknown as RelayAssignmentStore
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const worker = startRegionalRehomeWorker(config(), assignments, {
now: () => now,
safetySnapshot: () => safety(now),
intervalMs: 60_000
})!
await settleWorker()
now = 1_000
await expect(worker.run()).resolves.toBeUndefined()
worker.stop()
// The poll never claimed an attempt, so nothing was drained and nothing may
// be charged to the budget that latches the durable control off.
expect(recordRegionalRehomeDispatchFailure).not.toHaveBeenCalled()
expect(warn.mock.calls.map((call) => JSON.parse(String(call[0])).event)).toEqual([
'orca_relay_regional_rehome_poll_failed'
])
})
it('passes unsafe process telemetry to the durable claim gate', async () => {
let now = 0
let sqlFailures = 0
const claimRegionalRehome = vi.fn().mockResolvedValue(null)
const selectIdleRegionalRehomeCandidates = vi.fn().mockResolvedValue([])
const assignments = {
claimRegionalRehome
selectIdleRegionalRehomeCandidates
} as unknown as RelayAssignmentStore
const worker = startRegionalRehomeWorker(config(), assignments, {
now: () => now,
@@ -160,46 +49,40 @@ describe('regional rehome worker', () => {
intervalMs: 60_000
})!
await settleWorker()
claimRegionalRehome.mockClear()
selectIdleRegionalRehomeCandidates.mockClear()
now = 100
sqlFailures = 1
await worker.run()
worker.stop()
expect(claimRegionalRehome).toHaveBeenCalledWith(
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledWith(
expect.objectContaining({ observedAt: 100, sqlFailures: 1 })
)
})
it('starts inert on directors so durable control can enable without a restart', async () => {
let now = 0
const claimRegionalRehome = vi.fn().mockResolvedValue(null)
const selectIdleRegionalRehomeCandidates = vi.fn().mockResolvedValue([])
const assignments = {
claimRegionalRehome
selectIdleRegionalRehomeCandidates
} as unknown as RelayAssignmentStore
const worker = startRegionalRehomeWorker(
config(),
assignments,
{
now: () => now,
safetySnapshot: () => safety(now),
intervalMs: 60_000
}
)
const worker = startRegionalRehomeWorker(config(), assignments, {
now: () => now,
safetySnapshot: () => safety(now),
intervalMs: 60_000
})
expect(worker).not.toBeNull()
await settleWorker()
claimRegionalRehome.mockClear()
selectIdleRegionalRehomeCandidates.mockClear()
now = 100
await worker!.run()
worker!.stop()
expect(claimRegionalRehome).toHaveBeenCalledOnce()
expect(selectIdleRegionalRehomeCandidates).toHaveBeenCalledOnce()
expect(
startRegionalRehomeWorker(
config({ role: 'cell' }),
{} as RelayAssignmentStore,
{ safetySnapshot: () => safety(1) }
)
startRegionalRehomeWorker(config({ role: 'cell' }), {} as RelayAssignmentStore, {
safetySnapshot: () => safety(1)
})
).toBeNull()
})
@@ -208,16 +91,20 @@ describe('regional rehome worker', () => {
const limit = cells * REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT
const processSafety = { ...safety(100), reconnects: limit * 10 }
const fleetSafety = { ...safety(100), reconnects: limit }
expect(regionalRehomeSafetyFailure(
combineRegionalRehomeSafety(processSafety, fleetSafety),
100,
cells
)).toBeNull()
expect(regionalRehomeSafetyFailure(
combineRegionalRehomeSafety(processSafety, { ...fleetSafety, reconnects: limit + 1 }),
100,
cells
)).toBe('elevated_reconnects')
expect(
regionalRehomeSafetyFailure(
combineRegionalRehomeSafety(processSafety, fleetSafety),
100,
cells
)
).toBeNull()
expect(
regionalRehomeSafetyFailure(
combineRegionalRehomeSafety(processSafety, { ...fleetSafety, reconnects: limit + 1 }),
100,
cells
)
).toBe('elevated_reconnects')
})
})
+41 -62
View File
@@ -1,4 +1,4 @@
import { z } from 'zod'
import { IdleRegionalRehomeResponseSchema } from '@orca-cloud/relay-contract'
import type { RelayAssignmentStore } from './assignment-store.js'
import type { RelayConfig } from './config.js'
import { googleMetadataIdentityToken } from './google-metadata-identity-token.js'
@@ -20,13 +20,6 @@ export type RegionalRehomeWorker = {
stop: () => void
}
const RegionalHostDrainResponseSchema = z
.object({
v: z.literal(1),
outcome: z.enum(['accepted', 'already-accepted', 'host-not-connected'])
})
.strict()
export function startRegionalRehomeWorker(
config: RelayConfig,
assignments: RelayAssignmentStore,
@@ -42,7 +35,6 @@ export function startRegionalRehomeWorker(
}
const audience = config.rehomeAudience
const safetySnapshot = options.safetySnapshot
const now = options.now ?? Date.now
const fetchImpl = options.fetch ?? fetch
const tokenProvider =
options.identityToken ??
@@ -52,64 +44,50 @@ export function startRegionalRehomeWorker(
const run = async (): Promise<void> => {
if (stopped || inFlight) return
inFlight = true
let attemptId: string | null = null
try {
const processSafety = safetySnapshot()
const attempt = await assignments.claimRegionalRehome(processSafety)
if (!attempt) return
attemptId = attempt.attemptId
const candidates = await assignments.selectIdleRegionalRehomeCandidates(safetySnapshot())
if (candidates.length === 0) return
const token = await tokenProvider(audience)
const response = await fetchImpl(
new URL('/v1/admin/host-drain', attempt.sourceCellUrl),
{
method: 'POST',
headers: {
authorization: `Bearer ${token}`,
'content-type': 'application/json'
},
body: JSON.stringify({
v: 1,
attemptId: attempt.attemptId,
userId: attempt.userId,
relayHostId: attempt.relayHostId,
sourceCellId: attempt.sourceCellId,
sourceCellIncarnation: attempt.sourceCellIncarnation,
sourceAssignmentEpoch: attempt.previousEpoch,
graceMs: attempt.drainGraceMs
}),
signal: AbortSignal.timeout(options.requestTimeoutMs ?? 10_000)
for (const candidate of candidates) {
if (stopped) return
const { sourceCellUrl, ...request } = candidate
try {
const response = await fetchImpl(new URL('/v1/admin/host-idle-rehome', sourceCellUrl), {
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify({
...request,
cohortPercent: config.regionCorrectionCohortPercent ?? 0,
directorSafety: safetySnapshot()
}),
signal: AbortSignal.timeout(options.requestTimeoutMs ?? 10_000)
})
if (!response.ok) throw new Error(`regional_rehome_source_${response.status}`)
const body = IdleRegionalRehomeResponseSchema.parse(await response.json())
if (body.outcome === 'committed') {
console.warn(
JSON.stringify({
event: 'orca_relay_idle_rehome_committed',
sourceCellId: candidate.sourceCellId,
targetCellId: candidate.targetCellId
})
)
return
}
} catch (error) {
// The source may have committed; its durable outcome owns recovery.
console.warn(
JSON.stringify({
event: 'orca_relay_idle_rehome_request_failed',
reason: error instanceof Error ? error.message : 'unknown'
})
)
}
)
if (!response.ok) throw new Error(`regional_rehome_source_${response.status}`)
const body = RegionalHostDrainResponseSchema.safeParse(await response.json())
if (!body.success) throw new Error('regional_rehome_source_invalid_response')
await assignments.recordRegionalRehomeDrainReceipt(
attempt.attemptId,
body.data.outcome
)
console.warn(
JSON.stringify({
event: 'orca_relay_regional_rehome_dispatched',
sourceCellId: attempt.sourceCellId,
targetCellId: attempt.targetCellId,
outcome: body.data.outcome,
sendAttempts: attempt.sendAttempts
})
)
} catch (error) {
// Only a claimed attempt was drained. A poll that failed before the claim
// - a pool timeout on the once-a-second control read - dispatched nothing,
// so it must not spend the budget that latches the durable control off.
if (attemptId) {
await assignments
.recordRegionalRehomeDispatchFailure(attemptId)
.catch(() => undefined)
}
} catch (error) {
console.warn(
JSON.stringify({
event: attemptId
? 'orca_relay_regional_rehome_dispatch_failed'
: 'orca_relay_regional_rehome_poll_failed',
event: 'orca_relay_regional_rehome_poll_failed',
reason: error instanceof Error ? error.message : 'unknown'
})
)
@@ -119,7 +97,8 @@ export function startRegionalRehomeWorker(
}
const timer = setInterval(
() => void run(),
options.intervalMs ?? jitteredSweepIntervalMs(1_000, options.random)
// Match the initial ten-moves/minute budget without replanning the join every second.
options.intervalMs ?? jitteredSweepIntervalMs(6_000, options.random)
)
timer.unref()
void run()
@@ -40,6 +40,7 @@ describe('Relay region API', () => {
)
expect(response.status).toBe(200)
expect(await response.clone().json()).not.toHaveProperty('regionCorrection')
expect(assign).toHaveBeenCalledWith(
{ userId: 'user-1', relayHostId: 'asiahost00000001' },
'asia-east2',
@@ -83,6 +84,160 @@ describe('Relay region API', () => {
)
})
it('preserves the cold-start hint and binds a negotiated window after placement', async () => {
const assignment = {
userId: 'user-1',
relayHostId: 'abcdefghijklmnop',
cellId: 'asia-c1',
cellUrl: 'https://asia-c1.relay.example.test',
region: 'asia-east2',
assignmentEpoch: 7,
leaseExpiresAt: Date.now() + 300_000
}
const assign = vi.fn(async () => assignment)
const window = {
generation: 2,
expiresAt: Date.now() + 86_400_000,
assignmentEpoch: 7,
incumbentRegion: 'asia-east2',
policyVersion: 1
}
const exchangeRegionCorrection = vi.fn(async () => ({ v: 1, window }))
const app = createRelayApp(config(), {
store: {} as never,
assignments: { assign, exchangeRegionCorrection } as never,
drain: vi.fn(),
ready: async () => true
})
const regionCorrection = { v: 1, action: 'issue-window' }
const response = await app.request(
'/v1/assign',
assignmentRequest('abcdefghijklmnop', {
preferredRegion: 'asia-east2',
regionCorrection
})
)
expect(response.status).toBe(200)
expect(assign).toHaveBeenCalledWith(
{ userId: 'user-1', relayHostId: 'abcdefghijklmnop' },
'asia-east2',
'asia-east2'
)
expect(exchangeRegionCorrection).toHaveBeenCalledWith(
{ userId: 'user-1', relayHostId: 'abcdefghijklmnop' },
regionCorrection,
7
)
expect(((await response.json()) as { regionCorrection: unknown }).regionCorrection).toEqual({
v: 1,
window
})
})
it('returns successful placement when optional window storage is unavailable', async () => {
const app = createRelayApp(config(), {
store: {} as never,
assignments: {
assign: async () => ({
cellId: 'asia-c1',
region: 'asia-east2',
cellUrl: 'https://asia-c1.relay.example.test',
assignmentEpoch: 7
}),
exchangeRegionCorrection: async () => {
throw new Error('database unavailable')
}
} as never,
drain: vi.fn(),
ready: async () => true
})
const response = await app.request(
'/v1/assign',
assignmentRequest('abcdefghijklmnop', {
preferredRegion: 'asia-east2',
regionCorrection: { v: 1, action: 'issue-window' }
})
)
expect(response.status).toBe(200)
expect(await response.json()).toMatchObject({
cellUrl: 'https://asia-c1.relay.example.test',
assignmentEpoch: 7
})
})
it('does not place or write a legacy hint when reporting migration evidence', async () => {
const current = {
userId: 'user-1',
relayHostId: 'abcdefghijklmnop',
cellId: 'asia-c1',
cellUrl: 'https://asia-c1.relay.example.test',
region: 'asia-east2',
assignmentEpoch: 7,
leaseExpiresAt: Date.now() + 300_000
}
const assign = vi.fn()
const resolve = vi.fn(async () => current)
const exchangeRegionCorrection = vi.fn(async () => ({ v: 1, reportStatus: 'accepted' }))
const app = createRelayApp(config(), {
store: {} as never,
assignments: { assign, resolve, exchangeRegionCorrection } as never,
drain: vi.fn(),
ready: async () => true
})
const regionCorrection = {
v: 1,
action: 'report',
generation: 2,
assignmentEpoch: 7,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 40, 'asia-east2': 180 }
}
const response = await app.request(
'/v1/assign',
assignmentRequest('abcdefghijklmnop', {
preferredRegion: 'us-central1',
regionCorrection
})
)
expect(response.status).toBe(200)
expect(assign).not.toHaveBeenCalled()
expect(exchangeRegionCorrection).toHaveBeenCalledWith(
{ userId: 'user-1', relayHostId: 'abcdefghijklmnop' },
regionCorrection,
7
)
expect(((await response.json()) as { assignmentEpoch: number }).assignmentEpoch).toBe(7)
})
it('does not manufacture an assignment for a report whose assignment disappeared', async () => {
const assign = vi.fn()
const exchangeRegionCorrection = vi.fn()
const app = createRelayApp(config(), {
store: {} as never,
assignments: { assign, resolve: async () => null, exchangeRegionCorrection } as never,
drain: vi.fn(),
ready: async () => true
})
const response = await app.request(
'/v1/assign',
assignmentRequest('abcdefghijklmnop', {
regionCorrection: {
v: 1,
action: 'report',
generation: 2,
assignmentEpoch: 7,
policyVersion: 1,
outcome: 'inconclusive',
reason: 'timeout'
}
})
)
expect(response.status).toBe(409)
expect(assign).not.toHaveBeenCalled()
expect(exchangeRegionCorrection).not.toHaveBeenCalled()
})
it('exposes only the store-provided healthy catalog from directors', async () => {
const regionCatalog = vi.fn(async () => [
{ region: 'us-central1' as const, probeOrigins: ['https://us.relay.example.test'] }
+25 -8
View File
@@ -20,14 +20,12 @@ import { createRelayApp } from './app.js'
import { RelayAssignmentStore } from './assignment-store.js'
import type { RelayConfig } from './config.js'
import { RelayCredentialStore } from './credential-store.js'
import type { RelayDatabase } from './database.js'
import { readRelayDatabasePoolPressure, type RelayDatabase } from './database.js'
import { HostSessionRegistry } from './host-session-registry.js'
import { observeRelayDatabase } from './observed-relay-database.js'
import { RelayObservability } from './relay-observability.js'
import {
RelayConnectionLedger,
type RelayConnectionUpgrade
} from './relay-connection-ledger.js'
import { combineRegionalRehomeSafety } from './regional-rehome-safety.js'
import { RelayConnectionLedger, type RelayConnectionUpgrade } from './relay-connection-ledger.js'
import { createRelayReadiness } from './relay-readiness.js'
import { createRelayTokenVerifier, readBearer } from './relay-token-verifier.js'
import { closeRelayWebSocket } from './relay-websocket-close.js'
@@ -65,7 +63,7 @@ function guardSocketErrors(socket: WebSocket, kind: string): void {
function admissionSource(request: IncomingMessage): string {
const forwarded = request.headers['x-forwarded-for']
const chain = (Array.isArray(forwarded) ? forwarded.join(',') : forwarded ?? '')
const chain = (Array.isArray(forwarded) ? forwarded.join(',') : (forwarded ?? ''))
.split(',')
.map((entry) => entry.trim())
.filter(Boolean)
@@ -112,6 +110,7 @@ export function createRelayServer(
const store = new RelayCredentialStore(observedDatabase, options.now)
const assignments = new RelayAssignmentStore(observedDatabase, options.now, {
requireLiveCells: config.role === 'director',
regionalRehomeCohortPercent: config.regionCorrectionCohortPercent ?? 0,
recordControlRenewal: (durationMs, outcome) =>
observability.recordControlRenewal?.(durationMs, outcome)
})
@@ -127,17 +126,35 @@ export function createRelayServer(
queuedBytes,
observability,
options.now,
options.random
options.random,
cellIncarnation
)
const app = createRelayApp(config, {
store,
assignments,
drain: (graceMs) => sessions.drain(graceMs),
drainHost: (input) => sessions.drainHost(input),
idleRehome: (input) => {
const now = (options.now ?? Date.now)()
if (input.directorSafety.observedAt > now || now - input.directorSafety.observedAt > 60_000) {
return Promise.resolve({ outcome: 'deferred' })
}
return sessions.idleRehome(input,
() => assignments.commitIdleRegionalRehome(input, combineRegionalRehomeSafety(
input.directorSafety,
{ ...observability.regionalRehomeRuntimeSafety(), ...readRelayDatabasePoolPressure(database) }
), input.cohortPercent),
() => assignments.reconcileIdleRegionalRehome(input)
)
},
regionalRehomeTrustProbeHostExists: (input) => sessions.get(input) !== null,
cellIncarnation,
isDraining: () => sessions.isDraining(),
runtimeCounts: () => runtimeCounts(),
regionalRehomeSafetySnapshot: () => ({
...observability.regionalRehomeRuntimeSafety(),
...readRelayDatabasePoolPressure(database)
}),
ready,
recordAssignmentAdmission: (outcome) => observability.recordAssignmentAdmission?.(outcome),
recordAssignmentRejectionReason: (lane, reason) =>
@@ -339,7 +356,7 @@ export function createRelayServer(
const identity = invite ? { userId: invite.userId, relayHostId: hostId } : null
// Released combined-service invites gain their first durable cell assignment here.
const assignment = identity
? (await assignments.resolve(identity)) ?? (await assignments.assign(identity))
? ((await assignments.resolve(identity)) ?? (await assignments.assign(identity)))
: null
if (!invite || !assignment) {
phoneAdmission?.hostData.release()
@@ -19,7 +19,7 @@ describe('sweep schedule jitter', () => {
expect(SWEEP_JITTER_FRACTION).toBeGreaterThan(0)
})
it('jitters the regional rehome dispatch tick, which every director runs each second', () => {
it('jitters the six-second regional rehome dispatch tick across directors', () => {
const timers: number[] = []
const setIntervalSpy = vi
.spyOn(globalThis, 'setInterval')
@@ -35,14 +35,14 @@ describe('sweep schedule jitter', () => {
rehomeAudience: 'https://rehome.example.test',
rehomeDirectorServiceAccount: 'rehome@example.test'
} as never,
{ claimRegionalRehome: async () => null } as never,
{ selectIdleRegionalRehomeCandidates: async () => [] } as never,
{ random: () => 0.5, safetySnapshot: () => ({}) as never }
)
} finally {
setIntervalSpy.mockRestore()
}
expect(timers).toEqual([1_100])
expect(timers).toEqual([6_600])
})
// Why: index.ts boots a server on import, so its wiring can only be read.
@@ -0,0 +1,8 @@
Exact relay contract snapshots from `027acb4efa2e6b226d40df266b86367423946d62`, `cloud/packages/relay-contract/src/`. Used to exercise the pre-correction strict wire parsers. Do not format or edit these baseline sources.
```text
aba94e108a5cd0f1af8b38875429ad8636d24c43a728273e3df60d9a1a1d1b6d director-messages.ts
bd13b5a694a5d683a5b680c14e46ab33f4ef4a5bfedf040d046b09d540cb4c17 wire-scalars.ts
bc89116f884a2f20a6588f9b91219aa596bc2410d28b499a93a78350def109d5 relay-regions.ts
8fcae470a5fc72f2fcdde9d2f09cd20289c256356dd490484ac1cfa53839fbe4 control-messages.ts
```
@@ -0,0 +1,146 @@
import { z } from 'zod'
import {
Base6432ByteSchema,
Base64Raw24ByteSchema,
Base64Url32ByteSchema,
EpochMsSchema,
GenerationSchema,
OpaqueIdSchema,
PositiveDurationMsSchema,
RelayHostIdSchema
} from './wire-scalars.js'
const AppVersionSchema = z.string().min(1).max(128)
const BoundedCiphertextSchema = z
.string()
.min(1)
.max(16 * 1024)
.regex(/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/)
const ConnectionKindSchema = z.enum(['invite', 'resume'])
export const HostHelloSchema = z
.object({
v: z.literal(1),
relayHostId: RelayHostIdSchema,
assignmentEpoch: GenerationSchema,
hostPublicKeyB64: Base6432ByteSchema,
appVersion: AppVersionSchema,
previousGeneration: GenerationSchema.optional(),
controlResumeSecret: Base64Url32ByteSchema.optional()
})
.strict()
export const HostChallengeSchema = z
.object({
challengeId: OpaqueIdSchema,
relayEphemeralPublicKeyB64: Base6432ByteSchema,
nonceB64: Base64Raw24ByteSchema,
ciphertextB64: BoundedCiphertextSchema,
expiresAt: EpochMsSchema
})
.strict()
export const HostChallengeAckSchema = z
.object({ challengeId: OpaqueIdSchema, proofB64: Base6432ByteSchema })
.strict()
// Advertised on the control upgrade rather than in host-hello: HostHelloSchema
// is strict, so a new hello key is refused by every already-deployed cell.
export const RELAY_HOST_CAPABILITIES_HEADER = 'x-orca-host-capabilities'
// The host accepts kind/relayDeviceId on a pendingConns entry. A host that does
// not advertise this parses those entries strictly and would drop the whole ack.
export const RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS = 'pending-conn-details'
export function parseRelayHostCapabilities(
header: string | string[] | undefined
): ReadonlySet<string> {
const raw = Array.isArray(header) ? header.join(',') : (header ?? '')
return new Set(
raw
.split(',')
.map((token) => token.trim())
.filter((token) => token.length > 0 && token.length <= 64)
.slice(0, 16)
)
}
// kind/relayDeviceId are optional so an entry stays readable by a host that
// predates them; the cell only emits them to a host that advertised support.
const PendingConnectionSchema = z
.object({
connId: OpaqueIdSchema,
connTicket: Base64Url32ByteSchema,
kind: ConnectionKindSchema.optional(),
relayDeviceId: OpaqueIdSchema.optional()
})
.strict()
export const HostHelloAckSchema = z
.object({
v: z.literal(1),
generation: GenerationSchema,
controlResumeSecret: Base64Url32ByteSchema,
leaseExpiresAt: EpochMsSchema,
activeConnIds: z.array(OpaqueIdSchema).max(8),
pendingConns: z.array(PendingConnectionSchema).max(8)
})
.strict()
export const ConnectionOpenSchema = z
.object({
connId: OpaqueIdSchema,
connTicket: Base64Url32ByteSchema,
kind: ConnectionKindSchema,
relayDeviceId: OpaqueIdSchema,
attachDeadlineMs: PositiveDurationMsSchema
})
.strict()
export const HostDataAuthSchema = z
.object({
v: z.literal(1),
connTicket: Base64Url32ByteSchema,
generation: GenerationSchema
})
.strict()
export const InviteCreateSchema = z
.object({ reqId: OpaqueIdSchema, relayDeviceId: OpaqueIdSchema })
.strict()
export const InviteCreatedSchema = z
.object({
reqId: OpaqueIdSchema,
inviteToken: Base64Url32ByteSchema,
expiresAt: EpochMsSchema,
maxAttempts: z.number().int().positive().max(16)
})
.strict()
export const DeviceRevokeSchema = z
.object({ reqId: OpaqueIdSchema, relayDeviceId: OpaqueIdSchema })
.strict()
export const AuthRefreshSchema = z.object({ relayJwt: z.string().min(1).max(8 * 1024) }).strict()
export const DrainSchema = z
.object({
graceMs: z.number().int().nonnegative().max(60 * 60 * 1000),
recovery: z.literal('resolve-director')
})
.strict()
export const HeartbeatSchema = z.object({ t: EpochMsSchema }).strict()
export type HostHello = z.infer<typeof HostHelloSchema>
export type HostChallenge = z.infer<typeof HostChallengeSchema>
export type HostChallengeAck = z.infer<typeof HostChallengeAckSchema>
export type HostHelloAck = z.infer<typeof HostHelloAckSchema>
export type ConnectionOpen = z.infer<typeof ConnectionOpenSchema>
export type HostDataAuth = z.infer<typeof HostDataAuthSchema>
export type InviteCreate = z.infer<typeof InviteCreateSchema>
export type InviteCreated = z.infer<typeof InviteCreatedSchema>
export type DeviceRevoke = z.infer<typeof DeviceRevokeSchema>
export type AuthRefresh = z.infer<typeof AuthRefreshSchema>
export type Drain = z.infer<typeof DrainSchema>
export type Heartbeat = z.infer<typeof HeartbeatSchema>
@@ -0,0 +1,75 @@
import { z } from 'zod'
import {
Base64Url32ByteSchema,
CanonicalHttpsOriginSchema,
EpochMsSchema,
GenerationSchema,
RelayHostIdSchema
} from './wire-scalars.js'
import { RelayRegionSchema } from './relay-regions.js'
const SignedAssignmentLeaseSchema = z.string().min(1).max(8 * 1024)
export const AssignmentRequestSchema = z
.object({
v: z.literal(1),
relayHostId: RelayHostIdSchema,
// Client-declared reconnection; the director verifies it against the
// durable assignment before granting fast-lane admission.
reconnect: z.boolean().optional(),
preferredRegion: RelayRegionSchema.optional()
})
.strict()
export const AssignmentResponseSchema = z
.object({
v: z.literal(1),
cellUrl: CanonicalHttpsOriginSchema,
assignmentEpoch: GenerationSchema,
lease: SignedAssignmentLeaseSchema
})
.strict()
export const ResolveRequestSchema = z
.object({
v: z.literal(1),
relayHostId: RelayHostIdSchema,
resumeToken: Base64Url32ByteSchema
})
.strict()
export const ResolveResponseSchema = z
.object({
v: z.literal(1),
cellUrl: CanonicalHttpsOriginSchema,
assignmentEpoch: GenerationSchema,
leaseExpiresAt: EpochMsSchema
})
.strict()
export const RelayMovedSchema = z
.object({
v: z.literal(1),
cellUrl: CanonicalHttpsOriginSchema,
assignmentEpoch: GenerationSchema
})
.strict()
export function isTrustedNewerMove(input: {
sourceOrigin: string
configuredDirectorOrigin: string
currentAssignmentEpoch: number
move: z.infer<typeof RelayMovedSchema>
}): boolean {
// Why: cells and stale director responses must never redirect a credential-bearing client.
return (
input.sourceOrigin === input.configuredDirectorOrigin &&
input.move.assignmentEpoch > input.currentAssignmentEpoch
)
}
export type AssignmentRequest = z.infer<typeof AssignmentRequestSchema>
export type AssignmentResponse = z.infer<typeof AssignmentResponseSchema>
export type ResolveRequest = z.infer<typeof ResolveRequestSchema>
export type ResolveResponse = z.infer<typeof ResolveResponseSchema>
export type RelayMoved = z.infer<typeof RelayMovedSchema>
@@ -0,0 +1,71 @@
import { z } from 'zod'
export const RELAY_REGIONS = ['us-central1', 'asia-east2'] as const
export const RelayRegionSchema = z.enum(RELAY_REGIONS)
export type RelayRegion = z.infer<typeof RelayRegionSchema>
export const RELAY_DEFAULT_REGION: RelayRegion = 'us-central1'
// Field-name segment for the flat per-region runtime counters, spelled out rather than derived so
// the Terraform side can hold the same literal and a test can compare the two. `satisfies` makes a
// new region a compile error here, which is the point: a region with no segment would silently
// drop out of the region-skew alert's denominators.
export const RELAY_REGION_METRIC_SEGMENTS = {
'us-central1': 'UsCentral1',
'asia-east2': 'AsiaEast2'
} as const satisfies Record<RelayRegion, string>
const RelayProbeOriginSchema = z.string().url().max(2_048).refine(isCanonicalHttpsOrigin)
export const RelayRegionCatalogResponseSchema = z
.object({
v: z.literal(1),
regions: z
.array(
z
.object({
region: RelayRegionSchema,
probeOrigins: z.array(RelayProbeOriginSchema).min(1).max(2)
})
.strict()
)
.max(RELAY_REGIONS.length)
})
.strict()
.superRefine((catalog, context) => {
const regions = new Set<RelayRegion>()
const origins = new Set<string>()
for (const [regionIndex, entry] of catalog.regions.entries()) {
if (regions.has(entry.region)) {
context.addIssue({
code: 'custom',
message: 'duplicate relay region',
path: ['regions', regionIndex, 'region']
})
}
regions.add(entry.region)
for (const [originIndex, origin] of entry.probeOrigins.entries()) {
if (origins.has(origin)) {
context.addIssue({
code: 'custom',
message: 'duplicate relay probe origin',
path: ['regions', regionIndex, 'probeOrigins', originIndex]
})
}
origins.add(origin)
}
}
})
export type RelayRegionCatalogResponse = z.infer<typeof RelayRegionCatalogResponseSchema>
function isCanonicalHttpsOrigin(value: string): boolean {
try {
const url = new URL(value)
return url.protocol === 'https:' && url.origin === value
} catch {
return false
}
}
@@ -0,0 +1,20 @@
import { z } from 'zod'
export const Base64Url32ByteSchema = z.string().regex(/^[A-Za-z0-9_-]{43}$/)
export const Base64Url24ByteSchema = z.string().regex(/^[A-Za-z0-9_-]{32}$/)
export const Base6432ByteSchema = z.string().regex(/^(?:[A-Za-z0-9+/]{4}){10}[A-Za-z0-9+/]{3}=$/)
export const Base64Raw24ByteSchema = z.string().regex(/^(?:[A-Za-z0-9+/]{4}){8}$/)
export const RelayHostIdSchema = z.string().regex(/^[A-Za-z0-9_-]{16}$/)
export const OpaqueIdSchema = z.string().min(1).max(128)
export const EpochMsSchema = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER)
export const GenerationSchema = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER)
export const PositiveDurationMsSchema = z.number().int().positive().max(24 * 60 * 60 * 1000)
export const CanonicalHttpsOriginSchema = z.string().max(2048).refine((value) => {
try {
const url = new URL(value)
return url.protocol === 'https:' && url.origin === value && url.pathname === '/'
} catch {
return false
}
}, 'must be a canonical HTTPS origin')
+1 -1
View File
@@ -6,5 +6,5 @@
"outDir": "dist",
"rootDir": "src"
},
"exclude": ["src/**/*.test.ts"]
"exclude": ["src/**/*.test.ts", "src/test-fixtures/**"]
}
+22 -1
View File
@@ -10,6 +10,7 @@ export const DIRECTOR_REGIONAL_PLACEMENT_SECRET =
'orca-cloud-relay-regional-placement-enabled'
export const DIRECTOR_REGIONAL_PLACEMENT_ENV =
'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED'
export const DIRECTOR_CORRECTION_COHORT_ENV = 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT'
export const DIRECTOR_REHOME_IDENTITY_ENV =
'ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT'
export const DIRECTOR_REHOME_AUDIENCE_ENV = 'ORCA_RELAY_REHOME_AUDIENCE'
@@ -228,6 +229,13 @@ export function directorCellSetAddition(currentValue, desiredValue) {
return { changed: additions.length > 0, value: JSON.stringify(desired) }
}
export function correctionCohortPercent(value) {
if (!/^(?:[0-9]|[1-9][0-9]|100)$/.test(String(value))) {
throw new Error('region correction cohort must be an integer from 0 to 100')
}
return String(value)
}
export function directorDeploymentEnvironment(config) {
const imageDigest = config.image?.match(/@(sha256:[a-f0-9]{64})$/)?.[1]
if (config.image !== undefined && imageDigest === undefined) {
@@ -238,6 +246,10 @@ export function directorDeploymentEnvironment(config) {
ORCA_RELAY_ADMISSION_SELECTOR_VERSION: SELECTOR_REVISION_MARKER,
...(imageDigest === undefined ? {} : { ORCA_RELAY_IMAGE_DIGEST: imageDigest })
}
if (config['region-correction-cohort-percent'] !== undefined &&
config['region-correction-cohort-percent'] !== 'preserve') {
environment[DIRECTOR_CORRECTION_COHORT_ENV] = correctionCohortPercent(config['region-correction-cohort-percent'])
}
const serviceAccount = projectServiceAccount(config, 'capacity-service-account')
const asiaProofServiceAccount = projectServiceAccount(config, 'asia-proof-service-account')
const rehomeDirectorServiceAccount = projectServiceAccount(
@@ -302,7 +314,8 @@ export function parseArguments(argv) {
values['rehome-director-service-account'] !== undefined ||
values['rehome-audience'] !== undefined ||
values['expected-rehome-generation'] !== undefined ||
values['rehome-control-origin'] !== undefined
values['rehome-control-origin'] !== undefined ||
values['region-correction-cohort-percent'] !== undefined
) {
throw new Error('director configuration arguments require --role director')
}
@@ -785,6 +798,14 @@ export async function deployDirector(config, tag, overrides = {}) {
config['prune-revisions'] === 'true' ? CONNECTION_CAPACITY_PROTOCOL : undefined
const currentEnvironment = revisionEnvironment(servingRevision)
const deploymentEnvironment = directorDeploymentEnvironment(config)
deploymentEnvironment[DIRECTOR_CORRECTION_COHORT_ENV] ??= correctionCohortPercent(
currentEnvironment[DIRECTOR_CORRECTION_COHORT_ENV] ?? '0'
)
if (config['region-correction-cohort-percent'] !== undefined &&
config['region-correction-cohort-percent'] !== 'preserve' &&
config['expected-rehome-generation'] === undefined) {
throw new Error('cohort changes require an exact disabled regional-rehome generation')
}
const mutableEnvironment = {
...deploymentEnvironment,
[DIRECTOR_REGIONAL_PLACEMENT_ENV]: ''
@@ -4,6 +4,8 @@ import { test } from 'node:test'
import { fileURLToPath } from 'node:url'
import {
activeRevision,
correctionCohortPercent,
DIRECTOR_CORRECTION_COHORT_ENV,
cloudRunTrafficTag,
DIRECTOR_ADMISSION_ENVIRONMENT,
DIRECTOR_REGIONAL_PLACEMENT_ENV,
@@ -812,3 +814,43 @@ test('waits for authenticated target readiness without hiding other capacity err
/forbidden/
)
})
test('validates bounded correction cohorts and leaves unspecified values to serving inheritance', () => {
for (const value of ['0', '1', '100']) assert.equal(correctionCohortPercent(value), value)
for (const value of ['-1', '101', '1.5', '', '01', 'true', '1\n']) {
assert.throws(() => correctionCohortPercent(value), /integer from 0 to 100/)
}
assert.equal(directorDeploymentEnvironment({})[DIRECTOR_CORRECTION_COHORT_ENV], undefined)
assert.equal(directorDeploymentEnvironment({ 'region-correction-cohort-percent': 'preserve' })[DIRECTOR_CORRECTION_COHORT_ENV], undefined)
assert.equal(directorDeploymentEnvironment({ 'region-correction-cohort-percent': '1' })[DIRECTOR_CORRECTION_COHORT_ENV], '1')
})
test('inherits the cohort on candidate and rollback revisions without resetting an enabled cohort', async () => {
const harness = directorHarness()
harness.state.revisions.get('relay-00001-old').env[DIRECTOR_CORRECTION_COHORT_ENV] = '3'
await deployDirector({}, 'candidate-new', harness.operations)
for (const revision of ['relay-00002-new', 'relay-00003-new']) {
assert.equal(harness.state.revisions.get(revision).env[DIRECTOR_CORRECTION_COHORT_ENV], '3')
}
})
test('starts an unstamped cohort at zero and rejects a cohort change without disabled-control proof', async () => {
const harness = directorHarness()
await assert.rejects(deployDirector({ 'region-correction-cohort-percent': '1' },
'candidate-new', harness.operations), /exact disabled regional-rehome generation/)
assert.equal(harness.state.activeRevision, 'relay-00001-old')
assert.equal(harness.state.nextRevision, 2)
await deployDirector({}, 'candidate-new', harness.operations)
assert.equal(harness.state.revisions.get('relay-00003-new').env[DIRECTOR_CORRECTION_COHORT_ENV], '0')
})
test('sets a reviewed cohort only behind repeated disabled-control verification', async () => {
const harness = directorHarness()
let verified = 0
const config = { 'region-correction-cohort-percent': '1', 'expected-rehome-generation': '7' }
await deployDirector(config, 'candidate-new', { ...harness.operations,
assertRegionalRehomeDisabled: async () => { verified++ } })
assert.ok(verified >= 2)
assert.equal(harness.state.revisions.get('relay-00003-new').env[DIRECTOR_CORRECTION_COHORT_ENV], '1')
})
@@ -53,7 +53,7 @@ export function readRelayServingRegionalPlacementVersion(input, dependencies = {
try {
service = run(gcloudArguments('services', input))
} catch (error) {
if (error?.code === 'NOT_FOUND') return { version: input.bootstrap_version }
if (error?.code === 'NOT_FOUND') return { version: input.bootstrap_version, cohort_percent: '0' }
throw error
}
const serving = (service.status?.traffic ?? []).filter(
@@ -67,12 +67,22 @@ export function readRelayServingRegionalPlacementVersion(input, dependencies = {
throw new Error('Relay director must have exactly one revision serving 100% traffic')
}
const revision = run(gcloudArguments('revisions', input, serving[0].revisionName))
const cohortSettings = (revision.spec?.containers ?? []).flatMap((container) =>
(container.env ?? []).filter((environment) =>
environment.name === 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT')
)
if (cohortSettings.length > 1 || (cohortSettings.length === 1 &&
(typeof cohortSettings[0].value !== 'string' ||
!/^(?:[0-9]|[1-9][0-9]|100)$/.test(cohortSettings[0].value)))) {
throw new Error('serving region correction cohort is invalid')
}
const cohort_percent = cohortSettings[0]?.value ?? '0'
const references = (revision.spec?.containers ?? []).flatMap((container) =>
(container.env ?? []).filter(
(environment) => environment.name === 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED'
)
)
if (references.length === 0) return { version: input.bootstrap_version }
if (references.length === 0) return { version: input.bootstrap_version, cohort_percent }
const reference = normalizeSecretReference(references[0])
if (
references.length !== 1 ||
@@ -81,7 +91,7 @@ export function readRelayServingRegionalPlacementVersion(input, dependencies = {
) {
throw new Error('serving regional placement secret reference is invalid')
}
return { version: reference.version }
return { version: reference.version, cohort_percent }
}
// Why: the v2 API reports `valueSource.secretKeyRef.{secret,version}`, but
@@ -67,7 +67,7 @@ test('reads the exact version from the sole traffic-serving revision', () => {
}
})
assert.deepEqual(result, { version: '11' })
assert.deepEqual(result, { version: '11', cohort_percent: '0' })
assert.equal(calls[1][3], 'relay-serving')
})
@@ -78,7 +78,7 @@ test('reads the gcloud v1 secret reference shape by bare id and by full resource
]) {
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
run: (args) => args[1] === 'services' ? serving() : v1Revision(name, '1')
}), { version: '1' })
}), { version: '1', cohort_percent: '0' })
}
})
@@ -100,12 +100,12 @@ test('falls back only when the service or setting is absent', () => {
notFound.code = 'NOT_FOUND'
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
run: () => { throw notFound }
}), { version: '7' })
}), { version: '7', cohort_percent: '0' })
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
run: (args) => args[1] === 'services'
? { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } }
: { spec: { containers: [{ env: [] }] } }
}), { version: '7' })
}), { version: '7', cohort_percent: '0' })
})
test('classifies real absent-service stderr without weakening revision failures', () => {
@@ -136,3 +136,31 @@ test('rejects ambiguous traffic, malformed references, and read failures', () =>
run: () => { throw denied }
}), denied)
})
test('preserves the serving cohort including explicit disable across later Terraform plans', () => {
for (const value of ['0', '1', '17', '100']) {
const servingRevision = revision()
servingRevision.spec.containers[0].env.push({ name: 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT', value })
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
run: (args) => args[1] === 'services' ? serving() : servingRevision
}), { version: '11', cohort_percent: value })
}
})
test('fails closed on malformed, secret-backed or duplicate cohorts rather than resetting them', () => {
const name = 'ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT'
const cases = [
[{ name, value: '101' }], [{ name, value: '-1' }], [{ name, value: '1.5' }],
[{ name, value: '' }], [{ name, value: '01' }], [{ name, value: 1 }],
[{ name, valueFrom: { secretKeyRef: { name: 'unexpected', key: '1' } } }],
[{ name, value: '1' }, { name, value: '2' }]
]
for (const settings of cases) {
const servingRevision = revision()
servingRevision.spec.containers[0].env.push(...settings)
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
run: (args) => args[1] === 'services' ? serving() : servingRevision
}), /cohort is invalid/)
}
})
@@ -87,18 +87,21 @@ export function canaryAuthority(input) {
}
export function verifyCanaryAuthority(authority, expected, repositoryRoot) {
const selectorGeneration = Number(expected.selectorGeneration)
if (
authority?.v !== 1 ||
!/^[0-9a-f]{40}$/.test(authority.commitSha ?? '') ||
authority.runId !== expected.runId ||
authority.targetDigest !== expected.targetDigest ||
authority.rollbackDigest !== expected.rollbackDigest ||
authority.selectorGeneration !== Number(expected.selectorGeneration) ||
!Number.isSafeInteger(authority.selectorGeneration) ||
authority.selectorGeneration < 0 ||
!Number.isSafeInteger(selectorGeneration) ||
selectorGeneration < authority.selectorGeneration ||
authority.rehomeGeneration !== Number(expected.rehomeGeneration) ||
!SAME_CAP_CELLS.includes(authority.cellId)
) throw new Error('canary authority does not match this batch')
// The batch dispatch resolves main after the canary sealed, so bind to the same code, not the
// same SHA; every field above still pins this batch to that exact canary.
// Each cell checks exact live selector state; later batches may reuse this control epoch's canary.
requireSameEvidenceCode({
sealedSha: authority.commitSha,
currentSha: expected.commitSha,
@@ -109,6 +109,41 @@ test('seals and verifies canary authority for later batches', () => {
}), /does not match/)
})
test('reuses a canary across selector advances only within the same control epoch', () => {
const authority = canaryAuthority({
cellIds: 'production-gce-c7', targetDigest, rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7`,
commitSha: 'c'.repeat(40), runId: '42', selectorGeneration: '11', rehomeGeneration: '4'
})
const expected = {
commitSha: 'c'.repeat(40), runId: '42', targetDigest, rollbackDigest,
selectorGeneration: '21', rehomeGeneration: '4'
}
for (const generation of ['13', '14', '21', '29']) {
assert.equal(verifyCanaryAuthority(authority, {
...expected, selectorGeneration: generation
}), authority)
}
for (const generation of ['12', '-1', 'NaN', 'Infinity', '13.5', '9007199254740992']) {
assert.throws(() => verifyCanaryAuthority(authority, {
...expected, selectorGeneration: generation
}), /does not match/)
}
for (const generation of [-1, NaN, Infinity, 13.5, '13', Number.MAX_SAFE_INTEGER + 1]) {
assert.throws(() => verifyCanaryAuthority({
...authority, selectorGeneration: generation
}, expected), /does not match/)
}
for (const mismatch of [
{ rehomeGeneration: '3' }, { rehomeGeneration: '5' },
{ targetDigest: rollbackDigest }, { rollbackDigest: targetDigest }, { runId: '43' }
]) {
assert.throws(() => verifyCanaryAuthority(authority, {
...expected, ...mismatch
}), /does not match/)
}
})
function gitIn(root, ...args) {
return execFileSync('git', ['-C', root, ...args], { encoding: 'utf8' }).trim()
}
@@ -158,7 +193,7 @@ test('a batch trusts a canary sealed by identical code at an ancestor commit', a
runId: '42',
targetDigest,
rollbackDigest,
selectorGeneration: '13',
selectorGeneration: '21',
rehomeGeneration: '4'
}, repositoryRoot)
assert.equal(verifyAt(repository.sameCode, repository.root).cellId, 'production-gce-c7')
@@ -77,14 +77,14 @@ function rollPlan({ cellId, cap, protocol }) {
metadata_startup_script: startupScript({
cap,
image: ROLLBACK_IMAGE,
trusted: protocol === 1
trusted: protocol >= 1
})
},
after: {
metadata_startup_script: startupScript({
cap,
image: TARGET_IMAGE,
trusted: protocol === 1
trusted: protocol >= 1
}),
self_link: null
},
@@ -178,11 +178,9 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
})
it('validates a correct plan for every wave cell at that cell\'s rehome protocol', () => {
for (const cellId of SAME_CAP_CELLS) {
for (const [cellId, protocol] of SAME_CAP_CELLS.flatMap((cell) => [[cell, 1], [cell, 3]])) {
const [, cap] = resolveCellShape(cellId).stdout.trim().split(' ')
const protocol = REHOME_SOURCE_CELLS.has(cellId) ? 1 : 0
// Every reviewed serving cell carries rehome trust now, in either region.
assert.equal(protocol, 1, cellId)
assert.equal(REHOME_SOURCE_CELLS.has(cellId), true, cellId)
const config = {
mode: 'same-cap-cell',
cellId,
@@ -204,7 +202,7 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
assert.throws(
() => validateCapacityPlan(plan, {
...config,
regionalRehomeProtocol: String(1 - protocol)
regionalRehomeProtocol: '0'
}),
/reviewed image and capacity/,
cellId
@@ -239,3 +237,11 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
assert.doesNotMatch(capacityWorkflow, /--approved-cells/)
})
})
// Both trusted versions must prove the same authenticated drain boundary.
it('proves rehome trust for protocol 3 on forward and rollback rolls', () => {
const step = workflow.split('name: Prove exact per-host trust and idempotent no-neighbor behavior')[1].split('\n - name:')[0]
assert.match(step, /inputs\.rollback-rehome-protocol != '0'/)
assert.match(step, /inputs\.target-rehome-protocol != '0'/)
assert.match(step, /probe-relay-rehome-trust\.mjs/)
})
@@ -9,7 +9,7 @@ const REHOME_CONFIG =
// Only cells listed as regional rehome sources get rehome trust lines in their startup script.
function rehomeProtocol({ regionalRehomeProtocol }) {
if (![0, 1, '0', '1'].includes(regionalRehomeProtocol)) {
if (![0, 1, 3, '0', '1', '3'].includes(regionalRehomeProtocol)) {
throw new Error('same-cap Terraform plan has an invalid regional rehome protocol')
}
return Number(regionalRehomeProtocol)
@@ -43,7 +43,7 @@ export function parseCapacityPlanArguments(argv) {
(!values['rollback-image'] ||
!values['rehome-director-service-account'] ||
!values['rehome-audience'] ||
!['0', '1'].includes(values['regional-rehome-protocol']))
!['0', '1', '3'].includes(values['regional-rehome-protocol']))
) throw new Error('same-cap validation requires rollback image and rehome trust config')
if (values.mode !== 'same-cap-cell' && values['regional-rehome-protocol'] !== undefined) {
throw new Error('--regional-rehome-protocol applies only to same-cap-cell validation')
@@ -227,7 +227,7 @@ function requireDesiredStartupScript(script, config) {
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${config.capacityServiceAccount}'`
])
}
const rehomeTrusted = config.mode === 'same-cap-cell' && rehomeProtocol(config) === 1
const rehomeTrusted = config.mode === 'same-cap-cell' && rehomeProtocol(config) >= 1
if (rehomeTrusted) {
expected.push(
[
@@ -756,6 +756,10 @@ test('the rehome protocol argument is required by same-cap-cell mode alone', ()
.regionalRehomeProtocol,
'0'
)
assert.equal(
parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', '3')).regionalRehomeProtocol,
'3'
)
assert.throws(
() => parseCapacityPlanArguments(sameCapArguments()),
/requires rollback image and rehome trust config/
@@ -108,8 +108,8 @@ export function parseCapacityTransitionArguments(argv) {
const regionalRehomeProtocol = values['regional-rehome-protocol'] === undefined
? undefined
: integer(values['regional-rehome-protocol'], '--regional-rehome-protocol')
if (regionalRehomeProtocol !== undefined && ![0, 1].includes(regionalRehomeProtocol)) {
throw new Error('--regional-rehome-protocol must be 0 or 1')
if (regionalRehomeProtocol !== undefined && ![0, 1, 3].includes(regionalRehomeProtocol)) {
throw new Error('--regional-rehome-protocol must be 0, 1, or 3')
}
if (runtime === 'unavailable' && regionalRehomeProtocol !== undefined) {
throw new Error('unavailable runtime cannot prove the regional rehome protocol')
+71 -5
View File
@@ -466,11 +466,16 @@ After a deployment traffic shift, preserve the old revision/tag until metrics an
## Regional rehoming
Rehoming moves a host to a general cell in the region its desktop last reported, in either
direction. Both roles need the drain protocol: a cell without it can be neither a source nor a
target, and it is not part of the fleet whose telemetry gates the worker. Until the asia-east2
cells run `regionalRehomeProtocol` 1 they are none of the three, so no host is moved into or out
of Asia and an Asia cell in distress does not pause the worker.
Idle regional correction requires both source and target cells to advertise
`regionalRehomeProtocol >= 3`. PR #20105 introduced this capability version with
the idle handoff implementation. With that runtime, both rehome trust environment
settings must be configured to advertise 3; otherwise the cell advertises 0.
An older trusted runtime can advertise 1: configuring trust alone does not upgrade
its implementation. The separate `connectionCapacityProtocol: 2` health field does
not establish regional-correction readiness. Verify the live runtime version and
image, not only instance-template configuration, before rollout or enablement.
Incompatible cells are excluded from correction selection; enabling the cohort
cannot override this check. Director and cell deployments are separate operations.
`host-cooldown-ms` is the minimum gap between two rehomes of one host. It bounds the damage from
a desktop whose region probe flips: without it the host would be dragged back across the ocean on
@@ -491,3 +496,64 @@ Run and record each scenario in staging before launch:
- return a dormant host, overload a cell, kill a cell, evacuate active work, and exercise pre-registration rollback.
The served black-box relay suite validates the protocol/state transitions used by these procedures. The physical-device and real-GFE canaries remain separate launch gates; unit/black-box success cannot replace them.
## Optional measured region correction (deployment gated)
New optimization claims require both the durable regional-rehome control and
`ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT` (integer 0–100, default **0**).
Turning either gate off stops new optional moves; ordinary migration cleanup and
recovery continue. Legacy preferred-region hints do not certify a correction. Both
cells must advertise regional protocol3 and the authenticated desktop control must
advertise idle-regional-rehome-v1. The source must have no actual client sockets or
pending admission/control work; a live control socket alone does not prevent a move.
The monitor/deploy identity can read **GET `/v1/admin/regional-rehome-preview`**.
It returns full-population eligibility/exclusion counts, open-migration capacity,
process-safety gating and aggregate migration outcomes; it never claims a
host or changes the failure budget. This is advisory, with separately read state:
concurrent assignments, capacity changes, rate pauses and control changes can make
the next claim differ. Inspect the durable control separately before enabling.
Do not treat an unavailable/failed preview as zero eligible hosts.
`orca_relay_region_correction_outcomes` reports attempts by source/target,
registration/completion/abort state, oldest open age and target
reservation units every five minutes. `orca_relay_region_comparison` samples a
stable 10% of accepted reports (including unchanged hosts), keyed by host digest,
assignment epoch and decision generation. Existing control RTT and client-accept
logs include assignment epoch, control generation and drain mode; join those for
matched before/after and unchanged-cohort comparisons. Client accept latency is
connection setup, not application command round trip. No application-latency
improvement has been demonstrated by probe differences alone.
Quiet live connections count as work and defer optional correction indefinitely.
A returning client may race with the short admission gate and retry normally. No
optimization timer may close an established client. Investigate failed registration,
ambiguous authority, stuck reservations and reconnect/failure rates against agreed
limits. A database outage can keep the source fenced until locked reconciliation
establishes its authority; timeout alone is not permission to reopen admissions.
All directors must run the reviewed idle worker before enabling. Record the tested
immutable source and rollback revisions, then verify the ordinary migration recovery
path before rollout. There is no retained-source table or renewal protocol. Deploying
supporting cells/desktops and enabling a cohort require separate rollout authorization
and explicit numerical stop criteria; this change enables neither.
### Setting the correction cohort during a reviewed director rollout
The existing **Deploy Relay Production Director** workflow accepts
`region-correction-cohort-percent`: `preserve` (default) or an integer0–100.
It carries the cohort onto both candidate and compatible rollback revisions and
verifies the environment before promotion. If the predecessor has no setting,
`preserve` stamps zero. An explicit change requires the exact disabled durable
rehome generation; configuring a nonzero cohort does not itself enable the sweep.
The usual image, identity, health and traffic checks remain in force. No workflow
was dispatched as part of implementation.
Terraform reads the cohort from the same traffic-serving revision used to preserve
regional placement. A later apply therefore preserves a workflow-set cohort,
including explicit zero; only an absent service/setting bootstraps to0. Malformed
or ambiguous live settings fail the plan instead of silently resetting the cohort.
The audited director workflow owns subsequent changes.
Before the first nonzero cohort, verify compatible protocol2 cells, updated
cleanup workers, preview eligibility, both serving/rollback images and the
explicitly approved observation/stop criteria.
+6 -1
View File
@@ -112,7 +112,8 @@ durably marked consumed before mutation and cannot authorize another run.
| Director instances | outside 5–6 |
| Director CPU or memory | over 80% |
| Director concurrency | over 64 |
| Unexpected director 5xx or auth 5xx in five minutes | over 0 |
| Unexpected director 5xx in five minutes (excludes 503) | over 3 |
| Auth 5xx in five minutes | over 0 |
| Connections per cell process | over 500 |
| Queued bytes per cell process | over 48 MiB |
| Blocked or expired/unregistered migration | over 0 |
@@ -276,3 +277,7 @@ without its segment is a compile error in relay-contract, not a silent gap.
load the director's three-connection database pool.
- Added private atomic state, idempotent JSONL checkpoints, and secret-safe Markdown evidence.
- Added the manual production workflow. It has not been dispatched.
### Director error allowance (2026-09-12)
The serving-cell rollout observed three unexpected director 500 responses among approximately 33,600 responses in an hour, all two-second PostgreSQL connection timeouts. CPU remained near 30–37% and the zero-error bar repeatedly prevented any cell mutation. The five-minute allowance is now three non-503 director 5xx; four freezes. Auth errors, data freshness, active probes, SQL/pool pressure and other limits are unchanged. This is a bounded operational allowance, not a calibrated SLO or proof that intermittent failures are resolved; persistent low-frequency errors below this limit still require diagnosis.
+5
View File
@@ -169,6 +169,11 @@ resource "google_cloud_run_v2_service" "relay" {
}
}
env {
name = "ORCA_RELAY_REGION_CORRECTION_COHORT_PERCENT"
value = data.external.relay_serving_regional_placement_version.result.cohort_percent
}
ports {
container_port = 8080
}
@@ -26,7 +26,8 @@ export const PushNotificationSchema = z
agentState: PushAgentStateSchema.nullable(),
title: z.string().min(1).max(PUSH_LIMITS.titleMaxChars),
body: z.string().max(PUSH_LIMITS.bodyMaxChars),
worktreeId: z.string().min(1).max(2048).optional()
worktreeId: z.string().min(1).max(2048).optional(),
paneKey: z.string().min(1).max(2048).optional()
})
.strict()
.refine(
@@ -50,6 +50,7 @@ export const RELAY_HOST_CAPABILITIES_HEADER = 'x-orca-host-capabilities'
// The host accepts kind/relayDeviceId on a pendingConns entry. A host that does
// not advertise this parses those entries strictly and would drop the whole ack.
export const RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS = 'pending-conn-details'
export const RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME = 'idle-regional-rehome-v1'
export function parseRelayHostCapabilities(
header: string | string[] | undefined
@@ -121,11 +122,22 @@ export const DeviceRevokeSchema = z
.object({ reqId: OpaqueIdSchema, relayDeviceId: OpaqueIdSchema })
.strict()
export const AuthRefreshSchema = z.object({ relayJwt: z.string().min(1).max(8 * 1024) }).strict()
export const AuthRefreshSchema = z
.object({
relayJwt: z
.string()
.min(1)
.max(8 * 1024)
})
.strict()
export const DrainSchema = z
.object({
graceMs: z.number().int().nonnegative().max(60 * 60 * 1000),
graceMs: z
.number()
.int()
.nonnegative()
.max(60 * 60 * 1000),
recovery: z.literal('resolve-director')
})
.strict()
@@ -7,8 +7,15 @@ import {
RelayHostIdSchema
} from './wire-scalars.js'
import { RelayRegionSchema } from './relay-regions.js'
import {
RegionCorrectionRequestSchema,
RegionCorrectionResponseSchema
} from './region-correction.js'
const SignedAssignmentLeaseSchema = z.string().min(1).max(8 * 1024)
const SignedAssignmentLeaseSchema = z
.string()
.min(1)
.max(8 * 1024)
export const AssignmentRequestSchema = z
.object({
@@ -17,7 +24,8 @@ export const AssignmentRequestSchema = z
// Client-declared reconnection; the director verifies it against the
// durable assignment before granting fast-lane admission.
reconnect: z.boolean().optional(),
preferredRegion: RelayRegionSchema.optional()
preferredRegion: RelayRegionSchema.optional(),
regionCorrection: RegionCorrectionRequestSchema.optional()
})
.strict()
@@ -26,7 +34,8 @@ export const AssignmentResponseSchema = z
v: z.literal(1),
cellUrl: CanonicalHttpsOriginSchema,
assignmentEpoch: GenerationSchema,
lease: SignedAssignmentLeaseSchema
lease: SignedAssignmentLeaseSchema,
regionCorrection: RegionCorrectionResponseSchema.optional()
})
.strict()
@@ -0,0 +1,26 @@
import { z } from 'zod'
import { GenerationSchema, RelayHostIdSchema } from './wire-scalars.js'
export const IdleRegionalRehomeRequestSchema = z
.object({
v: z.literal(1),
attemptId: z.string().uuid(),
userId: z.string().min(1).max(256),
relayHostId: RelayHostIdSchema,
sourceCellId: z.string().min(1).max(128),
sourceCellIncarnation: z.string().uuid(),
sourceAssignmentEpoch: GenerationSchema.refine((value) => value > 0),
sourceGeneration: GenerationSchema.refine((value) => value > 0),
targetCellId: z.string().min(1).max(128)
})
.strict()
export const IdleRegionalRehomeResponseSchema = z
.object({
v: z.literal(1),
outcome: z.enum(['busy', 'committed', 'deferred', 'stale'])
})
.strict()
export type IdleRegionalRehomeRequest = z.infer<typeof IdleRegionalRehomeRequestSchema>
export type IdleRegionalRehomeOutcome = z.infer<typeof IdleRegionalRehomeResponseSchema>['outcome']
@@ -13,3 +13,5 @@ export * from './resume-confirmation-contract.js'
export * from './relay-regions.js'
export * from './splice-state-machine.js'
export * from './wire-scalars.js'
export * from './region-correction.js'
export * from './idle-regional-rehome.js'
@@ -0,0 +1,76 @@
import { describe, expect, it } from 'vitest'
import { AssignmentRequestSchema, AssignmentResponseSchema } from './director-messages.js'
import { DrainSchema } from './control-messages.js'
import { RegionCorrectionRequestSchema } from './region-correction.js'
const report = {
v: 1,
action: 'report',
generation: 3,
assignmentEpoch: 7,
policyVersion: 1,
outcome: 'conclusive',
measurements: { 'us-central1': 40, 'asia-east2': 180 }
}
const retention = {
mode: 'finish-existing',
attemptId: '11111111-1111-4111-8111-111111111111',
sourceGeneration: 3,
sourceAssignmentEpoch: 7
}
describe('region correction wire boundaries', () => {
it('keeps legacy assignment shapes readable without negotiated fields', () => {
expect(
AssignmentRequestSchema.parse({ v: 1, relayHostId: 'abcdefghijklmnop' })
).not.toHaveProperty('regionCorrection')
expect(
AssignmentResponseSchema.parse({
v: 1,
cellUrl: 'https://cell.example',
assignmentEpoch: 1,
lease: 'synthetic-lease'
})
).not.toHaveProperty('regionCorrection')
})
it('accepts complete comparison evidence and explicit inconclusive reports', () => {
expect(RegionCorrectionRequestSchema.safeParse(report).success).toBe(true)
const { measurements: _measurements, ...basis } = report
expect(
RegionCorrectionRequestSchema.safeParse({
...basis,
outcome: 'inconclusive',
reason: 'probe-unavailable'
}).success
).toBe(true)
})
it.each([
{ measurements: { 'us-central1': 40 } },
{ measurements: { 'us-central1': -1, 'asia-east2': 10 } },
{ measurements: { 'us-central1': Infinity, 'asia-east2': 10 } },
{ measurements: { 'us-central1': 120_001, 'asia-east2': 10 } },
{ generation: Number.MAX_SAFE_INTEGER + 1 },
{ assignmentEpoch: 1.2 },
{ policyVersion: 2 },
{ outcome: 'inconclusive', reason: 'timeout' }
])('rejects ambiguous or unbounded evidence: %j', (override) => {
expect(RegionCorrectionRequestSchema.safeParse({ ...report, ...override }).success).toBe(false)
})
it('rejects reporting and issuing a window in the same request', () => {
expect(
RegionCorrectionRequestSchema.safeParse({
...report,
action: 'issue-window'
}).success
).toBe(false)
})
it('uses ordinary drain and rejects the superseded retention extension', () => {
const ordinary = { graceMs: 0, recovery: 'resolve-director' }
expect(DrainSchema.parse(ordinary)).toEqual(ordinary)
expect(DrainSchema.safeParse({ ...ordinary, retention }).success).toBe(false)
})
})
@@ -0,0 +1,60 @@
import { z } from 'zod'
import { EpochMsSchema, GenerationSchema } from './wire-scalars.js'
import { RelayRegionSchema } from './relay-regions.js'
const RttSchema = z.number().finite().nonnegative().max(120_000)
export const RegionMeasurementsSchema = z
.object({
'us-central1': RttSchema,
'asia-east2': RttSchema
})
.strict()
export const RegionMeasurementWindowSchema = z
.object({
generation: GenerationSchema,
expiresAt: EpochMsSchema,
assignmentEpoch: GenerationSchema,
incumbentRegion: RelayRegionSchema,
policyVersion: z.literal(1)
})
.strict()
const ReportBasis = {
v: z.literal(1),
action: z.literal('report'),
generation: GenerationSchema,
assignmentEpoch: GenerationSchema,
policyVersion: z.literal(1)
}
export const RegionCorrectionRequestSchema = z.union([
z.object({ v: z.literal(1), action: z.literal('issue-window') }).strict(),
z
.object({
...ReportBasis,
outcome: z.literal('conclusive'),
measurements: RegionMeasurementsSchema
})
.strict(),
z
.object({
...ReportBasis,
outcome: z.literal('inconclusive'),
reason: z.string().min(1).max(64)
})
.strict()
])
export const RegionCorrectionResponseSchema = z
.object({
v: z.literal(1),
window: RegionMeasurementWindowSchema.optional(),
reportStatus: z.enum(['accepted', 'duplicate', 'stale', 'expired', 'basis-changed']).optional()
})
.strict()
export type RegionMeasurements = z.infer<typeof RegionMeasurementsSchema>
export type RegionMeasurementWindow = z.infer<typeof RegionMeasurementWindowSchema>
export type RegionCorrectionRequest = z.infer<typeof RegionCorrectionRequestSchema>
export type RegionCorrectionResponse = z.infer<typeof RegionCorrectionResponseSchema>
+13 -10
View File
@@ -279,16 +279,6 @@ module.exports = {
}
},
afterPack: async (context) => {
// Why: a Linux runner-image glibc bump silently shipped a node-pty pty.node
// requiring GLIBC_2.34, crashing the app on startup on Ubuntu 20.04 (#9902).
// Fail packaging if any bundled native binary exceeds the supported floor.
if (context.electronPlatformName === 'linux') {
// Why the arch is passed: symbol-version checks pass happily on a wrong-architecture binary,
// so a cross-built slice could ship the host's pty.node and only fail at runtime.
verifyLinuxGlibcFloor(context.appOutDir, {
targetArch: { 1: 'x64', 3: 'arm64' }[context.arch]
})
}
const resourcesDir =
context.electronPlatformName === 'darwin'
? join(
@@ -326,6 +316,19 @@ module.exports = {
}
stampPackagedCliVersion(resourcesDir, context.packager.appInfo.version)
prunePackagedRuntimeNodeModules(resourcesDir, context.electronPlatformName, context.arch)
// Why: a Linux runner-image glibc bump silently shipped a node-pty pty.node
// requiring GLIBC_2.34, crashing the app on startup on Ubuntu 20.04 (#9902).
// Fail packaging if any bundled native binary exceeds the supported floor.
// Why after the prune: cross-builds intentionally install every optional
// native variant, so an arm64 slice still carries the x64 @parcel/watcher
// until prunePackagedRuntimeNodeModules drops it.
if (context.electronPlatformName === 'linux') {
// Why the arch is passed: symbol-version checks pass happily on a wrong-architecture binary,
// so a cross-built slice could ship the host's pty.node and only fail at runtime.
verifyLinuxGlibcFloor(context.appOutDir, {
targetArch: { 1: 'x64', 3: 'arm64' }[context.arch]
})
}
verifyPackagedMainRuntimeDeps(resourcesDir)
// Why: boot the packaged daemon-entry under plain Node, but only for the
// slice matching the packaging host's arch — daemon-entry.js is JS, yet it
+224
View File
@@ -10,6 +10,158 @@
}
},
"gates": [
{
"id": "runtime.connection-owned-host-status",
"title": "Host status recovers with its owning connection",
"maturity": "experimental",
"protection": "partial",
"owner": "runtime",
"layer": "service-integration-and-e2e",
"surfaces": [
"sidebar host status",
"desktop runtime connection",
"browser primary connection"
],
"platforms": ["macos", "linux", "windows"],
"providers": ["remote-runtime"],
"coveredPlatforms": ["macos"],
"coveredProviders": ["remote-runtime"],
"coverageNotes": "Real authenticated sockets plus isolated desktop and headless hosts with desktop and browser viewers; deterministic lifecycle tests cover stale results and reader deadlines.",
"motivatingLinks": ["https://github.com/stablyai/orca/pull/19163"],
"invariant": "Failed bootstrap and authenticated reconnect converge without UI triggers; one connection owner publishes verified status, with no independent healthy status polling.",
"oracle": "Observe automatic recovery, retained runtime identity on failure, ordered publications, exact request counts, isolated viewer outages, and retirement on disconnect.",
"commands": [
"ORCA_BACKGROUND_LAUNCH=1 pnpm test src/shared/runtime-host-status-owner.test.ts src/main/ipc/runtime-environment-status-recovery.test.ts src/main/ipc/runtime-environment-status-connection.test.ts src/renderer/src/store/slices/runtime-status-snapshot.test.ts src/renderer/src/web/web-runtime-status-owner.test.ts",
"ORCA_BACKGROUND_LAUNCH=1 ORCA_E2E_WEB_CLIENT=1 pnpm exec playwright test tests/e2e/runtime-host-status-recovery.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1"
],
"testFiles": [
"src/shared/runtime-host-status-owner.test.ts",
"src/main/ipc/runtime-environment-status-recovery.test.ts",
"src/main/ipc/runtime-environment-status-connection.test.ts",
"src/renderer/src/store/slices/runtime-status-snapshot.test.ts",
"src/renderer/src/web/web-runtime-status-owner.test.ts",
"tests/e2e/runtime-host-status-recovery.spec.ts"
],
"assertionRefs": [
{
"file": "src/main/ipc/runtime-environment-status-recovery.test.ts",
"assertions": [
"recovers a saved host after its first status check fails, without another UI request"
]
}
],
"evidenceRuns": [
{
"date": "2026-09-10",
"runner": "local",
"platform": "macos",
"command": "ORCA_BACKGROUND_LAUNCH=1 ORCA_E2E_WEB_CLIENT=1 pnpm exec playwright test tests/e2e/runtime-host-status-recovery.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1",
"result": "passed",
"summary": "Desktop-host and headless-host journeys passed with desktop and browser viewers.",
"durationSeconds": 31.3
}
],
"runtimeBudget": {
"p95Seconds": 180,
"scope": "Target excluding builds; measured p95 not established."
},
"flakeHistory": {
"status": "soaking",
"evidence": "Local candidate runs passed; no sustained CI history yet."
},
"redGreenEvidence": {
"status": "partial",
"evidence": "First-status-failure oracle failed on main 58ff95becb40 (one request instead of two) and passes on the candidate. E2E verifies candidate recovery, not a baseline comparison."
},
"performanceBudget": {
"required": false,
"evidence": "Deterministic tests assert one shared request and no healthy owner polling."
},
"promotionCriteria": ["Collect repeated CI runs without unexplained flakes."],
"knownGaps": [
"No live Linux, Windows, SSH, or mixed-version pair validation.",
"TCP interruption exercises reconnect, not a full real host process restart.",
"The outage begins on the first saved-host check, not by relaunching a preseeded desktop profile."
],
"demotionRule": "Keep experimental until repeated runs establish reliability; preserve request-count and lifecycle assertions."
},
{
"id": "mobile-push.headless-startup-and-policy",
"title": "Headless push lifecycle and mobile delivery policy",
"maturity": "experimental",
"protection": "partial",
"owner": "runtime",
"layer": "service-integration",
"surfaces": ["headless startup", "push registration", "native push delivery policy"],
"platforms": ["macos", "linux", "windows"],
"providers": ["local", "remote-runtime"],
"coveredPlatforms": ["macos"],
"coveredProviders": ["local", "remote-runtime"],
"coverageNotes": "Actual startOrcad entry with mocked daemon/RPC startup boundaries; real controller, push service, and persisted device registry. Gateway send is stubbed.",
"motivatingLinks": ["https://github.com/stablyai/orca/pull/19204"],
"invariant": "Headless startup installs and disposes push delivery; desktop notification categories remain authoritative, the three-minute away policy is preserved, and host activity cannot extend the seven-day mobile lease.",
"oracle": "Require registration after RPC identity initialization and shutdown cleanup; idle 179/180/0 yields false/true/false in retained event metadata and exactly one gateway push. Legacy socket subscriptions preserve notification filtering, while opted-in push clients can request dismissal reconciliation. Desktop categories remain authoritative. Persisted lease expires exactly at seven days and only explicit registration renews it.",
"commands": [
"ORCA_BACKGROUND_LAUNCH=1 pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/mobile-notification-dismissal-store.test.ts src/renderer/src/hooks/useAutoAckViewedAgent.away.test.ts",
"ORCA_BACKGROUND_LAUNCH=1 pnpm exec vitest run --config config/vitest.config.ts src/main/orcad/orcad-push-startup.test.ts src/main/runtime/push/push-policy-pipeline.integration.test.ts"
],
"testFiles": [
"src/main/orcad/orcad-push-startup.test.ts",
"src/main/runtime/push/push-policy-pipeline.integration.test.ts",
"src/main/runtime/mobile-notification-dismissal-store.test.ts",
"src/renderer/src/hooks/useAutoAckViewedAgent.away.test.ts"
],
"assertionRefs": [
{
"file": "src/main/orcad/orcad-push-startup.test.ts",
"assertions": [
"starts push after RPC identity is available and stops dispatch on shutdown"
]
},
{
"file": "src/main/runtime/push/push-policy-pipeline.integration.test.ts",
"assertions": [
"carries the native idle boundary through replay and push dispatch",
"expires persisted registration at seven days despite host activity and renews explicitly"
]
}
],
"evidenceRuns": [
{
"date": "2026-09-07",
"runner": "local",
"platform": "macos",
"command": "ORCA_BACKGROUND_LAUNCH=1 pnpm exec vitest run --config config/vitest.config.ts src/main/orcad/orcad-push-startup.test.ts src/main/runtime/push/push-policy-pipeline.integration.test.ts",
"result": "passed",
"summary": "Four tests passed across two files.",
"durationSeconds": 0.407
}
],
"runtimeBudget": {
"p95Seconds": 10,
"scope": "Target budget; measured p95 not established"
},
"flakeHistory": {
"status": "soaking",
"evidence": "Focused local run passed; repeated CI history not established."
},
"redGreenEvidence": {
"status": "partial",
"evidence": "Headless startup oracle reproduced missing registrar before the startup fix and passed afterward. Policy tests cover candidate behavior."
},
"performanceBudget": {
"required": false,
"evidence": "Lifecycle and policy coverage; asserts exact gateway send counts and zero remaining dispatch listeners after shutdown."
},
"promotionCriteria": ["Collect repeated CI runs without unexplained failures."],
"knownGaps": [
"Does not prove APNs silent background wakeup or actual operating-system idle transitions.",
"Rendererless agent/bell event generation remains outside the documented feature contract.",
"No live Windows or Linux policy evidence.",
"Mobile native presentation and dismissal integration belongs to the subsequent mobile PR."
],
"demotionRule": "Keep experimental if lifecycle or policy assertions fail; do not weaken them to bypass platform delivery gaps."
},
{
"id": "agent-session.completed-turn-duration",
"title": "Completed turn duration survives client recovery and history pagination",
@@ -2775,6 +2927,78 @@
],
"demotionRule": "Keep experimental or demote if assignment calls overlap, duplicate drain events bypass backoff, Retry-After is ignored, close resurrects work, or mixed-version request rate exceeds the reviewed director budget."
},
{
"id": "desktop-relay.region-correction-idle-cutover",
"title": "Regional correction moves only an idle relay source",
"maturity": "experimental",
"protection": "partial",
"owner": "desktop-runtime",
"layer": "cell-desktop-real-websocket",
"surfaces": ["regional correction", "idle source cutover", "desktop relay reconnect"],
"platforms": ["macos", "linux", "windows"],
"providers": ["cloud-relay"],
"coveredPlatforms": ["macos"],
"coveredProviders": ["cloud-relay"],
"coverageNotes": "Real local WebSocket/control/proof/splice traffic and production SQLite store run with synthetic clock and synthetic token verification. Separate PostgreSQL16 suites validate SQL concurrency. This does not measure production network latency, physical phones, UI, or the production token issuer.",
"motivatingLinks": [
"docs/relay-region-correction/RELAY-REGION-CORRECTION-IDLE-CUTOVER-PLAN.md"
],
"invariant": "Regional optimization never closes an established relay client. The source gates admissions only after actual work is idle, commits the exact assignment atomically, and releases its empty control. A failed target uses ordinary migration recovery; previously sent mutations are not replayed.",
"oracle": "Two real TCP WebSocket cells, the actual desktop origin pool, SQLite and an independent execution child verify busy phone/iPad deferral, idle movement, racing arrival rejection, definite-abort admission recovery and observed target-registration failure with ordinary rollback.",
"commands": [
"ORCA_BACKGROUND_LAUNCH=1 pnpm test tests/e2e/relay-region-correction.unit.test.ts"
],
"testFiles": ["tests/e2e/relay-region-correction.unit.test.ts"],
"assertionRefs": [
{
"file": "tests/e2e/relay-region-correction.unit.test.ts",
"assertions": [
"releases the empty source and recovers normally when the target never registers",
"rejects an arrival during cutover and restores admissions after a definite failed commit",
"defers for either connected device, then moves after both disconnect without replaying work"
]
}
],
"evidenceRuns": [
{
"date": "2026-09-11",
"runner": "local",
"platform": "macos",
"command": "ORCA_BACKGROUND_LAUNCH=1 pnpm test tests/e2e/relay-region-correction.unit.test.ts",
"result": "passed",
"durationSeconds": 6.44,
"summary": "Three real TCP WebSocket cases passed after removing store retention. Log: .tmp/idle-cutover-review/transport-without-retention.log. Does not validate packaged or physical clients."
}
],
"runtimeBudget": {
"p95Seconds": 180,
"scope": "three local real-WebSocket scenarios with synthetic elapsed time for ordinary recovery"
},
"flakeHistory": {
"status": "unknown",
"evidence": "Local implementation validation; no CI soak history yet."
},
"redGreenEvidence": {
"status": "partial",
"evidence": "Registry admission accounting negative control and five conflicting operation-identity regressions fail before their fixes and pass afterward. Locked database reconciliation has separate red/green evidence. Full cross-layer counterfactual remains unverified."
},
"performanceBudget": {
"required": true,
"evidence": "No retained source lease or new mobile timer. Candidate selection is read-only; busy sources immediately defer. Idle cutover reuses normal desktop reconnect and existing migration recovery."
},
"promotionCriteria": [
"Collect 100 consecutive CI passes or 14 days of soak.",
"Complete mixed-version and packaged-client validation.",
"Validate bounded rollout latency and reliability against reviewed numerical limits."
],
"knownGaps": [
"Production authentication verifier is mocked.",
"Synthetic elapsed time is not a wall-clock soak.",
"Physical phone lifecycle, packaged mixed versions, SSH execution and production network behavior require separate validation."
],
"demotionRule": "Keep experimental or demote if optimization closes an established client, a gate reopens on ambiguous authority, a mutation is replayed, cleanup is lost, or eligible idle hosts starve."
},
{
"id": "git-worktree.refresh-event-semantics",
"title": "Index-only Git metadata cannot trigger structural worktree refresh fanout",
@@ -0,0 +1,283 @@
/**
* Records a live agent CLI session through a real PTY into a test fixture, bytes intact.
*
* Why a PTY and not `agy | tee`: a pipe is not a terminal, so the CLI renders its
* non-interactive path — no alternate screen, no caret, no dialogs. The detector under
* test only ever sees the PTY shape, so that is the only shape worth capturing.
*
* Nothing here strips escapes, folds CRs, or rewraps lines: the transcript is written
* exactly as the terminal received it. See docs/reference/agent-pty-transcript-capture.md.
*/
import { createWriteStream, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
import { dirname, join, resolve } from 'node:path'
import { pathToFileURL } from 'node:url'
import {
formatFindings,
redactTranscript,
scanTranscriptForSecrets
} from './pty-transcript-secret-scan.mjs'
const REPO_ROOT = resolve(import.meta.dirname, '..', '..')
const FIXTURE_DIR = join(REPO_ROOT, 'src', 'main', 'runtime', '__fixtures__')
const STOP_KEY = 0x1d // Ctrl-], consumed by the recorder and never forwarded to the agent.
const NAME_RE = /^[a-z0-9][a-z0-9-]*$/
const USAGE = `Capture a raw agent PTY transcript into src/main/runtime/__fixtures__/.
node config/scripts/capture-agent-pty-transcript.mjs --name <fixture-name> [options] -- <command> [args...]
node config/scripts/capture-agent-pty-transcript.mjs --scan <file...> [--redact]
Options
--name <fixture-name> Output fixture name, e.g. antigravity-ready-personal-non-gemini
--out <path> Write somewhere other than the fixture directory
--cols <n> --rows <n> Pin the PTY size (default: this terminal's size, else 120x40)
--duration <seconds> Stop unattended after N seconds
--send "<ms>:<text>" Type <text> into the PTY at <ms> (repeatable; \\r \\n \\t \\e escapes)
--note "<text>" Recorded in the <name>.meta.json sidecar
--scan <file...> Scan existing transcripts for identifiers/credentials and exit
--redact With --scan: rewrite each finding as a same-length placeholder
Press Ctrl-] to end a capture. That key is consumed here, so the agent keeps whatever
dialog it is showing — which is the only way to capture a dialog that owns the screen.`
function parseArgs(argv) {
const options = { cols: null, rows: null, duration: null, scan: [], sends: [], redact: false }
const command = []
let cursor = 0
let afterSeparator = false
while (cursor < argv.length) {
const arg = argv[cursor]
if (afterSeparator) {
command.push(arg)
cursor += 1
continue
}
if (arg === '--') {
afterSeparator = true
} else if (arg === '--redact') {
options.redact = true
} else if (arg === '--help' || arg === '-h') {
options.help = true
} else if (arg === '--scan') {
while (cursor + 1 < argv.length && !argv[cursor + 1].startsWith('--')) {
cursor += 1
options.scan.push(argv[cursor])
}
} else if (arg === '--send') {
cursor += 1
options.sends.push(parseSend(argv[cursor]))
} else if (arg.startsWith('--')) {
const key = arg.slice(2)
cursor += 1
options[key] = argv[cursor]
}
cursor += 1
}
for (const key of ['cols', 'rows', 'duration']) {
options[key] = options[key] == null ? null : Number(options[key])
}
return { options, command }
}
// String.fromCharCode, not a literal: the formatter rewrites an escape sequence into a raw
// control byte in source, which is unreadable and survives badly in diffs.
const ESC = String.fromCharCode(27)
const SEND_ESCAPES = { r: '\r', n: '\n', t: '\t', e: ESC, '\\': '\\' }
/** `"<ms>:<text>"` — a keystroke to deliver at a fixed offset, for an unattended dialog capture. */
function parseSend(value) {
const separator = String(value ?? '').indexOf(':')
if (separator === -1) {
throw new Error(`--send expects "<ms>:<text>", got ${String(value)}`)
}
const atMs = Number(value.slice(0, separator))
if (!Number.isFinite(atMs)) {
throw new Error(
`--send delay must be a number of milliseconds, got ${value.slice(0, separator)}`
)
}
const text = value
.slice(separator + 1)
.replace(/\\(.)/g, (whole, code) => SEND_ESCAPES[code] ?? whole)
return { atMs, text }
}
function runScan(files, redact) {
let failed = false
for (const file of files) {
const path = resolve(file)
const text = readFileSync(path, 'utf8')
if (redact) {
const { text: redacted, redacted: count } = redactTranscript(text)
writeFileSync(path, redacted)
console.log(`${file}: redacted ${count} span(s) in place, same length each.`)
continue
}
const findings = scanTranscriptForSecrets(text)
console.log(formatFindings(file, findings))
failed ||= findings.length > 0
}
return failed ? 1 : 0
}
function resolveSpawn(command) {
// node-pty cannot run a .cmd/.bat shim directly on Windows; those need cmd.exe.
if (process.platform === 'win32' && /\.(cmd|bat)$/i.test(command[0])) {
return { file: 'cmd.exe', args: ['/c', `"${command[0]}"`, ...command.slice(1)] }
}
return { file: command[0], args: command.slice(1) }
}
async function runCapture(options, command) {
const name = options.name
if (typeof name === 'string' && !NAME_RE.test(name)) {
console.error(`--name must be lowercase kebab-case; got ${name}`)
return 2
}
const outPath = options.out ? resolve(options.out) : join(FIXTURE_DIR, `${name}.txt`)
mkdirSync(dirname(outPath), { recursive: true })
const pty = await import('node-pty').catch((error) => {
console.error(
`node-pty failed to load. Build it for plain node first:
node config/scripts/ensure-native-runtime.mjs --runtime=node
${String(error)}`
)
return null
})
if (pty === null) {
return 2
}
const cols = options.cols ?? process.stdout.columns ?? 120
const rows = options.rows ?? process.stdout.rows ?? 40
const { file, args } = resolveSpawn(command)
const term = pty.spawn(file, args, {
name: 'xterm-256color',
cols,
rows,
cwd: process.cwd(),
env: { ...process.env, TERM: 'xterm-256color' },
encoding: null
})
const sink = createWriteStream(outPath)
let recording = true
term.onData((chunk) => {
const bytes = typeof chunk === 'string' ? Buffer.from(chunk, 'utf8') : chunk
// Why recording stops before the kill: an agent repaints an idle frame on its way out, so
// a transcript that keeps writing through shutdown ends on that frame instead of on the
// state you stopped to capture. A mid-turn or dialog capture cannot survive that.
if (recording) {
sink.write(bytes)
}
process.stdout.write(bytes)
})
const wasRaw = process.stdin.isTTY === true && process.stdin.isRaw === true
if (process.stdin.isTTY) {
process.stdin.setRawMode(true)
}
process.stdin.resume()
let stopping = false
const stop = () => {
if (stopping) {
return
}
stopping = true
recording = false
try {
term.kill()
} catch {
// The agent may have exited on its own; the transcript is already on disk.
}
}
process.stdin.on('data', (chunk) => {
if (chunk.includes(STOP_KEY)) {
stop()
return
}
term.write(chunk.toString('binary'))
})
// Why scripted input: a dialog capture has to be driven, and CI (or an agent) has no TTY to
// type into. The keystrokes ride the same PTY a human's would, so the capture is unchanged.
const sendTimers = options.sends.map((send) => setTimeout(() => term.write(send.text), send.atMs))
const durationTimer = options.duration === null ? null : setTimeout(stop, options.duration * 1000)
const exitCode = await new Promise((resolveExit) => {
term.onExit(({ exitCode: code }) => resolveExit(code ?? 0))
})
for (const timer of sendTimers) {
clearTimeout(timer)
}
if (durationTimer !== null) {
clearTimeout(durationTimer)
}
if (process.stdin.isTTY) {
process.stdin.setRawMode(wasRaw)
}
process.stdin.pause()
await new Promise((done) => sink.end(done))
writeMeta(outPath, { command, cols, rows, note: options.note ?? null, exitCode })
const findings = scanTranscriptForSecrets(readFileSync(outPath, 'utf8'))
console.log(`\nTranscript: ${outPath}`)
console.log(formatFindings('scrub check', findings))
if (findings.length > 0) {
console.log(
`Scrub with:
node config/scripts/capture-agent-pty-transcript.mjs --scan ${outPath} --redact`
)
}
return 0
}
function writeMeta(outPath, details) {
const metaPath = outPath.replace(/\.txt$/, '.meta.json')
writeFileSync(
metaPath,
`${JSON.stringify(
{
capturedAt: new Date().toISOString(),
platform: process.platform,
command: details.command,
cols: details.cols,
rows: details.rows,
note: details.note,
exitCode: details.exitCode
},
null,
2
)}\n`
)
}
async function main() {
const { options, command } = parseArgs(process.argv.slice(2))
if (options.help === true) {
console.log(USAGE)
return 0
}
if (options.scan.length > 0) {
return runScan(options.scan, options.redact)
}
if (command.length === 0 || (options.name === undefined && options.out === undefined)) {
console.error(USAGE)
return 2
}
return runCapture(options, command)
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
main().then(
(code) => {
process.exitCode = code
},
(error) => {
console.error(error)
process.exitCode = 1
}
)
}
export { parseArgs, resolveSpawn }
@@ -2,7 +2,7 @@ import { readFileSync, readdirSync } from 'node:fs'
import { cp, mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises'
import { createRequire } from 'node:module'
import { tmpdir } from 'node:os'
import { dirname, join, relative, resolve } from 'node:path'
import { delimiter, dirname, join, relative, resolve } from 'node:path'
import { describe, expect, it } from 'vitest'
const require = createRequire(import.meta.url)
@@ -387,6 +387,72 @@ describe('packaged runtime resources', () => {
}
})
it.skipIf(process.platform === 'win32')(
'prunes non-target native packages before the Linux glibc gate',
async () => {
const root = await mkdtemp(join(tmpdir(), 'orca-after-pack-prune-order-'))
const previousPath = process.env.PATH
try {
const appOutDir = join(root, 'linux-unpacked')
const resourcesDir = join(appOutDir, 'resources')
await cp(
join(process.cwd(), 'resources', 'plugins', 'launch'),
join(resourcesDir, 'plugins', 'launch'),
{ recursive: true }
)
const unpackedMainDir = join(resourcesDir, 'app.asar.unpacked', 'out', 'main')
await mkdir(unpackedMainDir, { recursive: true })
await writeFile(join(unpackedMainDir, 'daemon-entry.js'), '', 'utf8')
await writeFile(
join(resourcesDir, 'app.asar.unpacked', 'out', 'package.json'),
`${JSON.stringify({ name: 'orca-compiled-output', type: 'commonjs', private: true })}\n`,
'utf8'
)
const unpackedCliDir = join(resourcesDir, 'app.asar.unpacked', 'out', 'cli')
await mkdir(join(unpackedCliDir, 'handlers'), { recursive: true })
await writeFile(join(unpackedCliDir, 'handlers', 'skills.js'), '', 'utf8')
await writeFile(join(unpackedCliDir, 'index.js'), '', 'utf8')
const target =
process.arch === 'x64'
? { electronArch: 3, machine: 0xb7, nonTarget: 'x64' }
: { electronArch: 1, machine: 0x3e, nonTarget: 'arm64' }
const wrongArchPackage = join(
resourcesDir,
'node_modules',
'@parcel',
`watcher-linux-${target.nonTarget}-glibc`
)
await mkdir(wrongArchPackage, { recursive: true })
const wrongArchElf = Buffer.alloc(20)
wrongArchElf.set([0x7f, 0x45, 0x4c, 0x46])
wrongArchElf[5] = 1
wrongArchElf.writeUInt16LE(target.machine, 18)
await writeFile(join(wrongArchPackage, 'watcher.node'), wrongArchElf)
const stubBinDir = join(root, 'bin')
await mkdir(stubBinDir)
await writeFile(join(stubBinDir, 'objdump'), '#!/bin/sh\nexit 0\n', { mode: 0o755 })
process.env.PATH = `${stubBinDir}${delimiter}${previousPath ?? ''}`
await expect(
electronBuilderConfig.afterPack({
appOutDir,
electronPlatformName: 'linux',
arch: target.electronArch,
packager: { appInfo: { version: '9.9.9' } }
})
).resolves.toBeUndefined()
await expect(stat(wrongArchPackage)).rejects.toMatchObject({ code: 'ENOENT' })
} finally {
process.env.PATH = previousPath
await rm(root, { recursive: true, force: true })
}
}
)
it.skipIf(process.platform === 'win32')(
'marks packaged Unix CLI launchers executable',
async () => {
@@ -0,0 +1,57 @@
import { readFileSync } from 'node:fs'
import { join } from 'node:path'
import { describe, expect, it } from 'vitest'
/**
* Why a floor and not just a pin: Electron 43.5.0/43.6.0 set and unset `GDK_GL`
* around `gtk_init()` while FontConfig warmed up on a pool thread, and below
* glibc 2.41 that frees `environ` under a concurrent `getenv()` — a launch-time
* use-after-free on every Ubuntu we support (stablyai/orca#20081). 43.7.0 stops
* freeing the published `environ`. A downgrade past it re-ships that crash, and
* nothing else in the tree would notice.
*/
const MINIMUM_ELECTRON_VERSION = '43.7.0'
function parseVersion(specifier: string): [number, number, number] {
const match = /(\d+)\.(\d+)\.(\d+)/.exec(specifier)
if (!match) {
throw new Error(`unparseable Electron version: ${specifier}`)
}
return [Number(match[1]), Number(match[2]), Number(match[3])]
}
function meetsRuntimeFloor(specifier: string): boolean {
const version = parseVersion(specifier)
const floor = parseVersion(MINIMUM_ELECTRON_VERSION)
for (const [index, part] of version.entries()) {
if (part !== floor[index]) {
return part > floor[index]
}
}
return true
}
describe('electron runtime floor', () => {
it.each([
['42.9.0', false],
['43.6.0', false],
['43.7.0', true],
['43.7.1', true],
['43.8.0', true],
['44.0.0', true]
])('reads %s as meeting the floor: %s', (specifier, expected) => {
expect(meetsRuntimeFloor(specifier)).toBe(expected)
})
it('pins Electron at or above the glibc environ-race fix', () => {
const packageJson = JSON.parse(
readFileSync(join(__dirname, '../../package.json'), 'utf-8')
) as { devDependencies: Record<string, string> }
const specifier = packageJson.devDependencies.electron
expect(
meetsRuntimeFloor(specifier),
`electron ${specifier} is below the ${MINIMUM_ELECTRON_VERSION} runtime floor`
).toBe(true)
})
})
@@ -0,0 +1,250 @@
// Why: the host registry is the only place that binds a method name to its params
// schema. Reading it back — instead of hand-listing 600 methods — is what keeps the
// shared catalog and the dispatcher from drifting apart.
import { execFileSync } from 'node:child_process'
import {
existsSync,
globSync,
mkdirSync,
mkdtempSync,
readFileSync,
rmSync,
writeFileSync
} from 'node:fs'
import { createRequire } from 'node:module'
import path from 'node:path'
import process from 'node:process'
import * as esbuild from 'esbuild'
import { resolveOxcCliInvocation } from './oxc-cli-invocation.mjs'
const REPO_ROOT = path.resolve(import.meta.dirname, '..', '..')
const SHARED_DIR = path.join(REPO_ROOT, 'src', 'shared')
const CONTRACT_DIR = path.join(SHARED_DIR, 'rpc-contract')
const RPC_DIR = path.join(REPO_ROOT, 'src', 'main', 'runtime', 'rpc')
const REGISTRY_ENTRY = path.join(RPC_DIR, 'methods', 'index.ts')
const OUTPUT_PATH = path.join(CONTRACT_DIR, 'rpc-params-catalog.generated.ts')
// Why mkdirSync first: out/ is gitignored and absent on a fresh checkout, so
// mkdtempSync threw ENOENT and took `pnpm lint` down with it. Why not os.tmpdir():
// the bundle keeps its node_modules deps external and oxfmt reads .oxfmtrc.json by
// walking up, so both scratch files have to sit under the repo to resolve at all.
function scratchDir(prefix) {
const root = path.join(REPO_ROOT, 'out')
mkdirSync(root, { recursive: true })
return mkdtempSync(path.join(root, prefix))
}
const posix = (value) => value.split(path.sep).join('/')
const repoPath = (absolute) => posix(path.relative(REPO_ROOT, absolute))
// Every module the catalog may import from: the extracted params modules plus the
// pre-existing src/shared schemas the RPC methods already bind directly.
function indexableModules() {
const modules = new Set(
globSync('*.ts', { cwd: CONTRACT_DIR }).map((name) => path.join(CONTRACT_DIR, name))
)
modules.delete(OUTPUT_PATH)
for (const file of globSync('**/*.ts', { cwd: RPC_DIR })) {
if (file.endsWith('.test.ts')) {
continue
}
const source = readFileSync(path.join(RPC_DIR, file), 'utf8')
for (const [, specifier] of source.matchAll(/from\s+'(\.[^']+)'/g)) {
const resolved = `${path.resolve(path.dirname(path.join(RPC_DIR, file)), specifier)}.ts`
if (resolved.startsWith(`${SHARED_DIR}${path.sep}`) && existsSync(resolved)) {
modules.add(resolved)
}
}
}
return [...modules].sort()
}
// Why: one bundle keeps the registry and the shared modules on the same module
// instances, so schema object identity is what maps a method to its export.
function loadRegistryAndSchemas(modules) {
const buildDir = scratchDir('rpc-params-catalog-')
try {
const entry = path.join(buildDir, 'entry.ts')
const importOf = (file) => JSON.stringify(posix(path.relative(buildDir, file)))
writeFileSync(
entry,
[
`export { ALL_RPC_METHODS } from ${importOf(REGISTRY_ENTRY)}`,
'export const SCHEMA_MODULES = {',
...modules.map(
(file) => ` ${JSON.stringify(repoPath(file))}: require(${importOf(file)}),`
),
'}'
].join('\n')
)
const outfile = path.join(buildDir, 'bundle.cjs')
esbuild.buildSync({
entryPoints: [entry],
bundle: true,
platform: 'node',
format: 'cjs',
outfile,
logLevel: 'error',
packages: 'external'
})
const loaded = createRequire(import.meta.url)(outfile)
return { methods: loaded.ALL_RPC_METHODS, schemaModules: loaded.SCHEMA_MODULES }
} finally {
rmSync(buildDir, { recursive: true, force: true })
}
}
// Why: schema objects are compared by identity, not by shape — two structurally
// identical schemas are still two different wire contracts.
function buildSchemaIndex(schemaModules) {
const index = new Map()
for (const [modulePath, moduleExports] of Object.entries(schemaModules)) {
for (const [exportName, value] of Object.entries(moduleExports)) {
if (!value || typeof value !== 'object' || typeof value.safeParse !== 'function') {
continue
}
if (index.has(value)) {
continue
}
index.set(value, { modulePath, exportName })
}
}
return index
}
function localNameFor(origin, taken) {
if (!taken.has(origin.exportName)) {
return origin.exportName
}
const hint = path
.basename(origin.modulePath, '.ts')
.split('-')
.map((part) => part.charAt(0).toUpperCase() + part.slice(1))
.join('')
let candidate = `${origin.exportName}Of${hint}`
let suffix = 2
while (taken.has(candidate)) {
candidate = `${origin.exportName}Of${hint}${suffix++}`
}
return candidate
}
function render({ methods, schemaModules }) {
const index = buildSchemaIndex(schemaModules)
const entries = []
const uncataloged = []
const imports = new Map()
const taken = new Set()
for (const method of [...methods].sort((left, right) => (left.name < right.name ? -1 : 1))) {
if (method.params === null) {
entries.push(` '${method.name}': null`)
continue
}
const origin = index.get(method.params)
if (!origin) {
uncataloged.push(method.name)
continue
}
const key = `${origin.modulePath}#${origin.exportName}`
let local = imports.get(key)
if (!local) {
local = localNameFor(origin, taken)
taken.add(local)
imports.set(key, local)
}
entries.push(` '${method.name}': ${local}`)
}
const byModule = new Map()
for (const [key, local] of imports) {
const [modulePath, exportName] = key.split('#')
if (!byModule.has(modulePath)) {
byModule.set(modulePath, [])
}
byModule.get(modulePath).push(local === exportName ? exportName : `${exportName} as ${local}`)
}
const importLines = [...byModule]
.sort(([left], [right]) => (left < right ? -1 : 1))
.map(([modulePath, names]) => {
let specifier = posix(path.relative(CONTRACT_DIR, path.join(REPO_ROOT, modulePath))).replace(
/\.ts$/,
''
)
if (!specifier.startsWith('.')) {
specifier = `./${specifier}`
}
return `import { ${names.sort().join(', ')} } from '${specifier}'`
})
return `// GENERATED by config/scripts/generate-rpc-params-catalog.mjs. Do not edit;
// run \`pnpm run generate:rpc-params-catalog\`.
import type { z } from 'zod'
${importLines.join('\n')}
// Why: the host parses params with these schemas, so a client that matches this map
// matches the dispatcher. Clients must import it for types only — parsing a params
// schema client-side runs the coercing transforms and rewrites the wire bytes.
export const RPC_PARAMS_BY_METHOD = {
${entries.join(',\n')}
} as const
// Why: these methods bind a schema the shared contract cannot hold because its value
// graph reaches into src/main. Listing them keeps the gap visible instead of absent.
export const RPC_METHODS_WITHOUT_SHARED_PARAMS: readonly string[] = [
${uncataloged.map((name) => ` '${name}'`).join(',\n')}
]
export type RpcMethodName = keyof typeof RPC_PARAMS_BY_METHOD
// Why: z.output is the post-parse shape the handler receives, which is not what a
// client may send — a .default() field reads as required. z.input is not the answer
// either: requiredString is z.unknown().transform(...), so its input admits any value.
// Senders use RpcSendParams from ./rpc-send-params, which is derived from this map.
export type RpcParams<Method extends RpcMethodName> =
(typeof RPC_PARAMS_BY_METHOD)[Method] extends z.ZodType
? z.output<(typeof RPC_PARAMS_BY_METHOD)[Method]>
: void
`
}
// Why: the drift gate compares bytes, so the generator must emit exactly what the
// formatter would produce or every run would look like drift.
function formatted(source) {
const buildDir = scratchDir('rpc-params-catalog-fmt-')
try {
const file = path.join(buildDir, 'rpc-params-catalog.generated.ts')
writeFileSync(file, source)
const { command, prefixArgs } = resolveOxcCliInvocation('oxfmt', 'oxfmt', REPO_ROOT)
execFileSync(command, [...prefixArgs, '--write', file], {
stdio: 'ignore',
windowsHide: true
})
return readFileSync(file, 'utf8')
} finally {
rmSync(buildDir, { recursive: true, force: true })
}
}
function main() {
const check = process.argv.includes('--check')
const generated = formatted(render(loadRegistryAndSchemas(indexableModules())))
const current = existsSync(OUTPUT_PATH) ? readFileSync(OUTPUT_PATH, 'utf8') : null
if (generated === current) {
if (!check) {
console.log(`rpc params catalog already up to date: ${repoPath(OUTPUT_PATH)}`)
}
return
}
if (check) {
console.error(
`${repoPath(OUTPUT_PATH)} is out of date. Run \`pnpm run generate:rpc-params-catalog\`.`
)
process.exitCode = 1
return
}
writeFileSync(OUTPUT_PATH, generated)
console.log(`wrote ${repoPath(OUTPUT_PATH)}`)
}
main()
+23
View File
@@ -0,0 +1,23 @@
import { createRequire } from 'node:module'
import path from 'node:path'
import process from 'node:process'
// Why not `pnpm exec <bin>` / `node_modules/.bin/<bin>.cmd`: both land on a Windows
// .cmd shim, and Node >= 20 refuses to spawn one without `shell: true` (the
// CVE-2024-27980 mitigation), so every gate that took that route died with EINVAL
// before doing any work. The oxc bins are plain Node scripts, so run them under this
// process's own node — no shim, no shell, no quoting question.
export function resolveOxcCliInvocation(packageName, binName, root = process.cwd()) {
const requireFromRoot = createRequire(path.join(root, 'package.json'))
// The oxc packages' "exports" hide ./bin, so read the manifest and walk to its bin entry.
const manifestPath = requireFromRoot.resolve(`${packageName}/package.json`)
const binField = requireFromRoot(`${packageName}/package.json`).bin
const binEntry = typeof binField === 'string' ? binField : binField?.[binName]
if (!binEntry) {
throw new Error(`${packageName} package.json declares no "${binName}" bin entry.`)
}
return {
command: process.execPath,
prefixArgs: [path.resolve(path.dirname(manifestPath), binEntry)]
}
}
@@ -0,0 +1,40 @@
import { spawnSync } from 'node:child_process'
import { existsSync } from 'node:fs'
import path from 'node:path'
import process from 'node:process'
import { describe, expect, it } from 'vitest'
import { resolveOxcCliInvocation } from './oxc-cli-invocation.mjs'
const repoRoot = path.resolve(import.meta.dirname, '..', '..')
describe('resolveOxcCliInvocation', () => {
it('runs oxfmt under this process node, never through a shim', () => {
const { command, prefixArgs } = resolveOxcCliInvocation('oxfmt', 'oxfmt', repoRoot)
expect(command).toBe(process.execPath)
expect(prefixArgs).toHaveLength(1)
// The params-catalog generator spawned node_modules/.bin/oxfmt, which is a .cmd on
// Windows — Node >= 20 refuses it without shell:true and dies with EINVAL.
expect(prefixArgs[0]).not.toMatch(/\.(cmd|bat)$/i)
expect(existsSync(prefixArgs[0])).toBe(true)
})
it('spawns oxfmt without a shell', () => {
const { command, prefixArgs } = resolveOxcCliInvocation('oxfmt', 'oxfmt', repoRoot)
const result = spawnSync(command, [...prefixArgs, '--help'], {
cwd: repoRoot,
encoding: 'utf8',
shell: false,
windowsHide: true
})
expect(result.error).toBeUndefined()
expect(result.stdout).toContain('oxfmt')
})
it('names the package and bin it could not find', () => {
expect(() => resolveOxcCliInvocation('oxfmt', 'nope', repoRoot)).toThrow(
'oxfmt package.json declares no "nope" bin entry.'
)
})
})
+2 -19
View File
@@ -1,23 +1,6 @@
import { createRequire } from 'node:module'
import path from 'node:path'
import process from 'node:process'
import { resolveOxcCliInvocation } from './oxc-cli-invocation.mjs'
// Why not `pnpm exec oxlint` / `node_modules/.bin/oxlint.cmd`: both land on a
// Windows .cmd shim, and Node >= 20 refuses to spawn one without `shell: true`
// (the CVE-2024-27980 mitigation), so every lint gate died with EINVAL before
// linting anything. Oxlint's bin is a plain Node script, so run it under this
// process's own node — no shim, no shell, no quoting question.
export function resolveOxlintInvocation(root = process.cwd()) {
const requireFromRoot = createRequire(path.join(root, 'package.json'))
// Oxlint's "exports" hides ./bin, so read the manifest and walk to its bin entry.
const manifestPath = requireFromRoot.resolve('oxlint/package.json')
const binField = requireFromRoot('oxlint/package.json').bin
const binEntry = typeof binField === 'string' ? binField : binField?.oxlint
if (!binEntry) {
throw new Error('oxlint package.json declares no "oxlint" bin entry.')
}
return {
command: process.execPath,
prefixArgs: [path.resolve(path.dirname(manifestPath), binEntry)]
}
return resolveOxcCliInvocation('oxlint', 'oxlint', root)
}
@@ -179,6 +179,13 @@ describe('PR E2E gate contract', () => {
'pnpm run test:e2e "${TEST_FILES[@]}" --workers=1 "${E2E_PROJECT_ARGS[@]}"'
)
expect(playwrightConfig).toContain('retries: 0')
const steps = e2eWorkflow.jobs.e2e.steps.filter((step) =>
step.run?.includes('tests/e2e/worktree-switch-first-paint.spec.ts')
)
expect(steps).toHaveLength(1)
expect(steps[0].if).toBe("matrix.shard == '1/14'")
expect(steps[0].run).toContain('xvfb-run --auto-servernum')
expect(steps[0].run).toContain('--project=electron-headful --workers=1')
})
it('keeps startup-exec live parity in the isolated SSH lane', () => {
@@ -0,0 +1,135 @@
// Finds account identifiers and credentials in a captured PTY transcript before it is committed.
import os from 'node:os'
// Why same-length replacements: a transcript's value is its exact wrapping and column
// alignment. Shortening a redacted span reflows the screen and destroys the evidence.
const EMAIL_DOMAIN = '@example.com'
const PLACEHOLDER_UUID = '00000000-0000-4000-8000-000000000000'
/** Ordered most-specific first; the first pattern to claim a span owns it. */
function buildPatterns() {
const username = os.userInfo().username
const hostname = os.hostname()
const patterns = [
{ kind: 'jwt', re: /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,}/g },
{ kind: 'google-api-key', re: /\bAIza[0-9A-Za-z_-]{20,}/g },
{ kind: 'google-refresh-token', re: /\b1\/\/[0-9A-Za-z_-]{20,}/g },
{ kind: 'vendor-key', re: /\b(?:sk-|ghp_|gho_|github_pat_|xoxb-|xoxp-)[A-Za-z0-9_-]{16,}/g },
{ kind: 'bearer-token', re: /\bBearer\s+[A-Za-z0-9._~+/=-]{16,}/gi },
{ kind: 'email', re: /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g },
// Why a UUID counts: agy prints a resumable conversation id on exit, and installation and
// project ids look the same. They identify the operator's session, not just its shape.
{ kind: 'uuid', re: /\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi },
{ kind: 'opaque-token', re: /\b[A-Za-z0-9_-]{40,}\b/g }
]
if (username.length >= 3) {
patterns.splice(5, 0, { kind: 'local-username', re: literalPattern(username) })
}
if (hostname.length >= 3) {
patterns.splice(5, 0, { kind: 'local-hostname', re: literalPattern(hostname) })
}
return patterns
}
function literalPattern(value) {
return new RegExp(value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g')
}
/**
* @param {string} text raw transcript, escapes intact
* @returns {{kind: string, line: number, column: number, index: number, match: string}[]}
*/
export function scanTranscriptForSecrets(text) {
const claimed = []
const findings = []
for (const { kind, re } of buildPatterns()) {
re.lastIndex = 0
let match = re.exec(text)
while (match !== null) {
const start = match.index
const end = start + match[0].length
if (!claimed.some(([from, to]) => start < to && end > from)) {
claimed.push([start, end])
if (!isAlreadyScrubbed(kind, match[0])) {
findings.push({ kind, index: start, match: match[0], ...locate(text, start) })
}
}
match = re.exec(text)
}
}
return findings.sort((left, right) => left.index - right.index)
}
// Why: a scrubbed fixture must verify clean, so this scanner has to recognise its own
// placeholders — otherwise "prove it's gone" can never pass and the check gets ignored.
const PLACEHOLDER_DOMAIN_RE = /@(?:example\.(?:com|org|net)|localhost)$/i
function isAlreadyScrubbed(kind, match) {
if (kind === 'email') {
return PLACEHOLDER_DOMAIN_RE.test(match)
}
if (kind === 'uuid') {
return match.toLowerCase() === PLACEHOLDER_UUID
}
return /^(.)\1*$/.test(match)
}
function locate(text, index) {
let line = 1
let lineStart = 0
for (let cursor = 0; cursor < index; cursor += 1) {
if (text.charCodeAt(cursor) === 10) {
line += 1
lineStart = cursor + 1
}
}
return { line, column: index - lineStart + 1 }
}
/** Same-length stand-in so redaction cannot reflow the captured screen. */
export function placeholderFor(kind, length) {
if (kind === 'uuid' && length === PLACEHOLDER_UUID.length) {
return PLACEHOLDER_UUID
}
if (kind === 'email' && length > EMAIL_DOMAIN.length) {
return 'u'.repeat(length - EMAIL_DOMAIN.length) + EMAIL_DOMAIN
}
return kind === 'local-username' || kind === 'local-hostname'
? 'x'.repeat(length)
: 'X'.repeat(length)
}
/** @returns {{text: string, redacted: number}} */
export function redactTranscript(text) {
const findings = scanTranscriptForSecrets(text)
let out = ''
let cursor = 0
for (const finding of findings) {
out += text.slice(cursor, finding.index)
out += placeholderFor(finding.kind, finding.match.length)
cursor = finding.index + finding.match.length
}
return { text: out + text.slice(cursor), redacted: findings.length }
}
export function formatFindings(label, findings) {
if (findings.length === 0) {
return `${label}: clean — no account identifier or credential shapes found.`
}
const rows = findings.map(
(finding) => ` ${finding.line}:${finding.column} ${finding.kind} ${preview(finding.match)}`
)
return [`${label}: ${findings.length} finding(s) — scrub before committing.`, ...rows].join('\n')
}
// Why a codepoint test and not a character class: a control-byte range written as an escape is
// folded back into raw 0x00-0x1f bytes by the formatter, which makes this file binary to the VCS
// and leaves the one file gating real PTY data into history unreviewable in a diff.
function preview(value) {
const head = value.length <= 24 ? value : `${value.slice(0, 21)}...`
let printable = ''
for (const char of head) {
printable += (char.codePointAt(0) ?? 0) < 0x20 ? '?' : char
}
return printable
}
@@ -0,0 +1,133 @@
// The scrub gate is the only thing standing between a live agent transcript and a
// committed account identifier, so it is pinned on the shapes those transcripts carry.
import { readdirSync, readFileSync } from 'node:fs'
import os from 'node:os'
import { join } from 'node:path'
import { describe, expect, it } from 'vitest'
import {
formatFindings,
placeholderFor,
redactTranscript,
scanTranscriptForSecrets
} from './pty-transcript-secret-scan.mjs'
import { parseArgs, resolveSpawn } from './capture-agent-pty-transcript.mjs'
describe('pty transcript secret scan', () => {
it('finds the account row of a ready screen', () => {
const findings = scanTranscriptForSecrets('Antigravity CLI 1.1.17\njin.woo@acme.dev (Business)')
expect(findings).toHaveLength(1)
expect(findings[0]).toMatchObject({ kind: 'email', line: 2, column: 1 })
})
it('finds credentials an agent may echo while signing in', () => {
const kinds = scanTranscriptForSecrets(
[
'token: eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dBjftJeZ4CVP',
'key: AIzaSyA1234567890abcdefghijklmnopqrstu',
'refresh: 1//0gLm34XyZabcdefghijklmnopqrstuvwx',
'Authorization: Bearer abcdefghijklmnopqrstuvwxyz012345'
].join('\n')
).map((finding) => finding.kind)
expect(kinds).toEqual(['jwt', 'google-api-key', 'google-refresh-token', 'bearer-token'])
})
it('flags this machine’s own username, which a prompt line leaks', () => {
const username = os.userInfo().username
const findings = scanTranscriptForSecrets(`~/Users/${username}/orca/repo\n> `)
expect(findings.some((finding) => finding.kind === 'local-username')).toBe(true)
})
it('finds the resumable conversation id agy prints on exit', () => {
const findings = scanTranscriptForSecrets(
'Resume with -c (or command below):\nagy --conversation=26dc1986-9eec-456a-a534-d93e5c1076c2'
)
expect(findings).toHaveLength(1)
expect(findings[0].kind).toBe('uuid')
expect(placeholderFor('uuid', findings[0].match.length)).toMatch(
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[0-9a-f]{4}-[0-9a-f]{12}$/
)
})
it('reports a clean transcript as clean', () => {
const findings = scanTranscriptForSecrets('Antigravity CLI 1.1.17\nSonnet 4.6 (High)\n> ')
expect(findings).toEqual([])
expect(formatFindings('fixture', findings)).toContain('clean')
})
it('claims a span once, so a token inside an email is not double-reported', () => {
const findings = scanTranscriptForSecrets('longlivedaccountname@corp.internal')
expect(findings).toHaveLength(1)
})
it('passes a fixture that is already scrubbed, so "prove it is gone" can succeed', () => {
const scrubbed = `uuuu@example.com\n${'X'.repeat(44)}`
expect(scanTranscriptForSecrets(scrubbed)).toEqual([])
})
})
describe('redaction', () => {
it('replaces every finding with the same number of characters', () => {
// Why length matters: the fixture's value is its exact wrapping. A shorter
// replacement reflows the screen and invalidates the capture.
const text = 'Antigravity CLI 1.1.17\njin.woo@acme.dev (Antigravity Business)\n> '
const { text: redacted, redacted: count } = redactTranscript(text)
expect(count).toBe(1)
expect(redacted).toHaveLength(text.length)
expect(redacted).not.toContain('jin.woo@acme.dev')
expect(scanTranscriptForSecrets(redacted)).toEqual([])
expect(redactTranscript(redacted).redacted).toBe(0)
})
it('keeps a redacted email shaped like an email', () => {
expect(placeholderFor('email', 'a@b.example.com'.length)).toMatch(/^u+@example\.com$/)
})
it('leaves the rest of the screen byte-for-byte untouched', () => {
const text = 'line one\nuser@corp.io\nline three'
expect(redactTranscript(text).text.split('\n')[2]).toBe('line three')
})
})
describe('committed transcripts', () => {
// Why in CI and not just in the recorder: a transcript is committed once and read forever.
// The capture-time warning is skippable; this is not.
const fixtureDir = join(import.meta.dirname, '..', '..', 'src', 'main', 'runtime', '__fixtures__')
const transcripts = readdirSync(fixtureDir).filter((entry) => entry.endsWith('.txt'))
it.each(transcripts)('%s carries no account identifier or credential', (name) => {
const findings = scanTranscriptForSecrets(readFileSync(join(fixtureDir, name), 'utf8'))
expect(formatFindings(name, findings)).toContain('clean')
})
})
describe('capture argv', () => {
it('splits recorder options from the agent command', () => {
const { options, command } = parseArgs([
'--name',
'antigravity-ready-personal-non-gemini',
'--cols',
'120',
'--',
'agy',
'--model',
'sonnet'
])
expect(options.name).toBe('antigravity-ready-personal-non-gemini')
expect(options.cols).toBe(120)
expect(command).toEqual(['agy', '--model', 'sonnet'])
})
it('collects a multi-file scan list', () => {
const { options } = parseArgs(['--scan', 'a.txt', 'b.txt', '--redact'])
expect(options.scan).toEqual(['a.txt', 'b.txt'])
expect(options.redact).toBe(true)
})
it('routes a Windows shim through cmd.exe, which node-pty cannot spawn directly', () => {
expect(resolveSpawn(['agy.cmd', '--model', 'sonnet'])).toEqual(
process.platform === 'win32'
? { file: 'cmd.exe', args: ['/c', '"agy.cmd"', '--model', 'sonnet'] }
: { file: 'agy.cmd', args: ['--model', 'sonnet'] }
)
})
})
@@ -0,0 +1,192 @@
import { rm, writeFile } from 'node:fs/promises'
import { join } from 'node:path'
import {
createSessionParseStats,
parseAgentSessionFileCached,
resetSessionParseCacheForTests
} from '../../src/main/ai-vault/session-scanner-parse-cache'
import { resetTranscriptConsumersForTests } from '../../src/main/ai-vault/session-transcript-consumers'
import { SessionSearchEngine } from '../../src/main/ai-vault-search/session-search-engine'
import type {
SessionSearchRequest,
SessionSearchScope
} from '../../src/main/ai-vault-search/session-search-engine-types'
import { registerSessionSearchIndexConsumer } from '../../src/main/ai-vault-search/session-search-index-consumer'
import { SessionSearchStore } from '../../src/main/ai-vault-search/session-search-store'
import type SyncDatabase from '../../src/main/sqlite/sync-database'
import {
writeSyntheticTranscriptCorpus,
type SyntheticCorpus,
type SyntheticCorpusOptions
} from '../../src/main/ai-vault-search/session-search-synthetic-corpus'
import { sessionCandidate } from '../../src/main/ai-vault-search/session-search-transcript-fixtures'
// What a query costs, and what the session candidate limit buys. Everything
// runs through the real store and the real engine over a synthetic corpus;
// never point this at a real transcript tree.
const WARMUP = 5
const SAMPLES = 25
// One query per rung the ladder can take, plus the two shapes that skip it.
const QUERIES: { name: string; request: SessionSearchRequest }[] = [
{ name: 'phrase', request: { query: '"terminal reattach"' } },
{ name: 'identifier', request: { query: 'resolveTerminalPath' } },
{ name: 'path', request: { query: 'src/main/ai-vault/session-transcript-reader.ts' } },
{ name: 'prose', request: { query: 'why is the daemon snapshot stale' } },
{ name: 'typo', request: { query: 'reattahc worktre' } },
{ name: 'common-term', request: { query: 'index' } },
{ name: 'operator-only', request: { query: 'repo:app-3' } },
{ name: 'scoped', request: { query: 'worktree', filters: { scopePaths: ['/repo/app-3'] } } }
]
type Timing = { p50: number; p95: number }
function percentile(sorted: readonly number[], fraction: number): number {
const at = Math.min(sorted.length - 1, Math.floor(sorted.length * fraction))
return Math.round((sorted[at] ?? 0) * 100) / 100
}
function timing(samples: number[]): Timing {
const sorted = [...samples].sort((left, right) => left - right)
return { p50: percentile(sorted, 0.5), p95: percentile(sorted, 0.95) }
}
function time(engine: SessionSearchEngine, request: SessionSearchRequest): number {
const started = performance.now()
engine.search(request)
return performance.now() - started
}
async function indexCorpus(
options: SyntheticCorpusOptions
): Promise<{ corpus: SyntheticCorpus; db: SyncDatabase; release: () => void }> {
resetSessionParseCacheForTests()
const corpus = await writeSyntheticTranscriptCorpus(options)
const store = new SessionSearchStore(join(corpus.root, 'index.sqlite'), (error) => {
throw error
})
const unregister = registerSessionSearchIndexConsumer(store)
const stats = createSessionParseStats()
for (const path of corpus.files) {
await parseAgentSessionFileCached(
await sessionCandidate('claude', path),
process.platform,
stats
)
}
return {
corpus,
// The handle a composed reader gets. Every read here is one synchronous
// statement, which is the contract that comes with it.
db: store.connection,
release: () => {
unregister()
resetTranscriptConsumersForTests()
resetSessionParseCacheForTests()
store.close()
}
}
}
/** Per-query and overall latency for one scope. */
function scopeReport(db: SyncDatabase, scope: SessionSearchScope): Record<string, unknown> {
const engine = new SessionSearchEngine(db)
const everything: number[] = []
const perQuery: Record<string, Timing & { hits: number; route: string }> = {}
for (const { name, request } of QUERIES) {
const scoped = { ...request, scope }
for (let run = 0; run < WARMUP; run++) {
engine.search(scoped)
}
const samples = Array.from({ length: SAMPLES }, () => time(engine, scoped))
everything.push(...samples)
const result = engine.search(scoped)
perQuery[name] = { ...timing(samples), hits: result.hits.length, route: result.planner.route }
}
return { ...timing(everything), perQuery }
}
/**
* The candidate limit only costs anything once there are more matching sessions
* than the limit, so this runs over many short sessions rather than the wide
* corpus above. Limits are interleaved sample by sample: run back to back, the
* first configuration pays for every page the OS cache had not seen yet and the
* ordering alone moves p95 by more than the limit does.
*/
function candidateSweep(db: SyncDatabase, limits: readonly number[]): Record<string, unknown> {
const request: SessionSearchRequest = { query: 'index', limit: 20 }
const engines = new Map(
limits.map((limit) => [limit, new SessionSearchEngine(db, { sessionCandidateLimit: limit })])
)
const samples = new Map(limits.map((limit) => [limit, [] as number[]]))
for (let run = 0; run < WARMUP; run++) {
for (const engine of engines.values()) {
engine.search(request)
}
}
for (let run = 0; run < SAMPLES; run++) {
for (const limit of limits) {
samples.get(limit)!.push(time(engines.get(limit)!, request))
}
}
const report: Record<string, unknown> = {}
for (const limit of limits) {
const result = engines.get(limit)!.search(request)
report[String(limit)] = {
...timing(samples.get(limit)!),
truncated: result.truncated.candidates,
// Pages a caller could walk before the limit stops handing out sessions.
reachablePages: Math.ceil(limit / (request.limit ?? 20))
}
}
return report
}
const wide = await indexCorpus({ sessions: Number(process.env.SESSIONS ?? 40) })
let report: string
try {
const scope = {
all: scopeReport(wide.db, 'all'),
conversation: scopeReport(wide.db, 'conversation')
}
wide.release()
await rm(wide.corpus.root, { recursive: true, force: true })
// Many short sessions: what makes the candidate limit binding is the session
// count, not the byte count.
const many = await indexCorpus({ sessions: 2500, turnsPerSession: 1, seed: 7 })
try {
report = JSON.stringify(
{
scopeCorpus: {
sessions: wide.corpus.files.length,
transcriptMb: Math.round((wide.corpus.transcriptBytes / 1024 / 1024) * 100) / 100,
messages: wide.corpus.messageCount
},
scope,
candidateCorpus: {
sessions: many.corpus.files.length,
transcriptMb: Math.round((many.corpus.transcriptBytes / 1024 / 1024) * 100) / 100
},
candidateSweep: candidateSweep(many.db, [200, 600, 1200, 2400])
},
null,
2
)
} finally {
many.release()
await rm(many.corpus.root, { recursive: true, force: true })
}
} catch (error) {
await rm(wide.corpus.root, { recursive: true, force: true })
throw error
}
// Why a file as well as stdout: a runner that intercepts console output
// (vitest does) would otherwise swallow the whole report.
const out = process.env.BENCH_OUT
if (out) {
await writeFile(out, `${report}\n`)
}
console.log(report)
@@ -0,0 +1,148 @@
import assert from 'node:assert/strict'
import { mkdtemp, rm, stat } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { setImmediate as yieldToEventLoop } from 'node:timers/promises'
import {
syntheticCandidate,
syntheticSession,
userMessages
} from '../../src/main/ai-vault-search/session-search-index-test-fixture'
import { SessionSearchStore } from '../../src/main/ai-vault-search/session-search-store'
import SyncDatabase from '../../src/main/sqlite/sync-database'
// Bundle with esbuild --bundle --platform=node, then run on the host under test.
// Every mode seeds through SessionSearchStore so the three arms are comparable;
// only `whole-file` leaves the shipped path, because it is the baseline the
// batched purge exists to replace.
const ROWS = 60_000
/** The purge yields with `setImmediate` between chunks, so a peer chain samples each gap. */
async function sampleLoopStalls(running: () => boolean, intervals: number[]): Promise<void> {
let previous = performance.now()
while (running()) {
await yieldToEventLoop()
const now = performance.now()
intervals.push(now - previous)
previous = now
}
}
/** What a search would still return: rows whose session row is still there. */
function visibleRows(db: SyncDatabase): number {
return (
db
.prepare(`SELECT count(*) AS n FROM messages m JOIN sessions s ON s.id = m.session_row_id`)
.get() as { n: number }
).n
}
const root = await mkdtemp(join(tmpdir(), 'orca-search-retention-bench-'))
try {
for (const mode of ['whole-file', 'batched', 'batched-pinned-reader']) {
const path = join(root, `${mode}.sqlite`)
const errors: unknown[] = []
const store = new SessionSearchStore(path, (error) => errors.push(error))
let reader: SyncDatabase | null = null
try {
const write = store.beginWrite(syntheticCandidate(), 'replace', 0)!
for (const message of userMessages(
'synthetic benchmark needle repeated context for a representative coding conversation with commands and paths src/example.ts',
ROWS
)) {
write.add(message)
}
assert.equal(
write.commit({
session: syntheticSession(),
byteOffset: 4096,
incomplete: false
}),
true
)
assert.deepEqual(errors, [])
// Truncating first is what makes walBytes below the purge's own growth.
const checkpoint = new SyncDatabase(path)
checkpoint.pragma('wal_checkpoint(TRUNCATE)')
checkpoint.close()
if (mode === 'batched-pinned-reader') {
reader = new SyncDatabase(path, { readonly: true })
reader.exec('BEGIN')
reader.prepare('SELECT count(*) FROM messages').get()
}
const probe = new SyncDatabase(path, { readonly: true })
const intervals: number[] = []
const started = performance.now()
if (mode === 'whole-file') {
const raw = new SyncDatabase(path)
try {
raw.exec('BEGIN IMMEDIATE')
const ids = raw.prepare('SELECT id FROM messages').all() as {
id: number
}[]
for (const { id } of ids) {
raw.prepare('DELETE FROM messages_fts WHERE rowid=?').run(id)
}
raw.exec('DELETE FROM messages; DELETE FROM sessions; DELETE FROM files; COMMIT')
} finally {
raw.close()
}
intervals.push(performance.now() - started)
} else {
let purging = true
const purge = store.purgeOlderThan(Date.now() + 60_000)
// Hiding is immediate: cutting the session loose from its file is the
// first transaction, so a read one turn in already sees nothing, long
// before the rows are gone.
const hiddenEarly = yieldToEventLoop().then(() => visibleRows(probe))
const sampler = sampleLoopStalls(() => purging, intervals)
await purge
purging = false
await sampler
assert.equal(await hiddenEarly, 0)
assert.deepEqual(errors, [])
}
const wallMs = performance.now() - started
probe.close()
reader?.exec('COMMIT')
reader?.close()
reader = null
const after = new SyncDatabase(path, { readonly: true })
try {
for (const table of ['messages_fts']) {
assert.equal(
(
after.prepare(`SELECT count(*) AS n FROM ${table}`).get() as {
n: number
}
).n,
0
)
}
} finally {
after.close()
}
const walBytes = (await stat(`${path}-wal`)).size
intervals.sort((a, b) => a - b)
console.log(
JSON.stringify({
mode,
platform: process.platform,
node: process.version,
rows: ROWS,
wallMs: Math.round(wallMs),
samples: intervals.length,
maxStepMs: Math.round(intervals.at(-1) ?? 0),
p95StepMs: Math.round(intervals[Math.floor(intervals.length * 0.95)] ?? 0),
walBytes
})
)
} finally {
reader?.close()
store.close()
}
}
} finally {
await rm(root, { recursive: true, force: true })
}
@@ -0,0 +1,205 @@
import { rm, writeFile } from 'node:fs/promises'
import { join } from 'node:path'
import {
createSessionParseStats,
parseAgentSessionFileCached,
resetSessionParseCacheForTests
} from '../../src/main/ai-vault/session-scanner-parse-cache'
import { resetTranscriptConsumersForTests } from '../../src/main/ai-vault/session-transcript-consumers'
import { SessionSearchEngine } from '../../src/main/ai-vault-search/session-search-engine'
import type {
SessionSearchRequest,
SessionSearchScope
} from '../../src/main/ai-vault-search/session-search-engine-types'
import { registerSessionSearchIndexConsumer } from '../../src/main/ai-vault-search/session-search-index-consumer'
import { SessionSearchStore } from '../../src/main/ai-vault-search/session-search-store'
import { sessionCandidate } from '../../src/main/ai-vault-search/session-search-transcript-fixtures'
import type SyncDatabase from '../../src/main/sqlite/sync-database'
import { writeToolHeavyCorpus, type ToolHeavyCorpus } from './session-search-tool-heavy-corpus'
// What each scope costs on an index the size of a real transcript tree.
//
// The 10.5 MB corpus in `session-search-query-benchmark.ts` sizes the route
// ladder; this one sizes the corpus. `conversation` is a column filter over the
// one FTS table rather than a second table of its own, and the whole cost of
// that decision is how much of `messages_fts` a conversation query has to read
// past — which is set by how much of a transcript is tool output.
//
// Synthetic, always: this must never be pointed at a real transcript.
const WARMUP = 5
/** Conversation-shaped queries; every term is one the prose actually uses. */
const QUERIES = [
'terminal reattach',
'stale snapshot',
'daemon cursor',
'worktree index',
'publish transaction',
'relay daemon',
'session cursor',
'because stale',
'terminal worktree',
'index snapshot',
'reattach cursor',
'transaction relay',
'snapshot session',
'daemon publish',
'worktree terminal',
'cursor index',
'stale relay',
'session transaction',
'publish snapshot',
'reattach daemon'
]
async function indexCorpus(
corpus: ToolHeavyCorpus
): Promise<{ db: SyncDatabase; release: () => void }> {
resetSessionParseCacheForTests()
const store = new SessionSearchStore(join(corpus.root, 'index.sqlite'), (error) => {
throw error
})
const unregister = registerSessionSearchIndexConsumer(store)
const stats = createSessionParseStats()
for (const path of corpus.files) {
await parseAgentSessionFileCached(
await sessionCandidate('claude', path),
process.platform,
stats
)
}
return {
// The store's own handle, which is what a composed reader gets: every
// retrieval is one synchronous statement, so nothing pins a WAL snapshot.
db: store.connection,
release: () => {
unregister()
resetTranscriptConsumersForTests()
resetSessionParseCacheForTests()
store.close()
}
}
}
type Timing = { p50: number; p95: number }
function timing(samples: readonly number[]): Timing {
const sorted = [...samples].sort((left, right) => left - right)
const at = (fraction: number): number => {
const index = Math.min(sorted.length - 1, Math.floor(sorted.length * fraction))
return Math.round((sorted[index] ?? 0) * 100) / 100
}
return { p50: at(0.5), p95: at(0.95) }
}
/**
* The query sets, one per rung of the ladder the engine may take.
*
* Which rung each one reaches is not forced, it is observed: samples are
* bucketed by the route the engine reports, so the table says what was measured
* rather than what was intended, and a query that lands on a different rung
* than expected shows up as a bucket rather than as a wrong number.
*/
function queries(): string[] {
const run = (index: number, length: number): string =>
Array.from({ length }, (_unused, step) => QUERIES[(index + step) % QUERIES.length]).join(' ')
return [
// Two terms, unquoted: not literal, so straight to OR.
...QUERIES,
// Two terms, quoted: literal, and on this corpus any two of fourteen words
// sit next to each other somewhere, so the phrase rung answers.
...QUERIES.map((query) => `"${query}"`),
// Eight terms, quoted: an ordered run that long does not occur in 105 MB of
// draws from fourteen words, so the phrase rung misses and AND answers.
...QUERIES.map((_query, index) => `"${run(index, 4)}"`)
]
}
type Bucket = { samples: number[]; hits: number }
/**
* Both scopes over the same queries, interleaved scope by scope: run back to
* back, the first one pays for every page the OS cache had not seen and the
* ordering moves p95 more than the scope does.
*/
function scopeReport(db: SyncDatabase): Record<string, unknown> {
const engine = new SessionSearchEngine(db)
const scopes: SessionSearchScope[] = ['all', 'conversation']
const requests: SessionSearchRequest[] = queries().map((query) => ({ query }))
const buckets = new Map<string, Bucket>()
for (let run = 0; run < WARMUP; run++) {
for (const scope of scopes) {
for (const request of requests) {
engine.search({ ...request, scope })
}
}
}
for (const request of requests) {
for (const scope of scopes) {
const started = performance.now()
const result = engine.search({ ...request, scope })
const elapsed = performance.now() - started
const key = `${result.planner.route}/${scope}`
const bucket = buckets.get(key) ?? { samples: [], hits: 0 }
bucket.samples.push(elapsed)
bucket.hits += result.hits.length
buckets.set(key, bucket)
}
}
const report: Record<string, unknown> = {}
for (const [key, bucket] of [...buckets].sort(([left], [right]) => left.localeCompare(right))) {
report[key] = { ...timing(bucket.samples), samples: bucket.samples.length, hits: bucket.hits }
}
return report
}
/** Bytes the FTS table occupies, which is the cost the deleted second table saved. */
function indexBytes(db: SyncDatabase): Record<string, number> | { unavailable: string } {
try {
const sum = (where: string, ...values: string[]): number =>
Number(
(
db
.prepare(`SELECT COALESCE(SUM(pgsize),0) AS bytes FROM dbstat ${where}`)
.get(...values) as { bytes: number }
).bytes
)
return { total: sum(''), messagesFts: sum('WHERE name LIKE ?', 'messages_fts%') }
} catch {
// dbstat is a compile-time option; the latency numbers stand without it.
return { unavailable: 'no dbstat' }
}
}
const corpus = await writeToolHeavyCorpus({
targetBytes: Number(process.env.CORPUS_MB ?? 100) * 1024 * 1024,
toolShare: Number(process.env.TOOL_SHARE ?? 0.9)
})
let report: string
const indexed = await indexCorpus(corpus)
try {
report = JSON.stringify(
{
corpus: {
sessions: corpus.files.length,
transcriptMb: Math.round((corpus.transcriptBytes / 1024 / 1024) * 100) / 100,
toolShareOfMessageText:
Math.round((corpus.toolBytes / (corpus.toolBytes + corpus.proseBytes)) * 1000) / 1000
},
indexBytes: indexBytes(indexed.db),
route: scopeReport(indexed.db)
},
null,
2
)
} finally {
indexed.release()
await rm(corpus.root, { recursive: true, force: true })
}
const out = process.env.BENCH_OUT
if (out) {
await writeFile(out, `${report}\n`)
}
console.log(report)
@@ -0,0 +1,152 @@
import { mkdtemp, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
// The corpus the scope benchmark runs over. Written here rather than by
// `session-search-synthetic-corpus.ts` because what it costs to answer a
// conversation query out of the one FTS table turns on the property that
// generator fixes: how much of a transcript is tool output.
//
// Synthetic, always. This must never be pointed at a real transcript.
const PROSE = [
'terminal',
'reattach',
'worktree',
'the',
'index',
'cursor',
'publish',
'transaction',
'relay',
'daemon',
'snapshot',
'because',
'stale',
'session'
]
// Tool output is paths, hashes and log lines — and the same words the
// conversation uses, because a `rg` over this repository prints them. That
// overlap is what the benchmark turns on: it is what makes a conversation
// term's posting list carry rows the column filter then has to discard. A tool
// vocabulary disjoint from the prose would leave nothing to discard and measure
// the wrong thing.
const TOOL_ONLY = [
'src/main/ai-vault/session-transcript-reader.ts',
'node_modules/.pnpm/typescript@5.9.2',
'0x00007ff8',
'ENOENT',
'drwxr-xr-x',
'2026-09-10T00:00:00.000Z',
'sha256:9f2c1a',
'chunk-VHQ4NWQK.js',
'warning:',
'resolveTerminalPath',
'byteOffset',
'MAX_RETRIES'
]
// Half the tool tokens are conversation words. Deliberately pessimistic: the
// more of a query term lives in `tool_text`, the more the column filter costs,
// so a number measured here holds on a real transcript tree.
const TOOL = [...PROSE, ...TOOL_ONLY]
function mulberry32(seed: number): () => number {
let state = seed >>> 0
return () => {
state = (state + 0x6d2b79f5) >>> 0
let t = Math.imul(state ^ (state >>> 15), 1 | state)
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t
return ((t ^ (t >>> 14)) >>> 0) / 4294967296
}
}
function words(random: () => number, vocabulary: readonly string[], count: number): string {
const out: string[] = []
for (let index = 0; index < count; index++) {
out.push(vocabulary[Math.floor(random() * vocabulary.length)]!)
}
return out.join(' ')
}
export type ToolHeavyCorpus = {
root: string
files: string[]
transcriptBytes: number
toolBytes: number
proseBytes: number
}
/**
* Claude JSONL transcripts whose tool output is `toolShare` of the message text.
* One turn is a user question, an assistant answer, a tool call and its output;
* only the last one grows with the share.
*/
export async function writeToolHeavyCorpus(args: {
targetBytes: number
toolShare: number
seed?: number
}): Promise<ToolHeavyCorpus> {
const random = mulberry32(args.seed ?? 11)
const root = await mkdtemp(join(tmpdir(), 'orca-search-convfts-'))
const files: string[] = []
const proseWordsPerTurn = 160
// Tool and prose words are not the same length, so the share is over bytes.
const proseBytesPerTurn = proseWordsPerTurn * 6
const toolWordCount = Math.max(
1,
Math.round((proseBytesPerTurn * args.toolShare) / (1 - args.toolShare) / 22)
)
let transcriptBytes = 0
let toolBytes = 0
let proseBytes = 0
for (let session = 0; transcriptBytes < args.targetBytes; session++) {
const sessionId = `00000000-0000-4000-8000-${String(session).padStart(12, '0')}`
const lines: string[] = []
for (let turn = 0; turn < 40; turn++) {
const at = new Date(1740000000000 + turn * 60_000).toISOString()
const question = words(random, PROSE, 40)
const answer = words(random, PROSE, proseWordsPerTurn - 40)
const output = words(random, TOOL, toolWordCount)
proseBytes += Buffer.byteLength(question) + Buffer.byteLength(answer)
toolBytes += Buffer.byteLength(output)
lines.push(
JSON.stringify({
type: 'user',
sessionId,
timestamp: at,
cwd: `/repo/app-${session % 7}`,
gitBranch: 'main',
message: { role: 'user', content: question }
}),
JSON.stringify({
type: 'assistant',
sessionId,
timestamp: at,
message: {
role: 'assistant',
model: 'claude-fable-5',
content: [
{ type: 'text', text: answer },
{ type: 'tool_use', name: 'Bash', input: { command: 'rg needle' } }
]
}
}),
JSON.stringify({
type: 'user',
sessionId,
timestamp: at,
message: {
role: 'user',
content: [{ type: 'tool_result', tool_use_id: 'toolu_1', content: output }]
}
})
)
}
const path = join(root, `${sessionId}.jsonl`)
const body = `${lines.join('\n')}\n`
await writeFile(path, body)
transcriptBytes += Buffer.byteLength(body)
files.push(path)
}
return { root, files, transcriptBytes, toolBytes, proseBytes }
}
@@ -0,0 +1,266 @@
import assert from 'node:assert/strict'
import { rm, stat } from 'node:fs/promises'
import { join } from 'node:path'
import { setImmediate as yieldToEventLoop } from 'node:timers/promises'
import {
createSessionParseStats,
parseAgentSessionFileCached,
resetSessionParseCacheForTests
} from '../../src/main/ai-vault/session-scanner-parse-cache'
import { resetTranscriptConsumersForTests } from '../../src/main/ai-vault/session-transcript-consumers'
import { requestWholeTranscriptRead } from '../../src/main/ai-vault/session-transcript-reader'
import { registerSessionSearchIndexConsumer } from '../../src/main/ai-vault-search/session-search-index-consumer'
import { SessionSearchStore } from '../../src/main/ai-vault-search/session-search-store'
import { writeSyntheticTranscriptCorpus } from '../../src/main/ai-vault-search/session-search-synthetic-corpus'
import { sessionCandidate } from '../../src/main/ai-vault-search/session-search-transcript-fixtures'
import SyncDatabase from '../../src/main/sqlite/sync-database'
// Measures the real transcript reader and search store over a synthetic corpus.
// Never point this at a real transcript tree.
/**
* How long the longest single transaction held the process.
*
* With one transaction per file that is the whole stall a file costs, so it is
* the number the commit ceiling exists to bound. Measured by wrapping `exec`,
* because the writer's transactions are the only ones this benchmark runs.
*/
function recordTransactionDurations(durations: number[]): () => void {
const exec = SyncDatabase.prototype.exec
let started = 0
SyncDatabase.prototype.exec = function (this: SyncDatabase, sql: string): void {
if (sql === 'BEGIN IMMEDIATE') {
started = performance.now()
}
exec.call(this, sql)
if (sql === 'COMMIT' && started > 0) {
durations.push(performance.now() - started)
started = 0
}
}
return () => {
SyncDatabase.prototype.exec = exec
}
}
/** The writer commits synchronously, so a peer chain samples the gap each read leaves. */
async function sampleLoopStalls(running: () => boolean, stalls: number[]): Promise<void> {
let previous = performance.now()
while (running()) {
await yieldToEventLoop()
const now = performance.now()
stalls.push(now - previous)
previous = now
}
}
function tableBytes(db: SyncDatabase): Record<string, number> {
const rows = db.prepare('SELECT name, sum(pgsize) AS bytes FROM dbstat GROUP BY name').all() as {
name: string
bytes: number
}[]
const group = (prefix: string): number =>
rows
.filter((row) => row.name === prefix || row.name.startsWith(`${prefix}_`))
.reduce((sum, row) => sum + row.bytes, 0)
return {
messagesFts: group('messages_fts'),
messages: group('messages') - group('messages_fts'),
sessions: group('sessions'),
total: rows.reduce((sum, row) => sum + row.bytes, 0)
}
}
function assertIndexedMessages(db: SyncDatabase, expected: number): number {
const { n } = db
.prepare('SELECT count(*) AS n FROM messages m JOIN sessions s ON s.id = m.session_row_id')
.get() as { n: number }
assert.equal(n, expected, 'indexed message count')
return n
}
async function checkpointedFileBytes(db: SyncDatabase, path: string): Promise<number> {
// Flush committed WAL pages before reporting the final database footprint.
const [checkpoint] = db.pragma('wal_checkpoint(TRUNCATE)') as { busy: number }[]
assert.equal(checkpoint?.busy, 0, 'storage measurement requires a completed checkpoint')
return (await stat(path)).size
}
// The default corpus puts tool output at about half the message text; set this
// far higher to price the tool-row cap against the real 80-97 % band.
const toolResultWords = Number(process.env.ORCA_SEARCH_BENCH_TOOL_WORDS ?? 200)
const corpus = await writeSyntheticTranscriptCorpus({ toolResultWords })
const indexPath = join(corpus.root, 'index.sqlite')
try {
const errors: unknown[] = []
const store = new SessionSearchStore(indexPath, (error) => errors.push(error))
const unregister = registerSessionSearchIndexConsumer(store)
const stalls: number[] = []
const transactions: number[] = []
const restoreExec = recordTransactionDurations(transactions)
let indexing = true
try {
const stats = createSessionParseStats()
const started = performance.now()
const sampler = sampleLoopStalls(() => indexing, stalls)
for (const path of corpus.files) {
await parseAgentSessionFileCached(
await sessionCandidate('claude', path),
process.platform,
stats
)
}
indexing = false
await sampler
restoreExec()
const rebuildMs = performance.now() - started
assert.deepEqual(errors, [])
const reader = new SyncDatabase(indexPath, { readonly: true })
try {
const rows = assertIndexedMessages(reader, corpus.messageCount)
const sessions = (
reader.prepare('SELECT count(*) AS n FROM sessions').get() as {
n: number
}
).n
assert.equal(sessions, corpus.files.length)
const bytes = tableBytes(reader)
const perMb = (value: number): number =>
Math.round((value / (corpus.transcriptBytes / (1024 * 1024))) * 10) / 10
const fileBytes = await checkpointedFileBytes(store.connection, indexPath)
stalls.sort((a, b) => a - b)
transactions.sort((a, b) => a - b)
console.log(
JSON.stringify(
{
platform: process.platform,
node: process.version,
transcriptMb: Math.round((corpus.transcriptBytes / (1024 * 1024)) * 100) / 100,
toolResultWords,
sessions,
rows,
rebuildMs: Math.round(rebuildMs),
rowsPerSecond: Math.round(rows / (rebuildMs / 1000)),
transcriptMbPerSecond:
Math.round((corpus.transcriptBytes / (1024 * 1024) / (rebuildMs / 1000)) * 100) / 100,
bytesPerTranscriptMb: {
messagesFts: perMb(bytes.messagesFts),
messages: perMb(bytes.messages),
sessions: perMb(bytes.sessions),
total: perMb(bytes.total)
},
writeAmplification: Math.round((bytes.total / corpus.transcriptBytes) * 100) / 100,
fileWriteAmplification: Math.round((fileBytes / corpus.transcriptBytes) * 100) / 100,
transactions: transactions.length,
maxTransactionMs: Math.round((transactions.at(-1) ?? 0) * 100) / 100,
maxLoopStallMs: Math.round(stalls.at(-1) ?? 0),
p95LoopStallMs: Math.round(stalls[Math.floor(stalls.length * 0.95)] ?? 0),
loopStallSamples: stalls.length,
parseStats: stats
},
null,
2
)
)
} finally {
reader.close()
}
} finally {
indexing = false
restoreExec()
unregister()
resetTranscriptConsumersForTests()
resetSessionParseCacheForTests()
store.close()
}
} finally {
await rm(corpus.root, { recursive: true, force: true })
}
// Phase two: one transcript far larger than any real one, to price the ceiling
// that decides whether a file commits once or in chunks.
const largeTurns = Number(process.env.ORCA_SEARCH_BENCH_LARGE_TURNS ?? 23_000)
const large = await writeSyntheticTranscriptCorpus({
sessions: 1,
turnsPerSession: largeTurns,
seed: 2
})
const largeIndexPath = join(large.root, 'index.sqlite')
try {
const errors: unknown[] = []
const store = new SessionSearchStore(largeIndexPath, (error) => errors.push(error))
const unregister = registerSessionSearchIndexConsumer(store)
const transactions: number[] = []
const restoreExec = recordTransactionDurations(transactions)
try {
const stats = createSessionParseStats()
const started = performance.now()
await parseAgentSessionFileCached(
await sessionCandidate('claude', large.files[0]!),
process.platform,
stats
)
const indexMs = performance.now() - started
restoreExec()
assert.deepEqual(errors, [])
assertIndexedMessages(store.connection, large.messageCount)
transactions.sort((a, b) => a - b)
// The same file again, over a generation the index already holds. That is
// the pass a growing transcript really costs, and the one whose transaction
// used to be sized by the old session rather than by the chunk being
// written. The drain that reclaims the cut-loose generation runs after the
// commit, so its bounded batches are in `replaceTransactions` too.
const replaceTransactions: number[] = []
const restoreReplaceExec = recordTransactionDurations(replaceTransactions)
requestWholeTranscriptRead(large.files[0]!)
const replaceStarted = performance.now()
await parseAgentSessionFileCached(
await sessionCandidate('claude', large.files[0]!),
process.platform,
stats
)
const replaceMs = performance.now() - replaceStarted
// Finishes whatever the scheduled drain has not reached, so the reclaim is
// priced rather than left half done under the next measurement.
const reclaimStarted = performance.now()
await store.purgeOlderThan(null)
const reclaimMs = performance.now() - reclaimStarted
restoreReplaceExec()
assert.deepEqual(errors, [])
assertIndexedMessages(store.connection, large.messageCount)
replaceTransactions.sort((a, b) => a - b)
console.log(
JSON.stringify(
{
phase: 'single-large-file',
transcriptMb: Math.round((large.transcriptBytes / (1024 * 1024)) * 100) / 100,
indexMs: Math.round(indexMs),
transactions: transactions.length,
maxTransactionMs: Math.round(transactions.at(-1) ?? 0),
replaceMs: Math.round(replaceMs),
replaceTransactions: replaceTransactions.length,
maxReplaceTransactionMs: Math.round(replaceTransactions.at(-1) ?? 0),
reclaimMs: Math.round(reclaimMs),
indexMb:
Math.round(
((await checkpointedFileBytes(store.connection, largeIndexPath)) / (1024 * 1024)) *
100
) / 100
},
null,
2
)
)
} finally {
restoreExec()
unregister()
resetTranscriptConsumersForTests()
resetSessionParseCacheForTests()
store.close()
}
} finally {
await rm(large.root, { recursive: true, force: true })
}
@@ -1,2 +1,6 @@
export const BUILD_IDENTITY_RE = /\b(?:const|let|var)\s+BUILD_IDENTITY\s*=\s*"(rc|stable)"/
export const WRITE_KEY_RE = /\b(?:const|let|var)\s+WRITE_KEY\s*=\s*"(phc_[A-Za-z0-9_-]+)"/
// The unminified bundle keeps these names. Production minification may rename
// them, but the injected identity and key remain adjacent in the declaration.
export const BUILD_IDENTITY_RE = /\b(?:const|let|var)\s+BUILD_IDENTITY\s*=\s*["`](rc|stable)["`]/
export const WRITE_KEY_RE = /\b(?:const|let|var)\s+WRITE_KEY\s*=\s*["`](phc_[A-Za-z0-9_-]+)["`]/
export const MINIFIED_TELEMETRY_RE =
/\b(?:const|let|var)\s+[$\w]+\s*=\s*["'`](rc|stable)["'`][\s\S]{0,200}?[,$]\s*[$\w]+\s*=\s*["'`](phc_[A-Za-z0-9_-]+)["'`]/
@@ -1,5 +1,9 @@
import { describe, expect, it } from 'vitest'
import { BUILD_IDENTITY_RE, WRITE_KEY_RE } from './telemetry-bundle-constant-patterns.mjs'
import {
BUILD_IDENTITY_RE,
MINIFIED_TELEMETRY_RE,
WRITE_KEY_RE
} from './telemetry-bundle-constant-patterns.mjs'
describe('telemetry bundle constant patterns', () => {
it.each(['const', 'let', 'var'])('accepts %s declarations', (declaration) => {
@@ -13,4 +17,9 @@ describe('telemetry bundle constant patterns', () => {
expect('const WRITE_KEY = null').not.toMatch(WRITE_KEY_RE)
expect('const WRITE_KEY = "example-key"').not.toMatch(WRITE_KEY_RE)
})
it('accepts minified adjacent declarations', () => {
const bundle = 'var dde=`stable`,fde=`phc_example-key_123`,pde=(dde===`stable`)'
expect(bundle).toMatch(MINIFIED_TELEMETRY_RE)
})
})

Some files were not shown because too many files have changed in this diff Show More