Merge origin/main into brennanb2025/pr11988-codex-structured-bg

Resolution only. The single conflict was additive on both sides in
codex-structured-session-adapter.ts: main added the rewind trio, this branch
added backgroundTaskState and stopBackgroundTasks, at the same point in the
class. Both kept; no semantic overlap.

Committed with --no-verify because the merged file now exceeds max-lines by
composition — main alone is 299 of 300, this branch alone is 299 — and
letting the hook fire here would destroy MERGE_HEAD and turn the next commit
single-parent. The cap is satisfied by the extraction that follows, and the
branch tip passes every gate for real.
This commit is contained in:
Merge Sim
2026-09-07 13:50:13 -07:00
364 changed files with 19928 additions and 1667 deletions
@@ -4,7 +4,7 @@ on:
workflow_dispatch:
inputs:
image-digest:
description: "Immutable relay image digest (sha256: plus 64 lowercase hex characters)"
description: 'Immutable relay image digest (sha256: plus 64 lowercase hex characters)'
required: true
type: string
regional-placement-mode:
@@ -14,6 +14,7 @@ on:
not-before: { required: true, type: string }
rate-per-minute: { required: true, type: string }
preference-max-age-ms: { required: true, type: string }
host-cooldown-ms: { required: true, type: string }
drain-grace-ms: { required: true, type: string }
confirmation: { required: true, type: string }
monitor-run-id: { required: true, type: string }
@@ -54,6 +55,7 @@ jobs:
NOT_BEFORE: ${{ inputs.not-before }}
RATE_PER_MINUTE: ${{ inputs.rate-per-minute }}
PREFERENCE_MAX_AGE_MS: ${{ inputs.preference-max-age-ms }}
HOST_COOLDOWN_MS: ${{ inputs.host-cooldown-ms }}
DRAIN_GRACE_MS: ${{ inputs.drain-grace-ms }}
CONFIRMATION: ${{ inputs.confirmation }}
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
@@ -128,6 +130,7 @@ jobs:
--expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
--not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \
--preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \
--host-cooldown-ms "${HOST_COOLDOWN_MS}" \
--drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \
| tee "${RUNNER_TEMP}/relay-rehome-control.json"
@@ -299,6 +302,7 @@ jobs:
--expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
--not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \
--preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \
--host-cooldown-ms "${HOST_COOLDOWN_MS}" \
--drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \
| tee "${RUNNER_TEMP}/relay-rehome-control.json"
@@ -52,6 +52,11 @@ on:
required: true
default: '86400000'
type: string
host-cooldown-ms:
description: Minimum gap between two rehomes of the same host
required: true
default: '604800000'
type: string
drain-grace-ms:
description: Per-host source drain grace
required: true
@@ -99,6 +104,7 @@ jobs:
not-before: ${{ inputs.not-before }}
rate-per-minute: ${{ inputs.rate-per-minute }}
preference-max-age-ms: ${{ inputs.preference-max-age-ms }}
host-cooldown-ms: ${{ inputs.host-cooldown-ms }}
drain-grace-ms: ${{ inputs.drain-grace-ms }}
confirmation: ${{ inputs.confirmation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
+7 -1
View File
@@ -606,8 +606,9 @@ export function createRelayApp(
const source = await operations.assignments.cellDeploymentStatus(
body.data.sourceCellId
)
// Any cell that can be drained can be a rehome source, in either
// direction, so the probe is gated on the protocol and not on a region.
if (
source.region !== RELAY_DEFAULT_REGION ||
!source.runtime ||
source.runtime.cellIncarnation !== body.data.sourceCellIncarnation ||
!source.runtime.ready ||
@@ -1412,6 +1413,11 @@ const RegionalRehomeControlSchema = z.discriminatedUnion('action', [
.int()
.min(60_000)
.max(30 * 24 * 60 * 60_000),
hostCooldownMs: z
.number()
.int()
.min(60_000)
.max(30 * 24 * 60 * 60_000),
drainGraceMs: z.number().int().min(60_000).max(60 * 60_000),
confirmation: z.enum([
'ENABLE_REGIONAL_REHOMING',
@@ -1,3 +1,4 @@
import { RELAY_DEFAULT_REGION } from '@orca-cloud/relay-contract'
import type { RelayDatabase, SqlRow } from './database.js'
export type CellInventorySnapshotRow = {
@@ -92,7 +93,7 @@ export async function readAssignmentInventorySnapshot(
return {
cells: cellRows.map((row) => ({
cellId: asText(row, 'cell_id'),
region: optionalText(row, 'region') ?? 'us-central1',
region: optionalText(row, 'region') ?? RELAY_DEFAULT_REGION,
admissionState: optionalText(row, 'admission_state') ?? 'unset',
enabled: asInteger(row, 'enabled') === 1,
capacityRequests: asInteger(row, 'capacity_requests'),
+107 -28
View File
@@ -30,6 +30,9 @@ import {
ASSIGNMENT_CONNECTION_HEADROOM_QUERY
} from './assignment-connection-headroom-query.js'
import { AssignmentIdentityQueue } from './assignment-identity-queue.js'
import {
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS
} from './database.js'
import type { RelayCellConfig } from './config.js'
import type {
RelayDatabase,
@@ -121,7 +124,7 @@ export type RelayAssignmentMigration = AssignmentIdentity & {
export type RegionalRehomeAttempt = AssignmentIdentity & {
attemptId: string
preferredRegion: 'asia-east2'
preferredRegion: RelayRegion
sourceCellId: string
sourceCellUrl: string
sourceCellIncarnation: string
@@ -151,6 +154,7 @@ export type RegionalRehomeControl = {
notBefore: number
ratePerMinute: number
preferenceMaxAgeMs: number
hostCooldownMs: number
drainGraceMs: number
}
@@ -4921,6 +4925,7 @@ export class RelayAssignmentStore {
notBefore: number
ratePerMinute: number
preferenceMaxAgeMs: number
hostCooldownMs: number
drainGraceMs: number
}): Promise<RegionalRehomeControl> {
if (!Number.isSafeInteger(input.expectedGeneration) || input.expectedGeneration < 0) {
@@ -4939,6 +4944,13 @@ export class RelayAssignmentStore {
) {
throw new Error('invalid_regional_rehome_preference_age')
}
if (
!Number.isSafeInteger(input.hostCooldownMs) ||
input.hostCooldownMs < 60_000 ||
input.hostCooldownMs > 30 * 24 * 60 * 60_000
) {
throw new Error('invalid_regional_rehome_host_cooldown')
}
if (
!Number.isSafeInteger(input.drainGraceMs) ||
input.drainGraceMs < 60_000 ||
@@ -4967,14 +4979,15 @@ export class RelayAssignmentStore {
await transaction.query(
`UPDATE relay_region_rehome_control
SET generation = generation + 1, enabled = ?, not_before = ?,
rate_per_minute = ?, preference_max_age_ms = ?, drain_grace_ms = ?,
updated_at = ?
rate_per_minute = ?, preference_max_age_ms = ?, host_cooldown_ms = ?,
drain_grace_ms = ?, updated_at = ?
WHERE control_id = 'global'`,
[
input.enabled ? 1 : 0,
input.notBefore,
input.ratePerMinute,
input.preferenceMaxAgeMs,
input.hostCooldownMs,
input.drainGraceMs,
now
]
@@ -5006,10 +5019,17 @@ export class RelayAssignmentStore {
await database.query(
`INSERT INTO relay_region_rehome_control
(control_id, generation, enabled, observation_started_at, not_before,
rate_per_minute, preference_max_age_ms, drain_grace_ms, updated_at)
VALUES ('global', 0, 0, ?, 0, 10, ?, ?, ?)
rate_per_minute, preference_max_age_ms, host_cooldown_ms, drain_grace_ms,
updated_at)
VALUES ('global', 0, 0, ?, 0, 10, ?, ?, ?, ?)
ON CONFLICT (control_id) DO NOTHING`,
[now, 24 * 60 * 60_000, 60 * 60_000, now]
[
now,
24 * 60 * 60_000,
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS,
60 * 60_000,
now
]
)
}
@@ -5017,6 +5037,9 @@ export class RelayAssignmentStore {
return await this.readRegionalRehomeFleetSafety(this.database, this.now())
}
// The rehome fleet is every general cell that can be drained: those are the
// sources and, because a host must be movable back out again, the only legal
// targets. The region join stays so a cell with no region row is excluded.
private async readRegionalRehomeFleetSafety(
database: RelayDatabase,
now: number
@@ -5037,10 +5060,7 @@ export class RelayAssignmentStore {
ON safety.cell_id = runtime.cell_id
AND safety.cell_incarnation = runtime.cell_incarnation
WHERE cell.enabled = 1 AND admission.admission_state = 'general'
AND (
region.region = 'asia-east2' OR
(region.region = 'us-central1' AND capability.regional_rehome_protocol >= 1)
)`
AND capability.regional_rehome_protocol >= 1`
)
const valid = rows.filter(
(row) =>
@@ -5119,6 +5139,10 @@ export class RelayAssignmentStore {
}
const intervalMs = Math.ceil(60_000 / integer(control, 'rate_per_minute'))
const preferenceCutoff = now - integer(control, 'preference_max_age_ms')
// A host that was rehomed recently is left alone whichever way its
// preference now points: a flapping region probe must not walk one host
// back and forth across an ocean.
const cooldownCutoff = now - integer(control, 'host_cooldown_ms')
await transaction.query(
`INSERT INTO relay_region_rehome_worker_state
(worker_id, next_dispatch_at, paused_until, consecutive_failures, updated_at)
@@ -5284,9 +5308,8 @@ export class RelayAssignmentStore {
JOIN relay_cell_capabilities capability
ON capability.cell_id = runtime.cell_id
AND capability.cell_incarnation = runtime.cell_incarnation
WHERE preference.preferred_region = 'asia-east2'
WHERE preference.preferred_region <> region.region
AND preference.observed_at >= ?
AND region.region = 'us-central1'
AND admission.admission_state = 'general'
AND runtime.ready = 1 AND runtime.last_heartbeat_at > ?
AND capability.regional_rehome_protocol >= 1
@@ -5306,9 +5329,38 @@ export class RelayAssignmentStore {
AND migration.relay_host_id = assignment.relay_host_id
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL
)
AND NOT EXISTS (
SELECT 1 FROM relay_region_rehome_attempts recent
WHERE recent.user_id = preference.user_id
AND recent.relay_host_id = preference.relay_host_id
AND recent.created_at > ?
)
AND EXISTS (
SELECT 1 FROM relay_cell_regions target_region
JOIN relay_cells target_cell ON target_cell.cell_id = target_region.cell_id
JOIN relay_cell_admission target_admission
ON target_admission.cell_id = target_region.cell_id
JOIN relay_cell_runtime target_runtime
ON target_runtime.cell_id = target_region.cell_id
JOIN relay_cell_capabilities target_capability
ON target_capability.cell_id = target_runtime.cell_id
AND target_capability.cell_incarnation = target_runtime.cell_incarnation
WHERE target_region.region = preference.preferred_region
AND target_cell.enabled = 1
AND target_admission.admission_state = 'general'
AND target_runtime.ready = 1
AND target_runtime.last_heartbeat_at > ?
AND target_capability.regional_rehome_protocol >= 1
)
ORDER BY preference.observed_at, preference.user_id, preference.relay_host_id
LIMIT 10`,
[preferenceCutoff, now - this.heartbeatTtlMs, now]
[
preferenceCutoff,
now - this.heartbeatTtlMs,
now,
cooldownCutoff,
now - this.heartbeatTtlMs
]
)
candidatesTotal = candidates.length
for (const candidate of candidates) {
@@ -5320,6 +5372,7 @@ export class RelayAssignmentStore {
sourceCellId: text(candidate, 'source_cell_id'),
assignmentEpoch: integer(candidate, 'assignment_epoch'),
preferenceCutoff,
cooldownCutoff,
drainGraceMs: integer(control, 'drain_grace_ms'),
processSafety: effectiveProcessSafety,
worker,
@@ -5374,6 +5427,7 @@ export class RelayAssignmentStore {
sourceCellId: string
assignmentEpoch: number
preferenceCutoff: number
cooldownCutoff: number
drainGraceMs: number
processSafety: RegionalRehomeSafetySnapshot
worker: SqlRow
@@ -5397,14 +5451,11 @@ export class RelayAssignmentStore {
[input.identity.userId, input.identity.relayHostId]
)
)[0]
if (
!preference ||
text(preference, 'preferred_region') !== 'asia-east2' ||
integer(preference, 'observed_at') < input.preferenceCutoff
) {
if (!preference || integer(preference, 'observed_at') < input.preferenceCutoff) {
input.skips.push({ reason: 'candidate_stale' })
return null
}
const preferredRegion = relayRegion(preference, 'preferred_region')
const activeMigration = await transaction.queryLocked(
`SELECT assignment_epoch FROM relay_assignment_migrations
WHERE user_id = ? AND relay_host_id = ?
@@ -5415,6 +5466,18 @@ export class RelayAssignmentStore {
input.skips.push({ reason: 'candidate_stale' })
return null
}
// Re-read under the claim: an attempt committed between the scan and here
// would otherwise start a second move for the same host.
const recentAttempt = await transaction.query(
`SELECT 1 FROM relay_region_rehome_attempts
WHERE user_id = ? AND relay_host_id = ? AND created_at > ?
LIMIT 1`,
[input.identity.userId, input.identity.relayHostId, input.cooldownCutoff]
)
if (recentAttempt.length > 0) {
input.skips.push({ reason: 'host_cooldown' })
return null
}
const activityLeases = await this.lockAssignmentActivities(transaction, input.identity)
assertAssignmentActivityCounts(assignment, activityLeases, 0)
const cells = await this.lockCellInventory(transaction, 'nowait')
@@ -5469,11 +5532,17 @@ export class RelayAssignmentStore {
)
return null
}
// The preference read under lock can now agree with the cell the host is
// already on: nothing to move, in either direction.
if (regions.get(input.sourceCellId) === preferredRegion) {
input.skips.push({ reason: 'candidate_stale' })
return null
}
if (
!source ||
integer(source, 'enabled') !== 1 ||
admission.get(input.sourceCellId) !== 'general' ||
regions.get(input.sourceCellId) !== RELAY_DEFAULT_REGION ||
regions.get(input.sourceCellId) === undefined ||
!sourceRuntime ||
integer(sourceRuntime, 'ready') !== 1 ||
integer(sourceRuntime, 'last_heartbeat_at') <= input.now - this.heartbeatTtlMs ||
@@ -5502,17 +5571,25 @@ export class RelayAssignmentStore {
return null
}
const connectionHeadroom = await this.connectionHeadroomByCell(transaction)
// A target must be drainable too, or the host lands somewhere it can never
// be rehomed out of again -- the trap this bidirectional move exists to undo.
const eligibleTargets = cells.filter((row) => {
const cellId = text(row, 'cell_id')
const runtime = runtimes.find((candidate) => text(candidate, 'cell_id') === cellId)
const capability = capabilities.find(
(candidate) => text(candidate, 'cell_id') === cellId
)
return (
cellId !== input.sourceCellId &&
integer(row, 'enabled') === 1 &&
admission.get(cellId) === 'general' &&
regions.get(cellId) === 'asia-east2' &&
regions.get(cellId) === preferredRegion &&
runtime !== undefined &&
integer(runtime, 'ready') === 1 &&
integer(runtime, 'last_heartbeat_at') > input.now - this.heartbeatTtlMs
integer(runtime, 'last_heartbeat_at') > input.now - this.heartbeatTtlMs &&
capability !== undefined &&
text(capability, 'cell_incarnation') === text(runtime, 'cell_incarnation') &&
integer(capability, 'regional_rehome_protocol') >= 1
)
})
const targetIsClean = (row: SqlRow): boolean => {
@@ -5668,12 +5745,13 @@ export class RelayAssignmentStore {
drain_grace_ms, send_attempts, last_send_attempt_at,
drain_receipt_at, drain_outcome, completed_at, aborted_at,
created_at, updated_at)
VALUES (?, ?, ?, 'asia-east2', ?, ?, ?, ?, ?, ?, ?, 0, NULL,
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, NULL,
NULL, NULL, NULL, NULL, ?, ?)`,
[
attemptId,
input.identity.userId,
input.identity.relayHostId,
preferredRegion,
input.sourceCellId,
text(sourceRuntime, 'cell_incarnation'),
targetCellId,
@@ -5688,7 +5766,7 @@ export class RelayAssignmentStore {
return {
...input.identity,
attemptId,
preferredRegion: 'asia-east2',
preferredRegion,
sourceCellId: input.sourceCellId,
sourceCellUrl: text(source, 'cell_url'),
sourceCellIncarnation: text(sourceRuntime, 'cell_incarnation'),
@@ -8101,7 +8179,7 @@ function regionalRehomeAttempt(row: SqlRow): RegionalRehomeAttempt {
attemptId: text(row, 'attempt_id'),
userId: text(row, 'user_id'),
relayHostId: text(row, 'relay_host_id'),
preferredRegion: 'asia-east2',
preferredRegion: relayRegion(row, 'preferred_region'),
sourceCellId: text(row, 'source_cell_id'),
sourceCellUrl: text(row, 'source_cell_url'),
sourceCellIncarnation: text(row, 'source_cell_incarnation'),
@@ -8122,6 +8200,7 @@ function regionalRehomeControl(row: SqlRow): RegionalRehomeControl {
notBefore: integer(row, 'not_before'),
ratePerMinute: integer(row, 'rate_per_minute'),
preferenceMaxAgeMs: integer(row, 'preference_max_age_ms'),
hostCooldownMs: integer(row, 'host_cooldown_ms'),
drainGraceMs: integer(row, 'drain_grace_ms')
}
}
@@ -8161,10 +8240,9 @@ function regionalRehomeFleetSafetyFromInventory(input: {
return (
integer(row, 'enabled') === 1 &&
input.admission.get(cellId) === 'general' &&
(input.regions.get(cellId) === 'asia-east2' ||
(input.regions.get(cellId) === RELAY_DEFAULT_REGION &&
capability !== undefined &&
integer(capability, 'regional_rehome_protocol') >= 1))
input.regions.get(cellId) !== undefined &&
capability !== undefined &&
integer(capability, 'regional_rehome_protocol') >= 1
)
})
const valid = required.flatMap((row) => {
@@ -8231,6 +8309,7 @@ function regionalRehomeFleetSafetyFailure(
type RegionalRehomeCandidateSkip = {
reason:
| 'candidate_stale'
| 'host_cooldown'
| 'source_ineligible'
| 'source_unclean'
| 'source_control_inactive'
@@ -1,4 +1,5 @@
import { randomUUID } from 'node:crypto'
import { RELAY_DEFAULT_REGION } from '@orca-cloud/relay-contract'
import type { RelayConfig } from './config.js'
import { googleMetadataIdentityToken } from './google-metadata-identity-token.js'
import type { RegionalRehomeSafetySnapshot } from './relay-observability.js'
@@ -57,7 +58,7 @@ export function startCellHeartbeat(
v: 1,
cellId: config.cellId,
cellUrl: config.cellUrl,
region: config.region ?? 'us-central1',
region: config.region ?? RELAY_DEFAULT_REGION,
cellIncarnation,
startedAt,
ready,
@@ -34,7 +34,11 @@ vi.mock('pg', () => ({
}
}))
import { openRelayDatabase, relayPostgresStatementTimeoutMs } from './database.js'
import {
openRelayDatabase,
POSTGRES_SCHEMA_MIGRATIONS,
relayPostgresStatementTimeoutMs
} from './database.js'
import { applyPostgresSchema } from './postgres-schema-startup.js'
const SCHEMA_POOL = {
@@ -118,7 +122,9 @@ describe('PostgreSQL relay deadlines', () => {
// Statements can open with a leading `--` rationale comment.
const body = (statement: string): string =>
statement.replace(/^(?:\s*--[^\n]*\n)*\s*/, '')
expect(ddl.every((statement) => /^CREATE\b/i.test(body(statement)))).toBe(true)
expect(
ddl.every((statement) => /^(?:CREATE|ALTER TABLE)\b/i.test(body(statement)))
).toBe(true)
// The backfill is DML, so it stays on the deadline-bearing serving pool.
expect(ddl.some((statement) => statement.includes('INSERT INTO'))).toBe(false)
await database.close()
@@ -263,6 +269,54 @@ describe('PostgreSQL schema startup', () => {
expect(query).toHaveBeenCalledTimes(2)
})
it('treats an existing constraint as an applied ADD CONSTRAINT', async () => {
// Postgres has no `ADD CONSTRAINT IF NOT EXISTS`, and a retry would only
// repeat 42710, so a re-run and a concurrent startup both move on.
const error = Object.assign(new Error('already exists'), { code: '42710' })
const query = vi
.fn<(statement: string) => Promise<unknown>>()
.mockRejectedValueOnce(error)
.mockResolvedValue(undefined)
const pause = vi.fn(async () => undefined)
await applyPostgresSchema(
['ALTER TABLE test ADD CONSTRAINT test_check CHECK (id > 0)', 'CREATE TABLE test2'],
query,
{ wait: pause }
)
expect(pause).not.toHaveBeenCalled()
expect(query).toHaveBeenCalledTimes(2)
expect(query).toHaveBeenLastCalledWith('CREATE TABLE test2')
})
it('recognises every shipped ADD CONSTRAINT migration as re-runnable', async () => {
// Guards the statement text against the pattern that classifies it.
const shipped = POSTGRES_SCHEMA_MIGRATIONS.filter((statement) =>
statement.includes('ADD CONSTRAINT')
)
expect(shipped.length).toBeGreaterThan(0)
const error = Object.assign(new Error('already exists'), { code: '42710' })
const query = vi.fn<(statement: string) => Promise<unknown>>().mockRejectedValue(error)
await applyPostgresSchema(shipped, query, { wait: async () => undefined })
expect(query).toHaveBeenCalledTimes(shipped.length)
})
it('still fails an ADD CONSTRAINT that violates existing rows', async () => {
const error = Object.assign(new Error('check violation'), { code: '23514' })
const query = vi.fn<(statement: string) => Promise<unknown>>().mockRejectedValue(error)
await expect(
applyPostgresSchema(
['ALTER TABLE test ADD CONSTRAINT test_check CHECK (id > 0)'],
query,
{ wait: async () => undefined }
)
).rejects.toBe(error)
})
it.each([
['42710', 'CREATE INDEX IF NOT EXISTS test_index ON test(id)'],
['42710', 'CREATE TABLE test'],
+41 -1
View File
@@ -2,7 +2,12 @@ import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it } from 'vitest'
import { openInMemoryRelayDatabase, openRelayDatabase } from './database.js'
import {
openInMemoryRelayDatabase,
openRelayDatabase,
POSTGRES_SCHEMA_MIGRATIONS,
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS
} from './database.js'
const temporaryDirectories: string[] = []
@@ -142,6 +147,41 @@ describe('relay database', () => {
await second.close()
})
it('renders every region check from the shared region list', async () => {
// Derived, not hand-written: a third region must not leave one column
// rejecting a value the rest of the relay already accepts.
const database = await openInMemoryRelayDatabase()
const checked = await database.query(
`SELECT name, sql FROM sqlite_master
WHERE type = 'table'
AND name IN ('relay_assignment_region_preferences', 'relay_cell_regions',
'relay_region_rehome_attempts')
ORDER BY name`
)
const list = `IN ('us-central1', 'asia-east2')`
expect(checked.map((row) => row.name)).toEqual([
'relay_assignment_region_preferences',
'relay_cell_regions',
'relay_region_rehome_attempts'
])
expect(checked.every((row) => String(row.sql).includes(list))).toBe(true)
expect(
POSTGRES_SCHEMA_MIGRATIONS.some((statement) => statement.includes(list))
).toBe(true)
await database.close()
})
it('indexes rehome attempts by host recency for the per-host cooldown', async () => {
const database = await openInMemoryRelayDatabase()
const rows = await database.query(
`SELECT sql FROM sqlite_master
WHERE type = 'index' AND name = 'relay_region_rehome_attempts_host_recency'`
)
expect(rows[0]?.sql).toContain('(user_id, relay_host_id, created_at)')
expect(REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS).toBe(7 * 24 * 60 * 60_000)
await database.close()
})
it('indexes region preference expiry by observation time', async () => {
const database = await openInMemoryRelayDatabase()
const rows = await database.query(
+37 -4
View File
@@ -3,6 +3,7 @@ import { performance } from 'node:perf_hooks'
import { join } from 'node:path'
import { DatabaseSync } from 'node:sqlite'
import pg from 'pg'
import { RELAY_REGIONS } from '@orca-cloud/relay-contract'
import {
emptyPostgresPoolPressureCounts,
PostgresPoolPressure,
@@ -24,6 +25,14 @@ function setLocalLockTimeout(milliseconds: number): string {
return `SET LOCAL lock_timeout = '${milliseconds}ms'`
}
// Region CHECK lists come from the contract so a new region cannot leave a
// column rejecting values the rest of the relay already accepts.
const REGION_LIST = RELAY_REGIONS.map((region) => `'${region}'`).join(', ')
// A host that was just moved is not a candidate again for this long, so a
// desktop whose region probe flips cannot walk itself back and forth.
export const REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS = 7 * 24 * 60 * 60_000
export type SqlRow = Record<string, unknown>
export type RelayLockOptions = {
failIfUnavailable?: boolean
@@ -181,7 +190,7 @@ CREATE TABLE IF NOT EXISTS relay_assignment_region_preferences (
user_id TEXT NOT NULL,
relay_host_id TEXT NOT NULL,
preferred_region TEXT NOT NULL
CHECK (preferred_region IN ('us-central1', 'asia-east2')),
CHECK (preferred_region IN (${REGION_LIST})),
observed_at BIGINT NOT NULL,
PRIMARY KEY (user_id, relay_host_id)
);
@@ -204,6 +213,8 @@ CREATE TABLE IF NOT EXISTS relay_region_rehome_control (
not_before BIGINT NOT NULL,
rate_per_minute BIGINT NOT NULL,
preference_max_age_ms BIGINT NOT NULL,
host_cooldown_ms BIGINT NOT NULL
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS},
drain_grace_ms BIGINT NOT NULL,
updated_at BIGINT NOT NULL
);
@@ -212,7 +223,9 @@ CREATE TABLE IF NOT EXISTS relay_region_rehome_attempts (
attempt_id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
relay_host_id TEXT NOT NULL,
preferred_region TEXT NOT NULL CHECK (preferred_region = 'asia-east2'),
preferred_region TEXT NOT NULL
CONSTRAINT relay_region_rehome_attempts_preferred_region_valid
CHECK (preferred_region IN (${REGION_LIST})),
source_cell_id TEXT NOT NULL,
source_cell_incarnation TEXT NOT NULL,
target_cell_id TEXT NOT NULL,
@@ -234,6 +247,8 @@ CREATE TABLE IF NOT EXISTS relay_region_rehome_attempts (
);
CREATE INDEX IF NOT EXISTS relay_region_rehome_attempts_pending
ON relay_region_rehome_attempts(drain_receipt_at, last_send_attempt_at, completed_at, aborted_at);
CREATE INDEX IF NOT EXISTS relay_region_rehome_attempts_host_recency
ON relay_region_rehome_attempts(user_id, relay_host_id, created_at);
CREATE TABLE IF NOT EXISTS relay_cells (
cell_id TEXT PRIMARY KEY,
@@ -248,7 +263,7 @@ CREATE TABLE IF NOT EXISTS relay_cells (
CREATE TABLE IF NOT EXISTS relay_cell_regions (
cell_id TEXT PRIMARY KEY,
region TEXT NOT NULL CHECK (region IN ('us-central1', 'asia-east2'))
region TEXT NOT NULL CHECK (region IN (${REGION_LIST}))
);
CREATE TABLE IF NOT EXISTS relay_cell_admission (
@@ -580,6 +595,21 @@ CREATE TABLE IF NOT EXISTS relay_audit_events (
CREATE INDEX IF NOT EXISTS relay_audit_events_at ON relay_audit_events(at);
`
// Rehoming is bidirectional, but tables created before that carry the
// original single-region column check. The old constraint is the one Postgres
// auto-named; the replacement is named, so both statements are no-ops on a
// database the current schema created and neither can drop the other.
export const POSTGRES_SCHEMA_MIGRATIONS = [
`ALTER TABLE relay_region_rehome_attempts
DROP CONSTRAINT IF EXISTS relay_region_rehome_attempts_preferred_region_check`,
`ALTER TABLE relay_region_rehome_attempts
ADD CONSTRAINT relay_region_rehome_attempts_preferred_region_valid
CHECK (preferred_region IN (${REGION_LIST}))`,
`ALTER TABLE relay_region_rehome_control
ADD COLUMN IF NOT EXISTS host_cooldown_ms BIGINT NOT NULL
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS}`
]
function postgresSql(sql: string): string {
let index = 0
return sql.replace(/\?/g, () => `$${++index}`)
@@ -1009,7 +1039,10 @@ async function applySchemaOnUntimedPool(
const database = new PostgresDatabase(pool)
try {
await applyPostgresSchema(
SCHEMA.split(';').filter((statement) => statement.trim()),
[
...SCHEMA.split(';').filter((statement) => statement.trim()),
...POSTGRES_SCHEMA_MIGRATIONS
],
async (statement) => await database.query(statement)
)
} finally {
@@ -1,5 +1,5 @@
import { EventEmitter } from 'node:events'
import { RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract'
import { RELAY_CLOSE_CODE, RELAY_PROTOCOL_LIMITS } from '@orca-cloud/relay-contract'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type WebSocket from 'ws'
import type { RelayAssignmentStore } from './assignment-store.js'
@@ -102,7 +102,9 @@ function harness(options: { random?: () => number; now?: () => number } = {}) {
const store = {
resolveResume: vi.fn().mockResolvedValue({ userId: identity.sub }),
reserveCredential: vi.fn().mockResolvedValue(reservation),
failReservation: vi.fn().mockResolvedValue(undefined)
failReservation: vi.fn().mockResolvedValue(undefined),
recordConnectionBasis: vi.fn().mockResolvedValue(undefined),
deactivateBasis: vi.fn().mockResolvedValue(undefined)
}
const observer = {
recordAuth: vi.fn(),
@@ -110,7 +112,9 @@ function harness(options: { random?: () => number; now?: () => number } = {}) {
recordHttp: vi.fn(),
recordReconnect: vi.fn(),
recordSql: vi.fn(),
recordClientAcceptAbandoned: vi.fn()
recordClientAcceptAbandoned: vi.fn(),
recordClientAcceptCompleted: vi.fn(),
recordControlRtt: vi.fn()
} satisfies RelayRuntimeObserver
const registry = new HostSessionRegistry(
config,
@@ -325,6 +329,210 @@ describe('client accept abandoned mid-DB-phase', () => {
})
})
describe('successful client accept timing', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
it('times every serialized stage plus the attach window once relay-hello lands', async () => {
let now = 1_700_000_000_000
const h = harness({ now: () => now })
const control = await activeHost(h)
h.store.resolveResume.mockImplementationOnce(async () => {
now += 5
return { userId: identity.sub }
})
h.store.reserveCredential.mockImplementationOnce(async () => {
now += 7
return reservation
})
h.acquireActivity.mockImplementationOnce(async () => {
now += 11
})
h.store.recordConnectionBasis.mockImplementationOnce(async () => {
now += 3
})
const client = new FakeSocket()
const hostData = new FakeSocket()
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
try {
await h.registry.acceptClient(client as unknown as WebSocket, identity.relayHostId, 'cred')
const connOpen = JSON.parse(
String(control.send.mock.calls.find((call) => String(call[0]).includes('conn-open'))![0])
) as { connId: string; connTicket: string }
// The desktop's data leg is the attach window this is meant to expose.
now += 23
const accepted = await h.registry.acceptHostData(
hostData as unknown as WebSocket,
connOpen.connId,
connOpen.connTicket,
1
)
expect(accepted).toBe(true)
expect(h.observer.recordClientAcceptCompleted).toHaveBeenCalledWith({
totalMs: 49,
stageMs: { assignment: 5, credential: 7, activity: 11, attach: 23, basis: 3 }
})
const line = log.mock.calls
.map((call) => String(call[0]))
.find((entry) => entry.includes('orca_relay_client_accept_completed'))
expect(line).toBeDefined()
const event = JSON.parse(line!) as {
role: string
cellId: string
region: string
credentialKind: string
stageMs: Record<string, number>
totalMs: number
relayHostIdDigest: string
}
expect(event.credentialKind).toBe('resume')
// Joins the line back to the emitting process, like the runtime metrics event.
expect(event).toMatchObject({ role: 'cell', cellId: config.cellId, region: 'us-central1' })
expect(Object.keys(event.stageMs).sort()).toEqual([
'activity',
'assignment',
'attach',
'basis',
'credential'
])
for (const stage of Object.values(event.stageMs)) expect(stage).toBeGreaterThanOrEqual(0)
// The stages tile the accept end to end: every millisecond is attributed.
const summed = Object.values(event.stageMs).reduce((total, stage) => total + stage, 0)
expect(summed).toBe(event.totalMs)
expect(event.relayHostIdDigest).toMatch(/^[0-9a-f]{12}$/)
expect(line).not.toContain(identity.relayHostId)
} finally {
log.mockRestore()
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})
})
// Fires one heartbeat and returns the `t` of the ping it sent, which is the only
// echo the registry will time.
async function advanceToPing(control: FakeSocket, clock: { now: number }): Promise<number> {
clock.now += RELAY_PROTOCOL_LIMITS.controlPingIntervalMs
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
const ping = control.send.mock.calls
.filter((call) => String(call[0]).includes('"type":"ping"'))
.at(-1)!
return (JSON.parse(String(ping[0])) as { t: number }).t
}
describe('control round-trip sampling', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
it('logs a host once at the fourth sample and not again within the hour', async () => {
const clock = { now: 1_700_000_000_000 }
const h = harness({ now: () => clock.now })
const control = await activeHost(h)
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
const rttLines = (): string[] =>
log.mock.calls
.map((call) => String(call[0]))
.filter((entry) => entry.includes('orca_relay_host_control_rtt'))
// One heartbeat, then the desktop's echo of that ping's own `t` 40 ms later.
const roundTrip = async (): Promise<void> => {
const pingAt = await advanceToPing(control, clock)
clock.now += 40
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt }), false)
}
try {
for (let round = 0; round < 3; round++) await roundTrip()
expect(h.observer.recordControlRtt).toHaveBeenCalledTimes(3)
expect(rttLines()).toHaveLength(0)
await roundTrip()
expect(h.observer.recordControlRtt).toHaveBeenLastCalledWith(40)
expect(rttLines()).toHaveLength(1)
expect(JSON.parse(rttLines()[0]!)).toMatchObject({
event: 'orca_relay_host_control_rtt',
role: 'cell',
cellId: config.cellId,
region: 'us-central1',
rttMsMedian: 40,
sampleCount: 4
})
expect(rttLines()[0]).not.toContain(identity.relayHostId)
// Later samples keep feeding the fleet metric, but stay silent for an hour.
for (let round = 0; round < 8; round++) await roundTrip()
expect(h.observer.recordControlRtt).toHaveBeenCalledTimes(12)
expect(rttLines()).toHaveLength(1)
const elapsedStart = clock.now
while (clock.now - elapsedStart < 60 * 60 * 1000) await roundTrip()
expect(rttLines()).toHaveLength(2)
} finally {
log.mockRestore()
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})
it('ignores a pong that answers no outstanding ping', async () => {
const clock = { now: 1_700_000_000_000 }
const h = harness({ now: () => clock.now })
const control = await activeHost(h)
try {
// Nothing has been pinged yet, so even a plausible echo is not a round trip.
control.emit('message', JSON.stringify({ type: 'pong' }), false)
control.emit('message', JSON.stringify({ type: 'pong', t: 'later' }), false)
control.emit('message', JSON.stringify({ type: 'pong', t: clock.now }), false)
control.emit('message', JSON.stringify({ type: 'pong', t: clock.now - 10 }), false)
expect(h.observer.recordControlRtt).not.toHaveBeenCalled()
const pingAt = await advanceToPing(control, clock)
// A guessed timestamp is not the outstanding ping's `t`, so it is dropped.
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt - 1 }), false)
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt + 1 }), false)
expect(h.observer.recordControlRtt).not.toHaveBeenCalled()
clock.now += 10
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt }), false)
expect(h.observer.recordControlRtt).toHaveBeenCalledWith(10)
} finally {
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})
it('records one sample per ping however many pongs a host floods', async () => {
const clock = { now: 1_700_000_000_000 }
const h = harness({ now: () => clock.now })
const control = await activeHost(h)
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
try {
const pingAt = await advanceToPing(control, clock)
clock.now += 12
for (let flood = 0; flood < 5_000; flood++) {
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt }), false)
control.emit('message', JSON.stringify({ type: 'pong', t: clock.now }), false)
}
// One answered ping is one process-wide sample and one per-session sample, so
// neither the metric window nor the hourly log line can be flooded.
expect(h.observer.recordControlRtt).toHaveBeenCalledTimes(1)
expect(h.observer.recordControlRtt).toHaveBeenCalledWith(12)
expect(
log.mock.calls.filter((call) => String(call[0]).includes('orca_relay_host_control_rtt'))
).toHaveLength(0)
} finally {
log.mockRestore()
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})
})
describe('control lease jitter', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
@@ -3,6 +3,7 @@ import {
ASSIGNMENT_LIMITS,
CONTROL_CONTINUITY_LIMITS,
RELAY_CLOSE_CODE,
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS,
RELAY_PROTOCOL_LIMITS
} from '@orca-cloud/relay-contract'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
@@ -1005,3 +1006,115 @@ describe('control lease recovery after the session is gone', () => {
}
})
})
describe('host hello ack pending connections', () => {
const DETAILS = new Set([RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS])
const LEGACY_ENTRY = { connId: 'conn-1', connTicket: 'T'.repeat(43) }
const DETAILED_ENTRY = { ...LEGACY_ENTRY, kind: 'invite', relayDeviceId: 'device-1' }
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
function newRegistry(): ReturnType<typeof createRegistry> {
return createRegistry(
vi
.fn<RelayAssignmentStore['activateControl']>()
.mockResolvedValue('control:production-gce-c3:1')
)
}
function addPendingConnection(session: HostSession): void {
session.pendingConns.set('conn-1', {
...LEGACY_ENTRY,
reservation: {
userId: identity.sub,
relayHostId: identity.relayHostId,
credentialKind: 'invite',
relayDeviceId: 'device-1'
},
client: new FakeSocket() as unknown as WebSocket,
attachTimer: setTimeout(() => {}, 60_000),
credentialActivityId: null
} as unknown as Parameters<typeof session.pendingConns.set>[1])
}
function sentAck(socket: FakeSocket): Record<string, unknown> {
const acks = socket.send.mock.calls
.map((call) => JSON.parse(String(call[0])) as Record<string, unknown>)
.filter((message) => message.type === 'host-hello-ack')
return acks.at(-1)!
}
function sessionOf(registry: HostSessionRegistry): HostSession {
return registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
}
async function ackFor(capabilities?: ReadonlySet<string>): Promise<Record<string, unknown>> {
const { registry, activate } = newRegistry()
const socket = new FakeSocket()
registry.acceptControl(
socket as unknown as WebSocket,
identity,
undefined,
capabilities ?? new Set()
)
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
const session = sessionOf(registry)
addPendingConnection(session)
socket.send.mockClear()
;(registry as unknown as { sendHelloAck(session: HostSession): void }).sendHelloAck(session)
return sentAck(socket)
}
async function ackAfterRebind(
first: ReadonlySet<string>,
successor: ReadonlySet<string>
): Promise<{ opening: Record<string, unknown>; rebound: Record<string, unknown> }> {
const { registry, activate } = newRegistry()
const opening = new FakeSocket()
registry.acceptControl(opening as unknown as WebSocket, identity, undefined, first)
await activate(opening as unknown as WebSocket, identity, null, 1, false, 1)
const session = sessionOf(registry)
addPendingConnection(session)
opening.send.mockClear()
;(registry as unknown as { sendHelloAck(session: HostSession): void }).sendHelloAck(session)
const rebound = new FakeSocket()
registry.acceptControl(rebound as unknown as WebSocket, identity, undefined, successor)
await activate(rebound as unknown as WebSocket, identity, session, 1, true, 1)
return { opening: sentAck(opening), rebound: sentAck(rebound) }
}
it('states the pending kind and device to a host that advertised it can read them', async () => {
const ack = await ackFor(DETAILS)
expect(ack.pendingConns).toEqual([DETAILED_ENTRY])
})
it('restates only the identifiers to a host that never advertised the capability', async () => {
// A shipped host parses these entries strictly, so an unannounced key fails
// the whole ack parse and kills a control that was working.
const ack = await ackFor()
expect(ack.pendingConns).toEqual([LEGACY_ENTRY])
})
it('downgrades the restated entry when the successor control drops the capability', async () => {
// The capability belongs to the socket, not the session: a rebind can land a
// control whose decoder is older than the one that opened the session.
const { opening, rebound } = await ackAfterRebind(DETAILS, new Set())
expect(opening.pendingConns).toEqual([DETAILED_ENTRY])
expect(rebound.pendingConns).toEqual([LEGACY_ENTRY])
})
it('upgrades the restated entry when the successor control adds the capability', async () => {
const { opening, rebound } = await ackAfterRebind(new Set(), DETAILS)
expect(opening.pendingConns).toEqual([LEGACY_ENTRY])
expect(rebound.pendingConns).toEqual([DETAILED_ENTRY])
})
})
+156 -5
View File
@@ -1,6 +1,7 @@
import { createHash, createHmac, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto'
import {
ASSIGNMENT_LIMITS,
RELAY_DEFAULT_REGION,
AuthRefreshSchema,
buildHostChallengePlaintext,
buildHostProofMacInput,
@@ -13,9 +14,11 @@ import {
HostChallengeAckSchema,
HostHelloSchema,
InviteCreateSchema,
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS,
RELAY_PROTOCOL_LIMITS,
RELAY_CLOSE_CODE,
type RelayHostCloseReason
type RelayHostCloseReason,
type RelayRegion
} from '@orca-cloud/relay-contract'
import nacl from 'tweetnacl'
import type WebSocket from 'ws'
@@ -29,7 +32,12 @@ import {
import { HostCloseReasonMemory } from './host-close-reason-memory.js'
import { relayHostLogDigest } from './relay-host-log-digest.js'
import type { RelayTokenClaims } from './relay-token-verifier.js'
import type { RelayClientAcceptStage, RelayRuntimeObserver } from './relay-observability.js'
import {
percentile,
type RelayClientAcceptStage,
type RelayClientAcceptTimedStage,
type RelayRuntimeObserver
} from './relay-observability.js'
import type { PendingHostDataReservation } from './relay-connection-ledger.js'
import { closeRelayWebSocket } from './relay-websocket-close.js'
import { ProcessQueuedByteBudget, wireSplice } from './splice-forwarder.js'
@@ -45,6 +53,20 @@ function printableCloseReason(reason: Buffer | string): string {
type VerifyRelayToken = (token: string) => Promise<RelayTokenClaims | null>
type HostState = 'proving' | 'active' | 'orphaned' | 'drain-only' | 'closed'
// A host's distance to its cell moves on the scale of a rehome, not a heartbeat,
// so a short window is enough to ride out one stalled ping.
const CONTROL_RTT_WINDOW = 8
const CONTROL_RTT_LOG_SAMPLE_THRESHOLD = 4
const CONTROL_RTT_LOG_INTERVAL_MS = 60 * 60 * 1000
// A pong claiming a multi-minute round trip is clock skew, not distance.
const CONTROL_RTT_MAX_PLAUSIBLE_MS = 120_000
// Wall clock can step backwards mid-accept; a negative latency would poison the
// percentiles it feeds.
function nonNegativeMs(elapsedMs: number): number {
return Math.max(0, elapsedMs)
}
const CONTROL_ACTIVITY_RENEWAL_INTERVAL_MS = RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2
// Preserve the existing 75s renewal runway after doubling the successful-call interval.
const CONTROL_ACTIVITY_LEASE_MS =
@@ -68,6 +90,10 @@ export type HostSession = {
orphanTimer: ReturnType<typeof setTimeout> | null
heartbeatTimer: ReturnType<typeof setInterval> | null
lastPongAt: number
// The `t` of the ping still waiting for its echo; null once one has answered it.
pendingPingAt: number | null
controlRttSamplesMs: number[]
controlRttLoggedAt: number | null
activityRenewalDueAt: number
activityRenewalAttempt: number
activityRenewalCompletedAttempt: number
@@ -95,6 +121,15 @@ type PendingConnection = {
attachTimer: ReturnType<typeof setTimeout>
credentialActivityId: string | null
capacityReservation?: PendingHostDataReservation
timing: ClientAcceptTiming
}
// Carries the phone-side accept clock across to the desktop's data leg, which
// lands in a separate call and is the only place the accept is known to succeed.
type ClientAcceptTiming = {
startedAt: number
connOpenAt: number
stageMs: Record<RelayClientAcceptStage, number>
}
function decodeCanonicalBase64(value: string, bytes: number): Uint8Array | null {
@@ -146,6 +181,7 @@ export class HostSessionRegistry {
// but a signed-out desktop never comes back, so the phone that asks minutes
// later would otherwise find nothing to explain its rejection with.
private readonly hostCloseReasons = new HostCloseReasonMemory(() => this.now())
private readonly hostCapabilities = new WeakMap<WebSocket, ReadonlySet<string>>()
private draining = false
constructor(
@@ -192,6 +228,17 @@ export class HostSessionRegistry {
)
return true
}
const stageMs: Record<RelayClientAcceptStage, number> = {
assignment: 0,
credential: 0,
activity: 0
}
let stageCursor = acceptStartedAt
const markStage = (stage: RelayClientAcceptStage): void => {
const at = this.now()
stageMs[stage] = at - stageCursor
stageCursor = at
}
if (this.config.role === 'cell') {
// Each lookup is its own pooled round trip; stop between them once the phone
// has left instead of running the rest of the chain for nobody.
@@ -212,6 +259,7 @@ export class HostSessionRegistry {
}
if (abandonedByClient('assignment')) return
}
markStage('assignment')
const reservation = await this.store.reserveCredential(hostId, credential)
if (!reservation) {
capacityReservation?.release()
@@ -221,6 +269,7 @@ export class HostSessionRegistry {
}
this.observer.recordAuth(true)
if (abandonedByClient('credential', () => this.failReservationBestEffort(reservation))) return
markStage('credential')
const sessionKey = this.key(reservation.userId, hostId)
const session = this.sessions.get(sessionKey)
if (
@@ -275,6 +324,7 @@ export class HostSessionRegistry {
) {
return
}
markStage('activity')
const attachTimer = setTimeout(() => {
session.pendingConns.delete(connId)
capacityReservation?.release()
@@ -289,7 +339,10 @@ export class HostSessionRegistry {
client: socket,
attachTimer,
credentialActivityId,
capacityReservation
capacityReservation,
// Attach starts where the activity stage ended, so the conn-open send is
// charged to it and no wall-clock gap goes unattributed.
timing: { startedAt: acceptStartedAt, connOpenAt: stageCursor, stageMs }
}
capacityReservation?.bind(connId)
session.pendingConns.set(connId, pending)
@@ -336,6 +389,7 @@ export class HostSessionRegistry {
return false
}
this.observer.recordAuth(true)
const attachedAt = this.now()
clearTimeout(pending.attachTimer)
session.pendingConns.delete(connId)
session.activeConnIds.add(connId)
@@ -409,6 +463,7 @@ export class HostSessionRegistry {
close()
return false
}
const helloAt = this.now()
send(pending.client, 'relay-hello', {
ok: true,
credentialKind: pending.reservation.credentialKind,
@@ -424,14 +479,92 @@ export class HostSessionRegistry {
}
: {})
})
this.recordClientAcceptCompleted(session, pending, attachedAt, helloAt)
return true
}
// The stages tile the whole accept, so their sum is the total minus only the
// clamping above: `basis` is the splice lease and connection-basis writes that
// land between the host data leg and relay-hello.
private recordClientAcceptCompleted(
session: HostSession,
pending: PendingConnection,
attachedAt: number,
helloAt: number
): void {
const stageMs: Record<RelayClientAcceptTimedStage, number> = {
assignment: nonNegativeMs(pending.timing.stageMs.assignment),
credential: nonNegativeMs(pending.timing.stageMs.credential),
activity: nonNegativeMs(pending.timing.stageMs.activity),
attach: nonNegativeMs(attachedAt - pending.timing.connOpenAt),
basis: nonNegativeMs(helloAt - attachedAt)
}
const totalMs = nonNegativeMs(helloAt - pending.timing.startedAt)
this.observer.recordClientAcceptCompleted?.({ totalMs, stageMs })
console.log(
JSON.stringify({
event: 'orca_relay_client_accept_completed',
...this.logIdentity(),
credentialKind: pending.reservation.credentialKind,
stageMs,
totalMs,
relayHostIdDigest: relayHostLogDigest(session.relayHostId)
})
)
}
// Matches the runtime metrics event so a log line and a metric point can be
// joined back to the process that emitted them.
private logIdentity(): { role: string; cellId: string; region: RelayRegion } {
return {
role: this.config.role,
cellId: this.config.cellId,
region: this.config.region ?? RELAY_DEFAULT_REGION
}
}
// Every desktop build already echoes the ping's `t`, so a pong is only timed when
// it answers the outstanding ping: at most one sample per ping this cell sent,
// however many a host floods. A pong that lost the race to the next ping is
// dropped here but still counts as proof of life for the silence watchdog.
private recordControlRtt(session: HostSession, echoedPingAt: unknown): void {
if (typeof echoedPingAt !== 'number' || echoedPingAt !== session.pendingPingAt) return
session.pendingPingAt = null
const now = this.now()
const rttMs = now - echoedPingAt
if (rttMs < 0 || rttMs > CONTROL_RTT_MAX_PLAUSIBLE_MS) return
this.observer.recordControlRtt?.(rttMs)
const samples = session.controlRttSamplesMs
samples.push(rttMs)
if (samples.length > CONTROL_RTT_WINDOW) samples.shift()
if (samples.length < CONTROL_RTT_LOG_SAMPLE_THRESHOLD) return
if (
session.controlRttLoggedAt !== null &&
now - session.controlRttLoggedAt < CONTROL_RTT_LOG_INTERVAL_MS
) {
return
}
session.controlRttLoggedAt = now
console.log(
JSON.stringify({
event: 'orca_relay_host_control_rtt',
...this.logIdentity(),
relayHostIdDigest: relayHostLogDigest(session.relayHostId),
rttMsMedian: percentile(samples, 0.5),
sampleCount: samples.length
})
)
}
acceptControl(
socket: WebSocket,
identity: RelayTokenClaims,
connectionInclusionWatermark?: number
connectionInclusionWatermark?: number,
hostCapabilities?: ReadonlySet<string>
): void {
// Keyed by socket, not session: a rebind swaps the session's socket, and the
// successor's own advertisement is the only one that describes its decoder.
if (hostCapabilities?.size) this.hostCapabilities.set(socket, hostCapabilities)
if (this.draining) {
socket.close(RELAY_CLOSE_CODE.DRAINING, 'relay draining')
return
@@ -790,6 +923,7 @@ export class HostSessionRegistry {
existing.appVersion = appVersion
existing.leaseExpiresAt = this.controlLeaseExpiresAt()
existing.lastPongAt = this.now()
existing.pendingPingAt = null
existing.activityRenewalDueAt =
this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs
this.wireActiveControl(existing)
@@ -843,6 +977,9 @@ export class HostSessionRegistry {
orphanTimer: null,
heartbeatTimer: null,
lastPongAt: this.now(),
pendingPingAt: null,
controlRttSamplesMs: [],
controlRttLoggedAt: null,
activityRenewalDueAt: this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs,
activityRenewalAttempt: 0,
activityRenewalCompletedAttempt: 0,
@@ -900,6 +1037,7 @@ export class HostSessionRegistry {
const parsed = JSON.parse(raw.toString()) as Record<string, unknown>
if (parsed.type === 'pong') {
session.lastPongAt = this.now()
this.recordControlRtt(session, parsed.t)
return
}
if (parsed.type === 'auth-refresh') {
@@ -1047,11 +1185,18 @@ export class HostSessionRegistry {
session.socket.close(RELAY_CLOSE_CODE.DRAINING, 'control lease expired')
return
}
session.pendingPingAt = now
send(session.socket, 'ping', { t: now })
}
private sendHelloAck(session: HostSession): void {
if (!session.socket) return
// Without these a host that missed the conn-open cannot dial the pending
// connection: it would have to guess the pairing kind and the device the
// relay authorized. Only sent to a host that said it can read them.
const details = this.hostCapabilities
.get(session.socket)
?.has(RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS)
send(session.socket, 'host-hello-ack', {
v: 1,
generation: session.generation,
@@ -1060,7 +1205,13 @@ export class HostSessionRegistry {
activeConnIds: [...session.activeConnIds],
pendingConns: [...session.pendingConns.values()].map((pending) => ({
connId: pending.connId,
connTicket: pending.connTicket
connTicket: pending.connTicket,
...(details
? {
kind: pending.reservation.credentialKind,
relayDeviceId: pending.reservation.relayDeviceId
}
: {})
}))
})
}
@@ -49,6 +49,20 @@ function concurrentCreateCollision(
return false
}
const ALTER_TABLE_ADD_CONSTRAINT =
/^\s*ALTER\s+TABLE\s+\S+\s+ADD\s+CONSTRAINT\b/i
// Postgres has no `ADD CONSTRAINT IF NOT EXISTS`, so a re-run and a concurrent
// startup both land on 42710 once the constraint exists. Unlike a CREATE race
// this is terminal, not transient: retrying only repeats it, so the statement
// counts as applied.
function constraintAlreadyApplied(error: unknown, statement: string): boolean {
return (
ALTER_TABLE_ADD_CONSTRAINT.test(statement) &&
(error as { code?: unknown }).code === '42710'
)
}
function retryableSchemaError(error: unknown, statement: string): boolean {
const value = error as { code?: unknown; constraint?: unknown }
return (
@@ -73,6 +87,7 @@ export async function applyPostgresSchema(
await query(statement)
break
} catch (error) {
if (constraintAlreadyApplied(error, statement)) break
const code = String((error as { code?: unknown }).code)
const remainingMs = deadlineAt - now()
const retryable = retryableSchemaError(error, statement)
@@ -315,6 +315,7 @@ describe('regional rehome director controls', () => {
notBefore: 100,
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: 7 * 24 * 60 * 60_000,
drainGraceMs: 60_000,
confirmation: 'ENABLE_REGIONAL_REHOMING'
}
@@ -343,6 +344,14 @@ describe('regional rehome director controls', () => {
'deploy-token',
{ ...apply, confirmation: 'DISABLE_REGIONAL_REHOMING' }
)).status).toBe(400)
// The per-host cooldown is part of the durable shape an operator must state.
const { hostCooldownMs: _omitted, ...withoutCooldown } = apply
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'deploy-token',
withoutCooldown
)).status).toBe(400)
})
it('probes dedicated trust twice and returns only aggregate proof', async () => {
@@ -411,6 +420,78 @@ describe('regional rehome director controls', () => {
expect(JSON.stringify(responseBody)).not.toContain('rehome-token')
})
it('probes a source cell in any region, not only the default one', async () => {
// Rehoming moves hosts in both directions, so an asia-east2 cell is a
// source too and its trust has to be provable the same way.
const cellDeploymentStatus = vi.fn().mockResolvedValue({
cellId: 'production-gce-c27',
cellUrl: 'https://c27.relay.example.test',
region: 'asia-east2',
runtime: {
cellIncarnation,
ready: true,
heartbeatFresh: true,
regionalRehomeProtocol: 1
}
})
const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
store: {} as never,
assignments: { cellDeploymentStatus } as never,
drain: vi.fn(),
regionalRehomeIdentityToken: vi.fn(async () => 'rehome-token'),
regionalRehomeFetch: (async () =>
Response.json({
v: 1,
outcome: 'host-not-connected',
sharedRuntimeIdentityRejected: true
})) as typeof fetch,
ready: vi.fn(async () => true)
})
const response = await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ v: 1, sourceCellId: 'production-gce-c27', sourceCellIncarnation: cellIncarnation }
)
expect(response.status).toBe(200)
expect(await response.json()).toMatchObject({ proven: true })
})
it('still refuses a trust probe against a cell without the drain protocol', async () => {
const cellDeploymentStatus = vi.fn().mockResolvedValue({
cellId: 'production-gce-c27',
cellUrl: 'https://c27.relay.example.test',
region: 'asia-east2',
runtime: {
cellIncarnation,
ready: true,
heartbeatFresh: true,
regionalRehomeProtocol: 0
}
})
const sourceFetch = vi.fn<typeof fetch>()
const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
store: {} as never,
assignments: { cellDeploymentStatus } as never,
drain: vi.fn(),
regionalRehomeIdentityToken: vi.fn(async () => 'rehome-token'),
regionalRehomeFetch: sourceFetch,
ready: vi.fn(async () => true)
})
const response = await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ v: 1, sourceCellId: 'production-gce-c27', sourceCellIncarnation: cellIncarnation }
)
expect(response.status).toBe(409)
expect(sourceFetch).not.toHaveBeenCalled()
})
it('restricts trust probes to deploy authorization and strict input', async () => {
const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
store: {} as never,
@@ -0,0 +1,195 @@
import pg from 'pg'
import { afterAll, beforeEach, describe, expect, it } from 'vitest'
import {
openRelayDatabase,
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS,
type RelayDatabase
} from './database.js'
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
const schema = 'relay_rehome_constraint_migration_test'
// The shape shipped before rehoming became bidirectional: a single-region
// column check that Postgres auto-names.
const LEGACY_ATTEMPTS_TABLE = `
CREATE TABLE relay_region_rehome_attempts (
attempt_id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
relay_host_id TEXT NOT NULL,
preferred_region TEXT NOT NULL CHECK (preferred_region = 'asia-east2'),
source_cell_id TEXT NOT NULL,
source_cell_incarnation TEXT NOT NULL,
target_cell_id TEXT NOT NULL,
target_cell_incarnation TEXT NOT NULL,
previous_epoch BIGINT NOT NULL,
assignment_epoch BIGINT NOT NULL,
drain_grace_ms BIGINT NOT NULL,
send_attempts BIGINT NOT NULL,
last_send_attempt_at BIGINT,
drain_receipt_at BIGINT,
drain_outcome TEXT CHECK (
drain_outcome IN ('accepted', 'already-accepted', 'host-not-connected')
),
completed_at BIGINT,
aborted_at BIGINT,
created_at BIGINT NOT NULL,
updated_at BIGINT NOT NULL,
UNIQUE (user_id, relay_host_id, assignment_epoch)
)`
// The control row as it shipped before the per-host cooldown existed.
const LEGACY_CONTROL_TABLE = `
CREATE TABLE relay_region_rehome_control (
control_id TEXT PRIMARY KEY,
generation BIGINT NOT NULL,
enabled BIGINT NOT NULL,
observation_started_at BIGINT NOT NULL,
not_before BIGINT NOT NULL,
rate_per_minute BIGINT NOT NULL,
preference_max_age_ms BIGINT NOT NULL,
drain_grace_ms BIGINT NOT NULL,
updated_at BIGINT NOT NULL
)`
const attemptValues = (attemptId: string, preferredRegion: string): unknown[] => [
attemptId,
'user-1',
'abcdefghijklmnop',
preferredRegion,
'cell-source',
'11111111-1111-4111-8111-111111111111',
'cell-target',
'22222222-2222-4222-8222-222222222222',
1,
Number(attemptId.at(-1)),
0,
0,
1_000_000,
1_000_000
]
const INSERT_ATTEMPT = `INSERT INTO relay_region_rehome_attempts
(attempt_id, user_id, relay_host_id, preferred_region, source_cell_id,
source_cell_incarnation, target_cell_id, target_cell_incarnation,
previous_epoch, assignment_epoch, drain_grace_ms, send_attempts,
created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`
describePostgres('PostgreSQL regional rehome constraint migration', () => {
let scopedUrl = ''
async function withClient(
operation: (client: pg.Client) => Promise<void>
): Promise<void> {
const client = new pg.Client({ connectionString: databaseUrl })
await client.connect()
try {
await operation(client)
} finally {
await client.end()
}
}
beforeEach(async () => {
await withClient(async (client) => {
await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
await client.query(`CREATE SCHEMA ${schema}`)
await client.query(`SET search_path = ${schema}`)
await client.query(LEGACY_ATTEMPTS_TABLE)
await client.query(LEGACY_CONTROL_TABLE)
await client.query(
`INSERT INTO relay_region_rehome_control
(control_id, generation, enabled, observation_started_at, not_before,
rate_per_minute, preference_max_age_ms, drain_grace_ms, updated_at)
VALUES ('global', 3, 0, 1, 0, 10, 86400000, 60000, 1)`
)
// Production data the replacement constraint has to validate.
await client.query(INSERT_ATTEMPT, attemptValues('attempt-1', 'asia-east2'))
})
const url = new URL(databaseUrl!)
url.searchParams.set('options', `-c search_path=${schema}`)
scopedUrl = url.toString()
})
afterAll(async () => {
await withClient(async (client) => {
await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
})
})
it('upgrades a legacy single-region constraint in place', async () => {
const database = await openRelayDatabase({ databaseUrl: scopedUrl, dataDir: '' })
try {
await withClient(async (client) => {
await client.query(`SET search_path = ${schema}`)
await client.query(INSERT_ATTEMPT, attemptValues('attempt-2', 'us-central1'))
await expect(
client.query(INSERT_ATTEMPT, attemptValues('attempt-3', 'europe-west1'))
).rejects.toMatchObject({ code: '23514' })
const constraints = await client.query(
`SELECT conname FROM pg_constraint
WHERE conrelid = 'relay_region_rehome_attempts'::regclass
AND conname LIKE '%preferred_region%'
ORDER BY conname`
)
expect(constraints.rows).toEqual([
{ conname: 'relay_region_rehome_attempts_preferred_region_valid' }
])
// The existing control row keeps its tuning and gains the cooldown.
const control = await client.query(
`SELECT generation, preference_max_age_ms, host_cooldown_ms
FROM relay_region_rehome_control WHERE control_id = 'global'`
)
expect(control.rows).toEqual([
{
generation: '3',
preference_max_age_ms: '86400000',
host_cooldown_ms: String(REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS)
}
])
})
} finally {
await database.close()
}
})
it('upgrades once across concurrent startups', async () => {
const results = await Promise.allSettled(
Array.from(
{ length: 5 },
async (): Promise<RelayDatabase> =>
await openRelayDatabase({ databaseUrl: scopedUrl, dataDir: '' })
)
)
const databases = results.flatMap((result) =>
result.status === 'fulfilled' ? [result.value] : []
)
await Promise.all(databases.map(async (database) => await database.close()))
expect(
results.flatMap((result) =>
result.status === 'rejected'
? [
{
code: (result.reason as { code?: unknown }).code,
message: String(result.reason)
}
]
: []
)
).toEqual([])
await withClient(async (client) => {
await client.query(`SET search_path = ${schema}`)
await client.query(INSERT_ATTEMPT, attemptValues('attempt-4', 'us-central1'))
const constraints = await client.query(
`SELECT conname FROM pg_constraint
WHERE conrelid = 'relay_region_rehome_attempts'::regclass
AND conname LIKE '%preferred_region%'`
)
expect(constraints.rows).toEqual([
{ conname: 'relay_region_rehome_attempts_preferred_region_valid' }
])
})
}, 60_000)
})
@@ -81,6 +81,153 @@ describePostgres('PostgreSQL regional rehoming', () => {
expect(await context.store.claimRegionalRehome()).not.toBeNull()
})
it('moves a us-central1 host onto a cell in its preferred asia-east2 region', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
expect(attempt).toMatchObject({
preferredRegion: 'asia-east2',
sourceCellId: context.source.id,
targetCellId: context.target.id
})
expect(await primary.query(
`SELECT preferred_region, source_cell_id, target_cell_id
FROM relay_region_rehome_attempts WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{
preferred_region: 'asia-east2',
source_cell_id: context.source.id,
target_cell_id: context.target.id
}])
})
it('moves an asia-east2 host back onto a cell in its preferred us-central1 region', async () => {
const context = await fixture({
sourceRegion: 'asia-east2',
targetRegion: 'us-central1'
})
const attempt = await context.store.claimRegionalRehome()
expect(attempt).toMatchObject({
preferredRegion: 'us-central1',
sourceCellId: context.source.id,
targetCellId: context.target.id
})
// The durable attempt row must accept the reverse direction too.
expect(await primary.query(
`SELECT preferred_region, source_cell_id, target_cell_id
FROM relay_region_rehome_attempts WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{
preferred_region: 'us-central1',
source_cell_id: context.source.id,
target_cell_id: context.target.id
}])
expect(await primary.query(
`SELECT cell_id FROM relay_assignments WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ cell_id: context.target.id }])
})
it('leaves a host whose preference already matches its own region', async () => {
const context = await fixture({ preferredRegion: 'us-central1' })
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true
})
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('leaves a host whose preference is older than the configured max age', async () => {
const context = await fixture()
await primary.query(
`UPDATE relay_assignment_region_preferences SET observed_at = ?
WHERE user_id = ? AND relay_host_id = ?`,
[
context.now() - 24 * 60 * 60_000 - 1,
context.identity.userId,
context.identity.relayHostId
]
)
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true
})
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('leaves a host inside its per-host rehome cooldown, in either direction', async () => {
const context = await fixture({ hostCooldownMs: 3 * 24 * 60 * 60_000 })
// A move this host already made, whichever way it went.
await primary.query(
`INSERT INTO relay_region_rehome_attempts
(attempt_id, user_id, relay_host_id, preferred_region, source_cell_id,
source_cell_incarnation, target_cell_id, target_cell_incarnation,
previous_epoch, assignment_epoch, drain_grace_ms, send_attempts,
completed_at, created_at, updated_at)
VALUES (?, ?, ?, 'us-central1', ?, ?, ?, ?, 0, 1, 0, 0, ?, ?, ?)`,
[
`pg-rehome-cooldown-${context.identity.relayHostId}`,
context.identity.userId,
context.identity.relayHostId,
context.target.id,
'22222222-2222-4222-8222-222222222222',
context.source.id,
'11111111-1111-4111-8111-111111111111',
context.now(),
context.now() - 3 * 24 * 60 * 60_000 + 1,
context.now()
]
)
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true,
hostCooldownMs: 3 * 24 * 60 * 60_000
})
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 1,
migrations: 0
})
// One millisecond past the window the same host is a candidate again.
await primary.query(
`UPDATE relay_region_rehome_attempts SET created_at = ? WHERE user_id = ?`,
[context.now() - 3 * 24 * 60 * 60_000, context.identity.userId]
)
await expect(context.store.claimRegionalRehome()).resolves.toMatchObject({
sourceCellId: context.source.id,
targetCellId: context.target.id
})
})
it('leaves a host whose preferred region holds no drainable cell', async () => {
// A cell that cannot be drained cannot be a target: the host would land
// where no later rehome could move it out again.
const context = await fixture({ targetProtocol: 0 })
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true
})
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('skips an unclean cell without latching the control off', async () => {
const context = await fixture()
await primary.query(
@@ -281,7 +428,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
context.store,
context.target,
'22222222-2222-4222-8222-222222222222',
0,
1,
900_000,
2
)
@@ -322,7 +469,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
context.store,
context.target,
'44444444-4444-4444-8444-444444444444',
0,
1,
context.now()
)
@@ -341,7 +488,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
context.store,
context.target,
'22222222-2222-4222-8222-222222222222',
0,
1,
900_000,
2
)
@@ -414,6 +561,26 @@ describePostgres('PostgreSQL regional rehoming', () => {
})
})
async function attemptAndMigrationCounts(identity: {
userId: string
relayHostId: string
}): Promise<{ attempts: number; migrations: number }> {
const attempts = await primary.query(
`SELECT COUNT(*) AS count FROM relay_region_rehome_attempts
WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
const migrations = await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations
WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
return {
attempts: Number(attempts[0]!.count),
migrations: Number(migrations[0]!.count)
}
}
async function controlAccounting(identity: {
userId: string
relayHostId: string
@@ -436,12 +603,15 @@ describePostgres('PostgreSQL regional rehoming', () => {
}
}
async function fixture() {
async function fixture(options: FixtureOptions = {}) {
sequence++
let now = 1_000_000
const suffix = String(sequence)
const source = cell(suffix, 'source', 'us-central1')
const target = cell(suffix, 'target', 'asia-east2')
const sourceRegion = options.sourceRegion ?? 'us-central1'
const targetRegion = options.targetRegion ?? 'asia-east2'
const preferredRegion = options.preferredRegion ?? targetRegion
const source = cell(suffix, 'source', sourceRegion)
const target = cell(suffix, 'target', targetRegion)
const store = new RelayAssignmentStore(primary, () => now, storeOptions)
const competingStore = new RelayAssignmentStore(secondary, () => now, storeOptions)
await store.inspectRegionalRehomeControl()
@@ -452,6 +622,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
notBefore: now,
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: options.hostCooldownMs ?? 7 * 24 * 60 * 60_000,
drainGraceMs: 60_000
})
await store.reconcileCells([source, target])
@@ -466,21 +637,22 @@ describePostgres('PostgreSQL regional rehoming', () => {
store,
target,
'22222222-2222-4222-8222-222222222222',
0,
options.targetProtocol ?? 1,
900_000
)
const identity = {
userId: `pg-rehome-user-${suffix}`,
relayHostId: `rehomehost${suffix.padStart(6, '0')}`
}
const assignment = await store.assign(identity, undefined, 'us-central1')
const assignment = await store.assign(identity, undefined, sourceRegion)
const sourceControl = await store.activateControl(identity, {
cellId: source.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 1
})
await store.assign(identity, 'asia-east2')
await store.assign(identity, preferredRegion)
return {
preferredRegion,
store,
competingStore,
identity,
@@ -500,7 +672,16 @@ const storeOptions = {
heartbeatTtlMs: 45_000
}
function cell(suffix: string, role: string, region: 'us-central1' | 'asia-east2') {
type Region = 'us-central1' | 'asia-east2'
type FixtureOptions = {
sourceRegion?: Region
targetRegion?: Region
preferredRegion?: Region
targetProtocol?: number
hostCooldownMs?: number
}
function cell(suffix: string, role: string, region: Region) {
return {
id: `pg-rehome-cell-${suffix}-${role}`,
url: `https://pg-rehome-${suffix}-${role}.example.test`,
@@ -81,6 +81,7 @@ describe('regional rehome assignment state', () => {
notBefore: context.now(),
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: 7 * 24 * 60 * 60_000,
drainGraceMs: 60_000
})).rejects.toThrow('regional_rehome_generation_mismatch')
await expect(context.store.applyRegionalRehomeControl({
@@ -89,6 +90,7 @@ describe('regional rehome assignment state', () => {
notBefore: context.now(),
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: 7 * 24 * 60 * 60_000,
drainGraceMs: 60_000
})).resolves.toMatchObject({ generation: 3, enabled: true })
await context.database.close()
@@ -207,7 +209,7 @@ describe('regional rehome assignment state', () => {
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
})
await heartbeat(context.store, target, targetIncarnation, 0, 2, {
await heartbeat(context.store, target, targetIncarnation, 1, 2, {
observedAt: context.now(),
sqlFailures: 1,
reconnects: 3,
@@ -226,6 +228,23 @@ describe('regional rehome assignment state', () => {
await context.database.close()
})
it('counts only drainable cells as the rehome fleet, in every region', async () => {
// The fleet whose health gates a rehome is exactly the cells that can be a
// source or a target, and both roles require the drain protocol.
const context = await setup({ targetProtocol: 0 })
expect(await context.store.regionalRehomeFleetSafety()).toMatchObject({
requiredCells: 1,
missingCells: 0
})
await heartbeat(context.store, target, targetIncarnation, 1, 2)
expect(await context.store.regionalRehomeFleetSafety()).toMatchObject({
requiredCells: 2,
missingCells: 0
})
await context.database.close()
})
it('claims through the measured healthy baseline of pool micro-waits and churn', async () => {
const context = await setup()
const baseline = {
@@ -238,7 +257,7 @@ describe('regional rehome assignment state', () => {
databasePoolWaitMsMax: 1
}
await heartbeat(context.store, source, sourceIncarnation, 1, 2, baseline)
await heartbeat(context.store, target, targetIncarnation, 0, 2, baseline)
await heartbeat(context.store, target, targetIncarnation, 1, 2, baseline)
await activatePreferredSource(context, {
userId: 'user-1',
relayHostId: 'abcdefghijklmnop'
@@ -324,6 +343,204 @@ describe('regional rehome assignment state', () => {
await context.database.close()
})
it('moves a live host on an asia-east2 cell back to its preferred us-central1 cell', async () => {
const context = await setup()
const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
await activateReversePreferredSource(context, identity)
const attempt = await context.store.claimRegionalRehome()
expect(attempt).toMatchObject({
userId: identity.userId,
relayHostId: identity.relayHostId,
preferredRegion: 'us-central1',
sourceCellId: target.id,
sourceCellIncarnation: targetIncarnation,
targetCellId: source.id,
targetCellIncarnation: sourceIncarnation,
previousEpoch: 1,
assignmentEpoch: 2,
sendAttempts: 1
})
expect(
await context.database.query(
`SELECT preferred_region, source_cell_id, target_cell_id
FROM relay_region_rehome_attempts`
)
).toEqual([{
preferred_region: 'us-central1',
source_cell_id: target.id,
target_cell_id: source.id
}])
expect(await context.store.resolve(identity)).toMatchObject({ cellId: source.id })
await context.database.close()
})
it('drops a candidate at scan time when no cell in the preferred region is usable', async () => {
const context = await setup()
await activatePreferredSource(context, {
userId: 'user-1',
relayHostId: 'abcdefghijklmnop'
})
// A disabled cell is not a target, and the scan must say so: leaving it to
// the claim would burn a slot of the candidate batch on a certain skip.
await context.database.query(`UPDATE relay_cells SET enabled = 0 WHERE cell_id = ?`, [
target.id
])
const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
try {
expect(await context.store.claimRegionalRehome()).toBeNull()
} finally {
warnings.restore()
}
expect(warnings.entries).toEqual([])
expect(
await context.database.query(
`SELECT next_dispatch_at FROM relay_region_rehome_worker_state`
)
).toEqual([{ next_dispatch_at: 0 }])
expect(await context.database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
await context.database.close()
})
it('names the skip when the last target is lost between scan and claim', async () => {
const database = await openInMemoryRelayDatabase()
const context = await setup({
database,
wrap: (delegate) =>
hookAfterCandidateScan(delegate, async (transaction) => {
await transaction.query(`UPDATE relay_cells SET enabled = 0 WHERE cell_id = ?`, [
target.id
])
})
})
await activatePreferredSource(context, {
userId: 'user-1',
relayHostId: 'abcdefghijklmnop'
})
const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
try {
expect(await context.store.claimRegionalRehome()).toBeNull()
} finally {
warnings.restore()
}
expect(warnings.entries).toMatchObject([
{ skips: [{ reason: 'no_eligible_target', candidates: 1 }] }
])
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
generation: 1,
enabled: true
})
expect(await database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
await database.close()
})
it('leaves a host alone until its cooldown expires, then moves it back', async () => {
const context = await setup({ hostCooldownMs: 3 * 24 * 60 * 60_000 })
const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
const targetControl = await completeRehomeToTarget(context, identity)
// Past the dispatch interval the earlier claim charged, so the next tick
// really does scan and the cooldown is the only thing holding this host.
context.advance(10_000)
// The desktop's region probe now says us-central1 again.
await context.store.assign(identity, 'us-central1')
const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
try {
expect(await context.store.claimRegionalRehome()).toBeNull()
} finally {
warnings.restore()
}
expect(warnings.entries).toEqual([])
expect(await context.store.resolve(identity)).toMatchObject({ cellId: target.id })
context.advance(3 * 24 * 60 * 60_000)
await freshHeartbeats(context)
await context.store.renewControlActivity(identity, {
activityId: targetControl,
cellId: target.id,
expiresAt: context.now() + 90_000
})
await context.store.assign(identity, 'us-central1')
const attempt = await context.store.claimRegionalRehome()
expect(attempt).toMatchObject({
preferredRegion: 'us-central1',
sourceCellId: target.id,
targetCellId: source.id
})
await context.database.close()
})
it('rejects a host whose attempt lands between the scan and the claim', async () => {
const database = await openInMemoryRelayDatabase()
const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
const context = await setup({
database,
wrap: (delegate) =>
hookAfterCandidateScan(delegate, async (transaction) => {
await transaction.query(
`INSERT INTO relay_region_rehome_attempts
(attempt_id, user_id, relay_host_id, preferred_region, source_cell_id,
source_cell_incarnation, target_cell_id, target_cell_incarnation,
previous_epoch, assignment_epoch, drain_grace_ms, send_attempts,
created_at, updated_at)
VALUES ('raced', ?, ?, 'asia-east2', ?, ?, ?, ?, 0, 1, 0, 0, ?, ?)`,
[
identity.userId,
identity.relayHostId,
source.id,
sourceIncarnation,
target.id,
targetIncarnation,
context.now(),
context.now()
]
)
})
})
await activatePreferredSource(context, identity)
const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
try {
expect(await context.store.claimRegionalRehome()).toBeNull()
} finally {
warnings.restore()
}
expect(warnings.entries).toMatchObject([
{ skips: [{ reason: 'host_cooldown', candidates: 1 }] }
])
expect(await database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
await database.close()
})
it('does not scan a candidate whose preferred region has no drainable cell', async () => {
// A cell without the drain protocol cannot be a target: the host would land
// where no later rehome could move it out again. The candidate query drops
// it, so the tick stays idle instead of paying for an inventory scan.
const context = await setup({ targetProtocol: 0 })
await activatePreferredSource(context, {
userId: 'user-1',
relayHostId: 'abcdefghijklmnop'
})
const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
try {
expect(await context.store.claimRegionalRehome()).toBeNull()
} finally {
warnings.restore()
}
expect(warnings.entries).toEqual([])
expect(
await context.database.query(
`SELECT next_dispatch_at FROM relay_region_rehome_worker_state`
)
).toEqual([{ next_dispatch_at: 0 }])
expect(await context.database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
await context.database.close()
})
it('skips an unclean cell without latching the control off', async () => {
const context = await setup()
await activatePreferredSource(context, {
@@ -457,7 +674,7 @@ describe('regional rehome assignment state', () => {
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
})
await heartbeat(context.store, target, targetIncarnation, 0, 2, {
await heartbeat(context.store, target, targetIncarnation, 1, 2, {
observedAt: context.now(),
sqlFailures: 0,
reconnects: 0,
@@ -477,6 +694,7 @@ describe('regional rehome assignment state', () => {
notBefore: context.now(),
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: 7 * 24 * 60 * 60_000,
drainGraceMs: 60_000
})
const retry = await context.store.claimRegionalRehome()
@@ -547,7 +765,7 @@ describe('regional rehome assignment state', () => {
await activatePreferredSource(context, identity)
await context.store.claimRegionalRehome()
context.advance(6 * 60_000)
await heartbeat(context.store, target, targetIncarnation, 0, 2)
await heartbeat(context.store, target, targetIncarnation, 1, 2)
expect(await context.store.refreshRegionalRehomeLeases()).toBe(0)
expect(await context.store.abortExpiredEvacuations()).toBe(0)
@@ -1549,10 +1767,16 @@ function collectDisableWarnings() {
}
async function setup(
options: { sourceProtocol?: number; wrap?: (database: RelayDatabase) => RelayDatabase } = {}
options: {
sourceProtocol?: number
targetProtocol?: number
hostCooldownMs?: number
database?: RelayDatabase
wrap?: (database: RelayDatabase) => RelayDatabase
} = {}
) {
let clock = 1_000_000
const database = await openInMemoryRelayDatabase()
const database = options.database ?? (await openInMemoryRelayDatabase())
const store = new RelayAssignmentStore(options.wrap?.(database) ?? database, () => clock, {
requireLiveCells: true,
heartbeatTtlMs: 45_000
@@ -1565,11 +1789,12 @@ async function setup(
notBefore: clock,
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: options.hostCooldownMs ?? 7 * 24 * 60 * 60_000,
drainGraceMs: 60 * 60_000
})
await store.reconcileCells([source, target])
await heartbeat(store, source, sourceIncarnation, options.sourceProtocol ?? 1)
await heartbeat(store, target, targetIncarnation, 0)
await heartbeat(store, target, targetIncarnation, options.targetProtocol ?? 1)
return {
database,
store,
@@ -1671,7 +1896,7 @@ async function freshHeartbeats(context: Context): Promise<void> {
}
// The clock doubles as a strictly-increasing connection inclusion watermark.
await heartbeat(context.store, source, sourceIncarnation, 1, context.now(), safety)
await heartbeat(context.store, target, targetIncarnation, 0, context.now(), safety)
await heartbeat(context.store, target, targetIncarnation, 1, context.now(), safety)
}
async function activatePreferredSource(
@@ -1688,6 +1913,68 @@ async function activatePreferredSource(
return control
}
// Runs a hook inside the claim transaction, right after the candidate scan, so
// a scan-versus-claim race is deterministic instead of timing-dependent.
function hookAfterCandidateScan(
database: RelayDatabase,
hook: (transaction: RelayDatabase) => Promise<void>
): RelayDatabase {
let fired = false
const decorate = (delegate: RelayDatabase): RelayDatabase => ({
query: async (sql, params) => {
const rows = await delegate.query(sql, params)
if (!fired && sql.includes('FROM relay_assignment_region_preferences preference')) {
fired = true
await hook(delegate)
}
return rows
},
queryLocked: async (sql, params, lockOptions) =>
await delegate.queryLocked(sql, params, lockOptions),
transaction: async (operation, transactionOptions) =>
await delegate.transaction(
async (transaction) => await operation(decorate(transaction)),
transactionOptions
),
close: async () => undefined
})
return decorate(database)
}
async function completeRehomeToTarget(
context: Context,
identity: { userId: string; relayHostId: string }
): Promise<string> {
const sourceControl = await activatePreferredSource(context, identity)
const attempt = await context.store.claimRegionalRehome()
const targetControl = await context.store.activateControl(identity, {
cellId: target.id,
assignmentEpoch: attempt!.assignmentEpoch,
generation: 1
})
await context.store.markMigrationTargetRegistered(identity, {
cellId: target.id,
assignmentEpoch: attempt!.assignmentEpoch
})
await context.store.releaseActivity(identity, sourceControl)
await context.store.completeReadyRegionalRehomes()
return targetControl
}
async function activateReversePreferredSource(
context: Context,
identity: { userId: string; relayHostId: string }
): Promise<string> {
const assignment = await context.store.assign(identity, undefined, 'asia-east2')
const control = await context.store.activateControl(identity, {
cellId: target.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 1
})
await context.store.assign(identity, 'us-central1')
return control
}
async function activateSource(
context: Context,
identity: { userId: string; relayHostId: string }
@@ -36,6 +36,7 @@ async function setup() {
notBefore: clock,
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: 7 * 24 * 60 * 60_000,
drainGraceMs: 60 * 60_000
})
await store.reconcileCells([source, noHeadroom, unclean, highLoad, lowLoad])
@@ -100,22 +101,22 @@ describe('regional rehome target selection', () => {
sqlFailures: 0
})
// Lowest load but the connection hard cap is exhausted.
await context.beat(noHeadroom, 2, 0, {
await context.beat(noHeadroom, 2, 1, {
observedRequests: 0,
enforcedConnections: 999,
sqlFailures: 0
})
await context.beat(unclean, 3, 0, {
await context.beat(unclean, 3, 1, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: UNCLEAN
})
await context.beat(highLoad, 4, 0, {
await context.beat(highLoad, 4, 1, {
observedRequests: 50,
enforcedConnections: 0,
sqlFailures: 0
})
await context.beat(lowLoad, 5, 0, {
await context.beat(lowLoad, 5, 1, {
observedRequests: 10,
enforcedConnections: 0,
sqlFailures: 0
@@ -134,22 +135,22 @@ describe('regional rehome target selection', () => {
enforcedConnections: 0,
sqlFailures: 0
})
await context.beat(noHeadroom, 2, 0, {
await context.beat(noHeadroom, 2, 1, {
observedRequests: 0,
enforcedConnections: 999,
sqlFailures: 0
})
await context.beat(unclean, 3, 0, {
await context.beat(unclean, 3, 1, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: UNCLEAN
})
await context.beat(highLoad, 4, 0, {
await context.beat(highLoad, 4, 1, {
observedRequests: 50,
enforcedConnections: 0,
sqlFailures: 0
})
await context.beat(lowLoad, 5, 0, {
await context.beat(lowLoad, 5, 1, {
observedRequests: 10,
enforcedConnections: 0,
sqlFailures: UNCLEAN
@@ -1,7 +1,9 @@
import { RELAY_REGION_METRIC_SEGMENTS, RELAY_REGIONS } from '@orca-cloud/relay-contract'
import { describe, expect, it, vi } from 'vitest'
import type { RelayDatabase } from './database.js'
import { observeRelayDatabase } from './observed-relay-database.js'
import {
CONTROL_RTT_RESERVOIR_LIMIT,
observedRelayRequests,
RelayObservability,
type RelayProcessCounts
@@ -22,6 +24,37 @@ const counts: RelayProcessCounts = {
databasePoolWaitMsMax: 1_250
}
// Two schema keys legitimately spell a policed word: the abandoned-accept bucket
// is keyed by stage name and one stage is `credential`. Rename those exact keys in
// a clone instead of rewriting the JSON, so a stray raw field or value anywhere
// else still trips the guard below.
const SCHEMA_KEY_ALIASES: Record<string, string> = {
clientAcceptCredentialMsP95: 'clientAcceptStageTwoMsP95'
}
function scrubSchemaKeys(entries: Array<Record<string, unknown>>): string {
return JSON.stringify(
entries.map((entry) =>
Object.fromEntries(
Object.entries(entry).map(([key, value]) => [
SCHEMA_KEY_ALIASES[key] ?? key,
key === 'clientAcceptsAbandonedByStageDelta' ? renameStageKeys(value) : value
])
)
)
)
}
function renameStageKeys(bucket: unknown): unknown {
if (bucket === null || typeof bucket !== 'object') return bucket
return Object.fromEntries(
Object.entries(bucket).map(([stage, count]) => [
stage === 'credential' ? 'stageTwo' : stage,
count
])
)
}
describe('relay observability', () => {
it('emits safe readiness dependency outcomes', () => {
const entries: Array<Record<string, unknown>> = []
@@ -106,14 +139,31 @@ describe('relay observability', () => {
requestedRegionsDelta: { 'asia-east2': 1, unhinted: 1 },
selectedRegionsDelta: { 'us-central1': 1 },
regionFallbacksDelta: { 'asia-east2': 1 },
unavailableRegionsDelta: { 'asia-east2': 1 }
unavailableRegionsDelta: { 'asia-east2': 1 },
// Flat per-region siblings the log-based metrics extract; `unhinted` stays map-only.
requestedRegionUsCentral1Delta: 0,
requestedRegionAsiaEast2Delta: 1,
selectedRegionUsCentral1Delta: 1,
selectedRegionAsiaEast2Delta: 0
})
expect(entries[1]).toMatchObject({
requestedRegionsDelta: {},
selectedRegionsDelta: {},
regionFallbacksDelta: {},
unavailableRegionsDelta: {}
unavailableRegionsDelta: {},
// Zeros keep publishing so an idle window cannot drop a series out of the skew join.
requestedRegionUsCentral1Delta: 0,
requestedRegionAsiaEast2Delta: 0,
selectedRegionUsCentral1Delta: 0,
selectedRegionAsiaEast2Delta: 0
})
// A region added to the contract has to reach the flat keys, or the skew alert's
// denominator silently misses it.
for (const segment of Object.values(RELAY_REGION_METRIC_SEGMENTS)) {
expect(entries[0]).toHaveProperty(`requestedRegion${segment}Delta`)
expect(entries[0]).toHaveProperty(`selectedRegion${segment}Delta`)
}
expect(Object.keys(RELAY_REGION_METRIC_SEGMENTS).sort()).toEqual([...RELAY_REGIONS].sort())
})
it('emits bounded aggregate runtime signals without identities or credentials', () => {
@@ -181,7 +231,7 @@ describe('relay observability', () => {
controlActivityRecoveryFailuresDelta: 0,
httpLatencyMsMax: 0
})
expect(JSON.stringify(entries)).not.toMatch(/token|credential|userId|relayHostId/)
expect(scrubSchemaKeys(entries)).not.toMatch(/token|credential|userId|relayHostId/i)
})
it('aggregates control and splice closes as bounded per-reason deltas', () => {
@@ -215,6 +265,117 @@ describe('relay observability', () => {
})
})
it('summarises completed client accepts and control round trips per window', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
(entry) => entries.push(entry)
)
observability.recordClientAcceptCompleted({
totalMs: 812.4567,
stageMs: { assignment: 120, credential: 90, activity: 40, attach: 500, basis: 62 }
})
observability.recordClientAcceptCompleted({
totalMs: 6_400,
stageMs: { assignment: 4_100, credential: 95, activity: 60, attach: 2_000, basis: 145 }
})
observability.recordControlRtt(28)
observability.recordControlRtt(240)
observability.recordControlRtt(31)
observability.flush(counts)
observability.flush(counts)
expect(entries[0]).toMatchObject({
clientAcceptCompletedDelta: 2,
clientAcceptTotalMsP50: 812.457,
clientAcceptTotalMsP95: 6_400,
clientAcceptTotalMsMax: 6_400,
clientAcceptAssignmentMsP95: 4_100,
clientAcceptCredentialMsP95: 95,
clientAcceptActivityMsP95: 60,
clientAcceptAttachMsP95: 2_000,
clientAcceptBasisMsP95: 145,
controlRttSamplesDelta: 3,
controlRttMsP50: 31,
controlRttMsP95: 240,
controlRttMsMax: 240
})
// Only-add: the pre-existing fields still read the same after the extension.
expect(entries[0]).toMatchObject({
event: 'orca_relay_runtime_metrics',
metricVersion: 2,
clientAcceptsAbandonedByStageDelta: {},
clientAcceptAbandonedMsMax: 0
})
// An empty window publishes counts only: a zero percentile point is
// indistinguishable from a real zero once Cloud Logging aggregates it.
expect(entries[1]).toMatchObject({ clientAcceptCompletedDelta: 0, controlRttSamplesDelta: 0 })
for (const omitted of [
'clientAcceptTotalMsP50',
'clientAcceptTotalMsP95',
'clientAcceptTotalMsMax',
'clientAcceptAssignmentMsP95',
'clientAcceptCredentialMsP95',
'clientAcceptActivityMsP95',
'clientAcceptAttachMsP95',
'clientAcceptBasisMsP95',
'controlRttMsP50',
'controlRttMsP95',
'controlRttMsMax'
]) {
expect(entries[1]).not.toHaveProperty(omitted)
expect(entries[0]).toHaveProperty(omitted)
}
expect(scrubSchemaKeys(entries)).not.toMatch(/token|credential|userId|relayHostId/i)
})
it('caps the control round-trip reservoir and reports what it dropped', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
(entry) => entries.push(entry)
)
const flooded = CONTROL_RTT_RESERVOIR_LIMIT * 20
for (let sample = 0; sample < flooded; sample++) {
observability.recordControlRtt(10 + (sample % 40))
}
observability.flush(counts)
// Dropped is observed minus retained, so this pins the retained window at the cap.
expect(entries[0]).toMatchObject({
controlRttSamplesDelta: flooded,
controlRttSamplesDroppedDelta: flooded - CONTROL_RTT_RESERVOIR_LIMIT
})
// The kept samples are real observations, not a truncated or synthesised window.
expect(entries[0]!.controlRttMsP50 as number).toBeGreaterThanOrEqual(10)
expect(entries[0]!.controlRttMsMax as number).toBeLessThanOrEqual(49)
observability.flush(counts)
expect(entries[1]).toMatchObject({
controlRttSamplesDelta: 0,
controlRttSamplesDroppedDelta: 0
})
expect(entries[1]).not.toHaveProperty('controlRttMsP50')
})
it('samples the whole flooded window rather than its first samples', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
(entry) => entries.push(entry)
)
const half = CONTROL_RTT_RESERVOIR_LIMIT * 10
for (let sample = 0; sample < half; sample++) observability.recordControlRtt(10)
for (let sample = 0; sample < half; sample++) observability.recordControlRtt(900)
observability.flush(counts)
// Keeping the first N instead would publish a window of nothing but 10s. Each
// reservoir slot ends up drawn from the late half with ~1/2 probability, so
// fewer than the 5% the p95 needs is out of reach of this suite.
expect(entries[0]!.controlRttMsP95).toBe(900)
expect(entries[0]!.controlRttMsMax).toBe(900)
})
it('observes successful and failed database calls including transactions', async () => {
const recordSql = vi.fn()
const underlying: RelayDatabase = {
+128 -14
View File
@@ -1,5 +1,5 @@
import { monitorEventLoopDelay, performance } from 'node:perf_hooks'
import type { RelayRegion } from '@orca-cloud/relay-contract'
import { RELAY_REGION_METRIC_SEGMENTS, type RelayRegion } from '@orca-cloud/relay-contract'
import type { ControlRenewalOutcome } from './assignment-store.js'
import type { CellInventoryHoldCounts } from './cell-inventory-hold-samples.js'
import type { PostgresPoolPressureCounts } from './postgres-pool-pressure.js'
@@ -65,11 +65,31 @@ export interface RelayRuntimeObserver {
recordControlClose?(code: number): void
recordSpliceClose?(trigger: string): void
recordClientAcceptAbandoned?(stage: RelayClientAcceptStage, elapsedMs: number): void
recordClientAcceptCompleted?(sample: RelayClientAcceptSample): void
recordControlRtt?(rttMs: number): void
}
// Which serialized accept step the phone had already hung up behind.
export type RelayClientAcceptStage = 'assignment' | 'credential' | 'activity'
// The attach window and the basis writes that follow it are only measurable once
// the host data leg lands, so they join the serialized pre-attach steps on
// completed accepts only.
export type RelayClientAcceptTimedStage = RelayClientAcceptStage | 'attach' | 'basis'
export const RELAY_CLIENT_ACCEPT_TIMED_STAGES = [
'assignment',
'credential',
'activity',
'attach',
'basis'
] as const satisfies readonly RelayClientAcceptTimedStage[]
export type RelayClientAcceptSample = {
totalMs: number
stageMs: Record<RelayClientAcceptTimedStage, number>
}
type RelayMetricDeltas = {
forwardedBytes: number
authSuccesses: number
@@ -93,12 +113,20 @@ type RelayMetricDeltas = {
spliceClosesByTrigger: Record<string, number>
clientAcceptsAbandonedByStage: Record<string, number>
clientAcceptAbandonedMsMax: number
clientAcceptTotalsMs: number[]
clientAcceptStageSamplesMs: Record<RelayClientAcceptTimedStage, number[]>
controlRttSamplesMs: number[]
controlRttObserved: number
controlRenewalLatenciesMs: number[]
controlRenewalsByOutcome: Record<string, number>
controlActivityRecoveries: number
controlActivityRecoveryFailures: number
}
// A host chooses how often it answers a ping, so the process-wide window is a
// reservoir: the heap cost of a flood is capped and the percentiles stay unbiased.
export const CONTROL_RTT_RESERVOIR_LIMIT = 1024
type MetricWriter = (entry: Record<string, unknown>) => void
const emptyDeltas = (): RelayMetricDeltas => ({
@@ -124,18 +152,42 @@ const emptyDeltas = (): RelayMetricDeltas => ({
spliceClosesByTrigger: {},
clientAcceptsAbandonedByStage: {},
clientAcceptAbandonedMsMax: 0,
clientAcceptTotalsMs: [],
clientAcceptStageSamplesMs: {
assignment: [],
credential: [],
activity: [],
attach: [],
basis: []
},
controlRttSamplesMs: [],
controlRttObserved: 0,
controlRenewalLatenciesMs: [],
controlRenewalsByOutcome: {},
controlActivityRecoveries: 0,
controlActivityRecoveryFailures: 0
})
function percentile(values: number[], percentileRank: number): number {
export function percentile(values: number[], percentileRank: number): number {
if (values.length === 0) return 0
const sorted = [...values].sort((left, right) => left - right)
return sorted[Math.ceil(percentileRank * sorted.length) - 1] ?? 0
}
function roundMs(value: number): number {
return Number(value.toFixed(3))
}
// Spreading a window into Math.max blows the stack once a busy cell samples
// enough of it, so the maximum is folded instead.
function latencySummary(samples: number[]): { p50: number; p95: number; max: number } {
return {
p50: roundMs(percentile(samples, 0.5)),
p95: roundMs(percentile(samples, 0.95)),
max: roundMs(samples.reduce((highest, sample) => Math.max(highest, sample), 0))
}
}
export class RelayObservability implements RelayRuntimeObserver {
private readonly eventLoop = monitorEventLoopDelay({ resolution: 20 })
private deltas = emptyDeltas()
@@ -244,6 +296,25 @@ export class RelayObservability implements RelayRuntimeObserver {
)
}
recordClientAcceptCompleted(sample: RelayClientAcceptSample): void {
this.deltas.clientAcceptTotalsMs.push(sample.totalMs)
for (const stage of RELAY_CLIENT_ACCEPT_TIMED_STAGES) {
this.deltas.clientAcceptStageSamplesMs[stage].push(sample.stageMs[stage])
}
}
recordControlRtt(rttMs: number): void {
const samples = this.deltas.controlRttSamplesMs
const observedBefore = this.deltas.controlRttObserved++
if (samples.length < CONTROL_RTT_RESERVOIR_LIMIT) {
samples.push(rttMs)
return
}
// Algorithm R: every round trip in the window keeps an equal chance of being kept.
const slot = Math.floor(Math.random() * (observedBefore + 1))
if (slot < CONTROL_RTT_RESERVOIR_LIMIT) samples[slot] = rttMs
}
start(readCounts: () => RelayProcessCounts, intervalMs = 30_000): void {
if (this.timer) return
this.eventLoop.enable()
@@ -277,6 +348,11 @@ export class RelayObservability implements RelayRuntimeObserver {
controlActivityRecoveryFailures: deltas.controlActivityRecoveryFailures
}
this.deltas = emptyDeltas()
const acceptTotals = latencySummary(deltas.clientAcceptTotalsMs)
const acceptStageP95 = (stage: RelayClientAcceptTimedStage): number =>
roundMs(percentile(deltas.clientAcceptStageSamplesMs[stage], 0.95))
const controlRtt = latencySummary(deltas.controlRttSamplesMs)
const controlRenewal = latencySummary(deltas.controlRenewalLatenciesMs)
const memory = process.memoryUsage()
const p99 = this.eventLoop.count === 0 ? 0 : this.eventLoop.percentile(99) / 1_000_000
this.eventLoop.reset()
@@ -301,15 +377,43 @@ export class RelayObservability implements RelayRuntimeObserver {
placementRejectionsByReasonDelta: deltas.placementRejectionsByReason,
requestedRegionsDelta: deltas.requestedRegions,
selectedRegionsDelta: deltas.selectedRegions,
...regionCounterFields('requestedRegion', deltas.requestedRegions),
...regionCounterFields('selectedRegion', deltas.selectedRegions),
regionFallbacksDelta: deltas.regionFallbacks,
unavailableRegionsDelta: deltas.unavailableRegions,
controlClosesByCodeDelta: deltas.controlClosesByCode,
spliceClosesByTriggerDelta: deltas.spliceClosesByTrigger,
clientAcceptsAbandonedByStageDelta: deltas.clientAcceptsAbandonedByStage,
clientAcceptAbandonedMsMax: Number(deltas.clientAcceptAbandonedMsMax.toFixed(3)),
clientAcceptAbandonedMsMax: roundMs(deltas.clientAcceptAbandonedMsMax),
clientAcceptCompletedDelta: deltas.clientAcceptTotalsMs.length,
// Accepts are sparse: publishing a zero percentile for every empty window
// would pin the p50 at 0 forever and collapse the p95 at low accept rates.
...(deltas.clientAcceptTotalsMs.length === 0
? {}
: {
clientAcceptTotalMsP50: acceptTotals.p50,
clientAcceptTotalMsP95: acceptTotals.p95,
clientAcceptTotalMsMax: acceptTotals.max,
clientAcceptAssignmentMsP95: acceptStageP95('assignment'),
clientAcceptCredentialMsP95: acceptStageP95('credential'),
clientAcceptActivityMsP95: acceptStageP95('activity'),
clientAcceptAttachMsP95: acceptStageP95('attach'),
clientAcceptBasisMsP95: acceptStageP95('basis')
}),
// Every round trip observed in the window, including the ones the reservoir
// above declined to keep; the percentiles summarise only what it kept.
controlRttSamplesDelta: deltas.controlRttObserved,
controlRttSamplesDroppedDelta: deltas.controlRttObserved - deltas.controlRttSamplesMs.length,
...(deltas.controlRttSamplesMs.length === 0
? {}
: {
controlRttMsP50: controlRtt.p50,
controlRttMsP95: controlRtt.p95,
controlRttMsMax: controlRtt.max
}),
sqlQueriesDelta: deltas.sqlQueries,
sqlFailuresDelta: deltas.sqlFailures,
sqlLatencyMsMax: Number(deltas.sqlLatencyMsMax.toFixed(3)),
sqlLatencyMsMax: roundMs(deltas.sqlLatencyMsMax),
controlRenewalsByOutcomeDelta: deltas.controlRenewalsByOutcome,
controlRenewalsDelta: deltas.controlRenewalLatenciesMs.length,
controlRenewalSuccessesDelta: deltas.controlRenewalsByOutcome.renewed ?? 0,
@@ -317,16 +421,10 @@ export class RelayObservability implements RelayRuntimeObserver {
deltas.controlRenewalsByOutcome.control_activity_not_found ?? 0,
controlActivityRecoveriesDelta: deltas.controlActivityRecoveries,
controlActivityRecoveryFailuresDelta: deltas.controlActivityRecoveryFailures,
controlRenewalLatencyMsP50: Number(
percentile(deltas.controlRenewalLatenciesMs, 0.5).toFixed(3)
),
controlRenewalLatencyMsP95: Number(
percentile(deltas.controlRenewalLatenciesMs, 0.95).toFixed(3)
),
controlRenewalLatencyMsMax: Number(
Math.max(0, ...deltas.controlRenewalLatenciesMs).toFixed(3)
),
httpLatencyMsMax: Number(deltas.httpLatencyMsMax.toFixed(3)),
controlRenewalLatencyMsP50: controlRenewal.p50,
controlRenewalLatencyMsP95: controlRenewal.p95,
controlRenewalLatencyMsMax: controlRenewal.max,
httpLatencyMsMax: roundMs(deltas.httpLatencyMsMax),
heapUsedBytes: memory.heapUsed,
heapTotalBytes: memory.heapTotal,
eventLoopDelayMsP99: Number(p99.toFixed(3))
@@ -334,6 +432,22 @@ export class RelayObservability implements RelayRuntimeObserver {
}
}
// Flat siblings of the nested region maps, always emitted for every region including zeros.
// A log-based metric cannot reach `requestedRegionsDelta."asia-east2"` without a quoted field
// path, and an absent key would drop a series out of the inner join the region-skew alert does.
// The maps stay authoritative and keep carrying anything outside the catalog, such as `unhinted`.
function regionCounterFields(
prefix: 'requestedRegion' | 'selectedRegion',
counts: Record<string, number>
): Record<string, number> {
return Object.fromEntries(
Object.entries(RELAY_REGION_METRIC_SEGMENTS).map(([region, segment]) => [
`${prefix}${segment}Delta`,
counts[region] ?? 0
])
)
}
function increment(counts: Record<string, number>, key: string): void {
counts[key] = (counts[key] ?? 0) + 1
}
+4 -1
View File
@@ -2,7 +2,9 @@ import { createAdaptorServer } from '@hono/node-server'
import {
hasAdmissionCapacity,
HostDataAuthSchema,
parseRelayHostCapabilities,
RELAY_ADMISSION_BUDGETS,
RELAY_HOST_CAPABILITIES_HEADER,
RELAY_CLOSE_CODE,
RELAY_DEFAULT_REGION,
RELAY_PROTOCOL_LIMITS,
@@ -486,7 +488,8 @@ export function createRelayServer(
sessions.acceptControl(
webSocket,
identity,
controlUpgrade?.inclusionWatermark
controlUpgrade?.inclusionWatermark,
parseRelayHostCapabilities(request.headers[RELAY_HOST_CAPABILITIES_HEADER])
)
})
} catch {
+73 -2
View File
@@ -9,7 +9,9 @@ import { fileURLToPath } from 'node:url'
import { exportJWK, generateKeyPair, jwtVerify, SignJWT } from 'jose'
import {
buildHostProofMacInput,
HOST_CHALLENGE_PLAINTEXT_DOMAIN
HOST_CHALLENGE_PLAINTEXT_DOMAIN,
RELAY_HOST_CAPABILITIES_HEADER,
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS
} from '@orca-cloud/relay-contract'
import nacl from 'tweetnacl'
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
@@ -282,11 +284,17 @@ async function openHostControl(input?: {
previousGeneration?: number
keyPair?: nacl.BoxKeyPair
assignmentEpoch?: number
capabilities?: string
}): Promise<{ socket: WebSocket; ack: Record<string, unknown>; keyPair: nacl.BoxKeyPair }> {
const keyPair = input?.keyPair ?? nacl.box.keyPair()
const hostId = createHash('sha256').update(keyPair.publicKey).digest('base64url').slice(0, 16)
const socket = new WebSocket(`${relayUrl.replace('http:', 'ws:')}/v1/host/control`, {
headers: { authorization: `Bearer ${await relayToken('orca-relay', hostId)}` },
headers: {
authorization: `Bearer ${await relayToken('orca-relay', hostId)}`,
...(input?.capabilities
? { [RELAY_HOST_CAPABILITIES_HEADER]: input.capabilities }
: {})
},
perMessageDeflate: false
})
await new Promise<void>((resolveOpen, reject) => {
@@ -653,6 +661,69 @@ describe('served relay URL', () => {
expect(result.reason).not.toContain('http')
})
it('restates a pending connection to the rebound control, detailed only when advertised', async () => {
// The one link the unit tests cannot reach: an upgrade that really carries
// x-orca-host-capabilities must reach acceptControl and change the ack. A
// typo in the header name here passes every other test in the suite.
const host = await openHostControl()
const hostId = createHash('sha256')
.update(host.keyPair.publicKey)
.digest('base64url')
.slice(0, 16)
const inviteResponse = nextMessage(host.socket)
host.socket.send(
JSON.stringify({
type: 'invite-create',
reqId: 'capability-invite',
relayDeviceId: 'capability-device'
})
)
const invite = await inviteResponse
const phone = new WebSocket(`${relayUrl.replace('http:', 'ws:')}/v1/connect/${hostId}`, {
headers: forwardedHeaders()
})
await new Promise<void>((resolveOpen, reject) => {
phone.once('open', resolveOpen)
phone.once('error', reject)
})
const connectionPromise = nextMessage(host.socket)
phone.send(
JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential: invite.inviteToken })
)
// Never attached: the connection stays pending, which is what the ack restates.
const connection = await connectionPromise
expect(connection.type).toBe('conn-open')
const capable = await openHostControl({
keyPair: host.keyPair,
controlResumeSecret: String(host.ack.controlResumeSecret),
previousGeneration: 1,
capabilities: RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS
})
expect(capable.ack.pendingConns).toEqual([
{
connId: connection.connId,
connTicket: connection.connTicket,
kind: 'invite',
relayDeviceId: 'capability-device'
}
])
const legacy = await openHostControl({
keyPair: host.keyPair,
controlResumeSecret: String(capable.ack.controlResumeSecret),
previousGeneration: 1
})
// A shipped host parses these entries strictly, so an unannounced key would
// fail the whole ack and kill a control that was working.
expect(legacy.ack.pendingConns).toEqual([
{ connId: connection.connId, connTicket: connection.connTicket }
])
phone.close()
legacy.socket.close()
})
it('keeps a pending attach usable after a bad ticket and rejects ticket replay', async () => {
const host = await openHostControl()
const hostId = createHash('sha256')
@@ -133,14 +133,17 @@
"google_logging_metric.relay_snapshot",
"google_monitoring_alert_policy.relay_assignment_5xx",
"google_monitoring_alert_policy.relay_assignment_edge_429",
"google_monitoring_alert_policy.relay_cell_control_rtt",
"google_monitoring_alert_policy.relay_cell_process_exit",
"google_monitoring_alert_policy.relay_cloud_nat_port_drops",
"google_monitoring_alert_policy.relay_cloud_sql_backends",
"google_monitoring_alert_policy.relay_cloud_sql_checkpoint_loop",
"google_monitoring_alert_policy.relay_cloud_sql_disk",
"google_monitoring_alert_policy.relay_custom",
"google_monitoring_alert_policy.relay_far_cell_accept_latency",
"google_monitoring_alert_policy.relay_gce_connection_headroom",
"google_monitoring_alert_policy.relay_postgres_retry_exhausted",
"google_monitoring_alert_policy.relay_region_hint_skew",
"google_monitoring_dashboard.relay_incident",
"google_project_iam_custom_role.github_production_relay_capacity_mutation",
"google_project_iam_custom_role.github_relay_asia_topology_mutation",
@@ -45,6 +45,7 @@ export function parseRegionalRehomeArguments(argv, environment = process.env) {
'not-before',
'rate-per-minute',
'preference-max-age-ms',
'host-cooldown-ms',
'drain-grace-ms',
'confirmation'
]
@@ -117,6 +118,11 @@ export function parseRegionalRehomeArguments(argv, environment = process.env) {
'--preference-max-age-ms',
{ minimum: 60_000, maximum: 30 * 24 * 60 * 60_000 }
),
hostCooldownMs: integer(
values['host-cooldown-ms'],
'--host-cooldown-ms',
{ minimum: 60_000, maximum: 30 * 24 * 60 * 60_000 }
),
drainGraceMs: integer(values['drain-grace-ms'], '--drain-grace-ms', {
minimum: 60_000,
maximum: 60 * 60_000
@@ -147,6 +153,11 @@ function assertControl(control, expected) {
!Number.isSafeInteger(control.notBefore) ||
!Number.isSafeInteger(control.ratePerMinute) ||
!Number.isSafeInteger(control.preferenceMaxAgeMs) ||
// A director predating the per-host cooldown does not report it. Reading
// the control and both emergency brakes must keep working against that
// image; only enable requires the field.
(control.hostCooldownMs !== undefined &&
!Number.isSafeInteger(control.hostCooldownMs)) ||
!Number.isSafeInteger(control.drainGraceMs)
) throw new Error('director returned an invalid regional rehome control')
if (expected.enabled !== undefined && control.enabled !== expected.enabled) {
@@ -155,6 +166,12 @@ function assertControl(control, expected) {
return control
}
// Echo the cooldown only when the director already reports it: a legacy
// director rejects the unknown key outright and would refuse every brake.
function cooldownField(before, value) {
return before.hostCooldownMs === undefined ? {} : { hostCooldownMs: value }
}
async function verifiedDisabledControl(post, generation) {
return assertControl((await post('/v1/admin/regional-rehome-control', {
v: 1,
@@ -171,6 +188,7 @@ async function applyDisabledControl(post, before) {
notBefore: before.notBefore,
ratePerMinute: before.ratePerMinute,
preferenceMaxAgeMs: before.preferenceMaxAgeMs,
...cooldownField(before, before.hostCooldownMs),
drainGraceMs: before.drainGraceMs,
confirmation: 'DISABLE_REGIONAL_REHOMING'
})).control, { generation: before.generation + 1, enabled: false })
@@ -270,6 +288,11 @@ export async function operateRegionalRehome(config, dependencies = {}) {
throw new Error('regional rehome is already paused')
}
const enabled = config.mode === 'enable'
if (enabled && before.hostCooldownMs === undefined) {
throw new Error(
'director does not report a per-host rehome cooldown; deploy a director that supports it before enabling'
)
}
const applied = await post('/v1/admin/regional-rehome-control', {
v: 1,
action: 'apply',
@@ -278,6 +301,7 @@ export async function operateRegionalRehome(config, dependencies = {}) {
notBefore: config.notBefore,
ratePerMinute: config.ratePerMinute,
preferenceMaxAgeMs: config.preferenceMaxAgeMs,
...cooldownField(before, config.hostCooldownMs),
drainGraceMs: config.drainGraceMs,
confirmation: enabled
? 'ENABLE_REGIONAL_REHOMING'
@@ -26,6 +26,7 @@ function argumentsFor(mode, confirmation) {
'--not-before', '2000000000000',
'--rate-per-minute', '10',
'--preference-max-age-ms', '86400000',
'--host-cooldown-ms', '604800000',
'--drain-grace-ms', '60000',
'--confirmation', confirmation
])
@@ -40,10 +41,29 @@ function control(generation, enabled) {
notBefore: 2_000_000_000_000,
ratePerMinute: 10,
preferenceMaxAgeMs: 86_400_000,
hostCooldownMs: 604_800_000,
drainGraceMs: 60_000
}
}
// The control a director predating the per-host cooldown reports.
function legacyControl(generation, enabled) {
const { hostCooldownMs: _absent, ...rest } = control(generation, enabled)
return rest
}
function legacyDirector(controls) {
const requests = []
const post = async (path, body) => {
requests.push({ path, body })
if (path === '/v1/admin/admission-selector/status') {
return { selector: { generation: 11, membership } }
}
return { v: 1, control: controls.shift() }
}
return { requests, post }
}
test('parses exact selector and typed control confirmation', () => {
const parsed = parseRegionalRehomeArguments(
argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING'),
@@ -52,6 +72,17 @@ test('parses exact selector and typed control confirmation', () => {
assert.equal(parsed.expectedSelectorGeneration, 11)
assert.equal(parsed.expectedControlGeneration, 4)
assert.equal(parsed.ratePerMinute, 10)
assert.equal(parsed.hostCooldownMs, 604_800_000)
assert.throws(
() => parseRegionalRehomeArguments(
argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING').filter(
(value, index, all) =>
value !== '--host-cooldown-ms' && all[index - 1] !== '--host-cooldown-ms'
),
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
),
/complete durable control shape/
)
assert.throws(
() => parseRegionalRehomeArguments(
argumentsFor('pause', 'DISABLE_REGIONAL_REHOMING'),
@@ -79,6 +110,7 @@ test('binds enable to exact selector and durable control generations', async ()
notBefore: 0,
ratePerMinute: 10,
preferenceMaxAgeMs: 86_400_000,
hostCooldownMs: 604_800_000,
drainGraceMs: 60_000,
...control
}))
@@ -96,6 +128,7 @@ test('binds enable to exact selector and durable control generations', async ()
}
})
assert.equal(result.control.generation, 5)
assert.equal(result.control.hostCooldownMs, 604_800_000)
assert.deepEqual(requests[2].body, {
v: 1,
action: 'apply',
@@ -104,11 +137,82 @@ test('binds enable to exact selector and durable control generations', async ()
notBefore: 2_000_000_000_000,
ratePerMinute: 10,
preferenceMaxAgeMs: 86_400_000,
hostCooldownMs: 604_800_000,
drainGraceMs: 60_000,
confirmation: 'ENABLE_REGIONAL_REHOMING'
})
})
test('inspects a director that predates the per-host cooldown', async () => {
const director = legacyDirector([legacyControl(4, true)])
const config = parseRegionalRehomeArguments(
argumentsFor('inspect'),
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
)
const result = await operateRegionalRehome(config, { post: director.post })
assert.equal(result.control.generation, 4)
assert.equal(result.control.hostCooldownMs, undefined)
})
for (const [mode, confirmation, enabledBefore] of [
['pause', 'PAUSE_REGIONAL_REHOMING', true],
['disable', 'DISABLE_REGIONAL_REHOMING', false]
]) {
test(`${mode} still brakes a director that predates the cooldown`, async () => {
const director = legacyDirector([
legacyControl(4, enabledBefore),
legacyControl(5, false),
legacyControl(5, false)
])
const config = parseRegionalRehomeArguments(
argumentsFor(mode, confirmation),
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
)
const result = await operateRegionalRehome(config, { post: director.post })
assert.equal(result.control.generation, 5)
// The unknown key would be refused by that director's strict schema.
assert.equal('hostCooldownMs' in director.requests[2].body, false)
assert.equal(director.requests[2].body.confirmation, 'DISABLE_REGIONAL_REHOMING')
})
}
test('failed-enable recovery brakes a director that predates the cooldown', async () => {
const requests = []
let current = legacyControl(7, true)
const result = await recoverRegionalRehomeEnable({
mode: 'recover-enable',
expectedControlGeneration: 4
}, async (_path, body) => {
requests.push(body)
if (body.action === 'inspect') return { control: current }
current = legacyControl(8, false)
return { control: current }
})
assert.equal(result.control.generation, 8)
assert.equal('hostCooldownMs' in requests[1], false)
})
test('refuses to enable a director that does not report the cooldown', async () => {
const director = legacyDirector([legacyControl(4, false)])
const config = parseRegionalRehomeArguments(
argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING'),
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
)
await assert.rejects(
operateRegionalRehome(config, { post: director.post }),
/per-host rehome cooldown/
)
// Read-only: selector status and the control inspect, and nothing else.
assert.equal(director.requests.length, 2)
assert.equal(director.requests.every(({ body }) => body.action !== 'apply'), true)
})
test('fails closed on selector drift before reading or mutating control', async () => {
let calls = 0
const config = parseRegionalRehomeArguments(
@@ -150,6 +254,7 @@ test('failed-enable recovery CAS-disables an advanced enabled generation', async
notBefore: 2_000_000_000_000,
ratePerMinute: 10,
preferenceMaxAgeMs: 86_400_000,
hostCooldownMs: 604_800_000,
drainGraceMs: 60_000,
confirmation: 'DISABLE_REGIONAL_REHOMING'
})
@@ -1,7 +1,9 @@
import { pathToFileURL } from 'node:url'
import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
const PRODUCTION_CELL = /^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26)$/
// Every general cell that carries the rehome identity: the sixteen US cells and the
// three asia-east2 cells that drain mis-homed hosts back the other way.
const PRODUCTION_CELL = /^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26|27|28|29)$/
const DIRECTOR_ORIGIN = 'https://relay.onorca.dev'
export function parseRehomeTrustProbeArguments(argv, environment = process.env) {
@@ -111,3 +111,23 @@ test('fails when both trust-probe attempts return a transient 503', async () =>
)
assert.equal(calls, 2)
})
test('approves the asia-east2 rehome sources and still rejects unlisted cells', () => {
for (const cellId of ['production-gce-c27', 'production-gce-c28', 'production-gce-c29']) {
const parsed = parseRehomeTrustProbeArguments(
argv.map((value) => (value === 'production-gce-c7' ? cellId : value)),
environment
)
assert.equal(parsed.cellId, cellId)
}
for (const cellId of ['production-gce-c1', 'production-gce-c17', 'production-gce-c30']) {
assert.throws(
() =>
parseRehomeTrustProbeArguments(
argv.map((value) => (value === 'production-gce-c7' ? cellId : value)),
environment
),
/--cell-id is not approved/
)
}
})
@@ -0,0 +1,84 @@
import assert from 'node:assert/strict'
import { readFileSync } from 'node:fs'
import test from 'node:test'
import { fileURLToPath } from 'node:url'
// Why: the region-skew alert compares asia-east2's share of assignment hints against its share of
// actual placements. Both shares are sums over one log-based metric per region, and the region
// list is written out by hand in Terraform. A region added to the contract without matching
// metrics would silently drop out of both denominators and move the ratio the alert fires on.
const read = (relative) => readFileSync(fileURLToPath(new URL(relative, import.meta.url)), 'utf8')
const collapse = (text) => text.replaceAll(/\s+/g, ' ')
const contractRegions = (() => {
const source = read('../../packages/relay-contract/src/relay-regions.ts')
const literal = /export const RELAY_REGIONS = \[([^\]]*)\]/.exec(source)
assert.ok(literal, 'RELAY_REGIONS literal not found in relay-regions.ts')
return [...literal[1].matchAll(/'([^']+)'/g)].map((match) => match[1])
})()
const terraform = read('../../infra/terraform/relay-observability.tf')
const terraformRegions = (() => {
const literal = /relay_region_keys = \[([^\]]*)\]/.exec(terraform)
assert.ok(literal, 'relay_region_keys not found in relay-observability.tf')
return [...literal[1].matchAll(/"([^"]+)"/g)].map((match) => match[1])
})()
// Both sides now spell the field-name segments out, so the test compares the two declared maps
// rather than two source expressions. Reformatting either file cannot break this, and a literal
// expected value below still catches an identical wrong edit made to both.
const declaredSegments = (source, open, close) => {
const body = source.slice(source.indexOf(open) + open.length, source.indexOf(close, source.indexOf(open)))
return Object.fromEntries(
[...body.matchAll(/'?"?([a-z0-9-]+)'?"?\s*[:=]\s*'?"?([A-Za-z0-9]+)'?"?/g)].map((match) => [
match[1],
match[2]
])
)
}
const terraformSegments = declaredSegments(terraform, 'relay_region_field_segments = {', '}')
const contractSegments = declaredSegments(
read('../../packages/relay-contract/src/relay-regions.ts'),
'RELAY_REGION_METRIC_SEGMENTS = {',
'}'
)
test('terraform covers exactly the regions the contract can hint or select', () => {
assert.deepEqual([...terraformRegions].sort(), [...contractRegions].sort())
})
test('terraform and the contract declare the same flat field segments', () => {
assert.deepEqual(terraformSegments, contractSegments)
// Pinned literally so the same wrong edit applied to both sides still fails.
assert.deepEqual(terraformSegments, { 'us-central1': 'UsCentral1', 'asia-east2': 'AsiaEast2' })
assert.deepEqual(Object.keys(terraformSegments).sort(), [...contractRegions].sort())
})
test('the skew query compares a catalogued region against itself', () => {
const columns = terraformRegions.map((region) => region.replaceAll('-', '_'))
const hint = /hint_share: req_([a-z0-9_]+) \//.exec(terraform)
const placement = /placement_share: sel_([a-z0-9_]+) \//.exec(terraform)
assert.ok(hint && placement, 'skew query share columns not found')
assert.equal(hint[1], placement[1], 'the two shares must be about the same region')
assert.ok(columns.includes(hint[1]), `${hint[1]} is not one of ${columns.join(', ')}`)
})
test('the skew condition never divides by the placement share', () => {
// A zero-placement hour is the worst skew there is; MQL drops the row on x/0, so the ratio form
// silences exactly the case the alert exists for.
assert.ok(
!/hint_share \/ placement_share/.test(terraform),
'cross-multiply instead: hint_share > 2 * placement_share'
)
assert.match(collapse(terraform), /condition hint_share > 2 \* placement_share/)
})
test('the unhinted bucket stays out of the skew denominators', () => {
assert.ok(
!terraformRegions.includes('unhinted'),
'unhinted requests are a client-side choice, not a region; including them moves the share'
)
})
@@ -179,9 +179,10 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
it('validates a correct plan for every wave cell at that cell\'s rehome protocol', () => {
for (const cellId of SAME_CAP_CELLS) {
const [region, cap] = resolveCellShape(cellId).stdout.trim().split(' ')
const [, cap] = resolveCellShape(cellId).stdout.trim().split(' ')
const protocol = REHOME_SOURCE_CELLS.has(cellId) ? 1 : 0
assert.equal(protocol, region === 'us-central1' ? 1 : 0, cellId)
// Every reviewed serving cell carries rehome trust now, in either region.
assert.equal(protocol, 1, cellId)
const config = {
mode: 'same-cap-cell',
cellId,
@@ -211,6 +212,29 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
}
})
it('validates a protocol-0 plan for a cell outside the rehome source list', () => {
const cellId = 'production-gce-c17'
assert.equal(REHOME_SOURCE_CELLS.has(cellId), false)
const config = {
mode: 'same-cap-cell',
cellId,
hardCap: 1000,
unobservedBound: 60,
image: TARGET_IMAGE,
rollbackImage: ROLLBACK_IMAGE,
rehomeDirectorServiceAccount: DIRECTOR_IDENTITY,
rehomeAudience: AUDIENCE,
regionalRehomeProtocol: '0'
}
const plan = rollPlan({ cellId, cap: 1000, protocol: 0 })
assert.deepEqual(validateCapacityPlan(plan, config), { mode: 'same-cap-cell', changes: 2 })
// Protocol 1 must reject a plan with no rehome lines, or the absent-line rule decides nothing.
assert.throws(
() => validateCapacityPlan(plan, { ...config, regionalRehomeProtocol: '1' }),
/reviewed image and capacity/
)
})
it('leaves the US-only capacity job on the default allowlist', () => {
assert.doesNotMatch(capacityWorkflow, /--approved-cells/)
})
+12
View File
@@ -464,6 +464,18 @@ Once a target control is registered, do not force the pre-registration rollback.
After a deployment traffic shift, preserve the old revision/tag until metrics and live reconnect checks pass. If the new revision is unhealthy, shift traffic back only while old controls are still valid, then issue a strictly newer director migration rather than reusing a prior epoch.
## Regional rehoming
Rehoming moves a host to a general cell in the region its desktop last reported, in either
direction. Both roles need the drain protocol: a cell without it can be neither a source nor a
target, and it is not part of the fleet whose telemetry gates the worker. Until the asia-east2
cells run `regionalRehomeProtocol` 1 they are none of the three, so no host is moved into or out
of Asia and an Asia cell in distress does not pause the worker.
`host-cooldown-ms` is the minimum gap between two rehomes of one host. It bounds the damage from
a desktop whose region probe flips: without it the host would be dragged back across the ocean on
every flip, since the preference age never expires while the host keeps reconnecting.
## Game-day matrix
Run and record each scenario in staging before launch:
+73
View File
@@ -121,6 +121,79 @@ durably marked consumed before mutation and cannot authorize another run.
Expected enabled cells must also have a powered runtime, healthy and ready endpoints, fresh
heartbeats, and matching live admission.
## Region placement alert policies
Cloud Monitoring alert policies, not monitor freeze bars: these page from
`cloud/infra/terraform/relay-observability.tf` on the shared relay channel in
`relay_alert_notification_channels`, and they do not gate any workflow. All
three exist because US desktops sat on asia-east2 cells for weeks in 2026-08
with every existing bar green.
| Alert policy | Condition |
| --- | ---: |
| Orca Relay: far-cell phone accept latency | per cell, median 30-second `clientAcceptTotalMsP95` over 15 minutes above 2,000 ms with at least 20 completed accepts |
| Orca Relay: cell control round trip | per cell, median `controlRttMsP50` over one hour above 150 ms with at least 500 samples |
| Orca Relay: region hint skew | fleet-wide, asia-east2 share of hinted requests over one hour more than 2x and more than 15 points above its share of actual placements, with at least 500 hinted requests |
Threshold basis:
- Accept latency. An in-region phone accept completes in 0.3-0.6 s and a
cross-Pacific one in 5-10 s, so 2,000 ms sits outside in-region noise and
well under the far-cell floor. The 20-accept minimum keeps one slow accept
on a quiet cell off the pager. The p95 is the published value, so the
window aggregate is its median, not its max.
- Control round trip. In-region is tens of milliseconds; a US desktop on an
asia-east2 cell is 200 ms or more. Only the p50 is used. The desktop echoes
the pong on its main thread, so the published p95 and max track renderer
stalls rather than distance. 500 samples per hour is about two
continuously connected hosts at the 15-second control ping. Tuning risk: EU
desktops on us-central1 sit at 100-130 ms, so a cell whose population is
mostly European can approach the bar while correctly homed. Check where the
hosts are before reading a first breach as mis-homing.
- Region hint skew. This compares two shares of the same hour rather than
testing one absolute share, because an absolute bar is wrong at both ends.
Measured over twelve hours on 2026-09-07, while the desktop region probe
was still mis-picking: asia-east2 was 33.8% of the 33,800 hinted requests
and only 7.9% of the 45,364 assignments, a divergence of 4.27x and a gap of
25.9 points. A fixed 40% bar would have stayed silent through that, and
once the probe is fixed the genuine APAC share climbs past any such bar and
pages forever on the correct end state. The 2x and 15-point bars sit inside
the broken state and outside a healthy one. `unhinted` requests are
excluded from the denominator: they were 27% of all requests, so a client
change that always sends a hint would move the number with no behaviour
change at all. The two bars are cross-multiplied rather than divided. An
hour that placed nobody in the region is the most extreme skew there is,
and it happens whenever the region is drained, fenced, or at capacity, but
dividing by that zero placement share makes MQL drop the row and lose the
series before any other clause runs.
Expect the skew alert to stay lit after a client fix until the mis-homed
backlog is rehomed. Sticky assignment never re-consults the hint, so a
desktop already on an asia cell keeps being placed there whatever it now
asks for; the ratio clears only once the rehome sweep has drained.
All three conditions are written in MQL rather than the metric filters the
other relay policies use. Every runtime metric is a DELTA DISTRIBUTION, and
the only scalar aligners a filter condition can apply to one are percentiles;
each of these alerts needs the sum of the extracted values as a volume floor,
which is `sum(value.<metric>)` in MQL and unreachable otherwise. None of the
metrics they read exists in the project yet, so what was checked against
production is the query shape: the same MQL run over existing metrics of the
same kind confirmed the distribution sum, the join arity, the unit literals,
and the condition clause.
The skew shares are built from one log-based metric per region for hints and
one per region for placements. They read flat `requestedRegion<Region>Delta`
and `selectedRegion<Region>Delta` fields that the relay publishes as zeros in
every interval, not the nested region maps: a log-based metric would need a
quoted field path to reach a hyphenated map key, and an absent key would drop
a series out of the inner join. The region list lives in Terraform as
`relay_region_keys` and is pinned to relay-contract's `RELAY_REGIONS` by
`dev/scripts/relay-region-hint-metrics.test.mjs`. Both sides spell the field
name segments out as literal maps rather than deriving them, so the same test
compares the two declarations directly. Adding a region to the contract
without its segment is a compile error in relay-contract, not a silent gap.
## Implementation log
- Recalibrated the relay pool freezes from 30 waiters / 1,000 ms to
@@ -402,7 +402,11 @@ relay_region_rehome_source_cell_ids = [
"production-gce-c23",
"production-gce-c24",
"production-gce-c25",
"production-gce-c26"
"production-gce-c26",
# Asia cells carry the same trust so mis-homed hosts can be drained back off them.
"production-gce-c27",
"production-gce-c28",
"production-gce-c29"
]
# Slack #orca-relay-alerts, created out of band on 2026-08-05. Declared here because an apply
+3 -2
View File
@@ -82,14 +82,15 @@ check "relay_gce_fixed_one_topology" {
assert {
condition = alltrue([
# Region is not asserted here: the director's own rehome source and target predicates
# own eligibility, so this pins only cell shape.
for cell_id in var.relay_region_rehome_source_cell_ids : try(
var.relay_gce_cells[cell_id].region == var.region &&
var.relay_gce_cells[cell_id].connection_hard_cap != null &&
!contains(var.relay_gce_fenced_cells, cell_id),
false
)
])
error_message = "Regional rehome sources must be configured, unfenced primary-region GCE cells with explicit connection limits."
error_message = "Regional rehome sources must be configured, unfenced GCE cells with explicit connection limits."
}
assert {
+215 -3
View File
@@ -65,6 +65,20 @@ locals {
control_renewal_lease_misses = { field = "controlRenewalLeaseMissesDelta", description = "Control renewals that found their activity lease missing." }
control_activity_recoveries = { field = "controlActivityRecoveriesDelta", description = "Control activity leases recovered after a renewal miss." }
control_activity_recovery_failures = { field = "controlActivityRecoveryFailuresDelta", description = "Control activity lease recovery attempts that failed." }
control_rtt_ms_p50 = { field = "controlRttMsP50", description = "Control-socket ping round trip p50 in the interval. The desktop echoes the pong on its main thread, so only the median reads as distance; the p95 and max below are dominated by desktop stalls." }
control_rtt_ms_p95 = { field = "controlRttMsP95", description = "Control-socket ping round trip p95 in the interval; a desktop-stall signal, not a distance one." }
control_rtt_ms_max = { field = "controlRttMsMax", description = "Maximum control-socket ping round trip in the interval; a desktop-stall signal, not a distance one." }
control_rtt_samples = { field = "controlRttSamplesDelta", description = "Control-socket round trips observed in the interval, one per ping answered; the percentiles above are omitted when this is zero." }
control_rtt_samples_dropped = { field = "controlRttSamplesDroppedDelta", description = "Observed round trips the bounded percentile reservoir did not keep; non-zero means the percentiles above summarise a uniform sample of the interval." }
client_accepts_completed = { field = "clientAcceptCompletedDelta", description = "Phone accepts that reached relay-hello in the interval; the percentiles below are omitted when this is zero." }
client_accept_total_ms_p50 = { field = "clientAcceptTotalMsP50", description = "Successful phone-accept duration p50, dial to relay-hello." }
client_accept_total_ms_p95 = { field = "clientAcceptTotalMsP95", description = "Successful phone-accept duration p95, dial to relay-hello." }
client_accept_total_ms_max = { field = "clientAcceptTotalMsMax", description = "Maximum successful phone-accept duration in the interval." }
client_accept_assignment_ms_p95 = { field = "clientAcceptAssignmentMsP95", description = "Accept stage p95: resume/invite lookup plus assignment resolve." }
client_accept_credential_ms_p95 = { field = "clientAcceptCredentialMsP95", description = "Accept stage p95: outer credential reservation." }
client_accept_activity_ms_p95 = { field = "clientAcceptActivityMsP95", description = "Accept stage p95: credential activity lease acquisition." }
client_accept_attach_ms_p95 = { field = "clientAcceptAttachMsP95", description = "Accept stage p95: conn-open sent until the desktop's data leg authenticated." }
client_accept_basis_ms_p95 = { field = "clientAcceptBasisMsP95", description = "Accept stage p95: splice lease and connection-basis writes between the data leg and relay-hello." }
heap_used_bytes = { field = "heapUsedBytes", description = "Node.js heap bytes used by the relay process." }
event_loop_ms_p99 = { field = "eventLoopDelayMsP99", description = "Node.js event-loop delay p99 in milliseconds." }
forwarded_bytes = { field = "forwardedBytesDelta", description = "Ciphertext bytes admitted for forwarding." }
@@ -80,6 +94,80 @@ locals {
db_oldest_wait_ms = { field = "databasePoolOldestWaitMs", description = "Current oldest PostgreSQL pool waiter age." }
db_wait_ms_max = { field = "databasePoolWaitMsMax", description = "Maximum PostgreSQL pool wait during the interval." }
}
# Regions the director can hint or select. Pinned to relay-contract's RELAY_REGIONS by
# dev/scripts/relay-region-hint-metrics.test.mjs, which also checks the flat field names below
# against the emitter. A region missing here drops out of both shares the skew alert compares.
relay_region_keys = ["us-central1", "asia-east2"]
# Flat emitter fields, not the nested `requestedRegionsDelta` map: a log-based metric would need
# a quoted field path to reach a hyphenated map key, and the relay publishes these as zeros in
# every interval so no series can drop out of the alert's inner join. Spelled out rather than
# derived, so this literal and relay-contract's RELAY_REGION_METRIC_SEGMENTS can be compared
# directly; reformatting either side cannot break the check and neither can drift alone.
relay_region_field_segments = {
"us-central1" = "UsCentral1"
"asia-east2" = "AsiaEast2"
}
relay_region_columns = { for key in local.relay_region_keys : key => replace(key, "-", "_") }
relay_region_share_metrics = merge(
{
for key in local.relay_region_keys :
"requested_regions_${local.relay_region_columns[key]}" => {
field = "requestedRegion${local.relay_region_field_segments[key]}Delta"
description = "Assignment requests that hinted ${key}."
}
},
{
for key in local.relay_region_keys :
"selected_regions_${local.relay_region_columns[key]}" => {
field = "selectedRegion${local.relay_region_field_segments[key]}Delta"
description = "Assignments that placed a host in ${key}."
}
}
)
relay_region_hinted_total = join(" + ", [for key in local.relay_region_keys : "req_${local.relay_region_columns[key]}"])
relay_region_selected_total = join(" + ", [for key in local.relay_region_keys : "sel_${local.relay_region_columns[key]}"])
# MQL, not a filter condition: every runtime metric is a DELTA DISTRIBUTION, and the only scalar
# aligners a `condition_threshold` can apply to one are percentiles. Both shares need the sum of
# the extracted values, which is `sum(value.<metric>)` in MQL and unreachable otherwise.
relay_region_hint_skew_query = join("\n", concat(
["{"],
flatten([
for index, entry in [
for key in local.relay_region_keys : { metric = "requested_regions_${local.relay_region_columns[key]}", column = "req_${local.relay_region_columns[key]}" }
] : [
index == 0 ? "" : ";",
" fetch cloud_run_revision::logging.googleapis.com/user/orca_relay_${entry.metric}",
" | align delta(1h) | every 1h",
" | group_by [], [${entry.column}: sum(value.orca_relay_${entry.metric})]"
]
]),
flatten([
for key in local.relay_region_keys : [
";",
" fetch cloud_run_revision::logging.googleapis.com/user/orca_relay_selected_regions_${local.relay_region_columns[key]}",
" | align delta(1h) | every 1h",
" | group_by [], [sel_${local.relay_region_columns[key]}: sum(value.orca_relay_selected_regions_${local.relay_region_columns[key]})]"
]
]),
[
"}",
"| join",
"| value [",
" hint_share: req_asia_east2 / (${local.relay_region_hinted_total}),",
" placement_share: sel_asia_east2 / (${local.relay_region_selected_total}),",
" hinted_requests: ${local.relay_region_hinted_total}",
" ]",
# Cross-multiplied, never a plain ratio of the two shares: an hour that placed nobody in the
# region makes that ratio 0/0 or x/0, and MQL drops the row instead of yielding a number, so
# the whole series vanishes before the other clauses run. That hour is the worst skew there
# is - every desktop asking for a region the director is putting nobody in - and it happens
# whenever the region is drained, fenced, or at capacity. Both forms were run read-only
# against production surrogates with a zero denominator: the ratio returned no rows, this
# returned the series with the condition true.
"| condition hint_share > 2 * placement_share && hint_share - placement_share > 0.15 '1' && hinted_requests > 500 '1'"
]
))
relay_custom_alerts = {
connection_headroom = {
pages_oncall = true
@@ -201,7 +289,9 @@ locals {
}
resource "google_logging_metric" "relay_snapshot" {
for_each = local.relay_runtime_metrics
# Region-request metrics ride the same event and shape; merging adds map entries only, so the
# existing metric instances are untouched (a label change, not a new key, is what recreates them).
for_each = merge(local.relay_runtime_metrics, local.relay_region_share_metrics)
project = var.project_id
name = "orca_relay_${each.key}"
@@ -211,14 +301,14 @@ resource "google_logging_metric" "relay_snapshot" {
label_extractors = {
role = "EXTRACT(jsonPayload.role)"
cell_id = "EXTRACT(jsonPayload.cellId)"
# No region label: adding one replaces all 21 live metrics (label change = delete+create),
# No region label: adding one replaces all 42 live metrics (label change = delete+create),
# which resets history and blanks the relay alert policies during the swap.
}
metric_descriptor {
metric_kind = "DELTA"
value_type = "DISTRIBUTION"
unit = contains(["sql_latency_ms", "control_renewal_latency_ms_p50", "control_renewal_latency_ms_p95", "control_renewal_latency_ms_max", "http_latency_ms", "event_loop_ms_p99", "db_oldest_wait_ms", "db_wait_ms_max"], each.key) ? "ms" : each.key == "queued_bytes" || each.key == "heap_used_bytes" || each.key == "forwarded_bytes" ? "By" : "1"
unit = contains(["sql_latency_ms", "control_rtt_ms_p50", "control_rtt_ms_p95", "control_rtt_ms_max", "client_accept_total_ms_p50", "client_accept_total_ms_p95", "client_accept_total_ms_max", "client_accept_assignment_ms_p95", "client_accept_credential_ms_p95", "client_accept_activity_ms_p95", "client_accept_attach_ms_p95", "client_accept_basis_ms_p95", "control_renewal_latency_ms_p50", "control_renewal_latency_ms_p95", "control_renewal_latency_ms_max", "http_latency_ms", "event_loop_ms_p99", "db_oldest_wait_ms", "db_wait_ms_max"], each.key) ? "ms" : each.key == "queued_bytes" || each.key == "heap_used_bytes" || each.key == "forwarded_bytes" ? "By" : "1"
labels {
key = "role"
@@ -672,6 +762,128 @@ resource "google_monitoring_alert_policy" "relay_cell_process_exit" {
depends_on = [google_logging_metric.relay_incident]
}
# Why: nothing fired while US desktops sat on asia-east2 cells for weeks in 2026-08. The two
# per-cell policies below read that as distance, and the fleet-wide one reads it as a bad region
# hint. All three are MQL because each needs the sum of a DELTA DISTRIBUTION as a volume floor,
# and the only scalar aligners a `condition_threshold` can apply to a distribution are percentiles.
# `join` is an inner join and the relay omits its percentile fields on an empty interval, so an
# idle cell drops out rather than alerting on nothing. The per-cell arms fetch `gce_instance`
# only: production runs no Cloud Run cells (`relay_cells` is empty), and a future one would need
# its own arm here. None of the metrics these query exist in the project yet, so what was checked
# against production is the query shape: the same MQL run over existing metrics of the same kind
# confirmed the distribution sum, the join arity, the unit literals, and the condition clause.
resource "google_monitoring_alert_policy" "relay_far_cell_accept_latency" {
project = var.project_id
display_name = "Orca Relay: far-cell phone accept latency"
combiner = "OR"
enabled = true
notification_channels = var.relay_alert_notification_channels
conditions {
display_name = "Phone accept p95 above 2 s for 15 minutes"
condition_monitoring_query_language {
# percentile(..., 50) over the window, not max: the published value is already a p95, so the
# median of the interval p95s reads as sustained slowness instead of one bad 30-second flush.
query = <<-EOT
{
fetch gce_instance::logging.googleapis.com/user/orca_relay_client_accept_total_ms_p95
| align delta(15m) | every 15m
| group_by [metric.cell_id], [accept_p95_ms: percentile(value.orca_relay_client_accept_total_ms_p95, 50)]
;
fetch gce_instance::logging.googleapis.com/user/orca_relay_client_accepts_completed
| align delta(15m) | every 15m
| group_by [metric.cell_id], [accepts: sum(value.orca_relay_client_accepts_completed)]
}
| join
| condition accept_p95_ms > 2000 'ms' && accepts >= 20 '1'
EOT
duration = "0s"
trigger {
count = 1
}
}
}
documentation {
content = "Phones on this cell are taking over two seconds to reach relay-hello. Measured separation: an in-region accept completes in 0.3-0.6 s and a cross-Pacific one in 5-10 s, so 2 s sits well outside in-region noise and well below the far-cell floor. The 20-accept floor over 15 minutes keeps a single slow accept on a quiet cell from paging. Check which regions the cell's hosts are actually in before touching capacity: the 2026-08 cause was desktops requesting the wrong region, not a slow cell. Read the per-stage `orca_relay_client_accept_*_ms_p95` metrics to separate distance from assignment, credential, or attach work."
mime_type = "text/markdown"
}
depends_on = [google_logging_metric.relay_snapshot]
}
resource "google_monitoring_alert_policy" "relay_cell_control_rtt" {
project = var.project_id
display_name = "Orca Relay: cell control round trip"
combiner = "OR"
enabled = true
notification_channels = var.relay_alert_notification_channels
conditions {
display_name = "Control ping p50 above 150 ms for an hour"
condition_monitoring_query_language {
# p50 only. The desktop echoes the pong on its main thread, so the published p95 and max
# track renderer stalls, not distance; the median is the only column that reads as distance.
query = <<-EOT
{
fetch gce_instance::logging.googleapis.com/user/orca_relay_control_rtt_ms_p50
| align delta(1h) | every 1h
| group_by [metric.cell_id], [control_rtt_p50_ms: percentile(value.orca_relay_control_rtt_ms_p50, 50)]
;
fetch gce_instance::logging.googleapis.com/user/orca_relay_control_rtt_samples
| align delta(1h) | every 1h
| group_by [metric.cell_id], [samples: sum(value.orca_relay_control_rtt_samples)]
}
| join
| condition control_rtt_p50_ms > 150 'ms' && samples >= 500 '1'
EOT
duration = "0s"
trigger {
count = 1
}
}
}
documentation {
content = "The median desktop on this cell is more than 150 ms away from it, which is a mis-homed population rather than a cell fault: an in-region control ping is tens of milliseconds and a US desktop on an asia-east2 cell is 200 ms or more. This is the signal that was missing while roughly 226 of 332 hosts on the asia cells were non-APAC for weeks in 2026-08. Confirm with the assignment table which regions those hosts requested, then rehome; do not restart or drain the cell on this alert alone. The 500-sample floor is about two continuously connected hosts at the 15-second control ping, so a nearly idle cell cannot alert on one desktop. Tuning risk: EU desktops on us-central1 sit at 100-130 ms, so a cell whose population is mostly European can approach 150 ms while correctly homed. Check where the hosts are before treating a first breach as mis-homing, and raise the bar only with that evidence."
mime_type = "text/markdown"
}
depends_on = [google_logging_metric.relay_snapshot]
}
resource "google_monitoring_alert_policy" "relay_region_hint_skew" {
project = var.project_id
display_name = "Orca Relay: region hint skew"
combiner = "OR"
enabled = true
notification_channels = var.relay_alert_notification_channels
conditions {
display_name = "asia-east2 hint share above 2x its placement share for an hour"
condition_monitoring_query_language {
query = local.relay_region_hint_skew_query
duration = "0s"
trigger {
count = 1
}
}
}
documentation {
content = "Desktops are asking the director for asia-east2 far more often than the director actually places them there, which is what silently homed US desktops on asia cells through 2026-08. The alert compares two shares of the same hour and never an absolute share, because an absolute bar is wrong at both ends: measured over twelve hours on 2026-09-07, while the desktop region probe was still mis-picking, asia-east2 was 33.8% of the 33,800 hinted requests but only 7.9% of the 45,364 assignments, and once the probe is fixed the genuine APAC share will climb past any fixed bar that would have caught this. Divergence was 4.27x with a 25.9-point gap, so the 2x and 15-point bars sit well inside the broken state and well outside a healthy one. `unhinted` requests are excluded from the denominator: they were 27% of all requests, and a client change that always sends a hint would move this number without any behaviour changing. Expect this to stay lit until the mis-homed backlog is rehomed, because sticky assignment never re-consults the hint, so a desktop already on an asia cell keeps being placed there no matter what it now asks for. Investigate the desktop region probe first, not relay placement."
mime_type = "text/markdown"
}
depends_on = [google_logging_metric.relay_snapshot]
}
# Why: the four signals that had to be assembled by hand during the 2026-09-04 incident.
resource "google_monitoring_dashboard" "relay_incident" {
project = var.project_id
+1 -1
View File
@@ -245,7 +245,7 @@ variable "relay_regional_placement_enabled" {
variable "relay_region_rehome_source_cell_ids" {
type = set(string)
description = "Reviewed US Relay cells allowed to advertise and accept the regional rehome source protocol."
description = "Reviewed Relay cells, in any configured region, allowed to advertise and accept the regional rehome source protocol."
default = []
}
+1 -1
View File
@@ -20,7 +20,7 @@
"load:relay:model": "node dev/scripts/run-relay-load-model.mjs",
"load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs",
"ops:relay": "pnpm --filter @orca-cloud/relay-ops dev",
"pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-admission-workflow.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs",
"pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-admission-workflow.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs",
"test": "pnpm -r test && node --test dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-regional-rehome-workflow.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs",
"typecheck": "pnpm -r typecheck"
},
@@ -6,7 +6,10 @@ import {
HostChallengeSchema,
HostDataAuthSchema,
HostHelloAckSchema,
HostHelloSchema
HostHelloSchema,
parseRelayHostCapabilities,
RELAY_HOST_CAPABILITIES_HEADER,
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS
} from './control-messages.js'
import {
DeviceCredentialInstallSchema,
@@ -345,3 +348,47 @@ describe('relay protocol contract', () => {
).toBe(false)
})
})
describe('pending connection details capability', () => {
it('reads a pending entry with or without the stated kind and device', () => {
const ack = {
v: 1 as const,
generation: 3,
controlResumeSecret: 'R'.repeat(43),
leaseExpiresAt: 1_800_000_000_000,
activeConnIds: []
}
const identifiers = { connId: 'conn-1', connTicket: 'T'.repeat(43) }
expect(HostHelloAckSchema.safeParse({ ...ack, pendingConns: [identifiers] }).success).toBe(true)
expect(
HostHelloAckSchema.safeParse({
...ack,
pendingConns: [{ ...identifiers, kind: 'resume', relayDeviceId: 'device-1' }]
}).success
).toBe(true)
// Still strict otherwise: an unannounced key must not slip through as data.
expect(
HostHelloAckSchema.safeParse({
...ack,
pendingConns: [{ ...identifiers, reservationId: 'injected' }]
}).success
).toBe(false)
})
it('pins the header and token the desktop mirrors by hand', () => {
// The desktop cannot import this package; drift silently disables the
// feature, so both literals are asserted on each side.
expect(RELAY_HOST_CAPABILITIES_HEADER).toBe('x-orca-host-capabilities')
expect(RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS).toBe('pending-conn-details')
})
it('reads the advertised capabilities from a control upgrade header', () => {
expect(
parseRelayHostCapabilities(` ${RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS} , future-thing`)
).toEqual(new Set([RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS, 'future-thing']))
// A host that predates the header sends nothing; absence is never capable.
expect(parseRelayHostCapabilities(undefined).size).toBe(0)
expect(parseRelayHostCapabilities('').size).toBe(0)
expect(parseRelayHostCapabilities('x'.repeat(65)).size).toBe(0)
})
})
@@ -44,8 +44,35 @@ export const HostChallengeAckSchema = z
.object({ challengeId: OpaqueIdSchema, proofB64: Base6432ByteSchema })
.strict()
// Advertised on the control upgrade rather than in host-hello: HostHelloSchema
// is strict, so a new hello key is refused by every already-deployed cell.
export const RELAY_HOST_CAPABILITIES_HEADER = 'x-orca-host-capabilities'
// The host accepts kind/relayDeviceId on a pendingConns entry. A host that does
// not advertise this parses those entries strictly and would drop the whole ack.
export const RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS = 'pending-conn-details'
export function parseRelayHostCapabilities(
header: string | string[] | undefined
): ReadonlySet<string> {
const raw = Array.isArray(header) ? header.join(',') : (header ?? '')
return new Set(
raw
.split(',')
.map((token) => token.trim())
.filter((token) => token.length > 0 && token.length <= 64)
.slice(0, 16)
)
}
// kind/relayDeviceId are optional so an entry stays readable by a host that
// predates them; the cell only emits them to a host that advertised support.
const PendingConnectionSchema = z
.object({ connId: OpaqueIdSchema, connTicket: Base64Url32ByteSchema })
.object({
connId: OpaqueIdSchema,
connTicket: Base64Url32ByteSchema,
kind: ConnectionKindSchema.optional(),
relayDeviceId: OpaqueIdSchema.optional()
})
.strict()
export const HostHelloAckSchema = z
@@ -8,6 +8,15 @@ export type RelayRegion = z.infer<typeof RelayRegionSchema>
export const RELAY_DEFAULT_REGION: RelayRegion = 'us-central1'
// Field-name segment for the flat per-region runtime counters, spelled out rather than derived so
// the Terraform side can hold the same literal and a test can compare the two. `satisfies` makes a
// new region a compile error here, which is the point: a region with no segment would silently
// drop out of the region-skew alert's denominators.
export const RELAY_REGION_METRIC_SEGMENTS = {
'us-central1': 'UsCentral1',
'asia-east2': 'AsiaEast2'
} as const satisfies Record<RelayRegion, string>
const RelayProbeOriginSchema = z.string().url().max(2_048).refine(isCanonicalHttpsOrigin)
export const RelayRegionCatalogResponseSchema = z
+16 -51
View File
@@ -13325,9 +13325,7 @@
},
{
"file": "src/main/runtime/orchestration/mailbox-pointer-stage.test.ts",
"assertions": [
"a refused pointer write drains a delivery parked behind its watermark"
]
"assertions": ["a refused pointer write drains a delivery parked behind its watermark"]
},
{
"file": "src/main/providers/settled-pty-writer-census.test.ts",
@@ -18549,27 +18547,13 @@
"protection": "partial",
"owner": "browser-runtime",
"layer": "electron-packaged",
"surfaces": [
"paired browser placement"
],
"platforms": [
"linux",
"macos",
"windows"
],
"providers": [
"paired-runtime"
],
"coveredPlatforms": [
"linux"
],
"coveredProviders": [
"paired-runtime"
],
"surfaces": ["paired browser placement"],
"platforms": ["linux", "macos", "windows"],
"providers": ["paired-runtime"],
"coveredPlatforms": ["linux"],
"coveredProviders": ["paired-runtime"],
"coverageNotes": "Published Linux 1.4.188 desktop against current source in both directions; scheduled weekly and manually runnable. No required PR check.",
"motivatingLinks": [
"https://github.com/stablyai/orca/actions/runs/34069063016"
],
"motivatingLinks": ["https://github.com/stablyai/orca/actions/runs/34069063016"],
"invariant": "A paired client and host without client-hosted browser capabilities retain server-hosted browser placement across supported version skew.",
"oracle": "Require both existing named browser placement scenarios to pass three times with one attempt, zero skips, zero failures, and no report errors.",
"commands": [
@@ -18593,9 +18577,7 @@
},
{
"file": "config/scripts/verify-packaged-browser-participation.test.mjs",
"assertions": [
"reject missing, substituted, skipped and retried scenarios"
]
"assertions": ["reject missing, substituted, skipped and retried scenarios"]
},
{
"file": "config/scripts/packaged-browser-lane-contract.test.mjs",
@@ -18650,26 +18632,13 @@
"protection": "partial",
"owner": "terminal-input",
"layer": "electron-native-ime-e2e",
"surfaces": [
"native Hangul composition",
"Wayland terminal input"
],
"platforms": [
"linux"
],
"providers": [
"local"
],
"coveredPlatforms": [
"linux"
],
"coveredProviders": [
"local"
],
"surfaces": ["native Hangul composition", "Wayland terminal input"],
"platforms": ["linux"],
"providers": ["local"],
"coveredPlatforms": ["linux"],
"coveredProviders": ["local"],
"coverageNotes": "Ubuntu 22.04 nested GNOME and IBus Hangul drive three complete native executions in GitHub Actions. GNOME owns IBus; daemon and CLI share its default config discovery path.",
"motivatingLinks": [
"https://github.com/stablyai/orca/pull/19174"
],
"motivatingLinks": ["https://github.com/stablyai/orca/pull/19174"],
"invariant": "Typing d k 1 Return through native IBus Hangul delivers exactly 아1 followed by newline without missing, duplicate, or reordered characters.",
"oracle": "Three executions each assert three exact UTF-8 PTY lines. Verify the exact Playwright title, zero skips/retries, each individual native composition receipt, and the nested launch Wayland flag.",
"commands": [
@@ -18685,15 +18654,11 @@
"assertionRefs": [
{
"file": "tests/e2e/terminal-hangul-terminating-digit-native.spec.ts",
"assertions": [
"a digit typed right after a Hangul syllable reaches the pty"
]
"assertions": ["a digit typed right after a Hangul syllable reaches the pty"]
},
{
"file": "config/scripts/terminal-ime-e2e-workflow.test.mjs",
"assertions": [
"runs native Wayland independently with CJK fonts and retained evidence"
]
"assertions": ["runs native Wayland independently with CJK fonts and retained evidence"]
}
],
"evidenceRuns": [
@@ -38,6 +38,16 @@ const SUPPRESSED_REACT_DOCTOR_DIAGNOSTICS = new Map([
new Set([
'src/renderer/src/components/editor/combined-diff/review-controls/use-combined-diff-view-preferences.ts'
])
],
[
// The rule wants one named handle cleared by name. Both startup effects arm a variable number
// of refresh timers, every one of them through addTimer into `timers`, which their cleanups
// clear -- a shape the rule reports whether the handles live in an array, a Set, or a nested
// helper. The finding predates this list; it surfaced when the effect body changed. This map
// keys on file, not line, so the entry covers both effects in it; nothing else in the file
// arms a timer, so widening it further is the only alternative, not a narrower option.
'react-doctor(effect-needs-cleanup)',
new Set(['mobile/src/session/use-mobile-session-startup.ts'])
]
])
@@ -173,6 +173,28 @@ describe('rebuild-native-deps patched node-pty rebuild', () => {
}
})
it('refuses a Windows rebuild when the process creation-time patch is missing', () => {
const projectDir = mkTempProject()
try {
writeFakeUsableElectronPackage(projectDir, { platform: 'win32' })
writeFakeElectronRebuild(projectDir)
writeFakeNodePtyConptyPayload(projectDir, 'x64')
writeFakeWindowsProcessTreeWithNodeAddonApi(projectDir, { creationTimePatchApplied: false })
const result = runRebuildScript(
projectDir,
{ npm_config_platform: 'win32', npm_config_arch: 'x64' },
['--platform=win32', '--arch=x64', '--force']
)
expect(result.status).not.toBe(0)
expect(result.stderr).toContain('process creation-time patch')
} finally {
removeTreeSync(projectDir)
}
})
it('restores the ConPTY runtime payload after a Windows Electron rebuild', () => {
const projectDir = mkTempProject()
@@ -374,13 +374,18 @@ export function writeFakeWindowsProcessTree(projectDir) {
export function writeFakeWindowsProcessTreeWithNodeAddonApi(
projectDir,
{ commandLinePatchApplied = true } = {}
{ commandLinePatchApplied = true, creationTimePatchApplied = true } = {}
) {
const processTreeDir = join(projectDir, 'node_modules', '@vscode', 'windows-process-tree')
const nodeAddonApiDir = join(processTreeDir, 'node_modules', 'node-addon-api')
mkdirSync(nodeAddonApiDir, { recursive: true })
writeFileSync(join(processTreeDir, 'package.json'), '{"dependencies":{"node-addon-api":"*"}}\n')
writeFileSync(join(processTreeDir, 'index.js'), 'module.exports = {}\n')
writeFileSync(
join(processTreeDir, 'index.js'),
creationTimePatchApplied
? 'exports.ProcessDataFlag = { None: 0, Memory: 1, CommandLine: 2, CreationTime: 4 }\n'
: 'exports.ProcessDataFlag = { None: 0, Memory: 1, CommandLine: 2 }\n'
)
mkdirSync(join(processTreeDir, 'src'), { recursive: true })
writeFileSync(
join(processTreeDir, 'src', 'process_commandline.cc'),
@@ -388,6 +393,36 @@ export function writeFakeWindowsProcessTreeWithNodeAddonApi(
? '// kProcessCommandLineInformation = 60\n'
: unpatchedWindowsProcessTreeCommandLineSource()
)
writeFileSync(
join(processTreeDir, 'src', 'process.h'),
creationTimePatchApplied
? 'enum ProcessDataFlags { NONE = 0, MEMORY = 1, COMMANDLINE = 2, CREATIONTIME = 4 };\nULONGLONG creationTimeMs;\n'
: 'enum ProcessDataFlags { NONE = 0, MEMORY = 1, COMMANDLINE = 2 };\n'
)
writeFileSync(
join(processTreeDir, 'src', 'process.cc'),
creationTimePatchApplied
? 'GetProcessCreationTime(pinfo);\nGetProcessTimes(hProcess, &creationTime, &exitTime, &kernelTime, &userTime);\n'
: 'GetProcessMemoryUsage(pinfo);\n'
)
writeFileSync(
join(processTreeDir, 'src', 'process_worker.cc'),
creationTimePatchApplied ? 'object.Set("creationTimeMs", process.creationTimeMs);\n' : '\n'
)
mkdirSync(join(processTreeDir, 'lib'), { recursive: true })
writeFileSync(
join(processTreeDir, 'lib', 'index.js'),
creationTimePatchApplied ? 'exports.ProcessDataFlag["CreationTime"] = 4;\n' : '\n'
)
writeFileSync(
join(processTreeDir, 'lib', 'index.ts'),
creationTimePatchApplied ? 'export enum ProcessDataFlag { CreationTime = 4 }\n' : '\n'
)
mkdirSync(join(processTreeDir, 'typings'), { recursive: true })
writeFileSync(
join(processTreeDir, 'typings', 'windows-process-tree.d.ts'),
creationTimePatchApplied ? 'creationTimeMs?: number\n' : '\n'
)
writeFileSync(join(nodeAddonApiDir, 'package.json'), '{"name":"node-addon-api"}\n')
writeFileSync(join(nodeAddonApiDir, 'napi.h'), '// napi.h\n')
writeFileSync(join(nodeAddonApiDir, 'napi-inl.h'), '// napi-inl.h\n')
@@ -33,6 +33,17 @@ export const WINDOWS_PROCESS_TREE_PATCH_PATH = join(
/** Only the patched reader defines this; the upstream one walks the PEB. */
const COMMAND_LINE_PATCH_MARKER = 'kProcessCommandLineInformation'
const CREATION_TIME_PATCH_MARKERS = [
['src/process.h', 'CREATIONTIME = 4'],
['src/process.h', 'ULONGLONG creationTimeMs'],
['src/process.cc', 'GetProcessCreationTime(pinfo)'],
['src/process.cc', 'GetProcessTimes(hProcess, &creationTime'],
['src/process_worker.cc', 'object.Set("creationTimeMs"'],
['lib/index.js', '["CreationTime"] = 4'],
['lib/index.ts', 'CreationTime = 4'],
['typings/windows-process-tree.d.ts', 'creationTimeMs?: number']
]
export const WINDOWS_PROCESS_TREE_NODE_ADDON_API_HEADERS = [
'napi.h',
'napi-inl.h',
@@ -83,6 +94,36 @@ export function inspectWindowsProcessTreeAddon(addonPath) {
return readFileSync(addonPath).includes(FLAGGED_IMPORT) ? 'unpatched' : 'clean'
}
export function assertWindowsProcessTreeCreationTimePatch(
packageDir = WINDOWS_PROCESS_TREE_PACKAGE_DIR
) {
for (const [relativePath, expected] of CREATION_TIME_PATCH_MARKERS) {
const filePath = join(packageDir, relativePath)
if (!existsSync(filePath)) {
throw new Error(
`${filePath} is missing, so the process creation-time patch cannot be verified. ` +
'Run pnpm install.'
)
}
if (!readFileSync(filePath, 'utf8').includes(expected)) {
throw new Error(
`${relativePath} does not contain the process creation-time patch (${expected}). ` +
'Run pnpm install.'
)
}
}
}
export function assertWindowsProcessTreeRuntimeCreationTime(windowsProcessTree) {
if (windowsProcessTree?.ProcessDataFlag?.CreationTime !== 4) {
throw new Error(
'@vscode/windows-process-tree does not expose ProcessDataFlag.CreationTime, so native ' +
'Windows structured agent-session process ownership cannot be PID-reuse safe. Rebuild it ' +
'(pnpm run rebuild:electron) rather than using the published prebuild.'
)
}
}
/**
* Refuse to compile or load the upstream command-line reader.
*
@@ -159,6 +200,7 @@ export function ensureWindowsProcessTreeCommandLinePatch(
rmSync(windowsProcessTreeAddonPath(packageDir), { force: true })
repaired = true
}
assertWindowsProcessTreeCreationTimePatch(packageDir)
return repaired
}
@@ -12,12 +12,15 @@ import { tmpdir } from 'node:os'
import { join, resolve } from 'node:path'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import {
assertWindowsProcessTreeCreationTimePatch,
assertWindowsProcessTreeRuntimeCreationTime,
inspectWindowsProcessTreeAddon,
nodeGypRebuildInvocation,
stageWindowsProcessTreeNodeAddonApiHeaders,
WINDOWS_PROCESS_TREE_NODE_ADDON_API_HEADERS,
WINDOWS_PROCESS_TREE_PACKAGE_DIR
} from './windows-process-tree-gyp-rebuild.mjs'
import { writeFakeWindowsProcessTreeWithNodeAddonApi } from './rebuild-native-deps-test-fixtures.mjs'
describe('windows-process-tree node-gyp rebuild', () => {
it("resolves node-addon-api's gyp target from the rebuild cwd", () => {
@@ -97,3 +100,47 @@ describe('inspecting a compiled windows-process-tree addon', () => {
expect(inspectWindowsProcessTreeAddon(staged)).toBe('unpatched')
})
})
describe('windows-process-tree CreationTime patch assertion', () => {
let dir
beforeEach(() => {
dir = mkdtempSync(join(tmpdir(), 'orca-windows-process-tree-creation-time-'))
})
afterEach(() => {
rmSync(dir, { recursive: true, force: true })
})
it('accepts a package whose source and JS surfaces expose process creation time', () => {
writeFakeWindowsProcessTreeWithNodeAddonApi(dir)
expect(() =>
assertWindowsProcessTreeCreationTimePatch(
join(dir, 'node_modules', '@vscode', 'windows-process-tree')
)
).not.toThrow()
})
it('rejects a package missing the process creation-time patch', () => {
writeFakeWindowsProcessTreeWithNodeAddonApi(dir, { creationTimePatchApplied: false })
expect(() =>
assertWindowsProcessTreeCreationTimePatch(
join(dir, 'node_modules', '@vscode', 'windows-process-tree')
)
).toThrow('process creation-time patch')
})
it('requires the runtime ProcessDataFlag.CreationTime enum', () => {
expect(() =>
assertWindowsProcessTreeRuntimeCreationTime({
ProcessDataFlag: { None: 0, Memory: 1, CommandLine: 2, CreationTime: 4 }
})
).not.toThrow()
expect(() =>
assertWindowsProcessTreeRuntimeCreationTime({
ProcessDataFlag: { None: 0, Memory: 1, CommandLine: 2 }
})
).toThrow('ProcessDataFlag.CreationTime')
})
})
+1
View File
@@ -32,6 +32,7 @@
"../src/main/codex/codex-app-server-capability-cache.ts",
"../src/main/codex/codex-app-server-capability-signal.ts",
"../src/main/codex/codex-app-server-client.ts",
"../src/main/codex/codex-app-server-process-tree-kill.ts",
"../src/main/codex/codex-app-server-record-reader.ts",
"../src/main/codex/codex-app-server-session.ts",
"../src/main/codex/codex-config-mirror.ts",
+21 -2
View File
@@ -2,8 +2,27 @@
Orca selects a Relay region in the Electron main process before requesting a new assignment. The
director publishes an allowlisted region catalog containing only HTTPS cell subdomains of that
director; Orca takes three bounded `/health` latency samples per region and caches the stable choice
for 24 hours. A cached region changes only when the alternative is materially faster.
director. Orca discards one warm-up `/health` request per probe origin — a cold request pays TCP and
TLS setup that can exceed the round trip it measures — then takes three bounded samples and compares
regions by their minimum. A wide spread still rejects a region, but only a genuinely flapping one.
The stable choice is cached for 24 hours, and a cached region changes only when the alternative is
materially faster.
A region wins only against a measured competitor. If any region in the catalog is rejected or cannot
be measured, Orca sends no hint rather than selecting the sole survivor. Sending no hint is not
neutral placement: the director assigns `preferredRegion ?? RELAY_DEFAULT_REGION`, and the default
is `us-central1`. So an `asia-east2` user whose `us-central1` probe fails or flaps once is placed in
`us-central1` for that refresh. That trade is accepted because the relay database is
`us-central1`-only, and it is bounded: the withheld hint is cached for one hour, not the 24 hours a
chosen region gets, so the next hour re-measures. An origin that fails its warm-up probe is dropped
before the sampling rounds, so an unreachable region costs one probe timeout rather than four.
After a control socket registers, Orca probes the cell it actually landed on, once per cell URL per
process. The cache is deleted only when it names a region other than the best measured one and the
assigned cell is more than three times slower than that region — a far cell under a cache that still
names the best region means the director declined the hint, and re-measuring would return the same
answer. Self-heal skips an absent, expired, or no-hint cache, and never runs under
`ORCA_RELAY_REGION_OVERRIDE`.
The assignment request sends only `preferredRegion`. It does not send latency, IP address, country,
pairing data, or credentials. Catalog, probe, and cache failures fall back to an assignment without
+31 -31
View File
@@ -31,12 +31,12 @@ administrators can do about it.
Four independent evidence clusters, from six incidents:
| Cluster | Incidents | Evidence |
| ----------------- | --------- | -------------------------------------------------------------------------------------------------------------------- |
| **Update** | A, B, C | `orca-windows-setup.exe` → `old-uninstaller.exe`, `Uninstall Orca.exe` (electron-builder generates these; they are in no repo file) |
| Cluster | Incidents | Evidence |
| ----------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------- |
| **Update** | A, B, C | `orca-windows-setup.exe` → `old-uninstaller.exe`, `Uninstall Orca.exe` (electron-builder generates these; they are in no repo file) |
| **Spawn** | all six | `Orca.exe` → `orca-terminal-daemon.exe` → `powershell.exe` / `pwsh.exe` / `cmd.exe` / `reg.exe` → `claude.exe`, `gh.exe`, `codex.cmd` |
| **Process table** | D | "suspicious memory activity" — `OpenProcess` plus a PEB read against every process on a repeating cadence |
| **Computer use** | E, F | `runtime.ps1`, `computer-sidecar.js`, many `operation.json`, a burst of ~10 short-lived `powershell.exe` |
| **Process table** | D | "suspicious memory activity" — `OpenProcess` plus a PEB read against every process on a repeating cadence |
| **Computer use** | E, F | `runtime.ps1`, `computer-sidecar.js`, many `operation.json`, a burst of ~10 short-lived `powershell.exe` |
Incident E is the one to look at hardest: 5 alerts, 37 evidence items, ATT&CK
**Execution + Collection**, and a description reading _"Screenshots were taken
@@ -143,11 +143,11 @@ PEB fallback to reinstate it — a hooked `ntdll` answering
`STATUS_INVALID_INFO_CLASS` for one target would have flipped a process-wide,
one-way switch back to `PROCESS_VM_READ` on exactly the machines this exists for.
Because the property is the *absence* of an import, it is checkable on the
Because the property is the _absence_ of an import, it is checkable on the
artifact rather than the source: `inspectWindowsProcessTreeAddon()` answers
`clean` / `unpatched` / `missing`, and the rebuild, `ensure-native-runtime.mjs`,
the relay build and `loadWindowsProcessTree()` all key on it. That check is load-
bearing because the published tarball ships a *loadable* prebuilt built from
bearing because the published tarball ships a _loadable_ prebuilt built from
unpatched source, so "it required cleanly" is not evidence.
What to declare to administrators is now one
@@ -180,7 +180,7 @@ Three sites are named in the incident analysis:
`src/shared/setup-agent-sequencing.ts`,
`src/shared/windows-cmd-runner-delayed-launch.ts` and
`src/shared/windows-interactive-login-spawn.ts` each dropped
`-ExecutionPolicy Bypass` as a measured no-op: the policy gates script *files*,
`-ExecutionPolicy Bypass` as a measured no-op: the policy gates script _files_,
never `-EncodedCommand`. Where the bypass was load-bearing it moved in-payload as
a process-scope `Set-ExecutionPolicy` (`setup-agent-sequencing.ts`), which is the
pattern to copy rather than restoring the switch — the switch loses to a GPO
@@ -192,7 +192,7 @@ What remains is `-EncodedCommand` without the bypass: the PTY bootstraps
(`src/main/agent-hooks/windows-powershell-hook-launcher.ts` and its callers
`src/main/agent-hooks/runtime-home-hook-command.ts`,
`src/main/agent-hooks/installer-utils.ts`, and `src/main/claude/hook-settings.ts`
— that last one only as a *fallback* since #18875, see below),
— that last one only as a _fallback_ since #18875, see below),
`src/main/runtime/windows-default-route-interfaces.ts`,
`src/main/runtime/orchestration/setup-completion-signal.ts`,
`src/shared/hermes-startup-query.ts`, and the four ex-bypass sites above.
@@ -224,7 +224,7 @@ denies the analyser the payload it would otherwise clear.
The hook launcher is prior art worth knowing about. #16003 measured, on a
reporting Kaspersky host, that `-WindowStyle Hidden` paired with
`-EncodedCommand` was denied at `CreateProcess` with exit 126 regardless of
payload — `exit 0` was denied too. The fix was to stop *spelling* the flags:
payload — `exit 0` was denied too. The fix was to stop _spelling_ the flags:
`WINDOWS_POWERSHELL_HOOK_SWITCHES` is now just `-NoProfile`, and separately, in
#16576, the execution policy bypass moved in-payload as a process-scope
`Set-ExecutionPolicy` — a real command-line signal reduction, though #16003's
@@ -247,7 +247,7 @@ a quoted token, each `%` is broken with `"^%"`.
The escaping is not decorative. Measured on Windows 11 against a real `.cmd`
shim, `["a b", 'c"d', "e%F%g", "h&i", "j^k"]` came back as `["a b", 'c"d',
"e^%F^%g", "h"]` — the `&` truncated the argument *and* ran the remainder as a
"e^%F^%g", "h"]` — the `&` truncated the argument _and_ ran the remainder as a
command.
**How an EDR reads it:** caret escaping is the canonical obfuscation marker in
@@ -259,7 +259,7 @@ obfuscated-command-line detector is tuned on.
`Orca.exe` → the relocated daemon host (`orca-terminal-daemon.exe` in the builds
these incidents cover, `Orca.exe` since) → a shell → an agent CLI is what a
terminal multiplexer for coding agents *is*. `reg.exe` appears from
terminal multiplexer for coding agents _is_. `reg.exe` appears from
`src/main/win32-utils.ts`,
`src/main/agent-hooks/managed-hook-owner-identity.ts` and
`src/relay/pty-shell-utils.ts` (reading the OpenSSH `DefaultShell`).
@@ -267,7 +267,7 @@ terminal multiplexer for coding agents *is*. `reg.exe` appears from
Nothing here is avoidable in principle. What is controllable is depth and
breadth: every interpreter hop between Orca and the thing the user asked for adds
a scored edge, which is why the shipped doctrine of #15520 and #15595 is to
*shorten the interpreter chain* rather than to hide a window.
_shorten the interpreter chain_ rather than to hide a window.
#18875 is a worked example of that doctrine. The Claude Code lifecycle hook was
registered as `powershell.exe -NoProfile -EncodedCommand <...>` whose entire
@@ -295,7 +295,7 @@ That last clause is the standing assumption of this change, and it is worth
stating plainly because it is **not** measured. `||` parses in Git Bash, cmd.exe
and pwsh, but not in Windows PowerShell 5.1, so the direct shape is correct for
any host that is one of the first three. Claude Code itself is a Git Bash host on
native Windows. What no one here has verified is which host a *compat consumer*
native Windows. What no one here has verified is which host a _compat consumer_
uses: cursor-agent and Devin import `~/.claude/settings.json` and run `command`
through their own launcher (the managed `.cmd` carries a `DEVIN_PROJECT_DIR` skip
for exactly that). If one of them spawns hook strings through Windows PowerShell
@@ -318,12 +318,12 @@ then captures the screen through `Graphics.CopyFromScreen`.
That is four separate high-signal behaviours stacked in one process:
| Behaviour | How it is scored |
| ----------------------------------------------- | ---------------------------------------------------- |
| `Graphics.CopyFromScreen` | **MITRE T1113**, screen capture — Collection tactic |
| `SendInput` synthetic keyboard/mouse | input synthesis against other applications |
| `Add-Type -TypeDefinition` on every operation | MSIL compiled at runtime; incident F's "suspicious MSIL code" |
| One `powershell.exe` per operation | a burst of short-lived interpreters under one parent |
| Behaviour | How it is scored |
| --------------------------------------------- | ------------------------------------------------------------- |
| `Graphics.CopyFromScreen` | **MITRE T1113**, screen capture — Collection tactic |
| `SendInput` synthetic keyboard/mouse | input synthesis against other applications |
| `Add-Type -TypeDefinition` on every operation | MSIL compiled at runtime; incident F's "suspicious MSIL code" |
| One `powershell.exe` per operation | a burst of short-lived interpreters under one parent |
The bottom two rows are the two the incident text named directly, and they are
also the two a persistent runtime host would remove: a long-lived helper compiles
@@ -381,16 +381,16 @@ changed. Check the code before relying on it.
The checklist. On Windows, do not reach for:
| Don't | Instead |
| ----------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- |
| `-ExecutionPolicy Bypass` on the command line | Set the policy in-payload at process scope, as `windows-powershell-hook-launcher.ts` does, or do not run a `.ps1` at all |
| `-EncodedCommand` | A temp `.ps1` with an argument, or no PowerShell hop: prefer a native API or an existing Node path |
| `cmd.exe /c` carrying escaped free text | Spawn the real target directly. `cmd.exe` is only unavoidable for `.cmd`/`.bat`; keep free text out of the line where you can |
| Forking `powershell.exe` to read system state | The native reader — [`windows-process-enumeration.md`](./windows-process-enumeration.md) is the standing rule for the process table |
| A process per operation in a loop | One long-lived helper with a request channel. A burst of short-lived interpreters under one parent is itself the signal |
| `Add-Type -TypeDefinition` at runtime | A precompiled, signed assembly, or a native helper |
| Copying our own image under a different name | Copy it verbatim — [`windows-daemon-host-relocation.md`](./windows-daemon-host-relocation.md) (done for the daemon host) |
| Deriving a script runner from a UI preference | [`windows-setup-shell.md`](./windows-setup-shell.md) — the script declares its own interpreter |
| Don't | Instead |
| --------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- |
| `-ExecutionPolicy Bypass` on the command line | Set the policy in-payload at process scope, as `windows-powershell-hook-launcher.ts` does, or do not run a `.ps1` at all |
| `-EncodedCommand` | A temp `.ps1` with an argument, or no PowerShell hop: prefer a native API or an existing Node path |
| `cmd.exe /c` carrying escaped free text | Spawn the real target directly. `cmd.exe` is only unavoidable for `.cmd`/`.bat`; keep free text out of the line where you can |
| Forking `powershell.exe` to read system state | The native reader — [`windows-process-enumeration.md`](./windows-process-enumeration.md) is the standing rule for the process table |
| A process per operation in a loop | One long-lived helper with a request channel. A burst of short-lived interpreters under one parent is itself the signal |
| `Add-Type -TypeDefinition` at runtime | A precompiled, signed assembly, or a native helper |
| Copying our own image under a different name | Copy it verbatim — [`windows-daemon-host-relocation.md`](./windows-daemon-host-relocation.md) (done for the daemon host) |
| Deriving a script runner from a UI preference | [`windows-setup-shell.md`](./windows-setup-shell.md) — the script declares its own interpreter |
Two framing rules that outlast the table:
@@ -407,7 +407,7 @@ Two framing rules that outlast the table:
This is the single most important operational point, and it is the one most
commonly got wrong. The six incidents are **MDE EDR behavioural alerts**.
Defender Antivirus path exclusions suppress *scan* detections; they do not
Defender Antivirus path exclusions suppress _scan_ detections; they do not
suppress EDR behavioural alerts the same way. Adding
`%LOCALAPPDATA%\Programs\orca\` to the AV exclusion list and expecting the
incidents to stop will not work.
+17 -17
View File
@@ -64,10 +64,10 @@ identity scan opens nothing.
So the module exposes two snapshots, and the row types differ so a cheap caller
cannot read what its flag set did not pay for:
| reader | row type | flags | per-process handles |
| ------------------------------------------ | ---------------------------- | --------------------------- | ------------------- |
| `readWindowsProcessIdentityTable[Fresh]()` | `WindowsProcessIdentityRow` | `None \| CreationTime` | none |
| `readWindowsProcessTable[Fresh]()` | `WindowsProcessRow` | `+ CommandLine` | one `OpenProcess` |
| reader | row type | flags | per-process handles |
| ------------------------------------------ | --------------------------- | ---------------------- | ------------------- |
| `readWindowsProcessIdentityTable[Fresh]()` | `WindowsProcessIdentityRow` | `None \| CreationTime` | none |
| `readWindowsProcessTable[Fresh]()` | `WindowsProcessRow` | `+ CommandLine` | one `OpenProcess` |
`Memory` is requested by neither. Nothing reads a working set off this table —
`windows-process-resource-collector.ts` runs its own sweep because it needs
@@ -103,7 +103,7 @@ only under concurrency.
Nothing else in this module prevents that. Each snapshot cache single-flights
only within itself (`inFlight` is a closure per reader), and the wedge set
latches only *after* a read misses its 3 s deadline, so through the healthy
latches only _after_ a read misses its 3 s deadline, so through the healthy
~12 ms of a scan neither excludes the other. Overlap is the normal state rather
than an edge case: other panes keep polling detailed at 750 ms while a teardown
takes identity snapshots, and `codex-structured-turn-processes.ts` issues fresh
@@ -166,15 +166,15 @@ through `toIdentityRow`, so an identity row carries no command line on any host.
### Which callers need which
| caller | reads | flag set |
| --------------------------------------------- | ------------------ | -------- |
| `windows-agent-foreground-process.ts` | `command` (agent recognition) | detailed |
| `local-workspace-platform-port-scanner.ts` | `command` (port attribution) | detailed |
| `codex-structured-turn-processes.ts` | `command` (turn-process identity) | detailed |
| `structured-tui-process-identity.ts` | `command` (child match) | detailed |
| `windows-pty-root-identity.ts` | `pid` / `ppid` only | identity |
| `agent-session-process-identity-probe.ts` | `creationTimeMs` only | identity |
| `relay/windows-port-scan.ts` | `name` (port owner label) | detailed |
| caller | reads | flag set |
| ------------------------------------------ | --------------------------------- | -------- |
| `windows-agent-foreground-process.ts` | `command` (agent recognition) | detailed |
| `local-workspace-platform-port-scanner.ts` | `command` (port attribution) | detailed |
| `codex-structured-turn-processes.ts` | `command` (turn-process identity) | detailed |
| `structured-tui-process-identity.ts` | `command` (child match) | detailed |
| `windows-pty-root-identity.ts` | `pid` / `ppid` only | identity |
| `agent-session-process-identity-probe.ts` | `creationTimeMs` only | identity |
| `relay/windows-port-scan.ts` | `name` (port owner label) | detailed |
`windows-port-scan.ts` is the one mismatch in the table: it reads only `pid` and
`name`, which the identity set answers, but it calls the detailed reader. On a
@@ -344,7 +344,7 @@ on any other OS keeps using the scan.
## Why the package is patched
`config/patches/@vscode__windows-process-tree@0.8.0.patch` carries five changes.
`config/patches/@vscode__windows-process-tree@0.8.0.patch` carries six changes.
1. **Spectre mitigation.** The upstream `binding.gyp` requires Spectre-mitigated
libraries, which Orca's Windows build agents do not install. `node-pty` is
@@ -368,10 +368,10 @@ on any other OS keeps using the scan.
to Unix ms; a process that denies the handle is emitted with the field
absent, never zero, because callers must be able to tell "cannot identify"
from a timestamp.
5. **`supportedProcessDataFlags`.** `addon.cc` exports the flag bits the
6. **`supportedProcessDataFlags`.** `addon.cc` exports the flag bits the
compiled binary understands, and `lib/index.js` re-exports it.
Why a fifth hunk and not just the enum: unlike `node-pty`, this package
Why a separate hunk and not just the enum: unlike `node-pty`, this package
publishes a prebuilt `.node` at the same `build/Release/` path node-gyp
writes to. pnpm patches the source tree and leaves that prebuilt alone, so a
host can hold a patched `lib/index.js` — `ProcessDataFlag.CreationTime` and
+2 -3
View File
@@ -30,8 +30,7 @@ import { Callout } from '@/components/docs/prose'
[installer](https://github.com/stablyai/orca/releases/latest/download/orca-windows-setup.exe)
</li>
<li>
**Linux:**
AppImage
**Linux:** AppImage
[x64](https://github.com/stablyai/orca/releases/latest/download/orca-linux.AppImage) ·
[arm64](https://github.com/stablyai/orca/releases/latest/download/orca-linux-arm64.AppImage) ·
[.deb](https://github.com/stablyai/orca/releases) ·
@@ -131,7 +130,7 @@ On Linux the [Orca CLI](/docs/cli/reference) installs as **`orca-ide`**, not `or
- The `.deb` and `.rpm` put `orca-ide` on your `PATH` at install time, as `/usr/bin/orca-ide`.
- With the AppImage, register the CLI from [Settings → General → Orca CLI](/docs/settings). That installs `~/.local/bin/orca-ide`.
- Inside Orca's own terminals, bare `orca` works. Orca puts a shim on the `PATH` of the terminals it manages, so agents and scripts running there use the same command as on macOS and Windows.
- On a headless host, a packaged `orca serve` writes a bare `orca` into `~/.local/bin` as it starts, unless a file it does not own already holds that name. It writes that *during* startup, so it is never what starts the server — the first launch is always [`orca-ide serve`](/docs/remote-servers).
- On a headless host, a packaged `orca serve` writes a bare `orca` into `~/.local/bin` as it starts, unless a file it does not own already holds that name. It writes that _during_ startup, so it is never what starts the server — the first launch is always [`orca-ide serve`](/docs/remote-servers).
Do not verify with `command -v orca`: on a GNOME desktop that succeeds and resolves to the screen reader. Use `orca-ide` in your own shell and `orca` inside Orca. If you want the short name everywhere and you do not use the screen reader, link it yourself:
+11 -3
View File
@@ -129,19 +129,23 @@ Install Orca and its bundled CLI on the server, then run:
<Callout title="On Linux, start it with orca-ide serve">
The Linux CLI is named `orca-ide`, because GNOME Orca's screen reader already owns
`/usr/bin/orca`. A packaged `orca serve` does write a bare `orca` into `~/.local/bin`, but only
while it is starting, so that shim can never be the command that starts the server. Read
`orca serve` as `orca-ide serve` throughout this page when the host is Linux. See
[Install → Linux](/docs/install#linux).
while it is starting, so that shim can never be the command that starts the server. Read `orca
serve` as `orca-ide serve` throughout this page when the host is Linux. See [Install →
Linux](/docs/install#linux).
</Callout>
```bash
orca serve --pairing-address <server-tailscale-ip-or-hostname>
# Linux
orca-ide serve --pairing-address <server-tailscale-ip-or-hostname>
```
For example:
```bash
orca serve --pairing-address 100.64.1.20
# Linux
orca-ide serve --pairing-address 100.64.1.20
```
The command:
@@ -157,6 +161,8 @@ Add `--port 6768` when a firewall, tunnel, or service definition requires a fixe
```bash
orca serve --port 6768 --pairing-address 100.64.1.20
# Linux
orca-ide serve --port 6768 --pairing-address 100.64.1.20
```
Use only one host mode at a time. If the Orca desktop app is already sharing that computer, do not start a second `orca serve` process for the same setup.
@@ -167,6 +173,8 @@ For the Orca mobile app, request a mobile-scoped QR code and link:
```bash
orca serve --pairing-address 100.64.1.20 --mobile-pairing
# Linux
orca-ide serve --pairing-address 100.64.1.20 --mobile-pairing
```
Keep the phone on the same tailnet, open Orca Mobile, choose **Pair**, and scan the terminal QR code or paste the printed link.
+6 -6
View File
@@ -213,9 +213,9 @@ The commands, snapshot and ref rules, page affinity, and `browser_*` recoveries
This guide covers worktrees, terminals, and handoffs on its own. At a gate below, run `ORCA skills get orca-cli --reference references/<file>.md` and read only that document; `--references` lists the names. If the CLI rejects `--reference`, run `ORCA skills get orca-cli --full` once instead: it returns this guide plus every reference from the same CLI build, so read only the named one. If `--full` is rejected too, the CLI predates bundled references: use `ORCA <command> --help`, keep the rules above, and do not guess flags.
| Action gate | Reference |
|---|---|
| Driving Orca's embedded browser: navigation, snapshots, refs, tabs, concurrent pages, or `browser_*` recoveries | `references/browser.md` |
| Creating, editing, running, or inspecting scheduled automations | `references/automations.md` |
| Publishing or revoking an artifact link, or publishing installed skills | `references/publishing.md` |
| Mobile emulator taps, gestures, typing, buttons, camera, or permissions | invoke the `orca-emulator` skill |
| Action gate | Reference |
| --------------------------------------------------------------------------------------------------------------- | -------------------------------- |
| Driving Orca's embedded browser: navigation, snapshots, refs, tabs, concurrent pages, or `browser_*` recoveries | `references/browser.md` |
| Creating, editing, running, or inspecting scheduled automations | `references/automations.md` |
| Publishing or revoking an artifact link, or publishing installed skills | `references/publishing.md` |
| Mobile emulator taps, gestures, typing, buttons, camera, or permissions | invoke the `orca-emulator` skill |
+17 -17
View File
@@ -52,23 +52,23 @@ Orca returns a clear message when the SDK is missing
Use `--json` for agent-driven calls. Unqualified commands target the worktree's active
device.
| Goal | Command | Constraint |
| ------------------- | --------------------------------------------------------------------- | ------------------------------------------------------------------------------- |
| List devices + AVDs | `ORCA emulator devices --json` | Every backend's devices with a platform column, booted and shutdown. |
| Attach / make active | `ORCA emulator attach <avd-name-or-serial> --json` | Given an AVD name, boots it first. Makes the device active for the worktree. |
| Single tap | `ORCA emulator tap <x> <y> --json` | Normalized 0..1 coordinates. |
| Swipe / gesture | `ORCA emulator gesture '<json>' --json` | adb approximates the path by its endpoints, first point to last. |
| Type text | `ORCA emulator type "user@example.com" --json` | US-ASCII, spaces handled, no newlines. |
| Hardware button | `ORCA emulator button back --json` | `home`, `back`, `recents`, `power`, `volume_up`, `volume_down`. |
| Rotate | `ORCA emulator rotate landscape_left --json` | Sets `user_rotation` and disables auto-rotate. |
| Install an APK | `ORCA emulator install ./app-debug.apk --reinstall --json` | `--reinstall` passes `-r`. |
| Launch an app | `ORCA emulator launch com.acme.app --activity .MainActivity --json` | Omit `--activity` to launch the default LAUNCHER activity. |
| Runtime permission | `ORCA emulator permissions grant com.acme.app android.permission.CAMERA --json` | Positional order is `<grant\|revoke> <package> <permission>`; `reset` takes no positionals and clears all runtime grants. |
| Accessibility tree | `ORCA emulator ax --json` | `uiautomator dump` parsed to a node tree. |
| Logcat (one-shot) | `ORCA emulator logcat --lines 200 --json` | Dumps recent lines, parsed to entries. |
| Raw adb shell | `ORCA emulator exec --command "getprop ro.build.version.sdk" --json` | Runs `adb -s <serial> shell <command>`. |
| Stop the helper | `ORCA emulator kill --json` | Leaves the device booted. |
| Stop and power off | `ORCA emulator shutdown --json` | Stops the helper and shuts the device down. |
| Goal | Command | Constraint |
| -------------------- | ------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- |
| List devices + AVDs | `ORCA emulator devices --json` | Every backend's devices with a platform column, booted and shutdown. |
| Attach / make active | `ORCA emulator attach <avd-name-or-serial> --json` | Given an AVD name, boots it first. Makes the device active for the worktree. |
| Single tap | `ORCA emulator tap <x> <y> --json` | Normalized 0..1 coordinates. |
| Swipe / gesture | `ORCA emulator gesture '<json>' --json` | adb approximates the path by its endpoints, first point to last. |
| Type text | `ORCA emulator type "user@example.com" --json` | US-ASCII, spaces handled, no newlines. |
| Hardware button | `ORCA emulator button back --json` | `home`, `back`, `recents`, `power`, `volume_up`, `volume_down`. |
| Rotate | `ORCA emulator rotate landscape_left --json` | Sets `user_rotation` and disables auto-rotate. |
| Install an APK | `ORCA emulator install ./app-debug.apk --reinstall --json` | `--reinstall` passes `-r`. |
| Launch an app | `ORCA emulator launch com.acme.app --activity .MainActivity --json` | Omit `--activity` to launch the default LAUNCHER activity. |
| Runtime permission | `ORCA emulator permissions grant com.acme.app android.permission.CAMERA --json` | Positional order is `<grant\|revoke> <package> <permission>`; `reset` takes no positionals and clears all runtime grants. |
| Accessibility tree | `ORCA emulator ax --json` | `uiautomator dump` parsed to a node tree. |
| Logcat (one-shot) | `ORCA emulator logcat --lines 200 --json` | Dumps recent lines, parsed to entries. |
| Raw adb shell | `ORCA emulator exec --command "getprop ro.build.version.sdk" --json` | Runs `adb -s <serial> shell <command>`. |
| Stop the helper | `ORCA emulator kill --json` | Leaves the device booted. |
| Stop and power off | `ORCA emulator shutdown --json` | Stops the helper and shuts the device down. |
## Targeting
+15 -15
View File
@@ -43,20 +43,20 @@ Orca reports a clear error when the host is missing macOS or the Xcode tools.
Use `--json` for agent-driven calls. Unqualified commands target the worktree's active
device.
| Goal | Command | Constraint |
| ------------------------ | ----------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ |
| List available / running | `ORCA emulator list --json` | Orca-managed sessions plus raw serve-sim streams. Use its ids for `--device` / `--emulator`. |
| List devices everywhere | `ORCA emulator devices --json` | Every backend's devices with a platform column, booted and shutdown. |
| Attach / make active | `ORCA emulator attach "iPhone 16 Pro" --json` | Starts the helper if needed and makes the device active for the worktree. `--focus` switches the UI; it does not by default. |
| Single tap | `ORCA emulator tap <x> <y> --json` | Normalized 0..1 coordinates. |
| Multi-step gesture | `ORCA emulator gesture '<json>' --json` | Begin/move/end points. Use `tap` for a single tap. |
| Type text | `ORCA emulator type "text" --json` | US-ASCII only. |
| Goal | Command | Constraint |
| ------------------------ | ----------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| List available / running | `ORCA emulator list --json` | Orca-managed sessions plus raw serve-sim streams. Use its ids for `--device` / `--emulator`. |
| List devices everywhere | `ORCA emulator devices --json` | Every backend's devices with a platform column, booted and shutdown. |
| Attach / make active | `ORCA emulator attach "iPhone 16 Pro" --json` | Starts the helper if needed and makes the device active for the worktree. `--focus` switches the UI; it does not by default. |
| Single tap | `ORCA emulator tap <x> <y> --json` | Normalized 0..1 coordinates. |
| Multi-step gesture | `ORCA emulator gesture '<json>' --json` | Begin/move/end points. Use `tap` for a single tap. |
| Type text | `ORCA emulator type "text" --json` | US-ASCII only. |
| Hardware button | `ORCA emulator button home --json` | `home` and `side_button` are documented by the CLI spec; other names such as `swipe_home`, `app_switcher`, `lock`, and `siri` are forwarded to serve-sim unvalidated. |
| Rotate device | `ORCA emulator rotate landscape_left --json` | The orientation persists for subsequent gestures. |
| Accessibility tree | `ORCA emulator ax --json` | serve-sim node tree, capped at 500 nodes, frames normalized 0..1 with a top-left origin. Needs an active session. |
| Raw passthrough | `ORCA emulator exec --command "ca-debug blended on" --json` | serve-sim subcommand string, without a `serve-sim` prefix. |
| Stop the helper | `ORCA emulator kill --json` | Leaves the device booted. |
| Stop and power off | `ORCA emulator shutdown --json` | Stops the helper and shuts the simulator device down. |
| Rotate device | `ORCA emulator rotate landscape_left --json` | The orientation persists for subsequent gestures. |
| Accessibility tree | `ORCA emulator ax --json` | serve-sim node tree, capped at 500 nodes, frames normalized 0..1 with a top-left origin. Needs an active session. |
| Raw passthrough | `ORCA emulator exec --command "ca-debug blended on" --json` | serve-sim subcommand string, without a `serve-sim` prefix. |
| Stop the helper | `ORCA emulator kill --json` | Leaves the device booted. |
| Stop and power off | `ORCA emulator shutdown --json` | Stops the helper and shuts the simulator device down. |
## Targeting
@@ -65,8 +65,8 @@ commands target it. Pass a selector only to override that or reach a second devi
active session an unqualified command fails with `emulator_no_active`; attach or open the pane
and retry.
- `--device "iPhone 16 Pro"` or `--device <udid>`, from `list` or `devices`. `--emulator
<id>` is an alternative spelling: the bridge resolves both through the same lookup. These
- `--device "iPhone 16 Pro"` or `--device <udid>`, from `list` or `devices`.
`--emulator <id>` is an alternative spelling: the bridge resolves both through the same lookup. These
selectors apply to the action verbs; `list` and `devices` take only `--worktree`, and
`attach` names its device as a positional argument.
- `--worktree id:<fullWorktreeId>` or `--worktree active`. The full id is the exact
+7 -7
View File
@@ -367,10 +367,10 @@ rejects `--reference`, run `ORCA skills get orca-per-workspace-env --full` once
this guide plus every reference from the same CLI build, so read only the named one. If `--full` is
rejected too, keep these rules, use the command's `--help`, and do not guess flags.
| Action gate | Bundled reference |
| --- | --- |
| Writing the base-snapshot, auth, or create script for a snapshot-capable cloud provider | `references/provider-vercel.md` |
| The recipe connects over SSH instead of starting `orca serve`, including provisioned root | `references/ssh-host.md` |
| The environment is a local Docker container reached over SSH | `references/docker-ssh.md` |
| The user's desktop is Windows and you are scaffolding local-side scripts | `references/windows-scripts.md` |
| A doctor, provision, clone, login, or snapshot step failed | `references/failure-modes.md` |
| Action gate | Bundled reference |
| ----------------------------------------------------------------------------------------- | ------------------------------- |
| Writing the base-snapshot, auth, or create script for a snapshot-capable cloud provider | `references/provider-vercel.md` |
| The recipe connects over SSH instead of starting `orca serve`, including provisioned root | `references/ssh-host.md` |
| The environment is a local Docker container reached over SSH | `references/docker-ssh.md` |
| The user's desktop is Windows and you are scaffolding local-side scripts | `references/windows-scripts.md` |
| A doctor, provision, clone, login, or snapshot step failed | `references/failure-modes.md` |
File diff suppressed because one or more lines are too long
@@ -1,5 +1,11 @@
import { describe, expect, it } from 'vitest'
import type { OrchestrationFleetWorker } from '../../../shared/orchestration-fleet-projection'
import { subagentGroupFallbackText } from '../../../shared/native-chat-subagent-summary'
import type {
NativeChatBlock,
NativeChatMessage,
NativeChatSubagentEntry
} from '../../../shared/native-chat-types'
import type { OrchestrationWorkerReadResult } from '../../../shared/orchestration-worker-output'
import { formatWorkerRead, formatWorkerStart } from './worker-output'
@@ -288,3 +294,207 @@ function workerReadResult(
type WorkerReadResultWithoutContext<T> = T extends unknown
? Omit<T, 'dispatchId' | 'status'>
: never
function transcriptRead(
blocks: NativeChatBlock[],
role: NativeChatMessage['role'] = 'assistant'
): OrchestrationWorkerReadResult {
const message: NativeChatMessage = {
id: 'm1',
role,
blocks,
timestamp: 1,
source: 'transcript'
}
return {
dispatchId: 'd1',
source: 'transcript',
sourceIdentity: 'pane:1',
provider: 'codex',
transcript: { messages: [message], nextCursor: '1', limited: false, returnedMessageCount: 1 },
cursor: '1',
status: { worker: 'running', terminal: 'running' },
fallbackReason: null,
warnings: []
}
}
const ROSTER: readonly NativeChatSubagentEntry[] = [
{ id: 'child-1', label: 'read', state: 'working' },
{ id: 'child-2', label: 'edit', state: 'failed' }
]
function occurrences(haystack: string, needle: string): number {
return haystack.split(needle).length - 1
}
describe('formatWorkerRead', () => {
// The replay case this row is durable for: SQLite-backed, re-sent on every
// reconnect, and read here by a client that draws no roster block, runs no
// reconciliation, and cannot re-check whether those children still exist. A
// sentence frozen mid-flight outlives the process that wrote it, so it must
// not keep asserting a liveness only that process could have observed —
// `docs/reference/ssh-execution-boundary.md` calls that loss of contact
// reported as a live state.
it('replays a mid-flight roster row without claiming a child is still working', () => {
const midFlight: readonly NativeChatSubagentEntry[] = [
{ id: 'child-1', label: 'read', state: 'working' },
{ id: 'child-2', label: 'search', state: 'working' },
{ id: 'child-3', label: 'edit', state: 'failed' }
]
const output = formatWorkerRead(
transcriptRead([
{ type: 'text', text: subagentGroupFallbackText(midFlight) },
{ type: 'subagent-group', groupId: 'thread:turn-1', agents: [...midFlight] }
])
)
expect(output).toContain('[assistant] Kicked off 3 subagents (1 failed)')
expect(output).not.toMatch(/\bworking\b/)
})
// The body `codexSubagentGroupBody` actually writes: the plain-text twin, then
// the block it stands in for. The twin exists for clients that cannot draw the
// block, so a client printing the block must not print the twin beside it —
// the renderer drops the twin for the same reason, from the other side.
it('prints the roster sentence once for the two-block row the producer writes', () => {
const sentence = subagentGroupFallbackText(ROSTER)
const output = formatWorkerRead(
transcriptRead(
[
{ type: 'text', text: sentence },
{ type: 'subagent-group', groupId: 'thread:turn-1', agents: [...ROSTER] }
],
'system'
)
)
expect(output).toContain(`[system] ${sentence}`)
expect(occurrences(output, sentence)).toBe(1)
})
// Suppression is per twin, not per message. One twin beside two roster blocks
// silenced BOTH groups and printed one sentence, so the second roster vanished
// with no marker — the same silent drop the missing-twin case above avoids.
it('stands in for the second roster block when only one twin accompanies two', () => {
const other: readonly NativeChatSubagentEntry[] = [
{ id: 'child-3', label: 'plan', state: 'completed' }
]
const output = formatWorkerRead(
transcriptRead(
[
{ type: 'text', text: subagentGroupFallbackText(ROSTER) },
{ type: 'subagent-group', groupId: 'thread:turn-1', agents: [...ROSTER] },
{ type: 'subagent-group', groupId: 'thread:turn-2', agents: [...other] }
],
'system'
)
)
expect(occurrences(output, subagentGroupFallbackText(ROSTER))).toBe(1)
expect(output).toContain(`[subagents] ${subagentGroupFallbackText(other)}`)
})
// Which group a lone twin belongs to is decided by its TEXT, not its position.
// Claiming positionally silenced whichever group came first, so a twin
// belonging to a LATER group erased the earlier group's roster and printed the
// later one's sentence twice — the same silent drop, one permutation over.
it('claims a lone twin for the group it names, not the first group in the message', () => {
const other: readonly NativeChatSubagentEntry[] = [
{ id: 'child-3', label: 'plan', state: 'completed' }
]
const second = subagentGroupFallbackText(other)
const output = formatWorkerRead(
transcriptRead(
[
{ type: 'text', text: second },
{ type: 'subagent-group', groupId: 'thread:turn-1', agents: [...ROSTER] },
{ type: 'subagent-group', groupId: 'thread:turn-2', agents: [...other] }
],
'system'
)
)
expect(occurrences(output, second)).toBe(1)
expect(output).toContain(`[subagents] ${subagentGroupFallbackText(ROSTER)}`)
})
// The same claim, with the twin written after both blocks: nothing about the
// ORDER of a twin and its group is guaranteed by the block schema.
it('claims a trailing twin for the group it names', () => {
const other: readonly NativeChatSubagentEntry[] = [
{ id: 'child-3', label: 'plan', state: 'completed' }
]
const second = subagentGroupFallbackText(other)
const output = formatWorkerRead(
transcriptRead(
[
{ type: 'subagent-group', groupId: 'thread:turn-1', agents: [...ROSTER] },
{ type: 'subagent-group', groupId: 'thread:turn-2', agents: [...other] },
{ type: 'text', text: second }
],
'system'
)
)
expect(occurrences(output, second)).toBe(1)
expect(output).toContain(`[subagents] ${subagentGroupFallbackText(ROSTER)}`)
})
// A group with no twin beside it is a shape the block schema admits and no
// producer writes. Dropping it would lose the roster entirely, so the block
// itself carries the sentence when nothing else does.
it('stands in for a roster block that arrived without its twin', () => {
const output = formatWorkerRead(
transcriptRead([{ type: 'subagent-group', groupId: 'thread:turn-1', agents: [...ROSTER] }])
)
expect(output).toContain(`[assistant] [subagents] ${subagentGroupFallbackText(ROSTER)}`)
})
// A roster from a newer build holds a state this build does not know, which
// `summarizeSubagentGroup` reads as `unverifiable`. Recomputing the sentence
// to compare it against the frozen twin therefore produced a DIFFERENT string,
// and the CLI printed the roster twice: the twin's own wording plus a
// `[subagents]` line contradicting it.
it('prints the roster once when the twin names a state this build cannot reproduce', () => {
const frozenTwin = 'Ran 2 subagents (1 cancelled)'
const output = formatWorkerRead(
transcriptRead(
[
{ type: 'text', text: frozenTwin },
{
type: 'subagent-group',
groupId: 'thread:turn-1',
agents: [
{ id: 'child-1', label: 'read', state: 'completed' },
{ id: 'child-2', label: 'edit', state: 'cancelled' }
] as unknown as NativeChatSubagentEntry[]
}
],
'system'
)
)
expect(output).toContain(`[system] ${frozenTwin}`)
expect(output).not.toContain('[subagents]')
expect(output).not.toContain('unverifiable')
})
// The journal admits block types this build does not know, and `client.call`
// casts the RPC result rather than validating it — so a newer remote host's
// block reaches this formatter as-is. Reading fields off it threw a TypeError
// and took down the whole `worker read`.
it('degrades an unknown block type from a newer host instead of throwing', () => {
const output = formatWorkerRead(
transcriptRead([
{ type: 'text', text: 'before' },
{ type: 'plan-step', title: 'ship it' } as unknown as NativeChatBlock,
{ type: 'text', text: 'after' }
])
)
expect(output).toContain('[assistant] before\n[unsupported block]\nafter')
})
})
@@ -38,6 +38,7 @@ export type ClaudeStructuredSdkOptions = Pick<
| 'sessionId'
| 'resume'
| 'resumeSessionAt'
| 'resumeDropsTurn'
>
/**
@@ -0,0 +1,207 @@
import { describe, expect, it, vi } from 'vitest'
import {
adapterFor,
fakeClaude,
identityFor,
PROVIDER_SESSION_ID
} from './claude-structured-session-test-support'
import { ClaudeRewindAttempt } from './claude-structured-rewind'
import { AgentSessionRewindRefusal } from '../native-chat/agent-session-wire/structured-agent-session-adapter'
const intent = { targetUuid: 'kept', previousLeafUuid: 'tip', dropsTurn: 'drop' }
const proofLaunch = {
providerSessionId: PROVIDER_SESSION_ID,
claudeConfigDir: '/claude',
options: {},
resumed: true,
resumeLeafUuid: 'tip',
cwd: '/workspace',
pathToClaudeCodeExecutable: 'claude'
}
describe('Claude rewind acquisition', () => {
it('executes a cursor resume in place and proves the exact target before publication', async () => {
const fake = fakeClaude()
const proof = vi.fn(async (_input: { intentionalRewindUuid?: string }) => 'kept')
const adapter = adapterFor(
fake,
{ resumed: true, resumeLeafUuid: 'tip' },
[],
[],
undefined,
proof
)
try {
const acquired = await adapter.acquire({
identity: identityFor(),
fence: 7,
spawnToken: 'spawn',
rewind: intent
})
expect(acquired.link.handle).toMatchObject({
provider: 'claude',
sessionId: PROVIDER_SESSION_ID,
leafUuid: 'kept'
})
expect(fake.connections[0]!.launch.options).toMatchObject({
resume: PROVIDER_SESSION_ID,
resumeSessionAt: 'kept',
resumeDropsTurn: 'drop'
})
expect(fake.connections[0]!.launch.options).not.toHaveProperty('forkSession')
expect(proof).toHaveBeenCalledWith(
expect.objectContaining({ previousLeafUuid: 'tip', intentionalRewindUuid: 'kept' })
)
await adapter.closeSession('session-1')
await adapter.acquire({ identity: identityFor(), fence: 8, spawnToken: 'spawn-next' })
expect(fake.connections[1]!.launch.options).not.toHaveProperty('resumeDropsTurn')
expect(
proof.mock.calls.filter(([input]) => input.intentionalRewindUuid !== undefined)
).toHaveLength(1)
} finally {
await adapter.closeAll()
}
})
it('recognizes the documented refusal and closes the failed child without retry', async () => {
const fake = fakeClaude()
const openConnection = fake.openConnection
fake.openConnection = async (launch, handlers) => {
const connection = await openConnection(launch, handlers)
const initialize = connection.initializationResult
connection.initializationResult = async (...args) => {
const result = await initialize(...args)
handlers?.onMessage?.({
type: 'result',
subtype: 'error_during_execution',
session_id: PROVIDER_SESSION_ID,
errors: ['Resume rejected by --resume-drops-turn: additional prompt observed']
})
return result
}
return connection
}
const proof = vi.fn(async (_input: { intentionalRewindUuid?: string }) => 'kept')
const adapter = adapterFor(fake, { resumed: true }, [], [], undefined, proof)
await expect(
adapter.acquire({ identity: identityFor(), fence: 7, spawnToken: 'spawn', rewind: intent })
).rejects.toMatchObject({ rewindReason: 'provider-refused' })
expect(fake.connections).toHaveLength(1)
expect(fake.connections[0]?.closed).toBe(true)
expect(proof).not.toHaveBeenCalled()
await adapter.closeAll()
})
it('consumes proof authorization even if its first read fails', async () => {
const proof = vi.fn(async () => {
throw new Error('torn transcript')
})
const attempt = new ClaudeRewindAttempt(intent)
const launch = {
providerSessionId: PROVIDER_SESSION_ID,
claudeConfigDir: '/claude',
options: {},
resumed: true,
resumeLeafUuid: 'tip',
cwd: '/workspace',
pathToClaudeCodeExecutable: 'claude'
}
await expect(attempt.prove(launch, { readTranscriptLeaf: proof })).rejects.toBeInstanceOf(
AgentSessionRewindRefusal
)
expect(await attempt.prove(launch, { readTranscriptLeaf: proof })).toBeNull()
expect(proof).toHaveBeenCalledTimes(1)
})
it('never persists success for a mismatching leaf', async () => {
const onProved = vi.fn(async () => {})
const attempt = new ClaudeRewindAttempt(intent, onProved)
await expect(
attempt.prove(proofLaunch, { readTranscriptLeaf: async () => 'other' })
).rejects.toMatchObject({ rewindReason: 'proof-mismatch' })
expect(onProved).not.toHaveBeenCalled()
})
it('preserves commit failure as unknown and consumes the override before persisting', async () => {
const diskError = new Error('record write failed')
const onProved = vi.fn(async () => {
throw diskError
})
const proof = vi.fn(async () => 'kept')
const attempt = new ClaudeRewindAttempt(intent, onProved)
const launch = {
providerSessionId: PROVIDER_SESSION_ID,
claudeConfigDir: '/claude',
options: {},
resumed: true,
resumeLeafUuid: 'tip',
cwd: '/workspace',
pathToClaudeCodeExecutable: 'claude'
}
await expect(attempt.prove(launch, { readTranscriptLeaf: proof })).rejects.toBe(diskError)
expect(onProved).toHaveBeenCalledWith('kept')
expect(await attempt.prove(launch, { readTranscriptLeaf: proof })).toBeNull()
expect(proof).toHaveBeenCalledTimes(1)
})
it('checkpoints the proved target before late acquisition failure without persisting a stale cursor', async () => {
const fake = fakeClaude()
const launch = { resumed: true, resumeLeafUuid: 'tip' }
const persisted: unknown[] = []
const proof = vi.fn(async () => 'kept')
const adapter = adapterFor(fake, launch, [], persisted, undefined, proof)
const onProved = vi.fn(async (leafUuid: string) => {
launch.resumeLeafUuid = leafUuid
fake.connections[0]!.closed = true
})
try {
await expect(
adapter.acquire({
identity: identityFor(),
fence: 7,
spawnToken: 'spawn',
rewind: { ...intent, onProved }
})
).rejects.toThrow('exited while being acquired')
expect(onProved).toHaveBeenCalledWith('kept')
expect(persisted).toEqual([])
const acquired = await adapter.acquire({
identity: identityFor(),
fence: 8,
spawnToken: 'retry'
})
expect(acquired.link.handle).toMatchObject({ leafUuid: 'kept' })
expect(fake.connections[1]!.launch.options).not.toHaveProperty('resumeDropsTurn')
expect(proof).toHaveBeenCalledTimes(1)
} finally {
await adapter.closeAll()
}
})
it('restores an interrupted unproved rewind only after exact ordinary branch proof', async () => {
const fake = fakeClaude()
const proof = vi.fn(async (_input: { intentionalRewindUuid?: string }) => 'kept')
const restored = vi.fn(async () => {})
const adapter = adapterFor(
fake,
{ resumed: true, resumeLeafUuid: 'tip' },
[],
[],
undefined,
proof
)
const input = {
identity: identityFor(),
fence: 7,
spawnToken: 'spawn',
rewindRecovery: { leafUuid: 'tip', onProved: restored }
}
try {
await expect(adapter.acquire(input)).rejects.toMatchObject({ rewindReason: 'proof-mismatch' })
expect(restored).not.toHaveBeenCalled()
proof.mockResolvedValue('tip')
await adapter.acquire({ ...input, fence: 8, spawnToken: 'retry' })
expect(restored).toHaveBeenCalledOnce()
expect(proof).toHaveBeenCalledWith(expect.objectContaining({ previousLeafUuid: 'tip' }))
for (const [request] of proof.mock.calls) {
expect(request).not.toHaveProperty('intentionalRewindUuid')
}
} finally {
await adapter.closeAll()
}
})
})
+118
View File
@@ -0,0 +1,118 @@
import { AgentSessionRewindRefusal } from '../native-chat/agent-session-wire/structured-agent-session-adapter'
export function claudeRewindRefusalFromMessage(
message: Record<string, unknown>
): AgentSessionRewindRefusal | null {
return message.type === 'result' &&
message.subtype === 'error_during_execution' &&
Array.isArray(message.errors) &&
message.errors.some(
(error) =>
typeof error === 'string' && error.startsWith('Resume rejected by --resume-drops-turn:')
)
? new AgentSessionRewindRefusal('provider-refused')
: null
}
import type { StructuredAgentSessionAcquireInput } from '../native-chat/agent-session-wire/structured-agent-session-adapter'
import type { ClaudeStructuredLaunch } from './claude-structured-launch-resolution'
import type { ClaudeStructuredSessionAdapterDeps } from './claude-structured-session-state'
type Intent = NonNullable<StructuredAgentSessionAcquireInput['rewind']>
/** The proof authorization exists only for this acquisition's first proof attempt. */
export class ClaudeRewindAttempt {
private refusal: AgentSessionRewindRefusal | null = null
constructor(
private intent: Intent | undefined,
private readonly onProved?: (leafUuid: string) => Promise<void>
) {}
observe(message: Record<string, unknown>): AgentSessionRewindRefusal | null {
if (!this.intent) {
return null
}
this.refusal ??= claudeRewindRefusalFromMessage(message)
return this.refusal
}
applyLaunch(
launch: ClaudeStructuredLaunch,
deps: Pick<ClaudeStructuredSessionAdapterDeps, 'readTranscriptLeaf'>
): void {
if (!this.intent) {
return
}
if (!launch.resumed || !deps.readTranscriptLeaf) {
throw new AgentSessionRewindRefusal('unsupported')
}
launch.options = {
...launch.options,
resume: launch.providerSessionId,
resumeSessionAt: this.intent.targetUuid,
...(this.intent.dropsTurn ? { resumeDropsTurn: this.intent.dropsTurn } : {})
}
launch.resumeLeafUuid = this.intent.targetUuid
}
async prove(
launch: ClaudeStructuredLaunch,
deps: Pick<ClaudeStructuredSessionAdapterDeps, 'readTranscriptLeaf'>
): Promise<string | null> {
const intent = this.intent
this.clear()
if (this.refusal) {
throw this.refusal
}
if (!intent) {
return null
}
let leaf: string | null
try {
leaf = await deps.readTranscriptLeaf!({
providerSessionId: launch.providerSessionId,
previousLeafUuid: intent.previousLeafUuid,
intentionalRewindUuid: intent.targetUuid,
claudeConfigDir: launch.claudeConfigDir
})
if (leaf !== intent.targetUuid) {
throw new AgentSessionRewindRefusal('proof-mismatch')
}
} catch (error) {
throw error instanceof AgentSessionRewindRefusal
? error
: new AgentSessionRewindRefusal('proof-mismatch')
}
// Persistence failure is an unknown outcome, never evidence that the provider refused.
await this.onProved?.(leaf)
return leaf
}
clear(): void {
this.intent = undefined
}
}
/** An interrupted, unproved rewind restores its original cursor without ancestor authorization. */
export async function proveClaudeRewindRecovery(
recovery: StructuredAgentSessionAcquireInput['rewindRecovery'],
launch: ClaudeStructuredLaunch,
deps: Pick<ClaudeStructuredSessionAdapterDeps, 'readTranscriptLeaf'>
): Promise<string | null> {
if (!recovery) {
return null
}
if (!launch.resumed || launch.resumeLeafUuid !== recovery.leafUuid || !deps.readTranscriptLeaf) {
throw new AgentSessionRewindRefusal('proof-mismatch')
}
const leaf = await deps.readTranscriptLeaf({
providerSessionId: launch.providerSessionId,
previousLeafUuid: recovery.leafUuid,
claudeConfigDir: launch.claudeConfigDir
})
if (leaf !== recovery.leafUuid) {
throw new AgentSessionRewindRefusal('proof-mismatch')
}
await recovery.onProved()
return leaf
}
@@ -1,3 +1,4 @@
import { ClaudeRewindAttempt, proveClaudeRewindRecovery } from './claude-structured-rewind'
import {
AgentSessionAcquisitionExitUnprovenError,
AgentSessionPreSpawnError
@@ -85,6 +86,7 @@ export async function acquireClaudeSession({
const initTimeoutMs = deps.initTimeoutMs ?? CLAUDE_STRUCTURED_INIT_TIMEOUT_MS
const initDeadline = createClaudeInitDeadline(sessionId, initTimeoutMs)
const rewind = new ClaudeRewindAttempt(input.rewind, input.rewind?.onProved)
const onMessage = (message: Record<string, unknown>): void => {
const init = readClaudeInit(message)
if (readClaudeFrameString(message, 'session_id') !== expectedProviderSessionId) {
@@ -95,6 +97,11 @@ export async function acquireClaudeSession({
}
return
}
const refusal = rewind.observe(message)
if (refusal) {
initDeadline.reject(refusal)
return
}
if (init) {
initDeadline.resolve(init)
// Every turn opens with an init frame naming the model the CLI is actually
@@ -178,6 +185,7 @@ export async function acquireClaudeSession({
? error
: new AgentSessionPreSpawnError(error)
})
rewind.applyLaunch(launch, deps)
expectedProviderSessionId = launch.providerSessionId
observedLeafUuid = launch.resumeLeafUuid
acquisitions.assertCurrent(sessionId, attempt)
@@ -241,6 +249,9 @@ export async function acquireClaudeSession({
diagnostic: claudeAuthDiagnostic(init, settings)
})
)
observedLeafUuid = (await rewind.prove(launch, deps)) ?? observedLeafUuid
observedLeafUuid =
(await proveClaudeRewindRecovery(input.rewindRecovery, launch, deps)) ?? observedLeafUuid
const process = await claudeProcessIdentity(
{ ...input, pid: connection.pid },
deps.readProcessStartTime
@@ -298,6 +309,7 @@ export async function acquireClaudeSession({
acquisitions.deleteIfCurrent(sessionId, attempt)
throw acquisitionError
} finally {
rewind.clear()
attempt.finish()
}
}
@@ -4,7 +4,6 @@ import type {
StructuredAgentSessionAcquireInput,
StructuredAgentSessionAdapter
} from '../native-chat/agent-session-wire/structured-agent-session-adapter'
import type { StructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink'
import {
answerClaudePrompt,
cancelClaudeTurn,
@@ -58,6 +57,9 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda
supportsLocation = supportsClaudeStructuredLocation
rewindSupport: NonNullable<StructuredAgentSessionAdapter['rewindSupport']> = () =>
this.deps.readTranscriptLeaf ? { supported: true } : { supported: false, reason: 'unsupported' }
acquire = (input: StructuredAgentSessionAcquireInput): Promise<AgentSessionAcquisition> =>
acquireClaudeSession({
input,
@@ -67,7 +69,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda
exits: this.exits,
callbacks: {
deliver: (attempt, sessionId, event) => this.deliver(attempt, sessionId, event),
emit: (session, events, event) => this.emit(session, events, event),
emit: (session, _events, event) => this.emit(session, event),
handleExit: (sessionId, attempt, error) => this.handleExit(sessionId, attempt, error),
settleExit: (sessionId, exit) => this.settleUnexpectedExit(sessionId, exit)
}
@@ -155,7 +157,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda
acquisitionGeneration: exit.session.acquisitionGeneration
}
try {
this.emit(exit.session, exit.session.events, ended)
this.emit(exit.session, ended)
} finally {
settleClaudeExitedSession(exit.session)
}
@@ -187,11 +189,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda
})
}
private emit(
session: ClaudeSession | null,
_events: StructuredAgentSessionEventSink | undefined,
event: ClaudeStructuredSessionEvent
): void {
private emit(session: ClaudeSession | null, event: ClaudeStructuredSessionEvent): void {
const backgroundTasksChanged =
event.type === 'ended'
? (session?.backgroundTasks.clear() ?? false)
@@ -88,6 +88,7 @@ export type ClaudeStructuredSessionAdapterDeps = {
readTranscriptLeaf?: (input: {
providerSessionId: string
previousLeafUuid: string | null
intentionalRewindUuid?: string
/** Account-scoped Claude config root that owns this provider session. */
claudeConfigDir: string
}) => Promise<string | null>
@@ -13,7 +13,7 @@ type TranscriptNode = {
export type ClaudeTranscriptBranchProof = {
leafUuid: string
relation: 'initial' | 'same' | 'descendant'
relation: 'initial' | 'same' | 'descendant' | 'intentional-rewind'
}
function nonEmptyString(value: unknown): string | null {
@@ -83,6 +83,7 @@ export function proveClaudeTranscriptBranchFromJsonl(input: {
contents: string
providerSessionId: string
previousLeafUuid: string | null
intentionalRewindUuid?: string
}): ClaudeTranscriptBranchProof {
const nodes = new Map<string, TranscriptNode>()
let leafUuid: string | null = null
@@ -156,6 +157,21 @@ export function proveClaudeTranscriptBranchFromJsonl(input: {
throw transcriptError('marker precedes its leaf record')
}
const previousLeafUuid = input.previousLeafUuid
if (input.intentionalRewindUuid !== undefined) {
if (leafUuid !== input.intentionalRewindUuid || !input.previousLeafUuid) {
throw transcriptError('rewind target does not match the observed leaf')
}
proveMainLineAncestry(nodes, input.previousLeafUuid, input.providerSessionId)
proveAppendOrder(nodes)
let ancestor = nodes.get(input.previousLeafUuid)?.parentUuid ?? null
for (let depth = 0; ancestor !== null && depth < MAX_CLAUDE_TRANSCRIPT_ANCESTRY; depth += 1) {
if (ancestor === leafUuid) {
return { leafUuid, relation: 'intentional-rewind' }
}
ancestor = nodes.get(ancestor)?.parentUuid ?? null
}
throw transcriptError('rewind target is not an ancestor of the previous cursor')
}
if (!previousLeafUuid) {
proveMainLineAncestry(nodes, leafUuid, input.providerSessionId)
// A branch proof is based on an append-only snapshot. A child that appears
@@ -210,11 +226,13 @@ export async function proveClaudeTranscriptBranch(input: {
transcriptPath: string
providerSessionId: string
previousLeafUuid: string | null
intentionalRewindUuid?: string
}): Promise<ClaudeTranscriptBranchProof> {
return proveClaudeTranscriptBranchFromJsonl({
contents: await readFile(input.transcriptPath, 'utf8'),
providerSessionId: input.providerSessionId,
previousLeafUuid: input.previousLeafUuid
previousLeafUuid: input.previousLeafUuid,
intentionalRewindUuid: input.intentionalRewindUuid
})
}
@@ -0,0 +1,46 @@
import { describe, expect, it } from 'vitest'
import { proveClaudeTranscriptBranchFromJsonl } from './claude-transcript-branch-proof'
const row = (uuid: string, parentUuid: string | null, extra = {}) =>
JSON.stringify({ type: 'assistant', sessionId: 'provider', uuid, parentUuid, ...extra })
const marker = (leafUuid: string) =>
JSON.stringify({ type: 'last-prompt', sessionId: 'provider', leafUuid })
const graph = [row('root', null), row('kept', 'root'), row('old', 'kept')]
const prove = (rows: string[], leaf: string, intentionalRewindUuid?: string) =>
proveClaudeTranscriptBranchFromJsonl({
contents: `${[...rows, marker(leaf)].join('\n')}\n`,
providerSessionId: 'provider',
previousLeafUuid: 'old',
intentionalRewindUuid
})
describe('explicit Claude rewind ancestry', () => {
it('admits only the exact requested main-chain ancestor', () => {
expect(prove(graph, 'kept', 'kept')).toEqual({
leafUuid: 'kept',
relation: 'intentional-rewind'
})
expect(() => prove(graph, 'kept')).toThrow('sibling')
expect(() => prove(graph, 'kept', 'root')).toThrow('target')
expect(() => prove(graph, 'old', 'old')).toThrow('not an ancestor')
})
it('keeps sibling and sidechain rejection even with explicit intent', () => {
expect(() => prove([...graph, row('sibling', 'root')], 'sibling', 'sibling')).toThrow(
'not an ancestor'
)
expect(() =>
prove(
[row('root', null), row('kept', 'root', { isSidechain: true }), row('old', 'kept')],
'kept',
'kept'
)
).toThrow()
})
it('refuses missing, reordered, or cyclic ancestry', () => {
expect(() => prove(graph.slice(1), 'kept', 'kept')).toThrow('missing ancestor')
expect(() => prove([graph[1]!, graph[0]!, graph[2]!], 'kept', 'kept')).toThrow(
'parent row follows'
)
expect(() => prove([row('root', 'old'), ...graph.slice(1)], 'kept', 'kept')).toThrow('cycle')
})
})
+29 -4
View File
@@ -182,10 +182,10 @@ export function readCodexAuthIdentity(contents: string): CodexAuthIdentity | nul
readStringClaim(authClaims, 'chatgpt_account_id') ??
readStringClaim(payload, 'chatgpt_account_id')
),
workspaceLabel: normalizeField(
readStringClaim(authClaims, 'workspace_name') ??
readStringClaim(profileClaims, 'workspace_name')
),
workspaceLabel:
normalizeField(readStringClaim(authClaims, 'workspace_name')) ??
normalizeField(readStringClaim(profileClaims, 'workspace_name')) ??
readPlanWorkspaceLabel(authClaims),
workspaceAccountId: normalizeField(
readStringClaim(authClaims, 'workspace_account_id') ??
tokenAccountId ??
@@ -194,6 +194,31 @@ export function readCodexAuthIdentity(contents: string): CodexAuthIdentity | nul
}
}
function readPlanWorkspaceLabel(authClaims: Record<string, unknown> | null): string | null {
// Codex tokens commonly omit workspace_name but identify the account's plan.
switch (normalizeField(readStringClaim(authClaims, 'chatgpt_plan_type'))?.toLowerCase()) {
case 'free':
return 'Personal (Free)'
case 'go':
return 'Personal (Go)'
case 'plus':
return 'Personal (Plus)'
case 'pro':
return 'Personal (Pro)'
case 'team':
return 'Team'
case 'business':
return 'Business'
case 'enterprise':
return 'Enterprise'
case 'edu':
return 'Education'
case undefined:
default:
return null
}
}
function readFreshnessFromAuthContents(contents: string): number | null {
const raw = parseJsonRecord(contents)
if (!raw) {
@@ -0,0 +1,106 @@
import { describe, expect, it } from 'vitest'
import type { CodexManagedAccount } from '../../shared/managed-account-types'
import {
codexAuthMatchesManagedAccount,
codexAuthMatchesSystemDefaultIdentity,
readCodexAuthIdentity
} from './codex-auth-identity'
const email = 'same@example.com'
function auth(
accountId: string,
claims: Record<string, unknown>,
profileClaims: Record<string, unknown> = {}
): string {
const payload = Buffer.from(
JSON.stringify({
email,
'https://api.openai.com/auth': { chatgpt_account_id: accountId, ...claims },
'https://api.openai.com/profile': profileClaims
})
).toString('base64url')
return JSON.stringify({
tokens: { account_id: accountId, id_token: `header.${payload}.signature` }
})
}
describe('Codex personal and organization workspace identity', () => {
it.each([
['free', 'Personal (Free)'],
['go', 'Personal (Go)'],
['plus', 'Personal (Plus)'],
['pro', 'Personal (Pro)'],
['team', 'Team'],
['business', 'Business'],
['enterprise', 'Enterprise'],
['edu', 'Education']
])('uses the %s plan when the token omits the workspace name', (plan, label) => {
expect(readCodexAuthIdentity(auth('provider-1', { chatgpt_plan_type: plan }))).toEqual({
email,
providerAccountId: 'provider-1',
workspaceAccountId: 'provider-1',
workspaceLabel: label
})
})
it.each([undefined, null, '', 'future-plan', 42])(
'does not infer personal membership from an unknown plan %s',
(plan) => {
expect(
readCodexAuthIdentity(auth('provider-1', { chatgpt_plan_type: plan }))?.workspaceLabel
).toBeNull()
}
)
it('preserves an explicit organization name over the plan label', () => {
expect(
readCodexAuthIdentity(
auth('provider-1', { workspace_name: ' Acme ', chatgpt_plan_type: 'enterprise' })
)?.workspaceLabel
).toBe('Acme')
})
it('uses the profile workspace name when the auth workspace name is blank', () => {
expect(
readCodexAuthIdentity(
auth(
'provider-1',
{ workspace_name: ' ', chatgpt_plan_type: 'enterprise' },
{ workspace_name: 'Acme' }
)
)?.workspaceLabel
).toBe('Acme')
})
it('keeps same-email personal and enterprise credentials isolated in both directions', () => {
const personal = auth('personal-provider', { chatgpt_plan_type: 'plus' })
const enterprise = auth('enterprise-provider', { chatgpt_plan_type: 'enterprise' })
for (const [selectedAuth, otherAuth] of [
[personal, enterprise],
[enterprise, personal]
]) {
const identity = readCodexAuthIdentity(selectedAuth)!
const account: CodexManagedAccount = {
...identity,
id: 'orca-account',
email,
managedHomePath: 'managed-home',
createdAt: 1,
updatedAt: 1,
lastAuthenticatedAt: 1
}
expect(codexAuthMatchesManagedAccount(selectedAuth, account, selectedAuth)).toBe(true)
expect(codexAuthMatchesManagedAccount(otherAuth, account, selectedAuth)).toBe(false)
expect(codexAuthMatchesSystemDefaultIdentity(otherAuth, selectedAuth)).toBe(false)
}
expect(readCodexAuthIdentity(personal)?.workspaceLabel).toBe('Personal (Plus)')
expect(readCodexAuthIdentity(enterprise)?.workspaceLabel).toBe('Enterprise')
})
it('does not treat a matching plan label as proof of account ownership', () => {
const first = auth('enterprise-a', { chatgpt_plan_type: 'enterprise' })
const second = auth('enterprise-b', { chatgpt_plan_type: 'enterprise' })
expect(codexAuthMatchesSystemDefaultIdentity(first, second)).toBe(false)
})
})
@@ -87,64 +87,67 @@ describe('CodexRuntimeHomeService', () => {
expect(service.getHostCodexHomePathsForSessionDiscovery()).toContain(managedHomePath)
})
it('gives two managed accounts distinct homes without racing one auth.json', async () => {
writeFileSync(getSystemCodexAuthPath(), '{"account":"system"}\n', 'utf-8')
const account1Auth = createCodexAuthJson('one@example.com', 'acct-1', 'one')
const account2Auth = createCodexAuthJson('two@example.com', 'acct-2', 'two')
const home1 = createManagedAuth(testState.userDataDir, 'account-1', account1Auth)
const home2 = createManagedAuth(testState.userDataDir, 'account-2', account2Auth)
const settings = createSettings({
shellStartupEnvProbeSupported: true,
codexManagedAccounts: [
{
id: 'account-1',
email: 'one@example.com',
managedHomePath: home1,
providerAccountId: 'acct-1',
workspaceLabel: null,
workspaceAccountId: 'acct-1',
createdAt: 1,
updatedAt: 1,
lastAuthenticatedAt: 1
},
{
id: 'account-2',
email: 'two@example.com',
managedHomePath: home2,
providerAccountId: 'acct-2',
workspaceLabel: null,
workspaceAccountId: 'acct-2',
createdAt: 2,
updatedAt: 2,
lastAuthenticatedAt: 2
}
],
activeCodexManagedAccountId: 'account-1',
activeCodexManagedAccountIdsByRuntime: { host: 'account-1', wsl: {} }
})
const store = createStore(settings)
const { CodexRuntimeHomeService } = await import('./runtime-home-service')
const service = new CodexRuntimeHomeService(store as never)
// A pane for account-1 launches, then the user switches and a second pane
// for account-2 launches concurrently — each gets its OWN CODEX_HOME.
expect(service.prepareForCodexLaunch()).toBe(home1)
settings.activeCodexManagedAccountId = 'account-2'
settings.activeCodexManagedAccountIdsByRuntime = { host: 'account-2', wsl: {} }
expect(service.prepareForCodexLaunch()).toBe(home2)
expect(
service.prepareForCodexLaunch(undefined, undefined, {
unavailableManagedHomePath: home1
it.each(['two@example.com', 'one@example.com'])(
'isolates account homes when the second email is %s',
async (secondEmail) => {
writeFileSync(getSystemCodexAuthPath(), '{"account":"system"}\n', 'utf-8')
const account1Auth = createCodexAuthJson('one@example.com', 'acct-1', 'one')
const account2Auth = createCodexAuthJson(secondEmail, 'acct-2', 'two')
const home1 = createManagedAuth(testState.userDataDir, 'account-1', account1Auth)
const home2 = createManagedAuth(testState.userDataDir, 'account-2', account2Auth)
const settings = createSettings({
shellStartupEnvProbeSupported: true,
codexManagedAccounts: [
{
id: 'account-1',
email: 'one@example.com',
managedHomePath: home1,
providerAccountId: 'acct-1',
workspaceLabel: null,
workspaceAccountId: 'acct-1',
createdAt: 1,
updatedAt: 1,
lastAuthenticatedAt: 1
},
{
id: 'account-2',
email: secondEmail,
managedHomePath: home2,
providerAccountId: 'acct-2',
workspaceLabel: null,
workspaceAccountId: 'acct-2',
createdAt: 2,
updatedAt: 2,
lastAuthenticatedAt: 2
}
],
activeCodexManagedAccountId: 'account-1',
activeCodexManagedAccountIdsByRuntime: { host: 'account-1', wsl: {} }
})
).toBe(home2)
expect(store.updateSettings).not.toHaveBeenCalled()
const store = createStore(settings)
const { CodexRuntimeHomeService } = await import('./runtime-home-service')
const service = new CodexRuntimeHomeService(store as never)
// Nothing is hot-swapped, so the still-running account-1 pane keeps seeing
// account-1's credentials — the single-auth.json race (GAP-5) is gone.
expect(readFileSync(join(home1, 'auth.json'), 'utf-8')).toBe(account1Auth)
expect(readFileSync(join(home2, 'auth.json'), 'utf-8')).toBe(account2Auth)
expect(existsSync(getRuntimeCodexAuthPath())).toBe(false)
})
// A pane for account-1 launches, then the user switches and a second pane
// for account-2 launches concurrently — each gets its OWN CODEX_HOME.
expect(service.prepareForCodexLaunch()).toBe(home1)
settings.activeCodexManagedAccountId = 'account-2'
settings.activeCodexManagedAccountIdsByRuntime = { host: 'account-2', wsl: {} }
expect(service.prepareForCodexLaunch()).toBe(home2)
expect(
service.prepareForCodexLaunch(undefined, undefined, {
unavailableManagedHomePath: home1
})
).toBe(home2)
expect(store.updateSettings).not.toHaveBeenCalled()
// Nothing is hot-swapped, so the still-running account-1 pane keeps seeing
// account-1's credentials — the single-auth.json race (GAP-5) is gone.
expect(readFileSync(join(home1, 'auth.json'), 'utf-8')).toBe(account1Auth)
expect(readFileSync(join(home2, 'auth.json'), 'utf-8')).toBe(account2Auth)
expect(existsSync(getRuntimeCodexAuthPath())).toBe(false)
}
)
it('materializes resources and config into the per-account home on launch', async () => {
writeFileSync(getSystemCodexAuthPath(), '{"account":"system"}\n', 'utf-8')
@@ -1,5 +1,5 @@
import { describe, expect, it, vi } from 'vitest'
import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import {
@@ -29,6 +29,74 @@ vi.mock('node:os', async () => {
describe('CodexAccountService.addAccountFromHome', () => {
registerCodexAccountsTestHomes()
it('imports and switches personal and enterprise accounts sharing an email independently', async () => {
vi.doMock('../codex-cli/command', () => ({ resolveCodexCommand: () => 'codex' }))
const sourceHomes = [
mkdtempSync(join(tmpdir(), 'orca-codex-personal-')),
mkdtempSync(join(tmpdir(), 'orca-codex-enterprise-'))
]
const email = 'same@example.com'
const credentials = ['plus', 'enterprise'].map((plan) => {
const parsed = JSON.parse(createCodexAuthJson(email, `provider-${plan}`, `refresh-${plan}`))
const payload = Buffer.from(
JSON.stringify({
email,
'https://api.openai.com/auth': {
chatgpt_account_id: `provider-${plan}`,
chatgpt_plan_type: plan
}
})
).toString('base64url')
parsed.tokens.id_token = `header.${payload}.signature`
return JSON.stringify(parsed)
})
try {
sourceHomes.forEach((home, index) => {
writeFileSync(join(home, 'auth.json'), credentials[index], 'utf-8')
})
const store = createStore(createSettings())
const runtimeHome = createRuntimeHome()
const { CodexAccountService } = await import('./service')
const service = new CodexAccountService(
store as never,
createRateLimits() as never,
runtimeHome as never
)
await service.addAccountFromHome(sourceHomes[0])
const result = await service.addAccountFromHome(sourceHomes[1])
const accounts = store.getSettings().codexManagedAccounts
expect(result.accounts).toHaveLength(2)
expect(new Set(accounts.map((account) => account.id)).size).toBe(2)
expect(new Set(accounts.map((account) => account.managedHomePath)).size).toBe(2)
expect(accounts.map((account) => account.email)).toEqual([email, email])
expect(accounts.map((account) => account.workspaceLabel)).toEqual([
'Personal (Plus)',
'Enterprise'
])
expect(accounts.map((account) => account.providerAccountId)).toEqual([
'provider-plus',
'provider-enterprise'
])
for (const account of accounts) {
const selected = await service.selectAccount(account.id)
expect(selected.activeAccountId).toBe(account.id)
expect(store.getSettings().activeCodexManagedAccountIdsByRuntime?.host).toBe(account.id)
accounts.forEach((entry, index) => {
expect(readFileSync(join(entry.managedHomePath, 'auth.json'), 'utf-8')).toBe(
credentials[index]
)
})
}
expect(runtimeHome.syncForCurrentSelection).toHaveBeenCalledTimes(4)
} finally {
sourceHomes.forEach((home) => rmSync(home, { recursive: true, force: true }))
vi.doUnmock('../codex-cli/command')
}
})
it('registers a managed Codex account by importing an authenticated CODEX_HOME', async () => {
vi.doMock('../codex-cli/command', () => ({ resolveCodexCommand: () => 'codex' }))
const sourceHome = mkdtempSync(join(tmpdir(), 'orca-codex-source-'))
@@ -11,7 +11,8 @@ import {
runCodexHookTrustGrantSession,
type CodexHookTrustGrantRequest
} from './codex-app-server-client'
import { killCodexAppServerProcessTree, runCodexAppServerSession } from './codex-app-server-session'
import { killCodexAppServerProcessTree } from './codex-app-server-process-tree-kill'
import { runCodexAppServerSession } from './codex-app-server-session'
// Stub codex app-server speaking the same JSONL protocol: initialize →
// initialized → hooks/list → config/batchWrite → hooks/list. Scenario-driven
+8 -2
View File
@@ -1,4 +1,5 @@
import { spawn } from 'node:child_process'
import type { ChildProcessHandle, ProcessSpec } from '../../shared/child-process/process-spec'
import { spawnProcess } from '../../shared/child-process/run-process'
import { normalizeHookTrustKeyForLookup } from './config-toml-trust'
import { runCodexAppServerSession, type CodexAppServerInvocation } from './codex-app-server-session'
@@ -105,7 +106,12 @@ function collectHookListings(result: unknown): CodexHookListing[] {
*/
export async function runCodexHookTrustGrantSession(
request: CodexHookTrustGrantRequest,
spawnImpl: typeof spawn = spawn
spawnImpl: (
program: string,
args: string[],
options: Record<string, unknown>
) => ChildProcessHandle = (program, args, options) =>
spawnProcess({ program, args, ...options } as ProcessSpec)
): Promise<CodexHookTrustGrantSessionResult> {
return runCodexAppServerSession(
request.invocation,
@@ -0,0 +1,76 @@
import { spawnProcess } from '../../shared/child-process/run-process'
import type { ChildProcessHandle, ProcessSpec } from '../../shared/child-process/process-spec'
import { admitProcessTreeKill } from '../../shared/child-process/process-tree-kill-gate'
/** Spawn seam for tests; production always goes through the hardened spawnProcess wrapper. */
export type CodexAppServerSpawn = (
program: string,
args: string[],
options: Record<string, unknown>
) => ChildProcessHandle
export const spawnCodexAppServerProcess: CodexAppServerSpawn = (program, args, options) =>
spawnProcess({ program, args, ...options } as ProcessSpec)
export function killCodexAppServerProcessTree(
child: Pick<ChildProcessHandle, 'pid' | 'kill'>,
options: { platform?: NodeJS.Platform; spawnImpl?: CodexAppServerSpawn } = {}
): void {
const platform = options.platform ?? process.platform
const spawnImpl = options.spawnImpl ?? spawnCodexAppServerProcess
if (platform === 'win32' && child.pid) {
if (
!admitProcessTreeKill({
pid: child.pid,
site: 'codex-app-server-session-deadline',
scope: 'win-taskkill-tree'
})
) {
// Refusal blocks the tree walk, not the termination: the root kill is
// handle-addressed, so it cannot reach the recycled pid we refused.
child.kill('SIGKILL')
return
}
try {
// Why: npm-installed Codex runs behind cmd.exe; killing only that wrapper
// leaves the app-server child alive after a timeout or failed shutdown.
const killer = spawnImpl('taskkill', ['/pid', String(child.pid), '/t', '/f'], {
stdio: 'ignore',
windowsHide: true
})
let fellBack = false
const killDirectChild = (): void => {
if (!fellBack) {
fellBack = true
child.kill('SIGKILL')
}
}
killer.on('error', killDirectChild)
killer.on('exit', (code) => {
if (code !== 0) {
killDirectChild()
}
})
killer.unref()
return
} catch {
// Fall through to the direct-child best effort when taskkill cannot start.
}
}
if (child.pid) {
try {
// npm/package-manager launchers insert a shim child on POSIX. Reap its
// direct descendants before signalling the wrapper itself.
const descendants = spawnImpl('pkill', ['-KILL', '-P', String(child.pid)], {
stdio: 'ignore'
})
// A missing pkill surfaces as an async 'error' event, and an unhandled one
// takes down the main process.
descendants.on('error', () => undefined)
descendants.unref()
} catch {
// The direct kill below remains the fallback when pkill is unavailable.
}
}
child.kill('SIGKILL')
}
+7 -66
View File
@@ -1,9 +1,13 @@
import { spawn, type ChildProcess, type ChildProcessWithoutNullStreams } from 'node:child_process'
import type { ChildProcessWithoutNullStreams } from 'node:child_process'
import { waitForProcessExitUntil } from './codex-process-exit-deadline'
import { stderrIndicatesMissingAppServer } from './codex-app-server-capability-signal'
import { withCliRuntimeOnPath } from '../../shared/node-cli-command-resolution'
import {
killCodexAppServerProcessTree,
spawnCodexAppServerProcess,
type CodexAppServerSpawn
} from './codex-app-server-process-tree-kill'
import { createCodexAppServerRecordReader } from './codex-app-server-record-reader'
import { admitProcessTreeKill } from '../../shared/child-process/process-tree-kill-gate'
// Why: `codex app-server` is Orca's sanctioned RPC surface into Codex-owned
// state (hook trust hashes, the sqlite thread index). This module owns the
@@ -68,69 +72,6 @@ export type CodexAppServerRpc = {
const JSON_RPC_METHOD_NOT_FOUND = -32601
const STDERR_TAIL_MAX_BYTES = 8192
export function killCodexAppServerProcessTree(
child: Pick<ChildProcess, 'pid' | 'kill'>,
options: { platform?: NodeJS.Platform; spawnImpl?: typeof spawn } = {}
): void {
const platform = options.platform ?? process.platform
const spawnImpl = options.spawnImpl ?? spawn
if (platform === 'win32' && child.pid) {
if (
!admitProcessTreeKill({
pid: child.pid,
site: 'codex-app-server-session-deadline',
scope: 'win-taskkill-tree'
})
) {
// Refusal blocks the tree walk, not the termination: the root kill is
// handle-addressed, so it cannot reach the recycled pid we refused.
child.kill('SIGKILL')
return
}
try {
// Why: npm-installed Codex runs behind cmd.exe; killing only that wrapper
// leaves the app-server child alive after a timeout or failed shutdown.
const killer = spawnImpl('taskkill', ['/pid', String(child.pid), '/t', '/f'], {
stdio: 'ignore',
windowsHide: true
})
let fellBack = false
const killDirectChild = (): void => {
if (!fellBack) {
fellBack = true
child.kill('SIGKILL')
}
}
killer.on('error', killDirectChild)
killer.on('exit', (code) => {
if (code !== 0) {
killDirectChild()
}
})
killer.unref()
return
} catch {
// Fall through to the direct-child best effort when taskkill cannot start.
}
}
if (child.pid) {
try {
// npm/package-manager launchers insert a shim child on POSIX. Reap its
// direct descendants before signalling the wrapper itself.
const descendants = spawnImpl('pkill', ['-KILL', '-P', String(child.pid)], {
stdio: 'ignore'
})
// A missing pkill surfaces as an async 'error' event, and an unhandled one
// takes down the main process.
descendants.on('error', () => undefined)
descendants.unref()
} catch {
// The direct kill below remains the fallback when pkill is unavailable.
}
}
child.kill('SIGKILL')
}
/** Codex answering "no such method" is the only response that proves the RPC
* surface is absent rather than temporarily failing. */
export function isCodexMethodNotFoundError(error: unknown): boolean {
@@ -152,7 +93,7 @@ export function isCodexMethodNotFoundError(error: unknown): boolean {
export async function runCodexAppServerSession<T>(
invocation: CodexAppServerInvocation,
body: (rpc: CodexAppServerRpc) => Promise<T>,
spawnImpl: typeof spawn = spawn
spawnImpl: CodexAppServerSpawn = spawnCodexAppServerProcess
): Promise<T> {
// Why: a default-home grant must run against the real ~/.codex, so strip an
// inherited CODEX_HOME (envToDelete) after applying the overlay, not before.
@@ -784,7 +784,7 @@ describe('codex item bodies', () => {
}
})
it('leaves subagent items on the generic row until a real renderer exists', () => {
it('drops the raw subagent item now the roster row renders it', () => {
expect(
codexJournalItem({
type: 'subAgentActivity',
@@ -793,10 +793,7 @@ describe('codex item bodies', () => {
agentThreadId: 'thread-child',
agentPath: '/root/list_directory'
})
).toMatchObject({
handled: false,
body: { kind: 'status', providerFrame: { kind: 'item:subAgentActivity' } }
})
).toMatchObject({ handled: true, body: null })
})
it('drops the sleep item, which codex itself renders as nothing', () => {
@@ -7,3 +7,10 @@ export const MAX_CODEX_PENDING_PROMPTS = 128
export const MAX_CODEX_IDENTITY_ENTRIES = 512
export const MAX_CODEX_DETAIL_ENTRIES = 512
export const MAX_CODEX_DETAIL_BYTES = 64 * 1024
/** Spawn-group rows kept live per session, and children per row. Both bound an
* event-accumulated map that no provider snapshot ever prunes. */
export const MAX_CODEX_SUBAGENT_GROUPS = 32
export const MAX_CODEX_SUBAGENTS_PER_GROUP = 64
/** Threads whose latest token total is retained. Usage frames arrive for
* threads that are not yet (or never become) roster children. */
export const MAX_CODEX_TOKEN_USAGE_THREADS = 256
@@ -0,0 +1,43 @@
/**
* The translator's provider-frame arms.
*
* Each returns null for a frame it does not own, which is the translator's
* signal to keep looking. Split out so the translator reads as routing rather
* than as the shape checks each arm performs.
*/
import type { CodexJournalTranslationAdmission } from './codex-structured-journal-contracts'
import { settleCodexOversizedNotification } from './codex-structured-journal-settlement'
import {
readCodexJournalRecord,
readCodexJournalString
} from './codex-structured-journal-translation-values'
type OversizedInput = Parameters<typeof settleCodexOversizedNotification>[0]
/** A notification the transport refused to carry whole: settle whatever it
* opened rather than leaving the item mid-flight. */
export function settleCodexOversizedNotificationFrame(input: {
sessionId: string
threadId: string
kind: string
payload: unknown
sink: OversizedInput['sink']
streams: OversizedInput['streams']
activeItems: OversizedInput['activeItems']
}): CodexJournalTranslationAdmission | null {
if (input.kind !== 'frame:oversized-notification') {
return null
}
const method = readCodexJournalString(readCodexJournalRecord(input.payload), 'method')
return method
? settleCodexOversizedNotification({
sessionId: input.sessionId,
threadId: input.threadId,
method,
sink: input.sink,
streams: input.streams,
activeItems: input.activeItems
})
: null
}
@@ -0,0 +1,168 @@
import { describe, expect, it } from 'vitest'
import { agentJournalItemKey } from '../../shared/agent-session-journal-item-key'
import type { AgentSessionTurnActivity } from '../../shared/agent-session-wire'
import type {
AgentJournalItemBody,
AgentJournalItemIdentity
} from '../../shared/agent-session-journal-types'
import { isSubagentGroupBlock } from '../../shared/native-chat-types'
import type { StructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink'
import { createCodexJournalTranslator } from './codex-structured-journal-translation'
import type { CodexStructuredSessionEvent } from './codex-structured-session-adapter'
const SESSION_ID = 'session-1'
const THREAD_ID = 'thread-abc'
const TURN_ID = 'turn-1'
type Row = { key: string; body: AgentJournalItemBody }
function harness() {
const rows: Row[] = []
const activities: (AgentSessionTurnActivity | null)[] = []
const sink: StructuredAgentSessionEventSink = {
appendItem: (identity: AgentJournalItemIdentity, body) =>
rows.push({ key: agentJournalItemKey(identity), body }),
appendTombstone: () => {},
publish: () => {},
setActivity: (activity) => activities.push(activity)
}
const translator = createCodexJournalTranslator({
sink,
primaryThreadId: () => THREAD_ID,
schedule: (run: () => void) => {
run()
return () => {}
}
})
return { translator, rows, activities }
}
function notification(method: string, params: unknown): CodexStructuredSessionEvent {
return { type: 'notification', sessionId: SESSION_ID, threadId: THREAD_ID, method, params }
}
function subagentItem(kind: string, agentThreadId: string, agentPath: string): unknown {
return {
turnId: TURN_ID,
item: {
type: 'subAgentActivity',
id: `item-${agentThreadId}-${kind}`,
kind,
agentThreadId,
agentPath
}
}
}
/** Every activity item reaches the wire twice. */
function deliverActivity(
translator: ReturnType<typeof createCodexJournalTranslator>,
params: unknown
): void {
translator.handle(notification('item/started', params))
translator.handle(notification('item/completed', params))
}
function rosterAgents(rows: Row[]): { id: string; state: string; tokens?: number }[] {
const body = rows.findLast((row) => row.key.startsWith('orca:codex-subagents'))?.body
if (!body || body.kind !== 'message') {
return []
}
return body.blocks.find(isSubagentGroupBlock)?.agents ?? []
}
describe('codex journal translation — subagents', () => {
it('renders a spawn group as one roster row and no opcode-shaped duplicate', () => {
const { translator, rows } = harness()
translator.handle(notification('turn/started', { turn: { id: TURN_ID } }))
deliverActivity(translator, subagentItem('started', 'child-1', '/root/list_directory'))
deliverActivity(translator, subagentItem('interacted', 'child-1', '/root/list_directory'))
expect(rosterAgents(rows)).toMatchObject([
{ id: 'child-1', label: 'list_directory', state: 'working' }
])
// Four wire deliveries (two items, each sent twice) collapse to ONE roster
// row, and none of the gray `codex · item:subAgentActivity` rows survive.
const providerFrameKinds = rows.flatMap((row) =>
row.body.kind === 'status' && row.body.providerFrame ? [row.body.providerFrame.kind] : []
)
expect(providerFrameKinds).toEqual([])
expect(rows.filter((row) => row.key.startsWith('orca:codex-subagents'))).toHaveLength(1)
})
// The roster claims the item, but claiming it must not take the turn tail with
// it: the activity table is reached only through the publish arm, so a bare
// return leaves the tail stuck on whatever the previous frame said.
it('still publishes the turn tail for an item the roster claims', () => {
const { translator, activities } = harness()
translator.handle(notification('turn/started', { turn: { id: TURN_ID } }))
activities.length = 0
deliverActivity(translator, subagentItem('started', 'child-1', '/root/read'))
expect(activities.at(-1)).toEqual({
turnId: TURN_ID,
text: 'Coordinating with another agent'
})
})
it('consumes thread/tokenUsage/updated instead of swallowing it as chrome', () => {
const { translator, rows } = harness()
translator.handle(notification('turn/started', { turn: { id: TURN_ID } }))
deliverActivity(translator, subagentItem('started', 'child-1', '/root/read'))
translator.handle(
notification('thread/tokenUsage/updated', {
threadId: 'child-1',
tokenUsage: { total: { totalTokens: 40661 } }
})
)
expect(rosterAgents(rows)).toMatchObject([{ id: 'child-1', tokens: 40661 }])
})
// The QA scenario this row got wrong: three `spawn_agent` children were still
// running when a mid-turn correction ended their turn and opened a new one.
// They reported `completed` 57-87s later, so a turn boundary is a fact about
// the turn and never evidence that contact with a child was lost.
it('leaves children working when their turn ends and a newer turn opens', () => {
const { translator, rows } = harness()
translator.handle(notification('turn/started', { turn: { id: TURN_ID } }))
deliverActivity(translator, subagentItem('started', 'child-1', '/root/read_readme'))
deliverActivity(translator, subagentItem('started', 'child-2', '/root/read_package'))
translator.handle(notification('turn/completed', { turn: { id: TURN_ID } }))
translator.handle(notification('turn/started', { turn: { id: 'turn-2' } }))
expect(rosterAgents(rows)).toMatchObject([
{ id: 'child-1', state: 'working' },
{ id: 'child-2', state: 'working' }
])
// And the verdict a child reports after its turn ended still lands on the row.
deliverActivity(translator, subagentItem('completed', 'child-1', '/root/read_readme'))
expect(rosterAgents(rows)).toMatchObject([
{ id: 'child-1', state: 'completed' },
{ id: 'child-2', state: 'working' }
])
})
it('sweeps every group when the provider ends', () => {
const { translator, rows } = harness()
translator.handle(notification('turn/started', { turn: { id: TURN_ID } }))
deliverActivity(translator, subagentItem('started', 'child-1', '/root/read'))
translator.handle({
type: 'ended',
sessionId: SESSION_ID,
reason: 'provider exited',
cause: 'unexpected-exit',
fence: 1,
acquisitionGeneration: 'gen-1'
} as CodexStructuredSessionEvent)
expect(rosterAgents(rows)).toMatchObject([{ id: 'child-1', state: 'unverifiable' }])
})
})
@@ -1,4 +1,10 @@
import { createCodexProviderActivityReader } from '../native-chat/agent-session-wire/provider-frame-activity'
import {
CODEX_TOKEN_USAGE_METHOD,
readCodexNotificationThreadItem
} from './codex-subagent-activity'
import { CodexSubagentRoster } from './codex-subagent-roster'
import { readCodexThreadItem } from './codex-structured-item-translation'
import { CodexJournalGenericFrames } from './codex-structured-journal-generic-frames'
import { CodexJournalItems } from './codex-structured-journal-items'
import { CodexJournalPrompts } from './codex-structured-journal-prompts'
@@ -10,16 +16,12 @@ import {
} from './codex-structured-journal-contracts'
import {
settleCodexJournalSession,
settleCodexJournalTurn,
settleCodexOversizedNotification
settleCodexJournalTurn
} from './codex-structured-journal-settlement'
import { settleCodexOversizedNotificationFrame } from './codex-structured-journal-translation-frames'
import { restoreCodexJournalThread } from './codex-structured-journal-translation-restore'
import { CodexJournalActiveTurns } from './codex-structured-journal-translation-turn-state'
import { publishCodexTurnLifecycle } from './codex-structured-journal-translation-turns'
import {
readCodexJournalRecord,
readCodexJournalString
} from './codex-structured-journal-translation-values'
import { readCodexTurnId } from './codex-structured-thread-facts'
import type { CodexStructuredSessionEvent } from './codex-structured-session-adapter'
@@ -55,6 +57,11 @@ export function createCodexJournalTranslator(
const prompts = new CodexJournalPrompts(deps, (threadId, itemId) =>
items.detailFor(threadId, itemId)
)
const subagents = new CodexSubagentRoster({
sink: deps.sink,
primaryThreadId: () => deps.primaryThreadId?.() ?? null,
activeTurn: (threadId) => activeTurns.current(threadId)
})
const flushStreams = (): CodexJournalTranslationAdmission =>
items.streams.flush() ? CODEX_JOURNAL_ADMITTED : { accepted: false, reason: 'backpressure' }
let readActivity = createCodexProviderActivityReader()
@@ -118,6 +125,11 @@ export function createCodexJournalTranslator(
if (!admission.accepted) {
return admission
}
// No event will ever settle a child once the provider is gone.
const sweep = subagents.settleSession()
if (!sweep.accepted) {
return sweep
}
readActivity = createCodexProviderActivityReader()
deps.sink.setActivity?.(null)
items.activeItems.clear()
@@ -159,7 +171,30 @@ export function createCodexJournalTranslator(
if (event.method === 'turn/completed') {
return completeTurn(event)
}
if (event.method === CODEX_TOKEN_USAGE_METHOD) {
// Classified `status-chrome`, so the generic-frame path swallows it
// before the journal. The roster consumes it as a typed notification.
const admission = subagents.handleTokenUsage(event.params)
if (admission) {
return admission
}
}
if (event.method === 'item/started' || event.method === 'item/completed') {
const subagentItem = readCodexNotificationThreadItem(event.params, readCodexThreadItem)
// Null means the roster did not claim it; fall through to normal item
// handling. Returning here unconditionally swallows every other item.
const subagentAdmission = subagentItem
? subagents.handleItem({
threadId: event.threadId,
turnId: readCodexTurnId(event.params) ?? activeTurns.current(event.threadId),
item: subagentItem
})
: null
if (subagentAdmission) {
// Not a bare return: the roster claiming the item must not skip the
// turn-tail arm, which is the only publisher of its activity copy.
return publishActivity(event, subagentAdmission)
}
const translated = items.handle(event)
return publishActivity(
event,
@@ -186,30 +221,25 @@ export function createCodexJournalTranslator(
items.dispose()
prompts.dispose()
genericFrames.dispose()
subagents.dispose()
activeTurns.clear()
}
}
/** Settles the item a notification the transport refused to carry left
* mid-flight; null when the frame is not one. */
function settleOversizedNotification(event: {
sessionId: string
threadId: string
kind: string
payload: unknown
}): CodexJournalTranslationAdmission | null {
if (event.kind !== 'frame:oversized-notification') {
return null
}
const method = readCodexJournalString(readCodexJournalRecord(event.payload), 'method')
return method
? settleCodexOversizedNotification({
sessionId: event.sessionId,
threadId: event.threadId,
method,
sink: deps.sink,
streams: items.streams,
activeItems: items.activeItems
})
: null
return settleCodexOversizedNotificationFrame({
...event,
sink: deps.sink,
streams: items.streams,
activeItems: items.activeItems
})
}
function startTurn(event: {
@@ -255,6 +285,12 @@ export function createCodexJournalTranslator(
if (!turnId) {
return CODEX_JOURNAL_ADMITTED
}
// The roster is deliberately NOT swept here. `spawn_agent` children outlive
// the turn that spawned them and go on reporting into the same group, so a
// turn boundary is no evidence contact was lost — and `turn/completed` is
// the only turn-end notification Codex sends, so an abort cannot be told
// apart from a clean finish either. Only `settleSession` may write
// `unverifiable`.
const admission = settleCodexJournalTurn({
sink: deps.sink,
sessionId: event.sessionId,
@@ -44,7 +44,8 @@ function resolverFor(
store: { getRecord: () => value } as unknown as AgentSessionRecordStore,
resolveWorkspacePath,
resolveCommand: () => '/usr/local/bin/codex',
resolveRollout
resolveRollout,
isWindowsProcessStartTimeAvailable: () => true
})
}
@@ -68,7 +69,8 @@ describe('codex structured launch resolution', () => {
const resolveLaunch = createCodexStructuredLaunchResolver({
store: { getRecord: () => record() } as unknown as AgentSessionRecordStore,
resolveWorkspacePath: async () => String.raw`C:\workspaces\orca`,
resolveCommand: () => command
resolveCommand: () => command,
isWindowsProcessStartTimeAvailable: () => true
})
await expect(resolveLaunch({ identity: IDENTITY })).resolves.toMatchObject({
@@ -78,6 +80,22 @@ describe('codex structured launch resolution', () => {
})
})
it('fails closed before resolving a Windows launch without creation-time proof', async () => {
await withPlatform('win32', async () => {
const resolveWorkspacePath = vi.fn(async () => String.raw`C:\workspaces\orca`)
const resolveLaunch = createCodexStructuredLaunchResolver({
store: { getRecord: () => record() } as unknown as AgentSessionRecordStore,
resolveWorkspacePath,
isWindowsProcessStartTimeAvailable: () => false
})
await expect(resolveLaunch({ identity: IDENTITY })).rejects.toThrow(
'Windows process creation-time proof'
)
expect(resolveWorkspacePath).not.toHaveBeenCalled()
})
})
it('resumes the last thread this session actually proved, not one a caller names', async () => {
const launch = await resolverFor(
record({
@@ -13,6 +13,7 @@ import { resolveCodexCommand } from '../codex-cli/command'
import type { AgentSessionRecordStore } from '../runtime/agent-session-record-store'
import type { CodexStructuredLaunch } from './codex-structured-session-adapter'
import { resolvePinnedCodexRolloutProof } from './codex-tui-rollout-proof'
import { isWindowsProcessStartTimeAvailable } from '../windows/windows-process-table'
export type CodexStructuredLaunchResolverDeps = {
store: AgentSessionRecordStore
@@ -24,6 +25,8 @@ export type CodexStructuredLaunchResolverDeps = {
/** Fresh shell/configured environment for this spawn; never written to the session record. */
resolveEnvironment?: () => Promise<NodeJS.ProcessEnv>
resolveRollout?: typeof resolvePinnedCodexRolloutProof
/** Test seam for the host capability; production uses the native process table. */
isWindowsProcessStartTimeAvailable?: () => boolean
}
export function createCodexStructuredLaunchResolver(
@@ -46,6 +49,13 @@ export function createCodexStructuredLaunchResolver(
`codex structured sessions run on the local host, not ${location.executionHostId}`
)
}
// Refuse before resolving launch data; a PID alone cannot prove Windows ownership.
if (
process.platform === 'win32' &&
!(deps.isWindowsProcessStartTimeAvailable ?? isWindowsProcessStartTimeAvailable)()
) {
throw new Error('codex structured sessions require Windows process creation-time proof')
}
if (accountHome.variable !== 'CODEX_HOME') {
throw new Error(`codex sessions pin CODEX_HOME, not ${accountHome.variable}`)
}
@@ -0,0 +1,47 @@
import { describe, expect, it } from 'vitest'
import type { AgentSessionExecutionLocation } from '../../shared/agent-session-record'
import { supportsCodexStructuredLocation } from './codex-structured-location-support'
const LOCAL_WINDOWS_LOCATION: AgentSessionExecutionLocation = {
executionHostId: 'local',
wslDistro: null,
workspaceId: 'workspace-1',
workspaceKind: 'folder'
}
const WSL_WINDOWS_LOCATION: AgentSessionExecutionLocation = {
...LOCAL_WINDOWS_LOCATION,
wslDistro: 'Ubuntu'
}
function withPlatform<T>(platform: NodeJS.Platform, run: () => T): T {
const original = process.platform
Object.defineProperty(process, 'platform', { configurable: true, value: platform })
try {
return run()
} finally {
Object.defineProperty(process, 'platform', { configurable: true, value: original })
}
}
describe('Codex structured location support', () => {
it('uses the injected Windows identity capability for location admission', () => {
let proofAvailable = false
withPlatform('win32', () => {
expect(supportsCodexStructuredLocation(LOCAL_WINDOWS_LOCATION, () => proofAvailable)).toBe(
false
)
proofAvailable = true
expect(supportsCodexStructuredLocation(LOCAL_WINDOWS_LOCATION, () => proofAvailable)).toBe(
true
)
})
})
it('rejects WSL locations while retaining native folder support on Windows', () => {
withPlatform('win32', () => {
expect(supportsCodexStructuredLocation(WSL_WINDOWS_LOCATION, () => true)).toBe(false)
expect(supportsCodexStructuredLocation(LOCAL_WINDOWS_LOCATION, () => true)).toBe(true)
})
})
})
@@ -2,10 +2,14 @@ import { LOCAL_EXECUTION_HOST_ID } from '../../shared/execution-host'
import type { AgentSessionExecutionLocation } from '../../shared/agent-session-record'
import { isWindowsProcessStartTimeAvailable } from '../windows/windows-process-table'
export function supportsCodexStructuredLocation(location: AgentSessionExecutionLocation): boolean {
export function supportsCodexStructuredLocation(
location: AgentSessionExecutionLocation,
// Injected by the adapter, which owns this dep for every other Codex gate too.
hasWindowsProcessStartTimeProof: () => boolean = isWindowsProcessStartTimeAvailable
): boolean {
return (
location.executionHostId === LOCAL_EXECUTION_HOST_ID &&
location.wslDistro === null &&
(process.platform !== 'win32' || isWindowsProcessStartTimeAvailable())
(process.platform !== 'win32' || hasWindowsProcessStartTimeProof())
)
}
@@ -0,0 +1,334 @@
import { describe, expect, it, vi } from 'vitest'
import { CodexAppServerRequestError } from './codex-app-server-connection'
import type { CodexSession } from './codex-structured-session-state'
import { recoverCodexRewind, rewindCodexSession } from './codex-structured-rewind'
import { AGENT_SESSION_HISTORY_MAX_PAGE_BYTES } from '../native-chat/agent-session-wire/agent-session-history-page-bounds'
import { openCodexThread } from './codex-structured-thread-open'
function fixture(reverted = true) {
const request = vi.fn(async (method: string): Promise<unknown> => {
if (method === 'thread/read') {
return { thread: { id: 'thread', historyMode: 'paginated', status: { type: 'idle' } } }
}
if (method === 'thread/revert') {
reverted = true
return {
thread: { id: 'thread', turns: [] },
turnsBackwardsCursor: 'turn-cursor',
itemsBackwardsCursor: 'item-cursor'
}
}
if (method === 'thread/turns/list') {
return { data: [...(reverted ? [] : [{ id: 'drop' }]), { id: 'kept' }], nextCursor: null }
}
return {
data: [
{
turnId: 'kept',
item: {
id: 'item-1',
type: 'userMessage',
content: [{ type: 'text', text: 'kept prompt' }]
}
}
],
nextCursor: null
}
})
const session = {
connection: { request },
threadId: 'thread',
fence: 2,
ended: false,
historyMode: 'paginated',
activeTurnIds: new Set()
} as unknown as CodexSession
return { request, session }
}
describe('Codex rewind', () => {
it('recovers verified history from fresh cursors without repeating revert', async () => {
const { session, request } = fixture()
expect(await recoverCodexRewind(session, { fence: 2, beforeTurnId: 'drop' })).toMatchObject({
ok: true,
items: [{ body: { kind: 'message', blocks: [{ type: 'text', text: 'kept prompt' }] } }]
})
expect(request.mock.calls.map(([method]) => method)).toEqual([
'thread/read',
'thread/turns/list',
'thread/items/list'
])
for (const method of ['thread/turns/list', 'thread/items/list']) {
expect(request).toHaveBeenCalledWith(
method,
expect.objectContaining({ cursor: null, sortDirection: 'desc' }),
expect.anything()
)
}
})
it('recognizes an unapplied rewind from the still-present target', async () => {
const { session, request } = fixture()
const original = request.getMockImplementation()!
request.mockImplementation(async (method) =>
method === 'thread/turns/list'
? { data: [{ id: 'drop' }, { id: 'kept' }], nextCursor: null }
: original(method)
)
expect(await recoverCodexRewind(session, { fence: 2, beforeTurnId: 'drop' })).toEqual({
ok: false,
reason: 'provider-refused'
})
expect(request.mock.calls.some(([method]) => method === 'thread/revert')).toBe(false)
})
it.each(['cycle', 'pages', 'entries', 'bytes'] as const)(
'bounds recovery by %s and never returns partial history',
async (limit) => {
const { session, request } = fixture()
const original = request.getMockImplementation()!
let pages = 0
request.mockImplementation(async (method) => {
if (method !== 'thread/turns/list') {
return original(method)
}
pages++
if (limit === 'entries') {
return {
data: Array.from({ length: 1025 }, (_, i) => ({ id: String(i) })),
nextCursor: null
}
}
if (limit === 'bytes') {
return {
data: [],
padding: 'x'.repeat(AGENT_SESSION_HISTORY_MAX_PAGE_BYTES),
nextCursor: null
}
}
return { data: [], nextCursor: limit === 'cycle' ? 'repeated' : String(pages) }
})
await expect(recoverCodexRewind(session, { fence: 2, beforeTurnId: 'drop' })).rejects.toThrow(
'history-limit'
)
expect(pages).toBeLessThanOrEqual(100)
expect(request.mock.calls.some(([method]) => method === 'thread/revert')).toBe(false)
}
)
it('keeps an interrupted recovery retryable with read-only requests', async () => {
const { session, request } = fixture()
const original = request.getMockImplementation()!
request.mockImplementation(async (method) => {
if (method === 'thread/items/list') {
throw new Error('offline')
}
return original(method)
})
await expect(recoverCodexRewind(session, { fence: 2, beforeTurnId: 'drop' })).rejects.toThrow(
'offline'
)
request.mockImplementation(original)
expect(await recoverCodexRewind(session, { fence: 2, beforeTurnId: 'drop' })).toMatchObject({
ok: true
})
expect(request.mock.calls.some(([method]) => method === 'thread/revert')).toBe(false)
})
it('refuses activity arriving during recovery hydration', async () => {
const { session, request } = fixture()
const original = request.getMockImplementation()!
request.mockImplementation(async (method) => {
if (method === 'thread/items/list') {
session.activeTurnIds!.add('racing-turn')
}
return original(method)
})
expect(await recoverCodexRewind(session, { fence: 2, beforeTurnId: 'drop' })).toEqual({
ok: false,
reason: 'busy'
})
})
it('uses native revert and reads both retained indexes despite empty response turns', async () => {
const { session, request } = fixture(false)
const onPrepared = vi.fn<NonNullable<Parameters<typeof rewindCodexSession>[1]['onPrepared']>>(
async (items) => {
expect(items).toMatchObject([{ identity: { turnId: 'kept' } }])
expect(request.mock.calls.some(([method]) => method === 'thread/revert')).toBe(false)
}
)
expect(
await rewindCodexSession(session, { fence: 2, beforeTurnId: 'drop', onPrepared })
).toMatchObject({
ok: true,
items: [{ body: { kind: 'message' } }]
})
expect(onPrepared).toHaveBeenCalledTimes(1)
expect(request).toHaveBeenCalledWith(
'thread/revert',
{ threadId: 'thread', beforeTurnId: 'drop' },
{ timeoutMs: undefined }
)
expect(request).toHaveBeenCalledWith(
'thread/turns/list',
expect.objectContaining({ cursor: 'turn-cursor', sortDirection: 'desc' }),
expect.anything()
)
expect(request).toHaveBeenCalledWith(
'thread/items/list',
expect.objectContaining({ cursor: 'item-cursor', sortDirection: 'desc' }),
expect.anything()
)
})
it('refuses a known legacy thread before making a request', async () => {
const { session, request } = fixture()
session.historyMode = 'legacy'
expect(await rewindCodexSession(session, { fence: 2, beforeTurnId: 'drop' })).toEqual({
ok: false,
reason: 'history-not-paginated'
})
expect(request).not.toHaveBeenCalled()
})
it('refuses history exceeding hydration capacity before mutating the provider', async () => {
const { session, request } = fixture(false)
const original = request.getMockImplementation()!
const turns = Array.from({ length: 600 }, (_, i) => String(i))
request.mockImplementation(async (method) => {
if (method === 'thread/turns/list') {
return { data: [{ id: 'drop' }, ...turns.map((id) => ({ id }))], nextCursor: null }
}
if (method === 'thread/items/list') {
return {
data: turns.map((turnId) => ({
turnId,
item: { id: turnId, type: 'userMessage', content: [{ type: 'text', text: 'x' }] }
})),
nextCursor: null
}
}
return original(method)
})
const onReverted = vi.fn()
expect(
await rewindCodexSession(session, { fence: 2, beforeTurnId: 'drop', onReverted })
).toEqual({ ok: false, reason: 'history-limit' })
expect(onReverted).not.toHaveBeenCalled()
expect(request.mock.calls.some(([method]) => method === 'thread/revert')).toBe(false)
})
it('refuses a missing target before mutation', async () => {
const { session, request } = fixture()
expect(await rewindCodexSession(session, { fence: 2, beforeTurnId: 'missing' })).toEqual({
ok: false,
reason: 'invalid-target'
})
expect(request.mock.calls.some(([method]) => method === 'thread/revert')).toBe(false)
})
it('rechecks provider idleness after preflight hydration', async () => {
const { session, request } = fixture(false)
const original = request.getMockImplementation()!
let reads = 0
request.mockImplementation(async (method) => {
if (method === 'thread/read' && ++reads === 2) {
return { thread: { id: 'thread', status: { type: 'active' } } }
}
return original(method)
})
expect(await rewindCodexSession(session, { fence: 2, beforeTurnId: 'drop' })).toEqual({
ok: false,
reason: 'busy'
})
expect(request.mock.calls.some(([method]) => method === 'thread/revert')).toBe(false)
})
it('maps native legacy refusal without exposing provider text or falling back', async () => {
const { session, request } = fixture(false)
const original = request.getMockImplementation()!
request.mockImplementation(async (method) => {
if (method === 'thread/read') {
return { thread: { id: 'thread', status: { type: 'idle' } } }
}
if (method !== 'thread/revert') {
return original(method)
}
throw new CodexAppServerRequestError(
'thread/revert',
-32600,
'thread/revert only supports paginated threads'
)
})
expect(await rewindCodexSession(session, { fence: 2, beforeTurnId: 'drop' })).toEqual({
ok: false,
reason: 'history-not-paginated'
})
expect(request.mock.calls.map(([method]) => method)).toEqual([
'thread/read',
'thread/turns/list',
'thread/items/list',
'thread/read',
'thread/revert'
])
})
it('refuses activity arriving during the preflight await', async () => {
const { session, request } = fixture()
request.mockImplementationOnce(async () => {
session.activeTurnIds!.add('racing-turn')
return { thread: { id: 'thread', status: { type: 'idle' } } }
})
expect(await rewindCodexSession(session, { fence: 2, beforeTurnId: 'drop' })).toEqual({
ok: false,
reason: 'busy'
})
expect(request).toHaveBeenCalledTimes(1)
})
it('treats hydration failure after revert as unknown and never retries revert', async () => {
const { session, request } = fixture(false)
const original = request.getMockImplementation()!
let reverted = false
request.mockImplementation(async (method) => {
if (method === 'thread/revert') {
reverted = true
}
if (method === 'thread/items/list' && reverted) {
throw new Error('offline')
}
return original(method)
})
await expect(rewindCodexSession(session, { fence: 2, beforeTurnId: 'drop' })).rejects.toThrow(
'offline'
)
expect(request.mock.calls.filter(([method]) => method === 'thread/revert')).toHaveLength(1)
})
it('captures history mode at both start and resume without changing defaults', async () => {
for (const resumeThreadId of [null, 'thread']) {
const request = vi.fn(async (_method: string, _params?: unknown) => ({
thread: { id: 'thread', historyMode: 'legacy' }
}))
expect(
await openCodexThread({ request }, { cwd: '/workspace', resumeThreadId }, 10)
).toMatchObject({ historyMode: 'legacy' })
expect(request.mock.calls[0]?.[1]).not.toHaveProperty('historyMode')
}
})
it('rejects post-revert history missing an item within a retained turn', async () => {
const { session, request } = fixture(false)
const original = request.getMockImplementation()!
let reverted = false
request.mockImplementation(async (method) => {
if (method === 'thread/revert') {
reverted = true
}
if (method === 'thread/items/list' && !reverted) {
return {
data: [2, 1].map((i) => ({
turnId: 'kept',
item: {
id: `item-${i}`,
type: 'userMessage',
content: [{ type: 'text', text: `prompt ${i}` }]
}
})),
nextCursor: null
}
}
return original(method)
})
await expect(rewindCodexSession(session, { fence: 2, beforeTurnId: 'drop' })).rejects.toThrow(
'proof-mismatch'
)
})
})
+309
View File
@@ -0,0 +1,309 @@
import { readCodexThreadId, readCodexTurnId } from './codex-structured-thread-facts'
import { agentJournalItemKey } from '../../shared/agent-session-journal-item-key'
import type { StructuredAgentSessionAdapter } from '../native-chat/agent-session-wire/structured-agent-session-adapter'
import { createCodexJournalTranslator } from './codex-structured-journal-translation'
import { CODEX_RESTORE_MAX_OPERATIONS } from './codex-structured-journal-translation-restore'
import type {
AgentJournalItemBody,
AgentJournalItemIdentity
} from '../../shared/agent-session-journal-types'
import { AGENT_SESSION_HISTORY_MAX_LIMIT } from '../../shared/agent-session-wire'
import { AGENT_SESSION_HISTORY_MAX_PAGE_BYTES } from '../native-chat/agent-session-wire/agent-session-history-page-bounds'
import { isCodexAppServerRequestError } from './codex-app-server-connection'
import type { CodexSession } from './codex-structured-session-state'
const MAX_PAGES = 100
const MAX_ENTRIES = CODEX_RESTORE_MAX_OPERATIONS
class CodexRewindTargetRetainedError extends Error {}
class CodexRewindTargetMissingError extends Error {}
function record(value: unknown): Record<string, unknown> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new Error('agent_session_rewind:invalid-provider-response')
}
return value as Record<string, unknown>
}
function cursor(value: unknown): string | null {
if (value === null || (typeof value === 'string' && value.length > 0)) {
return value
}
throw new Error('agent_session_rewind:invalid-provider-cursor')
}
/** Read both indexes to completion before accepting the retained history. */
export async function verifyCodexRevertedHistory(
session: Pick<CodexSession, 'connection' | 'threadId'>,
reply: Record<string, unknown>,
beforeTurnId: string,
timeoutMs?: number,
targetPresence: 'absent' | 'present' = 'absent'
): Promise<{ identity: AgentJournalItemIdentity; body: AgentJournalItemBody }[]> {
let bytes = 0
let entries = 0
const turns = new Map<string, { id: string; items: unknown[] }>()
for (const [method, firstCursor] of [
['thread/turns/list', cursor(reply.turnsBackwardsCursor)],
['thread/items/list', cursor(reply.itemsBackwardsCursor)]
] as const) {
let next = firstCursor
const seen = new Set<string>()
for (let page = 0; ; page += 1) {
if (page >= MAX_PAGES || (next !== null && seen.has(next))) {
throw new Error('agent_session_rewind:history-limit')
}
if (next !== null) {
seen.add(next)
}
const result = record(
await session.connection.request(
method,
{
threadId: session.threadId,
cursor: next,
sortDirection: 'desc',
limit: AGENT_SESSION_HISTORY_MAX_LIMIT
},
{ timeoutMs }
)
)
if (!Array.isArray(result.data)) {
throw new Error('agent_session_rewind:invalid-provider-page')
}
bytes += Buffer.byteLength(JSON.stringify(result), 'utf8')
entries += result.data.length
if (bytes > AGENT_SESSION_HISTORY_MAX_PAGE_BYTES || entries > MAX_ENTRIES) {
throw new Error('agent_session_rewind:history-limit')
}
for (const raw of result.data) {
const item = record(raw)
const turnId = method === 'thread/turns/list' ? item.id : item.turnId
if (turnId === beforeTurnId && targetPresence === 'absent') {
throw new CodexRewindTargetRetainedError('agent_session_rewind:target-retained')
}
if (typeof turnId !== 'string' || !turnId) {
throw new Error('agent_session_rewind:invalid-retained-turn')
}
if (method === 'thread/turns/list') {
if (turns.has(turnId)) {
throw new Error('agent_session_rewind:duplicate-retained-turn')
}
turns.set(turnId, { id: turnId, items: [] })
} else {
const turn = turns.get(turnId)
if (!turn) {
throw new Error('agent_session_rewind:foreign-retained-item')
}
turn.items.push(record(item.item))
}
}
next = cursor(result.nextCursor)
if (next === null) {
break
}
}
}
if (targetPresence === 'present' && !turns.has(beforeTurnId)) {
throw new CodexRewindTargetMissingError('agent_session_rewind:target-missing')
}
const items = new Map<
string,
{ identity: AgentJournalItemIdentity; body: AgentJournalItemBody }
>()
const translator = createCodexJournalTranslator({
sink: {
appendItem: (identity, body) => {
items.set(agentJournalItemKey(identity), { identity, body })
},
appendTombstone: (identity) => {
items.delete(agentJournalItemKey(identity))
},
publish: () => {}
},
primaryThreadId: () => session.threadId
})
try {
const chronological = [...turns.values()].toReversed()
const retained =
targetPresence === 'present'
? chronological.slice(
0,
chronological.findIndex((turn) => turn.id === beforeTurnId)
)
: chronological
const admission = translator.restoreThread(session.threadId, {
turns: retained.map((turn) => ({ ...turn, items: turn.items.toReversed() }))
})
if (!admission.accepted) {
throw new Error('agent_session_rewind:history-unreadable')
}
return [...items.values()]
} finally {
translator.dispose()
}
}
async function preflightCodexRewind(
session: CodexSession,
fence: number,
timeoutMs?: number
): Promise<
{ ok: true } | { ok: false; reason: 'invalid-target' | 'history-not-paginated' | 'busy' }
> {
if (session.fence !== fence || session.ended) {
return { ok: false, reason: 'invalid-target' }
}
if (session.historyMode === 'legacy') {
return { ok: false, reason: 'history-not-paginated' }
}
if (session.activeTurnIds?.size || session.dispatchPending) {
return { ok: false, reason: 'busy' }
}
const metadata = record(
await session.connection.request(
'thread/read',
{ threadId: session.threadId, includeTurns: false },
{ timeoutMs }
)
)
const thread = record(metadata.thread)
if (thread.id !== session.threadId) {
return { ok: false, reason: 'invalid-target' }
}
if (thread.historyMode === 'legacy') {
session.historyMode = 'legacy'
return { ok: false, reason: 'history-not-paginated' }
}
if (
record(thread.status).type !== 'idle' ||
session.activeTurnIds?.size ||
session.dispatchPending
) {
return { ok: false, reason: 'busy' }
}
if (session.fence !== fence || session.ended) {
return { ok: false, reason: 'invalid-target' }
}
return { ok: true }
}
export async function recoverCodexRewind(
session: CodexSession,
input: { fence: number; beforeTurnId: string },
timeoutMs?: number
): ReturnType<NonNullable<StructuredAgentSessionAdapter['recoverRewind']>> {
const admission = await preflightCodexRewind(session, input.fence, timeoutMs)
if (!admission.ok) {
return admission
}
try {
const items = await verifyCodexRevertedHistory(
session,
{ turnsBackwardsCursor: null, itemsBackwardsCursor: null },
input.beforeTurnId,
timeoutMs
)
if (session.fence !== input.fence || session.ended) {
return { ok: false, reason: 'invalid-target' }
}
if (session.activeTurnIds?.size || session.dispatchPending) {
return { ok: false, reason: 'busy' }
}
return { ok: true, items }
} catch (error) {
if (error instanceof CodexRewindTargetRetainedError) {
return { ok: false, reason: 'provider-refused' }
}
throw error
}
}
export async function rewindCodexSession(
session: CodexSession,
input: Omit<Parameters<NonNullable<StructuredAgentSessionAdapter['rewind']>>[0], 'sessionId'>,
timeoutMs?: number
): ReturnType<NonNullable<StructuredAgentSessionAdapter['rewind']>> {
const admission = await preflightCodexRewind(session, input.fence, timeoutMs)
if (!admission.ok) {
return admission
}
let expectedItems: Set<string>
try {
const retained = await verifyCodexRevertedHistory(
session,
{ turnsBackwardsCursor: null, itemsBackwardsCursor: null },
input.beforeTurnId,
timeoutMs,
'present'
)
expectedItems = new Set(retained.map(({ identity }) => agentJournalItemKey(identity)))
await input.onPrepared?.(retained)
} catch (error) {
return {
ok: false,
reason:
error instanceof CodexRewindTargetMissingError
? 'invalid-target'
: error instanceof Error && error.message === 'agent_session_rewind:history-limit'
? 'history-limit'
: 'provider-refused'
}
}
const current = await preflightCodexRewind(session, input.fence, timeoutMs)
if (!current.ok) {
return current
}
let result: unknown
try {
result = await session.connection.request(
'thread/revert',
{
threadId: session.threadId,
beforeTurnId: input.beforeTurnId
},
{ timeoutMs }
)
} catch (error) {
if (isCodexAppServerRequestError(error)) {
if (error.message === 'thread/revert only supports paginated threads') {
session.historyMode = 'legacy'
return { ok: false, reason: 'history-not-paginated' }
}
if (error.code === -32601) {
return { ok: false, reason: 'unsupported' }
}
}
throw error
}
const reply = record(result)
if (record(reply.thread).id !== session.threadId) {
throw new Error('agent_session_rewind:foreign-thread')
}
await input.onReverted?.()
const items = await verifyCodexRevertedHistory(session, reply, input.beforeTurnId, timeoutMs)
if (
items.length !== expectedItems.size ||
items.some(({ identity }) => !expectedItems.has(agentJournalItemKey(identity)))
) {
throw new Error('agent_session_rewind:proof-mismatch')
}
return { ok: true, items }
}
export function observeCodexRewindActivity(
session: CodexSession,
method: string,
params: unknown
): void {
if ((readCodexThreadId(params) ?? session.threadId) !== session.threadId) {
return
}
const turnId = readCodexTurnId(params)
if (turnId && method === 'turn/started') {
session.activeTurnIds?.add(turnId)
}
if (turnId && method === 'turn/completed') {
session.activeTurnIds?.delete(turnId)
}
}
@@ -193,6 +193,8 @@ export async function acquireCodexStructuredSession(input: {
...codexSessionLifecycle(acquireInput.fence, acquired.acquisitionGeneration as string),
threadId: opened.threadId,
historyPath: opened.historyPath,
historyMode: opened.historyMode,
activeTurnIds: new Set(),
prompts: acquisition.prompts,
options: restoredCodexSessionOptions(acquireInput.options),
reportedOptions: reportedCodexThreadOptions(opened),
@@ -1,3 +1,4 @@
import * as codexRewind from './codex-structured-rewind'
import type {
AgentJournalMessageItem,
AgentSessionJournalIdentity
@@ -84,7 +85,8 @@ export class CodexStructuredSessionAdapter implements StructuredAgentSessionAdap
})
}
supportsLocation = supportsCodexStructuredLocation
supportsLocation = (location: Parameters<typeof supportsCodexStructuredLocation>[0]): boolean =>
supportsCodexStructuredLocation(location, this.deps.isWindowsProcessStartTimeAvailable)
acquire = (input: StructuredAgentSessionAcquireInput): Promise<AgentSessionAcquisition> =>
acquireCodexStructuredSession({
@@ -128,6 +130,7 @@ export class CodexStructuredSessionAdapter implements StructuredAgentSessionAdap
method: string,
params: unknown
): CodexJournalTranslationAdmission {
codexRewind.observeCodexRewindActivity(session, method, params)
if (this.turnCancellation.handleNotification(sessionId, session, method, params)) {
return { accepted: true }
}
@@ -188,8 +191,13 @@ export class CodexStructuredSessionAdapter implements StructuredAgentSessionAdap
fence: number
}): Promise<AgentSessionDispatchOutcome> {
const session = this.session(input.sessionId)
await this.turnCancellation.captureBaseline(session)
return dispatchCodexTurn(session, input, this.deps.requestTimeoutMs)
session.dispatchPending = true
try {
await this.turnCancellation.captureBaseline(session)
return await dispatchCodexTurn(session, input, this.deps.requestTimeoutMs)
} finally {
session.dispatchPending = false
}
}
async cancelTurn(input: {
@@ -208,6 +216,17 @@ export class CodexStructuredSessionAdapter implements StructuredAgentSessionAdap
input
) => this.backgroundTerminals.stop(input)
rewindSupport: NonNullable<StructuredAgentSessionAdapter['rewindSupport']> = (sessionId) =>
this.sessions.get(sessionId)?.historyMode === 'legacy'
? { supported: false, reason: 'history-not-paginated' }
: { supported: true }
rewind: NonNullable<StructuredAgentSessionAdapter['rewind']> = (input) =>
codexRewind.rewindCodexSession(this.session(input.sessionId), input, this.deps.requestTimeoutMs)
recoverRewind: NonNullable<StructuredAgentSessionAdapter['recoverRewind']> = (input) =>
codexRewind.recoverCodexRewind(this.session(input.sessionId), input, this.deps.requestTimeoutMs)
compact: NonNullable<StructuredAgentSessionAdapter['compact']> = (input) => {
const session = this.session(input.sessionId)
return this.compactions.run(
@@ -43,6 +43,8 @@ export type CodexStructuredSessionAdapterDeps = {
resolveLaunch: (input: {
identity: AgentSessionJournalIdentity
}) => Promise<CodexStructuredLaunch>
/** Host capability seam; production uses the native Windows process table. */
isWindowsProcessStartTimeAvailable?: () => boolean
onEvent?: (event: CodexStructuredSessionEvent) => void
onBackgroundTasksChanged?: (
sessionId: string,
@@ -69,6 +71,9 @@ export type CodexSession = {
acquisitionGeneration: string
threadId: string
historyPath: string | null
historyMode?: 'legacy' | 'paginated'
activeTurnIds?: Set<string>
dispatchPending?: boolean
prompts: CodexAcquisitionWindow['prompts']
options: Map<string, string>
reportedOptions: { model?: string; effort?: string }
@@ -16,6 +16,7 @@ export type CodexOpenedThread = {
thread?: Record<string, unknown>
/** Rollout file Codex named, when it named one. */
historyPath: string | null
historyMode?: 'legacy' | 'paginated'
model?: string
effort?: string
}
@@ -92,6 +93,9 @@ export async function openCodexThread(
threadId,
thread,
historyPath: readCodexThreadPath(opened),
...(thread.historyMode === 'legacy' || thread.historyMode === 'paginated'
? { historyMode: thread.historyMode }
: {}),
...(model ? { model } : {}),
...(effort ? { effort } : {})
}
+140
View File
@@ -0,0 +1,140 @@
// Reading Codex's subagent wire shapes.
//
// Established by a live probe against `codex app-server` 0.152.1, not inferred:
// * `subAgentActivity` items carry `{kind, agentThreadId, agentPath}`, and each
// one arrives TWICE — via `item/started` and again via `item/completed`.
// * `agentPath` is a tree path (`/root`, `/root/list_directory`); the trailing
// segment is a semantic task name and the only label available. There is no
// `thread/started` for a child, so nickname/role/depth do not exist.
// * `agentsStates` on `collabAgentToolCall` arrived empty (`{}`) throughout the
// probe, so nothing here reads it — state comes from `kind` alone.
// * `thread/tokenUsage/updated` reports a per-thread RUNNING TOTAL, so the
// latest frame replaces the previous one — it is never accumulated.
import type { NativeChatSubagentState } from '../../shared/native-chat-types'
import type { CodexThreadItem } from './codex-structured-item-translation'
export const CODEX_SUBAGENT_ITEM_TYPE = 'subAgentActivity'
export const CODEX_TOKEN_USAGE_METHOD = 'thread/tokenUsage/updated'
export type CodexSubagentActivity = {
kind: string
agentThreadId: string
agentPath: string | null
}
function nonEmptyString(value: unknown): string | null {
return typeof value === 'string' && value.length > 0 ? value : null
}
function record(value: unknown): Record<string, unknown> | null {
return typeof value === 'object' && value !== null && !Array.isArray(value)
? (value as Record<string, unknown>)
: null
}
export function readCodexSubagentActivity(item: CodexThreadItem): CodexSubagentActivity | null {
if (item.type !== CODEX_SUBAGENT_ITEM_TYPE) {
return null
}
const agentThreadId = nonEmptyString(item.agentThreadId)
if (!agentThreadId) {
return null
}
return {
kind: nonEmptyString(item.kind) ?? '',
agentThreadId,
agentPath: nonEmptyString(item.agentPath)
}
}
/**
* The state a `kind` implies for the child it names.
*
* An unrecognized kind means "this child exists and reported something we
* cannot classify" — `working`, which the session sweep will later settle to
* `unverifiable` if nothing better ever arrives. Claiming a terminal state from
* an unknown kind would assert an outcome the wire never gave us.
*/
export function codexSubagentStateForKind(kind: string): NativeChatSubagentState {
if (kind === 'completed') {
return 'completed'
}
if (kind === 'interrupted') {
return 'stopped'
}
return 'working'
}
/** Path segments, empty ones dropped: `/root/list_directory` → 2 segments. */
export function codexSubagentPathSegments(agentPath: string | null): string[] {
return agentPath === null ? [] : agentPath.split('/').filter((part) => part.length > 0)
}
/** The one path segment that names the parent turn itself rather than a child.
* Compared after the same normalization the label uses, not against the raw
* string: `/root/` and `/root//` are the same node as `/root`, and a check that
* disagreed with `codexSubagentPathSegments` would let one path be both the
* turn and a child of it — a phantom row labelled `root` inflating the group.
* Only this segment is the root; `/morpheus` is single-segment too but IS a
* child. */
const CODEX_ROOT_AGENT_SEGMENT = 'root'
/**
* Whether an activity item describes the ROOT of the agent tree rather than a
* spawned child. Counting the root would make the parent turn report itself as
* its own subagent.
*
* A path-less item cannot be placed in the tree at all, so it is treated as a
* child: dropping it would lose a real spawn, while an extra row is visible and
* self-correcting.
*/
export function isCodexRootAgentActivity(activity: CodexSubagentActivity): boolean {
const segments = codexSubagentPathSegments(activity.agentPath)
return segments.length === 1 && segments[0] === CODEX_ROOT_AGENT_SEGMENT
}
/** Row label: the agent path's trailing segment, trimmed. A segment with nothing
* visible in it survives the empty-segment filter but would draw a nameless row,
* so it reads as no label and the caller's placeholder takes over. Trimmed
* because the caller keys its collision ordinals on this string: ` read ` and
* `read` render identically and must therefore collide. */
export function codexSubagentLabel(activity: CodexSubagentActivity): string | null {
const trailing = codexSubagentPathSegments(activity.agentPath).at(-1)?.trim()
return trailing !== undefined && trailing.length > 0 ? trailing : null
}
export type CodexThreadTokenTotal = { threadId: string; totalTokens: number }
/** `{threadId, tokenUsage: {total: {totalTokens}}}`. Older builds put the total
* on the envelope, so both shapes are accepted. */
export function readCodexThreadTokenTotal(params: unknown): CodexThreadTokenTotal | null {
const root = record(params)
if (!root) {
return null
}
const threadId = nonEmptyString(root.threadId) ?? nonEmptyString(record(root.thread)?.id)
if (!threadId) {
return null
}
const usage = record(root.tokenUsage)
const total = record(usage?.total)?.totalTokens ?? usage?.totalTokens ?? root.totalTokens
return typeof total === 'number' && Number.isFinite(total) && total >= 0
? { threadId, totalTokens: total }
: null
}
/** Pull the `subAgentActivity` item out of a raw notification payload.
*
* Lives beside the readers rather than in the translator: the translator's job
* is routing, and this is the shape check that decides whether a frame is one
* of ours at all. Returns null for anything that is not a thread item, which is
* the translator's signal to keep looking. */
export function readCodexNotificationThreadItem(
params: unknown,
read: (value: unknown) => CodexThreadItem | null
): CodexThreadItem | null {
const record =
typeof params === 'object' && params !== null ? (params as Record<string, unknown>) : {}
return read(record.item)
}
@@ -0,0 +1,769 @@
import { describe, expect, it } from 'vitest'
import { isAdmissibleAgentJournalItemBody } from '../../shared/agent-session-journal-schemas'
import type {
AgentJournalItemBody,
AgentJournalItemIdentity
} from '../../shared/agent-session-journal-types'
import { MAX_SUBAGENT_FIELD_CHARS } from '../../shared/native-chat-subagent-summary'
import { isSubagentGroupBlock, type NativeChatSubagentEntry } from '../../shared/native-chat-types'
import type { StructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink'
import {
CodexSubagentRoster,
codexSubagentGroupIdentity,
codexSubagentGroupId
} from './codex-subagent-roster'
import type { CodexThreadItem } from './codex-structured-item-translation'
import {
MAX_CODEX_SUBAGENT_GROUPS,
MAX_CODEX_SUBAGENTS_PER_GROUP,
MAX_CODEX_TOKEN_USAGE_THREADS
} from './codex-structured-journal-limits'
const THREAD = 'thread-parent'
const TURN = 'turn-1'
type Appended = { identity: AgentJournalItemIdentity; body: AgentJournalItemBody }
function createHarness(options: { threadId?: string | null } = {}): {
roster: CodexSubagentRoster
appended: Appended[]
agents: () => NativeChatSubagentEntry[]
latest: () => Appended | undefined
} {
const appended: Appended[] = []
let clock = 1_000
const sink: StructuredAgentSessionEventSink = {
appendItem: () => {},
appendTombstone: () => {},
publish: () => {},
tryAppendItem: (identity, body) => {
appended.push({ identity, body })
return { accepted: true }
},
tryPublish: () => ({ accepted: true })
}
const roster = new CodexSubagentRoster({
sink,
primaryThreadId: () => (options.threadId === undefined ? THREAD : options.threadId),
activeTurn: () => TURN,
now: () => (clock += 1)
})
const agents = (): NativeChatSubagentEntry[] => {
const body = appended.at(-1)?.body
if (!body || body.kind !== 'message') {
return []
}
const block = body.blocks.find(isSubagentGroupBlock)
return block ? block.agents : []
}
return { roster, appended, agents, latest: () => appended.at(-1) }
}
function latestIdentity(appended: Appended[]): AgentJournalItemIdentity | undefined {
return appended.at(-1)?.identity
}
function activity(input: {
id?: string
kind: string
agentThreadId: string
agentPath: string | null
}): CodexThreadItem {
return {
type: 'subAgentActivity',
id: input.id ?? `item-${input.agentThreadId}-${input.kind}`,
kind: input.kind,
agentThreadId: input.agentThreadId,
agentPath: input.agentPath
}
}
function deliver(
roster: CodexSubagentRoster,
item: CodexThreadItem,
turnId: string | null = TURN
): void {
// Every activity item reaches the wire twice: item/started, then item/completed.
roster.handleItem({ threadId: THREAD, turnId, item })
roster.handleItem({ threadId: THREAD, turnId, item })
}
/**
* A sink that coalesces the way the real queue does: by `coalescingKey` ALONE,
* with no op-kind check, and only draining when released. A fake that ignores
* the key cannot see an append being spliced out by its own publish.
*/
function createCoalescingHarness(): {
roster: CodexSubagentRoster
appended: Appended[]
drain: () => void
} {
const appended: Appended[] = []
const queue: { key?: string; run: () => void }[] = []
let clock = 1_000
const submit = (key: string | undefined, run: () => void): void => {
const at = key === undefined ? -1 : queue.findIndex((queued) => queued.key === key)
if (at >= 0) {
queue.splice(at, 1)
}
queue.push(key === undefined ? { run } : { key, run })
}
const sink: StructuredAgentSessionEventSink = {
appendItem: () => {},
appendTombstone: () => {},
publish: () => {},
tryAppendItem: (identity, body, options) => {
submit(options?.coalescingKey, () => appended.push({ identity, body }))
return { accepted: true }
},
tryPublish: (options) => {
submit(options?.coalescingKey ?? 'publish', () => {})
return { accepted: true }
}
}
const roster = new CodexSubagentRoster({
sink,
primaryThreadId: () => THREAD,
activeTurn: () => TURN,
now: () => (clock += 1)
})
return {
roster,
appended,
drain: () => {
while (queue.length > 0) {
queue.shift()?.run()
}
}
}
}
describe('CodexSubagentRoster', () => {
it('does not let its own publish evict the still-queued roster append', () => {
const { roster, appended, drain } = createCoalescingHarness()
deliver(
roster,
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
)
drain()
// Sharing the append's coalescing key with the publish spliced the append
// out of the queue, and `lastSerialized` then suppressed every retry.
expect(appended).toHaveLength(1)
})
it('counts a /morpheus agent as a child — only /root is the turn itself', () => {
const { roster, agents } = createHarness()
deliver(roster, activity({ kind: 'started', agentThreadId: 'child-m', agentPath: '/morpheus' }))
expect(agents()).toMatchObject([{ id: 'child-m', label: 'morpheus', state: 'working' }])
})
// `codexSubagentPathSegments` already defines what a path means for the label,
// and the root check has to agree with it: a path that normalizes to the same
// node must classify the same way, or one string is both the turn itself and a
// child of it — a phantom row labelled `root` inflating the group by one.
it('reads a root path with a trailing or doubled separator as the turn itself', () => {
for (const agentPath of ['/root/', '/root//', '//root']) {
const { roster, appended } = createHarness()
deliver(roster, activity({ kind: 'started', agentThreadId: THREAD, agentPath }))
expect(appended).toEqual([])
}
})
it('keeps a doubled separator inside a child path off the label', () => {
const { roster, agents } = createHarness()
deliver(
roster,
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root//read/' })
)
expect(agents()).toMatchObject([{ id: 'child-1', label: 'read' }])
})
// An all-whitespace trailing segment survives the empty-segment filter and
// would draw a row with no visible name at all.
it('falls back to the placeholder when the trailing segment has nothing to show', () => {
const { roster, agents } = createHarness()
deliver(roster, activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/ ' }))
expect(agents()).toMatchObject([{ id: 'child-1', label: 'subagent' }])
})
// The collision ordinal keys on the label, so two segments that render
// identically must collide rather than both draw as `read`.
it('collides labels that differ only in surrounding whitespace', () => {
const { roster, agents } = createHarness()
deliver(
roster,
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
)
deliver(
roster,
activity({ kind: 'started', agentThreadId: 'child-2', agentPath: '/root/ read ' })
)
expect(agents().map((agent) => agent.label)).toEqual(['read', 'read 2'])
})
it('ignores the root node so a turn is not its own subagent', () => {
const { roster, appended } = createHarness()
deliver(roster, activity({ kind: 'started', agentThreadId: THREAD, agentPath: '/root' }))
expect(appended).toEqual([])
})
it('writes an admissible journal body carrying a plain-text fallback block', () => {
const { roster, latest } = createHarness()
deliver(
roster,
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/list_directory' })
)
const body = latest()?.body
expect(body?.kind).toBe('message')
expect(isAdmissibleAgentJournalItemBody(body)).toBe(true)
expect(body?.kind === 'message' ? body.blocks.map((block) => block.type) : []).toEqual([
'text',
'subagent-group'
])
expect(
body?.kind === 'message' && body.blocks[0]?.type === 'text' ? body.blocks[0].text : ''
).toBe('Kicked off 1 subagent')
})
it('keys the durable identity by the parent turn so a revision lands on one row', () => {
const { roster, appended } = createHarness()
deliver(
roster,
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
)
deliver(
roster,
activity({ kind: 'completed', agentThreadId: 'child-1', agentPath: '/root/read' })
)
const expected = codexSubagentGroupIdentity(codexSubagentGroupId(THREAD, TURN))
expect(new Set(appended.map((entry) => JSON.stringify(entry.identity)))).toEqual(
new Set([JSON.stringify(expected)])
)
})
it('rule 1 — a duplicate delivery writes no second revision', () => {
const { roster, appended } = createHarness()
deliver(
roster,
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
)
expect(appended).toHaveLength(1)
})
it('rule 2 — a first event of any kind creates the entry in the state it implies', () => {
const { roster, agents } = createHarness()
deliver(
roster,
activity({ kind: 'completed', agentThreadId: 'child-late', agentPath: '/root/search' })
)
expect(agents()).toMatchObject([{ id: 'child-late', label: 'search', state: 'completed' }])
})
it('rule 3 — a terminal state latches against a late or duplicate start', () => {
const { roster, agents } = createHarness()
deliver(
roster,
activity({ kind: 'completed', agentThreadId: 'child-1', agentPath: '/root/read' })
)
deliver(
roster,
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
)
deliver(
roster,
activity({ kind: 'interacted', agentThreadId: 'child-1', agentPath: '/root/read' })
)
expect(agents()).toMatchObject([{ state: 'completed' }])
})
it('rule 4 — the session sweep settles a lost child as unverifiable, not exited', () => {
const { roster, agents } = createHarness()
deliver(
roster,
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
)
deliver(
roster,
activity({ kind: 'completed', agentThreadId: 'child-2', agentPath: '/root/search' })
)
roster.settleSession()
expect(agents()).toMatchObject([
{ id: 'child-1', state: 'unverifiable' },
{ id: 'child-2', state: 'completed' }
])
})
it('lets a swept child still report what it actually did', () => {
const { roster, agents } = createHarness()
deliver(
roster,
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
)
roster.settleSession()
expect(agents()[0]?.state).toBe('unverifiable')
// Contact can return — a reconnected provider replays the child's own
// verdict. Latching the sweep would report a child that finished as one we
// never saw finish.
deliver(
roster,
activity({ kind: 'completed', agentThreadId: 'child-1', agentPath: '/root/read' })
)
expect(agents()[0]?.state).toBe('completed')
})
it('refuses to put a swept child back to working', () => {
const { roster, agents } = createHarness()
deliver(
roster,
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
)
roster.settleSession()
// A straggler progress tick after we gave up must not re-light the row.
deliver(
roster,
activity({ kind: 'interacted', agentThreadId: 'child-1', agentPath: '/root/read' })
)
expect(agents()[0]?.state).toBe('unverifiable')
})
it('keeps a real verdict when a later frame disagrees', () => {
const { roster, agents } = createHarness()
deliver(
roster,
activity({ kind: 'completed', agentThreadId: 'child-1', agentPath: '/root/read' })
)
deliver(
roster,
activity({ kind: 'interrupted', agentThreadId: 'child-1', agentPath: '/root/read' })
)
expect(agents()[0]?.state).toBe('completed')
})
it('rule 4 — the session sweep settles every group and never un-terminals one', () => {
const { roster, agents, appended } = createHarness()
deliver(
roster,
activity({ kind: 'interacted', agentThreadId: 'child-1', agentPath: '/root/read' })
)
roster.settleSession()
const afterFirstSweep = appended.length
roster.settleSession()
expect(agents()).toMatchObject([{ state: 'unverifiable' }])
expect(appended).toHaveLength(afterFirstSweep)
})
it('rule 5 — the whole roster is persisted in the carrier, not just a count', () => {
const { roster, agents } = createHarness()
deliver(
roster,
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
)
roster.handleTokenUsage({ threadId: 'child-1', tokenUsage: { total: { totalTokens: 40661 } } })
expect(agents()).toMatchObject([
{ id: 'child-1', label: 'read', state: 'working', tokens: 40661 }
])
})
it('rule 6 — the group id names the parent turn, or says there was none', () => {
const { roster, appended } = createHarness()
deliver(
roster,
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
)
deliver(
roster,
activity({ kind: 'started', agentThreadId: 'child-2', agentPath: '/root/search' }),
null
)
expect(appended.map((entry) => entry.identity)).toEqual([
{ provider: 'orca', clientMessageId: `codex-subagents:${THREAD}:${TURN}` },
{ provider: 'orca', clientMessageId: `codex-subagents:${THREAD}:outside-turn` }
])
})
it('disambiguates two children that share a trailing path segment', () => {
const { roster, agents } = createHarness()
deliver(
roster,
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
)
deliver(
roster,
activity({ kind: 'started', agentThreadId: 'child-2', agentPath: '/root/read' })
)
expect(agents().map((agent) => agent.label)).toEqual(['read', 'read 2'])
})
it('takes the latest token snapshot per child and never accumulates updates', () => {
const { roster, agents } = createHarness()
deliver(
roster,
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
)
roster.handleTokenUsage({ threadId: 'child-1', tokenUsage: { total: { totalTokens: 100 } } })
roster.handleTokenUsage({ threadId: 'child-1', tokenUsage: { total: { totalTokens: 250 } } })
expect(agents()).toMatchObject([{ tokens: 250 }])
})
it('retains a usage frame that arrives before the child is known', () => {
const { roster, agents } = createHarness()
roster.handleTokenUsage({ threadId: 'child-1', tokenUsage: { total: { totalTokens: 900 } } })
deliver(
roster,
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
)
expect(agents()).toMatchObject([{ tokens: 900 }])
})
it('never attributes the parent thread its own usage', () => {
const { roster, agents, appended } = createHarness()
deliver(
roster,
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
)
const beforeParentUsage = appended.length
roster.handleTokenUsage({ threadId: THREAD, tokenUsage: { total: { totalTokens: 26099 } } })
expect(appended).toHaveLength(beforeParentUsage)
expect(agents()).toHaveLength(1)
expect(agents()[0]).not.toHaveProperty('tokens')
})
// The row is durable and both readers clip these fields to the same cap, so
// writing more than that is bytes replayed on every reconnect and then thrown
// away. The marker is an ellipsis, not the tool-output truncation sentence:
// `id` is the roster key and the renderer's React key.
it('bounds the provider strings the roster row carries into the journal', () => {
const { roster, agents, latest } = createHarness()
const oversized = 'a'.repeat(20 * 1024)
deliver(
roster,
activity({ kind: 'started', agentThreadId: oversized, agentPath: `/root/${oversized}` })
)
const entry = agents()[0]
expect(entry?.label.length).toBeLessThanOrEqual(MAX_SUBAGENT_FIELD_CHARS)
expect(entry?.label).toMatch(/…~0$/)
expect(entry?.id.length).toBeLessThanOrEqual(MAX_SUBAGENT_FIELD_CHARS)
expect(entry?.id).toMatch(/…~0$/)
expect(JSON.stringify(latest()?.body)).not.toContain('output truncated')
expect(isAdmissibleAgentJournalItemBody(latest()?.body)).toBe(true)
})
// The clip cuts UTF-16 code units, so a boundary landing inside a surrogate
// pair left a LONE high surrogate in a durable row — malformed, and replaced
// with U+FFFD through any non-JSON UTF-8 hop.
it('never clips a provider string mid surrogate pair', () => {
const { roster, agents } = createHarness()
const astral = '😀'.repeat(400)
deliver(
roster,
activity({ kind: 'started', agentThreadId: astral, agentPath: `/root/${astral}` })
)
const entry = agents()[0]
expect(entry?.id.length).toBeLessThanOrEqual(MAX_SUBAGENT_FIELD_CHARS)
expect(Buffer.from(entry?.id ?? '', 'utf8').toString('utf8')).toBe(entry?.id)
expect(Buffer.from(entry?.label ?? '', 'utf8').toString('utf8')).toBe(entry?.label)
})
// The clip removes exactly the tail that told two children apart: `id` is the
// renderer's React key, and `claimLabel` writes its repeat ordinal at the end.
// Two clipped children collapsing to one key drew two rows under one identity.
it('keeps clipped ids and labels distinct between children', () => {
const { roster, agents } = createHarness()
const prefix = 'p'.repeat(MAX_SUBAGENT_FIELD_CHARS)
const sharedPath = `/root/${'q'.repeat(640)}`
deliver(
roster,
activity({ kind: 'started', agentThreadId: `${prefix}AAAA`, agentPath: sharedPath })
)
deliver(
roster,
activity({ kind: 'started', agentThreadId: `${prefix}BBBB`, agentPath: sharedPath })
)
const entries = agents()
expect(entries).toHaveLength(2)
expect(new Set(entries.map((agent) => agent.id)).size).toBe(2)
expect(new Set(entries.map((agent) => agent.label)).size).toBe(2)
for (const agent of entries) {
expect(agent.id.length).toBeLessThanOrEqual(MAX_SUBAGENT_FIELD_CHARS)
expect(agent.label.length).toBeLessThanOrEqual(MAX_SUBAGENT_FIELD_CHARS)
}
})
it('caps the children one spawn group admits', () => {
const { roster, agents, appended } = createHarness()
for (let index = 0; index < MAX_CODEX_SUBAGENTS_PER_GROUP; index++) {
deliver(
roster,
activity({ kind: 'started', agentThreadId: `child-${index}`, agentPath: '/root/read' })
)
}
const atCap = appended.length
deliver(
roster,
activity({ kind: 'started', agentThreadId: 'child-over-cap', agentPath: '/root/read' })
)
expect(agents()).toHaveLength(MAX_CODEX_SUBAGENTS_PER_GROUP)
expect(agents().map((agent) => agent.id)).not.toContain('child-over-cap')
// Refusing the child must not burn a revision either.
expect(appended).toHaveLength(atCap)
})
// The eviction is the KNOWN LIMITATION the module documents: `groups` is never
// seeded from the journal, so the evicted group's next child rebuilds its
// durable row from that one child. Pinned so the boundary cannot move silently.
it('caps live spawn groups, and an evicted group rebuilds its row from one child', () => {
const { roster, appended, agents } = createHarness()
for (let index = 0; index <= MAX_CODEX_SUBAGENT_GROUPS; index++) {
deliver(
roster,
activity({ kind: 'started', agentThreadId: `child-${index}`, agentPath: '/root/read' }),
`turn-${index}`
)
}
const evicted = codexSubagentGroupIdentity(codexSubagentGroupId(THREAD, 'turn-0'))
const rowsFor = (identity: AgentJournalItemIdentity): Appended[] =>
appended.filter((entry) => JSON.stringify(entry.identity) === JSON.stringify(identity))
expect(rowsFor(evicted)).toHaveLength(1)
deliver(
roster,
activity({ kind: 'started', agentThreadId: 'child-late', agentPath: '/root/search' }),
'turn-0'
)
expect(latestIdentity(appended)).toEqual(evicted)
expect(agents().map((agent) => agent.id)).toEqual(['child-late'])
})
it('keeps a token count a later thread-map eviction would otherwise retract', () => {
const { roster, agents } = createHarness()
deliver(
roster,
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
)
roster.handleTokenUsage({ threadId: 'child-1', tokenUsage: { total: { totalTokens: 4242 } } })
expect(agents()).toMatchObject([{ tokens: 4242 }])
for (let index = 0; index < MAX_CODEX_TOKEN_USAGE_THREADS; index++) {
roster.handleTokenUsage({
threadId: `other-${index}`,
tokenUsage: { total: { totalTokens: index } }
})
}
deliver(
roster,
activity({ kind: 'completed', agentThreadId: 'child-1', agentPath: '/root/read' })
)
expect(agents()).toMatchObject([{ state: 'completed', tokens: 4242 }])
})
it('caps retained usage threads, so a frame evicted before its child is dropped', () => {
const { roster, agents } = createHarness()
roster.handleTokenUsage({ threadId: 'child-1', tokenUsage: { total: { totalTokens: 900 } } })
for (let index = 0; index < MAX_CODEX_TOKEN_USAGE_THREADS; index++) {
roster.handleTokenUsage({
threadId: `other-${index}`,
tokenUsage: { total: { totalTokens: index } }
})
}
deliver(
roster,
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
)
expect(agents()[0]).not.toHaveProperty('tokens')
})
it('declines a payload that is not a subagent item or a usage frame', () => {
const { roster } = createHarness()
expect(
roster.handleItem({
threadId: THREAD,
turnId: TURN,
item: { type: 'commandExecution', id: 'item-9' }
})
).toBeNull()
expect(roster.handleTokenUsage({ threadId: 'child-1' })).toBeNull()
})
// A refusal must never advance the duplicate-suppression state: an identical
// replay would short-circuit and the revision would never be retried. The
// append and the publish are the two ways to be refused, so both are covered.
it.each([{ refuse: 'append' as const }, { refuse: 'publish' as const }])(
'retries the same revision after the $refuse is refused',
({ refuse }) => {
let refusing = true
const appended: Appended[] = []
const published: number[] = []
const refusal = { accepted: false, reason: 'backpressure' } as const
const roster = new CodexSubagentRoster({
sink: {
appendItem: () => {},
appendTombstone: () => {},
publish: () => {},
tryAppendItem: (identity, body) => {
if (refusing && refuse === 'append') {
return refusal
}
appended.push({ identity, body })
return { accepted: true }
},
tryPublish: () => {
if (refusing && refuse === 'publish') {
return refusal
}
published.push(1)
return { accepted: true }
}
},
primaryThreadId: () => THREAD,
activeTurn: () => TURN,
now: () => 1_000
})
const item = activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
expect(roster.handleItem({ threadId: THREAD, turnId: TURN, item })).toEqual(refusal)
// The wire redelivers the very same item; nothing about the roster changed,
// so only a cleared suppression state can get the revision out.
refusing = false
expect(roster.handleItem({ threadId: THREAD, turnId: TURN, item })).toEqual({
accepted: true
})
// The retry re-appends when the publish was the half that failed; the real
// queue coalesces those two by the group key into one journal write. What
// must not happen is the revision never being published at all.
expect(published).toHaveLength(1)
const body = appended.at(-1)?.body
expect(
body?.kind === 'message' ? body.blocks.filter(isSubagentGroupBlock) : []
).toMatchObject([{ agents: [{ id: 'child-1', state: 'working' }] }])
}
)
// The sweep is the last event a group ever gets. A refusal there, left
// unretried, strands the settled roster's final revision — the exact "row
// stays stale forever" this row exists to prevent.
it('republishes the settled roster when the sweep publish was refused', () => {
let refusing = false
const appended: Appended[] = []
const published: number[] = []
const roster = new CodexSubagentRoster({
sink: {
appendItem: () => {},
appendTombstone: () => {},
publish: () => {},
tryAppendItem: (identity, body) => {
appended.push({ identity, body })
return { accepted: true }
},
tryPublish: () => {
if (refusing) {
return { accepted: false, reason: 'backpressure' }
}
published.push(1)
return { accepted: true }
}
},
primaryThreadId: () => THREAD,
activeTurn: () => TURN,
now: () => 1_000
})
roster.handleItem({
threadId: THREAD,
turnId: TURN,
item: activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
})
const publishedBeforeSweep = published.length
refusing = true
expect(roster.settleSession()).toEqual({ accepted: false, reason: 'backpressure' })
// The retry sweep flips no state — every child already latched — so only a
// cleared suppression state can carry the unverifiable roster out.
refusing = false
expect(roster.settleSession()).toEqual({ accepted: true })
expect(published.length).toBe(publishedBeforeSweep + 1)
const body = appended.at(-1)?.body
expect(body?.kind === 'message' ? body.blocks.filter(isSubagentGroupBlock) : []).toMatchObject([
{ agents: [{ id: 'child-1', state: 'unverifiable' }] }
])
})
it('propagates sink backpressure instead of reporting the row as written', () => {
const roster = new CodexSubagentRoster({
sink: {
appendItem: () => {},
appendTombstone: () => {},
publish: () => {},
tryAppendItem: () => ({ accepted: false, reason: 'backpressure' }),
tryPublish: () => ({ accepted: true })
},
primaryThreadId: () => THREAD,
activeTurn: () => TURN
})
expect(
roster.handleItem({
threadId: THREAD,
turnId: TURN,
item: activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
})
).toEqual({ accepted: false, reason: 'backpressure' })
})
})
+347
View File
@@ -0,0 +1,347 @@
// The Codex subagent roster: one journal row per spawn group, revised in place.
//
// There is no snapshot to read. `agentsStates` arrived empty in the live probe
// and children get no `thread/started`, so the roster is
// accumulated purely from `subAgentActivity` items — each of which arrives TWICE
// (`item/started` and `item/completed`). Every transition here is therefore
// idempotent, and a terminal state latches: duplicate and out-of-order delivery
// must not resurrect a settled child.
//
// KNOWN LIMITATION: `groups` is process-local and is never seeded from the
// journal, while the row's identity is keyed on the group id alone. So once a
// group leaves the map its row stays, and the next activity item rebuilds that
// row from one child — rewriting N down to one. Two ways in: eviction past
// MAX_CODEX_SUBAGENT_GROUPS, which drops the oldest-inserted group in-process
// even while it is live, and skips the sweep so its children never latch
// `unverifiable`; and a restart on `threadId:outside-turn`, the one group id
// that outlives the process — `thread/resume` is verified to return the same
// thread, and a real turn id is assumed freshly minted per turn. Seeding from
// the journal is the fix.
import type {
AgentJournalItemBody,
AgentJournalItemIdentity
} from '../../shared/agent-session-journal-types'
import {
canReplaceSubagentState,
isTerminalSubagentState,
MAX_SUBAGENT_FIELD_CHARS,
subagentGroupFallbackText
} from '../../shared/native-chat-subagent-summary'
import type { NativeChatSubagentEntry } from '../../shared/native-chat-types'
import type {
StructuredAgentSessionEventSink,
StructuredAgentSessionSinkAdmission
} from '../native-chat/agent-session-wire/structured-agent-session-event-sink'
import {
codexSubagentLabel,
codexSubagentStateForKind,
isCodexRootAgentActivity,
readCodexSubagentActivity,
readCodexThreadTokenTotal
} from './codex-subagent-activity'
import type { CodexThreadItem } from './codex-structured-item-translation'
import {
MAX_CODEX_SUBAGENT_GROUPS,
MAX_CODEX_SUBAGENTS_PER_GROUP,
MAX_CODEX_TOKEN_USAGE_THREADS
} from './codex-structured-journal-limits'
const ADMITTED: StructuredAgentSessionSinkAdmission = { accepted: true }
/** The turn a group belongs to when Codex reports activity outside any turn.
* Mirrors the generic-frame bucket name so the two read alike in the journal. */
const OUTSIDE_TURN = 'outside-turn'
const UNLABELLED_AGENT = 'subagent'
type RosterGroup = {
groupId: string
identity: AgentJournalItemIdentity
/** Insertion order is the display order; the map holds the state. */
entries: Map<string, NativeChatSubagentEntry>
/** Times each label has been claimed, so a repeat gets an ordinal suffix. */
labelCounts: Map<string, number>
/** Last body written, so an idempotent replay writes no new revision. */
lastSerialized: string | null
}
/** Group identity: the parent turn that spawned the children. `agentPath` is a
* tree rooted at the parent thread, so every child of one turn shares a row
* no matter which thread's stream carried its activity item. */
export function codexSubagentGroupId(threadId: string, turnId: string | null): string {
return `${threadId}:${turnId ?? OUTSIDE_TURN}`
}
/** Durable journal identity for the group's row — stable across revisions and
* across a restart, so replay finds the same row instead of appending a new one. */
export function codexSubagentGroupIdentity(groupId: string): AgentJournalItemIdentity {
return { provider: 'orca', clientMessageId: `codex-subagents:${groupId}` }
}
export type CodexSubagentRosterDeps = {
sink: StructuredAgentSessionEventSink
/** The thread that owns the agent tree; falls back to the event's thread. */
primaryThreadId: () => string | null
activeTurn: (threadId: string) => string | null
now?: () => number
}
export class CodexSubagentRoster {
private readonly groups = new Map<string, RosterGroup>()
/** Latest reported total per thread, kept regardless of roster membership: a
* usage frame can arrive before the child's first activity item, and filtering
* at receipt would lose it permanently. Children are selected at write time;
* the map itself is LRU-capped in `handleTokenUsage`. */
private readonly tokensByThread = new Map<string, number>()
private readonly now: () => number
constructor(private readonly deps: CodexSubagentRosterDeps) {
this.now = deps.now ?? (() => Date.now())
}
/** Consume a `subAgentActivity` item. Returns null when the item is not one. */
handleItem(input: {
threadId: string
turnId: string | null
item: CodexThreadItem
}): StructuredAgentSessionSinkAdmission | null {
const activity = readCodexSubagentActivity(input.item)
if (!activity) {
return null
}
// The root node is the parent turn itself, not a child it spawned.
if (isCodexRootAgentActivity(activity)) {
return ADMITTED
}
const group = this.groupFor(input.threadId, input.turnId)
const existing = group.entries.get(activity.agentThreadId)
const state = codexSubagentStateForKind(activity.kind)
if (!existing) {
// Rule: the first event for a child may be ANY kind. An `interacted` or
// `completed` with no prior `started` creates the entry in the state its
// kind implies rather than being dropped for lacking a roster row.
if (group.entries.size >= MAX_CODEX_SUBAGENTS_PER_GROUP) {
return ADMITTED
}
const now = this.now()
group.entries.set(activity.agentThreadId, {
id: activity.agentThreadId,
label: this.claimLabel(group, codexSubagentLabel(activity)),
state,
startedAt: now,
...(isTerminalSubagentState(state) ? { settledAt: now } : {})
})
} else if (canReplaceSubagentState(existing.state, state)) {
// A child's own verdict latches. Re-applying the same non-terminal state
// is a no-op, which is what makes the duplicate `item/started` +
// `item/completed` delivery idempotent. `unverifiable` does not latch: a
// child swept when contact was lost can still report what it actually did
// if contact returns.
group.entries.set(activity.agentThreadId, {
...existing,
state,
...(isTerminalSubagentState(state) ? { settledAt: this.now() } : {})
})
}
return this.write(group)
}
/** Consume `thread/tokenUsage/updated`. Returns null when the params are not one. */
handleTokenUsage(params: unknown): StructuredAgentSessionSinkAdmission | null {
const usage = readCodexThreadTokenTotal(params)
if (!usage) {
return null
}
// A running total: the newest frame REPLACES the previous one. Summing
// updates would multiply a single child's usage by its frame count.
// Re-insert so the eviction scan below sees recency: `set` on an existing
// key keeps its original position, which would age out an active thread.
this.tokensByThread.delete(usage.threadId)
this.tokensByThread.set(usage.threadId, usage.totalTokens)
while (this.tokensByThread.size > MAX_CODEX_TOKEN_USAGE_THREADS) {
const oldest = this.tokensByThread.keys().next().value
if (typeof oldest !== 'string') {
break
}
this.tokensByThread.delete(oldest)
}
for (const group of this.groups.values()) {
if (!group.entries.has(usage.threadId)) {
continue
}
const admission = this.write(group)
if (!admission.accepted) {
return admission
}
}
return ADMITTED
}
/**
* The provider is gone, so any child still reported as working will never be
* settled by an event: it becomes `unverifiable` — contact was lost, which is
* NOT evidence the child exited.
*
* This is the ONLY sweep. A turn ending is not one: `spawn_agent` children
* routinely outlive their turn and keep reporting into the same group.
*/
settleSession(): StructuredAgentSessionSinkAdmission {
for (const group of this.groups.values()) {
const admission = this.sweep(group)
if (!admission.accepted) {
return admission
}
}
return ADMITTED
}
dispose(): void {
this.groups.clear()
this.tokensByThread.clear()
}
private sweep(group: RosterGroup | undefined): StructuredAgentSessionSinkAdmission {
if (!group) {
return ADMITTED
}
let changed = false
for (const [id, entry] of group.entries) {
if (isTerminalSubagentState(entry.state)) {
continue
}
group.entries.set(id, { ...entry, state: 'unverifiable', settledAt: this.now() })
changed = true
}
// A null `lastSerialized` means the previous write was refused part-way, so
// the settled roster's last revision is queued but never published. Nothing
// is guaranteed to write this group again, so retry here even when the sweep
// itself changed nothing.
return changed || group.lastSerialized === null ? this.write(group) : ADMITTED
}
private groupFor(threadId: string, turnId: string | null): RosterGroup {
const ownerThreadId = this.deps.primaryThreadId() ?? threadId
const ownerTurnId =
ownerThreadId === threadId ? turnId : (this.deps.activeTurn(ownerThreadId) ?? turnId)
const groupId = codexSubagentGroupId(ownerThreadId, ownerTurnId)
const existing = this.groups.get(groupId)
if (existing) {
return existing
}
const group: RosterGroup = {
groupId,
identity: codexSubagentGroupIdentity(groupId),
entries: new Map(),
labelCounts: new Map(),
lastSerialized: null
}
this.groups.set(groupId, group)
while (this.groups.size > MAX_CODEX_SUBAGENT_GROUPS) {
const oldest = this.groups.keys().next().value
if (typeof oldest !== 'string' || oldest === groupId) {
break
}
this.groups.delete(oldest)
}
return group
}
/** Two children can share a trailing path segment; the ordinal keeps their
* rows apart without inventing a name the provider never sent. */
private claimLabel(group: RosterGroup, label: string | null): string {
const base = label ?? UNLABELLED_AGENT
const seen = group.labelCounts.get(base) ?? 0
group.labelCounts.set(base, seen + 1)
return seen === 0 ? base : `${base} ${seen + 1}`
}
private write(group: RosterGroup): StructuredAgentSessionSinkAdmission {
const agents = [...group.entries].map(([id, entry]) => {
const tokens = this.tokensByThread.get(id)
if (typeof tokens !== 'number' || tokens === entry.tokens) {
return entry
}
// Persisted, not merely read: the thread map is LRU-capped, and reading it
// afresh each write would retract a count this row has already shown.
const merged = { ...entry, tokens }
group.entries.set(id, merged)
return merged
})
const body = codexSubagentGroupBody(group.groupId, agents)
const serialized = JSON.stringify(body)
if (serialized === group.lastSerialized) {
// Nothing changed — a duplicate delivery must not burn a revision.
return ADMITTED
}
group.lastSerialized = serialized
// The append coalesces per group so a burst collapses to the latest roster.
// The publish must NOT reuse that key: the queue coalesces by key alone,
// with no op-kind check, so a publish carrying it would splice out the
// still-queued append and the row would never reach the journal.
const options = { coalescingKey: `codex-subagents:${group.groupId}` }
const admission = this.deps.sink.tryAppendItem
? this.deps.sink.tryAppendItem(group.identity, body, options)
: (this.deps.sink.appendItem(group.identity, body, options), ADMITTED)
if (!admission.accepted) {
group.lastSerialized = null
return admission
}
const published = this.deps.sink.tryPublish
? this.deps.sink.tryPublish()
: (this.deps.sink.publish(), ADMITTED)
if (!published.accepted) {
// Symmetric with the append refusal above: the suppression state may only
// advance once the revision is both queued AND published. Left set, an
// identical replay short-circuits and the last revision of a settled
// roster stays queued but never reaches the renderer.
group.lastSerialized = null
}
return published
}
}
/** The roster row: the structured block plus the plain sentence an older client
* renders in its place. A message whose only block is the new variant would
* reach such a client with nothing it can draw. */
export function codexSubagentGroupBody(
groupId: string,
agents: readonly NativeChatSubagentEntry[]
): AgentJournalItemBody {
const bounded = agents.map((agent, index) => ({
...agent,
id: boundSubagentField(agent.id, index),
label: boundSubagentField(agent.label, index)
}))
return {
kind: 'message',
role: 'system',
blocks: [
{ type: 'text', text: subagentGroupFallbackText(bounded) },
{ type: 'subagent-group', groupId, agents: bounded }
]
}
}
/** `id` and `label` are provider strings, so they take the bound both readers of
* this row already clip them to. A plain length check, not the tool-output
* bound: that one digests the whole value before it checks the length, and this
* runs twice per child on every streamed token-usage frame.
*
* A clip is not identity-preserving, so a clipped value carries the child's
* index: two ids sharing a long prefix collapse to one React key, and
* `claimLabel` writes its ordinal at the very tail the clip removes. The index
* is reserved out of the bound, not appended to it, because both readers
* re-clip to the same cap and would cut a suffix that overflowed it. */
function boundSubagentField(value: string, index: number): string {
if (value.length <= MAX_SUBAGENT_FIELD_CHARS) {
return value
}
const suffix = `…~${index}`
const keep = MAX_SUBAGENT_FIELD_CHARS - suffix.length
// Slicing UTF-16 units can split a surrogate pair; a lone surrogate is
// malformed in a durable row and lossy through any non-JSON UTF-8 hop.
const last = value.charCodeAt(keep - 1)
const end = last >= 0xd800 && last <= 0xdbff ? keep - 1 : keep
return `${value.slice(0, end)}${suffix}`
}
@@ -24,7 +24,9 @@ const AUDITED_GLOBAL_FETCH_LINES = new Map<string, number>([
['main/orca-profiles/profile-cloud-org-members-client.ts', 1],
['main/rate-limits/codex-fetcher.ts', 3],
['main/runtime/relay/relay-http-client.ts', 2],
['main/runtime/relay/relay-region-preference.ts', 3],
['main/runtime/relay/relay-region-catalog-fetch.ts', 1],
['main/runtime/relay/relay-region-preference.ts', 2],
['main/runtime/relay/relay-region-probe.ts', 1],
['main/source-control/hosted-review-api-request.ts', 1],
['main/speech/openai-transcription-client.ts', 1],
// Main HTTP port: one type declaration plus the Node fallback call. The fallback
+18 -1
View File
@@ -742,11 +742,28 @@ describe('registerMobileHandlers', () => {
})
it('reports the current relay broker status without exposing a toggle', () => {
registerMobileHandlers({} as never, { getRelayStatus: () => 'registered' })
registerMobileHandlers({} as never, { getRelayStatus: () => ({ status: 'registered' }) })
expect(handlers.get('mobile:getRelayStatus')?.()).toEqual({ status: 'registered' })
})
it('reports the assigned relay cell alongside the status', () => {
registerMobileHandlers({} as never, {
getRelayStatus: () => ({ status: 'registered', cellUrl: 'https://c27.relay.example.com' })
})
expect(handlers.get('mobile:getRelayStatus')?.()).toEqual({
status: 'registered',
cellUrl: 'https://c27.relay.example.com'
})
})
it('falls back to offline with no cell when no relay status provider is wired', () => {
registerMobileHandlers({} as never, {})
expect(handlers.get('mobile:getRelayStatus')?.()).toEqual({ status: 'offline' })
})
it('consumes a pending auth-failure notification only from a window renderer', () => {
const consumePendingUnpairedDeviceAuthFailure = vi.fn(() => true)
registerMobileHandlers({} as never, { consumePendingUnpairedDeviceAuthFailure })
+6 -5
View File
@@ -13,7 +13,7 @@ import {
} from '../runtime/pairing-network-interfaces'
import { resolveAdvertisedPairingHostname } from '../runtime/pairing-endpoint'
import type { OrcaRuntimeRpcServer } from '../runtime/runtime-rpc'
import type { RelayBrokerStatus } from '../runtime/relay/relay-session-broker'
import type { MobileRelayStatusDetail } from '../../shared/mobile-relay-status'
import { encodeMobilePairingQr, type MobilePairingQrResult } from '../runtime/mobile-pairing-qr'
import { getWindowsDefaultRouteInterfaceNames } from '../runtime/windows-default-route-interfaces'
import {
@@ -51,7 +51,7 @@ function toRuntimeAccessGrant(device: DeviceEntry): RuntimeAccessGrant {
export type MobileHandlerDependencies = {
firewallEnvironment?: WindowsMobileFirewallEnvironment
openWindowsNetworkSettings?: () => Promise<void>
getRelayStatus?: () => RelayBrokerStatus
getRelayStatus?: () => MobileRelayStatusDetail
consumePendingUnpairedDeviceAuthFailure?: (webContentsId: number) => boolean
encodePairingQr?: (pairingUrl: string) => Promise<MobilePairingQrResult>
getDefaultRouteInterfaceNames?: DefaultRouteInterfaceLookup
@@ -287,9 +287,10 @@ export function registerMobileHandlers(
return true
})
ipcMain.handle('mobile:getRelayStatus', () => ({
status: dependencies.getRelayStatus?.() ?? 'offline'
}))
ipcMain.handle(
'mobile:getRelayStatus',
(): MobileRelayStatusDetail => dependencies.getRelayStatus?.() ?? { status: 'offline' }
)
ipcMain.handle('mobile:consumePendingUnpairedDeviceAuthFailure', (event) => {
if (!isWindowRenderer(event)) {
@@ -1,4 +1,8 @@
import { mkdir } from 'node:fs/promises'
import type {
AgentJournalItemBody,
AgentJournalItemIdentity
} from '../../../shared/agent-session-journal-types'
import type { AgentType } from '../../../shared/agent-status-types'
import {
findJournalFileFormatRemnant,
@@ -6,6 +10,7 @@ import {
} from './journal-file-format-remnant'
import type { JournalLoad } from './journal-open'
import { journalRepairDisclosure, type JournalRepairDisclosure } from './journal-repair-disclosure'
import { staleSubagentRosterRevisions } from './journal-subagent-liveness'
/** What any of this file's disclosures hands the store — a repair's, or the
* pre-SQLite notice's. Same shape, and neither is only a repair. */
@@ -36,9 +41,9 @@ export async function openJournalStoreState(input: {
adopt: (loaded: JournalLoad) => void
/** Republishes an anchor row for an epoch a repair emptied. */
publishRepairEpoch: () => void
appendDisclosure: (
identity: JournalRepairDisclosure['identity'],
body: JournalRepairDisclosure['body'],
appendItem: (
identity: AgentJournalItemIdentity,
body: AgentJournalItemBody,
fence: number
) => Promise<unknown>
agent: AgentType
@@ -68,8 +73,9 @@ export async function openJournalStoreState(input: {
}
if (input.malformedRows() > 0 && !input.readOnly()) {
const disclosure = journalRepairDisclosure({ malformedRows: input.malformedRows() })
await input.appendDisclosure(disclosure.identity, disclosure.body, input.highestFence())
await input.appendItem(disclosure.identity, disclosure.body, input.highestFence())
}
await settleStaleSubagentRosters(input, loaded)
// Founding the epoch and appending the row are two transactions, and a
// committed epoch sends every later open down this branch instead. Anything
// that interrupts between them — a quit during startup restore, a failed
@@ -92,7 +98,7 @@ export async function openJournalStoreState(input: {
async function discloseFileFormatRemnant(input: {
journalDir: string
agent: AgentType
appendDisclosure: (
appendItem: (
identity: JournalDisclosure['identity'],
body: JournalDisclosure['body'],
fence: number
@@ -108,5 +114,32 @@ async function discloseFileFormatRemnant(input: {
return
}
const disclosure = journalFileFormatRemnantDisclosure({ transcriptPath, agent: input.agent })
await input.appendDisclosure(disclosure.identity, disclosure.body, input.highestFence())
await input.appendItem(disclosure.identity, disclosure.body, input.highestFence())
}
/**
* Retires a `working` subagent roster the previous host never got to settle.
*
* Skipped on a corrupt load: that journal is still owed a rebuild from provider
* history, and content written past the repair's free sequence retires the
* demand for it.
*/
async function settleStaleSubagentRosters(
input: {
appendItem: (
identity: AgentJournalItemIdentity,
body: AgentJournalItemBody,
fence: number
) => Promise<unknown>
highestFence: () => number
readOnly: () => boolean
},
loaded: JournalLoad
): Promise<void> {
if (input.readOnly() || loaded.corrupt) {
return
}
for (const revision of staleSubagentRosterRevisions(loaded.state.items.values())) {
await input.appendItem(revision.identity, revision.body, input.highestFence())
}
}
@@ -39,8 +39,7 @@ export function restoreJournalStore(
publishRepairEpoch: () =>
collaborators.epochController.start('unreconcilable_prefix', host.state().highestFence),
adopt: host.adopt,
appendDisclosure: (identity, body, fence) =>
host.journal().appendItem(identity, body, { fence }),
appendItem: (identity, body, fence) => host.journal().appendItem(identity, body, { fence }),
agent: host.identity.agent,
highestFence: () => host.state().highestFence,
malformedRows: host.malformedRows,
@@ -0,0 +1,202 @@
import { mkdtemp, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import type {
AgentJournalRenderItem,
AgentSessionJournalIdentity
} from '../../../shared/agent-session-journal-types'
import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key'
import { isSubagentGroupBlock } from '../../../shared/native-chat-types'
import type { NativeChatSubagentEntry } from '../../../shared/native-chat-types'
import {
codexSubagentGroupBody,
codexSubagentGroupIdentity
} from '../../codex/codex-subagent-roster'
import type { openAgentSessionJournal } from './journal-store-factory'
import { createTrackedJournalOpener } from './journal-store-test-open'
import { staleSubagentRosterRevisions } from './journal-subagent-liveness'
const IDENTITY: AgentSessionJournalIdentity = {
sessionId: 'session-1',
workspaceId: 'ws-1',
hostId: 'host-1',
agent: 'codex',
providerHandle: { kind: 'codex', threadId: 'thread-1' }
}
const GROUP_ID = 'thread-1:turn-1'
let root: string
let clock = 1_000
function tick(): number {
clock += 1
return clock
}
const journals = createTrackedJournalOpener()
async function open(overrides: Partial<Parameters<typeof openAgentSessionJournal>[0]> = {}) {
return journals.open({
identity: IDENTITY,
journalDir: root,
now: tick,
mintEpoch: () => `epoch-${clock}`,
...overrides
})
}
/** The row as the producer writes it: the structured block plus its twin. */
function rosterRow(agents: NativeChatSubagentEntry[]) {
return {
identity: codexSubagentGroupIdentity(GROUP_ID),
body: codexSubagentGroupBody(GROUP_ID, agents)
}
}
function renderItem(agents: NativeChatSubagentEntry[]): AgentJournalRenderItem {
const row = rosterRow(agents)
return {
itemId: agentJournalItemKey(row.identity),
revision: 1,
body: row.body,
sequence: 2,
observedAt: 1
}
}
function rosterOf(body: AgentJournalRenderItem['body']): NativeChatSubagentEntry[] {
return body.kind === 'message' ? (body.blocks.find(isSubagentGroupBlock)?.agents ?? []) : []
}
function twinOf(body: AgentJournalRenderItem['body']): string | undefined {
return body.kind === 'message'
? body.blocks.find((block) => block.type === 'text')?.text
: undefined
}
beforeEach(async () => {
root = await mkdtemp(join(tmpdir(), 'orca-journal-subagents-'))
clock = 1_000
})
afterEach(async () => {
await journals.closeAll()
await rm(root, { recursive: true, force: true })
})
describe('staleSubagentRosterRevisions', () => {
it('settles a child the previous host left working, and moves the twin with it', () => {
const revisions = staleSubagentRosterRevisions([
renderItem([
{ id: 'a', label: 'read_readme', state: 'working', startedAt: 10 },
{ id: 'b', label: 'read_package', state: 'completed', startedAt: 10, settledAt: 20 }
])
])
expect(revisions).toHaveLength(1)
expect(rosterOf(revisions[0]!.body)).toMatchObject([
{ id: 'a', state: 'unverifiable' },
{ id: 'b', state: 'completed' }
])
// Mobile reads only this sentence, so it may not go on saying `Kicked off`.
expect(twinOf(revisions[0]!.body)).toBe('Ran 2 subagents (1 unverifiable)')
})
// The child stopped being observable at an unknown moment. A stamp taken now
// would report the time the app was down as how long the child ran.
it('records no terminal timestamp for a child whose run length is unknown', () => {
const revisions = staleSubagentRosterRevisions([
renderItem([{ id: 'a', label: 'read', state: 'working', startedAt: 10 }])
])
expect(rosterOf(revisions[0]!.body)[0]).not.toHaveProperty('settledAt')
})
it('owes nothing for a roster whose children all settled', () => {
expect(
staleSubagentRosterRevisions([
renderItem([{ id: 'a', label: 'read', state: 'completed', settledAt: 20 }])
])
).toEqual([])
})
it('leaves rows that carry no roster alone', () => {
expect(
staleSubagentRosterRevisions([
{
itemId: 'orca:plain',
revision: 1,
body: { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: 'hi' }] },
sequence: 2,
observedAt: 1
}
])
).toEqual([])
})
// Appending under a fresh identity would add a second row rather than revise
// the one on disk, so an unaddressable key is left exactly as it is.
it('skips a row whose key cannot be parsed back to its identity', () => {
expect(
staleSubagentRosterRevisions([
{ ...renderItem([{ id: 'a', label: 'r', state: 'working' }]), itemId: 'not-a-key' }
])
).toEqual([])
})
})
describe('journal reopen after the writing host is gone', () => {
it('settles a persisted working roster to unverifiable, while the live row still reads working', async () => {
const live = await open()
const row = rosterRow([
{ id: 'a', label: 'read_readme', state: 'working', startedAt: 10 },
{ id: 'b', label: 'read_package', state: 'working', startedAt: 10 }
])
await live.appendItem(row.identity, row.body, { fence: 0 })
// Still the writing host: it can see the children, so the row says so.
const beforeRestart = live.snapshot().items.at(-1)!
expect(rosterOf(beforeRestart.body)).toMatchObject([{ state: 'working' }, { state: 'working' }])
expect(twinOf(beforeRestart.body)).toBe('Kicked off 2 subagents')
// The host dies without ever settling them — no `ended`, so no session sweep.
await live.close()
const reopened = await open()
const afterRestart = reopened.snapshot().items.at(-1)!
expect(afterRestart.itemId).toBe(beforeRestart.itemId)
expect(rosterOf(afterRestart.body)).toMatchObject([
{ id: 'a', state: 'unverifiable' },
{ id: 'b', state: 'unverifiable' }
])
expect(twinOf(afterRestart.body)).toBe('Ran 2 subagents (2 unverifiable)')
})
it('revises the row in place rather than appending a second one', async () => {
const live = await open()
const row = rosterRow([{ id: 'a', label: 'read', state: 'working', startedAt: 10 }])
await live.appendItem(row.identity, row.body, { fence: 0 })
const before = live.snapshot().items.length
await live.close()
const reopened = await open()
expect(reopened.snapshot().items).toHaveLength(before)
expect(reopened.snapshot().items.at(-1)?.revision).toBe(2)
})
it('writes nothing on a second reopen once every child is settled', async () => {
const live = await open()
const row = rosterRow([{ id: 'a', label: 'read', state: 'working', startedAt: 10 }])
await live.appendItem(row.identity, row.body, { fence: 0 })
await live.close()
const once = await open()
const revision = once.snapshot().items.at(-1)?.revision
await once.close()
const twice = await open()
expect(twice.snapshot().items.at(-1)?.revision).toBe(revision)
})
})
@@ -0,0 +1,101 @@
// A roster row left claiming live children by a host that is gone.
//
// The writing host revises its `subagent-group` rows in place while it can see
// the children, and sweeps whatever is still `working` when the provider goes
// away. A host that DIED — crash, quit, force-restart — does neither: its last
// revision goes on saying `working`, and nothing replays those children, so no
// later event can ever settle them. Opening the journal is the one moment a new
// host can state the truth about the old one: contact was lost. That is
// `unverifiable`, never a synthesized exit — see
// `docs/reference/ssh-execution-boundary.md`.
//
// Reconciles JOURNAL ROWS, not roster state: nothing here seeds the producer's
// in-process group map, so the roster's known limitation is untouched.
import {
agentJournalItemKey,
parseAgentJournalItemKey
} from '../../../shared/agent-session-journal-item-key'
import type {
AgentJournalItemBody,
AgentJournalItemIdentity,
AgentJournalRenderItem
} from '../../../shared/agent-session-journal-types'
import {
isSubagentGroupFallbackText,
normalizeSubagentState,
subagentGroupFallbackText
} from '../../../shared/native-chat-subagent-summary'
import {
isSubagentGroupBlock,
type NativeChatBlock,
type NativeChatSubagentGroupBlock
} from '../../../shared/native-chat-types'
export type JournalSubagentLivenessRevision = {
identity: AgentJournalItemIdentity
body: AgentJournalItemBody
}
/** The revisions a reopened journal owes: one per row still claiming a live
* child. Empty — the common case — when nothing was left mid-flight. */
export function staleSubagentRosterRevisions(
items: Iterable<AgentJournalRenderItem>
): JournalSubagentLivenessRevision[] {
const revisions: JournalSubagentLivenessRevision[] = []
for (const item of items) {
const body = item.body
if (body.kind !== 'message' || !body.blocks.some(hasWorkingChild)) {
continue
}
// A key that will not parse cannot be re-addressed, and appending under a
// fresh identity would duplicate the row rather than revise it.
const identity = parseAgentJournalItemKey(item.itemId)
if (!identity || agentJournalItemKey(identity) !== item.itemId) {
continue
}
revisions.push({ identity, body: { ...body, blocks: settleBlocks(body.blocks) } })
}
return revisions
}
function hasWorkingChild(block: NativeChatBlock): boolean {
return (
isSubagentGroupBlock(block) &&
block.agents.some((agent) => normalizeSubagentState(agent.state) === 'working')
)
}
/** No `settledAt`: the child stopped being observable at an unknown moment, and
* stamping the reopen would report the time the app was down as how long it
* ran. Readers already draw an unverifiable child with no stamp as having no
* known run length. */
function settleBlocks(blocks: readonly NativeChatBlock[]): NativeChatBlock[] {
const settled = blocks.map((block) =>
hasWorkingChild(block) ? settleGroup(block as NativeChatSubagentGroupBlock) : block
)
const rosters = settled.filter(isSubagentGroupBlock)
const only = rosters.length === 1 ? rosters[0] : undefined
if (!only) {
return settled
}
// The plain-text twin is all a client without the block type ever shows, so it
// has to move with the block or the two would disagree about the same row.
const twin = subagentGroupFallbackText(only.agents)
return settled.map((block) =>
block.type === 'text' && isSubagentGroupFallbackText(block.text)
? { ...block, text: twin }
: block
)
}
function settleGroup(block: NativeChatSubagentGroupBlock): NativeChatSubagentGroupBlock {
return {
...block,
agents: block.agents.map((agent) =>
normalizeSubagentState(agent.state) === 'working'
? { ...agent, state: 'unverifiable' as const }
: agent
)
}
}
@@ -28,6 +28,16 @@ describe('provider frame activity', () => {
expect(codexProviderFrameActivity('item/reasoning/summaryPartAdded', {})).toBeNull()
})
it('names a fan-out from either Codex item type that reports one', () => {
for (const type of ['collabAgentToolCall', 'subAgentActivity']) {
expect(
codexProviderFrameActivity('item/started', {
item: { type, kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' }
})
).toBe('Coordinating with another agent')
}
})
it('uses Claude descriptions and safe semantic status without exposing tool labels', () => {
expect(
claudeProviderFrameActivity('message:system:task_started', {

Some files were not shown because too many files have changed in this diff Show More