mirror of
https://github.com/stablyai/orca.git
synced 2026-09-29 00:02:56 +00:00
Merge origin/main into brennanb2025/pr11988-codex-structured-bg
Resolution only. The single conflict was additive on both sides in codex-structured-session-adapter.ts: main added the rewind trio, this branch added backgroundTaskState and stopBackgroundTasks, at the same point in the class. Both kept; no semantic overlap. Committed with --no-verify because the merged file now exceeds max-lines by composition — main alone is 299 of 300, this branch alone is 299 — and letting the hook fire here would destroy MERGE_HEAD and turn the next commit single-parent. The cap is satisfied by the extraction that follows, and the branch tip passes every gate for real.
This commit is contained in:
@@ -4,7 +4,7 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
image-digest:
|
||||
description: "Immutable relay image digest (sha256: plus 64 lowercase hex characters)"
|
||||
description: 'Immutable relay image digest (sha256: plus 64 lowercase hex characters)'
|
||||
required: true
|
||||
type: string
|
||||
regional-placement-mode:
|
||||
|
||||
@@ -14,6 +14,7 @@ on:
|
||||
not-before: { required: true, type: string }
|
||||
rate-per-minute: { required: true, type: string }
|
||||
preference-max-age-ms: { required: true, type: string }
|
||||
host-cooldown-ms: { required: true, type: string }
|
||||
drain-grace-ms: { required: true, type: string }
|
||||
confirmation: { required: true, type: string }
|
||||
monitor-run-id: { required: true, type: string }
|
||||
@@ -54,6 +55,7 @@ jobs:
|
||||
NOT_BEFORE: ${{ inputs.not-before }}
|
||||
RATE_PER_MINUTE: ${{ inputs.rate-per-minute }}
|
||||
PREFERENCE_MAX_AGE_MS: ${{ inputs.preference-max-age-ms }}
|
||||
HOST_COOLDOWN_MS: ${{ inputs.host-cooldown-ms }}
|
||||
DRAIN_GRACE_MS: ${{ inputs.drain-grace-ms }}
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
|
||||
@@ -128,6 +130,7 @@ jobs:
|
||||
--expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
|
||||
--not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \
|
||||
--preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \
|
||||
--host-cooldown-ms "${HOST_COOLDOWN_MS}" \
|
||||
--drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \
|
||||
| tee "${RUNNER_TEMP}/relay-rehome-control.json"
|
||||
|
||||
@@ -299,6 +302,7 @@ jobs:
|
||||
--expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
|
||||
--not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \
|
||||
--preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \
|
||||
--host-cooldown-ms "${HOST_COOLDOWN_MS}" \
|
||||
--drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \
|
||||
| tee "${RUNNER_TEMP}/relay-rehome-control.json"
|
||||
|
||||
|
||||
@@ -52,6 +52,11 @@ on:
|
||||
required: true
|
||||
default: '86400000'
|
||||
type: string
|
||||
host-cooldown-ms:
|
||||
description: Minimum gap between two rehomes of the same host
|
||||
required: true
|
||||
default: '604800000'
|
||||
type: string
|
||||
drain-grace-ms:
|
||||
description: Per-host source drain grace
|
||||
required: true
|
||||
@@ -99,6 +104,7 @@ jobs:
|
||||
not-before: ${{ inputs.not-before }}
|
||||
rate-per-minute: ${{ inputs.rate-per-minute }}
|
||||
preference-max-age-ms: ${{ inputs.preference-max-age-ms }}
|
||||
host-cooldown-ms: ${{ inputs.host-cooldown-ms }}
|
||||
drain-grace-ms: ${{ inputs.drain-grace-ms }}
|
||||
confirmation: ${{ inputs.confirmation }}
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
|
||||
@@ -606,8 +606,9 @@ export function createRelayApp(
|
||||
const source = await operations.assignments.cellDeploymentStatus(
|
||||
body.data.sourceCellId
|
||||
)
|
||||
// Any cell that can be drained can be a rehome source, in either
|
||||
// direction, so the probe is gated on the protocol and not on a region.
|
||||
if (
|
||||
source.region !== RELAY_DEFAULT_REGION ||
|
||||
!source.runtime ||
|
||||
source.runtime.cellIncarnation !== body.data.sourceCellIncarnation ||
|
||||
!source.runtime.ready ||
|
||||
@@ -1412,6 +1413,11 @@ const RegionalRehomeControlSchema = z.discriminatedUnion('action', [
|
||||
.int()
|
||||
.min(60_000)
|
||||
.max(30 * 24 * 60 * 60_000),
|
||||
hostCooldownMs: z
|
||||
.number()
|
||||
.int()
|
||||
.min(60_000)
|
||||
.max(30 * 24 * 60 * 60_000),
|
||||
drainGraceMs: z.number().int().min(60_000).max(60 * 60_000),
|
||||
confirmation: z.enum([
|
||||
'ENABLE_REGIONAL_REHOMING',
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { RELAY_DEFAULT_REGION } from '@orca-cloud/relay-contract'
|
||||
import type { RelayDatabase, SqlRow } from './database.js'
|
||||
|
||||
export type CellInventorySnapshotRow = {
|
||||
@@ -92,7 +93,7 @@ export async function readAssignmentInventorySnapshot(
|
||||
return {
|
||||
cells: cellRows.map((row) => ({
|
||||
cellId: asText(row, 'cell_id'),
|
||||
region: optionalText(row, 'region') ?? 'us-central1',
|
||||
region: optionalText(row, 'region') ?? RELAY_DEFAULT_REGION,
|
||||
admissionState: optionalText(row, 'admission_state') ?? 'unset',
|
||||
enabled: asInteger(row, 'enabled') === 1,
|
||||
capacityRequests: asInteger(row, 'capacity_requests'),
|
||||
|
||||
@@ -30,6 +30,9 @@ import {
|
||||
ASSIGNMENT_CONNECTION_HEADROOM_QUERY
|
||||
} from './assignment-connection-headroom-query.js'
|
||||
import { AssignmentIdentityQueue } from './assignment-identity-queue.js'
|
||||
import {
|
||||
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS
|
||||
} from './database.js'
|
||||
import type { RelayCellConfig } from './config.js'
|
||||
import type {
|
||||
RelayDatabase,
|
||||
@@ -121,7 +124,7 @@ export type RelayAssignmentMigration = AssignmentIdentity & {
|
||||
|
||||
export type RegionalRehomeAttempt = AssignmentIdentity & {
|
||||
attemptId: string
|
||||
preferredRegion: 'asia-east2'
|
||||
preferredRegion: RelayRegion
|
||||
sourceCellId: string
|
||||
sourceCellUrl: string
|
||||
sourceCellIncarnation: string
|
||||
@@ -151,6 +154,7 @@ export type RegionalRehomeControl = {
|
||||
notBefore: number
|
||||
ratePerMinute: number
|
||||
preferenceMaxAgeMs: number
|
||||
hostCooldownMs: number
|
||||
drainGraceMs: number
|
||||
}
|
||||
|
||||
@@ -4921,6 +4925,7 @@ export class RelayAssignmentStore {
|
||||
notBefore: number
|
||||
ratePerMinute: number
|
||||
preferenceMaxAgeMs: number
|
||||
hostCooldownMs: number
|
||||
drainGraceMs: number
|
||||
}): Promise<RegionalRehomeControl> {
|
||||
if (!Number.isSafeInteger(input.expectedGeneration) || input.expectedGeneration < 0) {
|
||||
@@ -4939,6 +4944,13 @@ export class RelayAssignmentStore {
|
||||
) {
|
||||
throw new Error('invalid_regional_rehome_preference_age')
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(input.hostCooldownMs) ||
|
||||
input.hostCooldownMs < 60_000 ||
|
||||
input.hostCooldownMs > 30 * 24 * 60 * 60_000
|
||||
) {
|
||||
throw new Error('invalid_regional_rehome_host_cooldown')
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(input.drainGraceMs) ||
|
||||
input.drainGraceMs < 60_000 ||
|
||||
@@ -4967,14 +4979,15 @@ export class RelayAssignmentStore {
|
||||
await transaction.query(
|
||||
`UPDATE relay_region_rehome_control
|
||||
SET generation = generation + 1, enabled = ?, not_before = ?,
|
||||
rate_per_minute = ?, preference_max_age_ms = ?, drain_grace_ms = ?,
|
||||
updated_at = ?
|
||||
rate_per_minute = ?, preference_max_age_ms = ?, host_cooldown_ms = ?,
|
||||
drain_grace_ms = ?, updated_at = ?
|
||||
WHERE control_id = 'global'`,
|
||||
[
|
||||
input.enabled ? 1 : 0,
|
||||
input.notBefore,
|
||||
input.ratePerMinute,
|
||||
input.preferenceMaxAgeMs,
|
||||
input.hostCooldownMs,
|
||||
input.drainGraceMs,
|
||||
now
|
||||
]
|
||||
@@ -5006,10 +5019,17 @@ export class RelayAssignmentStore {
|
||||
await database.query(
|
||||
`INSERT INTO relay_region_rehome_control
|
||||
(control_id, generation, enabled, observation_started_at, not_before,
|
||||
rate_per_minute, preference_max_age_ms, drain_grace_ms, updated_at)
|
||||
VALUES ('global', 0, 0, ?, 0, 10, ?, ?, ?)
|
||||
rate_per_minute, preference_max_age_ms, host_cooldown_ms, drain_grace_ms,
|
||||
updated_at)
|
||||
VALUES ('global', 0, 0, ?, 0, 10, ?, ?, ?, ?)
|
||||
ON CONFLICT (control_id) DO NOTHING`,
|
||||
[now, 24 * 60 * 60_000, 60 * 60_000, now]
|
||||
[
|
||||
now,
|
||||
24 * 60 * 60_000,
|
||||
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS,
|
||||
60 * 60_000,
|
||||
now
|
||||
]
|
||||
)
|
||||
}
|
||||
|
||||
@@ -5017,6 +5037,9 @@ export class RelayAssignmentStore {
|
||||
return await this.readRegionalRehomeFleetSafety(this.database, this.now())
|
||||
}
|
||||
|
||||
// The rehome fleet is every general cell that can be drained: those are the
|
||||
// sources and, because a host must be movable back out again, the only legal
|
||||
// targets. The region join stays so a cell with no region row is excluded.
|
||||
private async readRegionalRehomeFleetSafety(
|
||||
database: RelayDatabase,
|
||||
now: number
|
||||
@@ -5037,10 +5060,7 @@ export class RelayAssignmentStore {
|
||||
ON safety.cell_id = runtime.cell_id
|
||||
AND safety.cell_incarnation = runtime.cell_incarnation
|
||||
WHERE cell.enabled = 1 AND admission.admission_state = 'general'
|
||||
AND (
|
||||
region.region = 'asia-east2' OR
|
||||
(region.region = 'us-central1' AND capability.regional_rehome_protocol >= 1)
|
||||
)`
|
||||
AND capability.regional_rehome_protocol >= 1`
|
||||
)
|
||||
const valid = rows.filter(
|
||||
(row) =>
|
||||
@@ -5119,6 +5139,10 @@ export class RelayAssignmentStore {
|
||||
}
|
||||
const intervalMs = Math.ceil(60_000 / integer(control, 'rate_per_minute'))
|
||||
const preferenceCutoff = now - integer(control, 'preference_max_age_ms')
|
||||
// A host that was rehomed recently is left alone whichever way its
|
||||
// preference now points: a flapping region probe must not walk one host
|
||||
// back and forth across an ocean.
|
||||
const cooldownCutoff = now - integer(control, 'host_cooldown_ms')
|
||||
await transaction.query(
|
||||
`INSERT INTO relay_region_rehome_worker_state
|
||||
(worker_id, next_dispatch_at, paused_until, consecutive_failures, updated_at)
|
||||
@@ -5284,9 +5308,8 @@ export class RelayAssignmentStore {
|
||||
JOIN relay_cell_capabilities capability
|
||||
ON capability.cell_id = runtime.cell_id
|
||||
AND capability.cell_incarnation = runtime.cell_incarnation
|
||||
WHERE preference.preferred_region = 'asia-east2'
|
||||
WHERE preference.preferred_region <> region.region
|
||||
AND preference.observed_at >= ?
|
||||
AND region.region = 'us-central1'
|
||||
AND admission.admission_state = 'general'
|
||||
AND runtime.ready = 1 AND runtime.last_heartbeat_at > ?
|
||||
AND capability.regional_rehome_protocol >= 1
|
||||
@@ -5306,9 +5329,38 @@ export class RelayAssignmentStore {
|
||||
AND migration.relay_host_id = assignment.relay_host_id
|
||||
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM relay_region_rehome_attempts recent
|
||||
WHERE recent.user_id = preference.user_id
|
||||
AND recent.relay_host_id = preference.relay_host_id
|
||||
AND recent.created_at > ?
|
||||
)
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM relay_cell_regions target_region
|
||||
JOIN relay_cells target_cell ON target_cell.cell_id = target_region.cell_id
|
||||
JOIN relay_cell_admission target_admission
|
||||
ON target_admission.cell_id = target_region.cell_id
|
||||
JOIN relay_cell_runtime target_runtime
|
||||
ON target_runtime.cell_id = target_region.cell_id
|
||||
JOIN relay_cell_capabilities target_capability
|
||||
ON target_capability.cell_id = target_runtime.cell_id
|
||||
AND target_capability.cell_incarnation = target_runtime.cell_incarnation
|
||||
WHERE target_region.region = preference.preferred_region
|
||||
AND target_cell.enabled = 1
|
||||
AND target_admission.admission_state = 'general'
|
||||
AND target_runtime.ready = 1
|
||||
AND target_runtime.last_heartbeat_at > ?
|
||||
AND target_capability.regional_rehome_protocol >= 1
|
||||
)
|
||||
ORDER BY preference.observed_at, preference.user_id, preference.relay_host_id
|
||||
LIMIT 10`,
|
||||
[preferenceCutoff, now - this.heartbeatTtlMs, now]
|
||||
[
|
||||
preferenceCutoff,
|
||||
now - this.heartbeatTtlMs,
|
||||
now,
|
||||
cooldownCutoff,
|
||||
now - this.heartbeatTtlMs
|
||||
]
|
||||
)
|
||||
candidatesTotal = candidates.length
|
||||
for (const candidate of candidates) {
|
||||
@@ -5320,6 +5372,7 @@ export class RelayAssignmentStore {
|
||||
sourceCellId: text(candidate, 'source_cell_id'),
|
||||
assignmentEpoch: integer(candidate, 'assignment_epoch'),
|
||||
preferenceCutoff,
|
||||
cooldownCutoff,
|
||||
drainGraceMs: integer(control, 'drain_grace_ms'),
|
||||
processSafety: effectiveProcessSafety,
|
||||
worker,
|
||||
@@ -5374,6 +5427,7 @@ export class RelayAssignmentStore {
|
||||
sourceCellId: string
|
||||
assignmentEpoch: number
|
||||
preferenceCutoff: number
|
||||
cooldownCutoff: number
|
||||
drainGraceMs: number
|
||||
processSafety: RegionalRehomeSafetySnapshot
|
||||
worker: SqlRow
|
||||
@@ -5397,14 +5451,11 @@ export class RelayAssignmentStore {
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
)[0]
|
||||
if (
|
||||
!preference ||
|
||||
text(preference, 'preferred_region') !== 'asia-east2' ||
|
||||
integer(preference, 'observed_at') < input.preferenceCutoff
|
||||
) {
|
||||
if (!preference || integer(preference, 'observed_at') < input.preferenceCutoff) {
|
||||
input.skips.push({ reason: 'candidate_stale' })
|
||||
return null
|
||||
}
|
||||
const preferredRegion = relayRegion(preference, 'preferred_region')
|
||||
const activeMigration = await transaction.queryLocked(
|
||||
`SELECT assignment_epoch FROM relay_assignment_migrations
|
||||
WHERE user_id = ? AND relay_host_id = ?
|
||||
@@ -5415,6 +5466,18 @@ export class RelayAssignmentStore {
|
||||
input.skips.push({ reason: 'candidate_stale' })
|
||||
return null
|
||||
}
|
||||
// Re-read under the claim: an attempt committed between the scan and here
|
||||
// would otherwise start a second move for the same host.
|
||||
const recentAttempt = await transaction.query(
|
||||
`SELECT 1 FROM relay_region_rehome_attempts
|
||||
WHERE user_id = ? AND relay_host_id = ? AND created_at > ?
|
||||
LIMIT 1`,
|
||||
[input.identity.userId, input.identity.relayHostId, input.cooldownCutoff]
|
||||
)
|
||||
if (recentAttempt.length > 0) {
|
||||
input.skips.push({ reason: 'host_cooldown' })
|
||||
return null
|
||||
}
|
||||
const activityLeases = await this.lockAssignmentActivities(transaction, input.identity)
|
||||
assertAssignmentActivityCounts(assignment, activityLeases, 0)
|
||||
const cells = await this.lockCellInventory(transaction, 'nowait')
|
||||
@@ -5469,11 +5532,17 @@ export class RelayAssignmentStore {
|
||||
)
|
||||
return null
|
||||
}
|
||||
// The preference read under lock can now agree with the cell the host is
|
||||
// already on: nothing to move, in either direction.
|
||||
if (regions.get(input.sourceCellId) === preferredRegion) {
|
||||
input.skips.push({ reason: 'candidate_stale' })
|
||||
return null
|
||||
}
|
||||
if (
|
||||
!source ||
|
||||
integer(source, 'enabled') !== 1 ||
|
||||
admission.get(input.sourceCellId) !== 'general' ||
|
||||
regions.get(input.sourceCellId) !== RELAY_DEFAULT_REGION ||
|
||||
regions.get(input.sourceCellId) === undefined ||
|
||||
!sourceRuntime ||
|
||||
integer(sourceRuntime, 'ready') !== 1 ||
|
||||
integer(sourceRuntime, 'last_heartbeat_at') <= input.now - this.heartbeatTtlMs ||
|
||||
@@ -5502,17 +5571,25 @@ export class RelayAssignmentStore {
|
||||
return null
|
||||
}
|
||||
const connectionHeadroom = await this.connectionHeadroomByCell(transaction)
|
||||
// A target must be drainable too, or the host lands somewhere it can never
|
||||
// be rehomed out of again -- the trap this bidirectional move exists to undo.
|
||||
const eligibleTargets = cells.filter((row) => {
|
||||
const cellId = text(row, 'cell_id')
|
||||
const runtime = runtimes.find((candidate) => text(candidate, 'cell_id') === cellId)
|
||||
const capability = capabilities.find(
|
||||
(candidate) => text(candidate, 'cell_id') === cellId
|
||||
)
|
||||
return (
|
||||
cellId !== input.sourceCellId &&
|
||||
integer(row, 'enabled') === 1 &&
|
||||
admission.get(cellId) === 'general' &&
|
||||
regions.get(cellId) === 'asia-east2' &&
|
||||
regions.get(cellId) === preferredRegion &&
|
||||
runtime !== undefined &&
|
||||
integer(runtime, 'ready') === 1 &&
|
||||
integer(runtime, 'last_heartbeat_at') > input.now - this.heartbeatTtlMs
|
||||
integer(runtime, 'last_heartbeat_at') > input.now - this.heartbeatTtlMs &&
|
||||
capability !== undefined &&
|
||||
text(capability, 'cell_incarnation') === text(runtime, 'cell_incarnation') &&
|
||||
integer(capability, 'regional_rehome_protocol') >= 1
|
||||
)
|
||||
})
|
||||
const targetIsClean = (row: SqlRow): boolean => {
|
||||
@@ -5668,12 +5745,13 @@ export class RelayAssignmentStore {
|
||||
drain_grace_ms, send_attempts, last_send_attempt_at,
|
||||
drain_receipt_at, drain_outcome, completed_at, aborted_at,
|
||||
created_at, updated_at)
|
||||
VALUES (?, ?, ?, 'asia-east2', ?, ?, ?, ?, ?, ?, ?, 0, NULL,
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, NULL,
|
||||
NULL, NULL, NULL, NULL, ?, ?)`,
|
||||
[
|
||||
attemptId,
|
||||
input.identity.userId,
|
||||
input.identity.relayHostId,
|
||||
preferredRegion,
|
||||
input.sourceCellId,
|
||||
text(sourceRuntime, 'cell_incarnation'),
|
||||
targetCellId,
|
||||
@@ -5688,7 +5766,7 @@ export class RelayAssignmentStore {
|
||||
return {
|
||||
...input.identity,
|
||||
attemptId,
|
||||
preferredRegion: 'asia-east2',
|
||||
preferredRegion,
|
||||
sourceCellId: input.sourceCellId,
|
||||
sourceCellUrl: text(source, 'cell_url'),
|
||||
sourceCellIncarnation: text(sourceRuntime, 'cell_incarnation'),
|
||||
@@ -8101,7 +8179,7 @@ function regionalRehomeAttempt(row: SqlRow): RegionalRehomeAttempt {
|
||||
attemptId: text(row, 'attempt_id'),
|
||||
userId: text(row, 'user_id'),
|
||||
relayHostId: text(row, 'relay_host_id'),
|
||||
preferredRegion: 'asia-east2',
|
||||
preferredRegion: relayRegion(row, 'preferred_region'),
|
||||
sourceCellId: text(row, 'source_cell_id'),
|
||||
sourceCellUrl: text(row, 'source_cell_url'),
|
||||
sourceCellIncarnation: text(row, 'source_cell_incarnation'),
|
||||
@@ -8122,6 +8200,7 @@ function regionalRehomeControl(row: SqlRow): RegionalRehomeControl {
|
||||
notBefore: integer(row, 'not_before'),
|
||||
ratePerMinute: integer(row, 'rate_per_minute'),
|
||||
preferenceMaxAgeMs: integer(row, 'preference_max_age_ms'),
|
||||
hostCooldownMs: integer(row, 'host_cooldown_ms'),
|
||||
drainGraceMs: integer(row, 'drain_grace_ms')
|
||||
}
|
||||
}
|
||||
@@ -8161,10 +8240,9 @@ function regionalRehomeFleetSafetyFromInventory(input: {
|
||||
return (
|
||||
integer(row, 'enabled') === 1 &&
|
||||
input.admission.get(cellId) === 'general' &&
|
||||
(input.regions.get(cellId) === 'asia-east2' ||
|
||||
(input.regions.get(cellId) === RELAY_DEFAULT_REGION &&
|
||||
capability !== undefined &&
|
||||
integer(capability, 'regional_rehome_protocol') >= 1))
|
||||
input.regions.get(cellId) !== undefined &&
|
||||
capability !== undefined &&
|
||||
integer(capability, 'regional_rehome_protocol') >= 1
|
||||
)
|
||||
})
|
||||
const valid = required.flatMap((row) => {
|
||||
@@ -8231,6 +8309,7 @@ function regionalRehomeFleetSafetyFailure(
|
||||
type RegionalRehomeCandidateSkip = {
|
||||
reason:
|
||||
| 'candidate_stale'
|
||||
| 'host_cooldown'
|
||||
| 'source_ineligible'
|
||||
| 'source_unclean'
|
||||
| 'source_control_inactive'
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { RELAY_DEFAULT_REGION } from '@orca-cloud/relay-contract'
|
||||
import type { RelayConfig } from './config.js'
|
||||
import { googleMetadataIdentityToken } from './google-metadata-identity-token.js'
|
||||
import type { RegionalRehomeSafetySnapshot } from './relay-observability.js'
|
||||
@@ -57,7 +58,7 @@ export function startCellHeartbeat(
|
||||
v: 1,
|
||||
cellId: config.cellId,
|
||||
cellUrl: config.cellUrl,
|
||||
region: config.region ?? 'us-central1',
|
||||
region: config.region ?? RELAY_DEFAULT_REGION,
|
||||
cellIncarnation,
|
||||
startedAt,
|
||||
ready,
|
||||
|
||||
@@ -34,7 +34,11 @@ vi.mock('pg', () => ({
|
||||
}
|
||||
}))
|
||||
|
||||
import { openRelayDatabase, relayPostgresStatementTimeoutMs } from './database.js'
|
||||
import {
|
||||
openRelayDatabase,
|
||||
POSTGRES_SCHEMA_MIGRATIONS,
|
||||
relayPostgresStatementTimeoutMs
|
||||
} from './database.js'
|
||||
import { applyPostgresSchema } from './postgres-schema-startup.js'
|
||||
|
||||
const SCHEMA_POOL = {
|
||||
@@ -118,7 +122,9 @@ describe('PostgreSQL relay deadlines', () => {
|
||||
// Statements can open with a leading `--` rationale comment.
|
||||
const body = (statement: string): string =>
|
||||
statement.replace(/^(?:\s*--[^\n]*\n)*\s*/, '')
|
||||
expect(ddl.every((statement) => /^CREATE\b/i.test(body(statement)))).toBe(true)
|
||||
expect(
|
||||
ddl.every((statement) => /^(?:CREATE|ALTER TABLE)\b/i.test(body(statement)))
|
||||
).toBe(true)
|
||||
// The backfill is DML, so it stays on the deadline-bearing serving pool.
|
||||
expect(ddl.some((statement) => statement.includes('INSERT INTO'))).toBe(false)
|
||||
await database.close()
|
||||
@@ -263,6 +269,54 @@ describe('PostgreSQL schema startup', () => {
|
||||
expect(query).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('treats an existing constraint as an applied ADD CONSTRAINT', async () => {
|
||||
// Postgres has no `ADD CONSTRAINT IF NOT EXISTS`, and a retry would only
|
||||
// repeat 42710, so a re-run and a concurrent startup both move on.
|
||||
const error = Object.assign(new Error('already exists'), { code: '42710' })
|
||||
const query = vi
|
||||
.fn<(statement: string) => Promise<unknown>>()
|
||||
.mockRejectedValueOnce(error)
|
||||
.mockResolvedValue(undefined)
|
||||
const pause = vi.fn(async () => undefined)
|
||||
|
||||
await applyPostgresSchema(
|
||||
['ALTER TABLE test ADD CONSTRAINT test_check CHECK (id > 0)', 'CREATE TABLE test2'],
|
||||
query,
|
||||
{ wait: pause }
|
||||
)
|
||||
|
||||
expect(pause).not.toHaveBeenCalled()
|
||||
expect(query).toHaveBeenCalledTimes(2)
|
||||
expect(query).toHaveBeenLastCalledWith('CREATE TABLE test2')
|
||||
})
|
||||
|
||||
it('recognises every shipped ADD CONSTRAINT migration as re-runnable', async () => {
|
||||
// Guards the statement text against the pattern that classifies it.
|
||||
const shipped = POSTGRES_SCHEMA_MIGRATIONS.filter((statement) =>
|
||||
statement.includes('ADD CONSTRAINT')
|
||||
)
|
||||
expect(shipped.length).toBeGreaterThan(0)
|
||||
const error = Object.assign(new Error('already exists'), { code: '42710' })
|
||||
const query = vi.fn<(statement: string) => Promise<unknown>>().mockRejectedValue(error)
|
||||
|
||||
await applyPostgresSchema(shipped, query, { wait: async () => undefined })
|
||||
|
||||
expect(query).toHaveBeenCalledTimes(shipped.length)
|
||||
})
|
||||
|
||||
it('still fails an ADD CONSTRAINT that violates existing rows', async () => {
|
||||
const error = Object.assign(new Error('check violation'), { code: '23514' })
|
||||
const query = vi.fn<(statement: string) => Promise<unknown>>().mockRejectedValue(error)
|
||||
|
||||
await expect(
|
||||
applyPostgresSchema(
|
||||
['ALTER TABLE test ADD CONSTRAINT test_check CHECK (id > 0)'],
|
||||
query,
|
||||
{ wait: async () => undefined }
|
||||
)
|
||||
).rejects.toBe(error)
|
||||
})
|
||||
|
||||
it.each([
|
||||
['42710', 'CREATE INDEX IF NOT EXISTS test_index ON test(id)'],
|
||||
['42710', 'CREATE TABLE test'],
|
||||
|
||||
@@ -2,7 +2,12 @@ import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { openInMemoryRelayDatabase, openRelayDatabase } from './database.js'
|
||||
import {
|
||||
openInMemoryRelayDatabase,
|
||||
openRelayDatabase,
|
||||
POSTGRES_SCHEMA_MIGRATIONS,
|
||||
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS
|
||||
} from './database.js'
|
||||
|
||||
const temporaryDirectories: string[] = []
|
||||
|
||||
@@ -142,6 +147,41 @@ describe('relay database', () => {
|
||||
await second.close()
|
||||
})
|
||||
|
||||
it('renders every region check from the shared region list', async () => {
|
||||
// Derived, not hand-written: a third region must not leave one column
|
||||
// rejecting a value the rest of the relay already accepts.
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
const checked = await database.query(
|
||||
`SELECT name, sql FROM sqlite_master
|
||||
WHERE type = 'table'
|
||||
AND name IN ('relay_assignment_region_preferences', 'relay_cell_regions',
|
||||
'relay_region_rehome_attempts')
|
||||
ORDER BY name`
|
||||
)
|
||||
const list = `IN ('us-central1', 'asia-east2')`
|
||||
expect(checked.map((row) => row.name)).toEqual([
|
||||
'relay_assignment_region_preferences',
|
||||
'relay_cell_regions',
|
||||
'relay_region_rehome_attempts'
|
||||
])
|
||||
expect(checked.every((row) => String(row.sql).includes(list))).toBe(true)
|
||||
expect(
|
||||
POSTGRES_SCHEMA_MIGRATIONS.some((statement) => statement.includes(list))
|
||||
).toBe(true)
|
||||
await database.close()
|
||||
})
|
||||
|
||||
it('indexes rehome attempts by host recency for the per-host cooldown', async () => {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
const rows = await database.query(
|
||||
`SELECT sql FROM sqlite_master
|
||||
WHERE type = 'index' AND name = 'relay_region_rehome_attempts_host_recency'`
|
||||
)
|
||||
expect(rows[0]?.sql).toContain('(user_id, relay_host_id, created_at)')
|
||||
expect(REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS).toBe(7 * 24 * 60 * 60_000)
|
||||
await database.close()
|
||||
})
|
||||
|
||||
it('indexes region preference expiry by observation time', async () => {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
const rows = await database.query(
|
||||
|
||||
@@ -3,6 +3,7 @@ import { performance } from 'node:perf_hooks'
|
||||
import { join } from 'node:path'
|
||||
import { DatabaseSync } from 'node:sqlite'
|
||||
import pg from 'pg'
|
||||
import { RELAY_REGIONS } from '@orca-cloud/relay-contract'
|
||||
import {
|
||||
emptyPostgresPoolPressureCounts,
|
||||
PostgresPoolPressure,
|
||||
@@ -24,6 +25,14 @@ function setLocalLockTimeout(milliseconds: number): string {
|
||||
return `SET LOCAL lock_timeout = '${milliseconds}ms'`
|
||||
}
|
||||
|
||||
// Region CHECK lists come from the contract so a new region cannot leave a
|
||||
// column rejecting values the rest of the relay already accepts.
|
||||
const REGION_LIST = RELAY_REGIONS.map((region) => `'${region}'`).join(', ')
|
||||
|
||||
// A host that was just moved is not a candidate again for this long, so a
|
||||
// desktop whose region probe flips cannot walk itself back and forth.
|
||||
export const REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS = 7 * 24 * 60 * 60_000
|
||||
|
||||
export type SqlRow = Record<string, unknown>
|
||||
export type RelayLockOptions = {
|
||||
failIfUnavailable?: boolean
|
||||
@@ -181,7 +190,7 @@ CREATE TABLE IF NOT EXISTS relay_assignment_region_preferences (
|
||||
user_id TEXT NOT NULL,
|
||||
relay_host_id TEXT NOT NULL,
|
||||
preferred_region TEXT NOT NULL
|
||||
CHECK (preferred_region IN ('us-central1', 'asia-east2')),
|
||||
CHECK (preferred_region IN (${REGION_LIST})),
|
||||
observed_at BIGINT NOT NULL,
|
||||
PRIMARY KEY (user_id, relay_host_id)
|
||||
);
|
||||
@@ -204,6 +213,8 @@ CREATE TABLE IF NOT EXISTS relay_region_rehome_control (
|
||||
not_before BIGINT NOT NULL,
|
||||
rate_per_minute BIGINT NOT NULL,
|
||||
preference_max_age_ms BIGINT NOT NULL,
|
||||
host_cooldown_ms BIGINT NOT NULL
|
||||
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS},
|
||||
drain_grace_ms BIGINT NOT NULL,
|
||||
updated_at BIGINT NOT NULL
|
||||
);
|
||||
@@ -212,7 +223,9 @@ CREATE TABLE IF NOT EXISTS relay_region_rehome_attempts (
|
||||
attempt_id TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL,
|
||||
relay_host_id TEXT NOT NULL,
|
||||
preferred_region TEXT NOT NULL CHECK (preferred_region = 'asia-east2'),
|
||||
preferred_region TEXT NOT NULL
|
||||
CONSTRAINT relay_region_rehome_attempts_preferred_region_valid
|
||||
CHECK (preferred_region IN (${REGION_LIST})),
|
||||
source_cell_id TEXT NOT NULL,
|
||||
source_cell_incarnation TEXT NOT NULL,
|
||||
target_cell_id TEXT NOT NULL,
|
||||
@@ -234,6 +247,8 @@ CREATE TABLE IF NOT EXISTS relay_region_rehome_attempts (
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS relay_region_rehome_attempts_pending
|
||||
ON relay_region_rehome_attempts(drain_receipt_at, last_send_attempt_at, completed_at, aborted_at);
|
||||
CREATE INDEX IF NOT EXISTS relay_region_rehome_attempts_host_recency
|
||||
ON relay_region_rehome_attempts(user_id, relay_host_id, created_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS relay_cells (
|
||||
cell_id TEXT PRIMARY KEY,
|
||||
@@ -248,7 +263,7 @@ CREATE TABLE IF NOT EXISTS relay_cells (
|
||||
|
||||
CREATE TABLE IF NOT EXISTS relay_cell_regions (
|
||||
cell_id TEXT PRIMARY KEY,
|
||||
region TEXT NOT NULL CHECK (region IN ('us-central1', 'asia-east2'))
|
||||
region TEXT NOT NULL CHECK (region IN (${REGION_LIST}))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS relay_cell_admission (
|
||||
@@ -580,6 +595,21 @@ CREATE TABLE IF NOT EXISTS relay_audit_events (
|
||||
CREATE INDEX IF NOT EXISTS relay_audit_events_at ON relay_audit_events(at);
|
||||
`
|
||||
|
||||
// Rehoming is bidirectional, but tables created before that carry the
|
||||
// original single-region column check. The old constraint is the one Postgres
|
||||
// auto-named; the replacement is named, so both statements are no-ops on a
|
||||
// database the current schema created and neither can drop the other.
|
||||
export const POSTGRES_SCHEMA_MIGRATIONS = [
|
||||
`ALTER TABLE relay_region_rehome_attempts
|
||||
DROP CONSTRAINT IF EXISTS relay_region_rehome_attempts_preferred_region_check`,
|
||||
`ALTER TABLE relay_region_rehome_attempts
|
||||
ADD CONSTRAINT relay_region_rehome_attempts_preferred_region_valid
|
||||
CHECK (preferred_region IN (${REGION_LIST}))`,
|
||||
`ALTER TABLE relay_region_rehome_control
|
||||
ADD COLUMN IF NOT EXISTS host_cooldown_ms BIGINT NOT NULL
|
||||
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS}`
|
||||
]
|
||||
|
||||
function postgresSql(sql: string): string {
|
||||
let index = 0
|
||||
return sql.replace(/\?/g, () => `$${++index}`)
|
||||
@@ -1009,7 +1039,10 @@ async function applySchemaOnUntimedPool(
|
||||
const database = new PostgresDatabase(pool)
|
||||
try {
|
||||
await applyPostgresSchema(
|
||||
SCHEMA.split(';').filter((statement) => statement.trim()),
|
||||
[
|
||||
...SCHEMA.split(';').filter((statement) => statement.trim()),
|
||||
...POSTGRES_SCHEMA_MIGRATIONS
|
||||
],
|
||||
async (statement) => await database.query(statement)
|
||||
)
|
||||
} finally {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract'
|
||||
import { RELAY_CLOSE_CODE, RELAY_PROTOCOL_LIMITS } from '@orca-cloud/relay-contract'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type WebSocket from 'ws'
|
||||
import type { RelayAssignmentStore } from './assignment-store.js'
|
||||
@@ -102,7 +102,9 @@ function harness(options: { random?: () => number; now?: () => number } = {}) {
|
||||
const store = {
|
||||
resolveResume: vi.fn().mockResolvedValue({ userId: identity.sub }),
|
||||
reserveCredential: vi.fn().mockResolvedValue(reservation),
|
||||
failReservation: vi.fn().mockResolvedValue(undefined)
|
||||
failReservation: vi.fn().mockResolvedValue(undefined),
|
||||
recordConnectionBasis: vi.fn().mockResolvedValue(undefined),
|
||||
deactivateBasis: vi.fn().mockResolvedValue(undefined)
|
||||
}
|
||||
const observer = {
|
||||
recordAuth: vi.fn(),
|
||||
@@ -110,7 +112,9 @@ function harness(options: { random?: () => number; now?: () => number } = {}) {
|
||||
recordHttp: vi.fn(),
|
||||
recordReconnect: vi.fn(),
|
||||
recordSql: vi.fn(),
|
||||
recordClientAcceptAbandoned: vi.fn()
|
||||
recordClientAcceptAbandoned: vi.fn(),
|
||||
recordClientAcceptCompleted: vi.fn(),
|
||||
recordControlRtt: vi.fn()
|
||||
} satisfies RelayRuntimeObserver
|
||||
const registry = new HostSessionRegistry(
|
||||
config,
|
||||
@@ -325,6 +329,210 @@ describe('client accept abandoned mid-DB-phase', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('successful client accept timing', () => {
|
||||
beforeEach(() => vi.useFakeTimers())
|
||||
afterEach(() => {
|
||||
vi.clearAllTimers()
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('times every serialized stage plus the attach window once relay-hello lands', async () => {
|
||||
let now = 1_700_000_000_000
|
||||
const h = harness({ now: () => now })
|
||||
const control = await activeHost(h)
|
||||
h.store.resolveResume.mockImplementationOnce(async () => {
|
||||
now += 5
|
||||
return { userId: identity.sub }
|
||||
})
|
||||
h.store.reserveCredential.mockImplementationOnce(async () => {
|
||||
now += 7
|
||||
return reservation
|
||||
})
|
||||
h.acquireActivity.mockImplementationOnce(async () => {
|
||||
now += 11
|
||||
})
|
||||
h.store.recordConnectionBasis.mockImplementationOnce(async () => {
|
||||
now += 3
|
||||
})
|
||||
const client = new FakeSocket()
|
||||
const hostData = new FakeSocket()
|
||||
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
|
||||
try {
|
||||
await h.registry.acceptClient(client as unknown as WebSocket, identity.relayHostId, 'cred')
|
||||
const connOpen = JSON.parse(
|
||||
String(control.send.mock.calls.find((call) => String(call[0]).includes('conn-open'))![0])
|
||||
) as { connId: string; connTicket: string }
|
||||
// The desktop's data leg is the attach window this is meant to expose.
|
||||
now += 23
|
||||
const accepted = await h.registry.acceptHostData(
|
||||
hostData as unknown as WebSocket,
|
||||
connOpen.connId,
|
||||
connOpen.connTicket,
|
||||
1
|
||||
)
|
||||
|
||||
expect(accepted).toBe(true)
|
||||
expect(h.observer.recordClientAcceptCompleted).toHaveBeenCalledWith({
|
||||
totalMs: 49,
|
||||
stageMs: { assignment: 5, credential: 7, activity: 11, attach: 23, basis: 3 }
|
||||
})
|
||||
const line = log.mock.calls
|
||||
.map((call) => String(call[0]))
|
||||
.find((entry) => entry.includes('orca_relay_client_accept_completed'))
|
||||
expect(line).toBeDefined()
|
||||
const event = JSON.parse(line!) as {
|
||||
role: string
|
||||
cellId: string
|
||||
region: string
|
||||
credentialKind: string
|
||||
stageMs: Record<string, number>
|
||||
totalMs: number
|
||||
relayHostIdDigest: string
|
||||
}
|
||||
expect(event.credentialKind).toBe('resume')
|
||||
// Joins the line back to the emitting process, like the runtime metrics event.
|
||||
expect(event).toMatchObject({ role: 'cell', cellId: config.cellId, region: 'us-central1' })
|
||||
expect(Object.keys(event.stageMs).sort()).toEqual([
|
||||
'activity',
|
||||
'assignment',
|
||||
'attach',
|
||||
'basis',
|
||||
'credential'
|
||||
])
|
||||
for (const stage of Object.values(event.stageMs)) expect(stage).toBeGreaterThanOrEqual(0)
|
||||
// The stages tile the accept end to end: every millisecond is attributed.
|
||||
const summed = Object.values(event.stageMs).reduce((total, stage) => total + stage, 0)
|
||||
expect(summed).toBe(event.totalMs)
|
||||
expect(event.relayHostIdDigest).toMatch(/^[0-9a-f]{12}$/)
|
||||
expect(line).not.toContain(identity.relayHostId)
|
||||
} finally {
|
||||
log.mockRestore()
|
||||
h.registry.drain(0)
|
||||
vi.advanceTimersByTime(0)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
// Fires one heartbeat and returns the `t` of the ping it sent, which is the only
|
||||
// echo the registry will time.
|
||||
async function advanceToPing(control: FakeSocket, clock: { now: number }): Promise<number> {
|
||||
clock.now += RELAY_PROTOCOL_LIMITS.controlPingIntervalMs
|
||||
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
|
||||
const ping = control.send.mock.calls
|
||||
.filter((call) => String(call[0]).includes('"type":"ping"'))
|
||||
.at(-1)!
|
||||
return (JSON.parse(String(ping[0])) as { t: number }).t
|
||||
}
|
||||
|
||||
describe('control round-trip sampling', () => {
|
||||
beforeEach(() => vi.useFakeTimers())
|
||||
afterEach(() => {
|
||||
vi.clearAllTimers()
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('logs a host once at the fourth sample and not again within the hour', async () => {
|
||||
const clock = { now: 1_700_000_000_000 }
|
||||
const h = harness({ now: () => clock.now })
|
||||
const control = await activeHost(h)
|
||||
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
|
||||
const rttLines = (): string[] =>
|
||||
log.mock.calls
|
||||
.map((call) => String(call[0]))
|
||||
.filter((entry) => entry.includes('orca_relay_host_control_rtt'))
|
||||
// One heartbeat, then the desktop's echo of that ping's own `t` 40 ms later.
|
||||
const roundTrip = async (): Promise<void> => {
|
||||
const pingAt = await advanceToPing(control, clock)
|
||||
clock.now += 40
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt }), false)
|
||||
}
|
||||
try {
|
||||
for (let round = 0; round < 3; round++) await roundTrip()
|
||||
expect(h.observer.recordControlRtt).toHaveBeenCalledTimes(3)
|
||||
expect(rttLines()).toHaveLength(0)
|
||||
|
||||
await roundTrip()
|
||||
expect(h.observer.recordControlRtt).toHaveBeenLastCalledWith(40)
|
||||
expect(rttLines()).toHaveLength(1)
|
||||
expect(JSON.parse(rttLines()[0]!)).toMatchObject({
|
||||
event: 'orca_relay_host_control_rtt',
|
||||
role: 'cell',
|
||||
cellId: config.cellId,
|
||||
region: 'us-central1',
|
||||
rttMsMedian: 40,
|
||||
sampleCount: 4
|
||||
})
|
||||
expect(rttLines()[0]).not.toContain(identity.relayHostId)
|
||||
|
||||
// Later samples keep feeding the fleet metric, but stay silent for an hour.
|
||||
for (let round = 0; round < 8; round++) await roundTrip()
|
||||
expect(h.observer.recordControlRtt).toHaveBeenCalledTimes(12)
|
||||
expect(rttLines()).toHaveLength(1)
|
||||
|
||||
const elapsedStart = clock.now
|
||||
while (clock.now - elapsedStart < 60 * 60 * 1000) await roundTrip()
|
||||
expect(rttLines()).toHaveLength(2)
|
||||
} finally {
|
||||
log.mockRestore()
|
||||
h.registry.drain(0)
|
||||
vi.advanceTimersByTime(0)
|
||||
}
|
||||
})
|
||||
|
||||
it('ignores a pong that answers no outstanding ping', async () => {
|
||||
const clock = { now: 1_700_000_000_000 }
|
||||
const h = harness({ now: () => clock.now })
|
||||
const control = await activeHost(h)
|
||||
try {
|
||||
// Nothing has been pinged yet, so even a plausible echo is not a round trip.
|
||||
control.emit('message', JSON.stringify({ type: 'pong' }), false)
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: 'later' }), false)
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: clock.now }), false)
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: clock.now - 10 }), false)
|
||||
expect(h.observer.recordControlRtt).not.toHaveBeenCalled()
|
||||
|
||||
const pingAt = await advanceToPing(control, clock)
|
||||
// A guessed timestamp is not the outstanding ping's `t`, so it is dropped.
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt - 1 }), false)
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt + 1 }), false)
|
||||
expect(h.observer.recordControlRtt).not.toHaveBeenCalled()
|
||||
|
||||
clock.now += 10
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt }), false)
|
||||
expect(h.observer.recordControlRtt).toHaveBeenCalledWith(10)
|
||||
} finally {
|
||||
h.registry.drain(0)
|
||||
vi.advanceTimersByTime(0)
|
||||
}
|
||||
})
|
||||
|
||||
it('records one sample per ping however many pongs a host floods', async () => {
|
||||
const clock = { now: 1_700_000_000_000 }
|
||||
const h = harness({ now: () => clock.now })
|
||||
const control = await activeHost(h)
|
||||
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
|
||||
try {
|
||||
const pingAt = await advanceToPing(control, clock)
|
||||
clock.now += 12
|
||||
for (let flood = 0; flood < 5_000; flood++) {
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt }), false)
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: clock.now }), false)
|
||||
}
|
||||
// One answered ping is one process-wide sample and one per-session sample, so
|
||||
// neither the metric window nor the hourly log line can be flooded.
|
||||
expect(h.observer.recordControlRtt).toHaveBeenCalledTimes(1)
|
||||
expect(h.observer.recordControlRtt).toHaveBeenCalledWith(12)
|
||||
expect(
|
||||
log.mock.calls.filter((call) => String(call[0]).includes('orca_relay_host_control_rtt'))
|
||||
).toHaveLength(0)
|
||||
} finally {
|
||||
log.mockRestore()
|
||||
h.registry.drain(0)
|
||||
vi.advanceTimersByTime(0)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('control lease jitter', () => {
|
||||
beforeEach(() => vi.useFakeTimers())
|
||||
afterEach(() => {
|
||||
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
ASSIGNMENT_LIMITS,
|
||||
CONTROL_CONTINUITY_LIMITS,
|
||||
RELAY_CLOSE_CODE,
|
||||
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS,
|
||||
RELAY_PROTOCOL_LIMITS
|
||||
} from '@orca-cloud/relay-contract'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
@@ -1005,3 +1006,115 @@ describe('control lease recovery after the session is gone', () => {
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('host hello ack pending connections', () => {
|
||||
const DETAILS = new Set([RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS])
|
||||
const LEGACY_ENTRY = { connId: 'conn-1', connTicket: 'T'.repeat(43) }
|
||||
const DETAILED_ENTRY = { ...LEGACY_ENTRY, kind: 'invite', relayDeviceId: 'device-1' }
|
||||
|
||||
beforeEach(() => vi.useFakeTimers())
|
||||
afterEach(() => {
|
||||
vi.clearAllTimers()
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
function newRegistry(): ReturnType<typeof createRegistry> {
|
||||
return createRegistry(
|
||||
vi
|
||||
.fn<RelayAssignmentStore['activateControl']>()
|
||||
.mockResolvedValue('control:production-gce-c3:1')
|
||||
)
|
||||
}
|
||||
|
||||
function addPendingConnection(session: HostSession): void {
|
||||
session.pendingConns.set('conn-1', {
|
||||
...LEGACY_ENTRY,
|
||||
reservation: {
|
||||
userId: identity.sub,
|
||||
relayHostId: identity.relayHostId,
|
||||
credentialKind: 'invite',
|
||||
relayDeviceId: 'device-1'
|
||||
},
|
||||
client: new FakeSocket() as unknown as WebSocket,
|
||||
attachTimer: setTimeout(() => {}, 60_000),
|
||||
credentialActivityId: null
|
||||
} as unknown as Parameters<typeof session.pendingConns.set>[1])
|
||||
}
|
||||
|
||||
function sentAck(socket: FakeSocket): Record<string, unknown> {
|
||||
const acks = socket.send.mock.calls
|
||||
.map((call) => JSON.parse(String(call[0])) as Record<string, unknown>)
|
||||
.filter((message) => message.type === 'host-hello-ack')
|
||||
return acks.at(-1)!
|
||||
}
|
||||
|
||||
function sessionOf(registry: HostSessionRegistry): HostSession {
|
||||
return registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
|
||||
}
|
||||
|
||||
async function ackFor(capabilities?: ReadonlySet<string>): Promise<Record<string, unknown>> {
|
||||
const { registry, activate } = newRegistry()
|
||||
const socket = new FakeSocket()
|
||||
registry.acceptControl(
|
||||
socket as unknown as WebSocket,
|
||||
identity,
|
||||
undefined,
|
||||
capabilities ?? new Set()
|
||||
)
|
||||
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
|
||||
const session = sessionOf(registry)
|
||||
addPendingConnection(session)
|
||||
socket.send.mockClear()
|
||||
;(registry as unknown as { sendHelloAck(session: HostSession): void }).sendHelloAck(session)
|
||||
return sentAck(socket)
|
||||
}
|
||||
|
||||
async function ackAfterRebind(
|
||||
first: ReadonlySet<string>,
|
||||
successor: ReadonlySet<string>
|
||||
): Promise<{ opening: Record<string, unknown>; rebound: Record<string, unknown> }> {
|
||||
const { registry, activate } = newRegistry()
|
||||
const opening = new FakeSocket()
|
||||
registry.acceptControl(opening as unknown as WebSocket, identity, undefined, first)
|
||||
await activate(opening as unknown as WebSocket, identity, null, 1, false, 1)
|
||||
const session = sessionOf(registry)
|
||||
addPendingConnection(session)
|
||||
opening.send.mockClear()
|
||||
;(registry as unknown as { sendHelloAck(session: HostSession): void }).sendHelloAck(session)
|
||||
|
||||
const rebound = new FakeSocket()
|
||||
registry.acceptControl(rebound as unknown as WebSocket, identity, undefined, successor)
|
||||
await activate(rebound as unknown as WebSocket, identity, session, 1, true, 1)
|
||||
return { opening: sentAck(opening), rebound: sentAck(rebound) }
|
||||
}
|
||||
|
||||
it('states the pending kind and device to a host that advertised it can read them', async () => {
|
||||
const ack = await ackFor(DETAILS)
|
||||
|
||||
expect(ack.pendingConns).toEqual([DETAILED_ENTRY])
|
||||
})
|
||||
|
||||
it('restates only the identifiers to a host that never advertised the capability', async () => {
|
||||
// A shipped host parses these entries strictly, so an unannounced key fails
|
||||
// the whole ack parse and kills a control that was working.
|
||||
const ack = await ackFor()
|
||||
|
||||
expect(ack.pendingConns).toEqual([LEGACY_ENTRY])
|
||||
})
|
||||
|
||||
it('downgrades the restated entry when the successor control drops the capability', async () => {
|
||||
// The capability belongs to the socket, not the session: a rebind can land a
|
||||
// control whose decoder is older than the one that opened the session.
|
||||
const { opening, rebound } = await ackAfterRebind(DETAILS, new Set())
|
||||
|
||||
expect(opening.pendingConns).toEqual([DETAILED_ENTRY])
|
||||
expect(rebound.pendingConns).toEqual([LEGACY_ENTRY])
|
||||
})
|
||||
|
||||
it('upgrades the restated entry when the successor control adds the capability', async () => {
|
||||
const { opening, rebound } = await ackAfterRebind(new Set(), DETAILS)
|
||||
|
||||
expect(opening.pendingConns).toEqual([LEGACY_ENTRY])
|
||||
expect(rebound.pendingConns).toEqual([DETAILED_ENTRY])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { createHash, createHmac, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto'
|
||||
import {
|
||||
ASSIGNMENT_LIMITS,
|
||||
RELAY_DEFAULT_REGION,
|
||||
AuthRefreshSchema,
|
||||
buildHostChallengePlaintext,
|
||||
buildHostProofMacInput,
|
||||
@@ -13,9 +14,11 @@ import {
|
||||
HostChallengeAckSchema,
|
||||
HostHelloSchema,
|
||||
InviteCreateSchema,
|
||||
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS,
|
||||
RELAY_PROTOCOL_LIMITS,
|
||||
RELAY_CLOSE_CODE,
|
||||
type RelayHostCloseReason
|
||||
type RelayHostCloseReason,
|
||||
type RelayRegion
|
||||
} from '@orca-cloud/relay-contract'
|
||||
import nacl from 'tweetnacl'
|
||||
import type WebSocket from 'ws'
|
||||
@@ -29,7 +32,12 @@ import {
|
||||
import { HostCloseReasonMemory } from './host-close-reason-memory.js'
|
||||
import { relayHostLogDigest } from './relay-host-log-digest.js'
|
||||
import type { RelayTokenClaims } from './relay-token-verifier.js'
|
||||
import type { RelayClientAcceptStage, RelayRuntimeObserver } from './relay-observability.js'
|
||||
import {
|
||||
percentile,
|
||||
type RelayClientAcceptStage,
|
||||
type RelayClientAcceptTimedStage,
|
||||
type RelayRuntimeObserver
|
||||
} from './relay-observability.js'
|
||||
import type { PendingHostDataReservation } from './relay-connection-ledger.js'
|
||||
import { closeRelayWebSocket } from './relay-websocket-close.js'
|
||||
import { ProcessQueuedByteBudget, wireSplice } from './splice-forwarder.js'
|
||||
@@ -45,6 +53,20 @@ function printableCloseReason(reason: Buffer | string): string {
|
||||
type VerifyRelayToken = (token: string) => Promise<RelayTokenClaims | null>
|
||||
type HostState = 'proving' | 'active' | 'orphaned' | 'drain-only' | 'closed'
|
||||
|
||||
// A host's distance to its cell moves on the scale of a rehome, not a heartbeat,
|
||||
// so a short window is enough to ride out one stalled ping.
|
||||
const CONTROL_RTT_WINDOW = 8
|
||||
const CONTROL_RTT_LOG_SAMPLE_THRESHOLD = 4
|
||||
const CONTROL_RTT_LOG_INTERVAL_MS = 60 * 60 * 1000
|
||||
// A pong claiming a multi-minute round trip is clock skew, not distance.
|
||||
const CONTROL_RTT_MAX_PLAUSIBLE_MS = 120_000
|
||||
|
||||
// Wall clock can step backwards mid-accept; a negative latency would poison the
|
||||
// percentiles it feeds.
|
||||
function nonNegativeMs(elapsedMs: number): number {
|
||||
return Math.max(0, elapsedMs)
|
||||
}
|
||||
|
||||
const CONTROL_ACTIVITY_RENEWAL_INTERVAL_MS = RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2
|
||||
// Preserve the existing 75s renewal runway after doubling the successful-call interval.
|
||||
const CONTROL_ACTIVITY_LEASE_MS =
|
||||
@@ -68,6 +90,10 @@ export type HostSession = {
|
||||
orphanTimer: ReturnType<typeof setTimeout> | null
|
||||
heartbeatTimer: ReturnType<typeof setInterval> | null
|
||||
lastPongAt: number
|
||||
// The `t` of the ping still waiting for its echo; null once one has answered it.
|
||||
pendingPingAt: number | null
|
||||
controlRttSamplesMs: number[]
|
||||
controlRttLoggedAt: number | null
|
||||
activityRenewalDueAt: number
|
||||
activityRenewalAttempt: number
|
||||
activityRenewalCompletedAttempt: number
|
||||
@@ -95,6 +121,15 @@ type PendingConnection = {
|
||||
attachTimer: ReturnType<typeof setTimeout>
|
||||
credentialActivityId: string | null
|
||||
capacityReservation?: PendingHostDataReservation
|
||||
timing: ClientAcceptTiming
|
||||
}
|
||||
|
||||
// Carries the phone-side accept clock across to the desktop's data leg, which
|
||||
// lands in a separate call and is the only place the accept is known to succeed.
|
||||
type ClientAcceptTiming = {
|
||||
startedAt: number
|
||||
connOpenAt: number
|
||||
stageMs: Record<RelayClientAcceptStage, number>
|
||||
}
|
||||
|
||||
function decodeCanonicalBase64(value: string, bytes: number): Uint8Array | null {
|
||||
@@ -146,6 +181,7 @@ export class HostSessionRegistry {
|
||||
// but a signed-out desktop never comes back, so the phone that asks minutes
|
||||
// later would otherwise find nothing to explain its rejection with.
|
||||
private readonly hostCloseReasons = new HostCloseReasonMemory(() => this.now())
|
||||
private readonly hostCapabilities = new WeakMap<WebSocket, ReadonlySet<string>>()
|
||||
private draining = false
|
||||
|
||||
constructor(
|
||||
@@ -192,6 +228,17 @@ export class HostSessionRegistry {
|
||||
)
|
||||
return true
|
||||
}
|
||||
const stageMs: Record<RelayClientAcceptStage, number> = {
|
||||
assignment: 0,
|
||||
credential: 0,
|
||||
activity: 0
|
||||
}
|
||||
let stageCursor = acceptStartedAt
|
||||
const markStage = (stage: RelayClientAcceptStage): void => {
|
||||
const at = this.now()
|
||||
stageMs[stage] = at - stageCursor
|
||||
stageCursor = at
|
||||
}
|
||||
if (this.config.role === 'cell') {
|
||||
// Each lookup is its own pooled round trip; stop between them once the phone
|
||||
// has left instead of running the rest of the chain for nobody.
|
||||
@@ -212,6 +259,7 @@ export class HostSessionRegistry {
|
||||
}
|
||||
if (abandonedByClient('assignment')) return
|
||||
}
|
||||
markStage('assignment')
|
||||
const reservation = await this.store.reserveCredential(hostId, credential)
|
||||
if (!reservation) {
|
||||
capacityReservation?.release()
|
||||
@@ -221,6 +269,7 @@ export class HostSessionRegistry {
|
||||
}
|
||||
this.observer.recordAuth(true)
|
||||
if (abandonedByClient('credential', () => this.failReservationBestEffort(reservation))) return
|
||||
markStage('credential')
|
||||
const sessionKey = this.key(reservation.userId, hostId)
|
||||
const session = this.sessions.get(sessionKey)
|
||||
if (
|
||||
@@ -275,6 +324,7 @@ export class HostSessionRegistry {
|
||||
) {
|
||||
return
|
||||
}
|
||||
markStage('activity')
|
||||
const attachTimer = setTimeout(() => {
|
||||
session.pendingConns.delete(connId)
|
||||
capacityReservation?.release()
|
||||
@@ -289,7 +339,10 @@ export class HostSessionRegistry {
|
||||
client: socket,
|
||||
attachTimer,
|
||||
credentialActivityId,
|
||||
capacityReservation
|
||||
capacityReservation,
|
||||
// Attach starts where the activity stage ended, so the conn-open send is
|
||||
// charged to it and no wall-clock gap goes unattributed.
|
||||
timing: { startedAt: acceptStartedAt, connOpenAt: stageCursor, stageMs }
|
||||
}
|
||||
capacityReservation?.bind(connId)
|
||||
session.pendingConns.set(connId, pending)
|
||||
@@ -336,6 +389,7 @@ export class HostSessionRegistry {
|
||||
return false
|
||||
}
|
||||
this.observer.recordAuth(true)
|
||||
const attachedAt = this.now()
|
||||
clearTimeout(pending.attachTimer)
|
||||
session.pendingConns.delete(connId)
|
||||
session.activeConnIds.add(connId)
|
||||
@@ -409,6 +463,7 @@ export class HostSessionRegistry {
|
||||
close()
|
||||
return false
|
||||
}
|
||||
const helloAt = this.now()
|
||||
send(pending.client, 'relay-hello', {
|
||||
ok: true,
|
||||
credentialKind: pending.reservation.credentialKind,
|
||||
@@ -424,14 +479,92 @@ export class HostSessionRegistry {
|
||||
}
|
||||
: {})
|
||||
})
|
||||
this.recordClientAcceptCompleted(session, pending, attachedAt, helloAt)
|
||||
return true
|
||||
}
|
||||
|
||||
// The stages tile the whole accept, so their sum is the total minus only the
|
||||
// clamping above: `basis` is the splice lease and connection-basis writes that
|
||||
// land between the host data leg and relay-hello.
|
||||
private recordClientAcceptCompleted(
|
||||
session: HostSession,
|
||||
pending: PendingConnection,
|
||||
attachedAt: number,
|
||||
helloAt: number
|
||||
): void {
|
||||
const stageMs: Record<RelayClientAcceptTimedStage, number> = {
|
||||
assignment: nonNegativeMs(pending.timing.stageMs.assignment),
|
||||
credential: nonNegativeMs(pending.timing.stageMs.credential),
|
||||
activity: nonNegativeMs(pending.timing.stageMs.activity),
|
||||
attach: nonNegativeMs(attachedAt - pending.timing.connOpenAt),
|
||||
basis: nonNegativeMs(helloAt - attachedAt)
|
||||
}
|
||||
const totalMs = nonNegativeMs(helloAt - pending.timing.startedAt)
|
||||
this.observer.recordClientAcceptCompleted?.({ totalMs, stageMs })
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_client_accept_completed',
|
||||
...this.logIdentity(),
|
||||
credentialKind: pending.reservation.credentialKind,
|
||||
stageMs,
|
||||
totalMs,
|
||||
relayHostIdDigest: relayHostLogDigest(session.relayHostId)
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// Matches the runtime metrics event so a log line and a metric point can be
|
||||
// joined back to the process that emitted them.
|
||||
private logIdentity(): { role: string; cellId: string; region: RelayRegion } {
|
||||
return {
|
||||
role: this.config.role,
|
||||
cellId: this.config.cellId,
|
||||
region: this.config.region ?? RELAY_DEFAULT_REGION
|
||||
}
|
||||
}
|
||||
|
||||
// Every desktop build already echoes the ping's `t`, so a pong is only timed when
|
||||
// it answers the outstanding ping: at most one sample per ping this cell sent,
|
||||
// however many a host floods. A pong that lost the race to the next ping is
|
||||
// dropped here but still counts as proof of life for the silence watchdog.
|
||||
private recordControlRtt(session: HostSession, echoedPingAt: unknown): void {
|
||||
if (typeof echoedPingAt !== 'number' || echoedPingAt !== session.pendingPingAt) return
|
||||
session.pendingPingAt = null
|
||||
const now = this.now()
|
||||
const rttMs = now - echoedPingAt
|
||||
if (rttMs < 0 || rttMs > CONTROL_RTT_MAX_PLAUSIBLE_MS) return
|
||||
this.observer.recordControlRtt?.(rttMs)
|
||||
const samples = session.controlRttSamplesMs
|
||||
samples.push(rttMs)
|
||||
if (samples.length > CONTROL_RTT_WINDOW) samples.shift()
|
||||
if (samples.length < CONTROL_RTT_LOG_SAMPLE_THRESHOLD) return
|
||||
if (
|
||||
session.controlRttLoggedAt !== null &&
|
||||
now - session.controlRttLoggedAt < CONTROL_RTT_LOG_INTERVAL_MS
|
||||
) {
|
||||
return
|
||||
}
|
||||
session.controlRttLoggedAt = now
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_host_control_rtt',
|
||||
...this.logIdentity(),
|
||||
relayHostIdDigest: relayHostLogDigest(session.relayHostId),
|
||||
rttMsMedian: percentile(samples, 0.5),
|
||||
sampleCount: samples.length
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
acceptControl(
|
||||
socket: WebSocket,
|
||||
identity: RelayTokenClaims,
|
||||
connectionInclusionWatermark?: number
|
||||
connectionInclusionWatermark?: number,
|
||||
hostCapabilities?: ReadonlySet<string>
|
||||
): void {
|
||||
// Keyed by socket, not session: a rebind swaps the session's socket, and the
|
||||
// successor's own advertisement is the only one that describes its decoder.
|
||||
if (hostCapabilities?.size) this.hostCapabilities.set(socket, hostCapabilities)
|
||||
if (this.draining) {
|
||||
socket.close(RELAY_CLOSE_CODE.DRAINING, 'relay draining')
|
||||
return
|
||||
@@ -790,6 +923,7 @@ export class HostSessionRegistry {
|
||||
existing.appVersion = appVersion
|
||||
existing.leaseExpiresAt = this.controlLeaseExpiresAt()
|
||||
existing.lastPongAt = this.now()
|
||||
existing.pendingPingAt = null
|
||||
existing.activityRenewalDueAt =
|
||||
this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs
|
||||
this.wireActiveControl(existing)
|
||||
@@ -843,6 +977,9 @@ export class HostSessionRegistry {
|
||||
orphanTimer: null,
|
||||
heartbeatTimer: null,
|
||||
lastPongAt: this.now(),
|
||||
pendingPingAt: null,
|
||||
controlRttSamplesMs: [],
|
||||
controlRttLoggedAt: null,
|
||||
activityRenewalDueAt: this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs,
|
||||
activityRenewalAttempt: 0,
|
||||
activityRenewalCompletedAttempt: 0,
|
||||
@@ -900,6 +1037,7 @@ export class HostSessionRegistry {
|
||||
const parsed = JSON.parse(raw.toString()) as Record<string, unknown>
|
||||
if (parsed.type === 'pong') {
|
||||
session.lastPongAt = this.now()
|
||||
this.recordControlRtt(session, parsed.t)
|
||||
return
|
||||
}
|
||||
if (parsed.type === 'auth-refresh') {
|
||||
@@ -1047,11 +1185,18 @@ export class HostSessionRegistry {
|
||||
session.socket.close(RELAY_CLOSE_CODE.DRAINING, 'control lease expired')
|
||||
return
|
||||
}
|
||||
session.pendingPingAt = now
|
||||
send(session.socket, 'ping', { t: now })
|
||||
}
|
||||
|
||||
private sendHelloAck(session: HostSession): void {
|
||||
if (!session.socket) return
|
||||
// Without these a host that missed the conn-open cannot dial the pending
|
||||
// connection: it would have to guess the pairing kind and the device the
|
||||
// relay authorized. Only sent to a host that said it can read them.
|
||||
const details = this.hostCapabilities
|
||||
.get(session.socket)
|
||||
?.has(RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS)
|
||||
send(session.socket, 'host-hello-ack', {
|
||||
v: 1,
|
||||
generation: session.generation,
|
||||
@@ -1060,7 +1205,13 @@ export class HostSessionRegistry {
|
||||
activeConnIds: [...session.activeConnIds],
|
||||
pendingConns: [...session.pendingConns.values()].map((pending) => ({
|
||||
connId: pending.connId,
|
||||
connTicket: pending.connTicket
|
||||
connTicket: pending.connTicket,
|
||||
...(details
|
||||
? {
|
||||
kind: pending.reservation.credentialKind,
|
||||
relayDeviceId: pending.reservation.relayDeviceId
|
||||
}
|
||||
: {})
|
||||
}))
|
||||
})
|
||||
}
|
||||
|
||||
@@ -49,6 +49,20 @@ function concurrentCreateCollision(
|
||||
return false
|
||||
}
|
||||
|
||||
const ALTER_TABLE_ADD_CONSTRAINT =
|
||||
/^\s*ALTER\s+TABLE\s+\S+\s+ADD\s+CONSTRAINT\b/i
|
||||
|
||||
// Postgres has no `ADD CONSTRAINT IF NOT EXISTS`, so a re-run and a concurrent
|
||||
// startup both land on 42710 once the constraint exists. Unlike a CREATE race
|
||||
// this is terminal, not transient: retrying only repeats it, so the statement
|
||||
// counts as applied.
|
||||
function constraintAlreadyApplied(error: unknown, statement: string): boolean {
|
||||
return (
|
||||
ALTER_TABLE_ADD_CONSTRAINT.test(statement) &&
|
||||
(error as { code?: unknown }).code === '42710'
|
||||
)
|
||||
}
|
||||
|
||||
function retryableSchemaError(error: unknown, statement: string): boolean {
|
||||
const value = error as { code?: unknown; constraint?: unknown }
|
||||
return (
|
||||
@@ -73,6 +87,7 @@ export async function applyPostgresSchema(
|
||||
await query(statement)
|
||||
break
|
||||
} catch (error) {
|
||||
if (constraintAlreadyApplied(error, statement)) break
|
||||
const code = String((error as { code?: unknown }).code)
|
||||
const remainingMs = deadlineAt - now()
|
||||
const retryable = retryableSchemaError(error, statement)
|
||||
|
||||
@@ -315,6 +315,7 @@ describe('regional rehome director controls', () => {
|
||||
notBefore: 100,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 24 * 60 * 60_000,
|
||||
hostCooldownMs: 7 * 24 * 60 * 60_000,
|
||||
drainGraceMs: 60_000,
|
||||
confirmation: 'ENABLE_REGIONAL_REHOMING'
|
||||
}
|
||||
@@ -343,6 +344,14 @@ describe('regional rehome director controls', () => {
|
||||
'deploy-token',
|
||||
{ ...apply, confirmation: 'DISABLE_REGIONAL_REHOMING' }
|
||||
)).status).toBe(400)
|
||||
// The per-host cooldown is part of the durable shape an operator must state.
|
||||
const { hostCooldownMs: _omitted, ...withoutCooldown } = apply
|
||||
expect((await postPath(
|
||||
app,
|
||||
'/v1/admin/regional-rehome-control',
|
||||
'deploy-token',
|
||||
withoutCooldown
|
||||
)).status).toBe(400)
|
||||
})
|
||||
|
||||
it('probes dedicated trust twice and returns only aggregate proof', async () => {
|
||||
@@ -411,6 +420,78 @@ describe('regional rehome director controls', () => {
|
||||
expect(JSON.stringify(responseBody)).not.toContain('rehome-token')
|
||||
})
|
||||
|
||||
it('probes a source cell in any region, not only the default one', async () => {
|
||||
// Rehoming moves hosts in both directions, so an asia-east2 cell is a
|
||||
// source too and its trust has to be provable the same way.
|
||||
const cellDeploymentStatus = vi.fn().mockResolvedValue({
|
||||
cellId: 'production-gce-c27',
|
||||
cellUrl: 'https://c27.relay.example.test',
|
||||
region: 'asia-east2',
|
||||
runtime: {
|
||||
cellIncarnation,
|
||||
ready: true,
|
||||
heartbeatFresh: true,
|
||||
regionalRehomeProtocol: 1
|
||||
}
|
||||
})
|
||||
const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
|
||||
store: {} as never,
|
||||
assignments: { cellDeploymentStatus } as never,
|
||||
drain: vi.fn(),
|
||||
regionalRehomeIdentityToken: vi.fn(async () => 'rehome-token'),
|
||||
regionalRehomeFetch: (async () =>
|
||||
Response.json({
|
||||
v: 1,
|
||||
outcome: 'host-not-connected',
|
||||
sharedRuntimeIdentityRejected: true
|
||||
})) as typeof fetch,
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
|
||||
const response = await postPath(
|
||||
app,
|
||||
'/v1/admin/regional-rehome-trust-probe',
|
||||
'deploy-token',
|
||||
{ v: 1, sourceCellId: 'production-gce-c27', sourceCellIncarnation: cellIncarnation }
|
||||
)
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(await response.json()).toMatchObject({ proven: true })
|
||||
})
|
||||
|
||||
it('still refuses a trust probe against a cell without the drain protocol', async () => {
|
||||
const cellDeploymentStatus = vi.fn().mockResolvedValue({
|
||||
cellId: 'production-gce-c27',
|
||||
cellUrl: 'https://c27.relay.example.test',
|
||||
region: 'asia-east2',
|
||||
runtime: {
|
||||
cellIncarnation,
|
||||
ready: true,
|
||||
heartbeatFresh: true,
|
||||
regionalRehomeProtocol: 0
|
||||
}
|
||||
})
|
||||
const sourceFetch = vi.fn<typeof fetch>()
|
||||
const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
|
||||
store: {} as never,
|
||||
assignments: { cellDeploymentStatus } as never,
|
||||
drain: vi.fn(),
|
||||
regionalRehomeIdentityToken: vi.fn(async () => 'rehome-token'),
|
||||
regionalRehomeFetch: sourceFetch,
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
|
||||
const response = await postPath(
|
||||
app,
|
||||
'/v1/admin/regional-rehome-trust-probe',
|
||||
'deploy-token',
|
||||
{ v: 1, sourceCellId: 'production-gce-c27', sourceCellIncarnation: cellIncarnation }
|
||||
)
|
||||
|
||||
expect(response.status).toBe(409)
|
||||
expect(sourceFetch).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('restricts trust probes to deploy authorization and strict input', async () => {
|
||||
const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
|
||||
store: {} as never,
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
import pg from 'pg'
|
||||
import { afterAll, beforeEach, describe, expect, it } from 'vitest'
|
||||
import {
|
||||
openRelayDatabase,
|
||||
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS,
|
||||
type RelayDatabase
|
||||
} from './database.js'
|
||||
|
||||
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
const describePostgres = databaseUrl ? describe : describe.skip
|
||||
const schema = 'relay_rehome_constraint_migration_test'
|
||||
|
||||
// The shape shipped before rehoming became bidirectional: a single-region
|
||||
// column check that Postgres auto-names.
|
||||
const LEGACY_ATTEMPTS_TABLE = `
|
||||
CREATE TABLE relay_region_rehome_attempts (
|
||||
attempt_id TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL,
|
||||
relay_host_id TEXT NOT NULL,
|
||||
preferred_region TEXT NOT NULL CHECK (preferred_region = 'asia-east2'),
|
||||
source_cell_id TEXT NOT NULL,
|
||||
source_cell_incarnation TEXT NOT NULL,
|
||||
target_cell_id TEXT NOT NULL,
|
||||
target_cell_incarnation TEXT NOT NULL,
|
||||
previous_epoch BIGINT NOT NULL,
|
||||
assignment_epoch BIGINT NOT NULL,
|
||||
drain_grace_ms BIGINT NOT NULL,
|
||||
send_attempts BIGINT NOT NULL,
|
||||
last_send_attempt_at BIGINT,
|
||||
drain_receipt_at BIGINT,
|
||||
drain_outcome TEXT CHECK (
|
||||
drain_outcome IN ('accepted', 'already-accepted', 'host-not-connected')
|
||||
),
|
||||
completed_at BIGINT,
|
||||
aborted_at BIGINT,
|
||||
created_at BIGINT NOT NULL,
|
||||
updated_at BIGINT NOT NULL,
|
||||
UNIQUE (user_id, relay_host_id, assignment_epoch)
|
||||
)`
|
||||
|
||||
// The control row as it shipped before the per-host cooldown existed.
|
||||
const LEGACY_CONTROL_TABLE = `
|
||||
CREATE TABLE relay_region_rehome_control (
|
||||
control_id TEXT PRIMARY KEY,
|
||||
generation BIGINT NOT NULL,
|
||||
enabled BIGINT NOT NULL,
|
||||
observation_started_at BIGINT NOT NULL,
|
||||
not_before BIGINT NOT NULL,
|
||||
rate_per_minute BIGINT NOT NULL,
|
||||
preference_max_age_ms BIGINT NOT NULL,
|
||||
drain_grace_ms BIGINT NOT NULL,
|
||||
updated_at BIGINT NOT NULL
|
||||
)`
|
||||
|
||||
const attemptValues = (attemptId: string, preferredRegion: string): unknown[] => [
|
||||
attemptId,
|
||||
'user-1',
|
||||
'abcdefghijklmnop',
|
||||
preferredRegion,
|
||||
'cell-source',
|
||||
'11111111-1111-4111-8111-111111111111',
|
||||
'cell-target',
|
||||
'22222222-2222-4222-8222-222222222222',
|
||||
1,
|
||||
Number(attemptId.at(-1)),
|
||||
0,
|
||||
0,
|
||||
1_000_000,
|
||||
1_000_000
|
||||
]
|
||||
|
||||
const INSERT_ATTEMPT = `INSERT INTO relay_region_rehome_attempts
|
||||
(attempt_id, user_id, relay_host_id, preferred_region, source_cell_id,
|
||||
source_cell_incarnation, target_cell_id, target_cell_incarnation,
|
||||
previous_epoch, assignment_epoch, drain_grace_ms, send_attempts,
|
||||
created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`
|
||||
|
||||
describePostgres('PostgreSQL regional rehome constraint migration', () => {
|
||||
let scopedUrl = ''
|
||||
|
||||
async function withClient(
|
||||
operation: (client: pg.Client) => Promise<void>
|
||||
): Promise<void> {
|
||||
const client = new pg.Client({ connectionString: databaseUrl })
|
||||
await client.connect()
|
||||
try {
|
||||
await operation(client)
|
||||
} finally {
|
||||
await client.end()
|
||||
}
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
await withClient(async (client) => {
|
||||
await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
|
||||
await client.query(`CREATE SCHEMA ${schema}`)
|
||||
await client.query(`SET search_path = ${schema}`)
|
||||
await client.query(LEGACY_ATTEMPTS_TABLE)
|
||||
await client.query(LEGACY_CONTROL_TABLE)
|
||||
await client.query(
|
||||
`INSERT INTO relay_region_rehome_control
|
||||
(control_id, generation, enabled, observation_started_at, not_before,
|
||||
rate_per_minute, preference_max_age_ms, drain_grace_ms, updated_at)
|
||||
VALUES ('global', 3, 0, 1, 0, 10, 86400000, 60000, 1)`
|
||||
)
|
||||
// Production data the replacement constraint has to validate.
|
||||
await client.query(INSERT_ATTEMPT, attemptValues('attempt-1', 'asia-east2'))
|
||||
})
|
||||
const url = new URL(databaseUrl!)
|
||||
url.searchParams.set('options', `-c search_path=${schema}`)
|
||||
scopedUrl = url.toString()
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
await withClient(async (client) => {
|
||||
await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
|
||||
})
|
||||
})
|
||||
|
||||
it('upgrades a legacy single-region constraint in place', async () => {
|
||||
const database = await openRelayDatabase({ databaseUrl: scopedUrl, dataDir: '' })
|
||||
try {
|
||||
await withClient(async (client) => {
|
||||
await client.query(`SET search_path = ${schema}`)
|
||||
await client.query(INSERT_ATTEMPT, attemptValues('attempt-2', 'us-central1'))
|
||||
await expect(
|
||||
client.query(INSERT_ATTEMPT, attemptValues('attempt-3', 'europe-west1'))
|
||||
).rejects.toMatchObject({ code: '23514' })
|
||||
const constraints = await client.query(
|
||||
`SELECT conname FROM pg_constraint
|
||||
WHERE conrelid = 'relay_region_rehome_attempts'::regclass
|
||||
AND conname LIKE '%preferred_region%'
|
||||
ORDER BY conname`
|
||||
)
|
||||
expect(constraints.rows).toEqual([
|
||||
{ conname: 'relay_region_rehome_attempts_preferred_region_valid' }
|
||||
])
|
||||
// The existing control row keeps its tuning and gains the cooldown.
|
||||
const control = await client.query(
|
||||
`SELECT generation, preference_max_age_ms, host_cooldown_ms
|
||||
FROM relay_region_rehome_control WHERE control_id = 'global'`
|
||||
)
|
||||
expect(control.rows).toEqual([
|
||||
{
|
||||
generation: '3',
|
||||
preference_max_age_ms: '86400000',
|
||||
host_cooldown_ms: String(REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS)
|
||||
}
|
||||
])
|
||||
})
|
||||
} finally {
|
||||
await database.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('upgrades once across concurrent startups', async () => {
|
||||
const results = await Promise.allSettled(
|
||||
Array.from(
|
||||
{ length: 5 },
|
||||
async (): Promise<RelayDatabase> =>
|
||||
await openRelayDatabase({ databaseUrl: scopedUrl, dataDir: '' })
|
||||
)
|
||||
)
|
||||
const databases = results.flatMap((result) =>
|
||||
result.status === 'fulfilled' ? [result.value] : []
|
||||
)
|
||||
await Promise.all(databases.map(async (database) => await database.close()))
|
||||
|
||||
expect(
|
||||
results.flatMap((result) =>
|
||||
result.status === 'rejected'
|
||||
? [
|
||||
{
|
||||
code: (result.reason as { code?: unknown }).code,
|
||||
message: String(result.reason)
|
||||
}
|
||||
]
|
||||
: []
|
||||
)
|
||||
).toEqual([])
|
||||
await withClient(async (client) => {
|
||||
await client.query(`SET search_path = ${schema}`)
|
||||
await client.query(INSERT_ATTEMPT, attemptValues('attempt-4', 'us-central1'))
|
||||
const constraints = await client.query(
|
||||
`SELECT conname FROM pg_constraint
|
||||
WHERE conrelid = 'relay_region_rehome_attempts'::regclass
|
||||
AND conname LIKE '%preferred_region%'`
|
||||
)
|
||||
expect(constraints.rows).toEqual([
|
||||
{ conname: 'relay_region_rehome_attempts_preferred_region_valid' }
|
||||
])
|
||||
})
|
||||
}, 60_000)
|
||||
})
|
||||
@@ -81,6 +81,153 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
expect(await context.store.claimRegionalRehome()).not.toBeNull()
|
||||
})
|
||||
|
||||
it('moves a us-central1 host onto a cell in its preferred asia-east2 region', async () => {
|
||||
const context = await fixture()
|
||||
|
||||
const attempt = await context.store.claimRegionalRehome()
|
||||
expect(attempt).toMatchObject({
|
||||
preferredRegion: 'asia-east2',
|
||||
sourceCellId: context.source.id,
|
||||
targetCellId: context.target.id
|
||||
})
|
||||
expect(await primary.query(
|
||||
`SELECT preferred_region, source_cell_id, target_cell_id
|
||||
FROM relay_region_rehome_attempts WHERE user_id = ?`,
|
||||
[context.identity.userId]
|
||||
)).toEqual([{
|
||||
preferred_region: 'asia-east2',
|
||||
source_cell_id: context.source.id,
|
||||
target_cell_id: context.target.id
|
||||
}])
|
||||
})
|
||||
|
||||
it('moves an asia-east2 host back onto a cell in its preferred us-central1 region', async () => {
|
||||
const context = await fixture({
|
||||
sourceRegion: 'asia-east2',
|
||||
targetRegion: 'us-central1'
|
||||
})
|
||||
|
||||
const attempt = await context.store.claimRegionalRehome()
|
||||
expect(attempt).toMatchObject({
|
||||
preferredRegion: 'us-central1',
|
||||
sourceCellId: context.source.id,
|
||||
targetCellId: context.target.id
|
||||
})
|
||||
// The durable attempt row must accept the reverse direction too.
|
||||
expect(await primary.query(
|
||||
`SELECT preferred_region, source_cell_id, target_cell_id
|
||||
FROM relay_region_rehome_attempts WHERE user_id = ?`,
|
||||
[context.identity.userId]
|
||||
)).toEqual([{
|
||||
preferred_region: 'us-central1',
|
||||
source_cell_id: context.source.id,
|
||||
target_cell_id: context.target.id
|
||||
}])
|
||||
expect(await primary.query(
|
||||
`SELECT cell_id FROM relay_assignments WHERE user_id = ?`,
|
||||
[context.identity.userId]
|
||||
)).toEqual([{ cell_id: context.target.id }])
|
||||
})
|
||||
|
||||
it('leaves a host whose preference already matches its own region', async () => {
|
||||
const context = await fixture({ preferredRegion: 'us-central1' })
|
||||
|
||||
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
|
||||
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
|
||||
generation: 1,
|
||||
enabled: true
|
||||
})
|
||||
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
|
||||
attempts: 0,
|
||||
migrations: 0
|
||||
})
|
||||
})
|
||||
|
||||
it('leaves a host whose preference is older than the configured max age', async () => {
|
||||
const context = await fixture()
|
||||
await primary.query(
|
||||
`UPDATE relay_assignment_region_preferences SET observed_at = ?
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[
|
||||
context.now() - 24 * 60 * 60_000 - 1,
|
||||
context.identity.userId,
|
||||
context.identity.relayHostId
|
||||
]
|
||||
)
|
||||
|
||||
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
|
||||
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
|
||||
generation: 1,
|
||||
enabled: true
|
||||
})
|
||||
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
|
||||
attempts: 0,
|
||||
migrations: 0
|
||||
})
|
||||
})
|
||||
|
||||
it('leaves a host inside its per-host rehome cooldown, in either direction', async () => {
|
||||
const context = await fixture({ hostCooldownMs: 3 * 24 * 60 * 60_000 })
|
||||
// A move this host already made, whichever way it went.
|
||||
await primary.query(
|
||||
`INSERT INTO relay_region_rehome_attempts
|
||||
(attempt_id, user_id, relay_host_id, preferred_region, source_cell_id,
|
||||
source_cell_incarnation, target_cell_id, target_cell_incarnation,
|
||||
previous_epoch, assignment_epoch, drain_grace_ms, send_attempts,
|
||||
completed_at, created_at, updated_at)
|
||||
VALUES (?, ?, ?, 'us-central1', ?, ?, ?, ?, 0, 1, 0, 0, ?, ?, ?)`,
|
||||
[
|
||||
`pg-rehome-cooldown-${context.identity.relayHostId}`,
|
||||
context.identity.userId,
|
||||
context.identity.relayHostId,
|
||||
context.target.id,
|
||||
'22222222-2222-4222-8222-222222222222',
|
||||
context.source.id,
|
||||
'11111111-1111-4111-8111-111111111111',
|
||||
context.now(),
|
||||
context.now() - 3 * 24 * 60 * 60_000 + 1,
|
||||
context.now()
|
||||
]
|
||||
)
|
||||
|
||||
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
|
||||
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
|
||||
generation: 1,
|
||||
enabled: true,
|
||||
hostCooldownMs: 3 * 24 * 60 * 60_000
|
||||
})
|
||||
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
|
||||
attempts: 1,
|
||||
migrations: 0
|
||||
})
|
||||
|
||||
// One millisecond past the window the same host is a candidate again.
|
||||
await primary.query(
|
||||
`UPDATE relay_region_rehome_attempts SET created_at = ? WHERE user_id = ?`,
|
||||
[context.now() - 3 * 24 * 60 * 60_000, context.identity.userId]
|
||||
)
|
||||
await expect(context.store.claimRegionalRehome()).resolves.toMatchObject({
|
||||
sourceCellId: context.source.id,
|
||||
targetCellId: context.target.id
|
||||
})
|
||||
})
|
||||
|
||||
it('leaves a host whose preferred region holds no drainable cell', async () => {
|
||||
// A cell that cannot be drained cannot be a target: the host would land
|
||||
// where no later rehome could move it out again.
|
||||
const context = await fixture({ targetProtocol: 0 })
|
||||
|
||||
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
|
||||
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
|
||||
generation: 1,
|
||||
enabled: true
|
||||
})
|
||||
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
|
||||
attempts: 0,
|
||||
migrations: 0
|
||||
})
|
||||
})
|
||||
|
||||
it('skips an unclean cell without latching the control off', async () => {
|
||||
const context = await fixture()
|
||||
await primary.query(
|
||||
@@ -281,7 +428,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
context.store,
|
||||
context.target,
|
||||
'22222222-2222-4222-8222-222222222222',
|
||||
0,
|
||||
1,
|
||||
900_000,
|
||||
2
|
||||
)
|
||||
@@ -322,7 +469,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
context.store,
|
||||
context.target,
|
||||
'44444444-4444-4444-8444-444444444444',
|
||||
0,
|
||||
1,
|
||||
context.now()
|
||||
)
|
||||
|
||||
@@ -341,7 +488,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
context.store,
|
||||
context.target,
|
||||
'22222222-2222-4222-8222-222222222222',
|
||||
0,
|
||||
1,
|
||||
900_000,
|
||||
2
|
||||
)
|
||||
@@ -414,6 +561,26 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
})
|
||||
})
|
||||
|
||||
async function attemptAndMigrationCounts(identity: {
|
||||
userId: string
|
||||
relayHostId: string
|
||||
}): Promise<{ attempts: number; migrations: number }> {
|
||||
const attempts = await primary.query(
|
||||
`SELECT COUNT(*) AS count FROM relay_region_rehome_attempts
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[identity.userId, identity.relayHostId]
|
||||
)
|
||||
const migrations = await primary.query(
|
||||
`SELECT COUNT(*) AS count FROM relay_assignment_migrations
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[identity.userId, identity.relayHostId]
|
||||
)
|
||||
return {
|
||||
attempts: Number(attempts[0]!.count),
|
||||
migrations: Number(migrations[0]!.count)
|
||||
}
|
||||
}
|
||||
|
||||
async function controlAccounting(identity: {
|
||||
userId: string
|
||||
relayHostId: string
|
||||
@@ -436,12 +603,15 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
}
|
||||
}
|
||||
|
||||
async function fixture() {
|
||||
async function fixture(options: FixtureOptions = {}) {
|
||||
sequence++
|
||||
let now = 1_000_000
|
||||
const suffix = String(sequence)
|
||||
const source = cell(suffix, 'source', 'us-central1')
|
||||
const target = cell(suffix, 'target', 'asia-east2')
|
||||
const sourceRegion = options.sourceRegion ?? 'us-central1'
|
||||
const targetRegion = options.targetRegion ?? 'asia-east2'
|
||||
const preferredRegion = options.preferredRegion ?? targetRegion
|
||||
const source = cell(suffix, 'source', sourceRegion)
|
||||
const target = cell(suffix, 'target', targetRegion)
|
||||
const store = new RelayAssignmentStore(primary, () => now, storeOptions)
|
||||
const competingStore = new RelayAssignmentStore(secondary, () => now, storeOptions)
|
||||
await store.inspectRegionalRehomeControl()
|
||||
@@ -452,6 +622,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
notBefore: now,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 24 * 60 * 60_000,
|
||||
hostCooldownMs: options.hostCooldownMs ?? 7 * 24 * 60 * 60_000,
|
||||
drainGraceMs: 60_000
|
||||
})
|
||||
await store.reconcileCells([source, target])
|
||||
@@ -466,21 +637,22 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
store,
|
||||
target,
|
||||
'22222222-2222-4222-8222-222222222222',
|
||||
0,
|
||||
options.targetProtocol ?? 1,
|
||||
900_000
|
||||
)
|
||||
const identity = {
|
||||
userId: `pg-rehome-user-${suffix}`,
|
||||
relayHostId: `rehomehost${suffix.padStart(6, '0')}`
|
||||
}
|
||||
const assignment = await store.assign(identity, undefined, 'us-central1')
|
||||
const assignment = await store.assign(identity, undefined, sourceRegion)
|
||||
const sourceControl = await store.activateControl(identity, {
|
||||
cellId: source.id,
|
||||
assignmentEpoch: assignment.assignmentEpoch,
|
||||
generation: 1
|
||||
})
|
||||
await store.assign(identity, 'asia-east2')
|
||||
await store.assign(identity, preferredRegion)
|
||||
return {
|
||||
preferredRegion,
|
||||
store,
|
||||
competingStore,
|
||||
identity,
|
||||
@@ -500,7 +672,16 @@ const storeOptions = {
|
||||
heartbeatTtlMs: 45_000
|
||||
}
|
||||
|
||||
function cell(suffix: string, role: string, region: 'us-central1' | 'asia-east2') {
|
||||
type Region = 'us-central1' | 'asia-east2'
|
||||
type FixtureOptions = {
|
||||
sourceRegion?: Region
|
||||
targetRegion?: Region
|
||||
preferredRegion?: Region
|
||||
targetProtocol?: number
|
||||
hostCooldownMs?: number
|
||||
}
|
||||
|
||||
function cell(suffix: string, role: string, region: Region) {
|
||||
return {
|
||||
id: `pg-rehome-cell-${suffix}-${role}`,
|
||||
url: `https://pg-rehome-${suffix}-${role}.example.test`,
|
||||
|
||||
@@ -81,6 +81,7 @@ describe('regional rehome assignment state', () => {
|
||||
notBefore: context.now(),
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 24 * 60 * 60_000,
|
||||
hostCooldownMs: 7 * 24 * 60 * 60_000,
|
||||
drainGraceMs: 60_000
|
||||
})).rejects.toThrow('regional_rehome_generation_mismatch')
|
||||
await expect(context.store.applyRegionalRehomeControl({
|
||||
@@ -89,6 +90,7 @@ describe('regional rehome assignment state', () => {
|
||||
notBefore: context.now(),
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 24 * 60 * 60_000,
|
||||
hostCooldownMs: 7 * 24 * 60 * 60_000,
|
||||
drainGraceMs: 60_000
|
||||
})).resolves.toMatchObject({ generation: 3, enabled: true })
|
||||
await context.database.close()
|
||||
@@ -207,7 +209,7 @@ describe('regional rehome assignment state', () => {
|
||||
databasePoolWaitersMax: 0,
|
||||
databasePoolWaitMsMax: 0
|
||||
})
|
||||
await heartbeat(context.store, target, targetIncarnation, 0, 2, {
|
||||
await heartbeat(context.store, target, targetIncarnation, 1, 2, {
|
||||
observedAt: context.now(),
|
||||
sqlFailures: 1,
|
||||
reconnects: 3,
|
||||
@@ -226,6 +228,23 @@ describe('regional rehome assignment state', () => {
|
||||
await context.database.close()
|
||||
})
|
||||
|
||||
it('counts only drainable cells as the rehome fleet, in every region', async () => {
|
||||
// The fleet whose health gates a rehome is exactly the cells that can be a
|
||||
// source or a target, and both roles require the drain protocol.
|
||||
const context = await setup({ targetProtocol: 0 })
|
||||
|
||||
expect(await context.store.regionalRehomeFleetSafety()).toMatchObject({
|
||||
requiredCells: 1,
|
||||
missingCells: 0
|
||||
})
|
||||
await heartbeat(context.store, target, targetIncarnation, 1, 2)
|
||||
expect(await context.store.regionalRehomeFleetSafety()).toMatchObject({
|
||||
requiredCells: 2,
|
||||
missingCells: 0
|
||||
})
|
||||
await context.database.close()
|
||||
})
|
||||
|
||||
it('claims through the measured healthy baseline of pool micro-waits and churn', async () => {
|
||||
const context = await setup()
|
||||
const baseline = {
|
||||
@@ -238,7 +257,7 @@ describe('regional rehome assignment state', () => {
|
||||
databasePoolWaitMsMax: 1
|
||||
}
|
||||
await heartbeat(context.store, source, sourceIncarnation, 1, 2, baseline)
|
||||
await heartbeat(context.store, target, targetIncarnation, 0, 2, baseline)
|
||||
await heartbeat(context.store, target, targetIncarnation, 1, 2, baseline)
|
||||
await activatePreferredSource(context, {
|
||||
userId: 'user-1',
|
||||
relayHostId: 'abcdefghijklmnop'
|
||||
@@ -324,6 +343,204 @@ describe('regional rehome assignment state', () => {
|
||||
await context.database.close()
|
||||
})
|
||||
|
||||
it('moves a live host on an asia-east2 cell back to its preferred us-central1 cell', async () => {
|
||||
const context = await setup()
|
||||
const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
|
||||
await activateReversePreferredSource(context, identity)
|
||||
|
||||
const attempt = await context.store.claimRegionalRehome()
|
||||
expect(attempt).toMatchObject({
|
||||
userId: identity.userId,
|
||||
relayHostId: identity.relayHostId,
|
||||
preferredRegion: 'us-central1',
|
||||
sourceCellId: target.id,
|
||||
sourceCellIncarnation: targetIncarnation,
|
||||
targetCellId: source.id,
|
||||
targetCellIncarnation: sourceIncarnation,
|
||||
previousEpoch: 1,
|
||||
assignmentEpoch: 2,
|
||||
sendAttempts: 1
|
||||
})
|
||||
expect(
|
||||
await context.database.query(
|
||||
`SELECT preferred_region, source_cell_id, target_cell_id
|
||||
FROM relay_region_rehome_attempts`
|
||||
)
|
||||
).toEqual([{
|
||||
preferred_region: 'us-central1',
|
||||
source_cell_id: target.id,
|
||||
target_cell_id: source.id
|
||||
}])
|
||||
expect(await context.store.resolve(identity)).toMatchObject({ cellId: source.id })
|
||||
await context.database.close()
|
||||
})
|
||||
|
||||
it('drops a candidate at scan time when no cell in the preferred region is usable', async () => {
|
||||
const context = await setup()
|
||||
await activatePreferredSource(context, {
|
||||
userId: 'user-1',
|
||||
relayHostId: 'abcdefghijklmnop'
|
||||
})
|
||||
// A disabled cell is not a target, and the scan must say so: leaving it to
|
||||
// the claim would burn a slot of the candidate batch on a certain skip.
|
||||
await context.database.query(`UPDATE relay_cells SET enabled = 0 WHERE cell_id = ?`, [
|
||||
target.id
|
||||
])
|
||||
|
||||
const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
|
||||
try {
|
||||
expect(await context.store.claimRegionalRehome()).toBeNull()
|
||||
} finally {
|
||||
warnings.restore()
|
||||
}
|
||||
expect(warnings.entries).toEqual([])
|
||||
expect(
|
||||
await context.database.query(
|
||||
`SELECT next_dispatch_at FROM relay_region_rehome_worker_state`
|
||||
)
|
||||
).toEqual([{ next_dispatch_at: 0 }])
|
||||
expect(await context.database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
|
||||
await context.database.close()
|
||||
})
|
||||
|
||||
it('names the skip when the last target is lost between scan and claim', async () => {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
const context = await setup({
|
||||
database,
|
||||
wrap: (delegate) =>
|
||||
hookAfterCandidateScan(delegate, async (transaction) => {
|
||||
await transaction.query(`UPDATE relay_cells SET enabled = 0 WHERE cell_id = ?`, [
|
||||
target.id
|
||||
])
|
||||
})
|
||||
})
|
||||
await activatePreferredSource(context, {
|
||||
userId: 'user-1',
|
||||
relayHostId: 'abcdefghijklmnop'
|
||||
})
|
||||
|
||||
const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
|
||||
try {
|
||||
expect(await context.store.claimRegionalRehome()).toBeNull()
|
||||
} finally {
|
||||
warnings.restore()
|
||||
}
|
||||
expect(warnings.entries).toMatchObject([
|
||||
{ skips: [{ reason: 'no_eligible_target', candidates: 1 }] }
|
||||
])
|
||||
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
|
||||
generation: 1,
|
||||
enabled: true
|
||||
})
|
||||
expect(await database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
|
||||
await database.close()
|
||||
})
|
||||
|
||||
it('leaves a host alone until its cooldown expires, then moves it back', async () => {
|
||||
const context = await setup({ hostCooldownMs: 3 * 24 * 60 * 60_000 })
|
||||
const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
|
||||
const targetControl = await completeRehomeToTarget(context, identity)
|
||||
// Past the dispatch interval the earlier claim charged, so the next tick
|
||||
// really does scan and the cooldown is the only thing holding this host.
|
||||
context.advance(10_000)
|
||||
// The desktop's region probe now says us-central1 again.
|
||||
await context.store.assign(identity, 'us-central1')
|
||||
|
||||
const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
|
||||
try {
|
||||
expect(await context.store.claimRegionalRehome()).toBeNull()
|
||||
} finally {
|
||||
warnings.restore()
|
||||
}
|
||||
expect(warnings.entries).toEqual([])
|
||||
expect(await context.store.resolve(identity)).toMatchObject({ cellId: target.id })
|
||||
|
||||
context.advance(3 * 24 * 60 * 60_000)
|
||||
await freshHeartbeats(context)
|
||||
await context.store.renewControlActivity(identity, {
|
||||
activityId: targetControl,
|
||||
cellId: target.id,
|
||||
expiresAt: context.now() + 90_000
|
||||
})
|
||||
await context.store.assign(identity, 'us-central1')
|
||||
|
||||
const attempt = await context.store.claimRegionalRehome()
|
||||
expect(attempt).toMatchObject({
|
||||
preferredRegion: 'us-central1',
|
||||
sourceCellId: target.id,
|
||||
targetCellId: source.id
|
||||
})
|
||||
await context.database.close()
|
||||
})
|
||||
|
||||
it('rejects a host whose attempt lands between the scan and the claim', async () => {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
|
||||
const context = await setup({
|
||||
database,
|
||||
wrap: (delegate) =>
|
||||
hookAfterCandidateScan(delegate, async (transaction) => {
|
||||
await transaction.query(
|
||||
`INSERT INTO relay_region_rehome_attempts
|
||||
(attempt_id, user_id, relay_host_id, preferred_region, source_cell_id,
|
||||
source_cell_incarnation, target_cell_id, target_cell_incarnation,
|
||||
previous_epoch, assignment_epoch, drain_grace_ms, send_attempts,
|
||||
created_at, updated_at)
|
||||
VALUES ('raced', ?, ?, 'asia-east2', ?, ?, ?, ?, 0, 1, 0, 0, ?, ?)`,
|
||||
[
|
||||
identity.userId,
|
||||
identity.relayHostId,
|
||||
source.id,
|
||||
sourceIncarnation,
|
||||
target.id,
|
||||
targetIncarnation,
|
||||
context.now(),
|
||||
context.now()
|
||||
]
|
||||
)
|
||||
})
|
||||
})
|
||||
await activatePreferredSource(context, identity)
|
||||
|
||||
const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
|
||||
try {
|
||||
expect(await context.store.claimRegionalRehome()).toBeNull()
|
||||
} finally {
|
||||
warnings.restore()
|
||||
}
|
||||
expect(warnings.entries).toMatchObject([
|
||||
{ skips: [{ reason: 'host_cooldown', candidates: 1 }] }
|
||||
])
|
||||
expect(await database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
|
||||
await database.close()
|
||||
})
|
||||
|
||||
it('does not scan a candidate whose preferred region has no drainable cell', async () => {
|
||||
// A cell without the drain protocol cannot be a target: the host would land
|
||||
// where no later rehome could move it out again. The candidate query drops
|
||||
// it, so the tick stays idle instead of paying for an inventory scan.
|
||||
const context = await setup({ targetProtocol: 0 })
|
||||
await activatePreferredSource(context, {
|
||||
userId: 'user-1',
|
||||
relayHostId: 'abcdefghijklmnop'
|
||||
})
|
||||
|
||||
const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
|
||||
try {
|
||||
expect(await context.store.claimRegionalRehome()).toBeNull()
|
||||
} finally {
|
||||
warnings.restore()
|
||||
}
|
||||
expect(warnings.entries).toEqual([])
|
||||
expect(
|
||||
await context.database.query(
|
||||
`SELECT next_dispatch_at FROM relay_region_rehome_worker_state`
|
||||
)
|
||||
).toEqual([{ next_dispatch_at: 0 }])
|
||||
expect(await context.database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
|
||||
await context.database.close()
|
||||
})
|
||||
|
||||
it('skips an unclean cell without latching the control off', async () => {
|
||||
const context = await setup()
|
||||
await activatePreferredSource(context, {
|
||||
@@ -457,7 +674,7 @@ describe('regional rehome assignment state', () => {
|
||||
databasePoolWaitersMax: 0,
|
||||
databasePoolWaitMsMax: 0
|
||||
})
|
||||
await heartbeat(context.store, target, targetIncarnation, 0, 2, {
|
||||
await heartbeat(context.store, target, targetIncarnation, 1, 2, {
|
||||
observedAt: context.now(),
|
||||
sqlFailures: 0,
|
||||
reconnects: 0,
|
||||
@@ -477,6 +694,7 @@ describe('regional rehome assignment state', () => {
|
||||
notBefore: context.now(),
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 24 * 60 * 60_000,
|
||||
hostCooldownMs: 7 * 24 * 60 * 60_000,
|
||||
drainGraceMs: 60_000
|
||||
})
|
||||
const retry = await context.store.claimRegionalRehome()
|
||||
@@ -547,7 +765,7 @@ describe('regional rehome assignment state', () => {
|
||||
await activatePreferredSource(context, identity)
|
||||
await context.store.claimRegionalRehome()
|
||||
context.advance(6 * 60_000)
|
||||
await heartbeat(context.store, target, targetIncarnation, 0, 2)
|
||||
await heartbeat(context.store, target, targetIncarnation, 1, 2)
|
||||
|
||||
expect(await context.store.refreshRegionalRehomeLeases()).toBe(0)
|
||||
expect(await context.store.abortExpiredEvacuations()).toBe(0)
|
||||
@@ -1549,10 +1767,16 @@ function collectDisableWarnings() {
|
||||
}
|
||||
|
||||
async function setup(
|
||||
options: { sourceProtocol?: number; wrap?: (database: RelayDatabase) => RelayDatabase } = {}
|
||||
options: {
|
||||
sourceProtocol?: number
|
||||
targetProtocol?: number
|
||||
hostCooldownMs?: number
|
||||
database?: RelayDatabase
|
||||
wrap?: (database: RelayDatabase) => RelayDatabase
|
||||
} = {}
|
||||
) {
|
||||
let clock = 1_000_000
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
const database = options.database ?? (await openInMemoryRelayDatabase())
|
||||
const store = new RelayAssignmentStore(options.wrap?.(database) ?? database, () => clock, {
|
||||
requireLiveCells: true,
|
||||
heartbeatTtlMs: 45_000
|
||||
@@ -1565,11 +1789,12 @@ async function setup(
|
||||
notBefore: clock,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 24 * 60 * 60_000,
|
||||
hostCooldownMs: options.hostCooldownMs ?? 7 * 24 * 60 * 60_000,
|
||||
drainGraceMs: 60 * 60_000
|
||||
})
|
||||
await store.reconcileCells([source, target])
|
||||
await heartbeat(store, source, sourceIncarnation, options.sourceProtocol ?? 1)
|
||||
await heartbeat(store, target, targetIncarnation, 0)
|
||||
await heartbeat(store, target, targetIncarnation, options.targetProtocol ?? 1)
|
||||
return {
|
||||
database,
|
||||
store,
|
||||
@@ -1671,7 +1896,7 @@ async function freshHeartbeats(context: Context): Promise<void> {
|
||||
}
|
||||
// The clock doubles as a strictly-increasing connection inclusion watermark.
|
||||
await heartbeat(context.store, source, sourceIncarnation, 1, context.now(), safety)
|
||||
await heartbeat(context.store, target, targetIncarnation, 0, context.now(), safety)
|
||||
await heartbeat(context.store, target, targetIncarnation, 1, context.now(), safety)
|
||||
}
|
||||
|
||||
async function activatePreferredSource(
|
||||
@@ -1688,6 +1913,68 @@ async function activatePreferredSource(
|
||||
return control
|
||||
}
|
||||
|
||||
// Runs a hook inside the claim transaction, right after the candidate scan, so
|
||||
// a scan-versus-claim race is deterministic instead of timing-dependent.
|
||||
function hookAfterCandidateScan(
|
||||
database: RelayDatabase,
|
||||
hook: (transaction: RelayDatabase) => Promise<void>
|
||||
): RelayDatabase {
|
||||
let fired = false
|
||||
const decorate = (delegate: RelayDatabase): RelayDatabase => ({
|
||||
query: async (sql, params) => {
|
||||
const rows = await delegate.query(sql, params)
|
||||
if (!fired && sql.includes('FROM relay_assignment_region_preferences preference')) {
|
||||
fired = true
|
||||
await hook(delegate)
|
||||
}
|
||||
return rows
|
||||
},
|
||||
queryLocked: async (sql, params, lockOptions) =>
|
||||
await delegate.queryLocked(sql, params, lockOptions),
|
||||
transaction: async (operation, transactionOptions) =>
|
||||
await delegate.transaction(
|
||||
async (transaction) => await operation(decorate(transaction)),
|
||||
transactionOptions
|
||||
),
|
||||
close: async () => undefined
|
||||
})
|
||||
return decorate(database)
|
||||
}
|
||||
|
||||
async function completeRehomeToTarget(
|
||||
context: Context,
|
||||
identity: { userId: string; relayHostId: string }
|
||||
): Promise<string> {
|
||||
const sourceControl = await activatePreferredSource(context, identity)
|
||||
const attempt = await context.store.claimRegionalRehome()
|
||||
const targetControl = await context.store.activateControl(identity, {
|
||||
cellId: target.id,
|
||||
assignmentEpoch: attempt!.assignmentEpoch,
|
||||
generation: 1
|
||||
})
|
||||
await context.store.markMigrationTargetRegistered(identity, {
|
||||
cellId: target.id,
|
||||
assignmentEpoch: attempt!.assignmentEpoch
|
||||
})
|
||||
await context.store.releaseActivity(identity, sourceControl)
|
||||
await context.store.completeReadyRegionalRehomes()
|
||||
return targetControl
|
||||
}
|
||||
|
||||
async function activateReversePreferredSource(
|
||||
context: Context,
|
||||
identity: { userId: string; relayHostId: string }
|
||||
): Promise<string> {
|
||||
const assignment = await context.store.assign(identity, undefined, 'asia-east2')
|
||||
const control = await context.store.activateControl(identity, {
|
||||
cellId: target.id,
|
||||
assignmentEpoch: assignment.assignmentEpoch,
|
||||
generation: 1
|
||||
})
|
||||
await context.store.assign(identity, 'us-central1')
|
||||
return control
|
||||
}
|
||||
|
||||
async function activateSource(
|
||||
context: Context,
|
||||
identity: { userId: string; relayHostId: string }
|
||||
|
||||
@@ -36,6 +36,7 @@ async function setup() {
|
||||
notBefore: clock,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 24 * 60 * 60_000,
|
||||
hostCooldownMs: 7 * 24 * 60 * 60_000,
|
||||
drainGraceMs: 60 * 60_000
|
||||
})
|
||||
await store.reconcileCells([source, noHeadroom, unclean, highLoad, lowLoad])
|
||||
@@ -100,22 +101,22 @@ describe('regional rehome target selection', () => {
|
||||
sqlFailures: 0
|
||||
})
|
||||
// Lowest load but the connection hard cap is exhausted.
|
||||
await context.beat(noHeadroom, 2, 0, {
|
||||
await context.beat(noHeadroom, 2, 1, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 999,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.beat(unclean, 3, 0, {
|
||||
await context.beat(unclean, 3, 1, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: UNCLEAN
|
||||
})
|
||||
await context.beat(highLoad, 4, 0, {
|
||||
await context.beat(highLoad, 4, 1, {
|
||||
observedRequests: 50,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.beat(lowLoad, 5, 0, {
|
||||
await context.beat(lowLoad, 5, 1, {
|
||||
observedRequests: 10,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: 0
|
||||
@@ -134,22 +135,22 @@ describe('regional rehome target selection', () => {
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.beat(noHeadroom, 2, 0, {
|
||||
await context.beat(noHeadroom, 2, 1, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 999,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.beat(unclean, 3, 0, {
|
||||
await context.beat(unclean, 3, 1, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: UNCLEAN
|
||||
})
|
||||
await context.beat(highLoad, 4, 0, {
|
||||
await context.beat(highLoad, 4, 1, {
|
||||
observedRequests: 50,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.beat(lowLoad, 5, 0, {
|
||||
await context.beat(lowLoad, 5, 1, {
|
||||
observedRequests: 10,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: UNCLEAN
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import { RELAY_REGION_METRIC_SEGMENTS, RELAY_REGIONS } from '@orca-cloud/relay-contract'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { RelayDatabase } from './database.js'
|
||||
import { observeRelayDatabase } from './observed-relay-database.js'
|
||||
import {
|
||||
CONTROL_RTT_RESERVOIR_LIMIT,
|
||||
observedRelayRequests,
|
||||
RelayObservability,
|
||||
type RelayProcessCounts
|
||||
@@ -22,6 +24,37 @@ const counts: RelayProcessCounts = {
|
||||
databasePoolWaitMsMax: 1_250
|
||||
}
|
||||
|
||||
// Two schema keys legitimately spell a policed word: the abandoned-accept bucket
|
||||
// is keyed by stage name and one stage is `credential`. Rename those exact keys in
|
||||
// a clone instead of rewriting the JSON, so a stray raw field or value anywhere
|
||||
// else still trips the guard below.
|
||||
const SCHEMA_KEY_ALIASES: Record<string, string> = {
|
||||
clientAcceptCredentialMsP95: 'clientAcceptStageTwoMsP95'
|
||||
}
|
||||
|
||||
function scrubSchemaKeys(entries: Array<Record<string, unknown>>): string {
|
||||
return JSON.stringify(
|
||||
entries.map((entry) =>
|
||||
Object.fromEntries(
|
||||
Object.entries(entry).map(([key, value]) => [
|
||||
SCHEMA_KEY_ALIASES[key] ?? key,
|
||||
key === 'clientAcceptsAbandonedByStageDelta' ? renameStageKeys(value) : value
|
||||
])
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
function renameStageKeys(bucket: unknown): unknown {
|
||||
if (bucket === null || typeof bucket !== 'object') return bucket
|
||||
return Object.fromEntries(
|
||||
Object.entries(bucket).map(([stage, count]) => [
|
||||
stage === 'credential' ? 'stageTwo' : stage,
|
||||
count
|
||||
])
|
||||
)
|
||||
}
|
||||
|
||||
describe('relay observability', () => {
|
||||
it('emits safe readiness dependency outcomes', () => {
|
||||
const entries: Array<Record<string, unknown>> = []
|
||||
@@ -106,14 +139,31 @@ describe('relay observability', () => {
|
||||
requestedRegionsDelta: { 'asia-east2': 1, unhinted: 1 },
|
||||
selectedRegionsDelta: { 'us-central1': 1 },
|
||||
regionFallbacksDelta: { 'asia-east2': 1 },
|
||||
unavailableRegionsDelta: { 'asia-east2': 1 }
|
||||
unavailableRegionsDelta: { 'asia-east2': 1 },
|
||||
// Flat per-region siblings the log-based metrics extract; `unhinted` stays map-only.
|
||||
requestedRegionUsCentral1Delta: 0,
|
||||
requestedRegionAsiaEast2Delta: 1,
|
||||
selectedRegionUsCentral1Delta: 1,
|
||||
selectedRegionAsiaEast2Delta: 0
|
||||
})
|
||||
expect(entries[1]).toMatchObject({
|
||||
requestedRegionsDelta: {},
|
||||
selectedRegionsDelta: {},
|
||||
regionFallbacksDelta: {},
|
||||
unavailableRegionsDelta: {}
|
||||
unavailableRegionsDelta: {},
|
||||
// Zeros keep publishing so an idle window cannot drop a series out of the skew join.
|
||||
requestedRegionUsCentral1Delta: 0,
|
||||
requestedRegionAsiaEast2Delta: 0,
|
||||
selectedRegionUsCentral1Delta: 0,
|
||||
selectedRegionAsiaEast2Delta: 0
|
||||
})
|
||||
// A region added to the contract has to reach the flat keys, or the skew alert's
|
||||
// denominator silently misses it.
|
||||
for (const segment of Object.values(RELAY_REGION_METRIC_SEGMENTS)) {
|
||||
expect(entries[0]).toHaveProperty(`requestedRegion${segment}Delta`)
|
||||
expect(entries[0]).toHaveProperty(`selectedRegion${segment}Delta`)
|
||||
}
|
||||
expect(Object.keys(RELAY_REGION_METRIC_SEGMENTS).sort()).toEqual([...RELAY_REGIONS].sort())
|
||||
})
|
||||
|
||||
it('emits bounded aggregate runtime signals without identities or credentials', () => {
|
||||
@@ -181,7 +231,7 @@ describe('relay observability', () => {
|
||||
controlActivityRecoveryFailuresDelta: 0,
|
||||
httpLatencyMsMax: 0
|
||||
})
|
||||
expect(JSON.stringify(entries)).not.toMatch(/token|credential|userId|relayHostId/)
|
||||
expect(scrubSchemaKeys(entries)).not.toMatch(/token|credential|userId|relayHostId/i)
|
||||
})
|
||||
|
||||
it('aggregates control and splice closes as bounded per-reason deltas', () => {
|
||||
@@ -215,6 +265,117 @@ describe('relay observability', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('summarises completed client accepts and control round trips per window', () => {
|
||||
const entries: Array<Record<string, unknown>> = []
|
||||
const observability = new RelayObservability(
|
||||
{ role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
|
||||
(entry) => entries.push(entry)
|
||||
)
|
||||
observability.recordClientAcceptCompleted({
|
||||
totalMs: 812.4567,
|
||||
stageMs: { assignment: 120, credential: 90, activity: 40, attach: 500, basis: 62 }
|
||||
})
|
||||
observability.recordClientAcceptCompleted({
|
||||
totalMs: 6_400,
|
||||
stageMs: { assignment: 4_100, credential: 95, activity: 60, attach: 2_000, basis: 145 }
|
||||
})
|
||||
observability.recordControlRtt(28)
|
||||
observability.recordControlRtt(240)
|
||||
observability.recordControlRtt(31)
|
||||
observability.flush(counts)
|
||||
observability.flush(counts)
|
||||
|
||||
expect(entries[0]).toMatchObject({
|
||||
clientAcceptCompletedDelta: 2,
|
||||
clientAcceptTotalMsP50: 812.457,
|
||||
clientAcceptTotalMsP95: 6_400,
|
||||
clientAcceptTotalMsMax: 6_400,
|
||||
clientAcceptAssignmentMsP95: 4_100,
|
||||
clientAcceptCredentialMsP95: 95,
|
||||
clientAcceptActivityMsP95: 60,
|
||||
clientAcceptAttachMsP95: 2_000,
|
||||
clientAcceptBasisMsP95: 145,
|
||||
controlRttSamplesDelta: 3,
|
||||
controlRttMsP50: 31,
|
||||
controlRttMsP95: 240,
|
||||
controlRttMsMax: 240
|
||||
})
|
||||
// Only-add: the pre-existing fields still read the same after the extension.
|
||||
expect(entries[0]).toMatchObject({
|
||||
event: 'orca_relay_runtime_metrics',
|
||||
metricVersion: 2,
|
||||
clientAcceptsAbandonedByStageDelta: {},
|
||||
clientAcceptAbandonedMsMax: 0
|
||||
})
|
||||
// An empty window publishes counts only: a zero percentile point is
|
||||
// indistinguishable from a real zero once Cloud Logging aggregates it.
|
||||
expect(entries[1]).toMatchObject({ clientAcceptCompletedDelta: 0, controlRttSamplesDelta: 0 })
|
||||
for (const omitted of [
|
||||
'clientAcceptTotalMsP50',
|
||||
'clientAcceptTotalMsP95',
|
||||
'clientAcceptTotalMsMax',
|
||||
'clientAcceptAssignmentMsP95',
|
||||
'clientAcceptCredentialMsP95',
|
||||
'clientAcceptActivityMsP95',
|
||||
'clientAcceptAttachMsP95',
|
||||
'clientAcceptBasisMsP95',
|
||||
'controlRttMsP50',
|
||||
'controlRttMsP95',
|
||||
'controlRttMsMax'
|
||||
]) {
|
||||
expect(entries[1]).not.toHaveProperty(omitted)
|
||||
expect(entries[0]).toHaveProperty(omitted)
|
||||
}
|
||||
expect(scrubSchemaKeys(entries)).not.toMatch(/token|credential|userId|relayHostId/i)
|
||||
})
|
||||
|
||||
it('caps the control round-trip reservoir and reports what it dropped', () => {
|
||||
const entries: Array<Record<string, unknown>> = []
|
||||
const observability = new RelayObservability(
|
||||
{ role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
|
||||
(entry) => entries.push(entry)
|
||||
)
|
||||
const flooded = CONTROL_RTT_RESERVOIR_LIMIT * 20
|
||||
for (let sample = 0; sample < flooded; sample++) {
|
||||
observability.recordControlRtt(10 + (sample % 40))
|
||||
}
|
||||
observability.flush(counts)
|
||||
|
||||
// Dropped is observed minus retained, so this pins the retained window at the cap.
|
||||
expect(entries[0]).toMatchObject({
|
||||
controlRttSamplesDelta: flooded,
|
||||
controlRttSamplesDroppedDelta: flooded - CONTROL_RTT_RESERVOIR_LIMIT
|
||||
})
|
||||
// The kept samples are real observations, not a truncated or synthesised window.
|
||||
expect(entries[0]!.controlRttMsP50 as number).toBeGreaterThanOrEqual(10)
|
||||
expect(entries[0]!.controlRttMsMax as number).toBeLessThanOrEqual(49)
|
||||
|
||||
observability.flush(counts)
|
||||
expect(entries[1]).toMatchObject({
|
||||
controlRttSamplesDelta: 0,
|
||||
controlRttSamplesDroppedDelta: 0
|
||||
})
|
||||
expect(entries[1]).not.toHaveProperty('controlRttMsP50')
|
||||
})
|
||||
|
||||
it('samples the whole flooded window rather than its first samples', () => {
|
||||
const entries: Array<Record<string, unknown>> = []
|
||||
const observability = new RelayObservability(
|
||||
{ role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
|
||||
(entry) => entries.push(entry)
|
||||
)
|
||||
const half = CONTROL_RTT_RESERVOIR_LIMIT * 10
|
||||
for (let sample = 0; sample < half; sample++) observability.recordControlRtt(10)
|
||||
for (let sample = 0; sample < half; sample++) observability.recordControlRtt(900)
|
||||
observability.flush(counts)
|
||||
|
||||
// Keeping the first N instead would publish a window of nothing but 10s. Each
|
||||
// reservoir slot ends up drawn from the late half with ~1/2 probability, so
|
||||
// fewer than the 5% the p95 needs is out of reach of this suite.
|
||||
expect(entries[0]!.controlRttMsP95).toBe(900)
|
||||
expect(entries[0]!.controlRttMsMax).toBe(900)
|
||||
})
|
||||
|
||||
it('observes successful and failed database calls including transactions', async () => {
|
||||
const recordSql = vi.fn()
|
||||
const underlying: RelayDatabase = {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { monitorEventLoopDelay, performance } from 'node:perf_hooks'
|
||||
import type { RelayRegion } from '@orca-cloud/relay-contract'
|
||||
import { RELAY_REGION_METRIC_SEGMENTS, type RelayRegion } from '@orca-cloud/relay-contract'
|
||||
import type { ControlRenewalOutcome } from './assignment-store.js'
|
||||
import type { CellInventoryHoldCounts } from './cell-inventory-hold-samples.js'
|
||||
import type { PostgresPoolPressureCounts } from './postgres-pool-pressure.js'
|
||||
@@ -65,11 +65,31 @@ export interface RelayRuntimeObserver {
|
||||
recordControlClose?(code: number): void
|
||||
recordSpliceClose?(trigger: string): void
|
||||
recordClientAcceptAbandoned?(stage: RelayClientAcceptStage, elapsedMs: number): void
|
||||
recordClientAcceptCompleted?(sample: RelayClientAcceptSample): void
|
||||
recordControlRtt?(rttMs: number): void
|
||||
}
|
||||
|
||||
// Which serialized accept step the phone had already hung up behind.
|
||||
export type RelayClientAcceptStage = 'assignment' | 'credential' | 'activity'
|
||||
|
||||
// The attach window and the basis writes that follow it are only measurable once
|
||||
// the host data leg lands, so they join the serialized pre-attach steps on
|
||||
// completed accepts only.
|
||||
export type RelayClientAcceptTimedStage = RelayClientAcceptStage | 'attach' | 'basis'
|
||||
|
||||
export const RELAY_CLIENT_ACCEPT_TIMED_STAGES = [
|
||||
'assignment',
|
||||
'credential',
|
||||
'activity',
|
||||
'attach',
|
||||
'basis'
|
||||
] as const satisfies readonly RelayClientAcceptTimedStage[]
|
||||
|
||||
export type RelayClientAcceptSample = {
|
||||
totalMs: number
|
||||
stageMs: Record<RelayClientAcceptTimedStage, number>
|
||||
}
|
||||
|
||||
type RelayMetricDeltas = {
|
||||
forwardedBytes: number
|
||||
authSuccesses: number
|
||||
@@ -93,12 +113,20 @@ type RelayMetricDeltas = {
|
||||
spliceClosesByTrigger: Record<string, number>
|
||||
clientAcceptsAbandonedByStage: Record<string, number>
|
||||
clientAcceptAbandonedMsMax: number
|
||||
clientAcceptTotalsMs: number[]
|
||||
clientAcceptStageSamplesMs: Record<RelayClientAcceptTimedStage, number[]>
|
||||
controlRttSamplesMs: number[]
|
||||
controlRttObserved: number
|
||||
controlRenewalLatenciesMs: number[]
|
||||
controlRenewalsByOutcome: Record<string, number>
|
||||
controlActivityRecoveries: number
|
||||
controlActivityRecoveryFailures: number
|
||||
}
|
||||
|
||||
// A host chooses how often it answers a ping, so the process-wide window is a
|
||||
// reservoir: the heap cost of a flood is capped and the percentiles stay unbiased.
|
||||
export const CONTROL_RTT_RESERVOIR_LIMIT = 1024
|
||||
|
||||
type MetricWriter = (entry: Record<string, unknown>) => void
|
||||
|
||||
const emptyDeltas = (): RelayMetricDeltas => ({
|
||||
@@ -124,18 +152,42 @@ const emptyDeltas = (): RelayMetricDeltas => ({
|
||||
spliceClosesByTrigger: {},
|
||||
clientAcceptsAbandonedByStage: {},
|
||||
clientAcceptAbandonedMsMax: 0,
|
||||
clientAcceptTotalsMs: [],
|
||||
clientAcceptStageSamplesMs: {
|
||||
assignment: [],
|
||||
credential: [],
|
||||
activity: [],
|
||||
attach: [],
|
||||
basis: []
|
||||
},
|
||||
controlRttSamplesMs: [],
|
||||
controlRttObserved: 0,
|
||||
controlRenewalLatenciesMs: [],
|
||||
controlRenewalsByOutcome: {},
|
||||
controlActivityRecoveries: 0,
|
||||
controlActivityRecoveryFailures: 0
|
||||
})
|
||||
|
||||
function percentile(values: number[], percentileRank: number): number {
|
||||
export function percentile(values: number[], percentileRank: number): number {
|
||||
if (values.length === 0) return 0
|
||||
const sorted = [...values].sort((left, right) => left - right)
|
||||
return sorted[Math.ceil(percentileRank * sorted.length) - 1] ?? 0
|
||||
}
|
||||
|
||||
function roundMs(value: number): number {
|
||||
return Number(value.toFixed(3))
|
||||
}
|
||||
|
||||
// Spreading a window into Math.max blows the stack once a busy cell samples
|
||||
// enough of it, so the maximum is folded instead.
|
||||
function latencySummary(samples: number[]): { p50: number; p95: number; max: number } {
|
||||
return {
|
||||
p50: roundMs(percentile(samples, 0.5)),
|
||||
p95: roundMs(percentile(samples, 0.95)),
|
||||
max: roundMs(samples.reduce((highest, sample) => Math.max(highest, sample), 0))
|
||||
}
|
||||
}
|
||||
|
||||
export class RelayObservability implements RelayRuntimeObserver {
|
||||
private readonly eventLoop = monitorEventLoopDelay({ resolution: 20 })
|
||||
private deltas = emptyDeltas()
|
||||
@@ -244,6 +296,25 @@ export class RelayObservability implements RelayRuntimeObserver {
|
||||
)
|
||||
}
|
||||
|
||||
recordClientAcceptCompleted(sample: RelayClientAcceptSample): void {
|
||||
this.deltas.clientAcceptTotalsMs.push(sample.totalMs)
|
||||
for (const stage of RELAY_CLIENT_ACCEPT_TIMED_STAGES) {
|
||||
this.deltas.clientAcceptStageSamplesMs[stage].push(sample.stageMs[stage])
|
||||
}
|
||||
}
|
||||
|
||||
recordControlRtt(rttMs: number): void {
|
||||
const samples = this.deltas.controlRttSamplesMs
|
||||
const observedBefore = this.deltas.controlRttObserved++
|
||||
if (samples.length < CONTROL_RTT_RESERVOIR_LIMIT) {
|
||||
samples.push(rttMs)
|
||||
return
|
||||
}
|
||||
// Algorithm R: every round trip in the window keeps an equal chance of being kept.
|
||||
const slot = Math.floor(Math.random() * (observedBefore + 1))
|
||||
if (slot < CONTROL_RTT_RESERVOIR_LIMIT) samples[slot] = rttMs
|
||||
}
|
||||
|
||||
start(readCounts: () => RelayProcessCounts, intervalMs = 30_000): void {
|
||||
if (this.timer) return
|
||||
this.eventLoop.enable()
|
||||
@@ -277,6 +348,11 @@ export class RelayObservability implements RelayRuntimeObserver {
|
||||
controlActivityRecoveryFailures: deltas.controlActivityRecoveryFailures
|
||||
}
|
||||
this.deltas = emptyDeltas()
|
||||
const acceptTotals = latencySummary(deltas.clientAcceptTotalsMs)
|
||||
const acceptStageP95 = (stage: RelayClientAcceptTimedStage): number =>
|
||||
roundMs(percentile(deltas.clientAcceptStageSamplesMs[stage], 0.95))
|
||||
const controlRtt = latencySummary(deltas.controlRttSamplesMs)
|
||||
const controlRenewal = latencySummary(deltas.controlRenewalLatenciesMs)
|
||||
const memory = process.memoryUsage()
|
||||
const p99 = this.eventLoop.count === 0 ? 0 : this.eventLoop.percentile(99) / 1_000_000
|
||||
this.eventLoop.reset()
|
||||
@@ -301,15 +377,43 @@ export class RelayObservability implements RelayRuntimeObserver {
|
||||
placementRejectionsByReasonDelta: deltas.placementRejectionsByReason,
|
||||
requestedRegionsDelta: deltas.requestedRegions,
|
||||
selectedRegionsDelta: deltas.selectedRegions,
|
||||
...regionCounterFields('requestedRegion', deltas.requestedRegions),
|
||||
...regionCounterFields('selectedRegion', deltas.selectedRegions),
|
||||
regionFallbacksDelta: deltas.regionFallbacks,
|
||||
unavailableRegionsDelta: deltas.unavailableRegions,
|
||||
controlClosesByCodeDelta: deltas.controlClosesByCode,
|
||||
spliceClosesByTriggerDelta: deltas.spliceClosesByTrigger,
|
||||
clientAcceptsAbandonedByStageDelta: deltas.clientAcceptsAbandonedByStage,
|
||||
clientAcceptAbandonedMsMax: Number(deltas.clientAcceptAbandonedMsMax.toFixed(3)),
|
||||
clientAcceptAbandonedMsMax: roundMs(deltas.clientAcceptAbandonedMsMax),
|
||||
clientAcceptCompletedDelta: deltas.clientAcceptTotalsMs.length,
|
||||
// Accepts are sparse: publishing a zero percentile for every empty window
|
||||
// would pin the p50 at 0 forever and collapse the p95 at low accept rates.
|
||||
...(deltas.clientAcceptTotalsMs.length === 0
|
||||
? {}
|
||||
: {
|
||||
clientAcceptTotalMsP50: acceptTotals.p50,
|
||||
clientAcceptTotalMsP95: acceptTotals.p95,
|
||||
clientAcceptTotalMsMax: acceptTotals.max,
|
||||
clientAcceptAssignmentMsP95: acceptStageP95('assignment'),
|
||||
clientAcceptCredentialMsP95: acceptStageP95('credential'),
|
||||
clientAcceptActivityMsP95: acceptStageP95('activity'),
|
||||
clientAcceptAttachMsP95: acceptStageP95('attach'),
|
||||
clientAcceptBasisMsP95: acceptStageP95('basis')
|
||||
}),
|
||||
// Every round trip observed in the window, including the ones the reservoir
|
||||
// above declined to keep; the percentiles summarise only what it kept.
|
||||
controlRttSamplesDelta: deltas.controlRttObserved,
|
||||
controlRttSamplesDroppedDelta: deltas.controlRttObserved - deltas.controlRttSamplesMs.length,
|
||||
...(deltas.controlRttSamplesMs.length === 0
|
||||
? {}
|
||||
: {
|
||||
controlRttMsP50: controlRtt.p50,
|
||||
controlRttMsP95: controlRtt.p95,
|
||||
controlRttMsMax: controlRtt.max
|
||||
}),
|
||||
sqlQueriesDelta: deltas.sqlQueries,
|
||||
sqlFailuresDelta: deltas.sqlFailures,
|
||||
sqlLatencyMsMax: Number(deltas.sqlLatencyMsMax.toFixed(3)),
|
||||
sqlLatencyMsMax: roundMs(deltas.sqlLatencyMsMax),
|
||||
controlRenewalsByOutcomeDelta: deltas.controlRenewalsByOutcome,
|
||||
controlRenewalsDelta: deltas.controlRenewalLatenciesMs.length,
|
||||
controlRenewalSuccessesDelta: deltas.controlRenewalsByOutcome.renewed ?? 0,
|
||||
@@ -317,16 +421,10 @@ export class RelayObservability implements RelayRuntimeObserver {
|
||||
deltas.controlRenewalsByOutcome.control_activity_not_found ?? 0,
|
||||
controlActivityRecoveriesDelta: deltas.controlActivityRecoveries,
|
||||
controlActivityRecoveryFailuresDelta: deltas.controlActivityRecoveryFailures,
|
||||
controlRenewalLatencyMsP50: Number(
|
||||
percentile(deltas.controlRenewalLatenciesMs, 0.5).toFixed(3)
|
||||
),
|
||||
controlRenewalLatencyMsP95: Number(
|
||||
percentile(deltas.controlRenewalLatenciesMs, 0.95).toFixed(3)
|
||||
),
|
||||
controlRenewalLatencyMsMax: Number(
|
||||
Math.max(0, ...deltas.controlRenewalLatenciesMs).toFixed(3)
|
||||
),
|
||||
httpLatencyMsMax: Number(deltas.httpLatencyMsMax.toFixed(3)),
|
||||
controlRenewalLatencyMsP50: controlRenewal.p50,
|
||||
controlRenewalLatencyMsP95: controlRenewal.p95,
|
||||
controlRenewalLatencyMsMax: controlRenewal.max,
|
||||
httpLatencyMsMax: roundMs(deltas.httpLatencyMsMax),
|
||||
heapUsedBytes: memory.heapUsed,
|
||||
heapTotalBytes: memory.heapTotal,
|
||||
eventLoopDelayMsP99: Number(p99.toFixed(3))
|
||||
@@ -334,6 +432,22 @@ export class RelayObservability implements RelayRuntimeObserver {
|
||||
}
|
||||
}
|
||||
|
||||
// Flat siblings of the nested region maps, always emitted for every region including zeros.
|
||||
// A log-based metric cannot reach `requestedRegionsDelta."asia-east2"` without a quoted field
|
||||
// path, and an absent key would drop a series out of the inner join the region-skew alert does.
|
||||
// The maps stay authoritative and keep carrying anything outside the catalog, such as `unhinted`.
|
||||
function regionCounterFields(
|
||||
prefix: 'requestedRegion' | 'selectedRegion',
|
||||
counts: Record<string, number>
|
||||
): Record<string, number> {
|
||||
return Object.fromEntries(
|
||||
Object.entries(RELAY_REGION_METRIC_SEGMENTS).map(([region, segment]) => [
|
||||
`${prefix}${segment}Delta`,
|
||||
counts[region] ?? 0
|
||||
])
|
||||
)
|
||||
}
|
||||
|
||||
function increment(counts: Record<string, number>, key: string): void {
|
||||
counts[key] = (counts[key] ?? 0) + 1
|
||||
}
|
||||
|
||||
@@ -2,7 +2,9 @@ import { createAdaptorServer } from '@hono/node-server'
|
||||
import {
|
||||
hasAdmissionCapacity,
|
||||
HostDataAuthSchema,
|
||||
parseRelayHostCapabilities,
|
||||
RELAY_ADMISSION_BUDGETS,
|
||||
RELAY_HOST_CAPABILITIES_HEADER,
|
||||
RELAY_CLOSE_CODE,
|
||||
RELAY_DEFAULT_REGION,
|
||||
RELAY_PROTOCOL_LIMITS,
|
||||
@@ -486,7 +488,8 @@ export function createRelayServer(
|
||||
sessions.acceptControl(
|
||||
webSocket,
|
||||
identity,
|
||||
controlUpgrade?.inclusionWatermark
|
||||
controlUpgrade?.inclusionWatermark,
|
||||
parseRelayHostCapabilities(request.headers[RELAY_HOST_CAPABILITIES_HEADER])
|
||||
)
|
||||
})
|
||||
} catch {
|
||||
|
||||
@@ -9,7 +9,9 @@ import { fileURLToPath } from 'node:url'
|
||||
import { exportJWK, generateKeyPair, jwtVerify, SignJWT } from 'jose'
|
||||
import {
|
||||
buildHostProofMacInput,
|
||||
HOST_CHALLENGE_PLAINTEXT_DOMAIN
|
||||
HOST_CHALLENGE_PLAINTEXT_DOMAIN,
|
||||
RELAY_HOST_CAPABILITIES_HEADER,
|
||||
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS
|
||||
} from '@orca-cloud/relay-contract'
|
||||
import nacl from 'tweetnacl'
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
|
||||
@@ -282,11 +284,17 @@ async function openHostControl(input?: {
|
||||
previousGeneration?: number
|
||||
keyPair?: nacl.BoxKeyPair
|
||||
assignmentEpoch?: number
|
||||
capabilities?: string
|
||||
}): Promise<{ socket: WebSocket; ack: Record<string, unknown>; keyPair: nacl.BoxKeyPair }> {
|
||||
const keyPair = input?.keyPair ?? nacl.box.keyPair()
|
||||
const hostId = createHash('sha256').update(keyPair.publicKey).digest('base64url').slice(0, 16)
|
||||
const socket = new WebSocket(`${relayUrl.replace('http:', 'ws:')}/v1/host/control`, {
|
||||
headers: { authorization: `Bearer ${await relayToken('orca-relay', hostId)}` },
|
||||
headers: {
|
||||
authorization: `Bearer ${await relayToken('orca-relay', hostId)}`,
|
||||
...(input?.capabilities
|
||||
? { [RELAY_HOST_CAPABILITIES_HEADER]: input.capabilities }
|
||||
: {})
|
||||
},
|
||||
perMessageDeflate: false
|
||||
})
|
||||
await new Promise<void>((resolveOpen, reject) => {
|
||||
@@ -653,6 +661,69 @@ describe('served relay URL', () => {
|
||||
expect(result.reason).not.toContain('http')
|
||||
})
|
||||
|
||||
it('restates a pending connection to the rebound control, detailed only when advertised', async () => {
|
||||
// The one link the unit tests cannot reach: an upgrade that really carries
|
||||
// x-orca-host-capabilities must reach acceptControl and change the ack. A
|
||||
// typo in the header name here passes every other test in the suite.
|
||||
const host = await openHostControl()
|
||||
const hostId = createHash('sha256')
|
||||
.update(host.keyPair.publicKey)
|
||||
.digest('base64url')
|
||||
.slice(0, 16)
|
||||
const inviteResponse = nextMessage(host.socket)
|
||||
host.socket.send(
|
||||
JSON.stringify({
|
||||
type: 'invite-create',
|
||||
reqId: 'capability-invite',
|
||||
relayDeviceId: 'capability-device'
|
||||
})
|
||||
)
|
||||
const invite = await inviteResponse
|
||||
const phone = new WebSocket(`${relayUrl.replace('http:', 'ws:')}/v1/connect/${hostId}`, {
|
||||
headers: forwardedHeaders()
|
||||
})
|
||||
await new Promise<void>((resolveOpen, reject) => {
|
||||
phone.once('open', resolveOpen)
|
||||
phone.once('error', reject)
|
||||
})
|
||||
const connectionPromise = nextMessage(host.socket)
|
||||
phone.send(
|
||||
JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential: invite.inviteToken })
|
||||
)
|
||||
// Never attached: the connection stays pending, which is what the ack restates.
|
||||
const connection = await connectionPromise
|
||||
expect(connection.type).toBe('conn-open')
|
||||
|
||||
const capable = await openHostControl({
|
||||
keyPair: host.keyPair,
|
||||
controlResumeSecret: String(host.ack.controlResumeSecret),
|
||||
previousGeneration: 1,
|
||||
capabilities: RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS
|
||||
})
|
||||
expect(capable.ack.pendingConns).toEqual([
|
||||
{
|
||||
connId: connection.connId,
|
||||
connTicket: connection.connTicket,
|
||||
kind: 'invite',
|
||||
relayDeviceId: 'capability-device'
|
||||
}
|
||||
])
|
||||
|
||||
const legacy = await openHostControl({
|
||||
keyPair: host.keyPair,
|
||||
controlResumeSecret: String(capable.ack.controlResumeSecret),
|
||||
previousGeneration: 1
|
||||
})
|
||||
// A shipped host parses these entries strictly, so an unannounced key would
|
||||
// fail the whole ack and kill a control that was working.
|
||||
expect(legacy.ack.pendingConns).toEqual([
|
||||
{ connId: connection.connId, connTicket: connection.connTicket }
|
||||
])
|
||||
|
||||
phone.close()
|
||||
legacy.socket.close()
|
||||
})
|
||||
|
||||
it('keeps a pending attach usable after a bad ticket and rejects ticket replay', async () => {
|
||||
const host = await openHostControl()
|
||||
const hostId = createHash('sha256')
|
||||
|
||||
@@ -133,14 +133,17 @@
|
||||
"google_logging_metric.relay_snapshot",
|
||||
"google_monitoring_alert_policy.relay_assignment_5xx",
|
||||
"google_monitoring_alert_policy.relay_assignment_edge_429",
|
||||
"google_monitoring_alert_policy.relay_cell_control_rtt",
|
||||
"google_monitoring_alert_policy.relay_cell_process_exit",
|
||||
"google_monitoring_alert_policy.relay_cloud_nat_port_drops",
|
||||
"google_monitoring_alert_policy.relay_cloud_sql_backends",
|
||||
"google_monitoring_alert_policy.relay_cloud_sql_checkpoint_loop",
|
||||
"google_monitoring_alert_policy.relay_cloud_sql_disk",
|
||||
"google_monitoring_alert_policy.relay_custom",
|
||||
"google_monitoring_alert_policy.relay_far_cell_accept_latency",
|
||||
"google_monitoring_alert_policy.relay_gce_connection_headroom",
|
||||
"google_monitoring_alert_policy.relay_postgres_retry_exhausted",
|
||||
"google_monitoring_alert_policy.relay_region_hint_skew",
|
||||
"google_monitoring_dashboard.relay_incident",
|
||||
"google_project_iam_custom_role.github_production_relay_capacity_mutation",
|
||||
"google_project_iam_custom_role.github_relay_asia_topology_mutation",
|
||||
|
||||
@@ -45,6 +45,7 @@ export function parseRegionalRehomeArguments(argv, environment = process.env) {
|
||||
'not-before',
|
||||
'rate-per-minute',
|
||||
'preference-max-age-ms',
|
||||
'host-cooldown-ms',
|
||||
'drain-grace-ms',
|
||||
'confirmation'
|
||||
]
|
||||
@@ -117,6 +118,11 @@ export function parseRegionalRehomeArguments(argv, environment = process.env) {
|
||||
'--preference-max-age-ms',
|
||||
{ minimum: 60_000, maximum: 30 * 24 * 60 * 60_000 }
|
||||
),
|
||||
hostCooldownMs: integer(
|
||||
values['host-cooldown-ms'],
|
||||
'--host-cooldown-ms',
|
||||
{ minimum: 60_000, maximum: 30 * 24 * 60 * 60_000 }
|
||||
),
|
||||
drainGraceMs: integer(values['drain-grace-ms'], '--drain-grace-ms', {
|
||||
minimum: 60_000,
|
||||
maximum: 60 * 60_000
|
||||
@@ -147,6 +153,11 @@ function assertControl(control, expected) {
|
||||
!Number.isSafeInteger(control.notBefore) ||
|
||||
!Number.isSafeInteger(control.ratePerMinute) ||
|
||||
!Number.isSafeInteger(control.preferenceMaxAgeMs) ||
|
||||
// A director predating the per-host cooldown does not report it. Reading
|
||||
// the control and both emergency brakes must keep working against that
|
||||
// image; only enable requires the field.
|
||||
(control.hostCooldownMs !== undefined &&
|
||||
!Number.isSafeInteger(control.hostCooldownMs)) ||
|
||||
!Number.isSafeInteger(control.drainGraceMs)
|
||||
) throw new Error('director returned an invalid regional rehome control')
|
||||
if (expected.enabled !== undefined && control.enabled !== expected.enabled) {
|
||||
@@ -155,6 +166,12 @@ function assertControl(control, expected) {
|
||||
return control
|
||||
}
|
||||
|
||||
// Echo the cooldown only when the director already reports it: a legacy
|
||||
// director rejects the unknown key outright and would refuse every brake.
|
||||
function cooldownField(before, value) {
|
||||
return before.hostCooldownMs === undefined ? {} : { hostCooldownMs: value }
|
||||
}
|
||||
|
||||
async function verifiedDisabledControl(post, generation) {
|
||||
return assertControl((await post('/v1/admin/regional-rehome-control', {
|
||||
v: 1,
|
||||
@@ -171,6 +188,7 @@ async function applyDisabledControl(post, before) {
|
||||
notBefore: before.notBefore,
|
||||
ratePerMinute: before.ratePerMinute,
|
||||
preferenceMaxAgeMs: before.preferenceMaxAgeMs,
|
||||
...cooldownField(before, before.hostCooldownMs),
|
||||
drainGraceMs: before.drainGraceMs,
|
||||
confirmation: 'DISABLE_REGIONAL_REHOMING'
|
||||
})).control, { generation: before.generation + 1, enabled: false })
|
||||
@@ -270,6 +288,11 @@ export async function operateRegionalRehome(config, dependencies = {}) {
|
||||
throw new Error('regional rehome is already paused')
|
||||
}
|
||||
const enabled = config.mode === 'enable'
|
||||
if (enabled && before.hostCooldownMs === undefined) {
|
||||
throw new Error(
|
||||
'director does not report a per-host rehome cooldown; deploy a director that supports it before enabling'
|
||||
)
|
||||
}
|
||||
const applied = await post('/v1/admin/regional-rehome-control', {
|
||||
v: 1,
|
||||
action: 'apply',
|
||||
@@ -278,6 +301,7 @@ export async function operateRegionalRehome(config, dependencies = {}) {
|
||||
notBefore: config.notBefore,
|
||||
ratePerMinute: config.ratePerMinute,
|
||||
preferenceMaxAgeMs: config.preferenceMaxAgeMs,
|
||||
...cooldownField(before, config.hostCooldownMs),
|
||||
drainGraceMs: config.drainGraceMs,
|
||||
confirmation: enabled
|
||||
? 'ENABLE_REGIONAL_REHOMING'
|
||||
|
||||
@@ -26,6 +26,7 @@ function argumentsFor(mode, confirmation) {
|
||||
'--not-before', '2000000000000',
|
||||
'--rate-per-minute', '10',
|
||||
'--preference-max-age-ms', '86400000',
|
||||
'--host-cooldown-ms', '604800000',
|
||||
'--drain-grace-ms', '60000',
|
||||
'--confirmation', confirmation
|
||||
])
|
||||
@@ -40,10 +41,29 @@ function control(generation, enabled) {
|
||||
notBefore: 2_000_000_000_000,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 86_400_000,
|
||||
hostCooldownMs: 604_800_000,
|
||||
drainGraceMs: 60_000
|
||||
}
|
||||
}
|
||||
|
||||
// The control a director predating the per-host cooldown reports.
|
||||
function legacyControl(generation, enabled) {
|
||||
const { hostCooldownMs: _absent, ...rest } = control(generation, enabled)
|
||||
return rest
|
||||
}
|
||||
|
||||
function legacyDirector(controls) {
|
||||
const requests = []
|
||||
const post = async (path, body) => {
|
||||
requests.push({ path, body })
|
||||
if (path === '/v1/admin/admission-selector/status') {
|
||||
return { selector: { generation: 11, membership } }
|
||||
}
|
||||
return { v: 1, control: controls.shift() }
|
||||
}
|
||||
return { requests, post }
|
||||
}
|
||||
|
||||
test('parses exact selector and typed control confirmation', () => {
|
||||
const parsed = parseRegionalRehomeArguments(
|
||||
argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING'),
|
||||
@@ -52,6 +72,17 @@ test('parses exact selector and typed control confirmation', () => {
|
||||
assert.equal(parsed.expectedSelectorGeneration, 11)
|
||||
assert.equal(parsed.expectedControlGeneration, 4)
|
||||
assert.equal(parsed.ratePerMinute, 10)
|
||||
assert.equal(parsed.hostCooldownMs, 604_800_000)
|
||||
assert.throws(
|
||||
() => parseRegionalRehomeArguments(
|
||||
argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING').filter(
|
||||
(value, index, all) =>
|
||||
value !== '--host-cooldown-ms' && all[index - 1] !== '--host-cooldown-ms'
|
||||
),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
),
|
||||
/complete durable control shape/
|
||||
)
|
||||
assert.throws(
|
||||
() => parseRegionalRehomeArguments(
|
||||
argumentsFor('pause', 'DISABLE_REGIONAL_REHOMING'),
|
||||
@@ -79,6 +110,7 @@ test('binds enable to exact selector and durable control generations', async ()
|
||||
notBefore: 0,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 86_400_000,
|
||||
hostCooldownMs: 604_800_000,
|
||||
drainGraceMs: 60_000,
|
||||
...control
|
||||
}))
|
||||
@@ -96,6 +128,7 @@ test('binds enable to exact selector and durable control generations', async ()
|
||||
}
|
||||
})
|
||||
assert.equal(result.control.generation, 5)
|
||||
assert.equal(result.control.hostCooldownMs, 604_800_000)
|
||||
assert.deepEqual(requests[2].body, {
|
||||
v: 1,
|
||||
action: 'apply',
|
||||
@@ -104,11 +137,82 @@ test('binds enable to exact selector and durable control generations', async ()
|
||||
notBefore: 2_000_000_000_000,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 86_400_000,
|
||||
hostCooldownMs: 604_800_000,
|
||||
drainGraceMs: 60_000,
|
||||
confirmation: 'ENABLE_REGIONAL_REHOMING'
|
||||
})
|
||||
})
|
||||
|
||||
test('inspects a director that predates the per-host cooldown', async () => {
|
||||
const director = legacyDirector([legacyControl(4, true)])
|
||||
const config = parseRegionalRehomeArguments(
|
||||
argumentsFor('inspect'),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
)
|
||||
|
||||
const result = await operateRegionalRehome(config, { post: director.post })
|
||||
|
||||
assert.equal(result.control.generation, 4)
|
||||
assert.equal(result.control.hostCooldownMs, undefined)
|
||||
})
|
||||
|
||||
for (const [mode, confirmation, enabledBefore] of [
|
||||
['pause', 'PAUSE_REGIONAL_REHOMING', true],
|
||||
['disable', 'DISABLE_REGIONAL_REHOMING', false]
|
||||
]) {
|
||||
test(`${mode} still brakes a director that predates the cooldown`, async () => {
|
||||
const director = legacyDirector([
|
||||
legacyControl(4, enabledBefore),
|
||||
legacyControl(5, false),
|
||||
legacyControl(5, false)
|
||||
])
|
||||
const config = parseRegionalRehomeArguments(
|
||||
argumentsFor(mode, confirmation),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
)
|
||||
|
||||
const result = await operateRegionalRehome(config, { post: director.post })
|
||||
|
||||
assert.equal(result.control.generation, 5)
|
||||
// The unknown key would be refused by that director's strict schema.
|
||||
assert.equal('hostCooldownMs' in director.requests[2].body, false)
|
||||
assert.equal(director.requests[2].body.confirmation, 'DISABLE_REGIONAL_REHOMING')
|
||||
})
|
||||
}
|
||||
|
||||
test('failed-enable recovery brakes a director that predates the cooldown', async () => {
|
||||
const requests = []
|
||||
let current = legacyControl(7, true)
|
||||
const result = await recoverRegionalRehomeEnable({
|
||||
mode: 'recover-enable',
|
||||
expectedControlGeneration: 4
|
||||
}, async (_path, body) => {
|
||||
requests.push(body)
|
||||
if (body.action === 'inspect') return { control: current }
|
||||
current = legacyControl(8, false)
|
||||
return { control: current }
|
||||
})
|
||||
|
||||
assert.equal(result.control.generation, 8)
|
||||
assert.equal('hostCooldownMs' in requests[1], false)
|
||||
})
|
||||
|
||||
test('refuses to enable a director that does not report the cooldown', async () => {
|
||||
const director = legacyDirector([legacyControl(4, false)])
|
||||
const config = parseRegionalRehomeArguments(
|
||||
argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING'),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
)
|
||||
|
||||
await assert.rejects(
|
||||
operateRegionalRehome(config, { post: director.post }),
|
||||
/per-host rehome cooldown/
|
||||
)
|
||||
// Read-only: selector status and the control inspect, and nothing else.
|
||||
assert.equal(director.requests.length, 2)
|
||||
assert.equal(director.requests.every(({ body }) => body.action !== 'apply'), true)
|
||||
})
|
||||
|
||||
test('fails closed on selector drift before reading or mutating control', async () => {
|
||||
let calls = 0
|
||||
const config = parseRegionalRehomeArguments(
|
||||
@@ -150,6 +254,7 @@ test('failed-enable recovery CAS-disables an advanced enabled generation', async
|
||||
notBefore: 2_000_000_000_000,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 86_400_000,
|
||||
hostCooldownMs: 604_800_000,
|
||||
drainGraceMs: 60_000,
|
||||
confirmation: 'DISABLE_REGIONAL_REHOMING'
|
||||
})
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
|
||||
|
||||
const PRODUCTION_CELL = /^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26)$/
|
||||
// Every general cell that carries the rehome identity: the sixteen US cells and the
|
||||
// three asia-east2 cells that drain mis-homed hosts back the other way.
|
||||
const PRODUCTION_CELL = /^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26|27|28|29)$/
|
||||
const DIRECTOR_ORIGIN = 'https://relay.onorca.dev'
|
||||
|
||||
export function parseRehomeTrustProbeArguments(argv, environment = process.env) {
|
||||
|
||||
@@ -111,3 +111,23 @@ test('fails when both trust-probe attempts return a transient 503', async () =>
|
||||
)
|
||||
assert.equal(calls, 2)
|
||||
})
|
||||
|
||||
test('approves the asia-east2 rehome sources and still rejects unlisted cells', () => {
|
||||
for (const cellId of ['production-gce-c27', 'production-gce-c28', 'production-gce-c29']) {
|
||||
const parsed = parseRehomeTrustProbeArguments(
|
||||
argv.map((value) => (value === 'production-gce-c7' ? cellId : value)),
|
||||
environment
|
||||
)
|
||||
assert.equal(parsed.cellId, cellId)
|
||||
}
|
||||
for (const cellId of ['production-gce-c1', 'production-gce-c17', 'production-gce-c30']) {
|
||||
assert.throws(
|
||||
() =>
|
||||
parseRehomeTrustProbeArguments(
|
||||
argv.map((value) => (value === 'production-gce-c7' ? cellId : value)),
|
||||
environment
|
||||
),
|
||||
/--cell-id is not approved/
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import test from 'node:test'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
// Why: the region-skew alert compares asia-east2's share of assignment hints against its share of
|
||||
// actual placements. Both shares are sums over one log-based metric per region, and the region
|
||||
// list is written out by hand in Terraform. A region added to the contract without matching
|
||||
// metrics would silently drop out of both denominators and move the ratio the alert fires on.
|
||||
|
||||
const read = (relative) => readFileSync(fileURLToPath(new URL(relative, import.meta.url)), 'utf8')
|
||||
const collapse = (text) => text.replaceAll(/\s+/g, ' ')
|
||||
|
||||
const contractRegions = (() => {
|
||||
const source = read('../../packages/relay-contract/src/relay-regions.ts')
|
||||
const literal = /export const RELAY_REGIONS = \[([^\]]*)\]/.exec(source)
|
||||
assert.ok(literal, 'RELAY_REGIONS literal not found in relay-regions.ts')
|
||||
return [...literal[1].matchAll(/'([^']+)'/g)].map((match) => match[1])
|
||||
})()
|
||||
|
||||
const terraform = read('../../infra/terraform/relay-observability.tf')
|
||||
|
||||
const terraformRegions = (() => {
|
||||
const literal = /relay_region_keys = \[([^\]]*)\]/.exec(terraform)
|
||||
assert.ok(literal, 'relay_region_keys not found in relay-observability.tf')
|
||||
return [...literal[1].matchAll(/"([^"]+)"/g)].map((match) => match[1])
|
||||
})()
|
||||
|
||||
// Both sides now spell the field-name segments out, so the test compares the two declared maps
|
||||
// rather than two source expressions. Reformatting either file cannot break this, and a literal
|
||||
// expected value below still catches an identical wrong edit made to both.
|
||||
const declaredSegments = (source, open, close) => {
|
||||
const body = source.slice(source.indexOf(open) + open.length, source.indexOf(close, source.indexOf(open)))
|
||||
return Object.fromEntries(
|
||||
[...body.matchAll(/'?"?([a-z0-9-]+)'?"?\s*[:=]\s*'?"?([A-Za-z0-9]+)'?"?/g)].map((match) => [
|
||||
match[1],
|
||||
match[2]
|
||||
])
|
||||
)
|
||||
}
|
||||
|
||||
const terraformSegments = declaredSegments(terraform, 'relay_region_field_segments = {', '}')
|
||||
const contractSegments = declaredSegments(
|
||||
read('../../packages/relay-contract/src/relay-regions.ts'),
|
||||
'RELAY_REGION_METRIC_SEGMENTS = {',
|
||||
'}'
|
||||
)
|
||||
|
||||
test('terraform covers exactly the regions the contract can hint or select', () => {
|
||||
assert.deepEqual([...terraformRegions].sort(), [...contractRegions].sort())
|
||||
})
|
||||
|
||||
test('terraform and the contract declare the same flat field segments', () => {
|
||||
assert.deepEqual(terraformSegments, contractSegments)
|
||||
// Pinned literally so the same wrong edit applied to both sides still fails.
|
||||
assert.deepEqual(terraformSegments, { 'us-central1': 'UsCentral1', 'asia-east2': 'AsiaEast2' })
|
||||
assert.deepEqual(Object.keys(terraformSegments).sort(), [...contractRegions].sort())
|
||||
})
|
||||
|
||||
test('the skew query compares a catalogued region against itself', () => {
|
||||
const columns = terraformRegions.map((region) => region.replaceAll('-', '_'))
|
||||
const hint = /hint_share: req_([a-z0-9_]+) \//.exec(terraform)
|
||||
const placement = /placement_share: sel_([a-z0-9_]+) \//.exec(terraform)
|
||||
assert.ok(hint && placement, 'skew query share columns not found')
|
||||
assert.equal(hint[1], placement[1], 'the two shares must be about the same region')
|
||||
assert.ok(columns.includes(hint[1]), `${hint[1]} is not one of ${columns.join(', ')}`)
|
||||
})
|
||||
|
||||
test('the skew condition never divides by the placement share', () => {
|
||||
// A zero-placement hour is the worst skew there is; MQL drops the row on x/0, so the ratio form
|
||||
// silences exactly the case the alert exists for.
|
||||
assert.ok(
|
||||
!/hint_share \/ placement_share/.test(terraform),
|
||||
'cross-multiply instead: hint_share > 2 * placement_share'
|
||||
)
|
||||
assert.match(collapse(terraform), /condition hint_share > 2 \* placement_share/)
|
||||
})
|
||||
|
||||
test('the unhinted bucket stays out of the skew denominators', () => {
|
||||
assert.ok(
|
||||
!terraformRegions.includes('unhinted'),
|
||||
'unhinted requests are a client-side choice, not a region; including them moves the share'
|
||||
)
|
||||
})
|
||||
@@ -179,9 +179,10 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
|
||||
|
||||
it('validates a correct plan for every wave cell at that cell\'s rehome protocol', () => {
|
||||
for (const cellId of SAME_CAP_CELLS) {
|
||||
const [region, cap] = resolveCellShape(cellId).stdout.trim().split(' ')
|
||||
const [, cap] = resolveCellShape(cellId).stdout.trim().split(' ')
|
||||
const protocol = REHOME_SOURCE_CELLS.has(cellId) ? 1 : 0
|
||||
assert.equal(protocol, region === 'us-central1' ? 1 : 0, cellId)
|
||||
// Every reviewed serving cell carries rehome trust now, in either region.
|
||||
assert.equal(protocol, 1, cellId)
|
||||
const config = {
|
||||
mode: 'same-cap-cell',
|
||||
cellId,
|
||||
@@ -211,6 +212,29 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('validates a protocol-0 plan for a cell outside the rehome source list', () => {
|
||||
const cellId = 'production-gce-c17'
|
||||
assert.equal(REHOME_SOURCE_CELLS.has(cellId), false)
|
||||
const config = {
|
||||
mode: 'same-cap-cell',
|
||||
cellId,
|
||||
hardCap: 1000,
|
||||
unobservedBound: 60,
|
||||
image: TARGET_IMAGE,
|
||||
rollbackImage: ROLLBACK_IMAGE,
|
||||
rehomeDirectorServiceAccount: DIRECTOR_IDENTITY,
|
||||
rehomeAudience: AUDIENCE,
|
||||
regionalRehomeProtocol: '0'
|
||||
}
|
||||
const plan = rollPlan({ cellId, cap: 1000, protocol: 0 })
|
||||
assert.deepEqual(validateCapacityPlan(plan, config), { mode: 'same-cap-cell', changes: 2 })
|
||||
// Protocol 1 must reject a plan with no rehome lines, or the absent-line rule decides nothing.
|
||||
assert.throws(
|
||||
() => validateCapacityPlan(plan, { ...config, regionalRehomeProtocol: '1' }),
|
||||
/reviewed image and capacity/
|
||||
)
|
||||
})
|
||||
|
||||
it('leaves the US-only capacity job on the default allowlist', () => {
|
||||
assert.doesNotMatch(capacityWorkflow, /--approved-cells/)
|
||||
})
|
||||
|
||||
@@ -464,6 +464,18 @@ Once a target control is registered, do not force the pre-registration rollback.
|
||||
|
||||
After a deployment traffic shift, preserve the old revision/tag until metrics and live reconnect checks pass. If the new revision is unhealthy, shift traffic back only while old controls are still valid, then issue a strictly newer director migration rather than reusing a prior epoch.
|
||||
|
||||
## Regional rehoming
|
||||
|
||||
Rehoming moves a host to a general cell in the region its desktop last reported, in either
|
||||
direction. Both roles need the drain protocol: a cell without it can be neither a source nor a
|
||||
target, and it is not part of the fleet whose telemetry gates the worker. Until the asia-east2
|
||||
cells run `regionalRehomeProtocol` 1 they are none of the three, so no host is moved into or out
|
||||
of Asia and an Asia cell in distress does not pause the worker.
|
||||
|
||||
`host-cooldown-ms` is the minimum gap between two rehomes of one host. It bounds the damage from
|
||||
a desktop whose region probe flips: without it the host would be dragged back across the ocean on
|
||||
every flip, since the preference age never expires while the host keeps reconnecting.
|
||||
|
||||
## Game-day matrix
|
||||
|
||||
Run and record each scenario in staging before launch:
|
||||
|
||||
@@ -121,6 +121,79 @@ durably marked consumed before mutation and cannot authorize another run.
|
||||
Expected enabled cells must also have a powered runtime, healthy and ready endpoints, fresh
|
||||
heartbeats, and matching live admission.
|
||||
|
||||
## Region placement alert policies
|
||||
|
||||
Cloud Monitoring alert policies, not monitor freeze bars: these page from
|
||||
`cloud/infra/terraform/relay-observability.tf` on the shared relay channel in
|
||||
`relay_alert_notification_channels`, and they do not gate any workflow. All
|
||||
three exist because US desktops sat on asia-east2 cells for weeks in 2026-08
|
||||
with every existing bar green.
|
||||
|
||||
| Alert policy | Condition |
|
||||
| --- | ---: |
|
||||
| Orca Relay: far-cell phone accept latency | per cell, median 30-second `clientAcceptTotalMsP95` over 15 minutes above 2,000 ms with at least 20 completed accepts |
|
||||
| Orca Relay: cell control round trip | per cell, median `controlRttMsP50` over one hour above 150 ms with at least 500 samples |
|
||||
| Orca Relay: region hint skew | fleet-wide, asia-east2 share of hinted requests over one hour more than 2x and more than 15 points above its share of actual placements, with at least 500 hinted requests |
|
||||
|
||||
Threshold basis:
|
||||
|
||||
- Accept latency. An in-region phone accept completes in 0.3-0.6 s and a
|
||||
cross-Pacific one in 5-10 s, so 2,000 ms sits outside in-region noise and
|
||||
well under the far-cell floor. The 20-accept minimum keeps one slow accept
|
||||
on a quiet cell off the pager. The p95 is the published value, so the
|
||||
window aggregate is its median, not its max.
|
||||
- Control round trip. In-region is tens of milliseconds; a US desktop on an
|
||||
asia-east2 cell is 200 ms or more. Only the p50 is used. The desktop echoes
|
||||
the pong on its main thread, so the published p95 and max track renderer
|
||||
stalls rather than distance. 500 samples per hour is about two
|
||||
continuously connected hosts at the 15-second control ping. Tuning risk: EU
|
||||
desktops on us-central1 sit at 100-130 ms, so a cell whose population is
|
||||
mostly European can approach the bar while correctly homed. Check where the
|
||||
hosts are before reading a first breach as mis-homing.
|
||||
- Region hint skew. This compares two shares of the same hour rather than
|
||||
testing one absolute share, because an absolute bar is wrong at both ends.
|
||||
Measured over twelve hours on 2026-09-07, while the desktop region probe
|
||||
was still mis-picking: asia-east2 was 33.8% of the 33,800 hinted requests
|
||||
and only 7.9% of the 45,364 assignments, a divergence of 4.27x and a gap of
|
||||
25.9 points. A fixed 40% bar would have stayed silent through that, and
|
||||
once the probe is fixed the genuine APAC share climbs past any such bar and
|
||||
pages forever on the correct end state. The 2x and 15-point bars sit inside
|
||||
the broken state and outside a healthy one. `unhinted` requests are
|
||||
excluded from the denominator: they were 27% of all requests, so a client
|
||||
change that always sends a hint would move the number with no behaviour
|
||||
change at all. The two bars are cross-multiplied rather than divided. An
|
||||
hour that placed nobody in the region is the most extreme skew there is,
|
||||
and it happens whenever the region is drained, fenced, or at capacity, but
|
||||
dividing by that zero placement share makes MQL drop the row and lose the
|
||||
series before any other clause runs.
|
||||
|
||||
Expect the skew alert to stay lit after a client fix until the mis-homed
|
||||
backlog is rehomed. Sticky assignment never re-consults the hint, so a
|
||||
desktop already on an asia cell keeps being placed there whatever it now
|
||||
asks for; the ratio clears only once the rehome sweep has drained.
|
||||
|
||||
All three conditions are written in MQL rather than the metric filters the
|
||||
other relay policies use. Every runtime metric is a DELTA DISTRIBUTION, and
|
||||
the only scalar aligners a filter condition can apply to one are percentiles;
|
||||
each of these alerts needs the sum of the extracted values as a volume floor,
|
||||
which is `sum(value.<metric>)` in MQL and unreachable otherwise. None of the
|
||||
metrics they read exists in the project yet, so what was checked against
|
||||
production is the query shape: the same MQL run over existing metrics of the
|
||||
same kind confirmed the distribution sum, the join arity, the unit literals,
|
||||
and the condition clause.
|
||||
|
||||
The skew shares are built from one log-based metric per region for hints and
|
||||
one per region for placements. They read flat `requestedRegion<Region>Delta`
|
||||
and `selectedRegion<Region>Delta` fields that the relay publishes as zeros in
|
||||
every interval, not the nested region maps: a log-based metric would need a
|
||||
quoted field path to reach a hyphenated map key, and an absent key would drop
|
||||
a series out of the inner join. The region list lives in Terraform as
|
||||
`relay_region_keys` and is pinned to relay-contract's `RELAY_REGIONS` by
|
||||
`dev/scripts/relay-region-hint-metrics.test.mjs`. Both sides spell the field
|
||||
name segments out as literal maps rather than deriving them, so the same test
|
||||
compares the two declarations directly. Adding a region to the contract
|
||||
without its segment is a compile error in relay-contract, not a silent gap.
|
||||
|
||||
## Implementation log
|
||||
|
||||
- Recalibrated the relay pool freezes from 30 waiters / 1,000 ms to
|
||||
|
||||
@@ -402,7 +402,11 @@ relay_region_rehome_source_cell_ids = [
|
||||
"production-gce-c23",
|
||||
"production-gce-c24",
|
||||
"production-gce-c25",
|
||||
"production-gce-c26"
|
||||
"production-gce-c26",
|
||||
# Asia cells carry the same trust so mis-homed hosts can be drained back off them.
|
||||
"production-gce-c27",
|
||||
"production-gce-c28",
|
||||
"production-gce-c29"
|
||||
]
|
||||
|
||||
# Slack #orca-relay-alerts, created out of band on 2026-08-05. Declared here because an apply
|
||||
|
||||
@@ -82,14 +82,15 @@ check "relay_gce_fixed_one_topology" {
|
||||
|
||||
assert {
|
||||
condition = alltrue([
|
||||
# Region is not asserted here: the director's own rehome source and target predicates
|
||||
# own eligibility, so this pins only cell shape.
|
||||
for cell_id in var.relay_region_rehome_source_cell_ids : try(
|
||||
var.relay_gce_cells[cell_id].region == var.region &&
|
||||
var.relay_gce_cells[cell_id].connection_hard_cap != null &&
|
||||
!contains(var.relay_gce_fenced_cells, cell_id),
|
||||
false
|
||||
)
|
||||
])
|
||||
error_message = "Regional rehome sources must be configured, unfenced primary-region GCE cells with explicit connection limits."
|
||||
error_message = "Regional rehome sources must be configured, unfenced GCE cells with explicit connection limits."
|
||||
}
|
||||
|
||||
assert {
|
||||
|
||||
@@ -65,6 +65,20 @@ locals {
|
||||
control_renewal_lease_misses = { field = "controlRenewalLeaseMissesDelta", description = "Control renewals that found their activity lease missing." }
|
||||
control_activity_recoveries = { field = "controlActivityRecoveriesDelta", description = "Control activity leases recovered after a renewal miss." }
|
||||
control_activity_recovery_failures = { field = "controlActivityRecoveryFailuresDelta", description = "Control activity lease recovery attempts that failed." }
|
||||
control_rtt_ms_p50 = { field = "controlRttMsP50", description = "Control-socket ping round trip p50 in the interval. The desktop echoes the pong on its main thread, so only the median reads as distance; the p95 and max below are dominated by desktop stalls." }
|
||||
control_rtt_ms_p95 = { field = "controlRttMsP95", description = "Control-socket ping round trip p95 in the interval; a desktop-stall signal, not a distance one." }
|
||||
control_rtt_ms_max = { field = "controlRttMsMax", description = "Maximum control-socket ping round trip in the interval; a desktop-stall signal, not a distance one." }
|
||||
control_rtt_samples = { field = "controlRttSamplesDelta", description = "Control-socket round trips observed in the interval, one per ping answered; the percentiles above are omitted when this is zero." }
|
||||
control_rtt_samples_dropped = { field = "controlRttSamplesDroppedDelta", description = "Observed round trips the bounded percentile reservoir did not keep; non-zero means the percentiles above summarise a uniform sample of the interval." }
|
||||
client_accepts_completed = { field = "clientAcceptCompletedDelta", description = "Phone accepts that reached relay-hello in the interval; the percentiles below are omitted when this is zero." }
|
||||
client_accept_total_ms_p50 = { field = "clientAcceptTotalMsP50", description = "Successful phone-accept duration p50, dial to relay-hello." }
|
||||
client_accept_total_ms_p95 = { field = "clientAcceptTotalMsP95", description = "Successful phone-accept duration p95, dial to relay-hello." }
|
||||
client_accept_total_ms_max = { field = "clientAcceptTotalMsMax", description = "Maximum successful phone-accept duration in the interval." }
|
||||
client_accept_assignment_ms_p95 = { field = "clientAcceptAssignmentMsP95", description = "Accept stage p95: resume/invite lookup plus assignment resolve." }
|
||||
client_accept_credential_ms_p95 = { field = "clientAcceptCredentialMsP95", description = "Accept stage p95: outer credential reservation." }
|
||||
client_accept_activity_ms_p95 = { field = "clientAcceptActivityMsP95", description = "Accept stage p95: credential activity lease acquisition." }
|
||||
client_accept_attach_ms_p95 = { field = "clientAcceptAttachMsP95", description = "Accept stage p95: conn-open sent until the desktop's data leg authenticated." }
|
||||
client_accept_basis_ms_p95 = { field = "clientAcceptBasisMsP95", description = "Accept stage p95: splice lease and connection-basis writes between the data leg and relay-hello." }
|
||||
heap_used_bytes = { field = "heapUsedBytes", description = "Node.js heap bytes used by the relay process." }
|
||||
event_loop_ms_p99 = { field = "eventLoopDelayMsP99", description = "Node.js event-loop delay p99 in milliseconds." }
|
||||
forwarded_bytes = { field = "forwardedBytesDelta", description = "Ciphertext bytes admitted for forwarding." }
|
||||
@@ -80,6 +94,80 @@ locals {
|
||||
db_oldest_wait_ms = { field = "databasePoolOldestWaitMs", description = "Current oldest PostgreSQL pool waiter age." }
|
||||
db_wait_ms_max = { field = "databasePoolWaitMsMax", description = "Maximum PostgreSQL pool wait during the interval." }
|
||||
}
|
||||
|
||||
# Regions the director can hint or select. Pinned to relay-contract's RELAY_REGIONS by
|
||||
# dev/scripts/relay-region-hint-metrics.test.mjs, which also checks the flat field names below
|
||||
# against the emitter. A region missing here drops out of both shares the skew alert compares.
|
||||
relay_region_keys = ["us-central1", "asia-east2"]
|
||||
# Flat emitter fields, not the nested `requestedRegionsDelta` map: a log-based metric would need
|
||||
# a quoted field path to reach a hyphenated map key, and the relay publishes these as zeros in
|
||||
# every interval so no series can drop out of the alert's inner join. Spelled out rather than
|
||||
# derived, so this literal and relay-contract's RELAY_REGION_METRIC_SEGMENTS can be compared
|
||||
# directly; reformatting either side cannot break the check and neither can drift alone.
|
||||
relay_region_field_segments = {
|
||||
"us-central1" = "UsCentral1"
|
||||
"asia-east2" = "AsiaEast2"
|
||||
}
|
||||
relay_region_columns = { for key in local.relay_region_keys : key => replace(key, "-", "_") }
|
||||
relay_region_share_metrics = merge(
|
||||
{
|
||||
for key in local.relay_region_keys :
|
||||
"requested_regions_${local.relay_region_columns[key]}" => {
|
||||
field = "requestedRegion${local.relay_region_field_segments[key]}Delta"
|
||||
description = "Assignment requests that hinted ${key}."
|
||||
}
|
||||
},
|
||||
{
|
||||
for key in local.relay_region_keys :
|
||||
"selected_regions_${local.relay_region_columns[key]}" => {
|
||||
field = "selectedRegion${local.relay_region_field_segments[key]}Delta"
|
||||
description = "Assignments that placed a host in ${key}."
|
||||
}
|
||||
}
|
||||
)
|
||||
relay_region_hinted_total = join(" + ", [for key in local.relay_region_keys : "req_${local.relay_region_columns[key]}"])
|
||||
relay_region_selected_total = join(" + ", [for key in local.relay_region_keys : "sel_${local.relay_region_columns[key]}"])
|
||||
# MQL, not a filter condition: every runtime metric is a DELTA DISTRIBUTION, and the only scalar
|
||||
# aligners a `condition_threshold` can apply to one are percentiles. Both shares need the sum of
|
||||
# the extracted values, which is `sum(value.<metric>)` in MQL and unreachable otherwise.
|
||||
relay_region_hint_skew_query = join("\n", concat(
|
||||
["{"],
|
||||
flatten([
|
||||
for index, entry in [
|
||||
for key in local.relay_region_keys : { metric = "requested_regions_${local.relay_region_columns[key]}", column = "req_${local.relay_region_columns[key]}" }
|
||||
] : [
|
||||
index == 0 ? "" : ";",
|
||||
" fetch cloud_run_revision::logging.googleapis.com/user/orca_relay_${entry.metric}",
|
||||
" | align delta(1h) | every 1h",
|
||||
" | group_by [], [${entry.column}: sum(value.orca_relay_${entry.metric})]"
|
||||
]
|
||||
]),
|
||||
flatten([
|
||||
for key in local.relay_region_keys : [
|
||||
";",
|
||||
" fetch cloud_run_revision::logging.googleapis.com/user/orca_relay_selected_regions_${local.relay_region_columns[key]}",
|
||||
" | align delta(1h) | every 1h",
|
||||
" | group_by [], [sel_${local.relay_region_columns[key]}: sum(value.orca_relay_selected_regions_${local.relay_region_columns[key]})]"
|
||||
]
|
||||
]),
|
||||
[
|
||||
"}",
|
||||
"| join",
|
||||
"| value [",
|
||||
" hint_share: req_asia_east2 / (${local.relay_region_hinted_total}),",
|
||||
" placement_share: sel_asia_east2 / (${local.relay_region_selected_total}),",
|
||||
" hinted_requests: ${local.relay_region_hinted_total}",
|
||||
" ]",
|
||||
# Cross-multiplied, never a plain ratio of the two shares: an hour that placed nobody in the
|
||||
# region makes that ratio 0/0 or x/0, and MQL drops the row instead of yielding a number, so
|
||||
# the whole series vanishes before the other clauses run. That hour is the worst skew there
|
||||
# is - every desktop asking for a region the director is putting nobody in - and it happens
|
||||
# whenever the region is drained, fenced, or at capacity. Both forms were run read-only
|
||||
# against production surrogates with a zero denominator: the ratio returned no rows, this
|
||||
# returned the series with the condition true.
|
||||
"| condition hint_share > 2 * placement_share && hint_share - placement_share > 0.15 '1' && hinted_requests > 500 '1'"
|
||||
]
|
||||
))
|
||||
relay_custom_alerts = {
|
||||
connection_headroom = {
|
||||
pages_oncall = true
|
||||
@@ -201,7 +289,9 @@ locals {
|
||||
}
|
||||
|
||||
resource "google_logging_metric" "relay_snapshot" {
|
||||
for_each = local.relay_runtime_metrics
|
||||
# Region-request metrics ride the same event and shape; merging adds map entries only, so the
|
||||
# existing metric instances are untouched (a label change, not a new key, is what recreates them).
|
||||
for_each = merge(local.relay_runtime_metrics, local.relay_region_share_metrics)
|
||||
|
||||
project = var.project_id
|
||||
name = "orca_relay_${each.key}"
|
||||
@@ -211,14 +301,14 @@ resource "google_logging_metric" "relay_snapshot" {
|
||||
label_extractors = {
|
||||
role = "EXTRACT(jsonPayload.role)"
|
||||
cell_id = "EXTRACT(jsonPayload.cellId)"
|
||||
# No region label: adding one replaces all 21 live metrics (label change = delete+create),
|
||||
# No region label: adding one replaces all 42 live metrics (label change = delete+create),
|
||||
# which resets history and blanks the relay alert policies during the swap.
|
||||
}
|
||||
|
||||
metric_descriptor {
|
||||
metric_kind = "DELTA"
|
||||
value_type = "DISTRIBUTION"
|
||||
unit = contains(["sql_latency_ms", "control_renewal_latency_ms_p50", "control_renewal_latency_ms_p95", "control_renewal_latency_ms_max", "http_latency_ms", "event_loop_ms_p99", "db_oldest_wait_ms", "db_wait_ms_max"], each.key) ? "ms" : each.key == "queued_bytes" || each.key == "heap_used_bytes" || each.key == "forwarded_bytes" ? "By" : "1"
|
||||
unit = contains(["sql_latency_ms", "control_rtt_ms_p50", "control_rtt_ms_p95", "control_rtt_ms_max", "client_accept_total_ms_p50", "client_accept_total_ms_p95", "client_accept_total_ms_max", "client_accept_assignment_ms_p95", "client_accept_credential_ms_p95", "client_accept_activity_ms_p95", "client_accept_attach_ms_p95", "client_accept_basis_ms_p95", "control_renewal_latency_ms_p50", "control_renewal_latency_ms_p95", "control_renewal_latency_ms_max", "http_latency_ms", "event_loop_ms_p99", "db_oldest_wait_ms", "db_wait_ms_max"], each.key) ? "ms" : each.key == "queued_bytes" || each.key == "heap_used_bytes" || each.key == "forwarded_bytes" ? "By" : "1"
|
||||
|
||||
labels {
|
||||
key = "role"
|
||||
@@ -672,6 +762,128 @@ resource "google_monitoring_alert_policy" "relay_cell_process_exit" {
|
||||
depends_on = [google_logging_metric.relay_incident]
|
||||
}
|
||||
|
||||
# Why: nothing fired while US desktops sat on asia-east2 cells for weeks in 2026-08. The two
|
||||
# per-cell policies below read that as distance, and the fleet-wide one reads it as a bad region
|
||||
# hint. All three are MQL because each needs the sum of a DELTA DISTRIBUTION as a volume floor,
|
||||
# and the only scalar aligners a `condition_threshold` can apply to a distribution are percentiles.
|
||||
# `join` is an inner join and the relay omits its percentile fields on an empty interval, so an
|
||||
# idle cell drops out rather than alerting on nothing. The per-cell arms fetch `gce_instance`
|
||||
# only: production runs no Cloud Run cells (`relay_cells` is empty), and a future one would need
|
||||
# its own arm here. None of the metrics these query exist in the project yet, so what was checked
|
||||
# against production is the query shape: the same MQL run over existing metrics of the same kind
|
||||
# confirmed the distribution sum, the join arity, the unit literals, and the condition clause.
|
||||
resource "google_monitoring_alert_policy" "relay_far_cell_accept_latency" {
|
||||
project = var.project_id
|
||||
display_name = "Orca Relay: far-cell phone accept latency"
|
||||
combiner = "OR"
|
||||
enabled = true
|
||||
notification_channels = var.relay_alert_notification_channels
|
||||
|
||||
conditions {
|
||||
display_name = "Phone accept p95 above 2 s for 15 minutes"
|
||||
|
||||
condition_monitoring_query_language {
|
||||
# percentile(..., 50) over the window, not max: the published value is already a p95, so the
|
||||
# median of the interval p95s reads as sustained slowness instead of one bad 30-second flush.
|
||||
query = <<-EOT
|
||||
{
|
||||
fetch gce_instance::logging.googleapis.com/user/orca_relay_client_accept_total_ms_p95
|
||||
| align delta(15m) | every 15m
|
||||
| group_by [metric.cell_id], [accept_p95_ms: percentile(value.orca_relay_client_accept_total_ms_p95, 50)]
|
||||
;
|
||||
fetch gce_instance::logging.googleapis.com/user/orca_relay_client_accepts_completed
|
||||
| align delta(15m) | every 15m
|
||||
| group_by [metric.cell_id], [accepts: sum(value.orca_relay_client_accepts_completed)]
|
||||
}
|
||||
| join
|
||||
| condition accept_p95_ms > 2000 'ms' && accepts >= 20 '1'
|
||||
EOT
|
||||
duration = "0s"
|
||||
|
||||
trigger {
|
||||
count = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
documentation {
|
||||
content = "Phones on this cell are taking over two seconds to reach relay-hello. Measured separation: an in-region accept completes in 0.3-0.6 s and a cross-Pacific one in 5-10 s, so 2 s sits well outside in-region noise and well below the far-cell floor. The 20-accept floor over 15 minutes keeps a single slow accept on a quiet cell from paging. Check which regions the cell's hosts are actually in before touching capacity: the 2026-08 cause was desktops requesting the wrong region, not a slow cell. Read the per-stage `orca_relay_client_accept_*_ms_p95` metrics to separate distance from assignment, credential, or attach work."
|
||||
mime_type = "text/markdown"
|
||||
}
|
||||
|
||||
depends_on = [google_logging_metric.relay_snapshot]
|
||||
}
|
||||
|
||||
resource "google_monitoring_alert_policy" "relay_cell_control_rtt" {
|
||||
project = var.project_id
|
||||
display_name = "Orca Relay: cell control round trip"
|
||||
combiner = "OR"
|
||||
enabled = true
|
||||
notification_channels = var.relay_alert_notification_channels
|
||||
|
||||
conditions {
|
||||
display_name = "Control ping p50 above 150 ms for an hour"
|
||||
|
||||
condition_monitoring_query_language {
|
||||
# p50 only. The desktop echoes the pong on its main thread, so the published p95 and max
|
||||
# track renderer stalls, not distance; the median is the only column that reads as distance.
|
||||
query = <<-EOT
|
||||
{
|
||||
fetch gce_instance::logging.googleapis.com/user/orca_relay_control_rtt_ms_p50
|
||||
| align delta(1h) | every 1h
|
||||
| group_by [metric.cell_id], [control_rtt_p50_ms: percentile(value.orca_relay_control_rtt_ms_p50, 50)]
|
||||
;
|
||||
fetch gce_instance::logging.googleapis.com/user/orca_relay_control_rtt_samples
|
||||
| align delta(1h) | every 1h
|
||||
| group_by [metric.cell_id], [samples: sum(value.orca_relay_control_rtt_samples)]
|
||||
}
|
||||
| join
|
||||
| condition control_rtt_p50_ms > 150 'ms' && samples >= 500 '1'
|
||||
EOT
|
||||
duration = "0s"
|
||||
|
||||
trigger {
|
||||
count = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
documentation {
|
||||
content = "The median desktop on this cell is more than 150 ms away from it, which is a mis-homed population rather than a cell fault: an in-region control ping is tens of milliseconds and a US desktop on an asia-east2 cell is 200 ms or more. This is the signal that was missing while roughly 226 of 332 hosts on the asia cells were non-APAC for weeks in 2026-08. Confirm with the assignment table which regions those hosts requested, then rehome; do not restart or drain the cell on this alert alone. The 500-sample floor is about two continuously connected hosts at the 15-second control ping, so a nearly idle cell cannot alert on one desktop. Tuning risk: EU desktops on us-central1 sit at 100-130 ms, so a cell whose population is mostly European can approach 150 ms while correctly homed. Check where the hosts are before treating a first breach as mis-homing, and raise the bar only with that evidence."
|
||||
mime_type = "text/markdown"
|
||||
}
|
||||
|
||||
depends_on = [google_logging_metric.relay_snapshot]
|
||||
}
|
||||
|
||||
resource "google_monitoring_alert_policy" "relay_region_hint_skew" {
|
||||
project = var.project_id
|
||||
display_name = "Orca Relay: region hint skew"
|
||||
combiner = "OR"
|
||||
enabled = true
|
||||
notification_channels = var.relay_alert_notification_channels
|
||||
|
||||
conditions {
|
||||
display_name = "asia-east2 hint share above 2x its placement share for an hour"
|
||||
|
||||
condition_monitoring_query_language {
|
||||
query = local.relay_region_hint_skew_query
|
||||
duration = "0s"
|
||||
|
||||
trigger {
|
||||
count = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
documentation {
|
||||
content = "Desktops are asking the director for asia-east2 far more often than the director actually places them there, which is what silently homed US desktops on asia cells through 2026-08. The alert compares two shares of the same hour and never an absolute share, because an absolute bar is wrong at both ends: measured over twelve hours on 2026-09-07, while the desktop region probe was still mis-picking, asia-east2 was 33.8% of the 33,800 hinted requests but only 7.9% of the 45,364 assignments, and once the probe is fixed the genuine APAC share will climb past any fixed bar that would have caught this. Divergence was 4.27x with a 25.9-point gap, so the 2x and 15-point bars sit well inside the broken state and well outside a healthy one. `unhinted` requests are excluded from the denominator: they were 27% of all requests, and a client change that always sends a hint would move this number without any behaviour changing. Expect this to stay lit until the mis-homed backlog is rehomed, because sticky assignment never re-consults the hint, so a desktop already on an asia cell keeps being placed there no matter what it now asks for. Investigate the desktop region probe first, not relay placement."
|
||||
mime_type = "text/markdown"
|
||||
}
|
||||
|
||||
depends_on = [google_logging_metric.relay_snapshot]
|
||||
}
|
||||
|
||||
# Why: the four signals that had to be assembled by hand during the 2026-09-04 incident.
|
||||
resource "google_monitoring_dashboard" "relay_incident" {
|
||||
project = var.project_id
|
||||
|
||||
@@ -245,7 +245,7 @@ variable "relay_regional_placement_enabled" {
|
||||
|
||||
variable "relay_region_rehome_source_cell_ids" {
|
||||
type = set(string)
|
||||
description = "Reviewed US Relay cells allowed to advertise and accept the regional rehome source protocol."
|
||||
description = "Reviewed Relay cells, in any configured region, allowed to advertise and accept the regional rehome source protocol."
|
||||
default = []
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@
|
||||
"load:relay:model": "node dev/scripts/run-relay-load-model.mjs",
|
||||
"load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs",
|
||||
"ops:relay": "pnpm --filter @orca-cloud/relay-ops dev",
|
||||
"pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-admission-workflow.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs",
|
||||
"pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-admission-workflow.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs",
|
||||
"test": "pnpm -r test && node --test dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-regional-rehome-workflow.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs",
|
||||
"typecheck": "pnpm -r typecheck"
|
||||
},
|
||||
|
||||
@@ -6,7 +6,10 @@ import {
|
||||
HostChallengeSchema,
|
||||
HostDataAuthSchema,
|
||||
HostHelloAckSchema,
|
||||
HostHelloSchema
|
||||
HostHelloSchema,
|
||||
parseRelayHostCapabilities,
|
||||
RELAY_HOST_CAPABILITIES_HEADER,
|
||||
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS
|
||||
} from './control-messages.js'
|
||||
import {
|
||||
DeviceCredentialInstallSchema,
|
||||
@@ -345,3 +348,47 @@ describe('relay protocol contract', () => {
|
||||
).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('pending connection details capability', () => {
|
||||
it('reads a pending entry with or without the stated kind and device', () => {
|
||||
const ack = {
|
||||
v: 1 as const,
|
||||
generation: 3,
|
||||
controlResumeSecret: 'R'.repeat(43),
|
||||
leaseExpiresAt: 1_800_000_000_000,
|
||||
activeConnIds: []
|
||||
}
|
||||
const identifiers = { connId: 'conn-1', connTicket: 'T'.repeat(43) }
|
||||
expect(HostHelloAckSchema.safeParse({ ...ack, pendingConns: [identifiers] }).success).toBe(true)
|
||||
expect(
|
||||
HostHelloAckSchema.safeParse({
|
||||
...ack,
|
||||
pendingConns: [{ ...identifiers, kind: 'resume', relayDeviceId: 'device-1' }]
|
||||
}).success
|
||||
).toBe(true)
|
||||
// Still strict otherwise: an unannounced key must not slip through as data.
|
||||
expect(
|
||||
HostHelloAckSchema.safeParse({
|
||||
...ack,
|
||||
pendingConns: [{ ...identifiers, reservationId: 'injected' }]
|
||||
}).success
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('pins the header and token the desktop mirrors by hand', () => {
|
||||
// The desktop cannot import this package; drift silently disables the
|
||||
// feature, so both literals are asserted on each side.
|
||||
expect(RELAY_HOST_CAPABILITIES_HEADER).toBe('x-orca-host-capabilities')
|
||||
expect(RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS).toBe('pending-conn-details')
|
||||
})
|
||||
|
||||
it('reads the advertised capabilities from a control upgrade header', () => {
|
||||
expect(
|
||||
parseRelayHostCapabilities(` ${RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS} , future-thing`)
|
||||
).toEqual(new Set([RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS, 'future-thing']))
|
||||
// A host that predates the header sends nothing; absence is never capable.
|
||||
expect(parseRelayHostCapabilities(undefined).size).toBe(0)
|
||||
expect(parseRelayHostCapabilities('').size).toBe(0)
|
||||
expect(parseRelayHostCapabilities('x'.repeat(65)).size).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -44,8 +44,35 @@ export const HostChallengeAckSchema = z
|
||||
.object({ challengeId: OpaqueIdSchema, proofB64: Base6432ByteSchema })
|
||||
.strict()
|
||||
|
||||
// Advertised on the control upgrade rather than in host-hello: HostHelloSchema
|
||||
// is strict, so a new hello key is refused by every already-deployed cell.
|
||||
export const RELAY_HOST_CAPABILITIES_HEADER = 'x-orca-host-capabilities'
|
||||
// The host accepts kind/relayDeviceId on a pendingConns entry. A host that does
|
||||
// not advertise this parses those entries strictly and would drop the whole ack.
|
||||
export const RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS = 'pending-conn-details'
|
||||
|
||||
export function parseRelayHostCapabilities(
|
||||
header: string | string[] | undefined
|
||||
): ReadonlySet<string> {
|
||||
const raw = Array.isArray(header) ? header.join(',') : (header ?? '')
|
||||
return new Set(
|
||||
raw
|
||||
.split(',')
|
||||
.map((token) => token.trim())
|
||||
.filter((token) => token.length > 0 && token.length <= 64)
|
||||
.slice(0, 16)
|
||||
)
|
||||
}
|
||||
|
||||
// kind/relayDeviceId are optional so an entry stays readable by a host that
|
||||
// predates them; the cell only emits them to a host that advertised support.
|
||||
const PendingConnectionSchema = z
|
||||
.object({ connId: OpaqueIdSchema, connTicket: Base64Url32ByteSchema })
|
||||
.object({
|
||||
connId: OpaqueIdSchema,
|
||||
connTicket: Base64Url32ByteSchema,
|
||||
kind: ConnectionKindSchema.optional(),
|
||||
relayDeviceId: OpaqueIdSchema.optional()
|
||||
})
|
||||
.strict()
|
||||
|
||||
export const HostHelloAckSchema = z
|
||||
|
||||
@@ -8,6 +8,15 @@ export type RelayRegion = z.infer<typeof RelayRegionSchema>
|
||||
|
||||
export const RELAY_DEFAULT_REGION: RelayRegion = 'us-central1'
|
||||
|
||||
// Field-name segment for the flat per-region runtime counters, spelled out rather than derived so
|
||||
// the Terraform side can hold the same literal and a test can compare the two. `satisfies` makes a
|
||||
// new region a compile error here, which is the point: a region with no segment would silently
|
||||
// drop out of the region-skew alert's denominators.
|
||||
export const RELAY_REGION_METRIC_SEGMENTS = {
|
||||
'us-central1': 'UsCentral1',
|
||||
'asia-east2': 'AsiaEast2'
|
||||
} as const satisfies Record<RelayRegion, string>
|
||||
|
||||
const RelayProbeOriginSchema = z.string().url().max(2_048).refine(isCanonicalHttpsOrigin)
|
||||
|
||||
export const RelayRegionCatalogResponseSchema = z
|
||||
|
||||
@@ -13325,9 +13325,7 @@
|
||||
},
|
||||
{
|
||||
"file": "src/main/runtime/orchestration/mailbox-pointer-stage.test.ts",
|
||||
"assertions": [
|
||||
"a refused pointer write drains a delivery parked behind its watermark"
|
||||
]
|
||||
"assertions": ["a refused pointer write drains a delivery parked behind its watermark"]
|
||||
},
|
||||
{
|
||||
"file": "src/main/providers/settled-pty-writer-census.test.ts",
|
||||
@@ -18549,27 +18547,13 @@
|
||||
"protection": "partial",
|
||||
"owner": "browser-runtime",
|
||||
"layer": "electron-packaged",
|
||||
"surfaces": [
|
||||
"paired browser placement"
|
||||
],
|
||||
"platforms": [
|
||||
"linux",
|
||||
"macos",
|
||||
"windows"
|
||||
],
|
||||
"providers": [
|
||||
"paired-runtime"
|
||||
],
|
||||
"coveredPlatforms": [
|
||||
"linux"
|
||||
],
|
||||
"coveredProviders": [
|
||||
"paired-runtime"
|
||||
],
|
||||
"surfaces": ["paired browser placement"],
|
||||
"platforms": ["linux", "macos", "windows"],
|
||||
"providers": ["paired-runtime"],
|
||||
"coveredPlatforms": ["linux"],
|
||||
"coveredProviders": ["paired-runtime"],
|
||||
"coverageNotes": "Published Linux 1.4.188 desktop against current source in both directions; scheduled weekly and manually runnable. No required PR check.",
|
||||
"motivatingLinks": [
|
||||
"https://github.com/stablyai/orca/actions/runs/34069063016"
|
||||
],
|
||||
"motivatingLinks": ["https://github.com/stablyai/orca/actions/runs/34069063016"],
|
||||
"invariant": "A paired client and host without client-hosted browser capabilities retain server-hosted browser placement across supported version skew.",
|
||||
"oracle": "Require both existing named browser placement scenarios to pass three times with one attempt, zero skips, zero failures, and no report errors.",
|
||||
"commands": [
|
||||
@@ -18593,9 +18577,7 @@
|
||||
},
|
||||
{
|
||||
"file": "config/scripts/verify-packaged-browser-participation.test.mjs",
|
||||
"assertions": [
|
||||
"reject missing, substituted, skipped and retried scenarios"
|
||||
]
|
||||
"assertions": ["reject missing, substituted, skipped and retried scenarios"]
|
||||
},
|
||||
{
|
||||
"file": "config/scripts/packaged-browser-lane-contract.test.mjs",
|
||||
@@ -18650,26 +18632,13 @@
|
||||
"protection": "partial",
|
||||
"owner": "terminal-input",
|
||||
"layer": "electron-native-ime-e2e",
|
||||
"surfaces": [
|
||||
"native Hangul composition",
|
||||
"Wayland terminal input"
|
||||
],
|
||||
"platforms": [
|
||||
"linux"
|
||||
],
|
||||
"providers": [
|
||||
"local"
|
||||
],
|
||||
"coveredPlatforms": [
|
||||
"linux"
|
||||
],
|
||||
"coveredProviders": [
|
||||
"local"
|
||||
],
|
||||
"surfaces": ["native Hangul composition", "Wayland terminal input"],
|
||||
"platforms": ["linux"],
|
||||
"providers": ["local"],
|
||||
"coveredPlatforms": ["linux"],
|
||||
"coveredProviders": ["local"],
|
||||
"coverageNotes": "Ubuntu 22.04 nested GNOME and IBus Hangul drive three complete native executions in GitHub Actions. GNOME owns IBus; daemon and CLI share its default config discovery path.",
|
||||
"motivatingLinks": [
|
||||
"https://github.com/stablyai/orca/pull/19174"
|
||||
],
|
||||
"motivatingLinks": ["https://github.com/stablyai/orca/pull/19174"],
|
||||
"invariant": "Typing d k 1 Return through native IBus Hangul delivers exactly 아1 followed by newline without missing, duplicate, or reordered characters.",
|
||||
"oracle": "Three executions each assert three exact UTF-8 PTY lines. Verify the exact Playwright title, zero skips/retries, each individual native composition receipt, and the nested launch Wayland flag.",
|
||||
"commands": [
|
||||
@@ -18685,15 +18654,11 @@
|
||||
"assertionRefs": [
|
||||
{
|
||||
"file": "tests/e2e/terminal-hangul-terminating-digit-native.spec.ts",
|
||||
"assertions": [
|
||||
"a digit typed right after a Hangul syllable reaches the pty"
|
||||
]
|
||||
"assertions": ["a digit typed right after a Hangul syllable reaches the pty"]
|
||||
},
|
||||
{
|
||||
"file": "config/scripts/terminal-ime-e2e-workflow.test.mjs",
|
||||
"assertions": [
|
||||
"runs native Wayland independently with CJK fonts and retained evidence"
|
||||
]
|
||||
"assertions": ["runs native Wayland independently with CJK fonts and retained evidence"]
|
||||
}
|
||||
],
|
||||
"evidenceRuns": [
|
||||
|
||||
@@ -38,6 +38,16 @@ const SUPPRESSED_REACT_DOCTOR_DIAGNOSTICS = new Map([
|
||||
new Set([
|
||||
'src/renderer/src/components/editor/combined-diff/review-controls/use-combined-diff-view-preferences.ts'
|
||||
])
|
||||
],
|
||||
[
|
||||
// The rule wants one named handle cleared by name. Both startup effects arm a variable number
|
||||
// of refresh timers, every one of them through addTimer into `timers`, which their cleanups
|
||||
// clear -- a shape the rule reports whether the handles live in an array, a Set, or a nested
|
||||
// helper. The finding predates this list; it surfaced when the effect body changed. This map
|
||||
// keys on file, not line, so the entry covers both effects in it; nothing else in the file
|
||||
// arms a timer, so widening it further is the only alternative, not a narrower option.
|
||||
'react-doctor(effect-needs-cleanup)',
|
||||
new Set(['mobile/src/session/use-mobile-session-startup.ts'])
|
||||
]
|
||||
])
|
||||
|
||||
|
||||
@@ -173,6 +173,28 @@ describe('rebuild-native-deps patched node-pty rebuild', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('refuses a Windows rebuild when the process creation-time patch is missing', () => {
|
||||
const projectDir = mkTempProject()
|
||||
|
||||
try {
|
||||
writeFakeUsableElectronPackage(projectDir, { platform: 'win32' })
|
||||
writeFakeElectronRebuild(projectDir)
|
||||
writeFakeNodePtyConptyPayload(projectDir, 'x64')
|
||||
writeFakeWindowsProcessTreeWithNodeAddonApi(projectDir, { creationTimePatchApplied: false })
|
||||
|
||||
const result = runRebuildScript(
|
||||
projectDir,
|
||||
{ npm_config_platform: 'win32', npm_config_arch: 'x64' },
|
||||
['--platform=win32', '--arch=x64', '--force']
|
||||
)
|
||||
|
||||
expect(result.status).not.toBe(0)
|
||||
expect(result.stderr).toContain('process creation-time patch')
|
||||
} finally {
|
||||
removeTreeSync(projectDir)
|
||||
}
|
||||
})
|
||||
|
||||
it('restores the ConPTY runtime payload after a Windows Electron rebuild', () => {
|
||||
const projectDir = mkTempProject()
|
||||
|
||||
|
||||
@@ -374,13 +374,18 @@ export function writeFakeWindowsProcessTree(projectDir) {
|
||||
|
||||
export function writeFakeWindowsProcessTreeWithNodeAddonApi(
|
||||
projectDir,
|
||||
{ commandLinePatchApplied = true } = {}
|
||||
{ commandLinePatchApplied = true, creationTimePatchApplied = true } = {}
|
||||
) {
|
||||
const processTreeDir = join(projectDir, 'node_modules', '@vscode', 'windows-process-tree')
|
||||
const nodeAddonApiDir = join(processTreeDir, 'node_modules', 'node-addon-api')
|
||||
mkdirSync(nodeAddonApiDir, { recursive: true })
|
||||
writeFileSync(join(processTreeDir, 'package.json'), '{"dependencies":{"node-addon-api":"*"}}\n')
|
||||
writeFileSync(join(processTreeDir, 'index.js'), 'module.exports = {}\n')
|
||||
writeFileSync(
|
||||
join(processTreeDir, 'index.js'),
|
||||
creationTimePatchApplied
|
||||
? 'exports.ProcessDataFlag = { None: 0, Memory: 1, CommandLine: 2, CreationTime: 4 }\n'
|
||||
: 'exports.ProcessDataFlag = { None: 0, Memory: 1, CommandLine: 2 }\n'
|
||||
)
|
||||
mkdirSync(join(processTreeDir, 'src'), { recursive: true })
|
||||
writeFileSync(
|
||||
join(processTreeDir, 'src', 'process_commandline.cc'),
|
||||
@@ -388,6 +393,36 @@ export function writeFakeWindowsProcessTreeWithNodeAddonApi(
|
||||
? '// kProcessCommandLineInformation = 60\n'
|
||||
: unpatchedWindowsProcessTreeCommandLineSource()
|
||||
)
|
||||
writeFileSync(
|
||||
join(processTreeDir, 'src', 'process.h'),
|
||||
creationTimePatchApplied
|
||||
? 'enum ProcessDataFlags { NONE = 0, MEMORY = 1, COMMANDLINE = 2, CREATIONTIME = 4 };\nULONGLONG creationTimeMs;\n'
|
||||
: 'enum ProcessDataFlags { NONE = 0, MEMORY = 1, COMMANDLINE = 2 };\n'
|
||||
)
|
||||
writeFileSync(
|
||||
join(processTreeDir, 'src', 'process.cc'),
|
||||
creationTimePatchApplied
|
||||
? 'GetProcessCreationTime(pinfo);\nGetProcessTimes(hProcess, &creationTime, &exitTime, &kernelTime, &userTime);\n'
|
||||
: 'GetProcessMemoryUsage(pinfo);\n'
|
||||
)
|
||||
writeFileSync(
|
||||
join(processTreeDir, 'src', 'process_worker.cc'),
|
||||
creationTimePatchApplied ? 'object.Set("creationTimeMs", process.creationTimeMs);\n' : '\n'
|
||||
)
|
||||
mkdirSync(join(processTreeDir, 'lib'), { recursive: true })
|
||||
writeFileSync(
|
||||
join(processTreeDir, 'lib', 'index.js'),
|
||||
creationTimePatchApplied ? 'exports.ProcessDataFlag["CreationTime"] = 4;\n' : '\n'
|
||||
)
|
||||
writeFileSync(
|
||||
join(processTreeDir, 'lib', 'index.ts'),
|
||||
creationTimePatchApplied ? 'export enum ProcessDataFlag { CreationTime = 4 }\n' : '\n'
|
||||
)
|
||||
mkdirSync(join(processTreeDir, 'typings'), { recursive: true })
|
||||
writeFileSync(
|
||||
join(processTreeDir, 'typings', 'windows-process-tree.d.ts'),
|
||||
creationTimePatchApplied ? 'creationTimeMs?: number\n' : '\n'
|
||||
)
|
||||
writeFileSync(join(nodeAddonApiDir, 'package.json'), '{"name":"node-addon-api"}\n')
|
||||
writeFileSync(join(nodeAddonApiDir, 'napi.h'), '// napi.h\n')
|
||||
writeFileSync(join(nodeAddonApiDir, 'napi-inl.h'), '// napi-inl.h\n')
|
||||
|
||||
@@ -33,6 +33,17 @@ export const WINDOWS_PROCESS_TREE_PATCH_PATH = join(
|
||||
/** Only the patched reader defines this; the upstream one walks the PEB. */
|
||||
const COMMAND_LINE_PATCH_MARKER = 'kProcessCommandLineInformation'
|
||||
|
||||
const CREATION_TIME_PATCH_MARKERS = [
|
||||
['src/process.h', 'CREATIONTIME = 4'],
|
||||
['src/process.h', 'ULONGLONG creationTimeMs'],
|
||||
['src/process.cc', 'GetProcessCreationTime(pinfo)'],
|
||||
['src/process.cc', 'GetProcessTimes(hProcess, &creationTime'],
|
||||
['src/process_worker.cc', 'object.Set("creationTimeMs"'],
|
||||
['lib/index.js', '["CreationTime"] = 4'],
|
||||
['lib/index.ts', 'CreationTime = 4'],
|
||||
['typings/windows-process-tree.d.ts', 'creationTimeMs?: number']
|
||||
]
|
||||
|
||||
export const WINDOWS_PROCESS_TREE_NODE_ADDON_API_HEADERS = [
|
||||
'napi.h',
|
||||
'napi-inl.h',
|
||||
@@ -83,6 +94,36 @@ export function inspectWindowsProcessTreeAddon(addonPath) {
|
||||
return readFileSync(addonPath).includes(FLAGGED_IMPORT) ? 'unpatched' : 'clean'
|
||||
}
|
||||
|
||||
export function assertWindowsProcessTreeCreationTimePatch(
|
||||
packageDir = WINDOWS_PROCESS_TREE_PACKAGE_DIR
|
||||
) {
|
||||
for (const [relativePath, expected] of CREATION_TIME_PATCH_MARKERS) {
|
||||
const filePath = join(packageDir, relativePath)
|
||||
if (!existsSync(filePath)) {
|
||||
throw new Error(
|
||||
`${filePath} is missing, so the process creation-time patch cannot be verified. ` +
|
||||
'Run pnpm install.'
|
||||
)
|
||||
}
|
||||
if (!readFileSync(filePath, 'utf8').includes(expected)) {
|
||||
throw new Error(
|
||||
`${relativePath} does not contain the process creation-time patch (${expected}). ` +
|
||||
'Run pnpm install.'
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function assertWindowsProcessTreeRuntimeCreationTime(windowsProcessTree) {
|
||||
if (windowsProcessTree?.ProcessDataFlag?.CreationTime !== 4) {
|
||||
throw new Error(
|
||||
'@vscode/windows-process-tree does not expose ProcessDataFlag.CreationTime, so native ' +
|
||||
'Windows structured agent-session process ownership cannot be PID-reuse safe. Rebuild it ' +
|
||||
'(pnpm run rebuild:electron) rather than using the published prebuild.'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuse to compile or load the upstream command-line reader.
|
||||
*
|
||||
@@ -159,6 +200,7 @@ export function ensureWindowsProcessTreeCommandLinePatch(
|
||||
rmSync(windowsProcessTreeAddonPath(packageDir), { force: true })
|
||||
repaired = true
|
||||
}
|
||||
assertWindowsProcessTreeCreationTimePatch(packageDir)
|
||||
|
||||
return repaired
|
||||
}
|
||||
|
||||
@@ -12,12 +12,15 @@ import { tmpdir } from 'node:os'
|
||||
import { join, resolve } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import {
|
||||
assertWindowsProcessTreeCreationTimePatch,
|
||||
assertWindowsProcessTreeRuntimeCreationTime,
|
||||
inspectWindowsProcessTreeAddon,
|
||||
nodeGypRebuildInvocation,
|
||||
stageWindowsProcessTreeNodeAddonApiHeaders,
|
||||
WINDOWS_PROCESS_TREE_NODE_ADDON_API_HEADERS,
|
||||
WINDOWS_PROCESS_TREE_PACKAGE_DIR
|
||||
} from './windows-process-tree-gyp-rebuild.mjs'
|
||||
import { writeFakeWindowsProcessTreeWithNodeAddonApi } from './rebuild-native-deps-test-fixtures.mjs'
|
||||
|
||||
describe('windows-process-tree node-gyp rebuild', () => {
|
||||
it("resolves node-addon-api's gyp target from the rebuild cwd", () => {
|
||||
@@ -97,3 +100,47 @@ describe('inspecting a compiled windows-process-tree addon', () => {
|
||||
expect(inspectWindowsProcessTreeAddon(staged)).toBe('unpatched')
|
||||
})
|
||||
})
|
||||
|
||||
describe('windows-process-tree CreationTime patch assertion', () => {
|
||||
let dir
|
||||
|
||||
beforeEach(() => {
|
||||
dir = mkdtempSync(join(tmpdir(), 'orca-windows-process-tree-creation-time-'))
|
||||
})
|
||||
afterEach(() => {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('accepts a package whose source and JS surfaces expose process creation time', () => {
|
||||
writeFakeWindowsProcessTreeWithNodeAddonApi(dir)
|
||||
|
||||
expect(() =>
|
||||
assertWindowsProcessTreeCreationTimePatch(
|
||||
join(dir, 'node_modules', '@vscode', 'windows-process-tree')
|
||||
)
|
||||
).not.toThrow()
|
||||
})
|
||||
|
||||
it('rejects a package missing the process creation-time patch', () => {
|
||||
writeFakeWindowsProcessTreeWithNodeAddonApi(dir, { creationTimePatchApplied: false })
|
||||
|
||||
expect(() =>
|
||||
assertWindowsProcessTreeCreationTimePatch(
|
||||
join(dir, 'node_modules', '@vscode', 'windows-process-tree')
|
||||
)
|
||||
).toThrow('process creation-time patch')
|
||||
})
|
||||
|
||||
it('requires the runtime ProcessDataFlag.CreationTime enum', () => {
|
||||
expect(() =>
|
||||
assertWindowsProcessTreeRuntimeCreationTime({
|
||||
ProcessDataFlag: { None: 0, Memory: 1, CommandLine: 2, CreationTime: 4 }
|
||||
})
|
||||
).not.toThrow()
|
||||
expect(() =>
|
||||
assertWindowsProcessTreeRuntimeCreationTime({
|
||||
ProcessDataFlag: { None: 0, Memory: 1, CommandLine: 2 }
|
||||
})
|
||||
).toThrow('ProcessDataFlag.CreationTime')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -32,6 +32,7 @@
|
||||
"../src/main/codex/codex-app-server-capability-cache.ts",
|
||||
"../src/main/codex/codex-app-server-capability-signal.ts",
|
||||
"../src/main/codex/codex-app-server-client.ts",
|
||||
"../src/main/codex/codex-app-server-process-tree-kill.ts",
|
||||
"../src/main/codex/codex-app-server-record-reader.ts",
|
||||
"../src/main/codex/codex-app-server-session.ts",
|
||||
"../src/main/codex/codex-config-mirror.ts",
|
||||
|
||||
@@ -2,8 +2,27 @@
|
||||
|
||||
Orca selects a Relay region in the Electron main process before requesting a new assignment. The
|
||||
director publishes an allowlisted region catalog containing only HTTPS cell subdomains of that
|
||||
director; Orca takes three bounded `/health` latency samples per region and caches the stable choice
|
||||
for 24 hours. A cached region changes only when the alternative is materially faster.
|
||||
director. Orca discards one warm-up `/health` request per probe origin — a cold request pays TCP and
|
||||
TLS setup that can exceed the round trip it measures — then takes three bounded samples and compares
|
||||
regions by their minimum. A wide spread still rejects a region, but only a genuinely flapping one.
|
||||
The stable choice is cached for 24 hours, and a cached region changes only when the alternative is
|
||||
materially faster.
|
||||
|
||||
A region wins only against a measured competitor. If any region in the catalog is rejected or cannot
|
||||
be measured, Orca sends no hint rather than selecting the sole survivor. Sending no hint is not
|
||||
neutral placement: the director assigns `preferredRegion ?? RELAY_DEFAULT_REGION`, and the default
|
||||
is `us-central1`. So an `asia-east2` user whose `us-central1` probe fails or flaps once is placed in
|
||||
`us-central1` for that refresh. That trade is accepted because the relay database is
|
||||
`us-central1`-only, and it is bounded: the withheld hint is cached for one hour, not the 24 hours a
|
||||
chosen region gets, so the next hour re-measures. An origin that fails its warm-up probe is dropped
|
||||
before the sampling rounds, so an unreachable region costs one probe timeout rather than four.
|
||||
|
||||
After a control socket registers, Orca probes the cell it actually landed on, once per cell URL per
|
||||
process. The cache is deleted only when it names a region other than the best measured one and the
|
||||
assigned cell is more than three times slower than that region — a far cell under a cache that still
|
||||
names the best region means the director declined the hint, and re-measuring would return the same
|
||||
answer. Self-heal skips an absent, expired, or no-hint cache, and never runs under
|
||||
`ORCA_RELAY_REGION_OVERRIDE`.
|
||||
|
||||
The assignment request sends only `preferredRegion`. It does not send latency, IP address, country,
|
||||
pairing data, or credentials. Catalog, probe, and cache failures fall back to an assignment without
|
||||
|
||||
@@ -31,12 +31,12 @@ administrators can do about it.
|
||||
|
||||
Four independent evidence clusters, from six incidents:
|
||||
|
||||
| Cluster | Incidents | Evidence |
|
||||
| ----------------- | --------- | -------------------------------------------------------------------------------------------------------------------- |
|
||||
| **Update** | A, B, C | `orca-windows-setup.exe` → `old-uninstaller.exe`, `Uninstall Orca.exe` (electron-builder generates these; they are in no repo file) |
|
||||
| Cluster | Incidents | Evidence |
|
||||
| ----------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| **Update** | A, B, C | `orca-windows-setup.exe` → `old-uninstaller.exe`, `Uninstall Orca.exe` (electron-builder generates these; they are in no repo file) |
|
||||
| **Spawn** | all six | `Orca.exe` → `orca-terminal-daemon.exe` → `powershell.exe` / `pwsh.exe` / `cmd.exe` / `reg.exe` → `claude.exe`, `gh.exe`, `codex.cmd` |
|
||||
| **Process table** | D | "suspicious memory activity" — `OpenProcess` plus a PEB read against every process on a repeating cadence |
|
||||
| **Computer use** | E, F | `runtime.ps1`, `computer-sidecar.js`, many `operation.json`, a burst of ~10 short-lived `powershell.exe` |
|
||||
| **Process table** | D | "suspicious memory activity" — `OpenProcess` plus a PEB read against every process on a repeating cadence |
|
||||
| **Computer use** | E, F | `runtime.ps1`, `computer-sidecar.js`, many `operation.json`, a burst of ~10 short-lived `powershell.exe` |
|
||||
|
||||
Incident E is the one to look at hardest: 5 alerts, 37 evidence items, ATT&CK
|
||||
**Execution + Collection**, and a description reading _"Screenshots were taken
|
||||
@@ -143,11 +143,11 @@ PEB fallback to reinstate it — a hooked `ntdll` answering
|
||||
`STATUS_INVALID_INFO_CLASS` for one target would have flipped a process-wide,
|
||||
one-way switch back to `PROCESS_VM_READ` on exactly the machines this exists for.
|
||||
|
||||
Because the property is the *absence* of an import, it is checkable on the
|
||||
Because the property is the _absence_ of an import, it is checkable on the
|
||||
artifact rather than the source: `inspectWindowsProcessTreeAddon()` answers
|
||||
`clean` / `unpatched` / `missing`, and the rebuild, `ensure-native-runtime.mjs`,
|
||||
the relay build and `loadWindowsProcessTree()` all key on it. That check is load-
|
||||
bearing because the published tarball ships a *loadable* prebuilt built from
|
||||
bearing because the published tarball ships a _loadable_ prebuilt built from
|
||||
unpatched source, so "it required cleanly" is not evidence.
|
||||
|
||||
What to declare to administrators is now one
|
||||
@@ -180,7 +180,7 @@ Three sites are named in the incident analysis:
|
||||
`src/shared/setup-agent-sequencing.ts`,
|
||||
`src/shared/windows-cmd-runner-delayed-launch.ts` and
|
||||
`src/shared/windows-interactive-login-spawn.ts` each dropped
|
||||
`-ExecutionPolicy Bypass` as a measured no-op: the policy gates script *files*,
|
||||
`-ExecutionPolicy Bypass` as a measured no-op: the policy gates script _files_,
|
||||
never `-EncodedCommand`. Where the bypass was load-bearing it moved in-payload as
|
||||
a process-scope `Set-ExecutionPolicy` (`setup-agent-sequencing.ts`), which is the
|
||||
pattern to copy rather than restoring the switch — the switch loses to a GPO
|
||||
@@ -192,7 +192,7 @@ What remains is `-EncodedCommand` without the bypass: the PTY bootstraps
|
||||
(`src/main/agent-hooks/windows-powershell-hook-launcher.ts` and its callers
|
||||
`src/main/agent-hooks/runtime-home-hook-command.ts`,
|
||||
`src/main/agent-hooks/installer-utils.ts`, and `src/main/claude/hook-settings.ts`
|
||||
— that last one only as a *fallback* since #18875, see below),
|
||||
— that last one only as a _fallback_ since #18875, see below),
|
||||
`src/main/runtime/windows-default-route-interfaces.ts`,
|
||||
`src/main/runtime/orchestration/setup-completion-signal.ts`,
|
||||
`src/shared/hermes-startup-query.ts`, and the four ex-bypass sites above.
|
||||
@@ -224,7 +224,7 @@ denies the analyser the payload it would otherwise clear.
|
||||
The hook launcher is prior art worth knowing about. #16003 measured, on a
|
||||
reporting Kaspersky host, that `-WindowStyle Hidden` paired with
|
||||
`-EncodedCommand` was denied at `CreateProcess` with exit 126 regardless of
|
||||
payload — `exit 0` was denied too. The fix was to stop *spelling* the flags:
|
||||
payload — `exit 0` was denied too. The fix was to stop _spelling_ the flags:
|
||||
`WINDOWS_POWERSHELL_HOOK_SWITCHES` is now just `-NoProfile`, and separately, in
|
||||
#16576, the execution policy bypass moved in-payload as a process-scope
|
||||
`Set-ExecutionPolicy` — a real command-line signal reduction, though #16003's
|
||||
@@ -247,7 +247,7 @@ a quoted token, each `%` is broken with `"^%"`.
|
||||
|
||||
The escaping is not decorative. Measured on Windows 11 against a real `.cmd`
|
||||
shim, `["a b", 'c"d', "e%F%g", "h&i", "j^k"]` came back as `["a b", 'c"d',
|
||||
"e^%F^%g", "h"]` — the `&` truncated the argument *and* ran the remainder as a
|
||||
"e^%F^%g", "h"]` — the `&` truncated the argument _and_ ran the remainder as a
|
||||
command.
|
||||
|
||||
**How an EDR reads it:** caret escaping is the canonical obfuscation marker in
|
||||
@@ -259,7 +259,7 @@ obfuscated-command-line detector is tuned on.
|
||||
|
||||
`Orca.exe` → the relocated daemon host (`orca-terminal-daemon.exe` in the builds
|
||||
these incidents cover, `Orca.exe` since) → a shell → an agent CLI is what a
|
||||
terminal multiplexer for coding agents *is*. `reg.exe` appears from
|
||||
terminal multiplexer for coding agents _is_. `reg.exe` appears from
|
||||
`src/main/win32-utils.ts`,
|
||||
`src/main/agent-hooks/managed-hook-owner-identity.ts` and
|
||||
`src/relay/pty-shell-utils.ts` (reading the OpenSSH `DefaultShell`).
|
||||
@@ -267,7 +267,7 @@ terminal multiplexer for coding agents *is*. `reg.exe` appears from
|
||||
Nothing here is avoidable in principle. What is controllable is depth and
|
||||
breadth: every interpreter hop between Orca and the thing the user asked for adds
|
||||
a scored edge, which is why the shipped doctrine of #15520 and #15595 is to
|
||||
*shorten the interpreter chain* rather than to hide a window.
|
||||
_shorten the interpreter chain_ rather than to hide a window.
|
||||
|
||||
#18875 is a worked example of that doctrine. The Claude Code lifecycle hook was
|
||||
registered as `powershell.exe -NoProfile -EncodedCommand <...>` whose entire
|
||||
@@ -295,7 +295,7 @@ That last clause is the standing assumption of this change, and it is worth
|
||||
stating plainly because it is **not** measured. `||` parses in Git Bash, cmd.exe
|
||||
and pwsh, but not in Windows PowerShell 5.1, so the direct shape is correct for
|
||||
any host that is one of the first three. Claude Code itself is a Git Bash host on
|
||||
native Windows. What no one here has verified is which host a *compat consumer*
|
||||
native Windows. What no one here has verified is which host a _compat consumer_
|
||||
uses: cursor-agent and Devin import `~/.claude/settings.json` and run `command`
|
||||
through their own launcher (the managed `.cmd` carries a `DEVIN_PROJECT_DIR` skip
|
||||
for exactly that). If one of them spawns hook strings through Windows PowerShell
|
||||
@@ -318,12 +318,12 @@ then captures the screen through `Graphics.CopyFromScreen`.
|
||||
|
||||
That is four separate high-signal behaviours stacked in one process:
|
||||
|
||||
| Behaviour | How it is scored |
|
||||
| ----------------------------------------------- | ---------------------------------------------------- |
|
||||
| `Graphics.CopyFromScreen` | **MITRE T1113**, screen capture — Collection tactic |
|
||||
| `SendInput` synthetic keyboard/mouse | input synthesis against other applications |
|
||||
| `Add-Type -TypeDefinition` on every operation | MSIL compiled at runtime; incident F's "suspicious MSIL code" |
|
||||
| One `powershell.exe` per operation | a burst of short-lived interpreters under one parent |
|
||||
| Behaviour | How it is scored |
|
||||
| --------------------------------------------- | ------------------------------------------------------------- |
|
||||
| `Graphics.CopyFromScreen` | **MITRE T1113**, screen capture — Collection tactic |
|
||||
| `SendInput` synthetic keyboard/mouse | input synthesis against other applications |
|
||||
| `Add-Type -TypeDefinition` on every operation | MSIL compiled at runtime; incident F's "suspicious MSIL code" |
|
||||
| One `powershell.exe` per operation | a burst of short-lived interpreters under one parent |
|
||||
|
||||
The bottom two rows are the two the incident text named directly, and they are
|
||||
also the two a persistent runtime host would remove: a long-lived helper compiles
|
||||
@@ -381,16 +381,16 @@ changed. Check the code before relying on it.
|
||||
|
||||
The checklist. On Windows, do not reach for:
|
||||
|
||||
| Don't | Instead |
|
||||
| ----------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `-ExecutionPolicy Bypass` on the command line | Set the policy in-payload at process scope, as `windows-powershell-hook-launcher.ts` does, or do not run a `.ps1` at all |
|
||||
| `-EncodedCommand` | A temp `.ps1` with an argument, or no PowerShell hop: prefer a native API or an existing Node path |
|
||||
| `cmd.exe /c` carrying escaped free text | Spawn the real target directly. `cmd.exe` is only unavoidable for `.cmd`/`.bat`; keep free text out of the line where you can |
|
||||
| Forking `powershell.exe` to read system state | The native reader — [`windows-process-enumeration.md`](./windows-process-enumeration.md) is the standing rule for the process table |
|
||||
| A process per operation in a loop | One long-lived helper with a request channel. A burst of short-lived interpreters under one parent is itself the signal |
|
||||
| `Add-Type -TypeDefinition` at runtime | A precompiled, signed assembly, or a native helper |
|
||||
| Copying our own image under a different name | Copy it verbatim — [`windows-daemon-host-relocation.md`](./windows-daemon-host-relocation.md) (done for the daemon host) |
|
||||
| Deriving a script runner from a UI preference | [`windows-setup-shell.md`](./windows-setup-shell.md) — the script declares its own interpreter |
|
||||
| Don't | Instead |
|
||||
| --------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `-ExecutionPolicy Bypass` on the command line | Set the policy in-payload at process scope, as `windows-powershell-hook-launcher.ts` does, or do not run a `.ps1` at all |
|
||||
| `-EncodedCommand` | A temp `.ps1` with an argument, or no PowerShell hop: prefer a native API or an existing Node path |
|
||||
| `cmd.exe /c` carrying escaped free text | Spawn the real target directly. `cmd.exe` is only unavoidable for `.cmd`/`.bat`; keep free text out of the line where you can |
|
||||
| Forking `powershell.exe` to read system state | The native reader — [`windows-process-enumeration.md`](./windows-process-enumeration.md) is the standing rule for the process table |
|
||||
| A process per operation in a loop | One long-lived helper with a request channel. A burst of short-lived interpreters under one parent is itself the signal |
|
||||
| `Add-Type -TypeDefinition` at runtime | A precompiled, signed assembly, or a native helper |
|
||||
| Copying our own image under a different name | Copy it verbatim — [`windows-daemon-host-relocation.md`](./windows-daemon-host-relocation.md) (done for the daemon host) |
|
||||
| Deriving a script runner from a UI preference | [`windows-setup-shell.md`](./windows-setup-shell.md) — the script declares its own interpreter |
|
||||
|
||||
Two framing rules that outlast the table:
|
||||
|
||||
@@ -407,7 +407,7 @@ Two framing rules that outlast the table:
|
||||
|
||||
This is the single most important operational point, and it is the one most
|
||||
commonly got wrong. The six incidents are **MDE EDR behavioural alerts**.
|
||||
Defender Antivirus path exclusions suppress *scan* detections; they do not
|
||||
Defender Antivirus path exclusions suppress _scan_ detections; they do not
|
||||
suppress EDR behavioural alerts the same way. Adding
|
||||
`%LOCALAPPDATA%\Programs\orca\` to the AV exclusion list and expecting the
|
||||
incidents to stop will not work.
|
||||
|
||||
@@ -64,10 +64,10 @@ identity scan opens nothing.
|
||||
So the module exposes two snapshots, and the row types differ so a cheap caller
|
||||
cannot read what its flag set did not pay for:
|
||||
|
||||
| reader | row type | flags | per-process handles |
|
||||
| ------------------------------------------ | ---------------------------- | --------------------------- | ------------------- |
|
||||
| `readWindowsProcessIdentityTable[Fresh]()` | `WindowsProcessIdentityRow` | `None \| CreationTime` | none |
|
||||
| `readWindowsProcessTable[Fresh]()` | `WindowsProcessRow` | `+ CommandLine` | one `OpenProcess` |
|
||||
| reader | row type | flags | per-process handles |
|
||||
| ------------------------------------------ | --------------------------- | ---------------------- | ------------------- |
|
||||
| `readWindowsProcessIdentityTable[Fresh]()` | `WindowsProcessIdentityRow` | `None \| CreationTime` | none |
|
||||
| `readWindowsProcessTable[Fresh]()` | `WindowsProcessRow` | `+ CommandLine` | one `OpenProcess` |
|
||||
|
||||
`Memory` is requested by neither. Nothing reads a working set off this table —
|
||||
`windows-process-resource-collector.ts` runs its own sweep because it needs
|
||||
@@ -103,7 +103,7 @@ only under concurrency.
|
||||
|
||||
Nothing else in this module prevents that. Each snapshot cache single-flights
|
||||
only within itself (`inFlight` is a closure per reader), and the wedge set
|
||||
latches only *after* a read misses its 3 s deadline, so through the healthy
|
||||
latches only _after_ a read misses its 3 s deadline, so through the healthy
|
||||
~12 ms of a scan neither excludes the other. Overlap is the normal state rather
|
||||
than an edge case: other panes keep polling detailed at 750 ms while a teardown
|
||||
takes identity snapshots, and `codex-structured-turn-processes.ts` issues fresh
|
||||
@@ -166,15 +166,15 @@ through `toIdentityRow`, so an identity row carries no command line on any host.
|
||||
|
||||
### Which callers need which
|
||||
|
||||
| caller | reads | flag set |
|
||||
| --------------------------------------------- | ------------------ | -------- |
|
||||
| `windows-agent-foreground-process.ts` | `command` (agent recognition) | detailed |
|
||||
| `local-workspace-platform-port-scanner.ts` | `command` (port attribution) | detailed |
|
||||
| `codex-structured-turn-processes.ts` | `command` (turn-process identity) | detailed |
|
||||
| `structured-tui-process-identity.ts` | `command` (child match) | detailed |
|
||||
| `windows-pty-root-identity.ts` | `pid` / `ppid` only | identity |
|
||||
| `agent-session-process-identity-probe.ts` | `creationTimeMs` only | identity |
|
||||
| `relay/windows-port-scan.ts` | `name` (port owner label) | detailed |
|
||||
| caller | reads | flag set |
|
||||
| ------------------------------------------ | --------------------------------- | -------- |
|
||||
| `windows-agent-foreground-process.ts` | `command` (agent recognition) | detailed |
|
||||
| `local-workspace-platform-port-scanner.ts` | `command` (port attribution) | detailed |
|
||||
| `codex-structured-turn-processes.ts` | `command` (turn-process identity) | detailed |
|
||||
| `structured-tui-process-identity.ts` | `command` (child match) | detailed |
|
||||
| `windows-pty-root-identity.ts` | `pid` / `ppid` only | identity |
|
||||
| `agent-session-process-identity-probe.ts` | `creationTimeMs` only | identity |
|
||||
| `relay/windows-port-scan.ts` | `name` (port owner label) | detailed |
|
||||
|
||||
`windows-port-scan.ts` is the one mismatch in the table: it reads only `pid` and
|
||||
`name`, which the identity set answers, but it calls the detailed reader. On a
|
||||
@@ -344,7 +344,7 @@ on any other OS keeps using the scan.
|
||||
|
||||
## Why the package is patched
|
||||
|
||||
`config/patches/@vscode__windows-process-tree@0.8.0.patch` carries five changes.
|
||||
`config/patches/@vscode__windows-process-tree@0.8.0.patch` carries six changes.
|
||||
|
||||
1. **Spectre mitigation.** The upstream `binding.gyp` requires Spectre-mitigated
|
||||
libraries, which Orca's Windows build agents do not install. `node-pty` is
|
||||
@@ -368,10 +368,10 @@ on any other OS keeps using the scan.
|
||||
to Unix ms; a process that denies the handle is emitted with the field
|
||||
absent, never zero, because callers must be able to tell "cannot identify"
|
||||
from a timestamp.
|
||||
5. **`supportedProcessDataFlags`.** `addon.cc` exports the flag bits the
|
||||
6. **`supportedProcessDataFlags`.** `addon.cc` exports the flag bits the
|
||||
compiled binary understands, and `lib/index.js` re-exports it.
|
||||
|
||||
Why a fifth hunk and not just the enum: unlike `node-pty`, this package
|
||||
Why a separate hunk and not just the enum: unlike `node-pty`, this package
|
||||
publishes a prebuilt `.node` at the same `build/Release/` path node-gyp
|
||||
writes to. pnpm patches the source tree and leaves that prebuilt alone, so a
|
||||
host can hold a patched `lib/index.js` — `ProcessDataFlag.CreationTime` and
|
||||
|
||||
@@ -30,8 +30,7 @@ import { Callout } from '@/components/docs/prose'
|
||||
[installer](https://github.com/stablyai/orca/releases/latest/download/orca-windows-setup.exe)
|
||||
</li>
|
||||
<li>
|
||||
**Linux:**
|
||||
AppImage
|
||||
**Linux:** AppImage
|
||||
[x64](https://github.com/stablyai/orca/releases/latest/download/orca-linux.AppImage) ·
|
||||
[arm64](https://github.com/stablyai/orca/releases/latest/download/orca-linux-arm64.AppImage) ·
|
||||
[.deb](https://github.com/stablyai/orca/releases) ·
|
||||
@@ -131,7 +130,7 @@ On Linux the [Orca CLI](/docs/cli/reference) installs as **`orca-ide`**, not `or
|
||||
- The `.deb` and `.rpm` put `orca-ide` on your `PATH` at install time, as `/usr/bin/orca-ide`.
|
||||
- With the AppImage, register the CLI from [Settings → General → Orca CLI](/docs/settings). That installs `~/.local/bin/orca-ide`.
|
||||
- Inside Orca's own terminals, bare `orca` works. Orca puts a shim on the `PATH` of the terminals it manages, so agents and scripts running there use the same command as on macOS and Windows.
|
||||
- On a headless host, a packaged `orca serve` writes a bare `orca` into `~/.local/bin` as it starts, unless a file it does not own already holds that name. It writes that *during* startup, so it is never what starts the server — the first launch is always [`orca-ide serve`](/docs/remote-servers).
|
||||
- On a headless host, a packaged `orca serve` writes a bare `orca` into `~/.local/bin` as it starts, unless a file it does not own already holds that name. It writes that _during_ startup, so it is never what starts the server — the first launch is always [`orca-ide serve`](/docs/remote-servers).
|
||||
|
||||
Do not verify with `command -v orca`: on a GNOME desktop that succeeds and resolves to the screen reader. Use `orca-ide` in your own shell and `orca` inside Orca. If you want the short name everywhere and you do not use the screen reader, link it yourself:
|
||||
|
||||
|
||||
@@ -129,19 +129,23 @@ Install Orca and its bundled CLI on the server, then run:
|
||||
<Callout title="On Linux, start it with orca-ide serve">
|
||||
The Linux CLI is named `orca-ide`, because GNOME Orca's screen reader already owns
|
||||
`/usr/bin/orca`. A packaged `orca serve` does write a bare `orca` into `~/.local/bin`, but only
|
||||
while it is starting, so that shim can never be the command that starts the server. Read
|
||||
`orca serve` as `orca-ide serve` throughout this page when the host is Linux. See
|
||||
[Install → Linux](/docs/install#linux).
|
||||
while it is starting, so that shim can never be the command that starts the server. Read `orca
|
||||
serve` as `orca-ide serve` throughout this page when the host is Linux. See [Install →
|
||||
Linux](/docs/install#linux).
|
||||
</Callout>
|
||||
|
||||
```bash
|
||||
orca serve --pairing-address <server-tailscale-ip-or-hostname>
|
||||
# Linux
|
||||
orca-ide serve --pairing-address <server-tailscale-ip-or-hostname>
|
||||
```
|
||||
|
||||
For example:
|
||||
|
||||
```bash
|
||||
orca serve --pairing-address 100.64.1.20
|
||||
# Linux
|
||||
orca-ide serve --pairing-address 100.64.1.20
|
||||
```
|
||||
|
||||
The command:
|
||||
@@ -157,6 +161,8 @@ Add `--port 6768` when a firewall, tunnel, or service definition requires a fixe
|
||||
|
||||
```bash
|
||||
orca serve --port 6768 --pairing-address 100.64.1.20
|
||||
# Linux
|
||||
orca-ide serve --port 6768 --pairing-address 100.64.1.20
|
||||
```
|
||||
|
||||
Use only one host mode at a time. If the Orca desktop app is already sharing that computer, do not start a second `orca serve` process for the same setup.
|
||||
@@ -167,6 +173,8 @@ For the Orca mobile app, request a mobile-scoped QR code and link:
|
||||
|
||||
```bash
|
||||
orca serve --pairing-address 100.64.1.20 --mobile-pairing
|
||||
# Linux
|
||||
orca-ide serve --pairing-address 100.64.1.20 --mobile-pairing
|
||||
```
|
||||
|
||||
Keep the phone on the same tailnet, open Orca Mobile, choose **Pair**, and scan the terminal QR code or paste the printed link.
|
||||
|
||||
@@ -213,9 +213,9 @@ The commands, snapshot and ref rules, page affinity, and `browser_*` recoveries
|
||||
|
||||
This guide covers worktrees, terminals, and handoffs on its own. At a gate below, run `ORCA skills get orca-cli --reference references/<file>.md` and read only that document; `--references` lists the names. If the CLI rejects `--reference`, run `ORCA skills get orca-cli --full` once instead: it returns this guide plus every reference from the same CLI build, so read only the named one. If `--full` is rejected too, the CLI predates bundled references: use `ORCA <command> --help`, keep the rules above, and do not guess flags.
|
||||
|
||||
| Action gate | Reference |
|
||||
|---|---|
|
||||
| Driving Orca's embedded browser: navigation, snapshots, refs, tabs, concurrent pages, or `browser_*` recoveries | `references/browser.md` |
|
||||
| Creating, editing, running, or inspecting scheduled automations | `references/automations.md` |
|
||||
| Publishing or revoking an artifact link, or publishing installed skills | `references/publishing.md` |
|
||||
| Mobile emulator taps, gestures, typing, buttons, camera, or permissions | invoke the `orca-emulator` skill |
|
||||
| Action gate | Reference |
|
||||
| --------------------------------------------------------------------------------------------------------------- | -------------------------------- |
|
||||
| Driving Orca's embedded browser: navigation, snapshots, refs, tabs, concurrent pages, or `browser_*` recoveries | `references/browser.md` |
|
||||
| Creating, editing, running, or inspecting scheduled automations | `references/automations.md` |
|
||||
| Publishing or revoking an artifact link, or publishing installed skills | `references/publishing.md` |
|
||||
| Mobile emulator taps, gestures, typing, buttons, camera, or permissions | invoke the `orca-emulator` skill |
|
||||
|
||||
@@ -52,23 +52,23 @@ Orca returns a clear message when the SDK is missing
|
||||
Use `--json` for agent-driven calls. Unqualified commands target the worktree's active
|
||||
device.
|
||||
|
||||
| Goal | Command | Constraint |
|
||||
| ------------------- | --------------------------------------------------------------------- | ------------------------------------------------------------------------------- |
|
||||
| List devices + AVDs | `ORCA emulator devices --json` | Every backend's devices with a platform column, booted and shutdown. |
|
||||
| Attach / make active | `ORCA emulator attach <avd-name-or-serial> --json` | Given an AVD name, boots it first. Makes the device active for the worktree. |
|
||||
| Single tap | `ORCA emulator tap <x> <y> --json` | Normalized 0..1 coordinates. |
|
||||
| Swipe / gesture | `ORCA emulator gesture '<json>' --json` | adb approximates the path by its endpoints, first point to last. |
|
||||
| Type text | `ORCA emulator type "user@example.com" --json` | US-ASCII, spaces handled, no newlines. |
|
||||
| Hardware button | `ORCA emulator button back --json` | `home`, `back`, `recents`, `power`, `volume_up`, `volume_down`. |
|
||||
| Rotate | `ORCA emulator rotate landscape_left --json` | Sets `user_rotation` and disables auto-rotate. |
|
||||
| Install an APK | `ORCA emulator install ./app-debug.apk --reinstall --json` | `--reinstall` passes `-r`. |
|
||||
| Launch an app | `ORCA emulator launch com.acme.app --activity .MainActivity --json` | Omit `--activity` to launch the default LAUNCHER activity. |
|
||||
| Runtime permission | `ORCA emulator permissions grant com.acme.app android.permission.CAMERA --json` | Positional order is `<grant\|revoke> <package> <permission>`; `reset` takes no positionals and clears all runtime grants. |
|
||||
| Accessibility tree | `ORCA emulator ax --json` | `uiautomator dump` parsed to a node tree. |
|
||||
| Logcat (one-shot) | `ORCA emulator logcat --lines 200 --json` | Dumps recent lines, parsed to entries. |
|
||||
| Raw adb shell | `ORCA emulator exec --command "getprop ro.build.version.sdk" --json` | Runs `adb -s <serial> shell <command>`. |
|
||||
| Stop the helper | `ORCA emulator kill --json` | Leaves the device booted. |
|
||||
| Stop and power off | `ORCA emulator shutdown --json` | Stops the helper and shuts the device down. |
|
||||
| Goal | Command | Constraint |
|
||||
| -------------------- | ------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- |
|
||||
| List devices + AVDs | `ORCA emulator devices --json` | Every backend's devices with a platform column, booted and shutdown. |
|
||||
| Attach / make active | `ORCA emulator attach <avd-name-or-serial> --json` | Given an AVD name, boots it first. Makes the device active for the worktree. |
|
||||
| Single tap | `ORCA emulator tap <x> <y> --json` | Normalized 0..1 coordinates. |
|
||||
| Swipe / gesture | `ORCA emulator gesture '<json>' --json` | adb approximates the path by its endpoints, first point to last. |
|
||||
| Type text | `ORCA emulator type "user@example.com" --json` | US-ASCII, spaces handled, no newlines. |
|
||||
| Hardware button | `ORCA emulator button back --json` | `home`, `back`, `recents`, `power`, `volume_up`, `volume_down`. |
|
||||
| Rotate | `ORCA emulator rotate landscape_left --json` | Sets `user_rotation` and disables auto-rotate. |
|
||||
| Install an APK | `ORCA emulator install ./app-debug.apk --reinstall --json` | `--reinstall` passes `-r`. |
|
||||
| Launch an app | `ORCA emulator launch com.acme.app --activity .MainActivity --json` | Omit `--activity` to launch the default LAUNCHER activity. |
|
||||
| Runtime permission | `ORCA emulator permissions grant com.acme.app android.permission.CAMERA --json` | Positional order is `<grant\|revoke> <package> <permission>`; `reset` takes no positionals and clears all runtime grants. |
|
||||
| Accessibility tree | `ORCA emulator ax --json` | `uiautomator dump` parsed to a node tree. |
|
||||
| Logcat (one-shot) | `ORCA emulator logcat --lines 200 --json` | Dumps recent lines, parsed to entries. |
|
||||
| Raw adb shell | `ORCA emulator exec --command "getprop ro.build.version.sdk" --json` | Runs `adb -s <serial> shell <command>`. |
|
||||
| Stop the helper | `ORCA emulator kill --json` | Leaves the device booted. |
|
||||
| Stop and power off | `ORCA emulator shutdown --json` | Stops the helper and shuts the device down. |
|
||||
|
||||
## Targeting
|
||||
|
||||
|
||||
@@ -43,20 +43,20 @@ Orca reports a clear error when the host is missing macOS or the Xcode tools.
|
||||
Use `--json` for agent-driven calls. Unqualified commands target the worktree's active
|
||||
device.
|
||||
|
||||
| Goal | Command | Constraint |
|
||||
| ------------------------ | ----------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ |
|
||||
| List available / running | `ORCA emulator list --json` | Orca-managed sessions plus raw serve-sim streams. Use its ids for `--device` / `--emulator`. |
|
||||
| List devices everywhere | `ORCA emulator devices --json` | Every backend's devices with a platform column, booted and shutdown. |
|
||||
| Attach / make active | `ORCA emulator attach "iPhone 16 Pro" --json` | Starts the helper if needed and makes the device active for the worktree. `--focus` switches the UI; it does not by default. |
|
||||
| Single tap | `ORCA emulator tap <x> <y> --json` | Normalized 0..1 coordinates. |
|
||||
| Multi-step gesture | `ORCA emulator gesture '<json>' --json` | Begin/move/end points. Use `tap` for a single tap. |
|
||||
| Type text | `ORCA emulator type "text" --json` | US-ASCII only. |
|
||||
| Goal | Command | Constraint |
|
||||
| ------------------------ | ----------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| List available / running | `ORCA emulator list --json` | Orca-managed sessions plus raw serve-sim streams. Use its ids for `--device` / `--emulator`. |
|
||||
| List devices everywhere | `ORCA emulator devices --json` | Every backend's devices with a platform column, booted and shutdown. |
|
||||
| Attach / make active | `ORCA emulator attach "iPhone 16 Pro" --json` | Starts the helper if needed and makes the device active for the worktree. `--focus` switches the UI; it does not by default. |
|
||||
| Single tap | `ORCA emulator tap <x> <y> --json` | Normalized 0..1 coordinates. |
|
||||
| Multi-step gesture | `ORCA emulator gesture '<json>' --json` | Begin/move/end points. Use `tap` for a single tap. |
|
||||
| Type text | `ORCA emulator type "text" --json` | US-ASCII only. |
|
||||
| Hardware button | `ORCA emulator button home --json` | `home` and `side_button` are documented by the CLI spec; other names such as `swipe_home`, `app_switcher`, `lock`, and `siri` are forwarded to serve-sim unvalidated. |
|
||||
| Rotate device | `ORCA emulator rotate landscape_left --json` | The orientation persists for subsequent gestures. |
|
||||
| Accessibility tree | `ORCA emulator ax --json` | serve-sim node tree, capped at 500 nodes, frames normalized 0..1 with a top-left origin. Needs an active session. |
|
||||
| Raw passthrough | `ORCA emulator exec --command "ca-debug blended on" --json` | serve-sim subcommand string, without a `serve-sim` prefix. |
|
||||
| Stop the helper | `ORCA emulator kill --json` | Leaves the device booted. |
|
||||
| Stop and power off | `ORCA emulator shutdown --json` | Stops the helper and shuts the simulator device down. |
|
||||
| Rotate device | `ORCA emulator rotate landscape_left --json` | The orientation persists for subsequent gestures. |
|
||||
| Accessibility tree | `ORCA emulator ax --json` | serve-sim node tree, capped at 500 nodes, frames normalized 0..1 with a top-left origin. Needs an active session. |
|
||||
| Raw passthrough | `ORCA emulator exec --command "ca-debug blended on" --json` | serve-sim subcommand string, without a `serve-sim` prefix. |
|
||||
| Stop the helper | `ORCA emulator kill --json` | Leaves the device booted. |
|
||||
| Stop and power off | `ORCA emulator shutdown --json` | Stops the helper and shuts the simulator device down. |
|
||||
|
||||
## Targeting
|
||||
|
||||
@@ -65,8 +65,8 @@ commands target it. Pass a selector only to override that or reach a second devi
|
||||
active session an unqualified command fails with `emulator_no_active`; attach or open the pane
|
||||
and retry.
|
||||
|
||||
- `--device "iPhone 16 Pro"` or `--device <udid>`, from `list` or `devices`. `--emulator
|
||||
<id>` is an alternative spelling: the bridge resolves both through the same lookup. These
|
||||
- `--device "iPhone 16 Pro"` or `--device <udid>`, from `list` or `devices`.
|
||||
`--emulator <id>` is an alternative spelling: the bridge resolves both through the same lookup. These
|
||||
selectors apply to the action verbs; `list` and `devices` take only `--worktree`, and
|
||||
`attach` names its device as a positional argument.
|
||||
- `--worktree id:<fullWorktreeId>` or `--worktree active`. The full id is the exact
|
||||
|
||||
@@ -367,10 +367,10 @@ rejects `--reference`, run `ORCA skills get orca-per-workspace-env --full` once
|
||||
this guide plus every reference from the same CLI build, so read only the named one. If `--full` is
|
||||
rejected too, keep these rules, use the command's `--help`, and do not guess flags.
|
||||
|
||||
| Action gate | Bundled reference |
|
||||
| --- | --- |
|
||||
| Writing the base-snapshot, auth, or create script for a snapshot-capable cloud provider | `references/provider-vercel.md` |
|
||||
| The recipe connects over SSH instead of starting `orca serve`, including provisioned root | `references/ssh-host.md` |
|
||||
| The environment is a local Docker container reached over SSH | `references/docker-ssh.md` |
|
||||
| The user's desktop is Windows and you are scaffolding local-side scripts | `references/windows-scripts.md` |
|
||||
| A doctor, provision, clone, login, or snapshot step failed | `references/failure-modes.md` |
|
||||
| Action gate | Bundled reference |
|
||||
| ----------------------------------------------------------------------------------------- | ------------------------------- |
|
||||
| Writing the base-snapshot, auth, or create script for a snapshot-capable cloud provider | `references/provider-vercel.md` |
|
||||
| The recipe connects over SSH instead of starting `orca serve`, including provisioned root | `references/ssh-host.md` |
|
||||
| The environment is a local Docker container reached over SSH | `references/docker-ssh.md` |
|
||||
| The user's desktop is Windows and you are scaffolding local-side scripts | `references/windows-scripts.md` |
|
||||
| A doctor, provision, clone, login, or snapshot step failed | `references/failure-modes.md` |
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -1,5 +1,11 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import type { OrchestrationFleetWorker } from '../../../shared/orchestration-fleet-projection'
|
||||
import { subagentGroupFallbackText } from '../../../shared/native-chat-subagent-summary'
|
||||
import type {
|
||||
NativeChatBlock,
|
||||
NativeChatMessage,
|
||||
NativeChatSubagentEntry
|
||||
} from '../../../shared/native-chat-types'
|
||||
import type { OrchestrationWorkerReadResult } from '../../../shared/orchestration-worker-output'
|
||||
import { formatWorkerRead, formatWorkerStart } from './worker-output'
|
||||
|
||||
@@ -288,3 +294,207 @@ function workerReadResult(
|
||||
type WorkerReadResultWithoutContext<T> = T extends unknown
|
||||
? Omit<T, 'dispatchId' | 'status'>
|
||||
: never
|
||||
|
||||
function transcriptRead(
|
||||
blocks: NativeChatBlock[],
|
||||
role: NativeChatMessage['role'] = 'assistant'
|
||||
): OrchestrationWorkerReadResult {
|
||||
const message: NativeChatMessage = {
|
||||
id: 'm1',
|
||||
role,
|
||||
blocks,
|
||||
timestamp: 1,
|
||||
source: 'transcript'
|
||||
}
|
||||
return {
|
||||
dispatchId: 'd1',
|
||||
source: 'transcript',
|
||||
sourceIdentity: 'pane:1',
|
||||
provider: 'codex',
|
||||
transcript: { messages: [message], nextCursor: '1', limited: false, returnedMessageCount: 1 },
|
||||
cursor: '1',
|
||||
status: { worker: 'running', terminal: 'running' },
|
||||
fallbackReason: null,
|
||||
warnings: []
|
||||
}
|
||||
}
|
||||
|
||||
const ROSTER: readonly NativeChatSubagentEntry[] = [
|
||||
{ id: 'child-1', label: 'read', state: 'working' },
|
||||
{ id: 'child-2', label: 'edit', state: 'failed' }
|
||||
]
|
||||
|
||||
function occurrences(haystack: string, needle: string): number {
|
||||
return haystack.split(needle).length - 1
|
||||
}
|
||||
|
||||
describe('formatWorkerRead', () => {
|
||||
// The replay case this row is durable for: SQLite-backed, re-sent on every
|
||||
// reconnect, and read here by a client that draws no roster block, runs no
|
||||
// reconciliation, and cannot re-check whether those children still exist. A
|
||||
// sentence frozen mid-flight outlives the process that wrote it, so it must
|
||||
// not keep asserting a liveness only that process could have observed —
|
||||
// `docs/reference/ssh-execution-boundary.md` calls that loss of contact
|
||||
// reported as a live state.
|
||||
it('replays a mid-flight roster row without claiming a child is still working', () => {
|
||||
const midFlight: readonly NativeChatSubagentEntry[] = [
|
||||
{ id: 'child-1', label: 'read', state: 'working' },
|
||||
{ id: 'child-2', label: 'search', state: 'working' },
|
||||
{ id: 'child-3', label: 'edit', state: 'failed' }
|
||||
]
|
||||
|
||||
const output = formatWorkerRead(
|
||||
transcriptRead([
|
||||
{ type: 'text', text: subagentGroupFallbackText(midFlight) },
|
||||
{ type: 'subagent-group', groupId: 'thread:turn-1', agents: [...midFlight] }
|
||||
])
|
||||
)
|
||||
|
||||
expect(output).toContain('[assistant] Kicked off 3 subagents (1 failed)')
|
||||
expect(output).not.toMatch(/\bworking\b/)
|
||||
})
|
||||
|
||||
// The body `codexSubagentGroupBody` actually writes: the plain-text twin, then
|
||||
// the block it stands in for. The twin exists for clients that cannot draw the
|
||||
// block, so a client printing the block must not print the twin beside it —
|
||||
// the renderer drops the twin for the same reason, from the other side.
|
||||
it('prints the roster sentence once for the two-block row the producer writes', () => {
|
||||
const sentence = subagentGroupFallbackText(ROSTER)
|
||||
const output = formatWorkerRead(
|
||||
transcriptRead(
|
||||
[
|
||||
{ type: 'text', text: sentence },
|
||||
{ type: 'subagent-group', groupId: 'thread:turn-1', agents: [...ROSTER] }
|
||||
],
|
||||
'system'
|
||||
)
|
||||
)
|
||||
|
||||
expect(output).toContain(`[system] ${sentence}`)
|
||||
expect(occurrences(output, sentence)).toBe(1)
|
||||
})
|
||||
|
||||
// Suppression is per twin, not per message. One twin beside two roster blocks
|
||||
// silenced BOTH groups and printed one sentence, so the second roster vanished
|
||||
// with no marker — the same silent drop the missing-twin case above avoids.
|
||||
it('stands in for the second roster block when only one twin accompanies two', () => {
|
||||
const other: readonly NativeChatSubagentEntry[] = [
|
||||
{ id: 'child-3', label: 'plan', state: 'completed' }
|
||||
]
|
||||
const output = formatWorkerRead(
|
||||
transcriptRead(
|
||||
[
|
||||
{ type: 'text', text: subagentGroupFallbackText(ROSTER) },
|
||||
{ type: 'subagent-group', groupId: 'thread:turn-1', agents: [...ROSTER] },
|
||||
{ type: 'subagent-group', groupId: 'thread:turn-2', agents: [...other] }
|
||||
],
|
||||
'system'
|
||||
)
|
||||
)
|
||||
|
||||
expect(occurrences(output, subagentGroupFallbackText(ROSTER))).toBe(1)
|
||||
expect(output).toContain(`[subagents] ${subagentGroupFallbackText(other)}`)
|
||||
})
|
||||
|
||||
// Which group a lone twin belongs to is decided by its TEXT, not its position.
|
||||
// Claiming positionally silenced whichever group came first, so a twin
|
||||
// belonging to a LATER group erased the earlier group's roster and printed the
|
||||
// later one's sentence twice — the same silent drop, one permutation over.
|
||||
it('claims a lone twin for the group it names, not the first group in the message', () => {
|
||||
const other: readonly NativeChatSubagentEntry[] = [
|
||||
{ id: 'child-3', label: 'plan', state: 'completed' }
|
||||
]
|
||||
const second = subagentGroupFallbackText(other)
|
||||
const output = formatWorkerRead(
|
||||
transcriptRead(
|
||||
[
|
||||
{ type: 'text', text: second },
|
||||
{ type: 'subagent-group', groupId: 'thread:turn-1', agents: [...ROSTER] },
|
||||
{ type: 'subagent-group', groupId: 'thread:turn-2', agents: [...other] }
|
||||
],
|
||||
'system'
|
||||
)
|
||||
)
|
||||
|
||||
expect(occurrences(output, second)).toBe(1)
|
||||
expect(output).toContain(`[subagents] ${subagentGroupFallbackText(ROSTER)}`)
|
||||
})
|
||||
|
||||
// The same claim, with the twin written after both blocks: nothing about the
|
||||
// ORDER of a twin and its group is guaranteed by the block schema.
|
||||
it('claims a trailing twin for the group it names', () => {
|
||||
const other: readonly NativeChatSubagentEntry[] = [
|
||||
{ id: 'child-3', label: 'plan', state: 'completed' }
|
||||
]
|
||||
const second = subagentGroupFallbackText(other)
|
||||
const output = formatWorkerRead(
|
||||
transcriptRead(
|
||||
[
|
||||
{ type: 'subagent-group', groupId: 'thread:turn-1', agents: [...ROSTER] },
|
||||
{ type: 'subagent-group', groupId: 'thread:turn-2', agents: [...other] },
|
||||
{ type: 'text', text: second }
|
||||
],
|
||||
'system'
|
||||
)
|
||||
)
|
||||
|
||||
expect(occurrences(output, second)).toBe(1)
|
||||
expect(output).toContain(`[subagents] ${subagentGroupFallbackText(ROSTER)}`)
|
||||
})
|
||||
|
||||
// A group with no twin beside it is a shape the block schema admits and no
|
||||
// producer writes. Dropping it would lose the roster entirely, so the block
|
||||
// itself carries the sentence when nothing else does.
|
||||
it('stands in for a roster block that arrived without its twin', () => {
|
||||
const output = formatWorkerRead(
|
||||
transcriptRead([{ type: 'subagent-group', groupId: 'thread:turn-1', agents: [...ROSTER] }])
|
||||
)
|
||||
|
||||
expect(output).toContain(`[assistant] [subagents] ${subagentGroupFallbackText(ROSTER)}`)
|
||||
})
|
||||
|
||||
// A roster from a newer build holds a state this build does not know, which
|
||||
// `summarizeSubagentGroup` reads as `unverifiable`. Recomputing the sentence
|
||||
// to compare it against the frozen twin therefore produced a DIFFERENT string,
|
||||
// and the CLI printed the roster twice: the twin's own wording plus a
|
||||
// `[subagents]` line contradicting it.
|
||||
it('prints the roster once when the twin names a state this build cannot reproduce', () => {
|
||||
const frozenTwin = 'Ran 2 subagents (1 cancelled)'
|
||||
const output = formatWorkerRead(
|
||||
transcriptRead(
|
||||
[
|
||||
{ type: 'text', text: frozenTwin },
|
||||
{
|
||||
type: 'subagent-group',
|
||||
groupId: 'thread:turn-1',
|
||||
agents: [
|
||||
{ id: 'child-1', label: 'read', state: 'completed' },
|
||||
{ id: 'child-2', label: 'edit', state: 'cancelled' }
|
||||
] as unknown as NativeChatSubagentEntry[]
|
||||
}
|
||||
],
|
||||
'system'
|
||||
)
|
||||
)
|
||||
|
||||
expect(output).toContain(`[system] ${frozenTwin}`)
|
||||
expect(output).not.toContain('[subagents]')
|
||||
expect(output).not.toContain('unverifiable')
|
||||
})
|
||||
|
||||
// The journal admits block types this build does not know, and `client.call`
|
||||
// casts the RPC result rather than validating it — so a newer remote host's
|
||||
// block reaches this formatter as-is. Reading fields off it threw a TypeError
|
||||
// and took down the whole `worker read`.
|
||||
it('degrades an unknown block type from a newer host instead of throwing', () => {
|
||||
const output = formatWorkerRead(
|
||||
transcriptRead([
|
||||
{ type: 'text', text: 'before' },
|
||||
{ type: 'plan-step', title: 'ship it' } as unknown as NativeChatBlock,
|
||||
{ type: 'text', text: 'after' }
|
||||
])
|
||||
)
|
||||
|
||||
expect(output).toContain('[assistant] before\n[unsupported block]\nafter')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -38,6 +38,7 @@ export type ClaudeStructuredSdkOptions = Pick<
|
||||
| 'sessionId'
|
||||
| 'resume'
|
||||
| 'resumeSessionAt'
|
||||
| 'resumeDropsTurn'
|
||||
>
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
adapterFor,
|
||||
fakeClaude,
|
||||
identityFor,
|
||||
PROVIDER_SESSION_ID
|
||||
} from './claude-structured-session-test-support'
|
||||
import { ClaudeRewindAttempt } from './claude-structured-rewind'
|
||||
import { AgentSessionRewindRefusal } from '../native-chat/agent-session-wire/structured-agent-session-adapter'
|
||||
|
||||
const intent = { targetUuid: 'kept', previousLeafUuid: 'tip', dropsTurn: 'drop' }
|
||||
const proofLaunch = {
|
||||
providerSessionId: PROVIDER_SESSION_ID,
|
||||
claudeConfigDir: '/claude',
|
||||
options: {},
|
||||
resumed: true,
|
||||
resumeLeafUuid: 'tip',
|
||||
cwd: '/workspace',
|
||||
pathToClaudeCodeExecutable: 'claude'
|
||||
}
|
||||
|
||||
describe('Claude rewind acquisition', () => {
|
||||
it('executes a cursor resume in place and proves the exact target before publication', async () => {
|
||||
const fake = fakeClaude()
|
||||
const proof = vi.fn(async (_input: { intentionalRewindUuid?: string }) => 'kept')
|
||||
const adapter = adapterFor(
|
||||
fake,
|
||||
{ resumed: true, resumeLeafUuid: 'tip' },
|
||||
[],
|
||||
[],
|
||||
undefined,
|
||||
proof
|
||||
)
|
||||
try {
|
||||
const acquired = await adapter.acquire({
|
||||
identity: identityFor(),
|
||||
fence: 7,
|
||||
spawnToken: 'spawn',
|
||||
rewind: intent
|
||||
})
|
||||
expect(acquired.link.handle).toMatchObject({
|
||||
provider: 'claude',
|
||||
sessionId: PROVIDER_SESSION_ID,
|
||||
leafUuid: 'kept'
|
||||
})
|
||||
expect(fake.connections[0]!.launch.options).toMatchObject({
|
||||
resume: PROVIDER_SESSION_ID,
|
||||
resumeSessionAt: 'kept',
|
||||
resumeDropsTurn: 'drop'
|
||||
})
|
||||
expect(fake.connections[0]!.launch.options).not.toHaveProperty('forkSession')
|
||||
expect(proof).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ previousLeafUuid: 'tip', intentionalRewindUuid: 'kept' })
|
||||
)
|
||||
await adapter.closeSession('session-1')
|
||||
await adapter.acquire({ identity: identityFor(), fence: 8, spawnToken: 'spawn-next' })
|
||||
expect(fake.connections[1]!.launch.options).not.toHaveProperty('resumeDropsTurn')
|
||||
expect(
|
||||
proof.mock.calls.filter(([input]) => input.intentionalRewindUuid !== undefined)
|
||||
).toHaveLength(1)
|
||||
} finally {
|
||||
await adapter.closeAll()
|
||||
}
|
||||
})
|
||||
it('recognizes the documented refusal and closes the failed child without retry', async () => {
|
||||
const fake = fakeClaude()
|
||||
const openConnection = fake.openConnection
|
||||
fake.openConnection = async (launch, handlers) => {
|
||||
const connection = await openConnection(launch, handlers)
|
||||
const initialize = connection.initializationResult
|
||||
connection.initializationResult = async (...args) => {
|
||||
const result = await initialize(...args)
|
||||
handlers?.onMessage?.({
|
||||
type: 'result',
|
||||
subtype: 'error_during_execution',
|
||||
session_id: PROVIDER_SESSION_ID,
|
||||
errors: ['Resume rejected by --resume-drops-turn: additional prompt observed']
|
||||
})
|
||||
return result
|
||||
}
|
||||
return connection
|
||||
}
|
||||
const proof = vi.fn(async (_input: { intentionalRewindUuid?: string }) => 'kept')
|
||||
const adapter = adapterFor(fake, { resumed: true }, [], [], undefined, proof)
|
||||
await expect(
|
||||
adapter.acquire({ identity: identityFor(), fence: 7, spawnToken: 'spawn', rewind: intent })
|
||||
).rejects.toMatchObject({ rewindReason: 'provider-refused' })
|
||||
expect(fake.connections).toHaveLength(1)
|
||||
expect(fake.connections[0]?.closed).toBe(true)
|
||||
expect(proof).not.toHaveBeenCalled()
|
||||
await adapter.closeAll()
|
||||
})
|
||||
it('consumes proof authorization even if its first read fails', async () => {
|
||||
const proof = vi.fn(async () => {
|
||||
throw new Error('torn transcript')
|
||||
})
|
||||
const attempt = new ClaudeRewindAttempt(intent)
|
||||
const launch = {
|
||||
providerSessionId: PROVIDER_SESSION_ID,
|
||||
claudeConfigDir: '/claude',
|
||||
options: {},
|
||||
resumed: true,
|
||||
resumeLeafUuid: 'tip',
|
||||
cwd: '/workspace',
|
||||
pathToClaudeCodeExecutable: 'claude'
|
||||
}
|
||||
await expect(attempt.prove(launch, { readTranscriptLeaf: proof })).rejects.toBeInstanceOf(
|
||||
AgentSessionRewindRefusal
|
||||
)
|
||||
expect(await attempt.prove(launch, { readTranscriptLeaf: proof })).toBeNull()
|
||||
expect(proof).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
it('never persists success for a mismatching leaf', async () => {
|
||||
const onProved = vi.fn(async () => {})
|
||||
const attempt = new ClaudeRewindAttempt(intent, onProved)
|
||||
await expect(
|
||||
attempt.prove(proofLaunch, { readTranscriptLeaf: async () => 'other' })
|
||||
).rejects.toMatchObject({ rewindReason: 'proof-mismatch' })
|
||||
expect(onProved).not.toHaveBeenCalled()
|
||||
})
|
||||
it('preserves commit failure as unknown and consumes the override before persisting', async () => {
|
||||
const diskError = new Error('record write failed')
|
||||
const onProved = vi.fn(async () => {
|
||||
throw diskError
|
||||
})
|
||||
const proof = vi.fn(async () => 'kept')
|
||||
const attempt = new ClaudeRewindAttempt(intent, onProved)
|
||||
const launch = {
|
||||
providerSessionId: PROVIDER_SESSION_ID,
|
||||
claudeConfigDir: '/claude',
|
||||
options: {},
|
||||
resumed: true,
|
||||
resumeLeafUuid: 'tip',
|
||||
cwd: '/workspace',
|
||||
pathToClaudeCodeExecutable: 'claude'
|
||||
}
|
||||
await expect(attempt.prove(launch, { readTranscriptLeaf: proof })).rejects.toBe(diskError)
|
||||
expect(onProved).toHaveBeenCalledWith('kept')
|
||||
expect(await attempt.prove(launch, { readTranscriptLeaf: proof })).toBeNull()
|
||||
expect(proof).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
it('checkpoints the proved target before late acquisition failure without persisting a stale cursor', async () => {
|
||||
const fake = fakeClaude()
|
||||
const launch = { resumed: true, resumeLeafUuid: 'tip' }
|
||||
const persisted: unknown[] = []
|
||||
const proof = vi.fn(async () => 'kept')
|
||||
const adapter = adapterFor(fake, launch, [], persisted, undefined, proof)
|
||||
const onProved = vi.fn(async (leafUuid: string) => {
|
||||
launch.resumeLeafUuid = leafUuid
|
||||
fake.connections[0]!.closed = true
|
||||
})
|
||||
try {
|
||||
await expect(
|
||||
adapter.acquire({
|
||||
identity: identityFor(),
|
||||
fence: 7,
|
||||
spawnToken: 'spawn',
|
||||
rewind: { ...intent, onProved }
|
||||
})
|
||||
).rejects.toThrow('exited while being acquired')
|
||||
expect(onProved).toHaveBeenCalledWith('kept')
|
||||
expect(persisted).toEqual([])
|
||||
const acquired = await adapter.acquire({
|
||||
identity: identityFor(),
|
||||
fence: 8,
|
||||
spawnToken: 'retry'
|
||||
})
|
||||
expect(acquired.link.handle).toMatchObject({ leafUuid: 'kept' })
|
||||
expect(fake.connections[1]!.launch.options).not.toHaveProperty('resumeDropsTurn')
|
||||
expect(proof).toHaveBeenCalledTimes(1)
|
||||
} finally {
|
||||
await adapter.closeAll()
|
||||
}
|
||||
})
|
||||
it('restores an interrupted unproved rewind only after exact ordinary branch proof', async () => {
|
||||
const fake = fakeClaude()
|
||||
const proof = vi.fn(async (_input: { intentionalRewindUuid?: string }) => 'kept')
|
||||
const restored = vi.fn(async () => {})
|
||||
const adapter = adapterFor(
|
||||
fake,
|
||||
{ resumed: true, resumeLeafUuid: 'tip' },
|
||||
[],
|
||||
[],
|
||||
undefined,
|
||||
proof
|
||||
)
|
||||
const input = {
|
||||
identity: identityFor(),
|
||||
fence: 7,
|
||||
spawnToken: 'spawn',
|
||||
rewindRecovery: { leafUuid: 'tip', onProved: restored }
|
||||
}
|
||||
try {
|
||||
await expect(adapter.acquire(input)).rejects.toMatchObject({ rewindReason: 'proof-mismatch' })
|
||||
expect(restored).not.toHaveBeenCalled()
|
||||
proof.mockResolvedValue('tip')
|
||||
await adapter.acquire({ ...input, fence: 8, spawnToken: 'retry' })
|
||||
expect(restored).toHaveBeenCalledOnce()
|
||||
expect(proof).toHaveBeenCalledWith(expect.objectContaining({ previousLeafUuid: 'tip' }))
|
||||
for (const [request] of proof.mock.calls) {
|
||||
expect(request).not.toHaveProperty('intentionalRewindUuid')
|
||||
}
|
||||
} finally {
|
||||
await adapter.closeAll()
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,118 @@
|
||||
import { AgentSessionRewindRefusal } from '../native-chat/agent-session-wire/structured-agent-session-adapter'
|
||||
|
||||
export function claudeRewindRefusalFromMessage(
|
||||
message: Record<string, unknown>
|
||||
): AgentSessionRewindRefusal | null {
|
||||
return message.type === 'result' &&
|
||||
message.subtype === 'error_during_execution' &&
|
||||
Array.isArray(message.errors) &&
|
||||
message.errors.some(
|
||||
(error) =>
|
||||
typeof error === 'string' && error.startsWith('Resume rejected by --resume-drops-turn:')
|
||||
)
|
||||
? new AgentSessionRewindRefusal('provider-refused')
|
||||
: null
|
||||
}
|
||||
|
||||
import type { StructuredAgentSessionAcquireInput } from '../native-chat/agent-session-wire/structured-agent-session-adapter'
|
||||
import type { ClaudeStructuredLaunch } from './claude-structured-launch-resolution'
|
||||
import type { ClaudeStructuredSessionAdapterDeps } from './claude-structured-session-state'
|
||||
|
||||
type Intent = NonNullable<StructuredAgentSessionAcquireInput['rewind']>
|
||||
|
||||
/** The proof authorization exists only for this acquisition's first proof attempt. */
|
||||
export class ClaudeRewindAttempt {
|
||||
private refusal: AgentSessionRewindRefusal | null = null
|
||||
constructor(
|
||||
private intent: Intent | undefined,
|
||||
private readonly onProved?: (leafUuid: string) => Promise<void>
|
||||
) {}
|
||||
|
||||
observe(message: Record<string, unknown>): AgentSessionRewindRefusal | null {
|
||||
if (!this.intent) {
|
||||
return null
|
||||
}
|
||||
this.refusal ??= claudeRewindRefusalFromMessage(message)
|
||||
return this.refusal
|
||||
}
|
||||
|
||||
applyLaunch(
|
||||
launch: ClaudeStructuredLaunch,
|
||||
deps: Pick<ClaudeStructuredSessionAdapterDeps, 'readTranscriptLeaf'>
|
||||
): void {
|
||||
if (!this.intent) {
|
||||
return
|
||||
}
|
||||
if (!launch.resumed || !deps.readTranscriptLeaf) {
|
||||
throw new AgentSessionRewindRefusal('unsupported')
|
||||
}
|
||||
launch.options = {
|
||||
...launch.options,
|
||||
resume: launch.providerSessionId,
|
||||
resumeSessionAt: this.intent.targetUuid,
|
||||
...(this.intent.dropsTurn ? { resumeDropsTurn: this.intent.dropsTurn } : {})
|
||||
}
|
||||
launch.resumeLeafUuid = this.intent.targetUuid
|
||||
}
|
||||
|
||||
async prove(
|
||||
launch: ClaudeStructuredLaunch,
|
||||
deps: Pick<ClaudeStructuredSessionAdapterDeps, 'readTranscriptLeaf'>
|
||||
): Promise<string | null> {
|
||||
const intent = this.intent
|
||||
this.clear()
|
||||
if (this.refusal) {
|
||||
throw this.refusal
|
||||
}
|
||||
if (!intent) {
|
||||
return null
|
||||
}
|
||||
let leaf: string | null
|
||||
try {
|
||||
leaf = await deps.readTranscriptLeaf!({
|
||||
providerSessionId: launch.providerSessionId,
|
||||
previousLeafUuid: intent.previousLeafUuid,
|
||||
intentionalRewindUuid: intent.targetUuid,
|
||||
claudeConfigDir: launch.claudeConfigDir
|
||||
})
|
||||
if (leaf !== intent.targetUuid) {
|
||||
throw new AgentSessionRewindRefusal('proof-mismatch')
|
||||
}
|
||||
} catch (error) {
|
||||
throw error instanceof AgentSessionRewindRefusal
|
||||
? error
|
||||
: new AgentSessionRewindRefusal('proof-mismatch')
|
||||
}
|
||||
// Persistence failure is an unknown outcome, never evidence that the provider refused.
|
||||
await this.onProved?.(leaf)
|
||||
return leaf
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.intent = undefined
|
||||
}
|
||||
}
|
||||
|
||||
/** An interrupted, unproved rewind restores its original cursor without ancestor authorization. */
|
||||
export async function proveClaudeRewindRecovery(
|
||||
recovery: StructuredAgentSessionAcquireInput['rewindRecovery'],
|
||||
launch: ClaudeStructuredLaunch,
|
||||
deps: Pick<ClaudeStructuredSessionAdapterDeps, 'readTranscriptLeaf'>
|
||||
): Promise<string | null> {
|
||||
if (!recovery) {
|
||||
return null
|
||||
}
|
||||
if (!launch.resumed || launch.resumeLeafUuid !== recovery.leafUuid || !deps.readTranscriptLeaf) {
|
||||
throw new AgentSessionRewindRefusal('proof-mismatch')
|
||||
}
|
||||
const leaf = await deps.readTranscriptLeaf({
|
||||
providerSessionId: launch.providerSessionId,
|
||||
previousLeafUuid: recovery.leafUuid,
|
||||
claudeConfigDir: launch.claudeConfigDir
|
||||
})
|
||||
if (leaf !== recovery.leafUuid) {
|
||||
throw new AgentSessionRewindRefusal('proof-mismatch')
|
||||
}
|
||||
await recovery.onProved()
|
||||
return leaf
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { ClaudeRewindAttempt, proveClaudeRewindRecovery } from './claude-structured-rewind'
|
||||
import {
|
||||
AgentSessionAcquisitionExitUnprovenError,
|
||||
AgentSessionPreSpawnError
|
||||
@@ -85,6 +86,7 @@ export async function acquireClaudeSession({
|
||||
const initTimeoutMs = deps.initTimeoutMs ?? CLAUDE_STRUCTURED_INIT_TIMEOUT_MS
|
||||
const initDeadline = createClaudeInitDeadline(sessionId, initTimeoutMs)
|
||||
|
||||
const rewind = new ClaudeRewindAttempt(input.rewind, input.rewind?.onProved)
|
||||
const onMessage = (message: Record<string, unknown>): void => {
|
||||
const init = readClaudeInit(message)
|
||||
if (readClaudeFrameString(message, 'session_id') !== expectedProviderSessionId) {
|
||||
@@ -95,6 +97,11 @@ export async function acquireClaudeSession({
|
||||
}
|
||||
return
|
||||
}
|
||||
const refusal = rewind.observe(message)
|
||||
if (refusal) {
|
||||
initDeadline.reject(refusal)
|
||||
return
|
||||
}
|
||||
if (init) {
|
||||
initDeadline.resolve(init)
|
||||
// Every turn opens with an init frame naming the model the CLI is actually
|
||||
@@ -178,6 +185,7 @@ export async function acquireClaudeSession({
|
||||
? error
|
||||
: new AgentSessionPreSpawnError(error)
|
||||
})
|
||||
rewind.applyLaunch(launch, deps)
|
||||
expectedProviderSessionId = launch.providerSessionId
|
||||
observedLeafUuid = launch.resumeLeafUuid
|
||||
acquisitions.assertCurrent(sessionId, attempt)
|
||||
@@ -241,6 +249,9 @@ export async function acquireClaudeSession({
|
||||
diagnostic: claudeAuthDiagnostic(init, settings)
|
||||
})
|
||||
)
|
||||
observedLeafUuid = (await rewind.prove(launch, deps)) ?? observedLeafUuid
|
||||
observedLeafUuid =
|
||||
(await proveClaudeRewindRecovery(input.rewindRecovery, launch, deps)) ?? observedLeafUuid
|
||||
const process = await claudeProcessIdentity(
|
||||
{ ...input, pid: connection.pid },
|
||||
deps.readProcessStartTime
|
||||
@@ -298,6 +309,7 @@ export async function acquireClaudeSession({
|
||||
acquisitions.deleteIfCurrent(sessionId, attempt)
|
||||
throw acquisitionError
|
||||
} finally {
|
||||
rewind.clear()
|
||||
attempt.finish()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ import type {
|
||||
StructuredAgentSessionAcquireInput,
|
||||
StructuredAgentSessionAdapter
|
||||
} from '../native-chat/agent-session-wire/structured-agent-session-adapter'
|
||||
import type { StructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink'
|
||||
import {
|
||||
answerClaudePrompt,
|
||||
cancelClaudeTurn,
|
||||
@@ -58,6 +57,9 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda
|
||||
|
||||
supportsLocation = supportsClaudeStructuredLocation
|
||||
|
||||
rewindSupport: NonNullable<StructuredAgentSessionAdapter['rewindSupport']> = () =>
|
||||
this.deps.readTranscriptLeaf ? { supported: true } : { supported: false, reason: 'unsupported' }
|
||||
|
||||
acquire = (input: StructuredAgentSessionAcquireInput): Promise<AgentSessionAcquisition> =>
|
||||
acquireClaudeSession({
|
||||
input,
|
||||
@@ -67,7 +69,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda
|
||||
exits: this.exits,
|
||||
callbacks: {
|
||||
deliver: (attempt, sessionId, event) => this.deliver(attempt, sessionId, event),
|
||||
emit: (session, events, event) => this.emit(session, events, event),
|
||||
emit: (session, _events, event) => this.emit(session, event),
|
||||
handleExit: (sessionId, attempt, error) => this.handleExit(sessionId, attempt, error),
|
||||
settleExit: (sessionId, exit) => this.settleUnexpectedExit(sessionId, exit)
|
||||
}
|
||||
@@ -155,7 +157,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda
|
||||
acquisitionGeneration: exit.session.acquisitionGeneration
|
||||
}
|
||||
try {
|
||||
this.emit(exit.session, exit.session.events, ended)
|
||||
this.emit(exit.session, ended)
|
||||
} finally {
|
||||
settleClaudeExitedSession(exit.session)
|
||||
}
|
||||
@@ -187,11 +189,7 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda
|
||||
})
|
||||
}
|
||||
|
||||
private emit(
|
||||
session: ClaudeSession | null,
|
||||
_events: StructuredAgentSessionEventSink | undefined,
|
||||
event: ClaudeStructuredSessionEvent
|
||||
): void {
|
||||
private emit(session: ClaudeSession | null, event: ClaudeStructuredSessionEvent): void {
|
||||
const backgroundTasksChanged =
|
||||
event.type === 'ended'
|
||||
? (session?.backgroundTasks.clear() ?? false)
|
||||
|
||||
@@ -88,6 +88,7 @@ export type ClaudeStructuredSessionAdapterDeps = {
|
||||
readTranscriptLeaf?: (input: {
|
||||
providerSessionId: string
|
||||
previousLeafUuid: string | null
|
||||
intentionalRewindUuid?: string
|
||||
/** Account-scoped Claude config root that owns this provider session. */
|
||||
claudeConfigDir: string
|
||||
}) => Promise<string | null>
|
||||
|
||||
@@ -13,7 +13,7 @@ type TranscriptNode = {
|
||||
|
||||
export type ClaudeTranscriptBranchProof = {
|
||||
leafUuid: string
|
||||
relation: 'initial' | 'same' | 'descendant'
|
||||
relation: 'initial' | 'same' | 'descendant' | 'intentional-rewind'
|
||||
}
|
||||
|
||||
function nonEmptyString(value: unknown): string | null {
|
||||
@@ -83,6 +83,7 @@ export function proveClaudeTranscriptBranchFromJsonl(input: {
|
||||
contents: string
|
||||
providerSessionId: string
|
||||
previousLeafUuid: string | null
|
||||
intentionalRewindUuid?: string
|
||||
}): ClaudeTranscriptBranchProof {
|
||||
const nodes = new Map<string, TranscriptNode>()
|
||||
let leafUuid: string | null = null
|
||||
@@ -156,6 +157,21 @@ export function proveClaudeTranscriptBranchFromJsonl(input: {
|
||||
throw transcriptError('marker precedes its leaf record')
|
||||
}
|
||||
const previousLeafUuid = input.previousLeafUuid
|
||||
if (input.intentionalRewindUuid !== undefined) {
|
||||
if (leafUuid !== input.intentionalRewindUuid || !input.previousLeafUuid) {
|
||||
throw transcriptError('rewind target does not match the observed leaf')
|
||||
}
|
||||
proveMainLineAncestry(nodes, input.previousLeafUuid, input.providerSessionId)
|
||||
proveAppendOrder(nodes)
|
||||
let ancestor = nodes.get(input.previousLeafUuid)?.parentUuid ?? null
|
||||
for (let depth = 0; ancestor !== null && depth < MAX_CLAUDE_TRANSCRIPT_ANCESTRY; depth += 1) {
|
||||
if (ancestor === leafUuid) {
|
||||
return { leafUuid, relation: 'intentional-rewind' }
|
||||
}
|
||||
ancestor = nodes.get(ancestor)?.parentUuid ?? null
|
||||
}
|
||||
throw transcriptError('rewind target is not an ancestor of the previous cursor')
|
||||
}
|
||||
if (!previousLeafUuid) {
|
||||
proveMainLineAncestry(nodes, leafUuid, input.providerSessionId)
|
||||
// A branch proof is based on an append-only snapshot. A child that appears
|
||||
@@ -210,11 +226,13 @@ export async function proveClaudeTranscriptBranch(input: {
|
||||
transcriptPath: string
|
||||
providerSessionId: string
|
||||
previousLeafUuid: string | null
|
||||
intentionalRewindUuid?: string
|
||||
}): Promise<ClaudeTranscriptBranchProof> {
|
||||
return proveClaudeTranscriptBranchFromJsonl({
|
||||
contents: await readFile(input.transcriptPath, 'utf8'),
|
||||
providerSessionId: input.providerSessionId,
|
||||
previousLeafUuid: input.previousLeafUuid
|
||||
previousLeafUuid: input.previousLeafUuid,
|
||||
intentionalRewindUuid: input.intentionalRewindUuid
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { proveClaudeTranscriptBranchFromJsonl } from './claude-transcript-branch-proof'
|
||||
|
||||
const row = (uuid: string, parentUuid: string | null, extra = {}) =>
|
||||
JSON.stringify({ type: 'assistant', sessionId: 'provider', uuid, parentUuid, ...extra })
|
||||
const marker = (leafUuid: string) =>
|
||||
JSON.stringify({ type: 'last-prompt', sessionId: 'provider', leafUuid })
|
||||
const graph = [row('root', null), row('kept', 'root'), row('old', 'kept')]
|
||||
const prove = (rows: string[], leaf: string, intentionalRewindUuid?: string) =>
|
||||
proveClaudeTranscriptBranchFromJsonl({
|
||||
contents: `${[...rows, marker(leaf)].join('\n')}\n`,
|
||||
providerSessionId: 'provider',
|
||||
previousLeafUuid: 'old',
|
||||
intentionalRewindUuid
|
||||
})
|
||||
|
||||
describe('explicit Claude rewind ancestry', () => {
|
||||
it('admits only the exact requested main-chain ancestor', () => {
|
||||
expect(prove(graph, 'kept', 'kept')).toEqual({
|
||||
leafUuid: 'kept',
|
||||
relation: 'intentional-rewind'
|
||||
})
|
||||
expect(() => prove(graph, 'kept')).toThrow('sibling')
|
||||
expect(() => prove(graph, 'kept', 'root')).toThrow('target')
|
||||
expect(() => prove(graph, 'old', 'old')).toThrow('not an ancestor')
|
||||
})
|
||||
it('keeps sibling and sidechain rejection even with explicit intent', () => {
|
||||
expect(() => prove([...graph, row('sibling', 'root')], 'sibling', 'sibling')).toThrow(
|
||||
'not an ancestor'
|
||||
)
|
||||
expect(() =>
|
||||
prove(
|
||||
[row('root', null), row('kept', 'root', { isSidechain: true }), row('old', 'kept')],
|
||||
'kept',
|
||||
'kept'
|
||||
)
|
||||
).toThrow()
|
||||
})
|
||||
it('refuses missing, reordered, or cyclic ancestry', () => {
|
||||
expect(() => prove(graph.slice(1), 'kept', 'kept')).toThrow('missing ancestor')
|
||||
expect(() => prove([graph[1]!, graph[0]!, graph[2]!], 'kept', 'kept')).toThrow(
|
||||
'parent row follows'
|
||||
)
|
||||
expect(() => prove([row('root', 'old'), ...graph.slice(1)], 'kept', 'kept')).toThrow('cycle')
|
||||
})
|
||||
})
|
||||
@@ -182,10 +182,10 @@ export function readCodexAuthIdentity(contents: string): CodexAuthIdentity | nul
|
||||
readStringClaim(authClaims, 'chatgpt_account_id') ??
|
||||
readStringClaim(payload, 'chatgpt_account_id')
|
||||
),
|
||||
workspaceLabel: normalizeField(
|
||||
readStringClaim(authClaims, 'workspace_name') ??
|
||||
readStringClaim(profileClaims, 'workspace_name')
|
||||
),
|
||||
workspaceLabel:
|
||||
normalizeField(readStringClaim(authClaims, 'workspace_name')) ??
|
||||
normalizeField(readStringClaim(profileClaims, 'workspace_name')) ??
|
||||
readPlanWorkspaceLabel(authClaims),
|
||||
workspaceAccountId: normalizeField(
|
||||
readStringClaim(authClaims, 'workspace_account_id') ??
|
||||
tokenAccountId ??
|
||||
@@ -194,6 +194,31 @@ export function readCodexAuthIdentity(contents: string): CodexAuthIdentity | nul
|
||||
}
|
||||
}
|
||||
|
||||
function readPlanWorkspaceLabel(authClaims: Record<string, unknown> | null): string | null {
|
||||
// Codex tokens commonly omit workspace_name but identify the account's plan.
|
||||
switch (normalizeField(readStringClaim(authClaims, 'chatgpt_plan_type'))?.toLowerCase()) {
|
||||
case 'free':
|
||||
return 'Personal (Free)'
|
||||
case 'go':
|
||||
return 'Personal (Go)'
|
||||
case 'plus':
|
||||
return 'Personal (Plus)'
|
||||
case 'pro':
|
||||
return 'Personal (Pro)'
|
||||
case 'team':
|
||||
return 'Team'
|
||||
case 'business':
|
||||
return 'Business'
|
||||
case 'enterprise':
|
||||
return 'Enterprise'
|
||||
case 'edu':
|
||||
return 'Education'
|
||||
case undefined:
|
||||
default:
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function readFreshnessFromAuthContents(contents: string): number | null {
|
||||
const raw = parseJsonRecord(contents)
|
||||
if (!raw) {
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import type { CodexManagedAccount } from '../../shared/managed-account-types'
|
||||
import {
|
||||
codexAuthMatchesManagedAccount,
|
||||
codexAuthMatchesSystemDefaultIdentity,
|
||||
readCodexAuthIdentity
|
||||
} from './codex-auth-identity'
|
||||
|
||||
const email = 'same@example.com'
|
||||
|
||||
function auth(
|
||||
accountId: string,
|
||||
claims: Record<string, unknown>,
|
||||
profileClaims: Record<string, unknown> = {}
|
||||
): string {
|
||||
const payload = Buffer.from(
|
||||
JSON.stringify({
|
||||
email,
|
||||
'https://api.openai.com/auth': { chatgpt_account_id: accountId, ...claims },
|
||||
'https://api.openai.com/profile': profileClaims
|
||||
})
|
||||
).toString('base64url')
|
||||
return JSON.stringify({
|
||||
tokens: { account_id: accountId, id_token: `header.${payload}.signature` }
|
||||
})
|
||||
}
|
||||
|
||||
describe('Codex personal and organization workspace identity', () => {
|
||||
it.each([
|
||||
['free', 'Personal (Free)'],
|
||||
['go', 'Personal (Go)'],
|
||||
['plus', 'Personal (Plus)'],
|
||||
['pro', 'Personal (Pro)'],
|
||||
['team', 'Team'],
|
||||
['business', 'Business'],
|
||||
['enterprise', 'Enterprise'],
|
||||
['edu', 'Education']
|
||||
])('uses the %s plan when the token omits the workspace name', (plan, label) => {
|
||||
expect(readCodexAuthIdentity(auth('provider-1', { chatgpt_plan_type: plan }))).toEqual({
|
||||
email,
|
||||
providerAccountId: 'provider-1',
|
||||
workspaceAccountId: 'provider-1',
|
||||
workspaceLabel: label
|
||||
})
|
||||
})
|
||||
|
||||
it.each([undefined, null, '', 'future-plan', 42])(
|
||||
'does not infer personal membership from an unknown plan %s',
|
||||
(plan) => {
|
||||
expect(
|
||||
readCodexAuthIdentity(auth('provider-1', { chatgpt_plan_type: plan }))?.workspaceLabel
|
||||
).toBeNull()
|
||||
}
|
||||
)
|
||||
|
||||
it('preserves an explicit organization name over the plan label', () => {
|
||||
expect(
|
||||
readCodexAuthIdentity(
|
||||
auth('provider-1', { workspace_name: ' Acme ', chatgpt_plan_type: 'enterprise' })
|
||||
)?.workspaceLabel
|
||||
).toBe('Acme')
|
||||
})
|
||||
|
||||
it('uses the profile workspace name when the auth workspace name is blank', () => {
|
||||
expect(
|
||||
readCodexAuthIdentity(
|
||||
auth(
|
||||
'provider-1',
|
||||
{ workspace_name: ' ', chatgpt_plan_type: 'enterprise' },
|
||||
{ workspace_name: 'Acme' }
|
||||
)
|
||||
)?.workspaceLabel
|
||||
).toBe('Acme')
|
||||
})
|
||||
|
||||
it('keeps same-email personal and enterprise credentials isolated in both directions', () => {
|
||||
const personal = auth('personal-provider', { chatgpt_plan_type: 'plus' })
|
||||
const enterprise = auth('enterprise-provider', { chatgpt_plan_type: 'enterprise' })
|
||||
for (const [selectedAuth, otherAuth] of [
|
||||
[personal, enterprise],
|
||||
[enterprise, personal]
|
||||
]) {
|
||||
const identity = readCodexAuthIdentity(selectedAuth)!
|
||||
const account: CodexManagedAccount = {
|
||||
...identity,
|
||||
id: 'orca-account',
|
||||
email,
|
||||
managedHomePath: 'managed-home',
|
||||
createdAt: 1,
|
||||
updatedAt: 1,
|
||||
lastAuthenticatedAt: 1
|
||||
}
|
||||
expect(codexAuthMatchesManagedAccount(selectedAuth, account, selectedAuth)).toBe(true)
|
||||
expect(codexAuthMatchesManagedAccount(otherAuth, account, selectedAuth)).toBe(false)
|
||||
expect(codexAuthMatchesSystemDefaultIdentity(otherAuth, selectedAuth)).toBe(false)
|
||||
}
|
||||
expect(readCodexAuthIdentity(personal)?.workspaceLabel).toBe('Personal (Plus)')
|
||||
expect(readCodexAuthIdentity(enterprise)?.workspaceLabel).toBe('Enterprise')
|
||||
})
|
||||
|
||||
it('does not treat a matching plan label as proof of account ownership', () => {
|
||||
const first = auth('enterprise-a', { chatgpt_plan_type: 'enterprise' })
|
||||
const second = auth('enterprise-b', { chatgpt_plan_type: 'enterprise' })
|
||||
expect(codexAuthMatchesSystemDefaultIdentity(first, second)).toBe(false)
|
||||
})
|
||||
})
|
||||
@@ -87,64 +87,67 @@ describe('CodexRuntimeHomeService', () => {
|
||||
expect(service.getHostCodexHomePathsForSessionDiscovery()).toContain(managedHomePath)
|
||||
})
|
||||
|
||||
it('gives two managed accounts distinct homes without racing one auth.json', async () => {
|
||||
writeFileSync(getSystemCodexAuthPath(), '{"account":"system"}\n', 'utf-8')
|
||||
const account1Auth = createCodexAuthJson('one@example.com', 'acct-1', 'one')
|
||||
const account2Auth = createCodexAuthJson('two@example.com', 'acct-2', 'two')
|
||||
const home1 = createManagedAuth(testState.userDataDir, 'account-1', account1Auth)
|
||||
const home2 = createManagedAuth(testState.userDataDir, 'account-2', account2Auth)
|
||||
const settings = createSettings({
|
||||
shellStartupEnvProbeSupported: true,
|
||||
codexManagedAccounts: [
|
||||
{
|
||||
id: 'account-1',
|
||||
email: 'one@example.com',
|
||||
managedHomePath: home1,
|
||||
providerAccountId: 'acct-1',
|
||||
workspaceLabel: null,
|
||||
workspaceAccountId: 'acct-1',
|
||||
createdAt: 1,
|
||||
updatedAt: 1,
|
||||
lastAuthenticatedAt: 1
|
||||
},
|
||||
{
|
||||
id: 'account-2',
|
||||
email: 'two@example.com',
|
||||
managedHomePath: home2,
|
||||
providerAccountId: 'acct-2',
|
||||
workspaceLabel: null,
|
||||
workspaceAccountId: 'acct-2',
|
||||
createdAt: 2,
|
||||
updatedAt: 2,
|
||||
lastAuthenticatedAt: 2
|
||||
}
|
||||
],
|
||||
activeCodexManagedAccountId: 'account-1',
|
||||
activeCodexManagedAccountIdsByRuntime: { host: 'account-1', wsl: {} }
|
||||
})
|
||||
const store = createStore(settings)
|
||||
const { CodexRuntimeHomeService } = await import('./runtime-home-service')
|
||||
const service = new CodexRuntimeHomeService(store as never)
|
||||
|
||||
// A pane for account-1 launches, then the user switches and a second pane
|
||||
// for account-2 launches concurrently — each gets its OWN CODEX_HOME.
|
||||
expect(service.prepareForCodexLaunch()).toBe(home1)
|
||||
settings.activeCodexManagedAccountId = 'account-2'
|
||||
settings.activeCodexManagedAccountIdsByRuntime = { host: 'account-2', wsl: {} }
|
||||
expect(service.prepareForCodexLaunch()).toBe(home2)
|
||||
expect(
|
||||
service.prepareForCodexLaunch(undefined, undefined, {
|
||||
unavailableManagedHomePath: home1
|
||||
it.each(['two@example.com', 'one@example.com'])(
|
||||
'isolates account homes when the second email is %s',
|
||||
async (secondEmail) => {
|
||||
writeFileSync(getSystemCodexAuthPath(), '{"account":"system"}\n', 'utf-8')
|
||||
const account1Auth = createCodexAuthJson('one@example.com', 'acct-1', 'one')
|
||||
const account2Auth = createCodexAuthJson(secondEmail, 'acct-2', 'two')
|
||||
const home1 = createManagedAuth(testState.userDataDir, 'account-1', account1Auth)
|
||||
const home2 = createManagedAuth(testState.userDataDir, 'account-2', account2Auth)
|
||||
const settings = createSettings({
|
||||
shellStartupEnvProbeSupported: true,
|
||||
codexManagedAccounts: [
|
||||
{
|
||||
id: 'account-1',
|
||||
email: 'one@example.com',
|
||||
managedHomePath: home1,
|
||||
providerAccountId: 'acct-1',
|
||||
workspaceLabel: null,
|
||||
workspaceAccountId: 'acct-1',
|
||||
createdAt: 1,
|
||||
updatedAt: 1,
|
||||
lastAuthenticatedAt: 1
|
||||
},
|
||||
{
|
||||
id: 'account-2',
|
||||
email: secondEmail,
|
||||
managedHomePath: home2,
|
||||
providerAccountId: 'acct-2',
|
||||
workspaceLabel: null,
|
||||
workspaceAccountId: 'acct-2',
|
||||
createdAt: 2,
|
||||
updatedAt: 2,
|
||||
lastAuthenticatedAt: 2
|
||||
}
|
||||
],
|
||||
activeCodexManagedAccountId: 'account-1',
|
||||
activeCodexManagedAccountIdsByRuntime: { host: 'account-1', wsl: {} }
|
||||
})
|
||||
).toBe(home2)
|
||||
expect(store.updateSettings).not.toHaveBeenCalled()
|
||||
const store = createStore(settings)
|
||||
const { CodexRuntimeHomeService } = await import('./runtime-home-service')
|
||||
const service = new CodexRuntimeHomeService(store as never)
|
||||
|
||||
// Nothing is hot-swapped, so the still-running account-1 pane keeps seeing
|
||||
// account-1's credentials — the single-auth.json race (GAP-5) is gone.
|
||||
expect(readFileSync(join(home1, 'auth.json'), 'utf-8')).toBe(account1Auth)
|
||||
expect(readFileSync(join(home2, 'auth.json'), 'utf-8')).toBe(account2Auth)
|
||||
expect(existsSync(getRuntimeCodexAuthPath())).toBe(false)
|
||||
})
|
||||
// A pane for account-1 launches, then the user switches and a second pane
|
||||
// for account-2 launches concurrently — each gets its OWN CODEX_HOME.
|
||||
expect(service.prepareForCodexLaunch()).toBe(home1)
|
||||
settings.activeCodexManagedAccountId = 'account-2'
|
||||
settings.activeCodexManagedAccountIdsByRuntime = { host: 'account-2', wsl: {} }
|
||||
expect(service.prepareForCodexLaunch()).toBe(home2)
|
||||
expect(
|
||||
service.prepareForCodexLaunch(undefined, undefined, {
|
||||
unavailableManagedHomePath: home1
|
||||
})
|
||||
).toBe(home2)
|
||||
expect(store.updateSettings).not.toHaveBeenCalled()
|
||||
|
||||
// Nothing is hot-swapped, so the still-running account-1 pane keeps seeing
|
||||
// account-1's credentials — the single-auth.json race (GAP-5) is gone.
|
||||
expect(readFileSync(join(home1, 'auth.json'), 'utf-8')).toBe(account1Auth)
|
||||
expect(readFileSync(join(home2, 'auth.json'), 'utf-8')).toBe(account2Auth)
|
||||
expect(existsSync(getRuntimeCodexAuthPath())).toBe(false)
|
||||
}
|
||||
)
|
||||
|
||||
it('materializes resources and config into the per-account home on launch', async () => {
|
||||
writeFileSync(getSystemCodexAuthPath(), '{"account":"system"}\n', 'utf-8')
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import {
|
||||
@@ -29,6 +29,74 @@ vi.mock('node:os', async () => {
|
||||
describe('CodexAccountService.addAccountFromHome', () => {
|
||||
registerCodexAccountsTestHomes()
|
||||
|
||||
it('imports and switches personal and enterprise accounts sharing an email independently', async () => {
|
||||
vi.doMock('../codex-cli/command', () => ({ resolveCodexCommand: () => 'codex' }))
|
||||
const sourceHomes = [
|
||||
mkdtempSync(join(tmpdir(), 'orca-codex-personal-')),
|
||||
mkdtempSync(join(tmpdir(), 'orca-codex-enterprise-'))
|
||||
]
|
||||
const email = 'same@example.com'
|
||||
const credentials = ['plus', 'enterprise'].map((plan) => {
|
||||
const parsed = JSON.parse(createCodexAuthJson(email, `provider-${plan}`, `refresh-${plan}`))
|
||||
const payload = Buffer.from(
|
||||
JSON.stringify({
|
||||
email,
|
||||
'https://api.openai.com/auth': {
|
||||
chatgpt_account_id: `provider-${plan}`,
|
||||
chatgpt_plan_type: plan
|
||||
}
|
||||
})
|
||||
).toString('base64url')
|
||||
parsed.tokens.id_token = `header.${payload}.signature`
|
||||
return JSON.stringify(parsed)
|
||||
})
|
||||
|
||||
try {
|
||||
sourceHomes.forEach((home, index) => {
|
||||
writeFileSync(join(home, 'auth.json'), credentials[index], 'utf-8')
|
||||
})
|
||||
const store = createStore(createSettings())
|
||||
const runtimeHome = createRuntimeHome()
|
||||
const { CodexAccountService } = await import('./service')
|
||||
const service = new CodexAccountService(
|
||||
store as never,
|
||||
createRateLimits() as never,
|
||||
runtimeHome as never
|
||||
)
|
||||
|
||||
await service.addAccountFromHome(sourceHomes[0])
|
||||
const result = await service.addAccountFromHome(sourceHomes[1])
|
||||
const accounts = store.getSettings().codexManagedAccounts
|
||||
expect(result.accounts).toHaveLength(2)
|
||||
expect(new Set(accounts.map((account) => account.id)).size).toBe(2)
|
||||
expect(new Set(accounts.map((account) => account.managedHomePath)).size).toBe(2)
|
||||
expect(accounts.map((account) => account.email)).toEqual([email, email])
|
||||
expect(accounts.map((account) => account.workspaceLabel)).toEqual([
|
||||
'Personal (Plus)',
|
||||
'Enterprise'
|
||||
])
|
||||
expect(accounts.map((account) => account.providerAccountId)).toEqual([
|
||||
'provider-plus',
|
||||
'provider-enterprise'
|
||||
])
|
||||
|
||||
for (const account of accounts) {
|
||||
const selected = await service.selectAccount(account.id)
|
||||
expect(selected.activeAccountId).toBe(account.id)
|
||||
expect(store.getSettings().activeCodexManagedAccountIdsByRuntime?.host).toBe(account.id)
|
||||
accounts.forEach((entry, index) => {
|
||||
expect(readFileSync(join(entry.managedHomePath, 'auth.json'), 'utf-8')).toBe(
|
||||
credentials[index]
|
||||
)
|
||||
})
|
||||
}
|
||||
expect(runtimeHome.syncForCurrentSelection).toHaveBeenCalledTimes(4)
|
||||
} finally {
|
||||
sourceHomes.forEach((home) => rmSync(home, { recursive: true, force: true }))
|
||||
vi.doUnmock('../codex-cli/command')
|
||||
}
|
||||
})
|
||||
|
||||
it('registers a managed Codex account by importing an authenticated CODEX_HOME', async () => {
|
||||
vi.doMock('../codex-cli/command', () => ({ resolveCodexCommand: () => 'codex' }))
|
||||
const sourceHome = mkdtempSync(join(tmpdir(), 'orca-codex-source-'))
|
||||
|
||||
@@ -11,7 +11,8 @@ import {
|
||||
runCodexHookTrustGrantSession,
|
||||
type CodexHookTrustGrantRequest
|
||||
} from './codex-app-server-client'
|
||||
import { killCodexAppServerProcessTree, runCodexAppServerSession } from './codex-app-server-session'
|
||||
import { killCodexAppServerProcessTree } from './codex-app-server-process-tree-kill'
|
||||
import { runCodexAppServerSession } from './codex-app-server-session'
|
||||
|
||||
// Stub codex app-server speaking the same JSONL protocol: initialize →
|
||||
// initialized → hooks/list → config/batchWrite → hooks/list. Scenario-driven
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { spawn } from 'node:child_process'
|
||||
import type { ChildProcessHandle, ProcessSpec } from '../../shared/child-process/process-spec'
|
||||
import { spawnProcess } from '../../shared/child-process/run-process'
|
||||
import { normalizeHookTrustKeyForLookup } from './config-toml-trust'
|
||||
import { runCodexAppServerSession, type CodexAppServerInvocation } from './codex-app-server-session'
|
||||
|
||||
@@ -105,7 +106,12 @@ function collectHookListings(result: unknown): CodexHookListing[] {
|
||||
*/
|
||||
export async function runCodexHookTrustGrantSession(
|
||||
request: CodexHookTrustGrantRequest,
|
||||
spawnImpl: typeof spawn = spawn
|
||||
spawnImpl: (
|
||||
program: string,
|
||||
args: string[],
|
||||
options: Record<string, unknown>
|
||||
) => ChildProcessHandle = (program, args, options) =>
|
||||
spawnProcess({ program, args, ...options } as ProcessSpec)
|
||||
): Promise<CodexHookTrustGrantSessionResult> {
|
||||
return runCodexAppServerSession(
|
||||
request.invocation,
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
import { spawnProcess } from '../../shared/child-process/run-process'
|
||||
import type { ChildProcessHandle, ProcessSpec } from '../../shared/child-process/process-spec'
|
||||
import { admitProcessTreeKill } from '../../shared/child-process/process-tree-kill-gate'
|
||||
|
||||
/** Spawn seam for tests; production always goes through the hardened spawnProcess wrapper. */
|
||||
export type CodexAppServerSpawn = (
|
||||
program: string,
|
||||
args: string[],
|
||||
options: Record<string, unknown>
|
||||
) => ChildProcessHandle
|
||||
|
||||
export const spawnCodexAppServerProcess: CodexAppServerSpawn = (program, args, options) =>
|
||||
spawnProcess({ program, args, ...options } as ProcessSpec)
|
||||
|
||||
export function killCodexAppServerProcessTree(
|
||||
child: Pick<ChildProcessHandle, 'pid' | 'kill'>,
|
||||
options: { platform?: NodeJS.Platform; spawnImpl?: CodexAppServerSpawn } = {}
|
||||
): void {
|
||||
const platform = options.platform ?? process.platform
|
||||
const spawnImpl = options.spawnImpl ?? spawnCodexAppServerProcess
|
||||
if (platform === 'win32' && child.pid) {
|
||||
if (
|
||||
!admitProcessTreeKill({
|
||||
pid: child.pid,
|
||||
site: 'codex-app-server-session-deadline',
|
||||
scope: 'win-taskkill-tree'
|
||||
})
|
||||
) {
|
||||
// Refusal blocks the tree walk, not the termination: the root kill is
|
||||
// handle-addressed, so it cannot reach the recycled pid we refused.
|
||||
child.kill('SIGKILL')
|
||||
return
|
||||
}
|
||||
try {
|
||||
// Why: npm-installed Codex runs behind cmd.exe; killing only that wrapper
|
||||
// leaves the app-server child alive after a timeout or failed shutdown.
|
||||
const killer = spawnImpl('taskkill', ['/pid', String(child.pid), '/t', '/f'], {
|
||||
stdio: 'ignore',
|
||||
windowsHide: true
|
||||
})
|
||||
let fellBack = false
|
||||
const killDirectChild = (): void => {
|
||||
if (!fellBack) {
|
||||
fellBack = true
|
||||
child.kill('SIGKILL')
|
||||
}
|
||||
}
|
||||
killer.on('error', killDirectChild)
|
||||
killer.on('exit', (code) => {
|
||||
if (code !== 0) {
|
||||
killDirectChild()
|
||||
}
|
||||
})
|
||||
killer.unref()
|
||||
return
|
||||
} catch {
|
||||
// Fall through to the direct-child best effort when taskkill cannot start.
|
||||
}
|
||||
}
|
||||
if (child.pid) {
|
||||
try {
|
||||
// npm/package-manager launchers insert a shim child on POSIX. Reap its
|
||||
// direct descendants before signalling the wrapper itself.
|
||||
const descendants = spawnImpl('pkill', ['-KILL', '-P', String(child.pid)], {
|
||||
stdio: 'ignore'
|
||||
})
|
||||
// A missing pkill surfaces as an async 'error' event, and an unhandled one
|
||||
// takes down the main process.
|
||||
descendants.on('error', () => undefined)
|
||||
descendants.unref()
|
||||
} catch {
|
||||
// The direct kill below remains the fallback when pkill is unavailable.
|
||||
}
|
||||
}
|
||||
child.kill('SIGKILL')
|
||||
}
|
||||
@@ -1,9 +1,13 @@
|
||||
import { spawn, type ChildProcess, type ChildProcessWithoutNullStreams } from 'node:child_process'
|
||||
import type { ChildProcessWithoutNullStreams } from 'node:child_process'
|
||||
import { waitForProcessExitUntil } from './codex-process-exit-deadline'
|
||||
import { stderrIndicatesMissingAppServer } from './codex-app-server-capability-signal'
|
||||
import { withCliRuntimeOnPath } from '../../shared/node-cli-command-resolution'
|
||||
import {
|
||||
killCodexAppServerProcessTree,
|
||||
spawnCodexAppServerProcess,
|
||||
type CodexAppServerSpawn
|
||||
} from './codex-app-server-process-tree-kill'
|
||||
import { createCodexAppServerRecordReader } from './codex-app-server-record-reader'
|
||||
import { admitProcessTreeKill } from '../../shared/child-process/process-tree-kill-gate'
|
||||
|
||||
// Why: `codex app-server` is Orca's sanctioned RPC surface into Codex-owned
|
||||
// state (hook trust hashes, the sqlite thread index). This module owns the
|
||||
@@ -68,69 +72,6 @@ export type CodexAppServerRpc = {
|
||||
const JSON_RPC_METHOD_NOT_FOUND = -32601
|
||||
const STDERR_TAIL_MAX_BYTES = 8192
|
||||
|
||||
export function killCodexAppServerProcessTree(
|
||||
child: Pick<ChildProcess, 'pid' | 'kill'>,
|
||||
options: { platform?: NodeJS.Platform; spawnImpl?: typeof spawn } = {}
|
||||
): void {
|
||||
const platform = options.platform ?? process.platform
|
||||
const spawnImpl = options.spawnImpl ?? spawn
|
||||
if (platform === 'win32' && child.pid) {
|
||||
if (
|
||||
!admitProcessTreeKill({
|
||||
pid: child.pid,
|
||||
site: 'codex-app-server-session-deadline',
|
||||
scope: 'win-taskkill-tree'
|
||||
})
|
||||
) {
|
||||
// Refusal blocks the tree walk, not the termination: the root kill is
|
||||
// handle-addressed, so it cannot reach the recycled pid we refused.
|
||||
child.kill('SIGKILL')
|
||||
return
|
||||
}
|
||||
try {
|
||||
// Why: npm-installed Codex runs behind cmd.exe; killing only that wrapper
|
||||
// leaves the app-server child alive after a timeout or failed shutdown.
|
||||
const killer = spawnImpl('taskkill', ['/pid', String(child.pid), '/t', '/f'], {
|
||||
stdio: 'ignore',
|
||||
windowsHide: true
|
||||
})
|
||||
let fellBack = false
|
||||
const killDirectChild = (): void => {
|
||||
if (!fellBack) {
|
||||
fellBack = true
|
||||
child.kill('SIGKILL')
|
||||
}
|
||||
}
|
||||
killer.on('error', killDirectChild)
|
||||
killer.on('exit', (code) => {
|
||||
if (code !== 0) {
|
||||
killDirectChild()
|
||||
}
|
||||
})
|
||||
killer.unref()
|
||||
return
|
||||
} catch {
|
||||
// Fall through to the direct-child best effort when taskkill cannot start.
|
||||
}
|
||||
}
|
||||
if (child.pid) {
|
||||
try {
|
||||
// npm/package-manager launchers insert a shim child on POSIX. Reap its
|
||||
// direct descendants before signalling the wrapper itself.
|
||||
const descendants = spawnImpl('pkill', ['-KILL', '-P', String(child.pid)], {
|
||||
stdio: 'ignore'
|
||||
})
|
||||
// A missing pkill surfaces as an async 'error' event, and an unhandled one
|
||||
// takes down the main process.
|
||||
descendants.on('error', () => undefined)
|
||||
descendants.unref()
|
||||
} catch {
|
||||
// The direct kill below remains the fallback when pkill is unavailable.
|
||||
}
|
||||
}
|
||||
child.kill('SIGKILL')
|
||||
}
|
||||
|
||||
/** Codex answering "no such method" is the only response that proves the RPC
|
||||
* surface is absent rather than temporarily failing. */
|
||||
export function isCodexMethodNotFoundError(error: unknown): boolean {
|
||||
@@ -152,7 +93,7 @@ export function isCodexMethodNotFoundError(error: unknown): boolean {
|
||||
export async function runCodexAppServerSession<T>(
|
||||
invocation: CodexAppServerInvocation,
|
||||
body: (rpc: CodexAppServerRpc) => Promise<T>,
|
||||
spawnImpl: typeof spawn = spawn
|
||||
spawnImpl: CodexAppServerSpawn = spawnCodexAppServerProcess
|
||||
): Promise<T> {
|
||||
// Why: a default-home grant must run against the real ~/.codex, so strip an
|
||||
// inherited CODEX_HOME (envToDelete) after applying the overlay, not before.
|
||||
|
||||
@@ -784,7 +784,7 @@ describe('codex item bodies', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('leaves subagent items on the generic row until a real renderer exists', () => {
|
||||
it('drops the raw subagent item now the roster row renders it', () => {
|
||||
expect(
|
||||
codexJournalItem({
|
||||
type: 'subAgentActivity',
|
||||
@@ -793,10 +793,7 @@ describe('codex item bodies', () => {
|
||||
agentThreadId: 'thread-child',
|
||||
agentPath: '/root/list_directory'
|
||||
})
|
||||
).toMatchObject({
|
||||
handled: false,
|
||||
body: { kind: 'status', providerFrame: { kind: 'item:subAgentActivity' } }
|
||||
})
|
||||
).toMatchObject({ handled: true, body: null })
|
||||
})
|
||||
|
||||
it('drops the sleep item, which codex itself renders as nothing', () => {
|
||||
|
||||
@@ -7,3 +7,10 @@ export const MAX_CODEX_PENDING_PROMPTS = 128
|
||||
export const MAX_CODEX_IDENTITY_ENTRIES = 512
|
||||
export const MAX_CODEX_DETAIL_ENTRIES = 512
|
||||
export const MAX_CODEX_DETAIL_BYTES = 64 * 1024
|
||||
/** Spawn-group rows kept live per session, and children per row. Both bound an
|
||||
* event-accumulated map that no provider snapshot ever prunes. */
|
||||
export const MAX_CODEX_SUBAGENT_GROUPS = 32
|
||||
export const MAX_CODEX_SUBAGENTS_PER_GROUP = 64
|
||||
/** Threads whose latest token total is retained. Usage frames arrive for
|
||||
* threads that are not yet (or never become) roster children. */
|
||||
export const MAX_CODEX_TOKEN_USAGE_THREADS = 256
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
/**
|
||||
* The translator's provider-frame arms.
|
||||
*
|
||||
* Each returns null for a frame it does not own, which is the translator's
|
||||
* signal to keep looking. Split out so the translator reads as routing rather
|
||||
* than as the shape checks each arm performs.
|
||||
*/
|
||||
|
||||
import type { CodexJournalTranslationAdmission } from './codex-structured-journal-contracts'
|
||||
import { settleCodexOversizedNotification } from './codex-structured-journal-settlement'
|
||||
import {
|
||||
readCodexJournalRecord,
|
||||
readCodexJournalString
|
||||
} from './codex-structured-journal-translation-values'
|
||||
|
||||
type OversizedInput = Parameters<typeof settleCodexOversizedNotification>[0]
|
||||
|
||||
/** A notification the transport refused to carry whole: settle whatever it
|
||||
* opened rather than leaving the item mid-flight. */
|
||||
export function settleCodexOversizedNotificationFrame(input: {
|
||||
sessionId: string
|
||||
threadId: string
|
||||
kind: string
|
||||
payload: unknown
|
||||
sink: OversizedInput['sink']
|
||||
streams: OversizedInput['streams']
|
||||
activeItems: OversizedInput['activeItems']
|
||||
}): CodexJournalTranslationAdmission | null {
|
||||
if (input.kind !== 'frame:oversized-notification') {
|
||||
return null
|
||||
}
|
||||
const method = readCodexJournalString(readCodexJournalRecord(input.payload), 'method')
|
||||
return method
|
||||
? settleCodexOversizedNotification({
|
||||
sessionId: input.sessionId,
|
||||
threadId: input.threadId,
|
||||
method,
|
||||
sink: input.sink,
|
||||
streams: input.streams,
|
||||
activeItems: input.activeItems
|
||||
})
|
||||
: null
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { agentJournalItemKey } from '../../shared/agent-session-journal-item-key'
|
||||
import type { AgentSessionTurnActivity } from '../../shared/agent-session-wire'
|
||||
import type {
|
||||
AgentJournalItemBody,
|
||||
AgentJournalItemIdentity
|
||||
} from '../../shared/agent-session-journal-types'
|
||||
import { isSubagentGroupBlock } from '../../shared/native-chat-types'
|
||||
import type { StructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink'
|
||||
import { createCodexJournalTranslator } from './codex-structured-journal-translation'
|
||||
import type { CodexStructuredSessionEvent } from './codex-structured-session-adapter'
|
||||
|
||||
const SESSION_ID = 'session-1'
|
||||
const THREAD_ID = 'thread-abc'
|
||||
const TURN_ID = 'turn-1'
|
||||
|
||||
type Row = { key: string; body: AgentJournalItemBody }
|
||||
|
||||
function harness() {
|
||||
const rows: Row[] = []
|
||||
const activities: (AgentSessionTurnActivity | null)[] = []
|
||||
const sink: StructuredAgentSessionEventSink = {
|
||||
appendItem: (identity: AgentJournalItemIdentity, body) =>
|
||||
rows.push({ key: agentJournalItemKey(identity), body }),
|
||||
appendTombstone: () => {},
|
||||
publish: () => {},
|
||||
setActivity: (activity) => activities.push(activity)
|
||||
}
|
||||
const translator = createCodexJournalTranslator({
|
||||
sink,
|
||||
primaryThreadId: () => THREAD_ID,
|
||||
schedule: (run: () => void) => {
|
||||
run()
|
||||
return () => {}
|
||||
}
|
||||
})
|
||||
return { translator, rows, activities }
|
||||
}
|
||||
|
||||
function notification(method: string, params: unknown): CodexStructuredSessionEvent {
|
||||
return { type: 'notification', sessionId: SESSION_ID, threadId: THREAD_ID, method, params }
|
||||
}
|
||||
|
||||
function subagentItem(kind: string, agentThreadId: string, agentPath: string): unknown {
|
||||
return {
|
||||
turnId: TURN_ID,
|
||||
item: {
|
||||
type: 'subAgentActivity',
|
||||
id: `item-${agentThreadId}-${kind}`,
|
||||
kind,
|
||||
agentThreadId,
|
||||
agentPath
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Every activity item reaches the wire twice. */
|
||||
function deliverActivity(
|
||||
translator: ReturnType<typeof createCodexJournalTranslator>,
|
||||
params: unknown
|
||||
): void {
|
||||
translator.handle(notification('item/started', params))
|
||||
translator.handle(notification('item/completed', params))
|
||||
}
|
||||
|
||||
function rosterAgents(rows: Row[]): { id: string; state: string; tokens?: number }[] {
|
||||
const body = rows.findLast((row) => row.key.startsWith('orca:codex-subagents'))?.body
|
||||
if (!body || body.kind !== 'message') {
|
||||
return []
|
||||
}
|
||||
return body.blocks.find(isSubagentGroupBlock)?.agents ?? []
|
||||
}
|
||||
|
||||
describe('codex journal translation — subagents', () => {
|
||||
it('renders a spawn group as one roster row and no opcode-shaped duplicate', () => {
|
||||
const { translator, rows } = harness()
|
||||
|
||||
translator.handle(notification('turn/started', { turn: { id: TURN_ID } }))
|
||||
deliverActivity(translator, subagentItem('started', 'child-1', '/root/list_directory'))
|
||||
deliverActivity(translator, subagentItem('interacted', 'child-1', '/root/list_directory'))
|
||||
|
||||
expect(rosterAgents(rows)).toMatchObject([
|
||||
{ id: 'child-1', label: 'list_directory', state: 'working' }
|
||||
])
|
||||
// Four wire deliveries (two items, each sent twice) collapse to ONE roster
|
||||
// row, and none of the gray `codex · item:subAgentActivity` rows survive.
|
||||
const providerFrameKinds = rows.flatMap((row) =>
|
||||
row.body.kind === 'status' && row.body.providerFrame ? [row.body.providerFrame.kind] : []
|
||||
)
|
||||
expect(providerFrameKinds).toEqual([])
|
||||
expect(rows.filter((row) => row.key.startsWith('orca:codex-subagents'))).toHaveLength(1)
|
||||
})
|
||||
|
||||
// The roster claims the item, but claiming it must not take the turn tail with
|
||||
// it: the activity table is reached only through the publish arm, so a bare
|
||||
// return leaves the tail stuck on whatever the previous frame said.
|
||||
it('still publishes the turn tail for an item the roster claims', () => {
|
||||
const { translator, activities } = harness()
|
||||
|
||||
translator.handle(notification('turn/started', { turn: { id: TURN_ID } }))
|
||||
activities.length = 0
|
||||
deliverActivity(translator, subagentItem('started', 'child-1', '/root/read'))
|
||||
|
||||
expect(activities.at(-1)).toEqual({
|
||||
turnId: TURN_ID,
|
||||
text: 'Coordinating with another agent'
|
||||
})
|
||||
})
|
||||
|
||||
it('consumes thread/tokenUsage/updated instead of swallowing it as chrome', () => {
|
||||
const { translator, rows } = harness()
|
||||
|
||||
translator.handle(notification('turn/started', { turn: { id: TURN_ID } }))
|
||||
deliverActivity(translator, subagentItem('started', 'child-1', '/root/read'))
|
||||
translator.handle(
|
||||
notification('thread/tokenUsage/updated', {
|
||||
threadId: 'child-1',
|
||||
tokenUsage: { total: { totalTokens: 40661 } }
|
||||
})
|
||||
)
|
||||
|
||||
expect(rosterAgents(rows)).toMatchObject([{ id: 'child-1', tokens: 40661 }])
|
||||
})
|
||||
|
||||
// The QA scenario this row got wrong: three `spawn_agent` children were still
|
||||
// running when a mid-turn correction ended their turn and opened a new one.
|
||||
// They reported `completed` 57-87s later, so a turn boundary is a fact about
|
||||
// the turn and never evidence that contact with a child was lost.
|
||||
it('leaves children working when their turn ends and a newer turn opens', () => {
|
||||
const { translator, rows } = harness()
|
||||
|
||||
translator.handle(notification('turn/started', { turn: { id: TURN_ID } }))
|
||||
deliverActivity(translator, subagentItem('started', 'child-1', '/root/read_readme'))
|
||||
deliverActivity(translator, subagentItem('started', 'child-2', '/root/read_package'))
|
||||
translator.handle(notification('turn/completed', { turn: { id: TURN_ID } }))
|
||||
translator.handle(notification('turn/started', { turn: { id: 'turn-2' } }))
|
||||
|
||||
expect(rosterAgents(rows)).toMatchObject([
|
||||
{ id: 'child-1', state: 'working' },
|
||||
{ id: 'child-2', state: 'working' }
|
||||
])
|
||||
|
||||
// And the verdict a child reports after its turn ended still lands on the row.
|
||||
deliverActivity(translator, subagentItem('completed', 'child-1', '/root/read_readme'))
|
||||
|
||||
expect(rosterAgents(rows)).toMatchObject([
|
||||
{ id: 'child-1', state: 'completed' },
|
||||
{ id: 'child-2', state: 'working' }
|
||||
])
|
||||
})
|
||||
|
||||
it('sweeps every group when the provider ends', () => {
|
||||
const { translator, rows } = harness()
|
||||
|
||||
translator.handle(notification('turn/started', { turn: { id: TURN_ID } }))
|
||||
deliverActivity(translator, subagentItem('started', 'child-1', '/root/read'))
|
||||
translator.handle({
|
||||
type: 'ended',
|
||||
sessionId: SESSION_ID,
|
||||
reason: 'provider exited',
|
||||
cause: 'unexpected-exit',
|
||||
fence: 1,
|
||||
acquisitionGeneration: 'gen-1'
|
||||
} as CodexStructuredSessionEvent)
|
||||
|
||||
expect(rosterAgents(rows)).toMatchObject([{ id: 'child-1', state: 'unverifiable' }])
|
||||
})
|
||||
})
|
||||
@@ -1,4 +1,10 @@
|
||||
import { createCodexProviderActivityReader } from '../native-chat/agent-session-wire/provider-frame-activity'
|
||||
import {
|
||||
CODEX_TOKEN_USAGE_METHOD,
|
||||
readCodexNotificationThreadItem
|
||||
} from './codex-subagent-activity'
|
||||
import { CodexSubagentRoster } from './codex-subagent-roster'
|
||||
import { readCodexThreadItem } from './codex-structured-item-translation'
|
||||
import { CodexJournalGenericFrames } from './codex-structured-journal-generic-frames'
|
||||
import { CodexJournalItems } from './codex-structured-journal-items'
|
||||
import { CodexJournalPrompts } from './codex-structured-journal-prompts'
|
||||
@@ -10,16 +16,12 @@ import {
|
||||
} from './codex-structured-journal-contracts'
|
||||
import {
|
||||
settleCodexJournalSession,
|
||||
settleCodexJournalTurn,
|
||||
settleCodexOversizedNotification
|
||||
settleCodexJournalTurn
|
||||
} from './codex-structured-journal-settlement'
|
||||
import { settleCodexOversizedNotificationFrame } from './codex-structured-journal-translation-frames'
|
||||
import { restoreCodexJournalThread } from './codex-structured-journal-translation-restore'
|
||||
import { CodexJournalActiveTurns } from './codex-structured-journal-translation-turn-state'
|
||||
import { publishCodexTurnLifecycle } from './codex-structured-journal-translation-turns'
|
||||
import {
|
||||
readCodexJournalRecord,
|
||||
readCodexJournalString
|
||||
} from './codex-structured-journal-translation-values'
|
||||
import { readCodexTurnId } from './codex-structured-thread-facts'
|
||||
import type { CodexStructuredSessionEvent } from './codex-structured-session-adapter'
|
||||
|
||||
@@ -55,6 +57,11 @@ export function createCodexJournalTranslator(
|
||||
const prompts = new CodexJournalPrompts(deps, (threadId, itemId) =>
|
||||
items.detailFor(threadId, itemId)
|
||||
)
|
||||
const subagents = new CodexSubagentRoster({
|
||||
sink: deps.sink,
|
||||
primaryThreadId: () => deps.primaryThreadId?.() ?? null,
|
||||
activeTurn: (threadId) => activeTurns.current(threadId)
|
||||
})
|
||||
const flushStreams = (): CodexJournalTranslationAdmission =>
|
||||
items.streams.flush() ? CODEX_JOURNAL_ADMITTED : { accepted: false, reason: 'backpressure' }
|
||||
let readActivity = createCodexProviderActivityReader()
|
||||
@@ -118,6 +125,11 @@ export function createCodexJournalTranslator(
|
||||
if (!admission.accepted) {
|
||||
return admission
|
||||
}
|
||||
// No event will ever settle a child once the provider is gone.
|
||||
const sweep = subagents.settleSession()
|
||||
if (!sweep.accepted) {
|
||||
return sweep
|
||||
}
|
||||
readActivity = createCodexProviderActivityReader()
|
||||
deps.sink.setActivity?.(null)
|
||||
items.activeItems.clear()
|
||||
@@ -159,7 +171,30 @@ export function createCodexJournalTranslator(
|
||||
if (event.method === 'turn/completed') {
|
||||
return completeTurn(event)
|
||||
}
|
||||
if (event.method === CODEX_TOKEN_USAGE_METHOD) {
|
||||
// Classified `status-chrome`, so the generic-frame path swallows it
|
||||
// before the journal. The roster consumes it as a typed notification.
|
||||
const admission = subagents.handleTokenUsage(event.params)
|
||||
if (admission) {
|
||||
return admission
|
||||
}
|
||||
}
|
||||
if (event.method === 'item/started' || event.method === 'item/completed') {
|
||||
const subagentItem = readCodexNotificationThreadItem(event.params, readCodexThreadItem)
|
||||
// Null means the roster did not claim it; fall through to normal item
|
||||
// handling. Returning here unconditionally swallows every other item.
|
||||
const subagentAdmission = subagentItem
|
||||
? subagents.handleItem({
|
||||
threadId: event.threadId,
|
||||
turnId: readCodexTurnId(event.params) ?? activeTurns.current(event.threadId),
|
||||
item: subagentItem
|
||||
})
|
||||
: null
|
||||
if (subagentAdmission) {
|
||||
// Not a bare return: the roster claiming the item must not skip the
|
||||
// turn-tail arm, which is the only publisher of its activity copy.
|
||||
return publishActivity(event, subagentAdmission)
|
||||
}
|
||||
const translated = items.handle(event)
|
||||
return publishActivity(
|
||||
event,
|
||||
@@ -186,30 +221,25 @@ export function createCodexJournalTranslator(
|
||||
items.dispose()
|
||||
prompts.dispose()
|
||||
genericFrames.dispose()
|
||||
subagents.dispose()
|
||||
activeTurns.clear()
|
||||
}
|
||||
}
|
||||
|
||||
/** Settles the item a notification the transport refused to carry left
|
||||
* mid-flight; null when the frame is not one. */
|
||||
function settleOversizedNotification(event: {
|
||||
sessionId: string
|
||||
threadId: string
|
||||
kind: string
|
||||
payload: unknown
|
||||
}): CodexJournalTranslationAdmission | null {
|
||||
if (event.kind !== 'frame:oversized-notification') {
|
||||
return null
|
||||
}
|
||||
const method = readCodexJournalString(readCodexJournalRecord(event.payload), 'method')
|
||||
return method
|
||||
? settleCodexOversizedNotification({
|
||||
sessionId: event.sessionId,
|
||||
threadId: event.threadId,
|
||||
method,
|
||||
sink: deps.sink,
|
||||
streams: items.streams,
|
||||
activeItems: items.activeItems
|
||||
})
|
||||
: null
|
||||
return settleCodexOversizedNotificationFrame({
|
||||
...event,
|
||||
sink: deps.sink,
|
||||
streams: items.streams,
|
||||
activeItems: items.activeItems
|
||||
})
|
||||
}
|
||||
|
||||
function startTurn(event: {
|
||||
@@ -255,6 +285,12 @@ export function createCodexJournalTranslator(
|
||||
if (!turnId) {
|
||||
return CODEX_JOURNAL_ADMITTED
|
||||
}
|
||||
// The roster is deliberately NOT swept here. `spawn_agent` children outlive
|
||||
// the turn that spawned them and go on reporting into the same group, so a
|
||||
// turn boundary is no evidence contact was lost — and `turn/completed` is
|
||||
// the only turn-end notification Codex sends, so an abort cannot be told
|
||||
// apart from a clean finish either. Only `settleSession` may write
|
||||
// `unverifiable`.
|
||||
const admission = settleCodexJournalTurn({
|
||||
sink: deps.sink,
|
||||
sessionId: event.sessionId,
|
||||
|
||||
@@ -44,7 +44,8 @@ function resolverFor(
|
||||
store: { getRecord: () => value } as unknown as AgentSessionRecordStore,
|
||||
resolveWorkspacePath,
|
||||
resolveCommand: () => '/usr/local/bin/codex',
|
||||
resolveRollout
|
||||
resolveRollout,
|
||||
isWindowsProcessStartTimeAvailable: () => true
|
||||
})
|
||||
}
|
||||
|
||||
@@ -68,7 +69,8 @@ describe('codex structured launch resolution', () => {
|
||||
const resolveLaunch = createCodexStructuredLaunchResolver({
|
||||
store: { getRecord: () => record() } as unknown as AgentSessionRecordStore,
|
||||
resolveWorkspacePath: async () => String.raw`C:\workspaces\orca`,
|
||||
resolveCommand: () => command
|
||||
resolveCommand: () => command,
|
||||
isWindowsProcessStartTimeAvailable: () => true
|
||||
})
|
||||
|
||||
await expect(resolveLaunch({ identity: IDENTITY })).resolves.toMatchObject({
|
||||
@@ -78,6 +80,22 @@ describe('codex structured launch resolution', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('fails closed before resolving a Windows launch without creation-time proof', async () => {
|
||||
await withPlatform('win32', async () => {
|
||||
const resolveWorkspacePath = vi.fn(async () => String.raw`C:\workspaces\orca`)
|
||||
const resolveLaunch = createCodexStructuredLaunchResolver({
|
||||
store: { getRecord: () => record() } as unknown as AgentSessionRecordStore,
|
||||
resolveWorkspacePath,
|
||||
isWindowsProcessStartTimeAvailable: () => false
|
||||
})
|
||||
|
||||
await expect(resolveLaunch({ identity: IDENTITY })).rejects.toThrow(
|
||||
'Windows process creation-time proof'
|
||||
)
|
||||
expect(resolveWorkspacePath).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
it('resumes the last thread this session actually proved, not one a caller names', async () => {
|
||||
const launch = await resolverFor(
|
||||
record({
|
||||
|
||||
@@ -13,6 +13,7 @@ import { resolveCodexCommand } from '../codex-cli/command'
|
||||
import type { AgentSessionRecordStore } from '../runtime/agent-session-record-store'
|
||||
import type { CodexStructuredLaunch } from './codex-structured-session-adapter'
|
||||
import { resolvePinnedCodexRolloutProof } from './codex-tui-rollout-proof'
|
||||
import { isWindowsProcessStartTimeAvailable } from '../windows/windows-process-table'
|
||||
|
||||
export type CodexStructuredLaunchResolverDeps = {
|
||||
store: AgentSessionRecordStore
|
||||
@@ -24,6 +25,8 @@ export type CodexStructuredLaunchResolverDeps = {
|
||||
/** Fresh shell/configured environment for this spawn; never written to the session record. */
|
||||
resolveEnvironment?: () => Promise<NodeJS.ProcessEnv>
|
||||
resolveRollout?: typeof resolvePinnedCodexRolloutProof
|
||||
/** Test seam for the host capability; production uses the native process table. */
|
||||
isWindowsProcessStartTimeAvailable?: () => boolean
|
||||
}
|
||||
|
||||
export function createCodexStructuredLaunchResolver(
|
||||
@@ -46,6 +49,13 @@ export function createCodexStructuredLaunchResolver(
|
||||
`codex structured sessions run on the local host, not ${location.executionHostId}`
|
||||
)
|
||||
}
|
||||
// Refuse before resolving launch data; a PID alone cannot prove Windows ownership.
|
||||
if (
|
||||
process.platform === 'win32' &&
|
||||
!(deps.isWindowsProcessStartTimeAvailable ?? isWindowsProcessStartTimeAvailable)()
|
||||
) {
|
||||
throw new Error('codex structured sessions require Windows process creation-time proof')
|
||||
}
|
||||
if (accountHome.variable !== 'CODEX_HOME') {
|
||||
throw new Error(`codex sessions pin CODEX_HOME, not ${accountHome.variable}`)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import type { AgentSessionExecutionLocation } from '../../shared/agent-session-record'
|
||||
import { supportsCodexStructuredLocation } from './codex-structured-location-support'
|
||||
|
||||
const LOCAL_WINDOWS_LOCATION: AgentSessionExecutionLocation = {
|
||||
executionHostId: 'local',
|
||||
wslDistro: null,
|
||||
workspaceId: 'workspace-1',
|
||||
workspaceKind: 'folder'
|
||||
}
|
||||
|
||||
const WSL_WINDOWS_LOCATION: AgentSessionExecutionLocation = {
|
||||
...LOCAL_WINDOWS_LOCATION,
|
||||
wslDistro: 'Ubuntu'
|
||||
}
|
||||
|
||||
function withPlatform<T>(platform: NodeJS.Platform, run: () => T): T {
|
||||
const original = process.platform
|
||||
Object.defineProperty(process, 'platform', { configurable: true, value: platform })
|
||||
try {
|
||||
return run()
|
||||
} finally {
|
||||
Object.defineProperty(process, 'platform', { configurable: true, value: original })
|
||||
}
|
||||
}
|
||||
|
||||
describe('Codex structured location support', () => {
|
||||
it('uses the injected Windows identity capability for location admission', () => {
|
||||
let proofAvailable = false
|
||||
withPlatform('win32', () => {
|
||||
expect(supportsCodexStructuredLocation(LOCAL_WINDOWS_LOCATION, () => proofAvailable)).toBe(
|
||||
false
|
||||
)
|
||||
proofAvailable = true
|
||||
expect(supportsCodexStructuredLocation(LOCAL_WINDOWS_LOCATION, () => proofAvailable)).toBe(
|
||||
true
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects WSL locations while retaining native folder support on Windows', () => {
|
||||
withPlatform('win32', () => {
|
||||
expect(supportsCodexStructuredLocation(WSL_WINDOWS_LOCATION, () => true)).toBe(false)
|
||||
expect(supportsCodexStructuredLocation(LOCAL_WINDOWS_LOCATION, () => true)).toBe(true)
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -2,10 +2,14 @@ import { LOCAL_EXECUTION_HOST_ID } from '../../shared/execution-host'
|
||||
import type { AgentSessionExecutionLocation } from '../../shared/agent-session-record'
|
||||
import { isWindowsProcessStartTimeAvailable } from '../windows/windows-process-table'
|
||||
|
||||
export function supportsCodexStructuredLocation(location: AgentSessionExecutionLocation): boolean {
|
||||
export function supportsCodexStructuredLocation(
|
||||
location: AgentSessionExecutionLocation,
|
||||
// Injected by the adapter, which owns this dep for every other Codex gate too.
|
||||
hasWindowsProcessStartTimeProof: () => boolean = isWindowsProcessStartTimeAvailable
|
||||
): boolean {
|
||||
return (
|
||||
location.executionHostId === LOCAL_EXECUTION_HOST_ID &&
|
||||
location.wslDistro === null &&
|
||||
(process.platform !== 'win32' || isWindowsProcessStartTimeAvailable())
|
||||
(process.platform !== 'win32' || hasWindowsProcessStartTimeProof())
|
||||
)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,334 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { CodexAppServerRequestError } from './codex-app-server-connection'
|
||||
import type { CodexSession } from './codex-structured-session-state'
|
||||
import { recoverCodexRewind, rewindCodexSession } from './codex-structured-rewind'
|
||||
import { AGENT_SESSION_HISTORY_MAX_PAGE_BYTES } from '../native-chat/agent-session-wire/agent-session-history-page-bounds'
|
||||
import { openCodexThread } from './codex-structured-thread-open'
|
||||
|
||||
function fixture(reverted = true) {
|
||||
const request = vi.fn(async (method: string): Promise<unknown> => {
|
||||
if (method === 'thread/read') {
|
||||
return { thread: { id: 'thread', historyMode: 'paginated', status: { type: 'idle' } } }
|
||||
}
|
||||
if (method === 'thread/revert') {
|
||||
reverted = true
|
||||
return {
|
||||
thread: { id: 'thread', turns: [] },
|
||||
turnsBackwardsCursor: 'turn-cursor',
|
||||
itemsBackwardsCursor: 'item-cursor'
|
||||
}
|
||||
}
|
||||
if (method === 'thread/turns/list') {
|
||||
return { data: [...(reverted ? [] : [{ id: 'drop' }]), { id: 'kept' }], nextCursor: null }
|
||||
}
|
||||
return {
|
||||
data: [
|
||||
{
|
||||
turnId: 'kept',
|
||||
item: {
|
||||
id: 'item-1',
|
||||
type: 'userMessage',
|
||||
content: [{ type: 'text', text: 'kept prompt' }]
|
||||
}
|
||||
}
|
||||
],
|
||||
nextCursor: null
|
||||
}
|
||||
})
|
||||
const session = {
|
||||
connection: { request },
|
||||
threadId: 'thread',
|
||||
fence: 2,
|
||||
ended: false,
|
||||
historyMode: 'paginated',
|
||||
activeTurnIds: new Set()
|
||||
} as unknown as CodexSession
|
||||
return { request, session }
|
||||
}
|
||||
|
||||
describe('Codex rewind', () => {
|
||||
it('recovers verified history from fresh cursors without repeating revert', async () => {
|
||||
const { session, request } = fixture()
|
||||
expect(await recoverCodexRewind(session, { fence: 2, beforeTurnId: 'drop' })).toMatchObject({
|
||||
ok: true,
|
||||
items: [{ body: { kind: 'message', blocks: [{ type: 'text', text: 'kept prompt' }] } }]
|
||||
})
|
||||
expect(request.mock.calls.map(([method]) => method)).toEqual([
|
||||
'thread/read',
|
||||
'thread/turns/list',
|
||||
'thread/items/list'
|
||||
])
|
||||
for (const method of ['thread/turns/list', 'thread/items/list']) {
|
||||
expect(request).toHaveBeenCalledWith(
|
||||
method,
|
||||
expect.objectContaining({ cursor: null, sortDirection: 'desc' }),
|
||||
expect.anything()
|
||||
)
|
||||
}
|
||||
})
|
||||
it('recognizes an unapplied rewind from the still-present target', async () => {
|
||||
const { session, request } = fixture()
|
||||
const original = request.getMockImplementation()!
|
||||
request.mockImplementation(async (method) =>
|
||||
method === 'thread/turns/list'
|
||||
? { data: [{ id: 'drop' }, { id: 'kept' }], nextCursor: null }
|
||||
: original(method)
|
||||
)
|
||||
expect(await recoverCodexRewind(session, { fence: 2, beforeTurnId: 'drop' })).toEqual({
|
||||
ok: false,
|
||||
reason: 'provider-refused'
|
||||
})
|
||||
expect(request.mock.calls.some(([method]) => method === 'thread/revert')).toBe(false)
|
||||
})
|
||||
it.each(['cycle', 'pages', 'entries', 'bytes'] as const)(
|
||||
'bounds recovery by %s and never returns partial history',
|
||||
async (limit) => {
|
||||
const { session, request } = fixture()
|
||||
const original = request.getMockImplementation()!
|
||||
let pages = 0
|
||||
request.mockImplementation(async (method) => {
|
||||
if (method !== 'thread/turns/list') {
|
||||
return original(method)
|
||||
}
|
||||
pages++
|
||||
if (limit === 'entries') {
|
||||
return {
|
||||
data: Array.from({ length: 1025 }, (_, i) => ({ id: String(i) })),
|
||||
nextCursor: null
|
||||
}
|
||||
}
|
||||
if (limit === 'bytes') {
|
||||
return {
|
||||
data: [],
|
||||
padding: 'x'.repeat(AGENT_SESSION_HISTORY_MAX_PAGE_BYTES),
|
||||
nextCursor: null
|
||||
}
|
||||
}
|
||||
return { data: [], nextCursor: limit === 'cycle' ? 'repeated' : String(pages) }
|
||||
})
|
||||
await expect(recoverCodexRewind(session, { fence: 2, beforeTurnId: 'drop' })).rejects.toThrow(
|
||||
'history-limit'
|
||||
)
|
||||
expect(pages).toBeLessThanOrEqual(100)
|
||||
expect(request.mock.calls.some(([method]) => method === 'thread/revert')).toBe(false)
|
||||
}
|
||||
)
|
||||
it('keeps an interrupted recovery retryable with read-only requests', async () => {
|
||||
const { session, request } = fixture()
|
||||
const original = request.getMockImplementation()!
|
||||
request.mockImplementation(async (method) => {
|
||||
if (method === 'thread/items/list') {
|
||||
throw new Error('offline')
|
||||
}
|
||||
return original(method)
|
||||
})
|
||||
await expect(recoverCodexRewind(session, { fence: 2, beforeTurnId: 'drop' })).rejects.toThrow(
|
||||
'offline'
|
||||
)
|
||||
request.mockImplementation(original)
|
||||
expect(await recoverCodexRewind(session, { fence: 2, beforeTurnId: 'drop' })).toMatchObject({
|
||||
ok: true
|
||||
})
|
||||
expect(request.mock.calls.some(([method]) => method === 'thread/revert')).toBe(false)
|
||||
})
|
||||
it('refuses activity arriving during recovery hydration', async () => {
|
||||
const { session, request } = fixture()
|
||||
const original = request.getMockImplementation()!
|
||||
request.mockImplementation(async (method) => {
|
||||
if (method === 'thread/items/list') {
|
||||
session.activeTurnIds!.add('racing-turn')
|
||||
}
|
||||
return original(method)
|
||||
})
|
||||
expect(await recoverCodexRewind(session, { fence: 2, beforeTurnId: 'drop' })).toEqual({
|
||||
ok: false,
|
||||
reason: 'busy'
|
||||
})
|
||||
})
|
||||
it('uses native revert and reads both retained indexes despite empty response turns', async () => {
|
||||
const { session, request } = fixture(false)
|
||||
const onPrepared = vi.fn<NonNullable<Parameters<typeof rewindCodexSession>[1]['onPrepared']>>(
|
||||
async (items) => {
|
||||
expect(items).toMatchObject([{ identity: { turnId: 'kept' } }])
|
||||
expect(request.mock.calls.some(([method]) => method === 'thread/revert')).toBe(false)
|
||||
}
|
||||
)
|
||||
expect(
|
||||
await rewindCodexSession(session, { fence: 2, beforeTurnId: 'drop', onPrepared })
|
||||
).toMatchObject({
|
||||
ok: true,
|
||||
items: [{ body: { kind: 'message' } }]
|
||||
})
|
||||
expect(onPrepared).toHaveBeenCalledTimes(1)
|
||||
expect(request).toHaveBeenCalledWith(
|
||||
'thread/revert',
|
||||
{ threadId: 'thread', beforeTurnId: 'drop' },
|
||||
{ timeoutMs: undefined }
|
||||
)
|
||||
expect(request).toHaveBeenCalledWith(
|
||||
'thread/turns/list',
|
||||
expect.objectContaining({ cursor: 'turn-cursor', sortDirection: 'desc' }),
|
||||
expect.anything()
|
||||
)
|
||||
expect(request).toHaveBeenCalledWith(
|
||||
'thread/items/list',
|
||||
expect.objectContaining({ cursor: 'item-cursor', sortDirection: 'desc' }),
|
||||
expect.anything()
|
||||
)
|
||||
})
|
||||
it('refuses a known legacy thread before making a request', async () => {
|
||||
const { session, request } = fixture()
|
||||
session.historyMode = 'legacy'
|
||||
expect(await rewindCodexSession(session, { fence: 2, beforeTurnId: 'drop' })).toEqual({
|
||||
ok: false,
|
||||
reason: 'history-not-paginated'
|
||||
})
|
||||
expect(request).not.toHaveBeenCalled()
|
||||
})
|
||||
it('refuses history exceeding hydration capacity before mutating the provider', async () => {
|
||||
const { session, request } = fixture(false)
|
||||
const original = request.getMockImplementation()!
|
||||
const turns = Array.from({ length: 600 }, (_, i) => String(i))
|
||||
request.mockImplementation(async (method) => {
|
||||
if (method === 'thread/turns/list') {
|
||||
return { data: [{ id: 'drop' }, ...turns.map((id) => ({ id }))], nextCursor: null }
|
||||
}
|
||||
if (method === 'thread/items/list') {
|
||||
return {
|
||||
data: turns.map((turnId) => ({
|
||||
turnId,
|
||||
item: { id: turnId, type: 'userMessage', content: [{ type: 'text', text: 'x' }] }
|
||||
})),
|
||||
nextCursor: null
|
||||
}
|
||||
}
|
||||
return original(method)
|
||||
})
|
||||
const onReverted = vi.fn()
|
||||
expect(
|
||||
await rewindCodexSession(session, { fence: 2, beforeTurnId: 'drop', onReverted })
|
||||
).toEqual({ ok: false, reason: 'history-limit' })
|
||||
expect(onReverted).not.toHaveBeenCalled()
|
||||
expect(request.mock.calls.some(([method]) => method === 'thread/revert')).toBe(false)
|
||||
})
|
||||
it('refuses a missing target before mutation', async () => {
|
||||
const { session, request } = fixture()
|
||||
expect(await rewindCodexSession(session, { fence: 2, beforeTurnId: 'missing' })).toEqual({
|
||||
ok: false,
|
||||
reason: 'invalid-target'
|
||||
})
|
||||
expect(request.mock.calls.some(([method]) => method === 'thread/revert')).toBe(false)
|
||||
})
|
||||
it('rechecks provider idleness after preflight hydration', async () => {
|
||||
const { session, request } = fixture(false)
|
||||
const original = request.getMockImplementation()!
|
||||
let reads = 0
|
||||
request.mockImplementation(async (method) => {
|
||||
if (method === 'thread/read' && ++reads === 2) {
|
||||
return { thread: { id: 'thread', status: { type: 'active' } } }
|
||||
}
|
||||
return original(method)
|
||||
})
|
||||
expect(await rewindCodexSession(session, { fence: 2, beforeTurnId: 'drop' })).toEqual({
|
||||
ok: false,
|
||||
reason: 'busy'
|
||||
})
|
||||
expect(request.mock.calls.some(([method]) => method === 'thread/revert')).toBe(false)
|
||||
})
|
||||
it('maps native legacy refusal without exposing provider text or falling back', async () => {
|
||||
const { session, request } = fixture(false)
|
||||
const original = request.getMockImplementation()!
|
||||
request.mockImplementation(async (method) => {
|
||||
if (method === 'thread/read') {
|
||||
return { thread: { id: 'thread', status: { type: 'idle' } } }
|
||||
}
|
||||
if (method !== 'thread/revert') {
|
||||
return original(method)
|
||||
}
|
||||
throw new CodexAppServerRequestError(
|
||||
'thread/revert',
|
||||
-32600,
|
||||
'thread/revert only supports paginated threads'
|
||||
)
|
||||
})
|
||||
expect(await rewindCodexSession(session, { fence: 2, beforeTurnId: 'drop' })).toEqual({
|
||||
ok: false,
|
||||
reason: 'history-not-paginated'
|
||||
})
|
||||
expect(request.mock.calls.map(([method]) => method)).toEqual([
|
||||
'thread/read',
|
||||
'thread/turns/list',
|
||||
'thread/items/list',
|
||||
'thread/read',
|
||||
'thread/revert'
|
||||
])
|
||||
})
|
||||
it('refuses activity arriving during the preflight await', async () => {
|
||||
const { session, request } = fixture()
|
||||
request.mockImplementationOnce(async () => {
|
||||
session.activeTurnIds!.add('racing-turn')
|
||||
return { thread: { id: 'thread', status: { type: 'idle' } } }
|
||||
})
|
||||
expect(await rewindCodexSession(session, { fence: 2, beforeTurnId: 'drop' })).toEqual({
|
||||
ok: false,
|
||||
reason: 'busy'
|
||||
})
|
||||
expect(request).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
it('treats hydration failure after revert as unknown and never retries revert', async () => {
|
||||
const { session, request } = fixture(false)
|
||||
const original = request.getMockImplementation()!
|
||||
let reverted = false
|
||||
request.mockImplementation(async (method) => {
|
||||
if (method === 'thread/revert') {
|
||||
reverted = true
|
||||
}
|
||||
if (method === 'thread/items/list' && reverted) {
|
||||
throw new Error('offline')
|
||||
}
|
||||
return original(method)
|
||||
})
|
||||
await expect(rewindCodexSession(session, { fence: 2, beforeTurnId: 'drop' })).rejects.toThrow(
|
||||
'offline'
|
||||
)
|
||||
expect(request.mock.calls.filter(([method]) => method === 'thread/revert')).toHaveLength(1)
|
||||
})
|
||||
it('captures history mode at both start and resume without changing defaults', async () => {
|
||||
for (const resumeThreadId of [null, 'thread']) {
|
||||
const request = vi.fn(async (_method: string, _params?: unknown) => ({
|
||||
thread: { id: 'thread', historyMode: 'legacy' }
|
||||
}))
|
||||
expect(
|
||||
await openCodexThread({ request }, { cwd: '/workspace', resumeThreadId }, 10)
|
||||
).toMatchObject({ historyMode: 'legacy' })
|
||||
expect(request.mock.calls[0]?.[1]).not.toHaveProperty('historyMode')
|
||||
}
|
||||
})
|
||||
it('rejects post-revert history missing an item within a retained turn', async () => {
|
||||
const { session, request } = fixture(false)
|
||||
const original = request.getMockImplementation()!
|
||||
let reverted = false
|
||||
request.mockImplementation(async (method) => {
|
||||
if (method === 'thread/revert') {
|
||||
reverted = true
|
||||
}
|
||||
if (method === 'thread/items/list' && !reverted) {
|
||||
return {
|
||||
data: [2, 1].map((i) => ({
|
||||
turnId: 'kept',
|
||||
item: {
|
||||
id: `item-${i}`,
|
||||
type: 'userMessage',
|
||||
content: [{ type: 'text', text: `prompt ${i}` }]
|
||||
}
|
||||
})),
|
||||
nextCursor: null
|
||||
}
|
||||
}
|
||||
return original(method)
|
||||
})
|
||||
await expect(rewindCodexSession(session, { fence: 2, beforeTurnId: 'drop' })).rejects.toThrow(
|
||||
'proof-mismatch'
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,309 @@
|
||||
import { readCodexThreadId, readCodexTurnId } from './codex-structured-thread-facts'
|
||||
import { agentJournalItemKey } from '../../shared/agent-session-journal-item-key'
|
||||
import type { StructuredAgentSessionAdapter } from '../native-chat/agent-session-wire/structured-agent-session-adapter'
|
||||
import { createCodexJournalTranslator } from './codex-structured-journal-translation'
|
||||
import { CODEX_RESTORE_MAX_OPERATIONS } from './codex-structured-journal-translation-restore'
|
||||
import type {
|
||||
AgentJournalItemBody,
|
||||
AgentJournalItemIdentity
|
||||
} from '../../shared/agent-session-journal-types'
|
||||
import { AGENT_SESSION_HISTORY_MAX_LIMIT } from '../../shared/agent-session-wire'
|
||||
import { AGENT_SESSION_HISTORY_MAX_PAGE_BYTES } from '../native-chat/agent-session-wire/agent-session-history-page-bounds'
|
||||
import { isCodexAppServerRequestError } from './codex-app-server-connection'
|
||||
import type { CodexSession } from './codex-structured-session-state'
|
||||
|
||||
const MAX_PAGES = 100
|
||||
const MAX_ENTRIES = CODEX_RESTORE_MAX_OPERATIONS
|
||||
|
||||
class CodexRewindTargetRetainedError extends Error {}
|
||||
class CodexRewindTargetMissingError extends Error {}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new Error('agent_session_rewind:invalid-provider-response')
|
||||
}
|
||||
return value as Record<string, unknown>
|
||||
}
|
||||
|
||||
function cursor(value: unknown): string | null {
|
||||
if (value === null || (typeof value === 'string' && value.length > 0)) {
|
||||
return value
|
||||
}
|
||||
throw new Error('agent_session_rewind:invalid-provider-cursor')
|
||||
}
|
||||
|
||||
/** Read both indexes to completion before accepting the retained history. */
|
||||
export async function verifyCodexRevertedHistory(
|
||||
session: Pick<CodexSession, 'connection' | 'threadId'>,
|
||||
reply: Record<string, unknown>,
|
||||
beforeTurnId: string,
|
||||
timeoutMs?: number,
|
||||
targetPresence: 'absent' | 'present' = 'absent'
|
||||
): Promise<{ identity: AgentJournalItemIdentity; body: AgentJournalItemBody }[]> {
|
||||
let bytes = 0
|
||||
let entries = 0
|
||||
const turns = new Map<string, { id: string; items: unknown[] }>()
|
||||
for (const [method, firstCursor] of [
|
||||
['thread/turns/list', cursor(reply.turnsBackwardsCursor)],
|
||||
['thread/items/list', cursor(reply.itemsBackwardsCursor)]
|
||||
] as const) {
|
||||
let next = firstCursor
|
||||
const seen = new Set<string>()
|
||||
for (let page = 0; ; page += 1) {
|
||||
if (page >= MAX_PAGES || (next !== null && seen.has(next))) {
|
||||
throw new Error('agent_session_rewind:history-limit')
|
||||
}
|
||||
if (next !== null) {
|
||||
seen.add(next)
|
||||
}
|
||||
const result = record(
|
||||
await session.connection.request(
|
||||
method,
|
||||
{
|
||||
threadId: session.threadId,
|
||||
cursor: next,
|
||||
sortDirection: 'desc',
|
||||
limit: AGENT_SESSION_HISTORY_MAX_LIMIT
|
||||
},
|
||||
{ timeoutMs }
|
||||
)
|
||||
)
|
||||
if (!Array.isArray(result.data)) {
|
||||
throw new Error('agent_session_rewind:invalid-provider-page')
|
||||
}
|
||||
bytes += Buffer.byteLength(JSON.stringify(result), 'utf8')
|
||||
entries += result.data.length
|
||||
if (bytes > AGENT_SESSION_HISTORY_MAX_PAGE_BYTES || entries > MAX_ENTRIES) {
|
||||
throw new Error('agent_session_rewind:history-limit')
|
||||
}
|
||||
for (const raw of result.data) {
|
||||
const item = record(raw)
|
||||
const turnId = method === 'thread/turns/list' ? item.id : item.turnId
|
||||
if (turnId === beforeTurnId && targetPresence === 'absent') {
|
||||
throw new CodexRewindTargetRetainedError('agent_session_rewind:target-retained')
|
||||
}
|
||||
if (typeof turnId !== 'string' || !turnId) {
|
||||
throw new Error('agent_session_rewind:invalid-retained-turn')
|
||||
}
|
||||
if (method === 'thread/turns/list') {
|
||||
if (turns.has(turnId)) {
|
||||
throw new Error('agent_session_rewind:duplicate-retained-turn')
|
||||
}
|
||||
turns.set(turnId, { id: turnId, items: [] })
|
||||
} else {
|
||||
const turn = turns.get(turnId)
|
||||
if (!turn) {
|
||||
throw new Error('agent_session_rewind:foreign-retained-item')
|
||||
}
|
||||
turn.items.push(record(item.item))
|
||||
}
|
||||
}
|
||||
next = cursor(result.nextCursor)
|
||||
if (next === null) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if (targetPresence === 'present' && !turns.has(beforeTurnId)) {
|
||||
throw new CodexRewindTargetMissingError('agent_session_rewind:target-missing')
|
||||
}
|
||||
const items = new Map<
|
||||
string,
|
||||
{ identity: AgentJournalItemIdentity; body: AgentJournalItemBody }
|
||||
>()
|
||||
const translator = createCodexJournalTranslator({
|
||||
sink: {
|
||||
appendItem: (identity, body) => {
|
||||
items.set(agentJournalItemKey(identity), { identity, body })
|
||||
},
|
||||
appendTombstone: (identity) => {
|
||||
items.delete(agentJournalItemKey(identity))
|
||||
},
|
||||
publish: () => {}
|
||||
},
|
||||
primaryThreadId: () => session.threadId
|
||||
})
|
||||
try {
|
||||
const chronological = [...turns.values()].toReversed()
|
||||
const retained =
|
||||
targetPresence === 'present'
|
||||
? chronological.slice(
|
||||
0,
|
||||
chronological.findIndex((turn) => turn.id === beforeTurnId)
|
||||
)
|
||||
: chronological
|
||||
const admission = translator.restoreThread(session.threadId, {
|
||||
turns: retained.map((turn) => ({ ...turn, items: turn.items.toReversed() }))
|
||||
})
|
||||
if (!admission.accepted) {
|
||||
throw new Error('agent_session_rewind:history-unreadable')
|
||||
}
|
||||
return [...items.values()]
|
||||
} finally {
|
||||
translator.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
async function preflightCodexRewind(
|
||||
session: CodexSession,
|
||||
fence: number,
|
||||
timeoutMs?: number
|
||||
): Promise<
|
||||
{ ok: true } | { ok: false; reason: 'invalid-target' | 'history-not-paginated' | 'busy' }
|
||||
> {
|
||||
if (session.fence !== fence || session.ended) {
|
||||
return { ok: false, reason: 'invalid-target' }
|
||||
}
|
||||
if (session.historyMode === 'legacy') {
|
||||
return { ok: false, reason: 'history-not-paginated' }
|
||||
}
|
||||
if (session.activeTurnIds?.size || session.dispatchPending) {
|
||||
return { ok: false, reason: 'busy' }
|
||||
}
|
||||
const metadata = record(
|
||||
await session.connection.request(
|
||||
'thread/read',
|
||||
{ threadId: session.threadId, includeTurns: false },
|
||||
{ timeoutMs }
|
||||
)
|
||||
)
|
||||
const thread = record(metadata.thread)
|
||||
if (thread.id !== session.threadId) {
|
||||
return { ok: false, reason: 'invalid-target' }
|
||||
}
|
||||
if (thread.historyMode === 'legacy') {
|
||||
session.historyMode = 'legacy'
|
||||
return { ok: false, reason: 'history-not-paginated' }
|
||||
}
|
||||
if (
|
||||
record(thread.status).type !== 'idle' ||
|
||||
session.activeTurnIds?.size ||
|
||||
session.dispatchPending
|
||||
) {
|
||||
return { ok: false, reason: 'busy' }
|
||||
}
|
||||
if (session.fence !== fence || session.ended) {
|
||||
return { ok: false, reason: 'invalid-target' }
|
||||
}
|
||||
return { ok: true }
|
||||
}
|
||||
|
||||
export async function recoverCodexRewind(
|
||||
session: CodexSession,
|
||||
input: { fence: number; beforeTurnId: string },
|
||||
timeoutMs?: number
|
||||
): ReturnType<NonNullable<StructuredAgentSessionAdapter['recoverRewind']>> {
|
||||
const admission = await preflightCodexRewind(session, input.fence, timeoutMs)
|
||||
if (!admission.ok) {
|
||||
return admission
|
||||
}
|
||||
try {
|
||||
const items = await verifyCodexRevertedHistory(
|
||||
session,
|
||||
{ turnsBackwardsCursor: null, itemsBackwardsCursor: null },
|
||||
input.beforeTurnId,
|
||||
timeoutMs
|
||||
)
|
||||
if (session.fence !== input.fence || session.ended) {
|
||||
return { ok: false, reason: 'invalid-target' }
|
||||
}
|
||||
if (session.activeTurnIds?.size || session.dispatchPending) {
|
||||
return { ok: false, reason: 'busy' }
|
||||
}
|
||||
return { ok: true, items }
|
||||
} catch (error) {
|
||||
if (error instanceof CodexRewindTargetRetainedError) {
|
||||
return { ok: false, reason: 'provider-refused' }
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
export async function rewindCodexSession(
|
||||
session: CodexSession,
|
||||
input: Omit<Parameters<NonNullable<StructuredAgentSessionAdapter['rewind']>>[0], 'sessionId'>,
|
||||
timeoutMs?: number
|
||||
): ReturnType<NonNullable<StructuredAgentSessionAdapter['rewind']>> {
|
||||
const admission = await preflightCodexRewind(session, input.fence, timeoutMs)
|
||||
if (!admission.ok) {
|
||||
return admission
|
||||
}
|
||||
let expectedItems: Set<string>
|
||||
try {
|
||||
const retained = await verifyCodexRevertedHistory(
|
||||
session,
|
||||
{ turnsBackwardsCursor: null, itemsBackwardsCursor: null },
|
||||
input.beforeTurnId,
|
||||
timeoutMs,
|
||||
'present'
|
||||
)
|
||||
expectedItems = new Set(retained.map(({ identity }) => agentJournalItemKey(identity)))
|
||||
await input.onPrepared?.(retained)
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false,
|
||||
reason:
|
||||
error instanceof CodexRewindTargetMissingError
|
||||
? 'invalid-target'
|
||||
: error instanceof Error && error.message === 'agent_session_rewind:history-limit'
|
||||
? 'history-limit'
|
||||
: 'provider-refused'
|
||||
}
|
||||
}
|
||||
const current = await preflightCodexRewind(session, input.fence, timeoutMs)
|
||||
if (!current.ok) {
|
||||
return current
|
||||
}
|
||||
let result: unknown
|
||||
try {
|
||||
result = await session.connection.request(
|
||||
'thread/revert',
|
||||
{
|
||||
threadId: session.threadId,
|
||||
beforeTurnId: input.beforeTurnId
|
||||
},
|
||||
{ timeoutMs }
|
||||
)
|
||||
} catch (error) {
|
||||
if (isCodexAppServerRequestError(error)) {
|
||||
if (error.message === 'thread/revert only supports paginated threads') {
|
||||
session.historyMode = 'legacy'
|
||||
return { ok: false, reason: 'history-not-paginated' }
|
||||
}
|
||||
if (error.code === -32601) {
|
||||
return { ok: false, reason: 'unsupported' }
|
||||
}
|
||||
}
|
||||
throw error
|
||||
}
|
||||
const reply = record(result)
|
||||
if (record(reply.thread).id !== session.threadId) {
|
||||
throw new Error('agent_session_rewind:foreign-thread')
|
||||
}
|
||||
await input.onReverted?.()
|
||||
const items = await verifyCodexRevertedHistory(session, reply, input.beforeTurnId, timeoutMs)
|
||||
if (
|
||||
items.length !== expectedItems.size ||
|
||||
items.some(({ identity }) => !expectedItems.has(agentJournalItemKey(identity)))
|
||||
) {
|
||||
throw new Error('agent_session_rewind:proof-mismatch')
|
||||
}
|
||||
return { ok: true, items }
|
||||
}
|
||||
|
||||
export function observeCodexRewindActivity(
|
||||
session: CodexSession,
|
||||
method: string,
|
||||
params: unknown
|
||||
): void {
|
||||
if ((readCodexThreadId(params) ?? session.threadId) !== session.threadId) {
|
||||
return
|
||||
}
|
||||
const turnId = readCodexTurnId(params)
|
||||
if (turnId && method === 'turn/started') {
|
||||
session.activeTurnIds?.add(turnId)
|
||||
}
|
||||
if (turnId && method === 'turn/completed') {
|
||||
session.activeTurnIds?.delete(turnId)
|
||||
}
|
||||
}
|
||||
@@ -193,6 +193,8 @@ export async function acquireCodexStructuredSession(input: {
|
||||
...codexSessionLifecycle(acquireInput.fence, acquired.acquisitionGeneration as string),
|
||||
threadId: opened.threadId,
|
||||
historyPath: opened.historyPath,
|
||||
historyMode: opened.historyMode,
|
||||
activeTurnIds: new Set(),
|
||||
prompts: acquisition.prompts,
|
||||
options: restoredCodexSessionOptions(acquireInput.options),
|
||||
reportedOptions: reportedCodexThreadOptions(opened),
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import * as codexRewind from './codex-structured-rewind'
|
||||
import type {
|
||||
AgentJournalMessageItem,
|
||||
AgentSessionJournalIdentity
|
||||
@@ -84,7 +85,8 @@ export class CodexStructuredSessionAdapter implements StructuredAgentSessionAdap
|
||||
})
|
||||
}
|
||||
|
||||
supportsLocation = supportsCodexStructuredLocation
|
||||
supportsLocation = (location: Parameters<typeof supportsCodexStructuredLocation>[0]): boolean =>
|
||||
supportsCodexStructuredLocation(location, this.deps.isWindowsProcessStartTimeAvailable)
|
||||
|
||||
acquire = (input: StructuredAgentSessionAcquireInput): Promise<AgentSessionAcquisition> =>
|
||||
acquireCodexStructuredSession({
|
||||
@@ -128,6 +130,7 @@ export class CodexStructuredSessionAdapter implements StructuredAgentSessionAdap
|
||||
method: string,
|
||||
params: unknown
|
||||
): CodexJournalTranslationAdmission {
|
||||
codexRewind.observeCodexRewindActivity(session, method, params)
|
||||
if (this.turnCancellation.handleNotification(sessionId, session, method, params)) {
|
||||
return { accepted: true }
|
||||
}
|
||||
@@ -188,8 +191,13 @@ export class CodexStructuredSessionAdapter implements StructuredAgentSessionAdap
|
||||
fence: number
|
||||
}): Promise<AgentSessionDispatchOutcome> {
|
||||
const session = this.session(input.sessionId)
|
||||
await this.turnCancellation.captureBaseline(session)
|
||||
return dispatchCodexTurn(session, input, this.deps.requestTimeoutMs)
|
||||
session.dispatchPending = true
|
||||
try {
|
||||
await this.turnCancellation.captureBaseline(session)
|
||||
return await dispatchCodexTurn(session, input, this.deps.requestTimeoutMs)
|
||||
} finally {
|
||||
session.dispatchPending = false
|
||||
}
|
||||
}
|
||||
|
||||
async cancelTurn(input: {
|
||||
@@ -208,6 +216,17 @@ export class CodexStructuredSessionAdapter implements StructuredAgentSessionAdap
|
||||
input
|
||||
) => this.backgroundTerminals.stop(input)
|
||||
|
||||
rewindSupport: NonNullable<StructuredAgentSessionAdapter['rewindSupport']> = (sessionId) =>
|
||||
this.sessions.get(sessionId)?.historyMode === 'legacy'
|
||||
? { supported: false, reason: 'history-not-paginated' }
|
||||
: { supported: true }
|
||||
|
||||
rewind: NonNullable<StructuredAgentSessionAdapter['rewind']> = (input) =>
|
||||
codexRewind.rewindCodexSession(this.session(input.sessionId), input, this.deps.requestTimeoutMs)
|
||||
|
||||
recoverRewind: NonNullable<StructuredAgentSessionAdapter['recoverRewind']> = (input) =>
|
||||
codexRewind.recoverCodexRewind(this.session(input.sessionId), input, this.deps.requestTimeoutMs)
|
||||
|
||||
compact: NonNullable<StructuredAgentSessionAdapter['compact']> = (input) => {
|
||||
const session = this.session(input.sessionId)
|
||||
return this.compactions.run(
|
||||
|
||||
@@ -43,6 +43,8 @@ export type CodexStructuredSessionAdapterDeps = {
|
||||
resolveLaunch: (input: {
|
||||
identity: AgentSessionJournalIdentity
|
||||
}) => Promise<CodexStructuredLaunch>
|
||||
/** Host capability seam; production uses the native Windows process table. */
|
||||
isWindowsProcessStartTimeAvailable?: () => boolean
|
||||
onEvent?: (event: CodexStructuredSessionEvent) => void
|
||||
onBackgroundTasksChanged?: (
|
||||
sessionId: string,
|
||||
@@ -69,6 +71,9 @@ export type CodexSession = {
|
||||
acquisitionGeneration: string
|
||||
threadId: string
|
||||
historyPath: string | null
|
||||
historyMode?: 'legacy' | 'paginated'
|
||||
activeTurnIds?: Set<string>
|
||||
dispatchPending?: boolean
|
||||
prompts: CodexAcquisitionWindow['prompts']
|
||||
options: Map<string, string>
|
||||
reportedOptions: { model?: string; effort?: string }
|
||||
|
||||
@@ -16,6 +16,7 @@ export type CodexOpenedThread = {
|
||||
thread?: Record<string, unknown>
|
||||
/** Rollout file Codex named, when it named one. */
|
||||
historyPath: string | null
|
||||
historyMode?: 'legacy' | 'paginated'
|
||||
model?: string
|
||||
effort?: string
|
||||
}
|
||||
@@ -92,6 +93,9 @@ export async function openCodexThread(
|
||||
threadId,
|
||||
thread,
|
||||
historyPath: readCodexThreadPath(opened),
|
||||
...(thread.historyMode === 'legacy' || thread.historyMode === 'paginated'
|
||||
? { historyMode: thread.historyMode }
|
||||
: {}),
|
||||
...(model ? { model } : {}),
|
||||
...(effort ? { effort } : {})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
// Reading Codex's subagent wire shapes.
|
||||
//
|
||||
// Established by a live probe against `codex app-server` 0.152.1, not inferred:
|
||||
// * `subAgentActivity` items carry `{kind, agentThreadId, agentPath}`, and each
|
||||
// one arrives TWICE — via `item/started` and again via `item/completed`.
|
||||
// * `agentPath` is a tree path (`/root`, `/root/list_directory`); the trailing
|
||||
// segment is a semantic task name and the only label available. There is no
|
||||
// `thread/started` for a child, so nickname/role/depth do not exist.
|
||||
// * `agentsStates` on `collabAgentToolCall` arrived empty (`{}`) throughout the
|
||||
// probe, so nothing here reads it — state comes from `kind` alone.
|
||||
// * `thread/tokenUsage/updated` reports a per-thread RUNNING TOTAL, so the
|
||||
// latest frame replaces the previous one — it is never accumulated.
|
||||
|
||||
import type { NativeChatSubagentState } from '../../shared/native-chat-types'
|
||||
import type { CodexThreadItem } from './codex-structured-item-translation'
|
||||
|
||||
export const CODEX_SUBAGENT_ITEM_TYPE = 'subAgentActivity'
|
||||
export const CODEX_TOKEN_USAGE_METHOD = 'thread/tokenUsage/updated'
|
||||
|
||||
export type CodexSubagentActivity = {
|
||||
kind: string
|
||||
agentThreadId: string
|
||||
agentPath: string | null
|
||||
}
|
||||
|
||||
function nonEmptyString(value: unknown): string | null {
|
||||
return typeof value === 'string' && value.length > 0 ? value : null
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> | null {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
||||
? (value as Record<string, unknown>)
|
||||
: null
|
||||
}
|
||||
|
||||
export function readCodexSubagentActivity(item: CodexThreadItem): CodexSubagentActivity | null {
|
||||
if (item.type !== CODEX_SUBAGENT_ITEM_TYPE) {
|
||||
return null
|
||||
}
|
||||
const agentThreadId = nonEmptyString(item.agentThreadId)
|
||||
if (!agentThreadId) {
|
||||
return null
|
||||
}
|
||||
return {
|
||||
kind: nonEmptyString(item.kind) ?? '',
|
||||
agentThreadId,
|
||||
agentPath: nonEmptyString(item.agentPath)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The state a `kind` implies for the child it names.
|
||||
*
|
||||
* An unrecognized kind means "this child exists and reported something we
|
||||
* cannot classify" — `working`, which the session sweep will later settle to
|
||||
* `unverifiable` if nothing better ever arrives. Claiming a terminal state from
|
||||
* an unknown kind would assert an outcome the wire never gave us.
|
||||
*/
|
||||
export function codexSubagentStateForKind(kind: string): NativeChatSubagentState {
|
||||
if (kind === 'completed') {
|
||||
return 'completed'
|
||||
}
|
||||
if (kind === 'interrupted') {
|
||||
return 'stopped'
|
||||
}
|
||||
return 'working'
|
||||
}
|
||||
|
||||
/** Path segments, empty ones dropped: `/root/list_directory` → 2 segments. */
|
||||
export function codexSubagentPathSegments(agentPath: string | null): string[] {
|
||||
return agentPath === null ? [] : agentPath.split('/').filter((part) => part.length > 0)
|
||||
}
|
||||
|
||||
/** The one path segment that names the parent turn itself rather than a child.
|
||||
* Compared after the same normalization the label uses, not against the raw
|
||||
* string: `/root/` and `/root//` are the same node as `/root`, and a check that
|
||||
* disagreed with `codexSubagentPathSegments` would let one path be both the
|
||||
* turn and a child of it — a phantom row labelled `root` inflating the group.
|
||||
* Only this segment is the root; `/morpheus` is single-segment too but IS a
|
||||
* child. */
|
||||
const CODEX_ROOT_AGENT_SEGMENT = 'root'
|
||||
|
||||
/**
|
||||
* Whether an activity item describes the ROOT of the agent tree rather than a
|
||||
* spawned child. Counting the root would make the parent turn report itself as
|
||||
* its own subagent.
|
||||
*
|
||||
* A path-less item cannot be placed in the tree at all, so it is treated as a
|
||||
* child: dropping it would lose a real spawn, while an extra row is visible and
|
||||
* self-correcting.
|
||||
*/
|
||||
export function isCodexRootAgentActivity(activity: CodexSubagentActivity): boolean {
|
||||
const segments = codexSubagentPathSegments(activity.agentPath)
|
||||
return segments.length === 1 && segments[0] === CODEX_ROOT_AGENT_SEGMENT
|
||||
}
|
||||
|
||||
/** Row label: the agent path's trailing segment, trimmed. A segment with nothing
|
||||
* visible in it survives the empty-segment filter but would draw a nameless row,
|
||||
* so it reads as no label and the caller's placeholder takes over. Trimmed
|
||||
* because the caller keys its collision ordinals on this string: ` read ` and
|
||||
* `read` render identically and must therefore collide. */
|
||||
export function codexSubagentLabel(activity: CodexSubagentActivity): string | null {
|
||||
const trailing = codexSubagentPathSegments(activity.agentPath).at(-1)?.trim()
|
||||
return trailing !== undefined && trailing.length > 0 ? trailing : null
|
||||
}
|
||||
|
||||
export type CodexThreadTokenTotal = { threadId: string; totalTokens: number }
|
||||
|
||||
/** `{threadId, tokenUsage: {total: {totalTokens}}}`. Older builds put the total
|
||||
* on the envelope, so both shapes are accepted. */
|
||||
export function readCodexThreadTokenTotal(params: unknown): CodexThreadTokenTotal | null {
|
||||
const root = record(params)
|
||||
if (!root) {
|
||||
return null
|
||||
}
|
||||
const threadId = nonEmptyString(root.threadId) ?? nonEmptyString(record(root.thread)?.id)
|
||||
if (!threadId) {
|
||||
return null
|
||||
}
|
||||
const usage = record(root.tokenUsage)
|
||||
const total = record(usage?.total)?.totalTokens ?? usage?.totalTokens ?? root.totalTokens
|
||||
return typeof total === 'number' && Number.isFinite(total) && total >= 0
|
||||
? { threadId, totalTokens: total }
|
||||
: null
|
||||
}
|
||||
|
||||
/** Pull the `subAgentActivity` item out of a raw notification payload.
|
||||
*
|
||||
* Lives beside the readers rather than in the translator: the translator's job
|
||||
* is routing, and this is the shape check that decides whether a frame is one
|
||||
* of ours at all. Returns null for anything that is not a thread item, which is
|
||||
* the translator's signal to keep looking. */
|
||||
export function readCodexNotificationThreadItem(
|
||||
params: unknown,
|
||||
read: (value: unknown) => CodexThreadItem | null
|
||||
): CodexThreadItem | null {
|
||||
const record =
|
||||
typeof params === 'object' && params !== null ? (params as Record<string, unknown>) : {}
|
||||
return read(record.item)
|
||||
}
|
||||
@@ -0,0 +1,769 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { isAdmissibleAgentJournalItemBody } from '../../shared/agent-session-journal-schemas'
|
||||
import type {
|
||||
AgentJournalItemBody,
|
||||
AgentJournalItemIdentity
|
||||
} from '../../shared/agent-session-journal-types'
|
||||
import { MAX_SUBAGENT_FIELD_CHARS } from '../../shared/native-chat-subagent-summary'
|
||||
import { isSubagentGroupBlock, type NativeChatSubagentEntry } from '../../shared/native-chat-types'
|
||||
import type { StructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink'
|
||||
import {
|
||||
CodexSubagentRoster,
|
||||
codexSubagentGroupIdentity,
|
||||
codexSubagentGroupId
|
||||
} from './codex-subagent-roster'
|
||||
import type { CodexThreadItem } from './codex-structured-item-translation'
|
||||
import {
|
||||
MAX_CODEX_SUBAGENT_GROUPS,
|
||||
MAX_CODEX_SUBAGENTS_PER_GROUP,
|
||||
MAX_CODEX_TOKEN_USAGE_THREADS
|
||||
} from './codex-structured-journal-limits'
|
||||
|
||||
const THREAD = 'thread-parent'
|
||||
const TURN = 'turn-1'
|
||||
|
||||
type Appended = { identity: AgentJournalItemIdentity; body: AgentJournalItemBody }
|
||||
|
||||
function createHarness(options: { threadId?: string | null } = {}): {
|
||||
roster: CodexSubagentRoster
|
||||
appended: Appended[]
|
||||
agents: () => NativeChatSubagentEntry[]
|
||||
latest: () => Appended | undefined
|
||||
} {
|
||||
const appended: Appended[] = []
|
||||
let clock = 1_000
|
||||
const sink: StructuredAgentSessionEventSink = {
|
||||
appendItem: () => {},
|
||||
appendTombstone: () => {},
|
||||
publish: () => {},
|
||||
tryAppendItem: (identity, body) => {
|
||||
appended.push({ identity, body })
|
||||
return { accepted: true }
|
||||
},
|
||||
tryPublish: () => ({ accepted: true })
|
||||
}
|
||||
const roster = new CodexSubagentRoster({
|
||||
sink,
|
||||
primaryThreadId: () => (options.threadId === undefined ? THREAD : options.threadId),
|
||||
activeTurn: () => TURN,
|
||||
now: () => (clock += 1)
|
||||
})
|
||||
const agents = (): NativeChatSubagentEntry[] => {
|
||||
const body = appended.at(-1)?.body
|
||||
if (!body || body.kind !== 'message') {
|
||||
return []
|
||||
}
|
||||
const block = body.blocks.find(isSubagentGroupBlock)
|
||||
return block ? block.agents : []
|
||||
}
|
||||
return { roster, appended, agents, latest: () => appended.at(-1) }
|
||||
}
|
||||
|
||||
function latestIdentity(appended: Appended[]): AgentJournalItemIdentity | undefined {
|
||||
return appended.at(-1)?.identity
|
||||
}
|
||||
|
||||
function activity(input: {
|
||||
id?: string
|
||||
kind: string
|
||||
agentThreadId: string
|
||||
agentPath: string | null
|
||||
}): CodexThreadItem {
|
||||
return {
|
||||
type: 'subAgentActivity',
|
||||
id: input.id ?? `item-${input.agentThreadId}-${input.kind}`,
|
||||
kind: input.kind,
|
||||
agentThreadId: input.agentThreadId,
|
||||
agentPath: input.agentPath
|
||||
}
|
||||
}
|
||||
|
||||
function deliver(
|
||||
roster: CodexSubagentRoster,
|
||||
item: CodexThreadItem,
|
||||
turnId: string | null = TURN
|
||||
): void {
|
||||
// Every activity item reaches the wire twice: item/started, then item/completed.
|
||||
roster.handleItem({ threadId: THREAD, turnId, item })
|
||||
roster.handleItem({ threadId: THREAD, turnId, item })
|
||||
}
|
||||
|
||||
/**
|
||||
* A sink that coalesces the way the real queue does: by `coalescingKey` ALONE,
|
||||
* with no op-kind check, and only draining when released. A fake that ignores
|
||||
* the key cannot see an append being spliced out by its own publish.
|
||||
*/
|
||||
function createCoalescingHarness(): {
|
||||
roster: CodexSubagentRoster
|
||||
appended: Appended[]
|
||||
drain: () => void
|
||||
} {
|
||||
const appended: Appended[] = []
|
||||
const queue: { key?: string; run: () => void }[] = []
|
||||
let clock = 1_000
|
||||
const submit = (key: string | undefined, run: () => void): void => {
|
||||
const at = key === undefined ? -1 : queue.findIndex((queued) => queued.key === key)
|
||||
if (at >= 0) {
|
||||
queue.splice(at, 1)
|
||||
}
|
||||
queue.push(key === undefined ? { run } : { key, run })
|
||||
}
|
||||
const sink: StructuredAgentSessionEventSink = {
|
||||
appendItem: () => {},
|
||||
appendTombstone: () => {},
|
||||
publish: () => {},
|
||||
tryAppendItem: (identity, body, options) => {
|
||||
submit(options?.coalescingKey, () => appended.push({ identity, body }))
|
||||
return { accepted: true }
|
||||
},
|
||||
tryPublish: (options) => {
|
||||
submit(options?.coalescingKey ?? 'publish', () => {})
|
||||
return { accepted: true }
|
||||
}
|
||||
}
|
||||
const roster = new CodexSubagentRoster({
|
||||
sink,
|
||||
primaryThreadId: () => THREAD,
|
||||
activeTurn: () => TURN,
|
||||
now: () => (clock += 1)
|
||||
})
|
||||
return {
|
||||
roster,
|
||||
appended,
|
||||
drain: () => {
|
||||
while (queue.length > 0) {
|
||||
queue.shift()?.run()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
describe('CodexSubagentRoster', () => {
|
||||
it('does not let its own publish evict the still-queued roster append', () => {
|
||||
const { roster, appended, drain } = createCoalescingHarness()
|
||||
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
)
|
||||
drain()
|
||||
|
||||
// Sharing the append's coalescing key with the publish spliced the append
|
||||
// out of the queue, and `lastSerialized` then suppressed every retry.
|
||||
expect(appended).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('counts a /morpheus agent as a child — only /root is the turn itself', () => {
|
||||
const { roster, agents } = createHarness()
|
||||
|
||||
deliver(roster, activity({ kind: 'started', agentThreadId: 'child-m', agentPath: '/morpheus' }))
|
||||
|
||||
expect(agents()).toMatchObject([{ id: 'child-m', label: 'morpheus', state: 'working' }])
|
||||
})
|
||||
|
||||
// `codexSubagentPathSegments` already defines what a path means for the label,
|
||||
// and the root check has to agree with it: a path that normalizes to the same
|
||||
// node must classify the same way, or one string is both the turn itself and a
|
||||
// child of it — a phantom row labelled `root` inflating the group by one.
|
||||
it('reads a root path with a trailing or doubled separator as the turn itself', () => {
|
||||
for (const agentPath of ['/root/', '/root//', '//root']) {
|
||||
const { roster, appended } = createHarness()
|
||||
|
||||
deliver(roster, activity({ kind: 'started', agentThreadId: THREAD, agentPath }))
|
||||
|
||||
expect(appended).toEqual([])
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps a doubled separator inside a child path off the label', () => {
|
||||
const { roster, agents } = createHarness()
|
||||
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root//read/' })
|
||||
)
|
||||
|
||||
expect(agents()).toMatchObject([{ id: 'child-1', label: 'read' }])
|
||||
})
|
||||
|
||||
// An all-whitespace trailing segment survives the empty-segment filter and
|
||||
// would draw a row with no visible name at all.
|
||||
it('falls back to the placeholder when the trailing segment has nothing to show', () => {
|
||||
const { roster, agents } = createHarness()
|
||||
|
||||
deliver(roster, activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/ ' }))
|
||||
|
||||
expect(agents()).toMatchObject([{ id: 'child-1', label: 'subagent' }])
|
||||
})
|
||||
|
||||
// The collision ordinal keys on the label, so two segments that render
|
||||
// identically must collide rather than both draw as `read`.
|
||||
it('collides labels that differ only in surrounding whitespace', () => {
|
||||
const { roster, agents } = createHarness()
|
||||
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
)
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: 'child-2', agentPath: '/root/ read ' })
|
||||
)
|
||||
|
||||
expect(agents().map((agent) => agent.label)).toEqual(['read', 'read 2'])
|
||||
})
|
||||
|
||||
it('ignores the root node so a turn is not its own subagent', () => {
|
||||
const { roster, appended } = createHarness()
|
||||
|
||||
deliver(roster, activity({ kind: 'started', agentThreadId: THREAD, agentPath: '/root' }))
|
||||
|
||||
expect(appended).toEqual([])
|
||||
})
|
||||
|
||||
it('writes an admissible journal body carrying a plain-text fallback block', () => {
|
||||
const { roster, latest } = createHarness()
|
||||
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/list_directory' })
|
||||
)
|
||||
|
||||
const body = latest()?.body
|
||||
expect(body?.kind).toBe('message')
|
||||
expect(isAdmissibleAgentJournalItemBody(body)).toBe(true)
|
||||
expect(body?.kind === 'message' ? body.blocks.map((block) => block.type) : []).toEqual([
|
||||
'text',
|
||||
'subagent-group'
|
||||
])
|
||||
expect(
|
||||
body?.kind === 'message' && body.blocks[0]?.type === 'text' ? body.blocks[0].text : ''
|
||||
).toBe('Kicked off 1 subagent')
|
||||
})
|
||||
|
||||
it('keys the durable identity by the parent turn so a revision lands on one row', () => {
|
||||
const { roster, appended } = createHarness()
|
||||
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
)
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'completed', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
)
|
||||
|
||||
const expected = codexSubagentGroupIdentity(codexSubagentGroupId(THREAD, TURN))
|
||||
expect(new Set(appended.map((entry) => JSON.stringify(entry.identity)))).toEqual(
|
||||
new Set([JSON.stringify(expected)])
|
||||
)
|
||||
})
|
||||
|
||||
it('rule 1 — a duplicate delivery writes no second revision', () => {
|
||||
const { roster, appended } = createHarness()
|
||||
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
)
|
||||
|
||||
expect(appended).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('rule 2 — a first event of any kind creates the entry in the state it implies', () => {
|
||||
const { roster, agents } = createHarness()
|
||||
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'completed', agentThreadId: 'child-late', agentPath: '/root/search' })
|
||||
)
|
||||
|
||||
expect(agents()).toMatchObject([{ id: 'child-late', label: 'search', state: 'completed' }])
|
||||
})
|
||||
|
||||
it('rule 3 — a terminal state latches against a late or duplicate start', () => {
|
||||
const { roster, agents } = createHarness()
|
||||
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'completed', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
)
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
)
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'interacted', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
)
|
||||
|
||||
expect(agents()).toMatchObject([{ state: 'completed' }])
|
||||
})
|
||||
|
||||
it('rule 4 — the session sweep settles a lost child as unverifiable, not exited', () => {
|
||||
const { roster, agents } = createHarness()
|
||||
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
)
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'completed', agentThreadId: 'child-2', agentPath: '/root/search' })
|
||||
)
|
||||
roster.settleSession()
|
||||
|
||||
expect(agents()).toMatchObject([
|
||||
{ id: 'child-1', state: 'unverifiable' },
|
||||
{ id: 'child-2', state: 'completed' }
|
||||
])
|
||||
})
|
||||
|
||||
it('lets a swept child still report what it actually did', () => {
|
||||
const { roster, agents } = createHarness()
|
||||
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
)
|
||||
roster.settleSession()
|
||||
expect(agents()[0]?.state).toBe('unverifiable')
|
||||
|
||||
// Contact can return — a reconnected provider replays the child's own
|
||||
// verdict. Latching the sweep would report a child that finished as one we
|
||||
// never saw finish.
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'completed', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
)
|
||||
expect(agents()[0]?.state).toBe('completed')
|
||||
})
|
||||
|
||||
it('refuses to put a swept child back to working', () => {
|
||||
const { roster, agents } = createHarness()
|
||||
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
)
|
||||
roster.settleSession()
|
||||
// A straggler progress tick after we gave up must not re-light the row.
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'interacted', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
)
|
||||
expect(agents()[0]?.state).toBe('unverifiable')
|
||||
})
|
||||
|
||||
it('keeps a real verdict when a later frame disagrees', () => {
|
||||
const { roster, agents } = createHarness()
|
||||
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'completed', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
)
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'interrupted', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
)
|
||||
expect(agents()[0]?.state).toBe('completed')
|
||||
})
|
||||
|
||||
it('rule 4 — the session sweep settles every group and never un-terminals one', () => {
|
||||
const { roster, agents, appended } = createHarness()
|
||||
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'interacted', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
)
|
||||
roster.settleSession()
|
||||
const afterFirstSweep = appended.length
|
||||
roster.settleSession()
|
||||
|
||||
expect(agents()).toMatchObject([{ state: 'unverifiable' }])
|
||||
expect(appended).toHaveLength(afterFirstSweep)
|
||||
})
|
||||
|
||||
it('rule 5 — the whole roster is persisted in the carrier, not just a count', () => {
|
||||
const { roster, agents } = createHarness()
|
||||
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
)
|
||||
roster.handleTokenUsage({ threadId: 'child-1', tokenUsage: { total: { totalTokens: 40661 } } })
|
||||
|
||||
expect(agents()).toMatchObject([
|
||||
{ id: 'child-1', label: 'read', state: 'working', tokens: 40661 }
|
||||
])
|
||||
})
|
||||
|
||||
it('rule 6 — the group id names the parent turn, or says there was none', () => {
|
||||
const { roster, appended } = createHarness()
|
||||
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
)
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: 'child-2', agentPath: '/root/search' }),
|
||||
null
|
||||
)
|
||||
|
||||
expect(appended.map((entry) => entry.identity)).toEqual([
|
||||
{ provider: 'orca', clientMessageId: `codex-subagents:${THREAD}:${TURN}` },
|
||||
{ provider: 'orca', clientMessageId: `codex-subagents:${THREAD}:outside-turn` }
|
||||
])
|
||||
})
|
||||
|
||||
it('disambiguates two children that share a trailing path segment', () => {
|
||||
const { roster, agents } = createHarness()
|
||||
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
)
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: 'child-2', agentPath: '/root/read' })
|
||||
)
|
||||
|
||||
expect(agents().map((agent) => agent.label)).toEqual(['read', 'read 2'])
|
||||
})
|
||||
|
||||
it('takes the latest token snapshot per child and never accumulates updates', () => {
|
||||
const { roster, agents } = createHarness()
|
||||
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
)
|
||||
roster.handleTokenUsage({ threadId: 'child-1', tokenUsage: { total: { totalTokens: 100 } } })
|
||||
roster.handleTokenUsage({ threadId: 'child-1', tokenUsage: { total: { totalTokens: 250 } } })
|
||||
|
||||
expect(agents()).toMatchObject([{ tokens: 250 }])
|
||||
})
|
||||
|
||||
it('retains a usage frame that arrives before the child is known', () => {
|
||||
const { roster, agents } = createHarness()
|
||||
|
||||
roster.handleTokenUsage({ threadId: 'child-1', tokenUsage: { total: { totalTokens: 900 } } })
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
)
|
||||
|
||||
expect(agents()).toMatchObject([{ tokens: 900 }])
|
||||
})
|
||||
|
||||
it('never attributes the parent thread its own usage', () => {
|
||||
const { roster, agents, appended } = createHarness()
|
||||
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
)
|
||||
const beforeParentUsage = appended.length
|
||||
roster.handleTokenUsage({ threadId: THREAD, tokenUsage: { total: { totalTokens: 26099 } } })
|
||||
|
||||
expect(appended).toHaveLength(beforeParentUsage)
|
||||
expect(agents()).toHaveLength(1)
|
||||
expect(agents()[0]).not.toHaveProperty('tokens')
|
||||
})
|
||||
|
||||
// The row is durable and both readers clip these fields to the same cap, so
|
||||
// writing more than that is bytes replayed on every reconnect and then thrown
|
||||
// away. The marker is an ellipsis, not the tool-output truncation sentence:
|
||||
// `id` is the roster key and the renderer's React key.
|
||||
it('bounds the provider strings the roster row carries into the journal', () => {
|
||||
const { roster, agents, latest } = createHarness()
|
||||
const oversized = 'a'.repeat(20 * 1024)
|
||||
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: oversized, agentPath: `/root/${oversized}` })
|
||||
)
|
||||
|
||||
const entry = agents()[0]
|
||||
expect(entry?.label.length).toBeLessThanOrEqual(MAX_SUBAGENT_FIELD_CHARS)
|
||||
expect(entry?.label).toMatch(/…~0$/)
|
||||
expect(entry?.id.length).toBeLessThanOrEqual(MAX_SUBAGENT_FIELD_CHARS)
|
||||
expect(entry?.id).toMatch(/…~0$/)
|
||||
expect(JSON.stringify(latest()?.body)).not.toContain('output truncated')
|
||||
expect(isAdmissibleAgentJournalItemBody(latest()?.body)).toBe(true)
|
||||
})
|
||||
|
||||
// The clip cuts UTF-16 code units, so a boundary landing inside a surrogate
|
||||
// pair left a LONE high surrogate in a durable row — malformed, and replaced
|
||||
// with U+FFFD through any non-JSON UTF-8 hop.
|
||||
it('never clips a provider string mid surrogate pair', () => {
|
||||
const { roster, agents } = createHarness()
|
||||
const astral = '😀'.repeat(400)
|
||||
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: astral, agentPath: `/root/${astral}` })
|
||||
)
|
||||
|
||||
const entry = agents()[0]
|
||||
expect(entry?.id.length).toBeLessThanOrEqual(MAX_SUBAGENT_FIELD_CHARS)
|
||||
expect(Buffer.from(entry?.id ?? '', 'utf8').toString('utf8')).toBe(entry?.id)
|
||||
expect(Buffer.from(entry?.label ?? '', 'utf8').toString('utf8')).toBe(entry?.label)
|
||||
})
|
||||
|
||||
// The clip removes exactly the tail that told two children apart: `id` is the
|
||||
// renderer's React key, and `claimLabel` writes its repeat ordinal at the end.
|
||||
// Two clipped children collapsing to one key drew two rows under one identity.
|
||||
it('keeps clipped ids and labels distinct between children', () => {
|
||||
const { roster, agents } = createHarness()
|
||||
const prefix = 'p'.repeat(MAX_SUBAGENT_FIELD_CHARS)
|
||||
const sharedPath = `/root/${'q'.repeat(640)}`
|
||||
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: `${prefix}AAAA`, agentPath: sharedPath })
|
||||
)
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: `${prefix}BBBB`, agentPath: sharedPath })
|
||||
)
|
||||
|
||||
const entries = agents()
|
||||
expect(entries).toHaveLength(2)
|
||||
expect(new Set(entries.map((agent) => agent.id)).size).toBe(2)
|
||||
expect(new Set(entries.map((agent) => agent.label)).size).toBe(2)
|
||||
for (const agent of entries) {
|
||||
expect(agent.id.length).toBeLessThanOrEqual(MAX_SUBAGENT_FIELD_CHARS)
|
||||
expect(agent.label.length).toBeLessThanOrEqual(MAX_SUBAGENT_FIELD_CHARS)
|
||||
}
|
||||
})
|
||||
|
||||
it('caps the children one spawn group admits', () => {
|
||||
const { roster, agents, appended } = createHarness()
|
||||
for (let index = 0; index < MAX_CODEX_SUBAGENTS_PER_GROUP; index++) {
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: `child-${index}`, agentPath: '/root/read' })
|
||||
)
|
||||
}
|
||||
const atCap = appended.length
|
||||
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: 'child-over-cap', agentPath: '/root/read' })
|
||||
)
|
||||
|
||||
expect(agents()).toHaveLength(MAX_CODEX_SUBAGENTS_PER_GROUP)
|
||||
expect(agents().map((agent) => agent.id)).not.toContain('child-over-cap')
|
||||
// Refusing the child must not burn a revision either.
|
||||
expect(appended).toHaveLength(atCap)
|
||||
})
|
||||
|
||||
// The eviction is the KNOWN LIMITATION the module documents: `groups` is never
|
||||
// seeded from the journal, so the evicted group's next child rebuilds its
|
||||
// durable row from that one child. Pinned so the boundary cannot move silently.
|
||||
it('caps live spawn groups, and an evicted group rebuilds its row from one child', () => {
|
||||
const { roster, appended, agents } = createHarness()
|
||||
for (let index = 0; index <= MAX_CODEX_SUBAGENT_GROUPS; index++) {
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: `child-${index}`, agentPath: '/root/read' }),
|
||||
`turn-${index}`
|
||||
)
|
||||
}
|
||||
const evicted = codexSubagentGroupIdentity(codexSubagentGroupId(THREAD, 'turn-0'))
|
||||
const rowsFor = (identity: AgentJournalItemIdentity): Appended[] =>
|
||||
appended.filter((entry) => JSON.stringify(entry.identity) === JSON.stringify(identity))
|
||||
expect(rowsFor(evicted)).toHaveLength(1)
|
||||
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: 'child-late', agentPath: '/root/search' }),
|
||||
'turn-0'
|
||||
)
|
||||
|
||||
expect(latestIdentity(appended)).toEqual(evicted)
|
||||
expect(agents().map((agent) => agent.id)).toEqual(['child-late'])
|
||||
})
|
||||
|
||||
it('keeps a token count a later thread-map eviction would otherwise retract', () => {
|
||||
const { roster, agents } = createHarness()
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
)
|
||||
roster.handleTokenUsage({ threadId: 'child-1', tokenUsage: { total: { totalTokens: 4242 } } })
|
||||
expect(agents()).toMatchObject([{ tokens: 4242 }])
|
||||
|
||||
for (let index = 0; index < MAX_CODEX_TOKEN_USAGE_THREADS; index++) {
|
||||
roster.handleTokenUsage({
|
||||
threadId: `other-${index}`,
|
||||
tokenUsage: { total: { totalTokens: index } }
|
||||
})
|
||||
}
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'completed', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
)
|
||||
|
||||
expect(agents()).toMatchObject([{ state: 'completed', tokens: 4242 }])
|
||||
})
|
||||
|
||||
it('caps retained usage threads, so a frame evicted before its child is dropped', () => {
|
||||
const { roster, agents } = createHarness()
|
||||
roster.handleTokenUsage({ threadId: 'child-1', tokenUsage: { total: { totalTokens: 900 } } })
|
||||
for (let index = 0; index < MAX_CODEX_TOKEN_USAGE_THREADS; index++) {
|
||||
roster.handleTokenUsage({
|
||||
threadId: `other-${index}`,
|
||||
tokenUsage: { total: { totalTokens: index } }
|
||||
})
|
||||
}
|
||||
|
||||
deliver(
|
||||
roster,
|
||||
activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
)
|
||||
|
||||
expect(agents()[0]).not.toHaveProperty('tokens')
|
||||
})
|
||||
|
||||
it('declines a payload that is not a subagent item or a usage frame', () => {
|
||||
const { roster } = createHarness()
|
||||
|
||||
expect(
|
||||
roster.handleItem({
|
||||
threadId: THREAD,
|
||||
turnId: TURN,
|
||||
item: { type: 'commandExecution', id: 'item-9' }
|
||||
})
|
||||
).toBeNull()
|
||||
expect(roster.handleTokenUsage({ threadId: 'child-1' })).toBeNull()
|
||||
})
|
||||
|
||||
// A refusal must never advance the duplicate-suppression state: an identical
|
||||
// replay would short-circuit and the revision would never be retried. The
|
||||
// append and the publish are the two ways to be refused, so both are covered.
|
||||
it.each([{ refuse: 'append' as const }, { refuse: 'publish' as const }])(
|
||||
'retries the same revision after the $refuse is refused',
|
||||
({ refuse }) => {
|
||||
let refusing = true
|
||||
const appended: Appended[] = []
|
||||
const published: number[] = []
|
||||
const refusal = { accepted: false, reason: 'backpressure' } as const
|
||||
const roster = new CodexSubagentRoster({
|
||||
sink: {
|
||||
appendItem: () => {},
|
||||
appendTombstone: () => {},
|
||||
publish: () => {},
|
||||
tryAppendItem: (identity, body) => {
|
||||
if (refusing && refuse === 'append') {
|
||||
return refusal
|
||||
}
|
||||
appended.push({ identity, body })
|
||||
return { accepted: true }
|
||||
},
|
||||
tryPublish: () => {
|
||||
if (refusing && refuse === 'publish') {
|
||||
return refusal
|
||||
}
|
||||
published.push(1)
|
||||
return { accepted: true }
|
||||
}
|
||||
},
|
||||
primaryThreadId: () => THREAD,
|
||||
activeTurn: () => TURN,
|
||||
now: () => 1_000
|
||||
})
|
||||
const item = activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
|
||||
expect(roster.handleItem({ threadId: THREAD, turnId: TURN, item })).toEqual(refusal)
|
||||
|
||||
// The wire redelivers the very same item; nothing about the roster changed,
|
||||
// so only a cleared suppression state can get the revision out.
|
||||
refusing = false
|
||||
expect(roster.handleItem({ threadId: THREAD, turnId: TURN, item })).toEqual({
|
||||
accepted: true
|
||||
})
|
||||
// The retry re-appends when the publish was the half that failed; the real
|
||||
// queue coalesces those two by the group key into one journal write. What
|
||||
// must not happen is the revision never being published at all.
|
||||
expect(published).toHaveLength(1)
|
||||
const body = appended.at(-1)?.body
|
||||
expect(
|
||||
body?.kind === 'message' ? body.blocks.filter(isSubagentGroupBlock) : []
|
||||
).toMatchObject([{ agents: [{ id: 'child-1', state: 'working' }] }])
|
||||
}
|
||||
)
|
||||
|
||||
// The sweep is the last event a group ever gets. A refusal there, left
|
||||
// unretried, strands the settled roster's final revision — the exact "row
|
||||
// stays stale forever" this row exists to prevent.
|
||||
it('republishes the settled roster when the sweep publish was refused', () => {
|
||||
let refusing = false
|
||||
const appended: Appended[] = []
|
||||
const published: number[] = []
|
||||
const roster = new CodexSubagentRoster({
|
||||
sink: {
|
||||
appendItem: () => {},
|
||||
appendTombstone: () => {},
|
||||
publish: () => {},
|
||||
tryAppendItem: (identity, body) => {
|
||||
appended.push({ identity, body })
|
||||
return { accepted: true }
|
||||
},
|
||||
tryPublish: () => {
|
||||
if (refusing) {
|
||||
return { accepted: false, reason: 'backpressure' }
|
||||
}
|
||||
published.push(1)
|
||||
return { accepted: true }
|
||||
}
|
||||
},
|
||||
primaryThreadId: () => THREAD,
|
||||
activeTurn: () => TURN,
|
||||
now: () => 1_000
|
||||
})
|
||||
roster.handleItem({
|
||||
threadId: THREAD,
|
||||
turnId: TURN,
|
||||
item: activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
})
|
||||
const publishedBeforeSweep = published.length
|
||||
|
||||
refusing = true
|
||||
expect(roster.settleSession()).toEqual({ accepted: false, reason: 'backpressure' })
|
||||
|
||||
// The retry sweep flips no state — every child already latched — so only a
|
||||
// cleared suppression state can carry the unverifiable roster out.
|
||||
refusing = false
|
||||
expect(roster.settleSession()).toEqual({ accepted: true })
|
||||
expect(published.length).toBe(publishedBeforeSweep + 1)
|
||||
const body = appended.at(-1)?.body
|
||||
expect(body?.kind === 'message' ? body.blocks.filter(isSubagentGroupBlock) : []).toMatchObject([
|
||||
{ agents: [{ id: 'child-1', state: 'unverifiable' }] }
|
||||
])
|
||||
})
|
||||
|
||||
it('propagates sink backpressure instead of reporting the row as written', () => {
|
||||
const roster = new CodexSubagentRoster({
|
||||
sink: {
|
||||
appendItem: () => {},
|
||||
appendTombstone: () => {},
|
||||
publish: () => {},
|
||||
tryAppendItem: () => ({ accepted: false, reason: 'backpressure' }),
|
||||
tryPublish: () => ({ accepted: true })
|
||||
},
|
||||
primaryThreadId: () => THREAD,
|
||||
activeTurn: () => TURN
|
||||
})
|
||||
|
||||
expect(
|
||||
roster.handleItem({
|
||||
threadId: THREAD,
|
||||
turnId: TURN,
|
||||
item: activity({ kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' })
|
||||
})
|
||||
).toEqual({ accepted: false, reason: 'backpressure' })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,347 @@
|
||||
// The Codex subagent roster: one journal row per spawn group, revised in place.
|
||||
//
|
||||
// There is no snapshot to read. `agentsStates` arrived empty in the live probe
|
||||
// and children get no `thread/started`, so the roster is
|
||||
// accumulated purely from `subAgentActivity` items — each of which arrives TWICE
|
||||
// (`item/started` and `item/completed`). Every transition here is therefore
|
||||
// idempotent, and a terminal state latches: duplicate and out-of-order delivery
|
||||
// must not resurrect a settled child.
|
||||
//
|
||||
// KNOWN LIMITATION: `groups` is process-local and is never seeded from the
|
||||
// journal, while the row's identity is keyed on the group id alone. So once a
|
||||
// group leaves the map its row stays, and the next activity item rebuilds that
|
||||
// row from one child — rewriting N down to one. Two ways in: eviction past
|
||||
// MAX_CODEX_SUBAGENT_GROUPS, which drops the oldest-inserted group in-process
|
||||
// even while it is live, and skips the sweep so its children never latch
|
||||
// `unverifiable`; and a restart on `threadId:outside-turn`, the one group id
|
||||
// that outlives the process — `thread/resume` is verified to return the same
|
||||
// thread, and a real turn id is assumed freshly minted per turn. Seeding from
|
||||
// the journal is the fix.
|
||||
|
||||
import type {
|
||||
AgentJournalItemBody,
|
||||
AgentJournalItemIdentity
|
||||
} from '../../shared/agent-session-journal-types'
|
||||
import {
|
||||
canReplaceSubagentState,
|
||||
isTerminalSubagentState,
|
||||
MAX_SUBAGENT_FIELD_CHARS,
|
||||
subagentGroupFallbackText
|
||||
} from '../../shared/native-chat-subagent-summary'
|
||||
import type { NativeChatSubagentEntry } from '../../shared/native-chat-types'
|
||||
import type {
|
||||
StructuredAgentSessionEventSink,
|
||||
StructuredAgentSessionSinkAdmission
|
||||
} from '../native-chat/agent-session-wire/structured-agent-session-event-sink'
|
||||
import {
|
||||
codexSubagentLabel,
|
||||
codexSubagentStateForKind,
|
||||
isCodexRootAgentActivity,
|
||||
readCodexSubagentActivity,
|
||||
readCodexThreadTokenTotal
|
||||
} from './codex-subagent-activity'
|
||||
import type { CodexThreadItem } from './codex-structured-item-translation'
|
||||
import {
|
||||
MAX_CODEX_SUBAGENT_GROUPS,
|
||||
MAX_CODEX_SUBAGENTS_PER_GROUP,
|
||||
MAX_CODEX_TOKEN_USAGE_THREADS
|
||||
} from './codex-structured-journal-limits'
|
||||
|
||||
const ADMITTED: StructuredAgentSessionSinkAdmission = { accepted: true }
|
||||
|
||||
/** The turn a group belongs to when Codex reports activity outside any turn.
|
||||
* Mirrors the generic-frame bucket name so the two read alike in the journal. */
|
||||
const OUTSIDE_TURN = 'outside-turn'
|
||||
|
||||
const UNLABELLED_AGENT = 'subagent'
|
||||
|
||||
type RosterGroup = {
|
||||
groupId: string
|
||||
identity: AgentJournalItemIdentity
|
||||
/** Insertion order is the display order; the map holds the state. */
|
||||
entries: Map<string, NativeChatSubagentEntry>
|
||||
/** Times each label has been claimed, so a repeat gets an ordinal suffix. */
|
||||
labelCounts: Map<string, number>
|
||||
/** Last body written, so an idempotent replay writes no new revision. */
|
||||
lastSerialized: string | null
|
||||
}
|
||||
|
||||
/** Group identity: the parent turn that spawned the children. `agentPath` is a
|
||||
* tree rooted at the parent thread, so every child of one turn shares a row
|
||||
* no matter which thread's stream carried its activity item. */
|
||||
export function codexSubagentGroupId(threadId: string, turnId: string | null): string {
|
||||
return `${threadId}:${turnId ?? OUTSIDE_TURN}`
|
||||
}
|
||||
|
||||
/** Durable journal identity for the group's row — stable across revisions and
|
||||
* across a restart, so replay finds the same row instead of appending a new one. */
|
||||
export function codexSubagentGroupIdentity(groupId: string): AgentJournalItemIdentity {
|
||||
return { provider: 'orca', clientMessageId: `codex-subagents:${groupId}` }
|
||||
}
|
||||
|
||||
export type CodexSubagentRosterDeps = {
|
||||
sink: StructuredAgentSessionEventSink
|
||||
/** The thread that owns the agent tree; falls back to the event's thread. */
|
||||
primaryThreadId: () => string | null
|
||||
activeTurn: (threadId: string) => string | null
|
||||
now?: () => number
|
||||
}
|
||||
|
||||
export class CodexSubagentRoster {
|
||||
private readonly groups = new Map<string, RosterGroup>()
|
||||
/** Latest reported total per thread, kept regardless of roster membership: a
|
||||
* usage frame can arrive before the child's first activity item, and filtering
|
||||
* at receipt would lose it permanently. Children are selected at write time;
|
||||
* the map itself is LRU-capped in `handleTokenUsage`. */
|
||||
private readonly tokensByThread = new Map<string, number>()
|
||||
private readonly now: () => number
|
||||
|
||||
constructor(private readonly deps: CodexSubagentRosterDeps) {
|
||||
this.now = deps.now ?? (() => Date.now())
|
||||
}
|
||||
|
||||
/** Consume a `subAgentActivity` item. Returns null when the item is not one. */
|
||||
handleItem(input: {
|
||||
threadId: string
|
||||
turnId: string | null
|
||||
item: CodexThreadItem
|
||||
}): StructuredAgentSessionSinkAdmission | null {
|
||||
const activity = readCodexSubagentActivity(input.item)
|
||||
if (!activity) {
|
||||
return null
|
||||
}
|
||||
// The root node is the parent turn itself, not a child it spawned.
|
||||
if (isCodexRootAgentActivity(activity)) {
|
||||
return ADMITTED
|
||||
}
|
||||
const group = this.groupFor(input.threadId, input.turnId)
|
||||
const existing = group.entries.get(activity.agentThreadId)
|
||||
const state = codexSubagentStateForKind(activity.kind)
|
||||
if (!existing) {
|
||||
// Rule: the first event for a child may be ANY kind. An `interacted` or
|
||||
// `completed` with no prior `started` creates the entry in the state its
|
||||
// kind implies rather than being dropped for lacking a roster row.
|
||||
if (group.entries.size >= MAX_CODEX_SUBAGENTS_PER_GROUP) {
|
||||
return ADMITTED
|
||||
}
|
||||
const now = this.now()
|
||||
group.entries.set(activity.agentThreadId, {
|
||||
id: activity.agentThreadId,
|
||||
label: this.claimLabel(group, codexSubagentLabel(activity)),
|
||||
state,
|
||||
startedAt: now,
|
||||
...(isTerminalSubagentState(state) ? { settledAt: now } : {})
|
||||
})
|
||||
} else if (canReplaceSubagentState(existing.state, state)) {
|
||||
// A child's own verdict latches. Re-applying the same non-terminal state
|
||||
// is a no-op, which is what makes the duplicate `item/started` +
|
||||
// `item/completed` delivery idempotent. `unverifiable` does not latch: a
|
||||
// child swept when contact was lost can still report what it actually did
|
||||
// if contact returns.
|
||||
group.entries.set(activity.agentThreadId, {
|
||||
...existing,
|
||||
state,
|
||||
...(isTerminalSubagentState(state) ? { settledAt: this.now() } : {})
|
||||
})
|
||||
}
|
||||
return this.write(group)
|
||||
}
|
||||
|
||||
/** Consume `thread/tokenUsage/updated`. Returns null when the params are not one. */
|
||||
handleTokenUsage(params: unknown): StructuredAgentSessionSinkAdmission | null {
|
||||
const usage = readCodexThreadTokenTotal(params)
|
||||
if (!usage) {
|
||||
return null
|
||||
}
|
||||
// A running total: the newest frame REPLACES the previous one. Summing
|
||||
// updates would multiply a single child's usage by its frame count.
|
||||
// Re-insert so the eviction scan below sees recency: `set` on an existing
|
||||
// key keeps its original position, which would age out an active thread.
|
||||
this.tokensByThread.delete(usage.threadId)
|
||||
this.tokensByThread.set(usage.threadId, usage.totalTokens)
|
||||
while (this.tokensByThread.size > MAX_CODEX_TOKEN_USAGE_THREADS) {
|
||||
const oldest = this.tokensByThread.keys().next().value
|
||||
if (typeof oldest !== 'string') {
|
||||
break
|
||||
}
|
||||
this.tokensByThread.delete(oldest)
|
||||
}
|
||||
for (const group of this.groups.values()) {
|
||||
if (!group.entries.has(usage.threadId)) {
|
||||
continue
|
||||
}
|
||||
const admission = this.write(group)
|
||||
if (!admission.accepted) {
|
||||
return admission
|
||||
}
|
||||
}
|
||||
return ADMITTED
|
||||
}
|
||||
|
||||
/**
|
||||
* The provider is gone, so any child still reported as working will never be
|
||||
* settled by an event: it becomes `unverifiable` — contact was lost, which is
|
||||
* NOT evidence the child exited.
|
||||
*
|
||||
* This is the ONLY sweep. A turn ending is not one: `spawn_agent` children
|
||||
* routinely outlive their turn and keep reporting into the same group.
|
||||
*/
|
||||
settleSession(): StructuredAgentSessionSinkAdmission {
|
||||
for (const group of this.groups.values()) {
|
||||
const admission = this.sweep(group)
|
||||
if (!admission.accepted) {
|
||||
return admission
|
||||
}
|
||||
}
|
||||
return ADMITTED
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
this.groups.clear()
|
||||
this.tokensByThread.clear()
|
||||
}
|
||||
|
||||
private sweep(group: RosterGroup | undefined): StructuredAgentSessionSinkAdmission {
|
||||
if (!group) {
|
||||
return ADMITTED
|
||||
}
|
||||
let changed = false
|
||||
for (const [id, entry] of group.entries) {
|
||||
if (isTerminalSubagentState(entry.state)) {
|
||||
continue
|
||||
}
|
||||
group.entries.set(id, { ...entry, state: 'unverifiable', settledAt: this.now() })
|
||||
changed = true
|
||||
}
|
||||
// A null `lastSerialized` means the previous write was refused part-way, so
|
||||
// the settled roster's last revision is queued but never published. Nothing
|
||||
// is guaranteed to write this group again, so retry here even when the sweep
|
||||
// itself changed nothing.
|
||||
return changed || group.lastSerialized === null ? this.write(group) : ADMITTED
|
||||
}
|
||||
|
||||
private groupFor(threadId: string, turnId: string | null): RosterGroup {
|
||||
const ownerThreadId = this.deps.primaryThreadId() ?? threadId
|
||||
const ownerTurnId =
|
||||
ownerThreadId === threadId ? turnId : (this.deps.activeTurn(ownerThreadId) ?? turnId)
|
||||
const groupId = codexSubagentGroupId(ownerThreadId, ownerTurnId)
|
||||
const existing = this.groups.get(groupId)
|
||||
if (existing) {
|
||||
return existing
|
||||
}
|
||||
const group: RosterGroup = {
|
||||
groupId,
|
||||
identity: codexSubagentGroupIdentity(groupId),
|
||||
entries: new Map(),
|
||||
labelCounts: new Map(),
|
||||
lastSerialized: null
|
||||
}
|
||||
this.groups.set(groupId, group)
|
||||
while (this.groups.size > MAX_CODEX_SUBAGENT_GROUPS) {
|
||||
const oldest = this.groups.keys().next().value
|
||||
if (typeof oldest !== 'string' || oldest === groupId) {
|
||||
break
|
||||
}
|
||||
this.groups.delete(oldest)
|
||||
}
|
||||
return group
|
||||
}
|
||||
|
||||
/** Two children can share a trailing path segment; the ordinal keeps their
|
||||
* rows apart without inventing a name the provider never sent. */
|
||||
private claimLabel(group: RosterGroup, label: string | null): string {
|
||||
const base = label ?? UNLABELLED_AGENT
|
||||
const seen = group.labelCounts.get(base) ?? 0
|
||||
group.labelCounts.set(base, seen + 1)
|
||||
return seen === 0 ? base : `${base} ${seen + 1}`
|
||||
}
|
||||
|
||||
private write(group: RosterGroup): StructuredAgentSessionSinkAdmission {
|
||||
const agents = [...group.entries].map(([id, entry]) => {
|
||||
const tokens = this.tokensByThread.get(id)
|
||||
if (typeof tokens !== 'number' || tokens === entry.tokens) {
|
||||
return entry
|
||||
}
|
||||
// Persisted, not merely read: the thread map is LRU-capped, and reading it
|
||||
// afresh each write would retract a count this row has already shown.
|
||||
const merged = { ...entry, tokens }
|
||||
group.entries.set(id, merged)
|
||||
return merged
|
||||
})
|
||||
const body = codexSubagentGroupBody(group.groupId, agents)
|
||||
const serialized = JSON.stringify(body)
|
||||
if (serialized === group.lastSerialized) {
|
||||
// Nothing changed — a duplicate delivery must not burn a revision.
|
||||
return ADMITTED
|
||||
}
|
||||
group.lastSerialized = serialized
|
||||
// The append coalesces per group so a burst collapses to the latest roster.
|
||||
// The publish must NOT reuse that key: the queue coalesces by key alone,
|
||||
// with no op-kind check, so a publish carrying it would splice out the
|
||||
// still-queued append and the row would never reach the journal.
|
||||
const options = { coalescingKey: `codex-subagents:${group.groupId}` }
|
||||
const admission = this.deps.sink.tryAppendItem
|
||||
? this.deps.sink.tryAppendItem(group.identity, body, options)
|
||||
: (this.deps.sink.appendItem(group.identity, body, options), ADMITTED)
|
||||
if (!admission.accepted) {
|
||||
group.lastSerialized = null
|
||||
return admission
|
||||
}
|
||||
const published = this.deps.sink.tryPublish
|
||||
? this.deps.sink.tryPublish()
|
||||
: (this.deps.sink.publish(), ADMITTED)
|
||||
if (!published.accepted) {
|
||||
// Symmetric with the append refusal above: the suppression state may only
|
||||
// advance once the revision is both queued AND published. Left set, an
|
||||
// identical replay short-circuits and the last revision of a settled
|
||||
// roster stays queued but never reaches the renderer.
|
||||
group.lastSerialized = null
|
||||
}
|
||||
return published
|
||||
}
|
||||
}
|
||||
|
||||
/** The roster row: the structured block plus the plain sentence an older client
|
||||
* renders in its place. A message whose only block is the new variant would
|
||||
* reach such a client with nothing it can draw. */
|
||||
export function codexSubagentGroupBody(
|
||||
groupId: string,
|
||||
agents: readonly NativeChatSubagentEntry[]
|
||||
): AgentJournalItemBody {
|
||||
const bounded = agents.map((agent, index) => ({
|
||||
...agent,
|
||||
id: boundSubagentField(agent.id, index),
|
||||
label: boundSubagentField(agent.label, index)
|
||||
}))
|
||||
return {
|
||||
kind: 'message',
|
||||
role: 'system',
|
||||
blocks: [
|
||||
{ type: 'text', text: subagentGroupFallbackText(bounded) },
|
||||
{ type: 'subagent-group', groupId, agents: bounded }
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
/** `id` and `label` are provider strings, so they take the bound both readers of
|
||||
* this row already clip them to. A plain length check, not the tool-output
|
||||
* bound: that one digests the whole value before it checks the length, and this
|
||||
* runs twice per child on every streamed token-usage frame.
|
||||
*
|
||||
* A clip is not identity-preserving, so a clipped value carries the child's
|
||||
* index: two ids sharing a long prefix collapse to one React key, and
|
||||
* `claimLabel` writes its ordinal at the very tail the clip removes. The index
|
||||
* is reserved out of the bound, not appended to it, because both readers
|
||||
* re-clip to the same cap and would cut a suffix that overflowed it. */
|
||||
function boundSubagentField(value: string, index: number): string {
|
||||
if (value.length <= MAX_SUBAGENT_FIELD_CHARS) {
|
||||
return value
|
||||
}
|
||||
const suffix = `…~${index}`
|
||||
const keep = MAX_SUBAGENT_FIELD_CHARS - suffix.length
|
||||
// Slicing UTF-16 units can split a surrogate pair; a lone surrogate is
|
||||
// malformed in a durable row and lossy through any non-JSON UTF-8 hop.
|
||||
const last = value.charCodeAt(keep - 1)
|
||||
const end = last >= 0xd800 && last <= 0xdbff ? keep - 1 : keep
|
||||
return `${value.slice(0, end)}${suffix}`
|
||||
}
|
||||
@@ -24,7 +24,9 @@ const AUDITED_GLOBAL_FETCH_LINES = new Map<string, number>([
|
||||
['main/orca-profiles/profile-cloud-org-members-client.ts', 1],
|
||||
['main/rate-limits/codex-fetcher.ts', 3],
|
||||
['main/runtime/relay/relay-http-client.ts', 2],
|
||||
['main/runtime/relay/relay-region-preference.ts', 3],
|
||||
['main/runtime/relay/relay-region-catalog-fetch.ts', 1],
|
||||
['main/runtime/relay/relay-region-preference.ts', 2],
|
||||
['main/runtime/relay/relay-region-probe.ts', 1],
|
||||
['main/source-control/hosted-review-api-request.ts', 1],
|
||||
['main/speech/openai-transcription-client.ts', 1],
|
||||
// Main HTTP port: one type declaration plus the Node fallback call. The fallback
|
||||
|
||||
@@ -742,11 +742,28 @@ describe('registerMobileHandlers', () => {
|
||||
})
|
||||
|
||||
it('reports the current relay broker status without exposing a toggle', () => {
|
||||
registerMobileHandlers({} as never, { getRelayStatus: () => 'registered' })
|
||||
registerMobileHandlers({} as never, { getRelayStatus: () => ({ status: 'registered' }) })
|
||||
|
||||
expect(handlers.get('mobile:getRelayStatus')?.()).toEqual({ status: 'registered' })
|
||||
})
|
||||
|
||||
it('reports the assigned relay cell alongside the status', () => {
|
||||
registerMobileHandlers({} as never, {
|
||||
getRelayStatus: () => ({ status: 'registered', cellUrl: 'https://c27.relay.example.com' })
|
||||
})
|
||||
|
||||
expect(handlers.get('mobile:getRelayStatus')?.()).toEqual({
|
||||
status: 'registered',
|
||||
cellUrl: 'https://c27.relay.example.com'
|
||||
})
|
||||
})
|
||||
|
||||
it('falls back to offline with no cell when no relay status provider is wired', () => {
|
||||
registerMobileHandlers({} as never, {})
|
||||
|
||||
expect(handlers.get('mobile:getRelayStatus')?.()).toEqual({ status: 'offline' })
|
||||
})
|
||||
|
||||
it('consumes a pending auth-failure notification only from a window renderer', () => {
|
||||
const consumePendingUnpairedDeviceAuthFailure = vi.fn(() => true)
|
||||
registerMobileHandlers({} as never, { consumePendingUnpairedDeviceAuthFailure })
|
||||
|
||||
@@ -13,7 +13,7 @@ import {
|
||||
} from '../runtime/pairing-network-interfaces'
|
||||
import { resolveAdvertisedPairingHostname } from '../runtime/pairing-endpoint'
|
||||
import type { OrcaRuntimeRpcServer } from '../runtime/runtime-rpc'
|
||||
import type { RelayBrokerStatus } from '../runtime/relay/relay-session-broker'
|
||||
import type { MobileRelayStatusDetail } from '../../shared/mobile-relay-status'
|
||||
import { encodeMobilePairingQr, type MobilePairingQrResult } from '../runtime/mobile-pairing-qr'
|
||||
import { getWindowsDefaultRouteInterfaceNames } from '../runtime/windows-default-route-interfaces'
|
||||
import {
|
||||
@@ -51,7 +51,7 @@ function toRuntimeAccessGrant(device: DeviceEntry): RuntimeAccessGrant {
|
||||
export type MobileHandlerDependencies = {
|
||||
firewallEnvironment?: WindowsMobileFirewallEnvironment
|
||||
openWindowsNetworkSettings?: () => Promise<void>
|
||||
getRelayStatus?: () => RelayBrokerStatus
|
||||
getRelayStatus?: () => MobileRelayStatusDetail
|
||||
consumePendingUnpairedDeviceAuthFailure?: (webContentsId: number) => boolean
|
||||
encodePairingQr?: (pairingUrl: string) => Promise<MobilePairingQrResult>
|
||||
getDefaultRouteInterfaceNames?: DefaultRouteInterfaceLookup
|
||||
@@ -287,9 +287,10 @@ export function registerMobileHandlers(
|
||||
return true
|
||||
})
|
||||
|
||||
ipcMain.handle('mobile:getRelayStatus', () => ({
|
||||
status: dependencies.getRelayStatus?.() ?? 'offline'
|
||||
}))
|
||||
ipcMain.handle(
|
||||
'mobile:getRelayStatus',
|
||||
(): MobileRelayStatusDetail => dependencies.getRelayStatus?.() ?? { status: 'offline' }
|
||||
)
|
||||
|
||||
ipcMain.handle('mobile:consumePendingUnpairedDeviceAuthFailure', (event) => {
|
||||
if (!isWindowRenderer(event)) {
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import { mkdir } from 'node:fs/promises'
|
||||
import type {
|
||||
AgentJournalItemBody,
|
||||
AgentJournalItemIdentity
|
||||
} from '../../../shared/agent-session-journal-types'
|
||||
import type { AgentType } from '../../../shared/agent-status-types'
|
||||
import {
|
||||
findJournalFileFormatRemnant,
|
||||
@@ -6,6 +10,7 @@ import {
|
||||
} from './journal-file-format-remnant'
|
||||
import type { JournalLoad } from './journal-open'
|
||||
import { journalRepairDisclosure, type JournalRepairDisclosure } from './journal-repair-disclosure'
|
||||
import { staleSubagentRosterRevisions } from './journal-subagent-liveness'
|
||||
|
||||
/** What any of this file's disclosures hands the store — a repair's, or the
|
||||
* pre-SQLite notice's. Same shape, and neither is only a repair. */
|
||||
@@ -36,9 +41,9 @@ export async function openJournalStoreState(input: {
|
||||
adopt: (loaded: JournalLoad) => void
|
||||
/** Republishes an anchor row for an epoch a repair emptied. */
|
||||
publishRepairEpoch: () => void
|
||||
appendDisclosure: (
|
||||
identity: JournalRepairDisclosure['identity'],
|
||||
body: JournalRepairDisclosure['body'],
|
||||
appendItem: (
|
||||
identity: AgentJournalItemIdentity,
|
||||
body: AgentJournalItemBody,
|
||||
fence: number
|
||||
) => Promise<unknown>
|
||||
agent: AgentType
|
||||
@@ -68,8 +73,9 @@ export async function openJournalStoreState(input: {
|
||||
}
|
||||
if (input.malformedRows() > 0 && !input.readOnly()) {
|
||||
const disclosure = journalRepairDisclosure({ malformedRows: input.malformedRows() })
|
||||
await input.appendDisclosure(disclosure.identity, disclosure.body, input.highestFence())
|
||||
await input.appendItem(disclosure.identity, disclosure.body, input.highestFence())
|
||||
}
|
||||
await settleStaleSubagentRosters(input, loaded)
|
||||
// Founding the epoch and appending the row are two transactions, and a
|
||||
// committed epoch sends every later open down this branch instead. Anything
|
||||
// that interrupts between them — a quit during startup restore, a failed
|
||||
@@ -92,7 +98,7 @@ export async function openJournalStoreState(input: {
|
||||
async function discloseFileFormatRemnant(input: {
|
||||
journalDir: string
|
||||
agent: AgentType
|
||||
appendDisclosure: (
|
||||
appendItem: (
|
||||
identity: JournalDisclosure['identity'],
|
||||
body: JournalDisclosure['body'],
|
||||
fence: number
|
||||
@@ -108,5 +114,32 @@ async function discloseFileFormatRemnant(input: {
|
||||
return
|
||||
}
|
||||
const disclosure = journalFileFormatRemnantDisclosure({ transcriptPath, agent: input.agent })
|
||||
await input.appendDisclosure(disclosure.identity, disclosure.body, input.highestFence())
|
||||
await input.appendItem(disclosure.identity, disclosure.body, input.highestFence())
|
||||
}
|
||||
|
||||
/**
|
||||
* Retires a `working` subagent roster the previous host never got to settle.
|
||||
*
|
||||
* Skipped on a corrupt load: that journal is still owed a rebuild from provider
|
||||
* history, and content written past the repair's free sequence retires the
|
||||
* demand for it.
|
||||
*/
|
||||
async function settleStaleSubagentRosters(
|
||||
input: {
|
||||
appendItem: (
|
||||
identity: AgentJournalItemIdentity,
|
||||
body: AgentJournalItemBody,
|
||||
fence: number
|
||||
) => Promise<unknown>
|
||||
highestFence: () => number
|
||||
readOnly: () => boolean
|
||||
},
|
||||
loaded: JournalLoad
|
||||
): Promise<void> {
|
||||
if (input.readOnly() || loaded.corrupt) {
|
||||
return
|
||||
}
|
||||
for (const revision of staleSubagentRosterRevisions(loaded.state.items.values())) {
|
||||
await input.appendItem(revision.identity, revision.body, input.highestFence())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,8 +39,7 @@ export function restoreJournalStore(
|
||||
publishRepairEpoch: () =>
|
||||
collaborators.epochController.start('unreconcilable_prefix', host.state().highestFence),
|
||||
adopt: host.adopt,
|
||||
appendDisclosure: (identity, body, fence) =>
|
||||
host.journal().appendItem(identity, body, { fence }),
|
||||
appendItem: (identity, body, fence) => host.journal().appendItem(identity, body, { fence }),
|
||||
agent: host.identity.agent,
|
||||
highestFence: () => host.state().highestFence,
|
||||
malformedRows: host.malformedRows,
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
import { mkdtemp, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import type {
|
||||
AgentJournalRenderItem,
|
||||
AgentSessionJournalIdentity
|
||||
} from '../../../shared/agent-session-journal-types'
|
||||
import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key'
|
||||
import { isSubagentGroupBlock } from '../../../shared/native-chat-types'
|
||||
import type { NativeChatSubagentEntry } from '../../../shared/native-chat-types'
|
||||
import {
|
||||
codexSubagentGroupBody,
|
||||
codexSubagentGroupIdentity
|
||||
} from '../../codex/codex-subagent-roster'
|
||||
import type { openAgentSessionJournal } from './journal-store-factory'
|
||||
import { createTrackedJournalOpener } from './journal-store-test-open'
|
||||
import { staleSubagentRosterRevisions } from './journal-subagent-liveness'
|
||||
|
||||
const IDENTITY: AgentSessionJournalIdentity = {
|
||||
sessionId: 'session-1',
|
||||
workspaceId: 'ws-1',
|
||||
hostId: 'host-1',
|
||||
agent: 'codex',
|
||||
providerHandle: { kind: 'codex', threadId: 'thread-1' }
|
||||
}
|
||||
|
||||
const GROUP_ID = 'thread-1:turn-1'
|
||||
|
||||
let root: string
|
||||
let clock = 1_000
|
||||
|
||||
function tick(): number {
|
||||
clock += 1
|
||||
return clock
|
||||
}
|
||||
|
||||
const journals = createTrackedJournalOpener()
|
||||
|
||||
async function open(overrides: Partial<Parameters<typeof openAgentSessionJournal>[0]> = {}) {
|
||||
return journals.open({
|
||||
identity: IDENTITY,
|
||||
journalDir: root,
|
||||
now: tick,
|
||||
mintEpoch: () => `epoch-${clock}`,
|
||||
...overrides
|
||||
})
|
||||
}
|
||||
|
||||
/** The row as the producer writes it: the structured block plus its twin. */
|
||||
function rosterRow(agents: NativeChatSubagentEntry[]) {
|
||||
return {
|
||||
identity: codexSubagentGroupIdentity(GROUP_ID),
|
||||
body: codexSubagentGroupBody(GROUP_ID, agents)
|
||||
}
|
||||
}
|
||||
|
||||
function renderItem(agents: NativeChatSubagentEntry[]): AgentJournalRenderItem {
|
||||
const row = rosterRow(agents)
|
||||
return {
|
||||
itemId: agentJournalItemKey(row.identity),
|
||||
revision: 1,
|
||||
body: row.body,
|
||||
sequence: 2,
|
||||
observedAt: 1
|
||||
}
|
||||
}
|
||||
|
||||
function rosterOf(body: AgentJournalRenderItem['body']): NativeChatSubagentEntry[] {
|
||||
return body.kind === 'message' ? (body.blocks.find(isSubagentGroupBlock)?.agents ?? []) : []
|
||||
}
|
||||
|
||||
function twinOf(body: AgentJournalRenderItem['body']): string | undefined {
|
||||
return body.kind === 'message'
|
||||
? body.blocks.find((block) => block.type === 'text')?.text
|
||||
: undefined
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
root = await mkdtemp(join(tmpdir(), 'orca-journal-subagents-'))
|
||||
clock = 1_000
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await journals.closeAll()
|
||||
await rm(root, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
describe('staleSubagentRosterRevisions', () => {
|
||||
it('settles a child the previous host left working, and moves the twin with it', () => {
|
||||
const revisions = staleSubagentRosterRevisions([
|
||||
renderItem([
|
||||
{ id: 'a', label: 'read_readme', state: 'working', startedAt: 10 },
|
||||
{ id: 'b', label: 'read_package', state: 'completed', startedAt: 10, settledAt: 20 }
|
||||
])
|
||||
])
|
||||
|
||||
expect(revisions).toHaveLength(1)
|
||||
expect(rosterOf(revisions[0]!.body)).toMatchObject([
|
||||
{ id: 'a', state: 'unverifiable' },
|
||||
{ id: 'b', state: 'completed' }
|
||||
])
|
||||
// Mobile reads only this sentence, so it may not go on saying `Kicked off`.
|
||||
expect(twinOf(revisions[0]!.body)).toBe('Ran 2 subagents (1 unverifiable)')
|
||||
})
|
||||
|
||||
// The child stopped being observable at an unknown moment. A stamp taken now
|
||||
// would report the time the app was down as how long the child ran.
|
||||
it('records no terminal timestamp for a child whose run length is unknown', () => {
|
||||
const revisions = staleSubagentRosterRevisions([
|
||||
renderItem([{ id: 'a', label: 'read', state: 'working', startedAt: 10 }])
|
||||
])
|
||||
|
||||
expect(rosterOf(revisions[0]!.body)[0]).not.toHaveProperty('settledAt')
|
||||
})
|
||||
|
||||
it('owes nothing for a roster whose children all settled', () => {
|
||||
expect(
|
||||
staleSubagentRosterRevisions([
|
||||
renderItem([{ id: 'a', label: 'read', state: 'completed', settledAt: 20 }])
|
||||
])
|
||||
).toEqual([])
|
||||
})
|
||||
|
||||
it('leaves rows that carry no roster alone', () => {
|
||||
expect(
|
||||
staleSubagentRosterRevisions([
|
||||
{
|
||||
itemId: 'orca:plain',
|
||||
revision: 1,
|
||||
body: { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: 'hi' }] },
|
||||
sequence: 2,
|
||||
observedAt: 1
|
||||
}
|
||||
])
|
||||
).toEqual([])
|
||||
})
|
||||
|
||||
// Appending under a fresh identity would add a second row rather than revise
|
||||
// the one on disk, so an unaddressable key is left exactly as it is.
|
||||
it('skips a row whose key cannot be parsed back to its identity', () => {
|
||||
expect(
|
||||
staleSubagentRosterRevisions([
|
||||
{ ...renderItem([{ id: 'a', label: 'r', state: 'working' }]), itemId: 'not-a-key' }
|
||||
])
|
||||
).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('journal reopen after the writing host is gone', () => {
|
||||
it('settles a persisted working roster to unverifiable, while the live row still reads working', async () => {
|
||||
const live = await open()
|
||||
const row = rosterRow([
|
||||
{ id: 'a', label: 'read_readme', state: 'working', startedAt: 10 },
|
||||
{ id: 'b', label: 'read_package', state: 'working', startedAt: 10 }
|
||||
])
|
||||
await live.appendItem(row.identity, row.body, { fence: 0 })
|
||||
|
||||
// Still the writing host: it can see the children, so the row says so.
|
||||
const beforeRestart = live.snapshot().items.at(-1)!
|
||||
expect(rosterOf(beforeRestart.body)).toMatchObject([{ state: 'working' }, { state: 'working' }])
|
||||
expect(twinOf(beforeRestart.body)).toBe('Kicked off 2 subagents')
|
||||
|
||||
// The host dies without ever settling them — no `ended`, so no session sweep.
|
||||
await live.close()
|
||||
|
||||
const reopened = await open()
|
||||
const afterRestart = reopened.snapshot().items.at(-1)!
|
||||
expect(afterRestart.itemId).toBe(beforeRestart.itemId)
|
||||
expect(rosterOf(afterRestart.body)).toMatchObject([
|
||||
{ id: 'a', state: 'unverifiable' },
|
||||
{ id: 'b', state: 'unverifiable' }
|
||||
])
|
||||
expect(twinOf(afterRestart.body)).toBe('Ran 2 subagents (2 unverifiable)')
|
||||
})
|
||||
|
||||
it('revises the row in place rather than appending a second one', async () => {
|
||||
const live = await open()
|
||||
const row = rosterRow([{ id: 'a', label: 'read', state: 'working', startedAt: 10 }])
|
||||
await live.appendItem(row.identity, row.body, { fence: 0 })
|
||||
const before = live.snapshot().items.length
|
||||
await live.close()
|
||||
|
||||
const reopened = await open()
|
||||
expect(reopened.snapshot().items).toHaveLength(before)
|
||||
expect(reopened.snapshot().items.at(-1)?.revision).toBe(2)
|
||||
})
|
||||
|
||||
it('writes nothing on a second reopen once every child is settled', async () => {
|
||||
const live = await open()
|
||||
const row = rosterRow([{ id: 'a', label: 'read', state: 'working', startedAt: 10 }])
|
||||
await live.appendItem(row.identity, row.body, { fence: 0 })
|
||||
await live.close()
|
||||
|
||||
const once = await open()
|
||||
const revision = once.snapshot().items.at(-1)?.revision
|
||||
await once.close()
|
||||
|
||||
const twice = await open()
|
||||
expect(twice.snapshot().items.at(-1)?.revision).toBe(revision)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,101 @@
|
||||
// A roster row left claiming live children by a host that is gone.
|
||||
//
|
||||
// The writing host revises its `subagent-group` rows in place while it can see
|
||||
// the children, and sweeps whatever is still `working` when the provider goes
|
||||
// away. A host that DIED — crash, quit, force-restart — does neither: its last
|
||||
// revision goes on saying `working`, and nothing replays those children, so no
|
||||
// later event can ever settle them. Opening the journal is the one moment a new
|
||||
// host can state the truth about the old one: contact was lost. That is
|
||||
// `unverifiable`, never a synthesized exit — see
|
||||
// `docs/reference/ssh-execution-boundary.md`.
|
||||
//
|
||||
// Reconciles JOURNAL ROWS, not roster state: nothing here seeds the producer's
|
||||
// in-process group map, so the roster's known limitation is untouched.
|
||||
|
||||
import {
|
||||
agentJournalItemKey,
|
||||
parseAgentJournalItemKey
|
||||
} from '../../../shared/agent-session-journal-item-key'
|
||||
import type {
|
||||
AgentJournalItemBody,
|
||||
AgentJournalItemIdentity,
|
||||
AgentJournalRenderItem
|
||||
} from '../../../shared/agent-session-journal-types'
|
||||
import {
|
||||
isSubagentGroupFallbackText,
|
||||
normalizeSubagentState,
|
||||
subagentGroupFallbackText
|
||||
} from '../../../shared/native-chat-subagent-summary'
|
||||
import {
|
||||
isSubagentGroupBlock,
|
||||
type NativeChatBlock,
|
||||
type NativeChatSubagentGroupBlock
|
||||
} from '../../../shared/native-chat-types'
|
||||
|
||||
export type JournalSubagentLivenessRevision = {
|
||||
identity: AgentJournalItemIdentity
|
||||
body: AgentJournalItemBody
|
||||
}
|
||||
|
||||
/** The revisions a reopened journal owes: one per row still claiming a live
|
||||
* child. Empty — the common case — when nothing was left mid-flight. */
|
||||
export function staleSubagentRosterRevisions(
|
||||
items: Iterable<AgentJournalRenderItem>
|
||||
): JournalSubagentLivenessRevision[] {
|
||||
const revisions: JournalSubagentLivenessRevision[] = []
|
||||
for (const item of items) {
|
||||
const body = item.body
|
||||
if (body.kind !== 'message' || !body.blocks.some(hasWorkingChild)) {
|
||||
continue
|
||||
}
|
||||
// A key that will not parse cannot be re-addressed, and appending under a
|
||||
// fresh identity would duplicate the row rather than revise it.
|
||||
const identity = parseAgentJournalItemKey(item.itemId)
|
||||
if (!identity || agentJournalItemKey(identity) !== item.itemId) {
|
||||
continue
|
||||
}
|
||||
revisions.push({ identity, body: { ...body, blocks: settleBlocks(body.blocks) } })
|
||||
}
|
||||
return revisions
|
||||
}
|
||||
|
||||
function hasWorkingChild(block: NativeChatBlock): boolean {
|
||||
return (
|
||||
isSubagentGroupBlock(block) &&
|
||||
block.agents.some((agent) => normalizeSubagentState(agent.state) === 'working')
|
||||
)
|
||||
}
|
||||
|
||||
/** No `settledAt`: the child stopped being observable at an unknown moment, and
|
||||
* stamping the reopen would report the time the app was down as how long it
|
||||
* ran. Readers already draw an unverifiable child with no stamp as having no
|
||||
* known run length. */
|
||||
function settleBlocks(blocks: readonly NativeChatBlock[]): NativeChatBlock[] {
|
||||
const settled = blocks.map((block) =>
|
||||
hasWorkingChild(block) ? settleGroup(block as NativeChatSubagentGroupBlock) : block
|
||||
)
|
||||
const rosters = settled.filter(isSubagentGroupBlock)
|
||||
const only = rosters.length === 1 ? rosters[0] : undefined
|
||||
if (!only) {
|
||||
return settled
|
||||
}
|
||||
// The plain-text twin is all a client without the block type ever shows, so it
|
||||
// has to move with the block or the two would disagree about the same row.
|
||||
const twin = subagentGroupFallbackText(only.agents)
|
||||
return settled.map((block) =>
|
||||
block.type === 'text' && isSubagentGroupFallbackText(block.text)
|
||||
? { ...block, text: twin }
|
||||
: block
|
||||
)
|
||||
}
|
||||
|
||||
function settleGroup(block: NativeChatSubagentGroupBlock): NativeChatSubagentGroupBlock {
|
||||
return {
|
||||
...block,
|
||||
agents: block.agents.map((agent) =>
|
||||
normalizeSubagentState(agent.state) === 'working'
|
||||
? { ...agent, state: 'unverifiable' as const }
|
||||
: agent
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,16 @@ describe('provider frame activity', () => {
|
||||
expect(codexProviderFrameActivity('item/reasoning/summaryPartAdded', {})).toBeNull()
|
||||
})
|
||||
|
||||
it('names a fan-out from either Codex item type that reports one', () => {
|
||||
for (const type of ['collabAgentToolCall', 'subAgentActivity']) {
|
||||
expect(
|
||||
codexProviderFrameActivity('item/started', {
|
||||
item: { type, kind: 'started', agentThreadId: 'child-1', agentPath: '/root/read' }
|
||||
})
|
||||
).toBe('Coordinating with another agent')
|
||||
}
|
||||
})
|
||||
|
||||
it('uses Claude descriptions and safe semantic status without exposing tool labels', () => {
|
||||
expect(
|
||||
claudeProviderFrameActivity('message:system:task_started', {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user