Merge remote-tracking branch 'origin/main' into brennanb2025/chatui-2-monitor-status

# Conflicts:
#	src/renderer/src/components/native-chat/NativeChatResolvedView.tsx
#	src/renderer/src/components/native-chat/NativeChatStructuredSession.tsx
#	src/renderer/src/i18n/locales/en.json
This commit is contained in:
Merge Sim
2026-09-04 18:10:39 -07:00
620 changed files with 40207 additions and 6979 deletions
@@ -91,7 +91,11 @@ jobs:
test -n "${CAPACITY_SERVICE_ACCOUNT}"
test -n "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}"
# Full history: the monitor evidence this job verifies is sealed at an ancestor commit,
# and the provenance check fails closed on a commit a shallow clone left out.
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: pnpm/action-setup@v4
with: { package_json_file: cloud/package.json }
@@ -87,13 +87,18 @@ jobs:
gate:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }}
runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 10
# Headroom for the full-history checkout the canary provenance check needs.
timeout-minutes: 15
environment: production
outputs:
cells: ${{ steps.wave.outputs.cells }}
job-mode: ${{ steps.wave.outputs.job-mode }}
steps:
# Full history: the canary authority a batch verifies is sealed at an ancestor commit, and
# the provenance check fails closed on a commit a shallow clone left out.
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/setup-node@v4
with: { node-version: 24 }
@@ -95,7 +95,11 @@ jobs:
;;
esac
# Full history: the monitor evidence this job verifies is sealed at an ancestor commit,
# and the provenance check fails closed on a commit a shallow clone left out.
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/setup-node@v4
with:
+1
View File
@@ -807,6 +807,7 @@ jobs:
src/main/agent-hooks/windows-hook-payload-delivery.test.ts
src/main/windows/windows-pty-job.win32.test.ts
src/main/windows/windows-host-job.win32.test.ts
src/main/windows-live-tree-kill.win32.test.ts
src/main/wsl/wsl-runner.test.ts
src/main/wsl/wsl-guest-environment.test.ts
src/main/wsl/wsl-invocation-boundary.test.ts
@@ -13,6 +13,41 @@ const runService = {
latestReadyRevision: 'projects/project/revisions/revision-one'
}
const sleepingStagingGcloud: GcloudClient = { accessToken: async () => 'a'.repeat(40) }
// Staging's Cloud SQL is stopped, so this inventory reads REST only and probes no endpoint.
type MigOutcome = 'ok' | 'throw' | 'missing'
const sleepingStagingFetch = (migOutcome: (migName: string) => MigOutcome): typeof fetch =>
async (input) => {
const url = new URL(String(input))
if (url.hostname === 'run.googleapis.com') return Response.json(runService)
if (url.hostname === 'sqladmin.googleapis.com') return Response.json({
state: 'STOPPED',
databaseVersion: 'POSTGRES_17',
settings: { activationPolicy: 'NEVER', availabilityType: 'ZONAL', tier: 'db-custom-1-3840' }
})
if (url.hostname === 'certificatemanager.googleapis.com') return Response.json({
managed: { domains: ['*.relay-staging.onorca.dev'], state: 'ACTIVE' }
})
if (url.pathname.includes('/instanceGroupManagers/')) {
const name = url.pathname.split('/').at(-1)!
const outcome = migOutcome(name)
if (outcome === 'throw') throw new TypeError('fetch failed')
if (outcome === 'missing') return new Response(null, { status: 404 })
return Response.json({
name,
targetSize: 0,
size: '0',
instanceGroup: `projects/project/zones/zone/instanceGroups/${name}`,
instanceTemplate: `projects/project/global/instanceTemplates/template-${name}`,
status: { isStable: true }
})
}
if (url.pathname.includes('/instanceTemplates/')) return Response.json({ properties: {} })
if (url.pathname.endsWith('/getHealth')) return Response.json([])
throw new Error(`Unexpected request to ${url.hostname}${url.pathname}`)
}
describe('readResourceInventory', () => {
it('does not delay a healthy endpoint sample', async () => {
let calls = 0
@@ -23,8 +58,10 @@ describe('readResourceInventory', () => {
calls += 1
return new Response(null, { status: 200 })
},
async () => {
waits += 1
{
wait: async () => {
waits += 1
}
}
)
@@ -45,8 +82,10 @@ describe('readResourceInventory', () => {
calls.set(path, call)
return new Response(null, { status: path === '/ready' && call === 1 ? 503 : 200 })
},
async (ms) => {
waits.push(ms)
{
wait: async (ms) => {
waits.push(ms)
}
}
)
@@ -65,17 +104,130 @@ describe('readResourceInventory', () => {
calls += 1
return new Response(null, { status: 503 })
},
async (ms) => {
waits.push(ms)
{
wait: async (ms) => {
waits.push(ms)
}
}
)
expect(result.health).toBe(false)
expect(result.ready).toBe(false)
expect(calls).toBe(4)
// A refusing endpoint is a reading, so only the independent retry runs.
expect(waits).toEqual([11_000])
})
it('treats a thrown fetch as no reading and re-asks that path once', async () => {
const calls: string[] = []
const waits: number[] = []
const result = await probeEndpointHealth(
'https://c9.relay.onorca.dev',
async (input) => {
const path = new URL(String(input)).pathname
calls.push(path)
if (path === '/health' && calls.filter((call) => call === '/health').length === 1) {
throw new TypeError('fetch failed')
}
return new Response(null, { status: 200 })
},
{
wait: async (ms) => {
waits.push(ms)
}
}
)
expect(result.health).toBe(true)
expect(result.ready).toBe(true)
expect(calls.filter((call) => call === '/health')).toEqual(['/health', '/health'])
expect(waits).toEqual([1_000])
})
it('fails closed when both attempts of a path throw', async () => {
const calls: string[] = []
const waits: number[] = []
const result = await probeEndpointHealth(
'https://c9.relay.onorca.dev',
async (input) => {
const path = new URL(String(input)).pathname
calls.push(path)
if (path === '/health') throw new TypeError('fetch failed')
return new Response(null, { status: 200 })
},
{
wait: async (ms) => {
waits.push(ms)
}
}
)
expect(result.health).toBe(false)
expect(calls.filter((call) => call === '/health')).toHaveLength(4)
expect(waits).toEqual([1_000, 11_000, 1_000])
})
it('accepts an auth-shaped endpoint that serves no readiness path', async () => {
const calls: string[] = []
const waits: number[] = []
const result = await probeEndpointHealth(
'https://login.onorca.dev',
async (input) => {
const path = new URL(String(input)).pathname
calls.push(path)
return new Response(null, { status: path === '/ready' ? 404 : 200 })
},
{
requiresReady: false,
wait: async (ms) => {
waits.push(ms)
}
}
)
expect(result.health).toBe(true)
expect(result.ready).toBeNull()
expect(calls).toEqual(['/health'])
expect(waits).toEqual([])
})
it('still requires readiness for the director and cells', async () => {
const waits: number[] = []
const result = await probeEndpointHealth(
'https://relay.onorca.dev',
async (input) => new Response(null, {
status: new URL(String(input)).pathname === '/ready' ? 503 : 200
}),
{
wait: async (ms) => {
waits.push(ms)
}
}
)
expect(result.health).toBe(true)
expect(result.ready).toBe(false)
expect(waits).toEqual([11_000])
})
it('measures latency as the answering round trip, not the retry delay', async () => {
let healthCalls = 0
const result = await probeEndpointHealth(
'https://c9.relay.onorca.dev',
async (input) => {
if (new URL(String(input)).pathname !== '/health') return new Response(null, { status: 200 })
healthCalls += 1
if (healthCalls === 1) throw new TypeError('fetch failed')
return new Response(null, { status: 200 })
},
{ wait: async (ms) => await new Promise((resolve) => setTimeout(resolve, Math.min(ms, 60))) }
)
expect(result.health).toBe(true)
expect(result.latencyMs).not.toBeNull()
expect(result.latencyMs!).toBeLessThan(60)
})
it('uses aggregate REST inventory without probing sleeping staging endpoints', async () => {
const gcloud: GcloudClient = { accessToken: async () => 'a'.repeat(40) }
let publicProbeCalls = 0
@@ -132,6 +284,74 @@ describe('readResourceInventory', () => {
expect(JSON.stringify(result)).not.toContain('SECRET_TEXT')
})
it('re-asks a MIG read that failed once before calling a cell powered-unknown', async () => {
const parkedCell = RELAY_OPS_ENVIRONMENTS.staging.cells[0]!
const waits: number[] = []
let parkedMigCalls = 0
const result = await readResourceInventory(
RELAY_OPS_ENVIRONMENTS.staging,
sleepingStagingGcloud,
sleepingStagingFetch((migName) => {
if (!migName.endsWith(parkedCell.hostname)) return 'ok'
parkedMigCalls += 1
return parkedMigCalls === 1 ? 'throw' : 'ok'
}),
{ wait: async (ms) => { waits.push(ms) } }
)
const parked = result.cells.find((cell) => cell.cellId === parkedCell.cellId)!
// The MIG was fine and parked at zero; one transient read must not erase that reading.
expect(parked.targetSize).toBe(0)
expect(parkedMigCalls).toBe(2)
expect(waits).toEqual([1_000])
expect(result.warnings).toEqual([])
})
it('reports a MIG unavailable only when the retry fails too', async () => {
const parkedCell = RELAY_OPS_ENVIRONMENTS.staging.cells[0]!
const waits: number[] = []
let parkedMigCalls = 0
const result = await readResourceInventory(
RELAY_OPS_ENVIRONMENTS.staging,
sleepingStagingGcloud,
sleepingStagingFetch((migName) => {
if (!migName.endsWith(parkedCell.hostname)) return 'ok'
parkedMigCalls += 1
return 'throw'
}),
{ wait: async (ms) => { waits.push(ms) } }
)
const parked = result.cells.find((cell) => cell.cellId === parkedCell.cellId)!
expect(parked.targetSize).toBeNull()
expect(parked.backendHealth).toBe('unknown')
expect(parkedMigCalls).toBe(2)
expect(waits).toEqual([1_000])
expect(result.warnings).toEqual([
`${parkedCell.hostname.toUpperCase()} MIG inventory is unavailable.`
])
})
it('does not re-ask a MIG read the API answered with 404', async () => {
const missingCell = RELAY_OPS_ENVIRONMENTS.staging.cells[0]!
const waits: number[] = []
let missingMigCalls = 0
const result = await readResourceInventory(
RELAY_OPS_ENVIRONMENTS.staging,
sleepingStagingGcloud,
sleepingStagingFetch((migName) => {
if (!migName.endsWith(missingCell.hostname)) return 'ok'
missingMigCalls += 1
return 'missing'
}),
{ wait: async (ms) => { waits.push(ms) } }
)
expect(result.cells.find((cell) => cell.cellId === missingCell.cellId)!.targetSize).toBeNull()
expect(missingMigCalls).toBe(1)
expect(waits).toEqual([])
})
it('represents missing credentials as unknown inventory, never sleeping', async () => {
const gcloud: GcloudClient = {
accessToken: async () => { throw new Error('sensitive context') }
+94 -23
View File
@@ -102,6 +102,9 @@ export type ResourceInventory = {
const unavailableEndpoint = (): EndpointHealth => ({ health: null, ready: null, latencyMs: null })
const independentEndpointRetryDelayMs = 11_000
const transientProbeRetryDelayMs = 1_000
const sleep = async (ms: number): Promise<void> =>
await new Promise((resolvePromise) => setTimeout(resolvePromise, ms))
function finalSegment(value: string): string {
return value.split('/').at(-1) ?? value
@@ -120,6 +123,12 @@ function parseService(value: unknown): ServiceInventory {
}
}
class GoogleApiError extends Error {
constructor(readonly status: number) {
super(`Google API returned ${status}`)
}
}
async function googleRequest(
fetchImpl: typeof fetch,
token: string,
@@ -134,45 +143,97 @@ async function googleRequest(
},
signal: AbortSignal.timeout(30_000)
})
if (!response.ok) throw new Error(`Google API returned ${response.status}`)
if (!response.ok) throw new GoogleApiError(response.status)
return await response.json()
}
async function endpointProbe(origin: string, fetchImpl: typeof fetch): Promise<EndpointHealth> {
const startedAt = performance.now()
const check = async (path: '/health' | '/ready'): Promise<boolean> => {
// A 404 is the API's answer about the resource; anything else is the absence of a reading, so re-ask.
async function readOnceMore(
read: () => Promise<unknown>,
wait: (ms: number) => Promise<void>
): Promise<unknown> {
try {
return await read()
} catch (error) {
if (error instanceof GoogleApiError && error.status === 404) throw error
await wait(transientProbeRetryDelayMs)
return await read()
}
}
// A reading the endpoint actually produced: ok is its answer, latencyMs is that answer's round trip.
type PathReading = { ok: boolean; latencyMs: number | null }
async function probePath(
origin: string,
path: '/health' | '/ready',
fetchImpl: typeof fetch,
wait: (ms: number) => Promise<void>
): Promise<PathReading> {
// null means the request never produced an answer (DNS/TCP/TLS failure or the 8s abort).
const attempt = async (): Promise<PathReading | null> => {
const startedAt = performance.now()
try {
const response = await fetchImpl(`${origin}${path}`, {
redirect: 'error',
signal: AbortSignal.timeout(8_000)
})
return response.ok
return { ok: response.ok, latencyMs: Math.round(performance.now() - startedAt) }
} catch {
return false
return null
}
}
const [health, ready] = await Promise.all([check('/health'), check('/ready')])
return { health, ready, latencyMs: Math.round(performance.now() - startedAt) }
const first = await attempt()
if (first) return first
// A thrown fetch is the absence of a reading, not an unhealthy answer, so re-ask before concluding.
await wait(transientProbeRetryDelayMs)
return (await attempt()) ?? { ok: false, latencyMs: null }
}
async function endpointProbe(
origin: string,
fetchImpl: typeof fetch,
requiresReady: boolean,
wait: (ms: number) => Promise<void>
): Promise<EndpointHealth> {
const [health, ready] = await Promise.all([
probePath(origin, '/health', fetchImpl, wait),
requiresReady ? probePath(origin, '/ready', fetchImpl, wait) : null
])
// Latency is the slowest answering round trip in this probe; retry delays are not serving latency.
const latencies = [health.latencyMs, ready?.latencyMs ?? null].filter(
(value): value is number => value !== null
)
return {
health: health.ok,
ready: ready ? ready.ok : null,
latencyMs: latencies.length > 0 ? Math.max(...latencies) : null
}
}
export type EndpointProbeOptions = {
// Auth serves no /ready by design, so it is judged on /health and latency alone.
requiresReady?: boolean
wait?: (ms: number) => Promise<void>
}
export async function probeEndpointHealth(
origin: string,
fetchImpl: typeof fetch,
wait: (ms: number) => Promise<void> = async (ms) =>
await new Promise((resolvePromise) => setTimeout(resolvePromise, ms))
options: EndpointProbeOptions = {}
): Promise<EndpointHealth> {
const first = await endpointProbe(origin, fetchImpl)
if (
first.health &&
first.ready &&
first.latencyMs !== null &&
first.latencyMs <= INCIDENT_MONITOR_THRESHOLDS.endpointLatencyMs
) {
return first
}
const requiresReady = options.requiresReady ?? true
const wait = options.wait ?? sleep
const accepted = (probe: EndpointHealth): boolean =>
probe.health === true &&
(!requiresReady || probe.ready === true) &&
probe.latencyMs !== null &&
probe.latencyMs <= INCIDENT_MONITOR_THRESHOLDS.endpointLatencyMs
const first = await endpointProbe(origin, fetchImpl, requiresReady, wait)
if (accepted(first)) return first
// Outwait Relay's ten-second readiness cache before treating the retry as independent.
await wait(independentEndpointRetryDelayMs)
return await endpointProbe(origin, fetchImpl)
return await endpointProbe(origin, fetchImpl, requiresReady, wait)
}
function imageDigest(template: z.infer<typeof TemplateSchema>): string | null {
@@ -285,11 +346,17 @@ function unavailableInventory(environment: RelayOpsEnvironment, warning: string)
}
}
export type ResourceInventoryOptions = {
wait?: (ms: number) => Promise<void>
}
export async function readResourceInventory(
environment: RelayOpsEnvironment,
gcloud: GcloudClient,
fetchImpl: typeof fetch = fetch
fetchImpl: typeof fetch = fetch,
options: ResourceInventoryOptions = {}
): Promise<ResourceInventory> {
const wait = options.wait ?? sleep
let token: string
try {
token = await gcloud.accessToken()
@@ -316,7 +383,10 @@ export async function readResourceInventory(
token,
`https://certificatemanager.googleapis.com/v1/projects/${environment.project}/locations/global/certificates/${environment.certificateName}`
),
...environment.cells.map((cell) => googleRequest(fetchImpl, token, migUrl(cell)))
// One transient Compute read must never become a verdict on a cell's power state.
...environment.cells.map((cell) =>
readOnceMore(async () => await googleRequest(fetchImpl, token, migUrl(cell)), wait)
)
])
const warnings: string[] = []
const directorValue = parsed(settled[0]!, RunServiceSchema, 'Director service inventory is unavailable.', warnings)
@@ -338,7 +408,8 @@ export async function readResourceInventory(
? [unavailableEndpoint(), unavailableEndpoint()]
: await Promise.all([
probeEndpointHealth(environment.directorOrigin, fetchImpl),
probeEndpointHealth(environment.authOrigin, fetchImpl)
// The auth service exposes no /ready, so requiring it would fail every first probe.
probeEndpointHealth(environment.authOrigin, fetchImpl, { requiresReady: false })
])
const cells = await Promise.all(environment.cells.map((cell, index) =>
readCell(environment, cell, migValues[index] ?? null, token, fetchImpl)
+61 -19
View File
@@ -645,9 +645,17 @@ export class RelayAssignmentStore {
): Promise<RelayAssignment | null> {
const now = this.now()
return await this.database.transaction(async (transaction) => {
const lockedCells = inventoryFirst
? await this.lockCellInventory(transaction, lockMode)
// Why: the retry exists to take a cell row before the assignment row, the
// order placement uses. It only ever needs the one cell this host is
// pinned to, so read the pin unlocked and lock that row alone; taking all
// 23 queued every sticky refresh in the fleet behind every other one.
const pinnedCellId = inventoryFirst
? await this.pinnedCellId(transaction, identity)
: undefined
const lockedCells =
pinnedCellId === undefined
? undefined
: await this.lockCellRows(transaction, [pinnedCellId], lockMode)
const existing = await this.assignmentRow(transaction, identity, inventoryFirst)
if (!existing) return null
const activityLeases = await this.lockAssignmentActivities(transaction, identity, true)
@@ -661,6 +669,11 @@ export class RelayAssignmentStore {
}
const currentCellId = text(existing, 'cell_id')
// The pin moved between the unlocked read and the assignment lock, so the
// row held is the wrong one. Same recovery as losing the lock: retry.
if (pinnedCellId !== undefined && pinnedCellId !== currentCellId) {
throw new Error('database_lock_unavailable')
}
const hadControl = holdsControlLease(
activityLeases,
currentCellId,
@@ -701,14 +714,9 @@ export class RelayAssignmentStore {
if (hadControl) {
await this.touchAssignment(transaction, identity, leaseExpiresAt, now)
} else {
const nextReservation = integer(currentRow, 'reserved_requests') + 1
if (nextReservation > integer(currentRow, 'capacity_requests')) {
throw new Error('relay_capacity_exhausted')
}
await transaction.query(
`UPDATE relay_cells SET reserved_requests = ?, updated_at = ? WHERE cell_id = ?`,
[nextReservation, now, currentCellId]
)
// Delta, not the value read from the snapshot: an absolute write here
// would clobber any concurrent movement of the same counter.
await this.adjustCellReservationAtomically(transaction, currentCellId, 1)
await this.adjustActivityCount(transaction, identity, 'control', 1, leaseExpiresAt, now)
await this.insertPendingControlLease(
transaction,
@@ -6864,13 +6872,24 @@ export class RelayAssignmentStore {
targetCellId
]
)
const cells = await this.lockCellInventory(transaction, 'pool-default')
// Only the two cells this repairs need holding. The id set below is an
// existence check against a table that only reconcileCells writes, so it
// reads unlocked instead of dragging the other 21 rows into the section.
const cellIds = new Set(
(await transaction.query(`SELECT cell_id FROM relay_cells`)).map((row) =>
text(row, 'cell_id')
)
)
const cells = await this.lockCellRows(
transaction,
[sourceCellId, targetCellId],
'pool-default'
)
const assignmentKeys = new Set(
assignments.map((row) =>
assignmentKey(text(row, 'user_id'), text(row, 'relay_host_id'))
)
)
const cellIds = new Set(cells.map((row) => text(row, 'cell_id')))
const assignmentCounts = new Map<
string,
{ counts: Record<AssignmentActivityKind, number>; leaseExpiresAt: number }
@@ -6924,9 +6943,7 @@ export class RelayAssignmentStore {
)
}
for (const row of cells.filter((cell) =>
[sourceCellId, targetCellId].includes(text(cell, 'cell_id'))
)) {
for (const row of cells) {
const cellId = text(row, 'cell_id')
const expected = cellUnits.get(cellId) ?? 0
if (expected > integer(row, 'capacity_requests')) {
@@ -6958,16 +6975,40 @@ export class RelayAssignmentStore {
// Per-connection paths touch one or two cells. Locking exactly those rows,
// in the same ascending order the inventory lock uses (ORDER BY fixes the
// row-lock order), keeps them off the fleet-wide lock without a cycle.
private async lockCellRows(database: RelayDatabase, cellIds: string[]): Promise<SqlRow[]> {
// The wait policy follows the caller for the same reason the inventory lock's
// does: a sweep must not fail terminally on ordinary contention. Hold time is
// deliberately not sampled here — the metric tracks the fleet-wide lock these
// rows replace, and mixing in short single-row holds would flatter it.
private async lockCellRows(
database: RelayDatabase,
cellIds: string[],
mode: CellInventoryLockMode = 'request'
): Promise<SqlRow[]> {
const distinct = [...new Set(cellIds)]
const { measureHoldMs: _sampled, ...wait } = cellInventoryLockOptions(mode)
return await database.queryLocked(
`SELECT * FROM relay_cells WHERE cell_id IN (${distinct.map(() => '?').join(', ')})
ORDER BY cell_id ASC`,
distinct,
{ lockTimeoutMs: CELL_INVENTORY_LOCK_TIMEOUT_MS }
wait
)
}
// Unlocked on purpose: this only names the row to lock next, and the caller
// re-checks the pin once the assignment row is held.
private async pinnedCellId(
database: RelayDatabase,
identity: AssignmentIdentity
): Promise<string | undefined> {
const row = (
await database.query(
`SELECT cell_id FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
)[0]
return row ? text(row, 'cell_id') : undefined
}
private async lockGeneralCellInventory(
database: RelayDatabase,
mode: CellInventoryLockMode
@@ -6986,10 +7027,11 @@ export class RelayAssignmentStore {
private async leastLoadedCell(
database: RelayDatabase,
lockedCells: SqlRow[] | undefined,
// Required: the one caller has already locked the inventory it selects from,
// and an optional parameter left a second fleet-wide lock reachable here.
rows: SqlRow[],
preferredRegion: RelayRegion
): Promise<CellRow | null> {
const rows = lockedCells ?? (await this.lockCellInventory(database, 'pool-default'))
const regions = new Map(
(await database.query(`SELECT cell_id, region FROM relay_cell_regions`)).map((row) => [
text(row, 'cell_id'),
@@ -18,7 +18,9 @@ type CensusEntry = { method: string; mode: CensusMode; reach: Reachability }
// assignment-store.ts, in source order. A new site fails this test until it is
// classified here, which is the point.
const CENSUS: CensusEntry[] = [
{ method: 'assignStickyOnce', mode: 'caller', reach: 'both' },
// assignStickyOnce is gone from this list: its retry now locks only the row
// the host is pinned to (lockCellRows), which is what a sticky refresh
// touches. Placement below is the one genuinely fleet-wide decision left.
{ method: 'assignOnce', mode: 'caller', reach: 'both' },
{ method: 'assignOnce', mode: 'caller', reach: 'both' },
{ method: 'assignOnce', mode: 'nowait', reach: 'both' },
@@ -49,8 +51,9 @@ const CENSUS: CensusEntry[] = [
{ method: 'abortExpiredEvacuations', mode: 'nowait', reach: 'sweep' },
{ method: 'releaseExpiredActivityLeases', mode: 'nowait', reach: 'sweep' },
{ method: 'releaseExpiredActivity', mode: 'nowait', reach: 'sweep' },
{ method: 'reconcileReservationAccounting', mode: 'pool-default', reach: 'both' },
{ method: 'leastLoadedCell', mode: 'pool-default', reach: 'both' }
// reconcileReservationAccounting and leastLoadedCell are gone too: the first
// repairs exactly two cells' counters and now holds only those rows, and the
// second selects from the inventory its single caller has already locked.
]
// Every inline `FROM relay_cells ... FOR UPDATE` outside the named lock helpers,
@@ -0,0 +1,206 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { openRelayDatabase, type RelayDatabase } from './database.js'
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
// Sorted ascending, and the host is pinned to the LAST id on purpose: the
// fleet-wide lock is one ordered scan, so it holds every earlier row while it
// waits on the pinned one. Pinning to the first id would make the two locking
// models indistinguishable.
const cells = ['a', 'b', 'c'].map((suffix) => ({
id: `percell-postgres-${suffix}`,
url: `https://percell-postgres-${suffix}.example.com`,
capacityRequests: 1_000,
connectionHardCap: 600 as const,
connectionUnobservedBound: 50
}))
const [cellA, cellB, cellC] = cells as [(typeof cells)[0], (typeof cells)[0], (typeof cells)[0]]
const identity = { userId: 'percell-postgres-user', relayHostId: 'percellhost00001' }
function heartbeat(cell: (typeof cells)[number]) {
return {
cellId: cell.id,
cellUrl: cell.url,
cellIncarnation: '11111111-1111-4111-8111-111111111111',
startedAt: 50,
ready: true,
observedRequests: 0,
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0,
connectionInclusionWatermark: 1,
connectionHardCap: 600 as const,
connectionUnobservedBound: 50
}
}
describePostgres('PostgreSQL per-cell inventory locking', () => {
const databases: RelayDatabase[] = []
beforeAll(async () => {
for (let index = 0; index < 3; index++) {
databases.push(await openRelayDatabase({ databaseUrl, dataDir: '' }))
}
})
async function removeTestRows(database: RelayDatabase): Promise<void> {
await database.query(
`DELETE FROM relay_control_connection_reservations WHERE user_id LIKE 'percell-postgres-%'`
)
for (const table of [
'relay_assignment_activity_leases',
'relay_post_drain_migration_pins',
'relay_assignment_migration_incarnations',
'relay_assignment_migrations',
'relay_assignment_region_preferences',
'relay_assignments'
]) {
await database.query(`DELETE FROM ${table} WHERE user_id LIKE 'percell-postgres-%'`)
}
for (const cell of cells) {
for (const table of [
'relay_cell_connection_snapshots',
'relay_cell_connection_runtime',
'relay_cell_connection_limits',
'relay_cell_runtime',
'relay_cells'
]) {
await database.query(`DELETE FROM ${table} WHERE cell_id = ?`, [cell.id])
}
}
}
afterAll(async () => {
if (databases[0]) await removeTestRows(databases[0])
for (const connection of databases) await connection.close()
})
async function pinHostToLastCell(store: RelayAssignmentStore): Promise<void> {
await store.reconcileCells(cells)
for (const cell of cells) await store.recordCellHeartbeat(heartbeat(cell))
await store.setCellEnabled(cellA.id, false)
await store.setCellEnabled(cellB.id, false)
const assignment = await store.assign(identity)
expect(assignment.cellId).toBe(cellC.id)
await store.setCellEnabled(cellA.id, true)
await store.setCellEnabled(cellB.id, true)
}
async function lockWaiterAppeared(database: RelayDatabase): Promise<boolean> {
const deadline = Date.now() + 4_000
while (Date.now() < deadline) {
const rows = await database.query(
`SELECT count(*) AS waiting FROM pg_stat_activity
WHERE datname = current_database() AND wait_event_type = 'Lock'`
)
if (Number(rows[0]!.waiting) > 0) return true
await new Promise((resolve) => setTimeout(resolve, 10))
}
return false
}
// Why: a sticky refresh whose first NOWAIT probe loses retries by taking a
// cell row before the assignment row. That retry used to take the whole
// inventory, so one busy cell stalled every other cell's reconnects.
it('waits only on the pinned cell row while refreshing a sticky assignment', async () => {
await removeTestRows(databases[0]!)
const store = new RelayAssignmentStore(databases[0]!, () => 100)
await pinHostToLastCell(store)
// A host whose control lease was already reaped still holds its pin; that
// is the shape that reaches the cell-row probe instead of touchAssignment.
await databases[0]!.query(
`DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`,
[identity.userId]
)
let releaseRow!: () => void
const rowReleased = new Promise<void>((resolve) => {
releaseRow = resolve
})
let rowHeld!: () => void
const rowHeldPromise = new Promise<void>((resolve) => {
rowHeld = resolve
})
const holder = databases[1]!.transaction(async (transaction) => {
await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cellC.id])
rowHeld()
await rowReleased
})
await rowHeldPromise
const refresh = store.assign(identity)
expect(await lockWaiterAppeared(databases[2]!)).toBe(true)
// The refresh is blocked on cell C. Every earlier row must still be free:
// the ordered fleet-wide scan would be holding both of them by now.
const heldWhileRefreshWaits: string[] = []
await databases[2]!.transaction(async (transaction) => {
for (const cell of [cellA, cellB]) {
try {
await transaction.queryLocked(
`SELECT * FROM relay_cells WHERE cell_id = ?`,
[cell.id],
{ failIfUnavailable: true }
)
} catch {
heldWhileRefreshWaits.push(cell.id)
}
}
})
releaseRow()
await holder
expect(heldWhileRefreshWaits).toEqual([])
expect((await refresh).cellId).toBe(cellC.id)
}, 15_000)
// Why: the counter moves by a delta now instead of an absolute value read
// from a snapshot, so concurrent movement on the same cell must still sum.
it('keeps a cell reservation exact under concurrent same-cell activity', async () => {
await removeTestRows(databases[0]!)
const store = new RelayAssignmentStore(databases[0]!, () => 100)
await store.reconcileCells(cells)
for (const cell of cells) await store.recordCellHeartbeat(heartbeat(cell))
await store.setCellEnabled(cellA.id, false)
await store.setCellEnabled(cellB.id, false)
const hosts = Array.from({ length: 6 }, (_, index) => ({
userId: `percell-postgres-user-${index}`,
relayHostId: `percellhost0000${index}`
}))
const stores = databases.map((database) => new RelayAssignmentStore(database, () => 100))
await Promise.all(hosts.map((host, index) => stores[index % stores.length]!.assign(host)))
// One splice each (2 units) on the same cell, from three connections at once.
await Promise.all(
hosts.map((host, index) =>
stores[index % stores.length]!.acquireActivity(host, {
activityId: `splice:percell-${index}`,
kind: 'splice',
cellId: cellC.id
})
)
)
const afterAcquire = await databases[0]!.query(
`SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`,
[cellC.id]
)
// 6 pending control grants + 6 splices at 2 units each.
expect(Number(afterAcquire[0]!.reserved_requests)).toBe(6 + 12)
await Promise.all(
hosts.map((host, index) =>
stores[index % stores.length]!.releaseActivity(host, `splice:percell-${index}`)
)
)
const afterRelease = await databases[0]!.query(
`SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`,
[cellC.id]
)
expect(Number(afterRelease[0]!.reserved_requests)).toBe(6)
await store.setCellEnabled(cellA.id, true)
await store.setCellEnabled(cellB.id, true)
}, 15_000)
})
@@ -2,7 +2,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
const fakes = vi.hoisted(() => ({
configs: [] as Array<Record<string, unknown>>,
query: vi.fn(async () => ({ rows: [], rowCount: 0 })),
// Pool construction and pool shutdown interleaved, so "the schema pool is
// gone before the serving pool opens" is checkable rather than assumed.
lifecycle: [] as string[],
query: vi.fn(async (_sql: string) => ({ rows: [], rowCount: 0 })),
release: vi.fn(),
end: vi.fn(async () => undefined)
}))
@@ -13,20 +16,37 @@ vi.mock('pg', () => ({
totalCount = 1
idleCount = 1
waitingCount = 0
end = fakes.end
on = vi.fn()
connect = vi.fn(async () => ({ query: fakes.query, release: fakes.release }))
private readonly label: string
constructor(config: Record<string, unknown>) {
fakes.configs.push(config)
this.label = `max=${String(config.max)} statement_timeout=${String(config.statement_timeout)}`
fakes.lifecycle.push(`open ${this.label}`)
}
async end(): Promise<void> {
fakes.lifecycle.push(`end ${this.label}`)
await fakes.end()
}
}
}
}))
import { openRelayDatabase } from './database.js'
import { openRelayDatabase, relayPostgresStatementTimeoutMs } from './database.js'
import { applyPostgresSchema } from './postgres-schema-startup.js'
const SCHEMA_POOL = {
max: 1,
application_name: 'orca-relay/director/director/schema',
connectionTimeoutMillis: 2_000,
// Why: DDL must not inherit the request deadline.
statement_timeout: 0,
lock_timeout: 1_000,
idle_in_transaction_session_timeout: 5_000
}
afterEach(() => {
vi.restoreAllMocks()
})
@@ -34,9 +54,11 @@ afterEach(() => {
describe('PostgreSQL relay deadlines', () => {
beforeEach(() => {
fakes.configs.length = 0
fakes.lifecycle.length = 0
fakes.query.mockClear()
fakes.release.mockClear()
fakes.end.mockClear()
delete process.env.ORCA_RELAY_POSTGRES_STATEMENT_TIMEOUT_MS
})
it('bounds pool acquisition, statements, locks, and abandoned transactions', async () => {
@@ -48,6 +70,7 @@ describe('PostgreSQL relay deadlines', () => {
})
expect(fakes.configs).toEqual([
expect.objectContaining(SCHEMA_POOL),
expect.objectContaining({
max: 3,
application_name: 'orca-relay/director/director',
@@ -59,6 +82,110 @@ describe('PostgreSQL relay deadlines', () => {
])
await database.close()
})
// Why: an untimed session left open would be a standing way for request work
// to escape the deadline this whole pool config exists to enforce.
it('closes the untimed schema pool before the serving pool opens', async () => {
const database = await openRelayDatabase({
databaseUrl: 'postgresql://relay:secret@127.0.0.1:5432/relay',
dataDir: './unused',
poolMax: 3,
applicationName: 'orca-relay/director/director'
})
expect(fakes.lifecycle).toEqual([
'open max=1 statement_timeout=0',
'end max=1 statement_timeout=0',
'open max=3 statement_timeout=5000'
])
await database.close()
})
it('applies the schema on the untimed pool, never on the serving one', async () => {
fakes.query.mockClear()
const ddl: string[] = []
fakes.query.mockImplementation(async (sql: string) => {
// Every statement issued before the serving pool exists is schema work.
if (fakes.lifecycle.length === 1) ddl.push(sql)
return { rows: [], rowCount: 0 }
})
const database = await openRelayDatabase({
databaseUrl: 'postgresql://relay:secret@127.0.0.1:5432/relay',
dataDir: './unused'
})
expect(ddl.length).toBeGreaterThan(0)
// Statements can open with a leading `--` rationale comment.
const body = (statement: string): string =>
statement.replace(/^(?:\s*--[^\n]*\n)*\s*/, '')
expect(ddl.every((statement) => /^CREATE\b/i.test(body(statement)))).toBe(true)
// The backfill is DML, so it stays on the deadline-bearing serving pool.
expect(ddl.some((statement) => statement.includes('INSERT INTO'))).toBe(false)
await database.close()
})
it('takes the serving statement deadline from the environment', async () => {
process.env.ORCA_RELAY_POSTGRES_STATEMENT_TIMEOUT_MS = '2500'
const database = await openRelayDatabase({
databaseUrl: 'postgresql://relay:secret@127.0.0.1:5432/relay',
dataDir: './unused'
})
expect(fakes.configs).toEqual([
expect.objectContaining({ statement_timeout: 0 }),
expect.objectContaining({ statement_timeout: 2_500 })
])
await database.close()
})
it.each(['0', '-1', '2.5', 'soon', ' '])(
'refuses %s as a statement deadline instead of running unbounded',
(value) => {
expect(() =>
relayPostgresStatementTimeoutMs({ ORCA_RELAY_POSTGRES_STATEMENT_TIMEOUT_MS: value })
).toThrow('invalid_statement_timeout')
}
)
it.each([undefined, ''])('defaults to 5s when the environment says %s', (value) => {
expect(
relayPostgresStatementTimeoutMs(
value === undefined ? {} : { ORCA_RELAY_POSTGRES_STATEMENT_TIMEOUT_MS: value }
)
).toBe(5_000)
})
// Why: a statement deadline that reaches the caller as a crash converts a
// transient stall into a failed assignment. It aborts the transaction exactly
// as a lock timeout does, so it belongs on the same bounded retry.
it('retries a statement timeout on a fresh client', async () => {
vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const database = await openRelayDatabase({
databaseUrl: 'postgresql://relay:secret@127.0.0.1:5432/relay',
dataDir: './unused'
})
let attempts = 0
const result = await database.transaction(async (transaction) => {
attempts += 1
if (attempts === 1) {
await transaction.query('SELECT 1')
throw Object.assign(new Error('canceling statement due to statement timeout'), {
code: '57014'
})
}
return 'committed'
})
expect(result).toBe('committed')
expect(attempts).toBe(2)
expect(console.warn).toHaveBeenCalledWith(
expect.stringContaining('"event":"orca_relay_postgres_transaction_retry"')
)
expect(console.warn).toHaveBeenCalledWith(expect.stringContaining('"code":"57014"'))
await database.close()
})
})
describe('PostgreSQL schema startup', () => {
@@ -0,0 +1,98 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { openRelayDatabase, type RelayDatabase } from './database.js'
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
const applicationName = 'orca-relay/statement-timeout-postgres'
describePostgres('PostgreSQL statement deadline', () => {
const databases: RelayDatabase[] = []
beforeAll(async () => {
databases.push(await openRelayDatabase({ databaseUrl, dataDir: '' }))
})
afterAll(async () => {
for (const database of databases) await database.close()
})
it('serves requests under the configured deadline', async () => {
const database = await openRelayDatabase({ databaseUrl, dataDir: '', statementTimeoutMs: 300 })
databases.push(database)
expect(await database.query(`SELECT current_setting('statement_timeout') AS statement_timeout`)).toEqual([
{ statement_timeout: '300ms' }
])
})
// Why: a real 57014 aborts the transaction exactly as a lock timeout does. If
// it escapes the bounded retry it becomes a failed assignment instead of a
// slow one.
it('retries a real statement timeout on a fresh client', async () => {
const database = await openRelayDatabase({ databaseUrl, dataDir: '', statementTimeoutMs: 300 })
databases.push(database)
let attempts = 0
const result = await database.transaction(async (transaction) => {
attempts += 1
if (attempts === 1) await transaction.query(`SELECT pg_sleep(2)`)
return attempts
})
expect(result).toBe(2)
}, 15_000)
// Why: DDL runs on its own untimed connection. relay_invites carries a
// CREATE INDEX IF NOT EXISTS, which (unlike CREATE TABLE IF NOT EXISTS)
// really does queue behind an ACCESS EXCLUSIVE lock on the table.
it('applies the schema behind a held ACCESS EXCLUSIVE lock', async () => {
let releaseTable!: () => void
const tableReleased = new Promise<void>((resolve) => {
releaseTable = resolve
})
let tableHeld!: () => void
const tableHeldPromise = new Promise<void>((resolve) => {
tableHeld = resolve
})
const holder = databases[0]!.transaction(async (transaction) => {
await transaction.query(`LOCK TABLE relay_invites IN ACCESS EXCLUSIVE MODE`)
tableHeld()
await tableReleased
})
await tableHeldPromise
const opening = openRelayDatabase({
databaseUrl,
dataDir: '',
applicationName,
// Far too short for a blocked DDL; the serving pool wears it, the schema
// connection must not.
statementTimeoutMs: 200
})
const blockedOnSchemaConnection = async (): Promise<boolean> => {
const deadline = Date.now() + 4_000
while (Date.now() < deadline) {
const rows = await databases[0]!.query(
`SELECT count(*) AS waiting FROM pg_stat_activity
WHERE datname = current_database() AND wait_event_type = 'Lock'
AND application_name = ?`,
[`${applicationName}/schema`]
)
if (Number(rows[0]!.waiting) > 0) return true
await new Promise((resolve) => setTimeout(resolve, 10))
}
return false
}
const blocked = await blockedOnSchemaConnection()
releaseTable()
await holder
const database = await opening
databases.push(database)
expect(blocked).toBe(true)
// The serving pool still carries the short deadline it was opened with.
expect(await database.query(`SELECT current_setting('statement_timeout') AS statement_timeout`)).toEqual([
{ statement_timeout: '200ms' }
])
}, 15_000)
})
+58 -10
View File
@@ -796,12 +796,34 @@ class PostgresTransaction implements RelayDatabase {
const POSTGRES_TRANSACTION_ATTEMPTS = 3
const POSTGRES_RETRY_MAX_DELAY_MS = 25
const POSTGRES_CONNECTION_TIMEOUT_MS = 2_000
const POSTGRES_STATEMENT_TIMEOUT_MS = 5_000
// Derivation: a control renewal must land inside its own 30s tick
// (RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2), and a transaction gets
// POSTGRES_TRANSACTION_ATTEMPTS tries, so the worst case a renewal can spend in
// Postgres is attempts * timeout. 5s keeps that at 15s, half the tick, and still
// leaves room for the connect timeout above.
export const POSTGRES_STATEMENT_TIMEOUT_MS = 5_000
const POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS = 5_000
export function relayPostgresStatementTimeoutMs(
env: NodeJS.ProcessEnv = process.env
): number {
const configured = env.ORCA_RELAY_POSTGRES_STATEMENT_TIMEOUT_MS
if (configured === undefined || configured === '') return POSTGRES_STATEMENT_TIMEOUT_MS
const milliseconds = Number(configured)
// 0 is PostgreSQL's "no timeout"; refusing it keeps the deadline this exists
// to enforce from being disabled by a typo in an environment variable.
if (!Number.isInteger(milliseconds) || milliseconds < 1) {
throw new Error('invalid_statement_timeout')
}
return milliseconds
}
function retryablePostgresTransactionError(error: unknown): boolean {
const code = String((error as { code?: unknown }).code)
return code === '40P01' || code === '40001' || code === '55P03'
// 57014 is the pool statement_timeout firing. It aborts the transaction the
// same way a lock timeout does, so it belongs on the bounded retry path
// rather than surfacing as a terminal failure to the caller.
return code === '40P01' || code === '40001' || code === '55P03' || code === '57014'
}
export function isRelayDatabaseTransientError(error: unknown): boolean {
@@ -963,11 +985,36 @@ async function applySchema(database: RelayDatabase): Promise<void> {
}
}
async function applySchemaWithPostgresRetries(database: RelayDatabase): Promise<void> {
await applyPostgresSchema(
SCHEMA.split(';').filter((statement) => statement.trim()),
async (statement) => await database.query(statement)
)
// Why: DDL is not a request. A CREATE INDEX on a grown table legitimately runs
// longer than the request statement_timeout, and inheriting that timeout would
// make every startup fail at the same statement instead of finishing once. One
// short-lived connection of its own, ended before the serving pool opens, keeps
// the untimed session off the request path entirely.
async function applySchemaOnUntimedPool(
databaseUrl: string,
applicationName: string | undefined
): Promise<void> {
const pool = new pg.Pool({
connectionString: databaseUrl,
max: 1,
application_name: applicationName ? `${applicationName}/schema` : undefined,
connectionTimeoutMillis: POSTGRES_CONNECTION_TIMEOUT_MS,
statement_timeout: 0,
// Kept: a DDL blocked behind another director's ACCESS EXCLUSIVE lock must
// yield to the bounded schema retry instead of holding the connection.
lock_timeout: POSTGRES_LOCK_TIMEOUT_MS,
idle_in_transaction_session_timeout: POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS
})
absorbPostgresIdleClientErrors(pool)
const database = new PostgresDatabase(pool)
try {
await applyPostgresSchema(
SCHEMA.split(';').filter((statement) => statement.trim()),
async (statement) => await database.query(statement)
)
} finally {
await database.close().catch(() => undefined)
}
}
async function backfillRelayCellRegions(database: RelayDatabase): Promise<void> {
@@ -983,15 +1030,17 @@ export async function openRelayDatabase(input: {
dataDir: string
poolMax?: number
applicationName?: string
statementTimeoutMs?: number
}): Promise<RelayDatabase> {
let database: RelayDatabase
if (input.databaseUrl) {
await applySchemaOnUntimedPool(input.databaseUrl, input.applicationName)
const pool = new pg.Pool({
connectionString: input.databaseUrl,
max: input.poolMax ?? 10,
application_name: input.applicationName,
connectionTimeoutMillis: POSTGRES_CONNECTION_TIMEOUT_MS,
statement_timeout: POSTGRES_STATEMENT_TIMEOUT_MS,
statement_timeout: input.statementTimeoutMs ?? relayPostgresStatementTimeoutMs(),
lock_timeout: POSTGRES_LOCK_TIMEOUT_MS,
idle_in_transaction_session_timeout: POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS
})
@@ -1004,8 +1053,7 @@ export async function openRelayDatabase(input: {
database = new SqliteDatabase(sqlite)
}
try {
if (input.databaseUrl) await applySchemaWithPostgresRetries(database)
else await applySchema(database)
if (!input.databaseUrl) await applySchema(database)
await backfillRelayCellRegions(database)
return database
} catch (error) {
@@ -0,0 +1,82 @@
import { ASSIGNMENT_LIMITS, RELAY_HOST_CLOSE_REASON } from '@orca-cloud/relay-contract'
import { describe, expect, it } from 'vitest'
import { HostCloseReasonMemory } from './host-close-reason-memory.js'
function memoryAt(clock: { now: number }): HostCloseReasonMemory {
return new HostCloseReasonMemory(() => clock.now)
}
describe('HostCloseReasonMemory', () => {
it('remembers only reasons it knows', () => {
const clock = { now: 1_000 }
const memory = memoryAt(clock)
memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
memory.record('b', 'quitting')
memory.record('c', Buffer.alloc(0))
memory.record('d', undefined)
expect(memory.read('a')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
expect(memory.read('b')).toBeNull()
expect(memory.read('c')).toBeNull()
expect(memory.read('d')).toBeNull()
})
it('accepts the reason as the Buffer a ws close delivers', () => {
const clock = { now: 1_000 }
const memory = memoryAt(clock)
memory.record('a', Buffer.from(RELAY_HOST_CLOSE_REASON.SIGNED_OUT))
expect(memory.read('a')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
})
it('expires an entry once its host may have been rebalanced away', () => {
const clock = { now: 1_000 }
const memory = memoryAt(clock)
memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
clock.now += ASSIGNMENT_LIMITS.dormantTtlMs - 1
expect(memory.read('a')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
clock.now += 1
expect(memory.read('a')).toBeNull()
expect(memory.size()).toBe(0)
})
it('forgets on demand', () => {
const clock = { now: 1_000 }
const memory = memoryAt(clock)
memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
memory.forget('a')
expect(memory.read('a')).toBeNull()
})
it('drops the oldest survivors rather than growing without bound', () => {
const clock = { now: 1_000 }
const memory = memoryAt(clock)
for (let index = 0; index < 50_050; index++) {
memory.record(`host-${index}`, RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
}
expect(memory.size()).toBe(50_000)
expect(memory.read('host-0')).toBeNull()
expect(memory.read('host-50049')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
})
it('re-recording refreshes recency so a live host is not evicted first', () => {
const clock = { now: 1_000 }
const memory = memoryAt(clock)
memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
memory.record('b', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
expect([...['a', 'b'].map((key) => memory.read(key))]).toEqual([
RELAY_HOST_CLOSE_REASON.SIGNED_OUT,
RELAY_HOST_CLOSE_REASON.SIGNED_OUT
])
expect(memory.size()).toBe(2)
})
})
@@ -0,0 +1,72 @@
import {
ASSIGNMENT_LIMITS,
relayHostCloseReasonFrom,
type RelayHostCloseReason
} from '@orca-cloud/relay-contract'
// Retention matches the dormant assignment TTL: past it the host may have been
// rebalanced onto another cell, so this cell is no longer the one a phone asks.
const RETENTION_MS = ASSIGNMENT_LIMITS.dormantTtlMs
// A fleet-wide auth outage signs out every host at once; the cap bounds that
// burst well above any single cell's host count without becoming a leak.
const MAX_ENTRIES = 50_000
// Why in-memory and not Postgres: a phone reaches the cell its host's assignment
// row already names, which is the same cell that watched the control socket
// close. Losing this on a cell restart degrades to the pre-existing generic
// verdict, so the failure mode is the old behaviour rather than a wrong one.
export class HostCloseReasonMemory {
private readonly entries = new Map<string, { reason: RelayHostCloseReason; expiresAt: number }>()
constructor(private readonly now: () => number = Date.now) {}
// Silently ignores anything that is not a known reason, which is every close
// from a host that predates the field and every abrupt 1006.
record(key: string, reason: unknown): void {
const parsed = relayHostCloseReasonFrom(reason)
if (!parsed) {
return
}
this.entries.delete(key)
this.entries.set(key, { reason: parsed, expiresAt: this.now() + RETENTION_MS })
this.evict()
}
forget(key: string): void {
this.entries.delete(key)
}
read(key: string): RelayHostCloseReason | null {
const entry = this.entries.get(key)
if (!entry) {
return null
}
if (entry.expiresAt <= this.now()) {
this.entries.delete(key)
return null
}
return entry.reason
}
size(): number {
return this.entries.size
}
private evict(): void {
const now = this.now()
for (const [key, entry] of this.entries) {
if (entry.expiresAt > now) {
break
}
this.entries.delete(key)
}
// Insertion order is recency order (record deletes before setting), so the
// head is always the oldest survivor.
for (const key of this.entries.keys()) {
if (this.entries.size <= MAX_ENTRIES) {
break
}
this.entries.delete(key)
}
}
}
+34 -6
View File
@@ -14,7 +14,8 @@ import {
HostHelloSchema,
InviteCreateSchema,
RELAY_PROTOCOL_LIMITS,
RELAY_CLOSE_CODE
RELAY_CLOSE_CODE,
type RelayHostCloseReason
} from '@orca-cloud/relay-contract'
import nacl from 'tweetnacl'
import type WebSocket from 'ws'
@@ -25,6 +26,7 @@ import {
RelayCredentialStore,
type CredentialReservation
} from './credential-store.js'
import { HostCloseReasonMemory } from './host-close-reason-memory.js'
import { relayHostLogDigest } from './relay-host-log-digest.js'
import type { RelayTokenClaims } from './relay-token-verifier.js'
import type { RelayRuntimeObserver } from './relay-observability.js'
@@ -130,6 +132,10 @@ const ACTIVATION_QUEUE_WAIT_MS = 30_000
export class HostSessionRegistry {
private readonly sessions = new Map<string, HostSession>()
private readonly activationQueues = new Map<string, Promise<void>>()
// Why it outlives `sessions`: the orphan grace deletes the session within 30s,
// but a signed-out desktop never comes back, so the phone that asks minutes
// later would otherwise find nothing to explain its rejection with.
private readonly hostCloseReasons = new HostCloseReasonMemory(() => this.now())
private draining = false
constructor(
@@ -175,7 +181,8 @@ export class HostSessionRegistry {
return
}
this.observer.recordAuth(true)
const session = this.sessions.get(this.key(reservation.userId, hostId))
const sessionKey = this.key(reservation.userId, hostId)
const session = this.sessions.get(sessionKey)
if (
!session ||
session.state !== 'active' ||
@@ -184,7 +191,13 @@ export class HostSessionRegistry {
) {
capacityReservation?.release()
await this.store.failReservation(reservation)
this.rejectClient(socket, RELAY_CLOSE_CODE.HOST_OFFLINE)
// The only rejection that can name a cause: the host is genuinely absent.
// The attach-deadline 4404 below fires while control is still connected.
this.rejectClient(
socket,
RELAY_CLOSE_CODE.HOST_OFFLINE,
this.hostCloseReasons.read(sessionKey)
)
return
}
if (session.activeConnIds.size + session.pendingConns.size >= 8) {
@@ -793,7 +806,10 @@ export class HostSessionRegistry {
regionalDrainTimer: null,
regionalDrainExpiresAt: null
}
this.sessions.set(this.key(identity.sub, identity.relayHostId), session)
const sessionKey = this.key(identity.sub, identity.relayHostId)
// A host that proved itself again is not signed out, whatever it said last.
this.hostCloseReasons.forget(sessionKey)
this.sessions.set(sessionKey, session)
this.wireActiveControl(session)
this.sendHelloAck(session)
}
@@ -813,6 +829,11 @@ export class HostSessionRegistry {
})
socket.once('close', (code, reason) => {
this.observer.recordControlClose?.(code)
// Guarded on identity: a predecessor retired by a rebind must not stamp a
// cause onto the live session that replaced it.
if (session.socket === socket) {
this.hostCloseReasons.record(this.key(session.identity.sub, session.relayHostId), reason)
}
// One line per control close makes reconnect churners attributable by
// host digest without exposing the raw relay host id.
console.warn(
@@ -1187,9 +1208,16 @@ export class HostSessionRegistry {
if (session.socket) send(session.socket, 'control-error', { ...(reqId ? { reqId } : {}), code })
}
private rejectClient(socket: WebSocket, code: number): void {
// hostCloseReason rides the WebSocket close reason, never relay-hello: every
// shipped phone parses relay-hello with a strict schema that rejects an
// unknown key, and none of them read the close reason at all.
private rejectClient(
socket: WebSocket,
code: number,
hostCloseReason?: RelayHostCloseReason | null
): void {
send(socket, 'relay-hello', { ok: false, code })
closeRelayWebSocket(socket, code, 'relay connection rejected')
closeRelayWebSocket(socket, code, hostCloseReason ?? 'relay connection rejected')
}
private releaseControlActivity(session: HostSession): void {
@@ -0,0 +1,206 @@
import { EventEmitter } from 'node:events'
import {
CONTROL_CONTINUITY_LIMITS,
RELAY_CLOSE_CODE,
RELAY_HOST_CLOSE_REASON
} from '@orca-cloud/relay-contract'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type WebSocket from 'ws'
import type { RelayAssignmentStore } from './assignment-store.js'
import type { RelayConfig } from './config.js'
import type { RelayCredentialStore } from './credential-store.js'
import { HostSessionRegistry } from './host-session-registry.js'
import type { RelayRuntimeObserver } from './relay-observability.js'
import type { RelayTokenClaims } from './relay-token-verifier.js'
import { ProcessQueuedByteBudget } from './splice-forwarder.js'
class FakeSocket extends EventEmitter {
readonly OPEN = 1
readonly CLOSED = 3
readyState = this.OPEN
readonly send = vi.fn()
readonly close = vi.fn((code?: number, reason?: string) => {
this.readyState = this.CLOSED
this.emit('close', code, Buffer.from(reason ?? ''))
})
readonly terminate = vi.fn(() => {
this.readyState = this.CLOSED
this.emit('close', 1006, Buffer.alloc(0))
})
}
const config = {
port: 8080,
publicUrl: 'https://relay-c3.example.com',
cellUrl: 'https://relay-c3.example.com',
authIssuer: 'https://auth.example.com',
authAudience: 'orca-relay',
jwksUrl: 'https://auth.example.com/jwks',
assignmentSigningKey: new Uint8Array(32),
role: 'cell',
cellId: 'production-gce-c3',
cells: []
} as unknown as RelayConfig
const identity = {
sub: 'user-1',
prof: 'profile-1',
org: 'org-1',
relayHostId: 'AbCdEf0123_-xyZ9'
} as unknown as RelayTokenClaims
const reservation = {
userId: identity.sub,
relayHostId: identity.relayHostId,
credentialKind: 'resume',
relayDeviceId: 'device-1',
leaseExpiresAt: Date.now() + 60_000
}
function createRegistry() {
const store = {
resolveResume: vi.fn().mockResolvedValue({ userId: identity.sub }),
reserveCredential: vi.fn().mockResolvedValue(reservation),
failReservation: vi.fn().mockResolvedValue(undefined)
}
const assignments = {
activateControl: vi.fn().mockResolvedValue('control:production-gce-c3:1'),
markMigrationTargetRegistered: vi.fn().mockResolvedValue(undefined),
resolve: vi.fn().mockResolvedValue({ cellId: config.cellId }),
acquireActivity: vi.fn().mockResolvedValue(undefined),
renewControlActivity: vi.fn().mockResolvedValue(undefined),
releaseActivity: vi.fn().mockResolvedValue(true)
} as unknown as RelayAssignmentStore
const observer = {
recordAuth: vi.fn(),
recordForwardedBytes: vi.fn(),
recordHttp: vi.fn(),
recordReconnect: vi.fn(),
recordSql: vi.fn(),
recordControlClose: vi.fn(),
recordSpliceClose: vi.fn()
} satisfies RelayRuntimeObserver
const registry = new HostSessionRegistry(
config,
vi.fn(),
store as unknown as RelayCredentialStore,
assignments,
new ProcessQueuedByteBudget(),
observer
)
const activate = (socket: WebSocket, generation: number): Promise<void> =>
(
registry as unknown as {
activate: (
socket: WebSocket,
identity: RelayTokenClaims,
existing: null,
generation: number,
rebind: boolean,
assignmentEpoch: number,
appVersion: string
) => Promise<void>
}
).activate(socket, identity, null, generation, false, 1, '1.4.173')
return { registry, activate }
}
async function dialPhone(registry: HostSessionRegistry): Promise<FakeSocket> {
const phone = new FakeSocket()
await registry.acceptClient(phone as unknown as WebSocket, identity.relayHostId, 'credential')
return phone
}
// The 4404 hello body is unchanged: every shipped phone parses it with a strict
// schema, so the cause has to ride the close frame instead.
const HOST_OFFLINE_HELLO = JSON.stringify({
type: 'relay-hello',
ok: false,
code: RELAY_CLOSE_CODE.HOST_OFFLINE
})
describe('host sign-out reason on phone rejection', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
it('names the sign-out to a phone that arrives after the host is gone', async () => {
const { registry, activate } = createRegistry()
const control = new FakeSocket()
await activate(control as unknown as WebSocket, 1)
control.close(1000, RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
vi.advanceTimersByTime(CONTROL_CONTINUITY_LIMITS.orphanGraceMs + 1)
const phone = await dialPhone(registry)
expect(phone.send).toHaveBeenCalledWith(HOST_OFFLINE_HELLO)
expect(phone.close).toHaveBeenCalledWith(
RELAY_CLOSE_CODE.HOST_OFFLINE,
RELAY_HOST_CLOSE_REASON.SIGNED_OUT
)
})
it('says nothing when the host died without naming a cause', async () => {
const { registry, activate } = createRegistry()
const control = new FakeSocket()
await activate(control as unknown as WebSocket, 1)
control.terminate()
vi.advanceTimersByTime(CONTROL_CONTINUITY_LIMITS.orphanGraceMs + 1)
const phone = await dialPhone(registry)
expect(phone.close).toHaveBeenCalledWith(
RELAY_CLOSE_CODE.HOST_OFFLINE,
'relay connection rejected'
)
})
it('ignores a close reason the host invented', async () => {
const { registry, activate } = createRegistry()
const control = new FakeSocket()
await activate(control as unknown as WebSocket, 1)
control.close(1000, 'signed-out-ish')
vi.advanceTimersByTime(CONTROL_CONTINUITY_LIMITS.orphanGraceMs + 1)
const phone = await dialPhone(registry)
expect(phone.close).toHaveBeenCalledWith(
RELAY_CLOSE_CODE.HOST_OFFLINE,
'relay connection rejected'
)
})
it('forgets the sign-out once the host proves itself again', async () => {
const { registry, activate } = createRegistry()
const control = new FakeSocket()
await activate(control as unknown as WebSocket, 1)
control.close(1000, RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
vi.advanceTimersByTime(CONTROL_CONTINUITY_LIMITS.orphanGraceMs + 1)
const reconnected = new FakeSocket()
await activate(reconnected as unknown as WebSocket, 2)
// Drop it abruptly, as a network death would, so only the stale memory
// could still name a cause.
reconnected.terminate()
vi.advanceTimersByTime(CONTROL_CONTINUITY_LIMITS.orphanGraceMs + 1)
const phone = await dialPhone(registry)
expect(phone.close).toHaveBeenCalledWith(
RELAY_CLOSE_CODE.HOST_OFFLINE,
'relay connection rejected'
)
})
// A live host is present: the 4404 there is an attach deadline, not absence.
it('never names a cause while the host control is connected', async () => {
const { registry, activate } = createRegistry()
const control = new FakeSocket()
await activate(control as unknown as WebSocket, 1)
const phone = await dialPhone(registry)
expect(phone.close).not.toHaveBeenCalled()
expect(control.send).toHaveBeenCalledWith(expect.stringContaining('"type":"conn-open"'))
})
})
@@ -503,12 +503,19 @@ describePostgres('PostgreSQL transaction recovery', () => {
const directorLockOrder: string[] = []
const assignmentDatabase = new TransactionProbeDatabase(database, async (phase, sql) => {
if (phase === 'before') {
if (sql.includes('FROM relay_assignments WHERE user_id = ?')) {
// Only locked statements reach this hook, so classifying the pin read
// is what proves it stays unlocked: if it ever grows a FOR UPDATE it
// shows up in the order below instead of silently joining the queue.
if (sql.includes('SELECT cell_id FROM relay_assignments')) {
directorLockOrder.push('pin-read')
} else if (sql.includes('FROM relay_assignments WHERE user_id = ?')) {
directorLockOrder.push('assignment')
} else if (sql.includes('FROM relay_assignment_activity_leases')) {
directorLockOrder.push('activity')
} else if (sql.includes('FROM relay_cells ORDER BY')) {
directorLockOrder.push('cell-inventory')
} else if (sql.includes('FROM relay_cells WHERE cell_id IN')) {
directorLockOrder.push('cell-rows')
} else if (sql.includes('FROM relay_cells WHERE cell_id = ?')) {
directorLockOrder.push('cell')
}
@@ -530,11 +537,14 @@ describePostgres('PostgreSQL transaction recovery', () => {
})
await expect(legacyTransaction).resolves.toBeUndefined()
expect(assignmentDatabase.attempts).toBe(2)
// The retry still takes a cell row before the assignment row — the order
// that avoids the legacy cycle — but only the pinned row, never the
// inventory.
expect(directorLockOrder).toEqual([
'assignment',
'activity',
'cell',
'cell-inventory',
'cell-rows',
'assignment',
'activity'
])
@@ -133,10 +133,15 @@
"google_logging_metric.relay_snapshot",
"google_monitoring_alert_policy.relay_assignment_5xx",
"google_monitoring_alert_policy.relay_assignment_edge_429",
"google_monitoring_alert_policy.relay_cell_process_exit",
"google_monitoring_alert_policy.relay_cloud_nat_port_drops",
"google_monitoring_alert_policy.relay_cloud_sql_backends",
"google_monitoring_alert_policy.relay_cloud_sql_checkpoint_loop",
"google_monitoring_alert_policy.relay_cloud_sql_disk",
"google_monitoring_alert_policy.relay_custom",
"google_monitoring_alert_policy.relay_gce_connection_headroom",
"google_monitoring_alert_policy.relay_postgres_retry_exhausted",
"google_monitoring_dashboard.relay_incident",
"google_project_iam_custom_role.github_production_relay_capacity_mutation",
"google_project_iam_custom_role.github_relay_asia_topology_mutation",
"google_project_iam_custom_role.github_relay_asia_topology_read",
@@ -0,0 +1,94 @@
import { spawnSync } from 'node:child_process'
import { fileURLToPath } from 'node:url'
import {
RELAY_REPOSITORY_ROOT,
relayTreePath,
relayWorkflowPath
} from './relay-repository.mjs'
const SHA = /^[a-f0-9]{40}$/
// Every file that decides how relay evidence is produced, sealed, verified, and then spent against
// production; identical content across two commits is what makes the older commit's verdict binding.
export const TRUSTED_EVIDENCE_CODE_PATHS = [
// Produces and seals the 15-minute dry-run evidence.
relayWorkflowPath('monitor-relay-production.yml'),
relayWorkflowPath('monitor-relay-production-job.yml'),
// Download it, verify its authority, and mutate production on it.
relayWorkflowPath('deploy-relay-production-same-cap.yml'),
relayWorkflowPath('deploy-relay-production-same-cap-job.yml'),
relayWorkflowPath('operate-relay-production-rehome.yml'),
relayWorkflowPath('operate-relay-production-rehome-job.yml'),
// Sealing, verification, the wave/canary authority, and the path constants below.
relayTreePath('dev/scripts/relay-evidence-code-provenance.mjs'),
relayTreePath('dev/scripts/relay-monitor-evidence.mjs'),
relayTreePath('dev/scripts/relay-production-same-cap-wave.mjs'),
relayTreePath('dev/scripts/relay-repository.mjs'),
// Every other script those jobs run against live production.
relayTreePath('dev/scripts/infra.mjs'),
relayTreePath('dev/scripts/operate-relay-regional-rehome.mjs'),
relayTreePath('dev/scripts/prepare-relay-production-capacity-canary.mjs'),
relayTreePath('dev/scripts/probe-relay-rehome-trust.mjs'),
relayTreePath('dev/scripts/validate-relay-capacity-plan.mjs'),
relayTreePath('dev/scripts/verify-relay-capacity-transition.mjs'),
// The monitor itself and the live preflight recheck, plus anything that changes their behaviour.
relayTreePath('apps/relay-ops'),
relayTreePath('package.json'),
relayTreePath('pnpm-lock.yaml'),
relayTreePath('pnpm-workspace.yaml'),
// The Cloud SQL rollout lease every mutation job takes and releases.
'.github/actions/cloud-sql-rollout-lease'
]
function git(root, args) {
const result = spawnSync('git', ['-C', root, ...args], { encoding: 'utf8' })
if (result.error) throw new Error('relay evidence provenance cannot run git')
return result
}
/**
* Accepts evidence sealed at a different commit only when the current commit descends from it and
* every trusted path is byte-identical, so the verdict provably came from this exact code. Anything
* git cannot answer (no checkout, unknown commit, shallow clone) fails closed.
*/
export function requireSameEvidenceCode({
sealedSha,
currentSha,
label,
repositoryRoot = fileURLToPath(RELAY_REPOSITORY_ROOT)
}) {
if (!SHA.test(sealedSha ?? '') || !SHA.test(currentSha ?? '')) {
throw new Error(`${label} commit is invalid`)
}
if (sealedSha === currentSha) return
if (git(repositoryRoot, ['rev-parse', '--git-dir']).status !== 0) {
throw new Error(`${label} commit cannot be compared without a git checkout`)
}
for (const sha of [sealedSha, currentSha]) {
if (git(repositoryRoot, ['rev-parse', '--verify', '--quiet', `${sha}^{commit}`]).status !== 0) {
throw new Error(
`${label} commit ${sha} is unknown to this checkout; check out with fetch-depth: 0`
)
}
}
const ancestry = git(repositoryRoot, ['merge-base', '--is-ancestor', sealedSha, currentSha])
if (ancestry.status === 1) {
throw new Error(`${label} commit ${sealedSha} is not an ancestor of ${currentSha}`)
}
if (ancestry.status !== 0) {
throw new Error(`${label} commit ancestry could not be determined`)
}
const diff = git(repositoryRoot, [
'diff',
'--name-only',
sealedSha,
currentSha,
'--',
...TRUSTED_EVIDENCE_CODE_PATHS
])
if (diff.status !== 0) throw new Error(`${label} commit comparison failed`)
const changed = diff.stdout.split('\n').filter(Boolean)
if (changed.length > 0) {
throw new Error(`${label} code changed after it was sealed: ${changed.join(',')}`)
}
}
+18 -5
View File
@@ -2,6 +2,7 @@ import { createHash } from 'node:crypto'
import { chmod, readFile, readdir, stat, writeFile } from 'node:fs/promises'
import { basename, join, resolve } from 'node:path'
import { pathToFileURL } from 'node:url'
import { requireSameEvidenceCode } from './relay-evidence-code-provenance.mjs'
const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{1,127}$/
const SHA = /^[a-f0-9]{40}$/
@@ -102,7 +103,7 @@ export async function createEvidenceManifest(argv) {
return manifest
}
async function readAndVerifyManifest(directory, expected) {
async function readAndVerifyManifest(directory, expected, sameCodeCommit) {
const manifest = JSON.parse(
await readFile(join(directory, 'evidence-manifest.json'), 'utf8')
)
@@ -111,11 +112,23 @@ async function readAndVerifyManifest(directory, expected) {
manifest.incidentId !== expected.incidentId ||
manifest.runId !== expected.runId ||
manifest.runAttempt !== expected.runAttempt ||
manifest.commitSha !== expected.commitSha ||
manifest.mode !== expected.mode
!SHA.test(manifest.commitSha ?? '') ||
manifest.mode !== expected.mode ||
(!sameCodeCommit && manifest.commitSha !== expected.commitSha)
) {
throw new Error('relay monitor evidence provenance does not match')
}
// Unrelated merges land on main every few minutes, so the deployer resolves a newer commit than
// the monitor it must trust; identical monitor and mutation code is the property the SHA stood in
// for. Restore and mutation keep the exact-SHA bind: both run at the commit that sealed them.
if (sameCodeCommit) {
requireSameEvidenceCode({
sealedSha: manifest.commitSha,
currentSha: expected.commitSha,
label: 'relay monitor evidence',
...sameCodeCommit
})
}
const names = Object.keys(manifest.files ?? {})
if (!names.includes(`${expected.incidentId}.state.json`)) {
throw new Error('relay monitor evidence has no durable state')
@@ -209,12 +222,12 @@ function validCompletedDryRunState(state, expected, nowMs, maxAgeMs) {
)
}
export async function verifyDryRunAuthority(argv, now = Date.now) {
export async function verifyDryRunAuthority(argv, now = Date.now, repositoryRoot) {
const values = argumentsByName(argv)
const directory = resolve(values.directory ?? '')
const expected = provenance(values)
if (expected.mode !== 'dry-run') throw new Error('relay mutation requires dry-run evidence')
const manifest = await readAndVerifyManifest(directory, expected)
const manifest = await readAndVerifyManifest(directory, expected, { repositoryRoot })
const state = JSON.parse(
await readFile(join(directory, `${expected.incidentId}.state.json`), 'utf8')
)
@@ -1,9 +1,15 @@
import assert from 'node:assert/strict'
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
import { execFileSync } from 'node:child_process'
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { dirname, join } from 'node:path'
import test from 'node:test'
import { relayWorkflowPath, relayWorkflowUrl } from './relay-repository.mjs'
import { TRUSTED_EVIDENCE_CODE_PATHS } from './relay-evidence-code-provenance.mjs'
import {
RELAY_REPOSITORY_ROOT,
relayWorkflowPath,
relayWorkflowUrl
} from './relay-repository.mjs'
import {
createEvidenceManifest,
verifyDryRunAuthority,
@@ -12,7 +18,7 @@ import {
} from './relay-monitor-evidence.mjs'
const now = Date.parse('2026-07-28T12:00:00.000Z')
const provenance = [
const provenanceFor = (commitSha) => [
'--incident-id',
'relay-123',
'--run-id',
@@ -20,10 +26,11 @@ const provenance = [
'--run-attempt',
'1',
'--commit-sha',
'a'.repeat(40),
commitSha,
'--mode',
'dry-run'
]
const provenance = provenanceFor('a'.repeat(40))
const selector = {
generation: 2,
membership: {
@@ -513,3 +520,157 @@ test('monitor uses a reusable job so exact job_workflow_ref is present', async (
assert.match(job, /workflow_call:/)
assert.match(job, /environment: production/)
})
function gitIn(root, ...args) {
return execFileSync('git', ['-C', root, ...args], { encoding: 'utf8' }).trim()
}
// A real repository shaped like main under unrelated merge traffic: one sealed commit, a
// descendant that only touched untrusted files, a descendant that touched the monitor, and a
// sibling that never descended from the seal.
async function trustedCodeRepository() {
const root = await mkdtemp(join(tmpdir(), 'relay-evidence-repository-'))
gitIn(root, 'init', '--quiet')
gitIn(root, 'config', 'user.email', 'relay@example.test')
gitIn(root, 'config', 'user.name', 'Relay Evidence Test')
gitIn(root, 'config', 'commit.gpgsign', 'false')
const commit = async (path, body, message) => {
await mkdir(dirname(join(root, path)), { recursive: true })
await writeFile(join(root, path), body)
gitIn(root, 'add', '--all')
gitIn(root, 'commit', '--quiet', '--no-verify', '--message', message)
return gitIn(root, 'rev-parse', 'HEAD')
}
const base = await commit(
'cloud/apps/relay-ops/src/incident-monitor.ts',
'export const v = 1\n',
'monitor'
)
const sealed = await commit('README.md', 'base\n', 'base')
const sameCode = await commit('README.md', 'an unrelated merge\n', 'unrelated')
const changedCode = await commit(
'cloud/apps/relay-ops/src/incident-monitor.ts',
'export const v = 2\n',
'monitor change'
)
// Branches before the seal, so the seal is not in its history even though its code matches.
gitIn(root, 'checkout', '--quiet', '--detach', base)
const sibling = await commit('README.md', 'a divergent line\n', 'divergent')
return { root, sealed, sameCode, changedCode, sibling }
}
const authorityAt = (directory, commitSha, repositoryRoot) => verifyDryRunAuthority(
[
'--directory',
directory,
...provenanceFor(commitSha),
'--required-migration-policy',
'strict'
],
() => now,
repositoryRoot
)
test('accepts dry-run evidence sealed by identical code at an ancestor commit', async () => {
const repository = await trustedCodeRepository()
const directory = await evidenceDirectory()
try {
await createEvidenceManifest([
'--directory',
directory,
...provenanceFor(repository.sealed)
])
// An exact match never consults git: a root with no checkout at all still verifies.
await assert.doesNotReject(authorityAt(directory, repository.sealed, directory))
await assert.doesNotReject(authorityAt(directory, repository.sameCode, repository.root))
} finally {
await rm(repository.root, { recursive: true, force: true })
await rm(directory, { recursive: true, force: true })
}
})
test('rejects dry-run evidence whose monitor code or lineage differs', async () => {
const repository = await trustedCodeRepository()
const directory = await evidenceDirectory()
try {
await createEvidenceManifest([
'--directory',
directory,
...provenanceFor(repository.sealed)
])
await assert.rejects(
authorityAt(directory, repository.changedCode, repository.root),
/code changed after it was sealed: cloud\/apps\/relay-ops\/src\/incident-monitor\.ts/
)
await assert.rejects(
authorityAt(directory, repository.sibling, repository.root),
/is not an ancestor of/
)
// Fails closed: a shallow clone that never fetched the sealed commit proves nothing.
await assert.rejects(
authorityAt(directory, 'f'.repeat(40), repository.root),
/unknown to this checkout/
)
// Fails closed: no checkout to compare against.
await assert.rejects(
authorityAt(directory, repository.sameCode, directory),
/cannot be compared without a git checkout/
)
} finally {
await rm(repository.root, { recursive: true, force: true })
await rm(directory, { recursive: true, force: true })
}
})
test('keeps restore and mutation bound to the exact sealing commit', async () => {
const repository = await trustedCodeRepository()
const directory = await evidenceDirectory()
try {
await createEvidenceManifest([
'--directory',
directory,
...provenanceFor(repository.sealed)
])
await assert.rejects(
verifyRestoredEvidence([
'--directory',
directory,
...provenanceFor(repository.sameCode)
]),
/provenance does not match/
)
await assert.rejects(
verifyMutationEvidence(
[
'--directory',
directory,
...provenanceFor(repository.sameCode),
'--mutation-mode',
'execute',
'--source-cell-id',
'c1',
'--director-origin',
'https://relay.example'
],
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' },
async () => Response.json({ selector }),
() => now
),
/provenance does not match/
)
} finally {
await rm(repository.root, { recursive: true, force: true })
await rm(directory, { recursive: true, force: true })
}
})
// A trusted path that no longer exists silently stops being compared, so the same-code rule would
// pass over code it was written to pin.
test('every trusted provenance path exists in this checkout', async () => {
for (const path of TRUSTED_EVIDENCE_CODE_PATHS) {
await assert.doesNotReject(
stat(new URL(path, RELAY_REPOSITORY_ROOT)),
`${path} is missing`
)
}
})
@@ -1,5 +1,6 @@
import { readFileSync } from 'node:fs'
import { pathToFileURL } from 'node:url'
import { requireSameEvidenceCode } from './relay-evidence-code-provenance.mjs'
export const SAME_CAP_CELLS = [
'production-gce-c7', 'production-gce-c8', 'production-gce-c9', 'production-gce-c10',
@@ -85,10 +86,10 @@ export function canaryAuthority(input) {
}
}
export function verifyCanaryAuthority(authority, expected) {
export function verifyCanaryAuthority(authority, expected, repositoryRoot) {
if (
authority?.v !== 1 ||
authority.commitSha !== expected.commitSha ||
!/^[0-9a-f]{40}$/.test(authority.commitSha ?? '') ||
authority.runId !== expected.runId ||
authority.targetDigest !== expected.targetDigest ||
authority.rollbackDigest !== expected.rollbackDigest ||
@@ -96,6 +97,14 @@ export function verifyCanaryAuthority(authority, expected) {
authority.rehomeGeneration !== Number(expected.rehomeGeneration) ||
!SAME_CAP_CELLS.includes(authority.cellId)
) throw new Error('canary authority does not match this batch')
// The batch dispatch resolves main after the canary sealed, so bind to the same code, not the
// same SHA; every field above still pins this batch to that exact canary.
requireSameEvidenceCode({
sealedSha: authority.commitSha,
currentSha: expected.commitSha,
label: 'relay same-cap canary authority',
repositoryRoot
})
return authority
}
@@ -1,4 +1,8 @@
import assert from 'node:assert/strict'
import { execFileSync } from 'node:child_process'
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { dirname, join } from 'node:path'
import { test } from 'node:test'
import {
canaryAuthority,
@@ -104,3 +108,66 @@ test('seals and verifies canary authority for later batches', () => {
rehomeGeneration: '4'
}), /does not match/)
})
function gitIn(root, ...args) {
return execFileSync('git', ['-C', root, ...args], { encoding: 'utf8' }).trim()
}
async function canaryRepository() {
const root = await mkdtemp(join(tmpdir(), 'relay-same-cap-canary-'))
gitIn(root, 'init', '--quiet')
gitIn(root, 'config', 'user.email', 'relay@example.test')
gitIn(root, 'config', 'user.name', 'Relay Wave Test')
gitIn(root, 'config', 'commit.gpgsign', 'false')
const commit = async (path, body, message) => {
await mkdir(dirname(join(root, path)), { recursive: true })
await writeFile(join(root, path), body)
gitIn(root, 'add', '--all')
gitIn(root, 'commit', '--quiet', '--no-verify', '--message', message)
return gitIn(root, 'rev-parse', 'HEAD')
}
const sealed = await commit(
'cloud/dev/scripts/relay-production-same-cap-wave.mjs',
'export const v = 1\n',
'wave'
)
const sameCode = await commit('README.md', 'an unrelated merge\n', 'unrelated')
const changedCode = await commit(
'cloud/dev/scripts/relay-production-same-cap-wave.mjs',
'export const v = 2\n',
'wave change'
)
return { root, sealed, sameCode, changedCode }
}
test('a batch trusts a canary sealed by identical code at an ancestor commit', async () => {
const repository = await canaryRepository()
try {
const authority = canaryAuthority({
cellIds: 'production-gce-c7',
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7`,
commitSha: repository.sealed,
runId: '42',
selectorGeneration: '11',
rehomeGeneration: '4'
})
const verifyAt = (commitSha, repositoryRoot) => verifyCanaryAuthority(authority, {
commitSha,
runId: '42',
targetDigest,
rollbackDigest,
selectorGeneration: '13',
rehomeGeneration: '4'
}, repositoryRoot)
assert.equal(verifyAt(repository.sameCode, repository.root).cellId, 'production-gce-c7')
assert.throws(
() => verifyAt(repository.changedCode, repository.root),
/code changed after it was sealed/
)
assert.throws(() => verifyAt('f'.repeat(40), repository.root), /unknown to this checkout/)
} finally {
await rm(repository.root, { recursive: true, force: true })
}
})
+15
View File
@@ -1,4 +1,6 @@
import { readFileSync } from 'node:fs'
import { relative } from 'node:path'
import { fileURLToPath } from 'node:url'
// Single place naming the repository the Relay workflows live in and where their files sit. The
// public-repo copy moves this tree under cloud/, prefixes every workflow filename, and changes the
@@ -11,6 +13,19 @@ export const RELAY_WORKFLOW_FILE_PREFIX = 'cloud-'
// this tree moves under cloud/, so the depth changes at the copy even though the layout does not.
export const RELAY_WORKFLOW_DIRECTORY = new URL('../../../.github/workflows/', import.meta.url)
// Repository root, derived from the one directory above that already tracks the copy's depth.
export const RELAY_REPOSITORY_ROOT = new URL('../../', RELAY_WORKFLOW_DIRECTORY)
// Repository-relative path for a file in this tree. The prefix is 'cloud/' here and empty where
// the tree is the repository root, so callers naming git paths never restate the layout.
export function relayTreePath(suffix) {
const prefix = relative(
fileURLToPath(RELAY_REPOSITORY_ROOT),
fileURLToPath(new URL('../../', import.meta.url))
).split(/[\\/]/).filter(Boolean)
return [...prefix, suffix].join('/')
}
export function relayWorkflowFile(name) {
return `${RELAY_WORKFLOW_FILE_PREFIX}${name}`
}
+1
View File
@@ -242,6 +242,7 @@ resource "google_compute_instance_template" "relay_gce_cell" {
artifact_registry_host = "${var.region}-docker.pkg.dev"
relay_image = each.value.image
cloud_sql_proxy_image = var.relay_gce_cloud_sql_proxy_image
cloud_sql_private_ip = var.relay_cloud_sql_private_ip
# Keep cell-only plans independent from unrelated database configuration drift.
cloud_sql_connection_name = local.relay_database_connection_name
})
@@ -42,6 +42,12 @@ resource "google_compute_router_nat" "relay_gce" {
router = google_compute_router.relay_gce[0].name
nat_ip_allocate_option = "AUTO_ONLY"
source_subnetwork_ip_ranges_to_nat = "LIST_OF_SUBNETWORKS"
# Cells reach Cloud SQL's public IP through this NAT. The static default of 64 ports per VM
# filled during the 2026-09-04 incident and every cell's proxy dial timed out at once.
enable_dynamic_port_allocation = true
enable_endpoint_independent_mapping = false
min_ports_per_vm = 64
max_ports_per_vm = 4096
subnetwork {
name = google_compute_subnetwork.relay_gce[0].id
@@ -85,6 +91,12 @@ resource "google_compute_router_nat" "relay_gce_additional" {
router = google_compute_router.relay_gce_additional[each.key].name
nat_ip_allocate_option = "AUTO_ONLY"
source_subnetwork_ip_ranges_to_nat = "LIST_OF_SUBNETWORKS"
# Cells reach Cloud SQL's public IP through this NAT. The static default of 64 ports per VM
# filled during the 2026-09-04 incident and every cell's proxy dial timed out at once.
enable_dynamic_port_allocation = true
enable_endpoint_independent_mapping = false
min_ports_per_vm = 64
max_ports_per_vm = 4096
subnetwork {
name = google_compute_subnetwork.relay_gce_additional[each.key].id
@@ -109,6 +109,9 @@ docker run --detach \
--user 0:0 \
--volume "$${cloudsql_dir}:/cloudsql" \
'${cloud_sql_proxy_image}' \
%{ if cloud_sql_private_ip ~}
--private-ip \
%{ endif ~}
--unix-socket=/cloudsql \
'${cloud_sql_connection_name}'
+289 -7
View File
@@ -37,6 +37,16 @@ locals {
description = "Relay PostgreSQL transactions that exhausted bounded retry."
filter = "((resource.type=\"cloud_run_revision\" AND (${local.relay_service_log_filter})) OR resource.type=\"gce_instance\") AND jsonPayload.event=\"orca_relay_postgres_transaction_exhausted\""
}
cell_process_exit = {
# The docker event stream is the only per-exit line: the relay's own crash footer only
# appears for unhandled rejections, and `container start` also counts healthy first boots.
description = "Relay cell container exits, one Docker `container die` event per process exit."
filter = "resource.type=\"gce_instance\" AND logName=\"projects/${var.project_id}/logs/cos_system\" AND jsonPayload.SYSLOG_IDENTIFIER=\"docker\" AND jsonPayload.MESSAGE:\"container die\" AND jsonPayload.MESSAGE:\"name=orca-relay)\""
}
cloud_sql_wal_checkpoint = {
description = "Cloud SQL checkpoints triggered by WAL volume instead of the timed schedule; a sustained run is the fsync loop that stalled every relay process at once on 2026-09-04."
filter = "resource.type=\"cloudsql_database\" AND resource.labels.database_id=\"${var.project_id}:${local.relay_database_instance_name}\" AND textPayload:\"checkpoint starting: wal\""
}
}
relay_runtime_metrics = {
@@ -201,7 +211,8 @@ resource "google_logging_metric" "relay_snapshot" {
label_extractors = {
role = "EXTRACT(jsonPayload.role)"
cell_id = "EXTRACT(jsonPayload.cellId)"
region = "EXTRACT(jsonPayload.region)"
# No region label: adding one replaces all 21 live metrics (label change = delete+create),
# which resets history and blanks the relay alert policies during the swap.
}
metric_descriptor {
@@ -220,12 +231,6 @@ resource "google_logging_metric" "relay_snapshot" {
value_type = "STRING"
description = "Durable relay cell identifier."
}
labels {
key = "region"
value_type = "STRING"
description = "Coarse Relay region."
}
}
bucket_options {
@@ -523,3 +528,280 @@ resource "google_monitoring_alert_policy" "relay_cloud_sql_backends" {
mime_type = "text/markdown"
}
}
resource "google_monitoring_alert_policy" "relay_cloud_sql_checkpoint_loop" {
project = var.project_id
display_name = "Orca Relay: Cloud SQL checkpoint loop"
combiner = "OR"
enabled = true
notification_channels = var.relay_alert_notification_channels
conditions {
display_name = "WAL-triggered checkpoints above 3 in 5 minutes"
condition_threshold {
filter = "resource.type=\"cloudsql_database\" AND metric.type=\"logging.googleapis.com/user/orca_relay_cloud_sql_wal_checkpoint\""
comparison = "COMPARISON_GT"
threshold_value = 3
duration = "300s"
aggregations {
alignment_period = "300s"
per_series_aligner = "ALIGN_SUM"
cross_series_reducer = "REDUCE_SUM"
}
trigger {
count = 1
}
}
}
documentation {
content = "Healthy operation is one timed checkpoint every 5 minutes. Repeated `checkpoint starting: wal` lines mean WAL is outrunning `max_wal_size` and every checkpoint fsync stalls all relay SQL for seconds. Check `checkpoint complete` sync= times and disk write throughput against the PD-SSD ceiling; the fix is disk size and `max_wal_size` in the Terraform root that owns the instance (orca-cloud `infra/terraform-foundation`)."
mime_type = "text/markdown"
}
depends_on = [google_logging_metric.relay_incident]
}
resource "google_monitoring_alert_policy" "relay_cloud_sql_disk" {
project = var.project_id
display_name = "Orca Relay: Cloud SQL disk utilization"
combiner = "OR"
enabled = true
notification_channels = var.relay_alert_notification_channels
conditions {
display_name = "Cloud SQL disk above 70%"
condition_threshold {
filter = "resource.type=\"cloudsql_database\" AND resource.label.\"database_id\"=\"${var.project_id}:${local.relay_database_instance_name}\" AND metric.type=\"cloudsql.googleapis.com/database/disk/utilization\""
comparison = "COMPARISON_GT"
threshold_value = 0.7
duration = "600s"
aggregations {
alignment_period = "300s"
per_series_aligner = "ALIGN_MAX"
}
trigger {
count = 1
}
}
}
documentation {
content = "The shared auth/relay Cloud SQL disk is filling. `refresh_tokens` is the largest table and grows without pruning; grow the disk (IOPS scale with size) before it reaches the WAL checkpoint loop, and prune revoked token rows."
mime_type = "text/markdown"
}
}
resource "google_monitoring_alert_policy" "relay_cloud_nat_port_drops" {
count = local.relay_gce_configured ? 1 : 0
project = var.project_id
display_name = "Orca Relay: Cloud NAT port exhaustion"
combiner = "OR"
enabled = true
notification_channels = var.relay_alert_notification_channels
conditions {
display_name = "NAT packets dropped for lack of ports"
condition_threshold {
filter = "resource.type=\"nat_gateway\" AND resource.label.\"gateway_name\"=monitoring.regex.full_match(\"${local.relay_gce_name}(-.*)?\") AND metric.type=\"router.googleapis.com/nat/dropped_sent_packets_count\" AND metric.label.\"reason\"=\"OUT_OF_RESOURCES\""
comparison = "COMPARISON_GT"
threshold_value = 0
duration = "120s"
aggregations {
alignment_period = "60s"
per_series_aligner = "ALIGN_SUM"
cross_series_reducer = "REDUCE_SUM"
group_by_fields = ["resource.label.\"gateway_name\""]
}
trigger {
count = 1
}
}
}
documentation {
content = "Relay cells reach Cloud SQL's public IP through this NAT. Port exhaustion makes every cell's Cloud SQL Auth Proxy dial time out at once, which reads as a fleet-wide SQL stall with a healthy database. Check `nat/port_usage` per VM and raise `max_ports_per_vm` in `relay-gce-foundation.tf`, or move the database to a private IP."
mime_type = "text/markdown"
}
}
resource "google_monitoring_alert_policy" "relay_cell_process_exit" {
project = var.project_id
display_name = "Orca Relay: cell process exits"
combiner = "OR"
enabled = true
notification_channels = var.relay_alert_notification_channels
conditions {
display_name = "Cell container exits above 3 in 15 minutes"
condition_threshold {
filter = "resource.type=\"gce_instance\" AND metric.type=\"logging.googleapis.com/user/orca_relay_cell_process_exit\""
comparison = "COMPARISON_GT"
threshold_value = 3
duration = "0s"
aggregations {
alignment_period = "900s"
per_series_aligner = "ALIGN_SUM"
cross_series_reducer = "REDUCE_SUM"
group_by_fields = ["resource.label.\"instance_id\""]
}
trigger {
count = 1
}
}
}
documentation {
content = "A Relay GCE cell restarted its container more than three times in 15 minutes. Each exit drops every host and phone on that cell, and 201 exits went unpaged over 48 h on 2026-09-04. The instance hostname is `relay-<cell>-<suffix>`; read `jsonPayload.MESSAGE` on `cos_system` for the exit code and the container's own stderr for the stack before blaming MIG autoheal or load. A same-capacity roll is the remedy when the running image is behind."
mime_type = "text/markdown"
}
depends_on = [google_logging_metric.relay_incident]
}
# Why: the four signals that had to be assembled by hand during the 2026-09-04 incident.
resource "google_monitoring_dashboard" "relay_incident" {
project = var.project_id
dashboard_json = jsonencode({
displayName = "Orca Relay: incident overview"
mosaicLayout = {
columns = 12
tiles = [
{
xPos = 0
yPos = 0
width = 3
height = 4
widget = {
title = "Cloud SQL WAL checkpoints"
xyChart = {
dataSets = [{
plotType = "LINE"
targetAxis = "Y1"
timeSeriesQuery = {
timeSeriesFilter = {
filter = "metric.type=\"logging.googleapis.com/user/orca_relay_cloud_sql_wal_checkpoint\" AND resource.type=\"cloudsql_database\""
aggregation = {
alignmentPeriod = "300s"
perSeriesAligner = "ALIGN_SUM"
crossSeriesReducer = "REDUCE_SUM"
}
}
}
}]
yAxis = {
label = "checkpoints"
scale = "LINEAR"
}
}
}
},
{
xPos = 3
yPos = 0
width = 3
height = 4
widget = {
title = "Cloud NAT dropped packets"
xyChart = {
dataSets = [{
plotType = "LINE"
targetAxis = "Y1"
timeSeriesQuery = {
timeSeriesFilter = {
filter = "metric.type=\"router.googleapis.com/nat/dropped_sent_packets_count\" AND resource.type=\"nat_gateway\" AND resource.label.\"gateway_name\"=monitoring.regex.full_match(\"${local.relay_gce_name}(-.*)?\")"
aggregation = {
alignmentPeriod = "60s"
perSeriesAligner = "ALIGN_SUM"
crossSeriesReducer = "REDUCE_SUM"
groupByFields = ["resource.label.\"gateway_name\"", "metric.label.\"reason\""]
}
}
}
}]
yAxis = {
label = "packets"
scale = "LINEAR"
}
}
}
},
{
xPos = 6
yPos = 0
width = 3
height = 4
widget = {
title = "Auth refresh 401s"
xyChart = {
dataSets = [{
plotType = "LINE"
targetAxis = "Y1"
timeSeriesQuery = {
timeSeriesFilter = {
filter = "metric.type=\"logging.googleapis.com/user/orca_auth_refresh_401\""
aggregation = {
alignmentPeriod = "300s"
perSeriesAligner = "ALIGN_SUM"
crossSeriesReducer = "REDUCE_SUM"
}
}
}
}]
yAxis = {
label = "rejections"
scale = "LINEAR"
}
}
}
},
{
xPos = 9
yPos = 0
width = 3
height = 4
widget = {
title = "Standing desktop controls (fleet sum)"
xyChart = {
dataSets = [{
plotType = "LINE"
targetAxis = "Y1"
timeSeriesQuery = {
timeSeriesFilter = {
# ALIGN_MEAN, not ALIGN_SUM: each process reports its standing control count once per interval.
filter = "metric.type=\"logging.googleapis.com/user/orca_relay_controls\""
aggregation = {
alignmentPeriod = "300s"
perSeriesAligner = "ALIGN_MEAN"
crossSeriesReducer = "REDUCE_SUM"
}
}
}
}]
yAxis = {
label = "controls"
scale = "LINEAR"
}
}
}
}
]
}
})
depends_on = [google_logging_metric.relay_incident, google_logging_metric.relay_snapshot]
}
+6
View File
@@ -468,6 +468,12 @@ variable "relay_gce_fenced_cells" {
default = []
}
variable "relay_cloud_sql_private_ip" {
type = bool
description = "Dial Cloud SQL over its private IP inside this VPC instead of its public IP through Cloud NAT. Requires the foundation root's private services access peering to be applied first; a cell that cannot reach the private IP never becomes ready."
default = false
}
variable "relay_gce_cloud_sql_proxy_image" {
type = string
description = "Digest-pinned Cloud SQL Auth Proxy image used by private relay workers."
@@ -0,0 +1,18 @@
// Mirror of src/shared/relay-host-close-reason.ts in the Orca app repo half.
// A host control socket may close with one of these as its WebSocket close
// reason; the cell records it so a later phone rejection can name the cause.
// Anything else (including the empty reason of an abrupt 1006) means "unknown",
// which is what every peer that predates this file sends.
export const RELAY_HOST_CLOSE_REASON = {
SIGNED_OUT: 'signed-out'
} as const
export type RelayHostCloseReason =
(typeof RELAY_HOST_CLOSE_REASON)[keyof typeof RELAY_HOST_CLOSE_REASON]
const REASONS: readonly string[] = Object.values(RELAY_HOST_CLOSE_REASON)
export function relayHostCloseReasonFrom(value: unknown): RelayHostCloseReason | null {
const text = typeof value === 'string' ? value : (value?.toString() ?? '')
return REASONS.includes(text) ? (text as RelayHostCloseReason) : null
}
@@ -5,6 +5,7 @@ export * from './control-messages.js'
export * from './control-continuity.js'
export * from './credential-messages.js'
export * from './director-messages.js'
export * from './host-close-reason.js'
export * from './host-proof-transcript.js'
export * from './persistence-invariants.js'
export * from './protocol-limits.js'
+1
View File
@@ -219,6 +219,7 @@ const WINDOWS_PACKAGE_TESTS = [
'src/main/agent-hooks/windows-hook-payload-delivery.test.ts',
'src/main/windows/windows-pty-job.win32.test.ts',
'src/main/windows/windows-host-job.win32.test.ts',
'src/main/windows-live-tree-kill.win32.test.ts',
'src/main/wsl/wsl-runner.test.ts',
'src/main/wsl/wsl-guest-environment.test.ts',
'src/main/wsl/wsl-invocation-boundary.test.ts',
@@ -11,7 +11,12 @@ import { repairTranslatedValue } from './locale-translation-policy.mjs'
const SOURCE_EXTENSIONS = new Set(['.ts', '.tsx', '.js', '.jsx', '.mts', '.cts'])
const SKIP_PATH_PARTS = new Set(['.git', 'dist', 'node_modules', 'out', '__snapshots__', 'assets'])
const LOCALIZATION_FUNCTION_NAMES = new Set(['t', 'translate', 'translateMain', 'translateSearchKeyword'])
const LOCALIZATION_FUNCTION_NAMES = new Set([
't',
'translate',
'translateMain',
'translateSearchKeyword'
])
const PLACEHOLDER_RE = /\{\{[^}]+\}\}/g
const LOCALES_RELATIVE_DIR = path.join('src', 'renderer', 'src', 'i18n', 'locales')
export const LOCALIZATION_SOURCE_ROOTS = [
+6 -1
View File
@@ -19,6 +19,7 @@ type MobileHomeHostListProps = {
hostAttempts: Record<string, number>
hostLastConnected: Record<string, number | null>
hostPairingRejected: Record<string, boolean>
hostSignedOut: Record<string, boolean>
hostPaths: Record<string, MobileConnectionPath>
hostPendingPaths: Record<string, MobileConnectionPath | null>
hosts: HostCatalogEntry[]
@@ -40,6 +41,7 @@ export function MobileHomeHostList(props: MobileHomeHostListProps) {
hostAttempts={props.hostAttempts}
hostLastConnected={props.hostLastConnected}
hostPairingRejected={props.hostPairingRejected}
hostSignedOut={props.hostSignedOut}
hostPaths={props.hostPaths}
hostPendingPaths={props.hostPendingPaths}
hostStates={props.hostStates}
@@ -54,6 +56,7 @@ export function MobileHomeHostList(props: MobileHomeHostListProps) {
props.hostAttempts,
props.hostLastConnected,
props.hostPairingRejected,
props.hostSignedOut,
props.hostPaths,
props.hostPendingPaths,
props.hostStates,
@@ -91,6 +94,7 @@ type MobileHomeHostRowProps = Pick<
| 'hostAttempts'
| 'hostLastConnected'
| 'hostPairingRejected'
| 'hostSignedOut'
| 'hostPaths'
| 'hostPendingPaths'
| 'hostStates'
@@ -113,7 +117,8 @@ const MobileHomeHostRow = memo(function MobileHomeHostRow(props: MobileHomeHostR
lastConnectedAt: props.hostLastConnected[item.id] ?? null,
endpoint: item.endpoint,
pendingPath: props.hostPendingPaths[item.id] ?? null,
pairingRejected: props.hostPairingRejected[item.id] ?? false
pairingRejected: props.hostPairingRejected[item.id] ?? false,
hostSignedOut: props.hostSignedOut[item.id] ?? false
})
const open = useCallback(() => onOpen(item), [item, onOpen])
const longPress = useCallback(() => onLongPress(item), [item, onLongPress])
+1
View File
@@ -143,6 +143,7 @@ export function MobileHomeScreen() {
hostAttempts={data.hostAttempts}
hostLastConnected={data.hostLastConnected}
hostPairingRejected={data.hostPairingRejected}
hostSignedOut={data.hostSignedOut}
hostPaths={data.hostPaths}
hostPendingPaths={data.hostPendingPaths}
hosts={data.sortedHostCatalog}
@@ -5,12 +5,14 @@ export type HomeHostConnectionProjectionEntry = {
path: MobileConnectionPath
pendingPath: MobileConnectionPath | null
pairingRejected: boolean
hostSignedOut: boolean
}
export type HomeHostConnectionProjection = {
hostPaths: Record<string, MobileConnectionPath>
hostPendingPaths: Record<string, MobileConnectionPath | null>
hostPairingRejected: Record<string, boolean>
hostSignedOut: Record<string, boolean>
}
/** Build all host lookup maps while reading each connection entry once. */
@@ -22,16 +24,19 @@ export function projectHomeHostConnections(
const hostPaths = Object.create(null) as Record<string, MobileConnectionPath>
const hostPendingPaths = Object.create(null) as Record<string, MobileConnectionPath | null>
const hostPairingRejected = Object.create(null) as Record<string, boolean>
const hostSignedOut = Object.create(null) as Record<string, boolean>
for (const { hostId, path, pendingPath, pairingRejected } of entries) {
for (const { hostId, path, pendingPath, pairingRejected, hostSignedOut: signedOut } of entries) {
hostPaths[hostId] = path
hostPendingPaths[hostId] = pendingPath
hostPairingRejected[hostId] = pairingRejected
hostSignedOut[hostId] = signedOut
}
Object.setPrototypeOf(hostPaths, Object.prototype)
Object.setPrototypeOf(hostPendingPaths, Object.prototype)
Object.setPrototypeOf(hostPairingRejected, Object.prototype)
Object.setPrototypeOf(hostSignedOut, Object.prototype)
return { hostPaths, hostPendingPaths, hostPairingRejected }
return { hostPaths, hostPendingPaths, hostPairingRejected, hostSignedOut }
}
+1
View File
@@ -179,6 +179,7 @@ export function useMobileHomeData() {
connectedHosts,
hostCatalog,
hostPairingRejected: hostConnectionProjection.hostPairingRejected,
hostSignedOut: hostConnectionProjection.hostSignedOut,
hostPaths: hostConnectionProjection.hostPaths,
hostPendingPaths: hostConnectionProjection.hostPendingPaths,
primaryHost,
@@ -41,6 +41,7 @@ const MODEL_DESCRIPTOR: SessionOptionDescriptor = {
]
},
valueSource: 'reported',
transport: 'catalog',
settable: true
}
@@ -57,6 +58,7 @@ const EFFORT_DESCRIPTOR: SessionOptionDescriptor = {
]
},
valueSource: 'dispatched',
transport: 'catalog',
settable: true
}
@@ -66,6 +68,7 @@ const FAST_MODE_DESCRIPTOR: SessionOptionDescriptor = {
category: 'mode',
kind: { type: 'boolean', currentValue: false },
valueSource: 'reported',
transport: 'catalog',
settable: true
}
@@ -227,6 +230,7 @@ describe('MobileNativeChatSessionOptionPickers', () => {
...MODEL_DESCRIPTOR,
kind: { type: 'select', choices: [] },
valueSource: 'unknown',
transport: 'catalog',
action: { type: 'agent-picker' }
}
])
@@ -236,6 +240,46 @@ describe('MobileNativeChatSessionOptionPickers', () => {
expect(invokeAction).toHaveBeenCalledWith('model')
})
// The terminal transport can only learn the outcome by parsing the screen back,
// so the sheet admits the value is unconfirmed; the structured transport reports
// it every turn, which makes the same caption noise there.
it.each([
{ transport: 'catalog' as const, caption: true },
{ transport: 'agent-session' as const, caption: false }
])('captions a dispatched value only on the terminal transport', async (scenario) => {
mount([
MODEL_DESCRIPTOR,
{ ...EFFORT_DESCRIPTOR, valueSource: 'dispatched', transport: scenario.transport }
])
await act(async () => pill('Model').props.onPress())
await act(async () => rowByText('Effort').props.onPress())
const captions = renderer!.root
.findAll((node) => node.type === 'Text')
.filter(
(node) =>
(node.props as { children?: unknown }).children === 'Sent to the agent — not confirmed'
)
expect(captions.length > 0).toBe(scenario.caption)
})
it.each(['catalog', 'agent-session'] as const)(
'does not caption a reported value on the %s transport',
async (transport) => {
mount([MODEL_DESCRIPTOR, { ...EFFORT_DESCRIPTOR, valueSource: 'reported', transport }])
await act(async () => pill('Model').props.onPress())
await act(async () => rowByText('Effort').props.onPress())
expect(
renderer!.root
.findAll((node) => node.type === 'Text')
.some(
(node) =>
(node.props as { children?: unknown }).children ===
'Sent to the agent — not confirmed'
)
).toBe(false)
}
)
it('locks the pills while the agent is working', () => {
mount([MODEL_DESCRIPTOR, EFFORT_DESCRIPTOR], true)
expect(pill('Model').props).toMatchObject({ disabled: true })
@@ -3,9 +3,10 @@ import { ActivityIndicator, Keyboard, Pressable, StyleSheet, Text, View } from '
import { ChevronLeft, X } from 'lucide-react-native'
import { BottomDrawer } from '../components/BottomDrawer'
import { colors, radii, spacing, typography } from '../theme/mobile-theme'
import type {
SessionOptionDescriptor,
SessionOptionValue
import {
sessionOptionDispatchUnconfirmed,
type SessionOptionDescriptor,
type SessionOptionValue
} from '../../../src/shared/native-chat-session-options'
import {
mobileModelPillLabel,
@@ -119,7 +120,7 @@ export function MobileNativeChatSessionOptionPickers({
) : null}
</View>
</View>
{activeDescriptor.valueSource === 'dispatched' ? (
{sessionOptionDispatchUnconfirmed(activeDescriptor) ? (
<SessionOptionCaption>Sent to the agent — not confirmed</SessionOptionCaption>
) : null}
{reason ? <SessionOptionCaption>{reason}</SessionOptionCaption> : null}
@@ -168,7 +168,8 @@ export function useMobileNativeChatSessionOptions(args: {
models: activeModels(catalog, record),
record,
mode: 'live',
modelLabel: 'Model'
modelLabel: 'Model',
liveTransport: 'catalog'
})
}, [agent, catalog, scopeKey, version])
@@ -30,14 +30,16 @@ export function useConnectionPathStatus(hostId: string | undefined): {
export function useRelayRecoveryStatus(hostId: string | undefined): {
pendingPath: MobileConnectionPath | null
pairingRejected: boolean
hostSignedOut: boolean
} {
return useHostMetric(
hostId,
(context, id) => ({
pendingPath: context.getPendingPath(id),
pairingRejected: context.isPairingRejected(id)
pairingRejected: context.isPairingRejected(id),
hostSignedOut: context.isHostSignedOut(id)
}),
{ pendingPath: null, pairingRejected: false }
{ pendingPath: null, pairingRejected: false, hostSignedOut: false }
)
}
+2 -2
View File
@@ -533,12 +533,12 @@ describe('useAllHostClients', () => {
await Promise.resolve()
})
act(() => client.emitPendingPath('relay'))
expect(status).toEqual({ pendingPath: 'relay', pairingRejected: false })
expect(status).toEqual({ pendingPath: 'relay', pairingRejected: false, hostSignedOut: false })
// Why: the desktop refusing the credential is a status-only change — no
// transport state moves, so only the connection-path signal can carry it.
act(() => client.emitPairingRejected(true))
expect(status).toEqual({ pendingPath: 'relay', pairingRejected: true })
expect(status).toEqual({ pendingPath: 'relay', pairingRejected: true, hostSignedOut: false })
act(() => renderer.unmount())
})
+19
View File
@@ -29,6 +29,10 @@ const STALE_SINCE_LAST_CONNECT_MS = 60_000
// instead of leaving the user staring at a generic "Can't connect".
const TAILSCALE_HINT = 'check Tailscale'
// No hint field: the remedy is the label, and appending "— check Tailscale" to
// it would be wrong advice for a desktop that is reachable but signed out.
const SIGNED_OUT_LABEL = 'Desktop signed out — sign in to Orca on your desktop to reconnect'
export type ConnectionVerdict =
| { kind: 'normal'; label: string }
| { kind: 'warning'; label: string; hint?: string } // "Can't connect"
@@ -54,6 +58,10 @@ export function classifyConnection(args: {
// The desktop has repeatedly refused this device's relay credential — retrying
// cannot fix it, so it outranks any "still connecting" reading (STA-4681).
pairingRejected?: boolean
// The relay says the desktop's last control close named its own Orca Cloud
// sign-out. Retrying is still correct and still happens on the same cadence,
// but only the desktop's owner can end it, so the label has to say so.
hostSignedOut?: boolean
nowMs?: number
}): ConnectionVerdict {
const { state, reconnectAttempts, lastConnectedAt } = args
@@ -70,6 +78,17 @@ export function classifyConnection(args: {
return { kind: 'normal', label: 'Connected' }
}
// Ahead of the attempt thresholds: this is evidence, not an inference from a
// failure streak, and waiting twelve dials to show it wastes the whole point.
// Below auth-failed because a revoked pairing cannot be fixed by signing in.
if (args.hostSignedOut) {
return {
kind: 'unreachable',
label: SIGNED_OUT_LABEL,
reason: lastConnectedAt == null ? 'never-connected' : 'stale'
}
}
// A disconnected pending path can survive a cleared retry timer during a
// lifecycle race. Only narrate Relay while dialing or after a retry has
// recorded progress; otherwise the idle transport must read Disconnected.
@@ -89,10 +89,16 @@ export function createHostClientSelectors(
getPendingPath: (hostId: string): MobileConnectionPath | null =>
clientPendingPath(entries.get(hostId)?.client),
isPairingRejected: (hostId: string): boolean =>
clientPairingRejected(entries.get(hostId)?.client)
clientPairingRejected(entries.get(hostId)?.client),
isHostSignedOut: (hostId: string): boolean => clientHostSignedOut(entries.get(hostId)?.client)
}
}
export function clientHostSignedOut(client: RpcClient | undefined): boolean {
const logical = client as Partial<StableLogicalRpcClient> | undefined
return logical?.isHostSignedOut?.() ?? false
}
export function clientPairingRejected(client: RpcClient | undefined): boolean {
const logical = client as Partial<StableLogicalRpcClient> | undefined
return logical?.isPairingRejected?.() ?? false
@@ -5,6 +5,7 @@ export class LogicalClientConnectionPath {
private recovery: MobileConnectionPath | null = null
private recoveryAttempt = 0
private pairingRejected = false
private hostSignedOut = false
private readonly listeners = new Set<() => void>()
constructor(private readonly isConnected: () => boolean) {}
@@ -35,12 +36,23 @@ export class LogicalClientConnectionPath {
})
}
isHostSignedOut(): boolean {
return this.hostSignedOut
}
setHostSignedOut(signedOut: boolean): void {
this.update(() => {
this.hostSignedOut = signedOut
})
}
clearAfterConnected(): void {
this.migration = null
this.recovery = null
this.recoveryAttempt = 0
// Why: an authenticated session is the desktop accepting this device.
this.pairingRejected = false
this.hostSignedOut = false
}
setRecovery(path: MobileConnectionPath | null, attempt?: number): void {
@@ -69,11 +81,13 @@ export class LogicalClientConnectionPath {
const previousPath = this.pending()
const previousAttempt = this.reconnectAttempt(0)
const previousRejected = this.pairingRejected
const previousSignedOut = this.hostSignedOut
apply()
if (
previousPath === this.pending() &&
previousAttempt === this.reconnectAttempt(0) &&
previousRejected === this.pairingRejected
previousRejected === this.pairingRejected &&
previousSignedOut === this.hostSignedOut
) {
return
}
@@ -86,7 +86,7 @@ function createSupervisor(
): MobileEndpointSupervisor {
return new MobileEndpointSupervisor(logical, host, {
openDirect: (endpoint) => connect(endpoint, host.deviceToken, host.publicKeyB64, { onLog }),
openRelay: (relay, credential, confirmReqId) =>
openRelay: (relay, credential, confirmReqId, onHostCloseReason) =>
connectMobileRelayRpcSession({
relay,
resumeToken: credential.token,
@@ -94,6 +94,7 @@ function createSupervisor(
resumeConfirmReqId: confirmReqId,
deviceToken: host.deviceToken,
desktopPublicKeyB64: host.publicKeyB64,
onHostCloseReason,
onLog
}),
resolveRelay: resolveMobileRelayEndpoint,
@@ -1,4 +1,5 @@
import type { MobileRelayEndpoint } from '../../../src/shared/mobile-relay-credential-contract'
import type { RelayHostCloseReason } from '../../../src/shared/relay-host-close-reason'
import type { MobileRelayCredentialBundle } from './mobile-relay-credential-bundle'
import type { MobileRelayRpcSession } from './mobile-relay-rpc-session'
import type { resolveMobileRelayEndpoint } from './mobile-relay-resume-director'
@@ -10,7 +11,8 @@ export type MobileEndpointSupervisorDependencies = {
openRelay: (
relay: MobileRelayEndpoint,
credential: { token: string; version: number },
confirmReqId: string
confirmReqId: string,
onHostCloseReason?: (reason: RelayHostCloseReason) => void
) => MobileRelayRpcSession
resolveRelay: typeof resolveMobileRelayEndpoint
readBundle: (hostId: string) => Promise<MobileRelayCredentialBundle | null>
@@ -134,10 +134,22 @@ export class FakeLogicalClient extends FakeSession implements StableLogicalRpcCl
}
})
isPairingRejected = () => this.pairingRejected
private hostSignedOut = false
setHostSignedOut = vi.fn((signedOut: boolean) => {
if (this.hostSignedOut === signedOut) {
return
}
this.hostSignedOut = signedOut
for (const listener of this.pathListeners) {
listener()
}
})
isHostSignedOut = () => this.hostSignedOut
// Mirrors LogicalClientConnectionPath.clearAfterConnected.
publishState(state: ConnectionState): void {
if (state === 'connected') {
this.pairingRejected = false
this.hostSignedOut = false
}
super.publishState(state)
}
@@ -185,7 +185,12 @@ describe('mobile endpoint supervisor', () => {
await supervisor.start()
expect(deps.resolveRelay).toHaveBeenCalledOnce()
expect(openRelay).toHaveBeenLastCalledWith(resolved, expect.any(Object), expect.any(String))
expect(openRelay).toHaveBeenLastCalledWith(
resolved,
expect.any(Object),
expect.any(String),
expect.any(Function)
)
expect(deps.saveHost).toHaveBeenCalledWith(
expect.objectContaining({ relay: resolved, endpoint: host.endpoint })
)
@@ -556,7 +561,8 @@ describe('mobile endpoint supervisor', () => {
expect(openRelay).toHaveBeenLastCalledWith(
relay,
expect.objectContaining({ version: 3 }),
expect.any(String)
expect.any(String),
expect.any(Function)
)
supervisor.stop()
})
@@ -603,7 +609,8 @@ describe('mobile endpoint supervisor', () => {
expect(openRelay).toHaveBeenLastCalledWith(
relay,
expect.objectContaining({ version: 3 }),
expect.any(String)
expect.any(String),
expect.any(Function)
)
supervisor.stop()
})
@@ -2,6 +2,10 @@ import {
RelayPhoneHelloSchema,
type RelayPhoneHello
} from '../../../src/shared/mobile-relay-phone-protocol'
import {
relayHostCloseReasonFrom,
type RelayHostCloseReason
} from '../../../src/shared/relay-host-close-reason'
import { MobileE2EEV2ClientSession } from './mobile-e2ee-v2-client-session'
import { MobileE2EEV2PhysicalChannel } from './mobile-e2ee-v2-physical-channel'
import { websocketPayloadToUint8 } from './websocket-payload-bytes'
@@ -26,6 +30,12 @@ type MobileRelayE2eeLinkOptions = {
onText: (plaintext: string) => void
onBinary: (plaintext: Uint8Array) => void
onHello?: (hello: Extract<RelayPhoneHello, { ok: true }>) => void
// The cell's account of why the desktop is absent, read off the close frame.
// Reported separately from onError because a rejection is delivered as both a
// relay-hello and a close, and which one the runtime dispatches first is not
// ordered — only the close carries the reason, and it must not be lost to
// that race.
onHostCloseReason?: (reason: RelayHostCloseReason) => void
// Fired once relay-auth is on the wire: from here the cell owns the wait.
onOpen?: () => void
onError: (error: Error) => void
@@ -129,6 +139,11 @@ export class MobileRelayE2eeLink {
clearTimeout(this.transportErrorTimer)
this.transportErrorTimer = null
}
// Ahead of fail(), which no-ops once the hello already reported this close.
const hostCloseReason = relayHostCloseReasonFrom(event.reason)
if (hostCloseReason) {
this.options.onHostCloseReason?.(hostCloseReason)
}
this.fail(new RelayOuterError(event.code || 1006))
}
}
@@ -13,6 +13,7 @@ import { RelayDialStageTracker, type RelayDialStageSource } from './relay-dial-s
import { RelayPendingRequests } from './relay-pending-requests'
import { RpcSessionLivenessWatchdog } from './rpc-session-liveness-watchdog'
import { settleMobileRuntimeCapabilities } from './mobile-runtime-capability-negotiation'
import type { RelayHostCloseReason } from '../../../src/shared/relay-host-close-reason'
import type { RpcClient } from './rpc-client'
import type { ConnectionLogSink, ConnectionState, RpcResponse } from './types'
@@ -40,6 +41,7 @@ export function connectMobileRelayRpcSession(args: {
desktopPublicKeyB64: string
requestTimeoutMs?: number
createSocket?: (url: string) => WebSocket
onHostCloseReason?: (reason: RelayHostCloseReason) => void
onLog?: ConnectionLogSink
}): MobileRelayRpcSession {
const requestTimeoutMs = args.requestTimeoutMs ?? 30_000
@@ -69,6 +71,7 @@ export function connectMobileRelayRpcSession(args: {
deviceToken: args.deviceToken,
desktopPublicKeyB64: args.desktopPublicKeyB64,
createSocket: args.createSocket,
onHostCloseReason: args.onHostCloseReason,
onOpen: () => dialStage.advance('awaiting-hello'),
onHello: (hello) => {
if (
@@ -145,10 +145,22 @@ class FakeLogicalClient extends FakeSession implements StableLogicalRpcClient {
}
})
isPairingRejected = () => this.pairingRejected
private hostSignedOut = false
setHostSignedOut = vi.fn((signedOut: boolean) => {
if (this.hostSignedOut === signedOut) {
return
}
this.hostSignedOut = signedOut
for (const listener of this.pathListeners) {
listener()
}
})
isHostSignedOut = () => this.hostSignedOut
// Mirrors LogicalClientConnectionPath.clearAfterConnected.
publishState(state: ConnectionState): void {
if (state === 'connected') {
this.pairingRejected = false
this.hostSignedOut = false
}
super.publishState(state)
}
@@ -264,7 +276,8 @@ describe('relay runtime recovery without direct connectivity', () => {
expect(openRelay).toHaveBeenLastCalledWith(
relay,
expect.objectContaining({ version: 3 }),
expect.any(String)
expect.any(String),
expect.any(Function)
)
expect(logical.getActivePath()).toBe('relay')
supervisor.stop()
@@ -353,7 +366,8 @@ describe('relay runtime recovery without direct connectivity', () => {
expect(deps.openRelay).toHaveBeenLastCalledWith(
relay,
expect.objectContaining({ version: 2 }),
expect.any(String)
expect.any(String),
expect.any(Function)
)
expect(logical.getActivePath()).toBe('relay')
supervisor.stop()
@@ -382,7 +396,8 @@ describe('relay runtime recovery without direct connectivity', () => {
expect(openRelay).toHaveBeenLastCalledWith(
relay,
expect.objectContaining({ version: 1 }),
expect.any(String)
expect.any(String),
expect.any(Function)
)
expect(logical.getActivePath()).toBe('relay')
supervisor.stop()
@@ -10,6 +10,7 @@ import type { MobileRelayCredentialBundle } from './mobile-relay-credential-bund
import type { RelayReconnectController } from './mobile-relay-reconnect-controller'
import type { StableLogicalRpcClient } from './stable-logical-rpc-client'
import type { MobileRelayEndpoint } from '../../../src/shared/mobile-relay-credential-contract'
import { RELAY_HOST_CLOSE_REASON } from '../../../src/shared/relay-host-close-reason'
import type { HostProfile } from './types'
type EstablishResult = { ok: true } | { ok: false; error: Error }
@@ -100,7 +101,16 @@ export class MobileRelaySessionEstablisher {
const session = args.openRelay(
relay,
credential,
`confirm-${encodeBase64Url(args.randomBytes(16))}`
`confirm-${encodeBase64Url(args.randomBytes(16))}`,
// Latched on the logical client, not on the dial result: the close that
// carries the reason can land after this dial has already reported its
// failure. Clearing is clearAfterConnected's job, so any path that
// reaches connected retires it.
(reason) => {
if (reason === RELAY_HOST_CLOSE_REASON.SIGNED_OUT) {
args.logical.setHostSignedOut(true)
}
}
)
try {
// Why: backgrounding or a direct winner withdraws this dial before cutover.
@@ -0,0 +1,60 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { MOBILE_RELAY_CLOSE_CODE } from '../../../src/shared/mobile-relay-close-codes'
import { RELAY_HOST_CLOSE_REASON } from '../../../src/shared/relay-host-close-reason'
import { RelayOuterError } from './mobile-relay-e2ee-link'
import {
dependencies,
FakeLogicalClient,
FakeRelaySession,
host
} from './mobile-endpoint-supervisor-test-fakes'
import { MobileEndpointSupervisor } from './mobile-endpoint-supervisor'
vi.mock('react-native', () => ({ Platform: { OS: 'ios' } }))
vi.mock('expo-secure-store', () => ({ WHEN_UNLOCKED_THIS_DEVICE_ONLY: 'when-unlocked' }))
vi.mock('expo-crypto', () => ({ getRandomBytes: (length: number) => new Uint8Array(length) }))
// The reason travels from the cell's close frame to the screens. This covers
// the production wiring between them: the supervisor's own openRelay callback.
describe('a signed-out desktop reaches the phone verdict', () => {
beforeEach(() => {
vi.useFakeTimers()
vi.setSystemTime(new Date('2026-07-13T12:00:00Z'))
})
afterEach(() => vi.useRealTimers())
function supervisorOver(closeReason: string | null) {
const logical = new FakeLogicalClient('disconnected', 'lan')
const deps = dependencies({
openDirect: vi.fn(() => new FakeRelaySession('disconnected')),
openRelay: vi.fn((_relay, _credential, _confirmReqId, onHostCloseReason) => {
if (closeReason) {
onHostCloseReason?.(closeReason as never)
}
return new FakeRelaySession(
'disconnected',
new RelayOuterError(MOBILE_RELAY_CLOSE_CODE.HOST_OFFLINE)
)
})
})
return { logical, supervisor: new MobileEndpointSupervisor(logical, host, deps) }
}
it('latches the sign-out the cell reported', async () => {
const { logical, supervisor } = supervisorOver(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
await supervisor.start()
await vi.waitFor(() => expect(logical.isHostSignedOut()).toBe(true))
supervisor.stop()
})
it('stays quiet for an ordinary host-offline rejection', async () => {
const { logical, supervisor } = supervisorOver(null)
await supervisor.start()
expect(logical.isHostSignedOut()).toBe(false)
supervisor.stop()
})
})
@@ -0,0 +1,216 @@
import { describe, expect, it, vi } from 'vitest'
vi.mock('./mobile-e2ee-v2-client-session', () => ({
MobileE2EEV2ClientSession: { create: () => ({}) }
}))
vi.mock('./mobile-e2ee-v2-physical-channel', () => ({
MobileE2EEAuthenticationError: class extends Error {},
MobileE2EEV2PhysicalChannel: class {
start = vi.fn()
handleMessage = vi.fn(async () => {})
sendText = vi.fn(() => true)
sendBinary = vi.fn(() => true)
dispose = vi.fn()
}
}))
import { RELAY_HOST_CLOSE_REASON } from '../../../src/shared/relay-host-close-reason'
import { MOBILE_RELAY_CLOSE_CODE } from '../../../src/shared/mobile-relay-close-codes'
import { classifyConnection, verdictDisplayLabel } from './connection-health'
import { MobileRelayE2eeLink, RelayOuterError } from './mobile-relay-e2ee-link'
import { LogicalClientConnectionPath } from './logical-client-connection-path'
import { RelayReconnectController } from './mobile-relay-reconnect-controller'
const SIGNED_OUT_LABEL = 'Desktop signed out — sign in to Orca on your desktop to reconnect'
class FakeSocket {
static readonly OPEN = 1
readonly OPEN = FakeSocket.OPEN
readyState = FakeSocket.OPEN
bufferedAmount = 0
onopen: (() => void) | null = null
onmessage: ((event: { data: unknown }) => void) | null = null
onerror: (() => void) | null = null
onclose: ((event: { code: number; reason: string }) => void) | null = null
send = vi.fn()
close = vi.fn()
}
function linkOver(
socket: FakeSocket,
onHostCloseReason: (reason: string) => void,
onError: (error: Error) => void
): MobileRelayE2eeLink {
return new MobileRelayE2eeLink({
endpoint: { cellUrl: 'https://relay-c1.onorca.dev', relayHostId: 'AbCdEf0123_-xyZ9' },
credential: 'credential',
expectedCredentialKind: 'resume',
deviceToken: 'device-token',
desktopPublicKeyB64: 'desktop-key',
onAuthenticated: vi.fn(),
onText: vi.fn(),
onBinary: vi.fn(),
onHostCloseReason,
onError,
createSocket: () => socket as unknown as WebSocket
})
}
describe('relay close reason on the phone', () => {
it('reports the cell close reason and still fails with 4404', () => {
const socket = new FakeSocket()
const onHostCloseReason = vi.fn()
const onError = vi.fn()
linkOver(socket, onHostCloseReason, onError)
socket.onclose?.({
code: MOBILE_RELAY_CLOSE_CODE.HOST_OFFLINE,
reason: RELAY_HOST_CLOSE_REASON.SIGNED_OUT
})
expect(onHostCloseReason).toHaveBeenCalledWith(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
expect(onError).toHaveBeenCalledWith(new RelayOuterError(MOBILE_RELAY_CLOSE_CODE.HOST_OFFLINE))
})
// An old cell sends its constant, and every other close sends nothing.
it('reports nothing for a reason it does not know', () => {
const socket = new FakeSocket()
const onHostCloseReason = vi.fn()
linkOver(socket, onHostCloseReason, vi.fn())
socket.onclose?.({
code: MOBILE_RELAY_CLOSE_CODE.HOST_OFFLINE,
reason: 'relay connection rejected'
})
expect(onHostCloseReason).not.toHaveBeenCalled()
})
// The rejection arrives as a relay-hello AND a close, in an unordered pair.
// Whichever lands first, the reason must survive.
it('still reports the reason when the hello already failed the link', async () => {
const socket = new FakeSocket()
const onHostCloseReason = vi.fn()
linkOver(socket, onHostCloseReason, vi.fn())
socket.onmessage?.({
data: JSON.stringify({
type: 'relay-hello',
ok: false,
code: MOBILE_RELAY_CLOSE_CODE.HOST_OFFLINE
})
})
await Promise.resolve()
await Promise.resolve()
socket.onclose?.({
code: MOBILE_RELAY_CLOSE_CODE.HOST_OFFLINE,
reason: RELAY_HOST_CLOSE_REASON.SIGNED_OUT
})
expect(onHostCloseReason).toHaveBeenCalledWith(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
})
})
describe('the signed-out signal on the logical client', () => {
it('publishes on change and retires when any path reaches connected', () => {
const path = new LogicalClientConnectionPath(() => false)
const changes = vi.fn()
path.subscribe(changes)
path.setHostSignedOut(true)
path.setHostSignedOut(true)
expect(path.isHostSignedOut()).toBe(true)
expect(changes).toHaveBeenCalledTimes(1)
path.clearAfterConnected()
expect(path.isHostSignedOut()).toBe(false)
})
})
describe('RelayReconnectController cadence', () => {
// The reason changes no recovery decision; 4404 keeps the host-offline
// backoff it has always had, so a phone on this build retries exactly as
// often as one that never hears the reason.
it('keeps the host-offline retry delay for a 4404', () => {
const delays: number[] = []
const controller = new RelayReconnectController(
{
now: () => 0,
randomBytes: () => new Uint8Array([0, 0]),
setTimer: ((callback: () => void, delay: number) => {
delays.push(delay)
return 1 as unknown as ReturnType<typeof setTimeout>
}) as unknown as typeof setTimeout,
clearTimer: (() => {}) as unknown as typeof clearTimeout
},
vi.fn()
)
controller.registerFailure(new RelayOuterError(MOBILE_RELAY_CLOSE_CODE.HOST_OFFLINE))
// hostOfflineDelayMs' 5s floor, not the 250ms transport-backoff floor.
expect(delays.at(-1)).toBe(5_000)
})
})
describe('classifyConnection with a signed-out desktop', () => {
const base = { reconnectAttempts: 0, lastConnectedAt: null, hostSignedOut: true }
it('says so from the first failed dial instead of "Connecting via Relay…"', () => {
const verdict = classifyConnection({
...base,
state: 'connecting',
pendingPath: 'relay'
})
expect(verdict).toEqual({
kind: 'unreachable',
label: SIGNED_OUT_LABEL,
reason: 'never-connected'
})
expect(verdictDisplayLabel(verdict)).toBe(SIGNED_OUT_LABEL)
})
it('replaces "Can\'t reach desktop" on the direct path too', () => {
expect(
classifyConnection({ ...base, state: 'reconnecting', reconnectAttempts: 20 }).label
).toBe(SIGNED_OUT_LABEL)
})
it('reads as stale once this session had been connected', () => {
expect(
classifyConnection({ ...base, state: 'reconnecting', lastConnectedAt: 1, nowMs: 2 }).reason
).toBe('stale')
})
// A Tailscale endpoint cannot make "sign in on your desktop" better advice.
it('never appends the Tailscale hint', () => {
expect(
classifyConnection({ ...base, state: 'reconnecting', endpoint: '100.64.0.1' })
).not.toHaveProperty('hint')
})
it('never outranks a connected session', () => {
expect(classifyConnection({ ...base, state: 'connected' }).label).toBe('Connected')
})
// Re-pairing, not signing in, is the remedy when the pairing itself is dead.
it('never outranks a revoked pairing', () => {
expect(classifyConnection({ ...base, state: 'reconnecting', pairingRejected: true }).kind).toBe(
'auth-failed'
)
})
it('leaves every other verdict alone when the desktop is not signed out', () => {
expect(
classifyConnection({
state: 'connecting',
reconnectAttempts: 0,
lastConnectedAt: null,
pendingPath: 'relay',
hostSignedOut: false
}).label
).toBe('Connecting via Relay…')
})
})
@@ -24,6 +24,7 @@ export type RpcClientContextValue = {
getActivePath: (hostId: string) => MobileConnectionPath
getPendingPath: (hostId: string) => MobileConnectionPath | null
isPairingRejected: (hostId: string) => boolean
isHostSignedOut: (hostId: string) => boolean
subscribeHostState: (hostId: string, listener: (state: ConnectionState) => void) => () => void
getAllClients: () => { hostId: string; client: RpcClient }[]
subscribeAllHosts: (listener: () => void) => () => void
@@ -55,6 +55,9 @@ export type StableLogicalRpcClient = RpcClient & {
// Latched when the desktop has repeatedly refused this device's relay credential.
setPairingRejected(rejected: boolean): void
isPairingRejected(): boolean
// Latched when the relay named the desktop's own sign-out as the reason it is absent.
setHostSignedOut(signedOut: boolean): void
isHostSignedOut(): boolean
// Recovery attempts share this signal so status-only changes rerender.
onConnectionPathChange(listener: () => void): () => void
getGeneration(): number
@@ -282,6 +285,8 @@ export function createStableLogicalRpcClient(
setRecoveryAttempt: (attempt) => connectionPath.setRecoveryAttempt(attempt),
setPairingRejected: (rejected) => connectionPath.setPairingRejected(rejected),
isPairingRejected: () => connectionPath.isPairingRejected(),
setHostSignedOut: (signedOut) => connectionPath.setHostSignedOut(signedOut),
isHostSignedOut: () => connectionPath.isHostSignedOut(),
onConnectionPathChange: (listener) => connectionPath.subscribe(listener),
getGeneration: () => generation
}
+3 -1
View File
@@ -138,6 +138,7 @@ export function useAllHostClients(hostIds: string[], options?: UseAllHostClients
path: MobileConnectionPath
pendingPath: MobileConnectionPath | null
pairingRejected: boolean
hostSignedOut: boolean
}>((hostId) => {
const client = clientsByHostId.get(hostId)
return client
@@ -148,7 +149,8 @@ export function useAllHostClients(hostIds: string[], options?: UseAllHostClients
state: ctx.getState(hostId),
path: ctx.getActivePath(hostId),
pendingPath: ctx.getPendingPath(hostId),
pairingRejected: ctx.isPairingRejected(hostId)
pairingRejected: ctx.isPairingRejected(hostId),
hostSignedOut: ctx.isHostSignedOut(hostId)
}
]
: []
+1
View File
@@ -153,6 +153,7 @@
"repro:live-remote-realistic-freeze": "node config/scripts/live-remote-realistic-freeze-repro.mjs"
},
"dependencies": {
"@anthropic-ai/claude-agent-sdk": "0.3.251",
"@electron-toolkit/preload": "^3.0.2",
"@electron-toolkit/utils": "^4.0.0",
"@floating-ui/dom": "1.7.6",
+117 -16
View File
@@ -122,6 +122,9 @@ importers:
.:
dependencies:
'@anthropic-ai/claude-agent-sdk':
specifier: 0.3.251
version: 0.3.251(@anthropic-ai/sdk@0.122.0(zod@4.5.4))(@modelcontextprotocol/sdk@1.30.0(supports-color@7.2.0)(zod@4.5.4))(zod@4.5.4)
'@electron-toolkit/preload':
specifier: ^3.0.2
version: 3.0.2(electron@43.4.1(supports-color@7.2.0))
@@ -535,6 +538,23 @@ packages:
'@antfu/install-pkg@1.1.0':
resolution: {integrity: sha512-MGQsmw10ZyI+EJo45CdSER4zEb+p31LpDAFp2Z3gkSd1yqVZGi0Ebx++YTEMonJy4oChEMLsxZ64j8FH6sSqtQ==}
'@anthropic-ai/claude-agent-sdk@0.3.251':
resolution: {integrity: sha512-DqSi8mH2tQYRlVV0G+lJnQ/WbjJZ/a+8cJ3vPuYoqh8esIIvXHm1ZOXV1UPGsFYRnbBytEoiSGitguEXd+sQ+Q==}
engines: {node: '>=18.0.0'}
peerDependencies:
'@anthropic-ai/sdk': '>=0.93.0'
'@modelcontextprotocol/sdk': ^1.29.0
zod: ^4.0.0
'@anthropic-ai/sdk@0.122.0':
resolution: {integrity: sha512-GGPNftt0caaz9MDlmNQGHX8855Ojaduyy5pm9Sm1h7HalCn0cWNb5/bweadJF+4yzbal+QL6ztBa09WAAOzLmQ==}
hasBin: true
peerDependencies:
zod: ^3.25.0 || ^4.0.0
peerDependenciesMeta:
zod:
optional: true
'@babel/code-frame@7.29.7':
resolution: {integrity: sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==}
engines: {node: '>=6.9.0'}
@@ -2628,6 +2648,9 @@ packages:
resolution: {integrity: sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==}
engines: {node: '>=18'}
'@stablelib/base64@1.0.1':
resolution: {integrity: sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==}
'@stablyai/playwright-base@2.1.14':
resolution: {integrity: sha512-/iAgMW5tC0ETDo3mFyTzszRrD7rGFIT4fgDgtZxqa9vPhiTLix/1+GeOOBNY0uS+XRLFY0Uc/irsC3XProL47g==}
engines: {node: '>=18'}
@@ -4493,6 +4516,9 @@ packages:
resolution: {integrity: sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==}
engines: {node: '>=8.6.0'}
fast-sha256@1.3.0:
resolution: {integrity: sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==}
fast-string-truncated-width@3.0.3:
resolution: {integrity: sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g==}
@@ -5039,6 +5065,10 @@ packages:
json-parse-even-better-errors@2.3.1:
resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==}
json-schema-to-ts@3.1.1:
resolution: {integrity: sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==}
engines: {node: '>=16'}
json-schema-traverse@1.0.0:
resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==}
@@ -6425,6 +6455,9 @@ packages:
stackback@0.0.2:
resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==}
standardwebhooks@1.1.1:
resolution: {integrity: sha512-bCbX9ZEyFkWPsRz7Bl3NuQUJohmwGSev/yhr7vhaGPlc4AfIrspIRa6cPTBuI1ItmrTDJ4d/S2hCsfe4+vQGnQ==}
stat-mode@1.0.0:
resolution: {integrity: sha512-jH9EhtKIjuXZ2cWxmXS8ZP80XyC3iasQxMDV8jzhNJpfDb7VbQLVW4Wvsxz9QZvzV+G4YoSfBUVKDOyxLzi/sg==}
engines: {node: '>= 6'}
@@ -6608,6 +6641,9 @@ packages:
truncate-utf8-bytes@1.0.2:
resolution: {integrity: sha512-95Pu1QXQvruGEhv62XCMO3Mm90GscOCClvrIUwCM0PYOXK3kaF3l3sIHxx71ThJfcbM2O5Au6SO3AWCSEfW4mQ==}
ts-algebra@2.0.0:
resolution: {integrity: sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==}
ts-dedent@2.2.0:
resolution: {integrity: sha512-q5W7tVM71e2xjHZTlgfTDoPF/SmqKG5hddq9SzR49CH2hayqRKJtQ4mtRlSxKaJlR/+9rEM+mnBHf7I2/BQcpQ==}
engines: {node: '>=6.10'}
@@ -7007,6 +7043,16 @@ packages:
zwitch@2.0.4:
resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==}
ignoredOptionalDependencies:
- '@anthropic-ai/claude-agent-sdk-darwin-arm64'
- '@anthropic-ai/claude-agent-sdk-darwin-x64'
- '@anthropic-ai/claude-agent-sdk-linux-arm64'
- '@anthropic-ai/claude-agent-sdk-linux-arm64-musl'
- '@anthropic-ai/claude-agent-sdk-linux-x64'
- '@anthropic-ai/claude-agent-sdk-linux-x64-musl'
- '@anthropic-ai/claude-agent-sdk-win32-arm64'
- '@anthropic-ai/claude-agent-sdk-win32-x64'
snapshots:
'@adobe/css-tools@4.5.0': {}
@@ -7016,6 +7062,19 @@ snapshots:
package-manager-detector: 1.6.0
tinyexec: 1.1.2
'@anthropic-ai/claude-agent-sdk@0.3.251(@anthropic-ai/sdk@0.122.0(zod@4.5.4))(@modelcontextprotocol/sdk@1.30.0(supports-color@7.2.0)(zod@4.5.4))(zod@4.5.4)':
dependencies:
'@anthropic-ai/sdk': 0.122.0(zod@4.5.4)
'@modelcontextprotocol/sdk': 1.30.0(supports-color@7.2.0)(zod@4.5.4)
zod: 4.5.4
'@anthropic-ai/sdk@0.122.0(zod@4.5.4)':
dependencies:
json-schema-to-ts: 3.1.1
standardwebhooks: 1.1.1
optionalDependencies:
zod: 4.5.4
'@babel/code-frame@7.29.7':
dependencies:
'@babel/helper-validator-identifier': 7.29.7
@@ -7669,6 +7728,28 @@ snapshots:
dependencies:
'@chevrotain/types': 11.1.2
'@modelcontextprotocol/sdk@1.30.0(supports-color@7.2.0)(zod@4.5.4)':
dependencies:
'@hono/node-server': 2.1.0(hono@4.13.0)
ajv: 8.20.0
ajv-formats: 3.0.1(ajv@8.20.0)
content-type: 1.0.5
cors: 2.8.6
cross-spawn: 7.0.6
eventsource: 3.0.7
eventsource-parser: 3.0.8
express: 5.2.1(supports-color@7.2.0)
express-rate-limit: 8.5.2(express@5.2.1(supports-color@7.2.0))
hono: 4.13.0
jose: 6.2.3
json-schema-typed: 8.0.2
pkce-challenge: 5.0.1
raw-body: 3.0.2
zod: 4.5.4
zod-to-json-schema: 3.25.2(zod@4.5.4)
transitivePeerDependencies:
- supports-color
'@modelcontextprotocol/sdk@1.30.0(zod@3.25.76)':
dependencies:
'@hono/node-server': 2.1.0(hono@4.13.0)
@@ -7679,8 +7760,8 @@ snapshots:
cross-spawn: 7.0.6
eventsource: 3.0.7
eventsource-parser: 3.0.8
express: 5.2.1
express-rate-limit: 8.5.2(express@5.2.1)
express: 5.2.1(supports-color@7.2.0)
express-rate-limit: 8.5.2(express@5.2.1(supports-color@7.2.0))
hono: 4.13.0
jose: 6.2.3
json-schema-typed: 8.0.2
@@ -8917,6 +8998,8 @@ snapshots:
'@sindresorhus/merge-streams@4.0.0': {}
'@stablelib/base64@1.0.1': {}
'@stablyai/playwright-base@2.1.14(@playwright/test@1.59.1)(zod@4.5.4)':
dependencies:
'@playwright/test': 1.59.1
@@ -9923,7 +10006,7 @@ snapshots:
bluebird@3.7.2: {}
body-parser@2.3.0:
body-parser@2.3.0(supports-color@7.2.0):
dependencies:
bytes: 3.1.2
content-type: 2.0.0
@@ -10779,15 +10862,15 @@ snapshots:
exponential-backoff@3.1.3: {}
express-rate-limit@8.5.2(express@5.2.1):
express-rate-limit@8.5.2(express@5.2.1(supports-color@7.2.0)):
dependencies:
express: 5.2.1
express: 5.2.1(supports-color@7.2.0)
ip-address: 10.4.0
express@5.2.1:
express@5.2.1(supports-color@7.2.0):
dependencies:
accepts: 2.0.0
body-parser: 2.3.0
body-parser: 2.3.0(supports-color@7.2.0)
content-disposition: 1.1.0
content-type: 1.0.5
cookie: 0.7.2
@@ -10797,7 +10880,7 @@ snapshots:
encodeurl: 2.0.0
escape-html: 1.0.3
etag: 1.8.1
finalhandler: 2.1.1
finalhandler: 2.1.1(supports-color@7.2.0)
fresh: 2.0.0
http-errors: 2.0.1
merge-descriptors: 2.0.0
@@ -10808,9 +10891,9 @@ snapshots:
proxy-addr: 2.0.7
qs: 6.15.2
range-parser: 1.2.1
router: 2.2.0
send: 1.2.1
serve-static: 2.2.1
router: 2.2.0(supports-color@7.2.0)
send: 1.2.1(supports-color@7.2.0)
serve-static: 2.2.1(supports-color@7.2.0)
statuses: 2.0.2
type-is: 2.1.0
vary: 1.1.2
@@ -10831,6 +10914,8 @@ snapshots:
merge2: 1.4.1
micromatch: 4.0.8
fast-sha256@1.3.0: {}
fast-string-truncated-width@3.0.3: {}
fast-string-width@3.0.2:
@@ -10871,7 +10956,7 @@ snapshots:
dependencies:
to-regex-range: 5.0.1
finalhandler@2.1.1:
finalhandler@2.1.1(supports-color@7.2.0):
dependencies:
debug: 4.4.3(supports-color@7.2.0)
encodeurl: 2.0.0
@@ -11445,6 +11530,11 @@ snapshots:
json-parse-even-better-errors@2.3.1: {}
json-schema-to-ts@3.1.1:
dependencies:
'@babel/runtime': 7.29.7
ts-algebra: 2.0.0
json-schema-traverse@1.0.0: {}
json-schema-typed@8.0.2: {}
@@ -13037,7 +13127,7 @@ snapshots:
points-on-curve: 0.2.0
points-on-path: 0.2.1
router@2.2.0:
router@2.2.0(supports-color@7.2.0):
dependencies:
debug: 4.4.3(supports-color@7.2.0)
depd: 2.0.0
@@ -13080,7 +13170,7 @@ snapshots:
semver@7.8.1: {}
send@1.2.1:
send@1.2.1(supports-color@7.2.0):
dependencies:
debug: 4.4.3(supports-color@7.2.0)
encodeurl: 2.0.0
@@ -13107,12 +13197,12 @@ snapshots:
transitivePeerDependencies:
- typescript
serve-static@2.2.1:
serve-static@2.2.1(supports-color@7.2.0):
dependencies:
encodeurl: 2.0.0
escape-html: 1.0.3
parseurl: 1.3.3
send: 1.2.1
send: 1.2.1(supports-color@7.2.0)
transitivePeerDependencies:
- supports-color
@@ -13267,6 +13357,11 @@ snapshots:
stackback@0.0.2: {}
standardwebhooks@1.1.1:
dependencies:
'@stablelib/base64': 1.0.1
fast-sha256: 1.3.0
stat-mode@1.0.0: {}
state-local@1.0.7: {}
@@ -13437,6 +13532,8 @@ snapshots:
dependencies:
utf8-byte-length: 1.0.5
ts-algebra@2.0.0: {}
ts-dedent@2.2.0: {}
ts-morph@26.0.0:
@@ -13786,6 +13883,10 @@ snapshots:
dependencies:
zod: 3.25.76
zod-to-json-schema@3.25.2(zod@4.5.4):
dependencies:
zod: 4.5.4
zod@3.25.76: {}
zod@4.5.4: {}
+14
View File
@@ -12,6 +12,20 @@ minimumReleaseAgeExclude:
- zod@4.5.4
shamefullyHoist: true
# Orca always launches the user's own resolved Claude CLI via
# pathToClaudeCodeExecutable, so the SDK's bundled ~95 MB-per-platform CLI
# binaries must never be installed. Excluding them is what makes the path
# override mandatory rather than merely preferred.
ignoredOptionalDependencies:
- '@anthropic-ai/claude-agent-sdk-darwin-arm64'
- '@anthropic-ai/claude-agent-sdk-darwin-x64'
- '@anthropic-ai/claude-agent-sdk-linux-arm64'
- '@anthropic-ai/claude-agent-sdk-linux-arm64-musl'
- '@anthropic-ai/claude-agent-sdk-linux-x64'
- '@anthropic-ai/claude-agent-sdk-linux-x64-musl'
- '@anthropic-ai/claude-agent-sdk-win32-arm64'
- '@anthropic-ai/claude-agent-sdk-win32-x64'
supportedArchitectures:
os:
- current
@@ -127,10 +127,6 @@ __orca_osc133_precmd() {
unset __orca_in_command
fi
printf "\033]133;A\007"
# Why: emit the shell-ready marker here (not a trailing PROMPT_COMMAND entry)
# so a framework that must be last in PROMPT_COMMAND — bash-preexec — is not
# displaced by one of Orca's own hooks.
[[ -n "$__orca_ready_marker" ]] && printf "\033]777;orca-shell-ready\007"
return "$exit_code"
}
__orca_osc133_preexec() {
@@ -188,6 +184,11 @@ __orca_osc133_epilogue() {
unset __orca_in_prompt_command
__orca_adopt_outer_debug_trap
trap '__orca_osc133_preexec' DEBUG
# Readline renders PS1 after entering raw mode; prompt hooks still run in cooked mode.
if [[ -n "$__orca_ready_marker" ]]; then
PS1="${PS1-}"'\[\e]777;orca-shell-ready\a\]'
__orca_ready_marker=""
fi
}
__orca_normalize_prompt_command_part() {
local __orca_value="$1" __orca_output_name="$2" __orca_character __orca_chunk
@@ -22,6 +22,20 @@ function workingStatus(): AgentAwakeStatus {
}
}
describe('AgentAwakeService status array ownership', () => {
it('does not observe rows appended to the caller array after setStatuses', () => {
const service = new AgentAwakeService()
service.setMode('auto')
const statuses: AgentAwakeStatus[] = [workingStatus()]
service.setStatuses(statuses)
const before = service.getWorkingAgentCount()
statuses.push(workingStatus(), workingStatus())
expect(service.getWorkingAgentCount()).toBe(before)
})
})
function createBlocker() {
const startedIds = new Set<number>()
let nextId = 1
+4 -2
View File
@@ -105,7 +105,8 @@ export class AgentAwakeService {
}
setStatuses(statuses: AgentAwakeStatus[]): void {
this.statuses = statuses.map((status) => ({ ...status }))
// Copy the array, not every row: the hook server allocates each row fresh per event.
this.statuses = [...statuses]
this.refresh('status-change')
}
@@ -171,7 +172,8 @@ export class AgentAwakeService {
private getEligibleRunningStatusCount(): number {
const now = this.now()
return this.statuses.filter((status) => this.isWakeEligible(status, now)).length
// Counted in place: the filtered array was only ever measured, and this runs per hook event.
return this.statuses.reduce((count, s) => count + (this.isWakeEligible(s, now) ? 1 : 0), 0)
}
private isWakeEligible(status: AgentAwakeStatus, now: number): boolean {
+17 -1
View File
@@ -4,6 +4,7 @@ import type { AutomationPrecheck, AutomationPrecheckResult } from '../../shared/
import { MAX_AUTOMATION_PRECHECK_OUTPUT_CHARS } from '../../shared/automation-precheck'
import { getSshConnectionManager } from '../ipc/ssh'
import { shellEscape } from '../ssh/ssh-connection-utils'
import { admitSelfInitiatedTreeKill } from '../own-chromium-tree-kill-guard'
type AutomationPrecheckExecutionTarget =
| {
@@ -73,7 +74,10 @@ function failedPrecheckResult(
})
}
function killLocalPrecheckProcessTree(child: ChildProcess): ReturnType<typeof setTimeout> | null {
/** Exported for the refusal-fallback test; the timeout path is otherwise unreachable. */
export function killLocalPrecheckProcessTree(
child: ChildProcess
): ReturnType<typeof setTimeout> | null {
const pid = child.pid
if (!pid) {
child.kill()
@@ -81,6 +85,18 @@ function killLocalPrecheckProcessTree(child: ChildProcess): ReturnType<typeof se
}
if (process.platform === 'win32') {
if (
!admitSelfInitiatedTreeKill({
pid,
site: 'automation-precheck-timeout',
scope: 'win-taskkill-tree'
})
) {
// Refusal blocks the tree walk, not the termination: killing the root by
// handle cannot reach a recycled pid, and a timed-out precheck must stop.
child.kill()
return null
}
try {
// Why: shell prechecks can launch child processes; taskkill walks the
// Windows process tree so timeout means the command is actually stopped.
@@ -0,0 +1,168 @@
import { describe, expect, it } from 'vitest'
import type { GlobalSettings } from '../../shared/global-settings-types'
import type { ClaudeManagedAccount } from '../../shared/managed-account-types'
import {
CLAUDE_AUTH_ENV_VARS,
hasClaudeAuthEnvConflict,
shouldStripClaudeAuthEnvForAccount
} from './environment'
import {
normalizeTuiAgentEnvRecord,
resolveTuiAgentLaunchEnv
} from '../../shared/tui-agent-launch-defaults'
import { claudeStructuredAuthPolicyForSettings } from './claude-structured-auth-policy'
const HOST_ACCOUNT = { id: 'host-a', managedAuthRuntime: 'host' } as ClaudeManagedAccount
const WSL_ACCOUNT = { id: 'wsl-b', managedAuthRuntime: 'wsl' } as ClaudeManagedAccount
const LEGACY_ACCOUNT = { id: 'legacy-c' } as ClaudeManagedAccount
function settings(
overrides: Partial<
Pick<
GlobalSettings,
| 'claudeManagedAccounts'
| 'activeClaudeManagedAccountId'
| 'activeClaudeManagedAccountIdsByRuntime'
>
>
): Parameters<typeof claudeStructuredAuthPolicyForSettings>[0] {
return {
claudeManagedAccounts: [HOST_ACCOUNT, WSL_ACCOUNT, LEGACY_ACCOUNT],
activeClaudeManagedAccountId: null,
...overrides
} as Parameters<typeof claudeStructuredAuthPolicyForSettings>[0]
}
// The predicate now backs BOTH transports (runtime-auth-preparation.ts and the
// structured wiring), so it needs a test of its own: forcing it to a constant used
// to leave ~1000 tests green.
describe('shouldStripClaudeAuthEnvForAccount', () => {
it('does not strip when no managed account is selected', () => {
expect(shouldStripClaudeAuthEnvForAccount([HOST_ACCOUNT], null)).toBe(false)
expect(shouldStripClaudeAuthEnvForAccount([HOST_ACCOUNT], undefined)).toBe(false)
expect(shouldStripClaudeAuthEnvForAccount([HOST_ACCOUNT], '')).toBe(false)
})
it('strips for a host-managed account', () => {
expect(shouldStripClaudeAuthEnvForAccount([HOST_ACCOUNT, WSL_ACCOUNT], 'host-a')).toBe(true)
})
it('strips for an account with no explicit runtime (the legacy host shape)', () => {
expect(shouldStripClaudeAuthEnvForAccount([LEGACY_ACCOUNT], 'legacy-c')).toBe(true)
})
it('does not strip for a WSL-managed account, matching runtime-auth-preparation', () => {
expect(shouldStripClaudeAuthEnvForAccount([HOST_ACCOUNT, WSL_ACCOUNT], 'wsl-b')).toBe(false)
})
it('strips for a selected id no account list explains', () => {
// Fail-safe: an id we cannot resolve is treated as a pinned account, never as
// "no account", so an unreadable settings blob cannot open the strip.
expect(shouldStripClaudeAuthEnvForAccount([HOST_ACCOUNT], 'deleted-d')).toBe(true)
expect(shouldStripClaudeAuthEnvForAccount(undefined, 'deleted-d')).toBe(true)
expect(shouldStripClaudeAuthEnvForAccount([], 'deleted-d')).toBe(true)
})
})
describe('claudeStructuredAuthPolicyForSettings', () => {
it('reads the host runtime selection, not the legacy flat field alone', () => {
expect(
claudeStructuredAuthPolicyForSettings(
settings({
activeClaudeManagedAccountId: 'host-a',
activeClaudeManagedAccountIdsByRuntime: { host: null, wsl: {} }
})
)
).toEqual({ stripAuthEnv: true })
})
it('strips when a host account is pinned by runtime selection', () => {
expect(
claudeStructuredAuthPolicyForSettings(
settings({ activeClaudeManagedAccountIdsByRuntime: { host: 'host-a', wsl: {} } })
)
).toEqual({ stripAuthEnv: true })
})
it('does not strip for system auth, so an API-key-only user keeps their sign-in', () => {
expect(claudeStructuredAuthPolicyForSettings(settings({}))).toEqual({ stripAuthEnv: false })
})
it('ignores a WSL-only selection: the structured child is always a native host process', () => {
expect(
claudeStructuredAuthPolicyForSettings(
settings({
activeClaudeManagedAccountIdsByRuntime: { host: null, wsl: { Ubuntu: 'wsl-b' } }
})
)
).toEqual({ stripAuthEnv: false })
})
})
describe('the strip vocabulary the policy governs', () => {
it('covers every Anthropic auth variable the terminal path knows about', () => {
// A new auth var added to the list without a matching refusal/strip path is the
// shape of the leak this lane already shipped once.
expect([...CLAUDE_AUTH_ENV_VARS]).toEqual([
'ANTHROPIC_API_KEY',
'ANTHROPIC_AUTH_TOKEN',
'CLAUDE_CODE_OAUTH_TOKEN',
'AWS_BEARER_TOKEN_BEDROCK'
])
})
})
// The refusal has to cover exactly what the strip removes. Anything narrower lets an
// override reach the child that applyClaudeEnvPatch would have deleted.
describe('hasClaudeAuthEnvConflict matches the strip it guards', () => {
it('refuses each Anthropic auth variable', () => {
for (const key of CLAUDE_AUTH_ENV_VARS) {
expect(hasClaudeAuthEnvConflict({ [key]: 'v' }, 'linux')).toBe(true)
}
})
// `ANTHROPIC_API_KEY=` in the agent env box is how a user blanks a variable, and the
// settings pipeline preserves the empty value (agent-default-env-draft.ts assigns
// everything after the `=`; normalizeTuiAgentEnvRecord drops empty KEYS only). An
// empty value cannot beat the pinned account and the strip removes the name anyway,
// so refusing it would break a terminal launch that works today for no security gain.
it('admits an override whose value is empty, the documented way to blank a variable', () => {
expect(hasClaudeAuthEnvConflict({ ANTHROPIC_API_KEY: '' }, 'linux')).toBe(false)
expect(hasClaudeAuthEnvConflict({ anthropic_api_key: '' }, 'win32')).toBe(false)
expect(hasClaudeAuthEnvConflict({ ANTHROPIC_CUSTOM_HEADERS: '' }, 'linux')).toBe(false)
})
it('still refuses the same names once they carry a value', () => {
expect(hasClaudeAuthEnvConflict({ ANTHROPIC_API_KEY: 'sk-ant' }, 'linux')).toBe(true)
})
// The end-to-end shape the regression actually took: settings text -> normalized
// record -> launch env -> the predicate the terminal preflight gates on.
it('admits a blanked variable all the way from the settings record', () => {
const configured = normalizeTuiAgentEnvRecord({ claude: { ANTHROPIC_API_KEY: '' } })
const launchEnv = resolveTuiAgentLaunchEnv('claude', configured)
expect(launchEnv).toEqual({ ANTHROPIC_API_KEY: '' })
expect(hasClaudeAuthEnvConflict(launchEnv, 'linux')).toBe(false)
})
it('folds case on win32, where the OS does', () => {
expect(hasClaudeAuthEnvConflict({ anthropic_api_key: 'sk-lower' }, 'win32')).toBe(true)
expect(hasClaudeAuthEnvConflict({ Anthropic_Custom_Headers: 'x-api-key: v' }, 'win32')).toBe(
true
)
})
it('keeps env names case-sensitive off win32', () => {
expect(hasClaudeAuthEnvConflict({ anthropic_api_key: 'sk-lower' }, 'linux')).toBe(false)
})
it('admits non-auth Anthropic settings on both platforms', () => {
expect(hasClaudeAuthEnvConflict({ ANTHROPIC_BASE_URL: 'https://gw.test' }, 'linux')).toBe(false)
expect(hasClaudeAuthEnvConflict({ ANTHROPIC_BASE_URL: 'https://gw.test' }, 'win32')).toBe(false)
expect(hasClaudeAuthEnvConflict({ ANTHROPIC_CUSTOM_HEADERS: 'X-Trace: 1' }, 'linux')).toBe(
false
)
expect(hasClaudeAuthEnvConflict(undefined, 'linux')).toBe(false)
})
})
@@ -0,0 +1,37 @@
import type { GlobalSettings } from '../../shared/global-settings-types'
import { shouldStripClaudeAuthEnvForAccount } from './environment'
import { getSelectedClaudeAccountIdForTarget } from './runtime-selection'
/** The structured mirror of the terminal preflight's `prepareClaudeAuth` result:
* the one field a launch resolution needs from the managed-account state. */
export type ClaudeStructuredAuthPolicy = {
stripAuthEnv: boolean
}
/**
* The only supported way to build a structured launch's auth policy.
*
* It exists as a named function rather than an inline object at the wiring site so
* that the settings-to-policy mapping is testable on its own: the one production
* wiring lives in a `@ts-nocheck` file, where neither the compiler nor a type test
* can see a dropped field.
*
* Structured Claude always spawns a native local-host child — the launch resolver
* refuses any record with a remote execution host or a WSL distro — so the host
* selection, not the platform default target, owns its auth.
*/
export function claudeStructuredAuthPolicyForSettings(
settings: Pick<
GlobalSettings,
| 'claudeManagedAccounts'
| 'activeClaudeManagedAccountId'
| 'activeClaudeManagedAccountIdsByRuntime'
>
): ClaudeStructuredAuthPolicy {
return {
stripAuthEnv: shouldStripClaudeAuthEnvForAccount(
settings.claudeManagedAccounts,
getSelectedClaudeAccountIdForTarget(settings, { runtime: 'host' })
)
}
}
+94 -7
View File
@@ -1,3 +1,5 @@
import type { ClaudeManagedAccount } from '../../shared/managed-account-types'
export const CLAUDE_AUTH_ENV_VARS = [
'ANTHROPIC_API_KEY',
'ANTHROPIC_AUTH_TOKEN',
@@ -13,14 +15,21 @@ export type ClaudeEnvPatch = {
export function applyClaudeEnvPatch(
baseEnv: Record<string, string>,
patch: ClaudeEnvPatch,
options?: { stripAuthEnv?: boolean }
options?: { stripAuthEnv?: boolean; platform?: NodeJS.Platform }
): Record<string, string> {
if (options?.stripAuthEnv) {
for (const key of CLAUDE_AUTH_ENV_VARS) {
delete baseEnv[key]
}
if (isAuthLikeCustomHeaders(baseEnv.ANTHROPIC_CUSTOM_HEADERS)) {
delete baseEnv.ANTHROPIC_CUSTOM_HEADERS
const platform = options.platform ?? process.platform
for (const key of Object.keys(baseEnv)) {
const normalized = platform === 'win32' ? key.toUpperCase() : key
if (
(platform === 'win32' && CLAUDE_AUTH_ENV_VARS.some((authKey) => authKey === normalized)) ||
(normalized === 'ANTHROPIC_CUSTOM_HEADERS' && isAuthLikeCustomHeaders(baseEnv[key]))
) {
delete baseEnv[key]
}
}
}
@@ -34,16 +43,94 @@ export function applyClaudeEnvPatch(
return baseEnv
}
export function hasClaudeAuthEnvConflict(env: Record<string, string> | undefined): boolean {
if (!env) {
/** One string for every transport, so a terminal launch and a structured launch
* cannot drift into telling the user two different things about one refusal. */
export const CLAUDE_AUTH_ENV_CONFLICT_MESSAGE =
'This Claude launch defines explicit Anthropic auth environment variables. Remove those overrides before using a managed Claude account.'
export const CLAUDE_AUTH_SWITCH_IN_PROGRESS_MESSAGE =
'A Claude account switch is in progress. Try again after it finishes.'
/**
* Whether a launch on the host runtime must drop inherited Anthropic auth.
*
* Only a pinned host-managed account owns the credential, so only it may strip:
* with no managed account the user's own `ANTHROPIC_*` is their sign-in, and
* removing it signs them out of a CLI that would otherwise have worked.
*/
export function shouldStripClaudeAuthEnvForAccount(
accounts: readonly ClaudeManagedAccount[] | undefined,
activeAccountId: string | null | undefined
): boolean {
if (!activeAccountId) {
return false
}
return (
CLAUDE_AUTH_ENV_VARS.some((key) => Boolean(env[key])) ||
isAuthLikeCustomHeaders(env.ANTHROPIC_CUSTOM_HEADERS)
(accounts ?? []).find((account) => account.id === activeAccountId)?.managedAuthRuntime !== 'wsl'
)
}
/**
* Whether a launch's explicit env carries Anthropic auth a managed account must own.
*
* The key comparison mirrors applyClaudeEnvPatch's strip exactly: case-insensitive on
* win32, where the OS folds env names so `anthropic_api_key` is an effective
* `ANTHROPIC_API_KEY`, and case-sensitive elsewhere. A refusal narrower than the strip
* lets an override through that the strip would have removed.
*
* A non-empty value is what makes it a conflict. `ANTHROPIC_API_KEY=` in the agent env
* box is how a user blanks a variable — the settings pipeline preserves that empty value
* (normalizeTuiAgentEnvRecord drops empty KEYS only) — and an empty override can neither
* authenticate nor beat the pinned account, while the strip removes the name regardless.
* Refusing it would break a terminal launch that works today for no security gain.
*/
/**
* The inherited Anthropic auth a non-stripping launch has to carry forward explicitly.
*
* applyClaudeEnvPatch always strips the inherited half of a child env, and the
* configured half is what overrides it — so a system-auth user's own key only survives
* if the caller puts it back deliberately. Returns the exact keys present, so a
* win32 `anthropic_api_key` is carried under the name the OS actually has.
*/
export function claudeAuthEnvCarriedForward(
inherited: NodeJS.ProcessEnv,
platform: NodeJS.Platform = process.platform
): Record<string, string> {
const carried: Record<string, string> = {}
for (const [key, value] of Object.entries(inherited)) {
if (value === undefined) {
continue
}
const normalized = platform === 'win32' ? key.toUpperCase() : key
if (
CLAUDE_AUTH_ENV_VARS.some((authKey) => authKey === normalized) ||
(normalized === 'ANTHROPIC_CUSTOM_HEADERS' && isAuthLikeCustomHeaders(value))
) {
carried[key] = value
}
}
return carried
}
export function hasClaudeAuthEnvConflict(
env: Record<string, string> | undefined,
platform: NodeJS.Platform = process.platform
): boolean {
if (!env) {
return false
}
for (const [key, value] of Object.entries(env)) {
const normalized = platform === 'win32' ? key.toUpperCase() : key
if (value && CLAUDE_AUTH_ENV_VARS.some((authKey) => authKey === normalized)) {
return true
}
if (normalized === 'ANTHROPIC_CUSTOM_HEADERS' && isAuthLikeCustomHeaders(value)) {
return true
}
}
return false
}
function isAuthLikeCustomHeaders(value: string | undefined): boolean {
if (!value) {
return false
+73
View File
@@ -5,6 +5,13 @@ const liveClaudePtyIds = new Set<string>()
// survived the app restart inside the daemon.
const seededUnconfirmedPtyIds = new Set<string>()
let switchInProgress = false
// Woken by endClaudeAuthSwitch so a caller past the point of no return can wait the
// swap out instead of refusing. See whenClaudeAuthSwitchSettles.
const switchSettledListeners = new Set<() => void>()
/** A managed account swap is a credential-file rewrite, not a network round trip;
* anything past this is a wedged switch, and refusing beats waiting forever. */
export const CLAUDE_AUTH_SWITCH_SETTLE_TIMEOUT_MS = 15_000
export type ClaudeLivePtyPersistence = {
addClaudeLivePtySessionId(sessionId: string): void
@@ -81,6 +88,35 @@ export function markClaudePtyExited(ptyId: string): void {
notifyDrainedOnTransition(hadLivePtys)
}
/**
* Register a structured Claude child with the same gate the terminal path uses.
*
* The gate is what makes the managed OAuth refresh defer instead of rotating a
* single-use refresh token out from under a running Claude (runtime-auth-sync.ts).
* A structured session's child is as much a live Claude as a PTY's is, so it has to
* hold the gate too — otherwise a refresh mid-turn breaks its next API call while an
* identical terminal session is protected.
*
* Deliberately not persisted, unlike markClaudePtySpawned: these children are direct
* children of this process and cannot survive a restart, so seeding them back on the
* next launch would hold the gate closed for a process that is provably gone.
*/
export function markClaudeStructuredChildSpawned(childKey: string): void {
liveClaudePtyIds.add(structuredChildGateId(childKey))
}
export function markClaudeStructuredChildExited(childKey: string): void {
const hadLivePtys = liveClaudePtyIds.size > 0
liveClaudePtyIds.delete(structuredChildGateId(childKey))
notifyDrainedOnTransition(hadLivePtys)
}
// Namespaced so a structured child can never collide with a daemon PTY session id,
// which confirmSeededClaudeLivePtys reconciles against the daemon's own list.
function structuredChildGateId(childKey: string): string {
return `claude-structured:${childKey}`
}
export function hasLiveClaudePtys(): boolean {
return liveClaudePtyIds.size > 0
}
@@ -93,7 +129,44 @@ export function beginClaudeAuthSwitch(): void {
}
export function endClaudeAuthSwitch(): void {
const wasInProgress = switchInProgress
switchInProgress = false
if (!wasInProgress) {
return
}
// Each listener removes itself as it settles; Set iteration is defined over that.
for (const listener of switchSettledListeners) {
listener()
}
}
/**
* Resolves `true` once no account switch is running, `false` if one is still running
* at the deadline.
*
* Exists for callers that have already done irreversible work — a structured acquire
* has closed the old child by the time it resolves its launch, so turning a switch
* into a refusal there strands the user with a dead session and no replacement.
* Waiting for the swap and then launching against it is the recoverable answer;
* refusing is only correct when nothing has been torn down yet.
*/
export function whenClaudeAuthSwitchSettles(
timeoutMs = CLAUDE_AUTH_SWITCH_SETTLE_TIMEOUT_MS
): Promise<boolean> {
if (!switchInProgress) {
return Promise.resolve(true)
}
return new Promise<boolean>((resolve) => {
const settle = (settled: boolean): void => {
switchSettledListeners.delete(listener)
clearTimeout(timer)
resolve(settled)
}
const listener = (): void => settle(true)
switchSettledListeners.add(listener)
const timer = setTimeout(() => settle(false), timeoutMs)
timer.unref?.()
})
}
export function isClaudeAuthSwitchInProgress(): boolean {
@@ -2,6 +2,7 @@ import { join } from 'node:path'
import type { ClaudeManagedAccount } from '../../../shared/managed-account-types'
import { resolveLocalAccountRuntimeTarget } from '../../../shared/local-account-runtime'
import { parseWslUncPath } from '../../../shared/wsl-paths'
import { shouldStripClaudeAuthEnvForAccount } from '../environment'
import { getDefaultWslDistro, getWslHome } from '../../wsl'
import {
getSelectedClaudeAccountIdForTarget,
@@ -69,7 +70,10 @@ export class ClaudeRuntimeAuthPreparationService extends ClaudeRuntimeAuthSnapsh
wslDistro: null,
wslLinuxConfigDir: null,
envPatch: paths.envPatch,
stripAuthEnv: Boolean(activeAccountId && activeAccount?.managedAuthRuntime !== 'wsl'),
stripAuthEnv: shouldStripClaudeAuthEnvForAccount(
settings.claudeManagedAccounts,
activeAccountId
),
managedRefreshDeferredByLivePty: Boolean(
activeAccountId &&
activeAccount?.managedAuthRuntime !== 'wsl' &&
@@ -0,0 +1,40 @@
import { describe, expect, it } from 'vitest'
import { parseClaudeUsageRecord } from './transcript-record-parser'
function assistantLine(overrides: Record<string, unknown> = {}): string {
return JSON.stringify({
type: 'assistant',
sessionId: 'session-1',
timestamp: '2026-01-01T00:00:00.000Z',
message: { usage: { input_tokens: 3, output_tokens: 5 } },
...overrides
})
}
describe('assistant-record prefilter', () => {
it('still parses an ordinary assistant record', () => {
expect(parseClaudeUsageRecord(assistantLine())?.inputTokens).toBe(3)
})
it('rejects a user record that never mentions assistant', () => {
const userLine = JSON.stringify({
type: 'user',
sessionId: 'session-1',
timestamp: '2026-01-01T00:00:00.000Z',
message: { content: 'x'.repeat(200) }
})
expect(parseClaudeUsageRecord(userLine)).toBeNull()
})
it('rejects a non-assistant record that happens to contain the word assistant', () => {
const userLine = JSON.stringify({
type: 'user',
sessionId: 'session-1',
timestamp: '2026-01-01T00:00:00.000Z',
message: { content: 'ask the assistant about this' }
})
expect(parseClaudeUsageRecord(userLine)).toBeNull()
})
})
@@ -84,10 +84,30 @@ function dedupeClaudeUsageTurns(
return deduped
}
/**
* Necessary condition for `JSON.parse(line).type === 'assistant'`, checked before the parse.
*
* Sound for any transcript written by a standard JSON serializer: `JSON.stringify` (which writes
* these files) escapes only quotes, backslashes and control characters, never ASCII letters, so
* the decoded value can only be `assistant` if the line spells it literally. The gate over-admits
* freely — the `parsed.type` check below stays authoritative.
*
* A `\u`-escape fallback was measured and rejected: it costs a second full-line scan and made
* transcripts whose tool results contain control characters 1.43x slower overall.
*/
function mayEncodeAssistantType(line: string): boolean {
return line.includes('assistant')
}
function parseClaudeUsageSourceRecord(
line: string,
fallbackSessionId: string | null = null
): ClaudeUsageParsedSourceTurn | null {
// Only assistant records carry usage, but transcripts interleave user/tool-result lines that
// routinely embed whole files. Reject those before paying for a full parse.
if (!mayEncodeAssistantType(line)) {
return null
}
let parsed: ClaudeUsageSourceRecord
try {
parsed = JSON.parse(line) as ClaudeUsageSourceRecord
@@ -0,0 +1,144 @@
// Scripted stand-in for the Claude Code CLI, driven by the SDK contract-pin
// tests. It speaks just enough stream-json to satisfy the SDK: it answers every
// inbound control_request with a success control_response, records everything it
// observes to a report file, and plays back the steps listed in a scenario file.
//
// Env contract (set by the test):
// ORCA_SDK_CONTRACT_SCENARIO_PATH — JSON file
// { steps: Step[], controlResponses?: { [subtype]: <response> } } where a Step is
// { emit: <frame> } | { awaitUserMessage: true } | { stderr: <text> } |
// { awaitControlResponse: <request_id> } | { delayMs: <n> } | { exit: <code> }
// ORCA_SDK_CONTRACT_REPORT_PATH — where argv/env observations are written
// ORCA_SDK_CONTRACT_IGNORE_SIGTERM — trap SIGTERM/SIGINT and outlive stdin close
// ORCA_SDK_CONTRACT_IGNORE_CONTROL_REQUESTS — record control requests but never answer
// ORCA_SDK_CONTRACT_DESCENDANT — fork an idle grandchild and report its pid
import { spawn } from 'node:child_process'
import { readFileSync, writeFileSync } from 'node:fs'
import { createInterface } from 'node:readline'
const scenarioPath = process.env.ORCA_SDK_CONTRACT_SCENARIO_PATH
const reportPath = process.env.ORCA_SDK_CONTRACT_REPORT_PATH
const report = {
argv: process.argv.slice(1),
execPath: process.execPath,
controlRequests: [],
controlResponses: [],
userMessages: [],
descendantPid: null
}
const writeReport = () => {
if (reportPath) {
writeFileSync(reportPath, JSON.stringify(report))
}
}
// Written immediately so a test can prove which script the SDK executed even if
// the session dies before the scenario completes.
writeReport()
const scenario = scenarioPath ? JSON.parse(readFileSync(scenarioPath, 'utf8')) : { steps: [] }
if (process.env.ORCA_SDK_CONTRACT_IGNORE_SIGTERM) {
process.on('SIGTERM', () => {})
process.on('SIGINT', () => {})
setInterval(() => {}, 1_000_000)
}
if (process.env.ORCA_SDK_CONTRACT_DESCENDANT) {
const descendant = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000000)'], {
stdio: 'ignore'
})
descendant.unref()
report.descendantPid = descendant.pid ?? null
writeReport()
}
const emit = (frame) => process.stdout.write(`${JSON.stringify(frame)}\n`)
const waiters = []
const settle = (kind, requestId) => {
for (let i = waiters.length - 1; i >= 0; i--) {
const waiter = waiters[i]
if (
waiter.kind === kind &&
(waiter.requestId === undefined || waiter.requestId === requestId)
) {
waiters.splice(i, 1)
waiter.resolve()
}
}
}
const waitFor = (kind, requestId) => {
if (kind === 'user' && report.userMessages.length > 0) {
return Promise.resolve()
}
if (
kind === 'control_response' &&
report.controlResponses.some((frame) => frame.response?.request_id === requestId)
) {
return Promise.resolve()
}
return new Promise((resolve) => waiters.push({ kind, requestId, resolve }))
}
createInterface({ input: process.stdin }).on('line', (line) => {
let frame
try {
frame = JSON.parse(line)
} catch {
return
}
if (frame.type === 'control_request') {
report.controlRequests.push(frame)
writeReport()
if (process.env.ORCA_SDK_CONTRACT_IGNORE_CONTROL_REQUESTS) {
return
}
emit({
type: 'control_response',
response: {
subtype: 'success',
request_id: frame.request_id,
response: scenario.controlResponses?.[frame.request?.subtype] ?? {
commands: [],
models: []
}
}
})
return
}
if (frame.type === 'control_response') {
report.controlResponses.push(frame)
writeReport()
settle('control_response', frame.response?.request_id)
return
}
if (frame.type === 'user') {
report.userMessages.push(frame)
writeReport()
settle('user')
}
})
// Never outlive a wedged test: the readline subscription would otherwise hold
// this process open forever if the SDK side stops driving the scenario.
setTimeout(() => process.exit(3), 20_000).unref()
for (const step of scenario.steps) {
if (step.emit) {
emit(step.emit)
} else if (step.stderr !== undefined) {
process.stderr.write(step.stderr)
} else if (step.awaitUserMessage) {
await waitFor('user')
} else if (step.awaitControlResponse !== undefined) {
await waitFor('control_response', step.awaitControlResponse)
} else if (step.delayMs) {
await new Promise((resolve) => setTimeout(resolve, step.delayMs))
} else if (step.exit !== undefined) {
// A CLI that refuses to start: leave with its own status, stderr already written.
writeReport()
process.exit(step.exit)
}
}
writeReport()
process.exit(0)
@@ -0,0 +1,519 @@
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { createRequire } from 'node:module'
import { tmpdir } from 'node:os'
import { dirname, join } from 'node:path'
import {
query,
type CanUseTool,
type Options,
type SDKUserMessage,
type SpawnedProcess as SdkSpawnedProcess,
type SpawnOptions as SdkSpawnOptions
} from '@anthropic-ai/claude-agent-sdk'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { spawnProcess } from '../../shared/child-process/run-process'
import type { AgentSessionRecord } from '../../shared/agent-session-record'
import { LOCAL_EXECUTION_HOST_ID } from '../../shared/execution-host'
import type { AgentSessionRecordStore } from '../runtime/agent-session-record-store'
import { claudeQuerySettingsReader } from './claude-agent-sdk-control-requests'
import { createClaudeStructuredLaunchResolver } from './claude-structured-launch-resolution'
// Contract pins for @anthropic-ai/claude-agent-sdk, run against the real SDK
// driving a scripted fake CLI (never the real Claude binary). These tests exist
// to catch a future SDK version drifting under Orca: unknown-frame pass-through,
// spawner env fidelity, argument parity with the pre-SDK argv,
// permission-callback semantics, and executable-path override.
const FAKE_CLI = join(__dirname, '__fixtures__', 'claude-agent-sdk-scripted-cli.mjs')
const SESSION_ID = '5348c19f-6a54-4c2e-9c68-9c2b1a3d4e5f'
const LEAF_UUID = 'ad0f7c9e-1b2c-4d3e-8f90-abc123def456'
const PINNED_SDK_VERSION = '0.3.251'
const SDK_PLATFORM_PACKAGE_BASENAMES = [
'claude-agent-sdk-darwin-arm64',
'claude-agent-sdk-darwin-x64',
'claude-agent-sdk-linux-arm64',
'claude-agent-sdk-linux-arm64-musl',
'claude-agent-sdk-linux-x64',
'claude-agent-sdk-linux-x64-musl',
'claude-agent-sdk-win32-arm64',
'claude-agent-sdk-win32-x64'
]
/**
* The exact argv the hand-rolled transport built before the SDK swap. Frozen here
* as the parity oracle: CLAUDE_STRUCTURED_BASE_OPTIONS has to keep producing it.
*/
const PRE_SDK_ARGV = [
'-p',
'--input-format',
'stream-json',
'--output-format',
'stream-json',
'--include-partial-messages',
'--verbose',
'--replay-user-messages',
'--permission-prompt-tool',
'stdio',
'--setting-sources',
'user,project,local'
]
const RESULT_FRAME = {
type: 'result',
subtype: 'success',
is_error: false,
duration_ms: 1,
duration_api_ms: 1,
num_turns: 1,
result: 'ok',
session_id: SESSION_ID,
total_cost_usd: 0,
usage: { input_tokens: 1, output_tokens: 1 },
uuid: 'uuid-result-1'
}
type ScenarioStep = Record<string, unknown>
type SpawnSeen = {
command: string
args: string[]
cwd: string | undefined
env: Record<string, string | undefined>
}
type ScriptedCliReport = {
argv: string[]
execPath: string
controlRequests: { request_id: string; request: { subtype: string } }[]
controlResponses: { response: { request_id: string; response?: Record<string, unknown> } }[]
userMessages: Record<string, unknown>[]
}
const scratchDirs: string[] = []
afterEach(() => {
vi.unstubAllEnvs()
for (const dir of scratchDirs.splice(0)) {
rmSync(dir, { recursive: true, force: true })
}
})
function scriptScenario(
steps: ScenarioStep[],
controlResponses: Record<string, unknown> = {}
): {
scenarioPath: string
reportPath: string
cwd: string
readReport: () => ScriptedCliReport
} {
const dir = mkdtempSync(join(tmpdir(), 'claude-sdk-contract-'))
scratchDirs.push(dir)
const scenarioPath = join(dir, 'scenario.json')
const reportPath = join(dir, 'report.json')
writeFileSync(scenarioPath, JSON.stringify({ steps, controlResponses }))
return {
scenarioPath,
reportPath,
cwd: dir,
readReport: () => JSON.parse(readFileSync(reportPath, 'utf8')) as ScriptedCliReport
}
}
function scenarioEnv(scenario: { scenarioPath: string; reportPath: string }) {
return {
PATH: process.env.PATH,
ORCA_SDK_CONTRACT_SCENARIO_PATH: scenario.scenarioPath,
ORCA_SDK_CONTRACT_REPORT_PATH: scenario.reportPath
}
}
function recordingSpawner(spawns: SpawnSeen[]) {
return (opts: SdkSpawnOptions): SdkSpawnedProcess => {
spawns.push({
command: opts.command,
args: [...opts.args],
cwd: opts.cwd,
env: { ...opts.env }
})
return spawnProcess({
program: opts.command,
args: opts.args,
cwd: opts.cwd,
env: opts.env as NodeJS.ProcessEnv,
signal: opts.signal
}) as unknown as SdkSpawnedProcess
}
}
function resolvedLaunch(launchArgs: string[]) {
const record = {
sessionId: 'contract-pin-session',
provider: 'claude',
location: {
executionHostId: LOCAL_EXECUTION_HOST_ID,
wslDistro: null,
workspaceId: 'workspace-1',
workspaceKind: 'folder'
},
accountHome: { variable: 'CLAUDE_CONFIG_DIR', path: '/home/work/.claude' },
providerHandleChain: [],
launchArgs
} as unknown as AgentSessionRecord
return createClaudeStructuredLaunchResolver({
store: { getRecord: () => record } as unknown as AgentSessionRecordStore,
resolveWorkspacePath: async () => '/repos/workspace-1',
resolveCommand: () => FAKE_CLI,
resolveAuthPolicy: () => ({ stripAuthEnv: true })
})({ identity: { sessionId: record.sessionId } as never })
}
function singleUserTurn(): AsyncIterable<SDKUserMessage> {
return (async function* () {
yield {
type: 'user',
message: { role: 'user', content: [{ type: 'text', text: 'hello' }] },
parent_tool_use_id: null,
session_id: SESSION_ID
} as SDKUserMessage
// Hold input open; the stream ends when the scripted CLI exits, and an
// unresolved bare promise does not keep the event loop alive.
await new Promise<void>(() => {})
})()
}
async function drainQuery(options: Options): Promise<Record<string, unknown>[]> {
const messages: Record<string, unknown>[] = []
for await (const message of query({ prompt: singleUserTurn(), options })) {
messages.push(message as unknown as Record<string, unknown>)
}
return messages
}
/** Expand `--flag=value` argv entries so both SDK spellings compare equal. */
function normalizeArgv(args: string[]): string[] {
return args.flatMap((arg) => {
if (!arg.startsWith('--')) {
return [arg]
}
const eq = arg.indexOf('=')
return eq === -1 ? [arg] : [arg.slice(0, eq), arg.slice(eq + 1)]
})
}
/** Group the pre-SDK argv into flag/value pairs. */
function flagTable(args: readonly string[]): { flag: string; value: string | null }[] {
const table: { flag: string; value: string | null }[] = []
for (let i = 0; i < args.length; i++) {
const flag = args[i]!
const next = args[i + 1]
if (next !== undefined && !next.startsWith('-')) {
table.push({ flag, value: next })
i++
} else {
table.push({ flag, value: null })
}
}
return table
}
describe('Claude Agent SDK contract pins', () => {
it('yields unknown types, unknown fields and unknown content blocks verbatim, and consumes keep_alive', async () => {
const unknownTopLevel = {
type: 'message_kind_from_the_future',
session_id: SESSION_ID,
uuid: 'uuid-unknown-1',
payload: { alpha: 1, nested: { flags: ['a', 'b'] } }
}
const assistantWithUnknowns = {
type: 'assistant',
message: {
id: 'msg-1',
type: 'message',
role: 'assistant',
model: 'claude-x',
content: [
{ type: 'text', text: 'hello back' },
{ type: 'content_block_from_the_future', payload: { depth: 3 } }
],
stop_reason: null,
stop_sequence: null,
usage: { input_tokens: 1, output_tokens: 2 }
},
parent_tool_use_id: null,
uuid: 'uuid-assistant-1',
session_id: SESSION_ID,
field_from_the_future: 'preserved'
}
const scenario = scriptScenario([
{ awaitUserMessage: true },
{ emit: { type: 'keep_alive' } },
{ emit: unknownTopLevel },
{ emit: assistantWithUnknowns },
{ emit: RESULT_FRAME }
])
const spawns: SpawnSeen[] = []
const messages = await drainQuery({
pathToClaudeCodeExecutable: FAKE_CLI,
cwd: scenario.cwd,
env: scenarioEnv(scenario),
spawnClaudeCodeProcess: recordingSpawner(spawns)
})
expect(messages.find((m) => m.uuid === 'uuid-unknown-1')).toEqual(unknownTopLevel)
expect(messages.find((m) => m.uuid === 'uuid-assistant-1')).toEqual(assistantWithUnknowns)
// The SDK intercepts keep_alive internally — a liveness signal must never
// be derived from it reaching the consumer, because it does not.
expect(messages.some((m) => m.type === 'keep_alive')).toBe(false)
expect(messages.some((m) => m.type === 'result')).toBe(true)
})
it('hands the custom spawner exactly the caller-supplied env, plus the two pinned SDK mutations', async () => {
vi.stubEnv('ANTHROPIC_API_KEY', 'ambient-key-must-not-leak')
const scenario = scriptScenario([{ awaitUserMessage: true }, { emit: RESULT_FRAME }])
const spawns: SpawnSeen[] = []
await drainQuery({
pathToClaudeCodeExecutable: FAKE_CLI,
cwd: scenario.cwd,
env: {
...scenarioEnv(scenario),
CLAUDE_CONFIG_DIR: '/pinned/claude-config',
ORCA_AGENT_SESSION_SPAWN_TOKEN: 'spawn-token-1',
NODE_OPTIONS: '--max-old-space-size=64'
},
spawnClaudeCodeProcess: recordingSpawner(spawns)
})
const env = spawns[0]!.env
// Supplied values arrive verbatim: the config-dir pin and spawn token are
// observable at this boundary, so Orca's auth scrubbing stays assertable.
expect(env.CLAUDE_CONFIG_DIR).toBe('/pinned/claude-config')
expect(env.ORCA_AGENT_SESSION_SPAWN_TOKEN).toBe('spawn-token-1')
// Ambient process.env is NOT merged in when env is supplied.
expect(env.ANTHROPIC_API_KEY).toBeUndefined()
// The SDK's two documented mutations, pinned so a change is noticed.
expect(env.CLAUDE_CODE_ENTRYPOINT).toBe('sdk-ts')
expect('NODE_OPTIONS' in env).toBe(false)
})
it('inherits process.env into the child when env is omitted — the ambient-auth sharp edge', async () => {
const scenario = scriptScenario([{ awaitUserMessage: true }, { emit: RESULT_FRAME }])
vi.stubEnv('ORCA_SDK_CONTRACT_SCENARIO_PATH', scenario.scenarioPath)
vi.stubEnv('ORCA_SDK_CONTRACT_REPORT_PATH', scenario.reportPath)
vi.stubEnv('ORCA_SDK_CONTRACT_AMBIENT_CANARY', 'inherited-from-process-env')
const spawns: SpawnSeen[] = []
await drainQuery({
pathToClaudeCodeExecutable: FAKE_CLI,
cwd: scenario.cwd,
spawnClaudeCodeProcess: recordingSpawner(spawns)
})
// Omitting env reproduces the ambient-auth-leak failure mode: the child
// sees everything in process.env. Orca must therefore always pass an
// explicit, fully-constructed env.
expect(spawns[0]!.env.ORCA_SDK_CONTRACT_AMBIENT_CANARY).toBe('inherited-from-process-env')
})
it('emits --replay-user-messages only through extraArgs, never on its own', async () => {
const scenario = scriptScenario([{ awaitUserMessage: true }, { emit: RESULT_FRAME }])
const bareSpawns: SpawnSeen[] = []
await drainQuery({
pathToClaudeCodeExecutable: FAKE_CLI,
cwd: scenario.cwd,
env: scenarioEnv(scenario),
spawnClaudeCodeProcess: recordingSpawner(bareSpawns)
})
expect(bareSpawns[0]!.args).not.toContain('--replay-user-messages')
const replayScenario = scriptScenario([{ awaitUserMessage: true }, { emit: RESULT_FRAME }])
const replaySpawns: SpawnSeen[] = []
await drainQuery({
pathToClaudeCodeExecutable: FAKE_CLI,
cwd: replayScenario.cwd,
env: scenarioEnv(replayScenario),
extraArgs: { 'replay-user-messages': null },
spawnClaudeCodeProcess: recordingSpawner(replaySpawns)
})
const replayArgs = replaySpawns[0]!.args
expect(replayArgs.filter((arg) => arg === '--replay-user-messages')).toHaveLength(1)
})
it('produces a matching CLI flag for every pre-SDK argv entry', async () => {
const scenario = scriptScenario([{ awaitUserMessage: true }, { emit: RESULT_FRAME }])
const spawns: SpawnSeen[] = []
// Driven by the real resolver, so the argv walk covers the durable-launchArgs
// translation and its merge order, not a hand-written options literal.
const launch = await resolvedLaunch(['--model', 'claude-sonnet-4-5', '--effort', 'high'])
await drainQuery({
...launch.options,
pathToClaudeCodeExecutable: FAKE_CLI,
cwd: scenario.cwd,
env: scenarioEnv(scenario),
canUseTool: (async () => ({ behavior: 'deny', message: 'unused' })) as CanUseTool,
spawnClaudeCodeProcess: recordingSpawner(spawns)
})
expect(spawns).toHaveLength(1)
const argv = normalizeArgv(spawns[0]!.args)
// Typed-first translation must not also spell the flag through extraArgs.
for (const flag of ['--model', '--effort']) {
expect(
argv.filter((arg) => arg === flag),
`${flag} occurrences`
).toHaveLength(1)
}
expect(argv[argv.indexOf('--model') + 1]).toBe('claude-sonnet-4-5')
expect(argv[argv.indexOf('--effort') + 1]).toBe('high')
// Headless print mode is the SDK's only mode; `query()` never passes `-p`,
// and if the SDK ever started passing it this pin would notice.
const impliedByHeadlessQuery = new Set(['-p'])
for (const entry of flagTable(PRE_SDK_ARGV)) {
if (impliedByHeadlessQuery.has(entry.flag)) {
expect(argv, `${entry.flag} is implied, never spelled`).not.toContain(entry.flag)
continue
}
const at = argv.indexOf(entry.flag)
expect(at, `SDK argv is missing ${entry.flag}`).toBeGreaterThanOrEqual(0)
if (entry.value !== null) {
expect(argv[at + 1], `value of ${entry.flag}`).toBe(entry.value)
}
}
// The launch resolver always carries one of --session-id / --resume.
const sessionAt = argv.indexOf('--session-id')
expect(sessionAt).toBeGreaterThanOrEqual(0)
expect(argv[sessionAt + 1]).toBe(launch.providerSessionId)
})
it('still exposes the runtime get_settings reader the auth diagnostic depends on', async () => {
// 0.3.251 ships getSettings() but redacts it from the Query declaration. This pin
// is the drift alarm: if a bump drops or reshapes it, the diagnostic degrades and
// this test says so instead of the degradation shipping silently.
const settings = { env: { ANTHROPIC_BASE_URL: 'https://settings.example.test' } }
const scenario = scriptScenario([{ delayMs: 3_000 }], { get_settings: settings })
const session = query({
prompt: singleUserTurn(),
options: {
pathToClaudeCodeExecutable: FAKE_CLI,
cwd: scenario.cwd,
env: scenarioEnv(scenario)
}
})
try {
const read = claudeQuerySettingsReader(session)
expect(read, 'the SDK no longer exposes get_settings at runtime').not.toBeNull()
await expect(read?.()).resolves.toEqual(settings)
} finally {
await session.return(undefined)
}
})
it('maps resume identity to --resume and --resume-session-at', async () => {
const scenario = scriptScenario([{ awaitUserMessage: true }, { emit: RESULT_FRAME }])
const spawns: SpawnSeen[] = []
await drainQuery({
pathToClaudeCodeExecutable: FAKE_CLI,
cwd: scenario.cwd,
env: scenarioEnv(scenario),
resume: SESSION_ID,
resumeSessionAt: LEAF_UUID,
spawnClaudeCodeProcess: recordingSpawner(spawns)
})
const argv = normalizeArgv(spawns[0]!.args)
const resumeAt = argv.indexOf('--resume')
expect(resumeAt).toBeGreaterThanOrEqual(0)
expect(argv[resumeAt + 1]).toBe(SESSION_ID)
const leafAt = argv.indexOf('--resume-session-at')
expect(leafAt).toBeGreaterThanOrEqual(0)
expect(argv[leafAt + 1]).toBe(LEAF_UUID)
})
it('gives canUseTool the wire request_id and fires its abort signal on control_cancel_request', async () => {
const scenario = scriptScenario([
{ awaitUserMessage: true },
{
emit: {
type: 'control_request',
request_id: 'perm-421',
request: {
subtype: 'can_use_tool',
tool_name: 'Bash',
input: { command: 'echo hi' },
tool_use_id: 'tool-use-9'
}
}
},
{ delayMs: 120 },
{ emit: { type: 'control_cancel_request', request_id: 'perm-421' } },
{ awaitControlResponse: 'perm-421' },
{ emit: RESULT_FRAME }
])
const seen: { toolName: string; requestId: string; toolUseID: string }[] = []
let abortFired = false
const canUseTool: CanUseTool = (toolName, _input, { signal, requestId, toolUseID }) => {
seen.push({ toolName, requestId, toolUseID })
return new Promise((resolve) => {
signal.addEventListener('abort', () => {
abortFired = true
resolve({ behavior: 'deny', message: 'cancelled by test' })
})
})
}
const spawns: SpawnSeen[] = []
await drainQuery({
pathToClaudeCodeExecutable: FAKE_CLI,
cwd: scenario.cwd,
env: scenarioEnv(scenario),
canUseTool,
spawnClaudeCodeProcess: recordingSpawner(spawns)
})
expect(seen).toEqual([{ toolName: 'Bash', requestId: 'perm-421', toolUseID: 'tool-use-9' }])
expect(abortFired).toBe(true)
// The callback's settlement is written back onto the wire against the same id.
const settled = scenario
.readReport()
.controlResponses.find((frame) => frame.response.request_id === 'perm-421')
expect(settled?.response.response?.behavior).toBe('deny')
// Exactly one process spawn per query, control traffic included.
expect(spawns).toHaveLength(1)
})
it('runs the executable given via pathToClaudeCodeExecutable under the default spawner', async () => {
const scenario = scriptScenario([{ awaitUserMessage: true }, { emit: RESULT_FRAME }])
const messages = await drainQuery({
pathToClaudeCodeExecutable: FAKE_CLI,
cwd: scenario.cwd,
env: scenarioEnv(scenario)
})
expect(messages.some((m) => m.type === 'result')).toBe(true)
const report = scenario.readReport()
// The SDK executed exactly the script we pointed it at — no bundled binary.
expect(report.argv[0]).toBe(FAKE_CLI)
expect(report.execPath).toContain('node')
// And the streaming handshake went to it: the SDK sent its initialize
// control request to our script.
expect(report.controlRequests.some((frame) => frame.request.subtype === 'initialize')).toBe(
true
)
})
it('pins the SDK version the contract was verified against', () => {
const sdkEntry = createRequire(__filename).resolve('@anthropic-ai/claude-agent-sdk')
const manifest = JSON.parse(readFileSync(join(dirname(sdkEntry), 'package.json'), 'utf8')) as {
version: string
}
expect(manifest.version).toBe(PINNED_SDK_VERSION)
})
it('keeps the eight bundled CLI platform binaries out of the install', () => {
const sdkEntry = createRequire(__filename).resolve('@anthropic-ai/claude-agent-sdk')
// The SDK's own scoped directory is where pnpm would link its optional
// platform packages; ignoredOptionalDependencies must keep them all absent.
const scopeDir = dirname(dirname(sdkEntry))
for (const basename of SDK_PLATFORM_PACKAGE_BASENAMES) {
expect(
existsSync(join(scopeDir, basename, 'package.json')),
`${basename} must not be installed`
).toBe(false)
}
})
})
@@ -0,0 +1,154 @@
import type {
PermissionMode,
Query,
SDKControlInterruptResponse
} from '@anthropic-ai/claude-agent-sdk'
export class ClaudeControlRequestError extends Error {
constructor(
readonly subtype: string,
message: string
) {
super(message)
this.name = 'ClaudeControlRequestError'
}
}
export const CLAUDE_DEFAULT_REQUEST_TIMEOUT_MS = 30_000
/** The SDK closes a query out from under an in-flight control request with this exact message. */
const QUERY_CLOSED_MESSAGE = 'Query closed before response received'
/** 0.3.251 ships getSettings() but redacts it from the Query declaration; the typeof guard below is its degradation path. */
type ClaudeQuerySettingsReader = { getSettings?: () => Promise<unknown> }
export function claudeQuerySettingsReader(query: Query): (() => Promise<unknown>) | null {
const reader = (query as unknown as ClaudeQuerySettingsReader).getSettings
return typeof reader === 'function' ? reader.bind(query) : null
}
/**
* cancel_async_message is a runtime Query method the shipped 0.3.251 declaration omits;
* it withdraws a single still-queued async user message by uuid so an interrupted turn
* cannot spawn a later unexpected turn. The typeof guard is its degradation path.
*/
type ClaudeQueryAsyncCanceller = { cancelAsyncMessage?: (uuid: string) => Promise<unknown> }
export function claudeQueryAsyncCanceller(
query: Query
): ((uuid: string) => Promise<unknown>) | null {
const cancel = (query as unknown as ClaudeQueryAsyncCanceller).cancelAsyncMessage
return typeof cancel === 'function' ? cancel.bind(query) : null
}
export type ClaudeControlOptions = { timeoutMs?: number }
/**
* Run one native Query control method under Orca's deadline and error classification.
*
* The SDK owns correlation but applies no deadline, so the timeout stays here — and its
* message is load-bearing: the init proof matches on `claude initialize request timed out`.
* A closed query is a transport failure, not the CLI rejecting the request, so only the
* latter is re-thrown as a `ClaudeControlRequestError` a caller may surface as a rejection.
*/
export function runClaudeControl<T>(
subtype: string,
run: () => Promise<T>,
timeoutMs: number = CLAUDE_DEFAULT_REQUEST_TIMEOUT_MS
): Promise<T> {
let timer: ReturnType<typeof setTimeout> | null = null
const deadline = new Promise<never>((_resolve, reject) => {
timer = setTimeout(() => reject(new Error(`claude ${subtype} request timed out`)), timeoutMs)
timer.unref?.()
})
return Promise.race([
Promise.resolve()
.then(run)
.catch((error: unknown) => {
const message = error instanceof Error ? error.message : String(error)
if (error instanceof ClaudeControlRequestError || message === QUERY_CLOSED_MESSAGE) {
throw error
}
throw new ClaudeControlRequestError(subtype, message)
}),
deadline
]).finally(() => {
if (timer) {
clearTimeout(timer)
}
})
}
/** The native control surface Orca drives, one method per Query control request. */
export type ClaudeControlSurface = {
interrupt: (
options?: ClaudeControlOptions & { cancelQueued?: boolean }
) => Promise<SDKControlInterruptResponse | undefined>
cancelAsyncMessage: (uuid: string, options?: ClaudeControlOptions) => Promise<void>
setModel: (model: string | undefined, options?: ClaudeControlOptions) => Promise<void>
setPermissionMode: (mode: PermissionMode, options?: ClaudeControlOptions) => Promise<void>
applyFlagSettings: (
settings: Parameters<Query['applyFlagSettings']>[0],
options?: ClaudeControlOptions
) => Promise<void>
supportedModels: (options?: ClaudeControlOptions) => Promise<unknown[]>
initializationResult: (options?: ClaudeControlOptions) => Promise<unknown>
getSettings: (options?: ClaudeControlOptions) => Promise<unknown>
}
type InterruptingQuery = {
interrupt: (options?: {
cancelQueued?: boolean
}) => Promise<SDKControlInterruptResponse | undefined>
}
export function createClaudeControlSurface(query: Query): ClaudeControlSurface {
return {
interrupt: (options) =>
runClaudeControl(
'interrupt',
() =>
(query as unknown as InterruptingQuery).interrupt(
options?.cancelQueued ? { cancelQueued: true } : undefined
),
options?.timeoutMs
),
cancelAsyncMessage: (uuid, options) => {
const cancel = claudeQueryAsyncCanceller(query)
return cancel
? runClaudeControl('cancel_async_message', () => cancel(uuid), options?.timeoutMs).then(
() => {}
)
: Promise.resolve()
},
setModel: (model, options) =>
runClaudeControl('set_model', () => query.setModel(model), options?.timeoutMs).then(() => {}),
setPermissionMode: (mode, options) =>
runClaudeControl(
'set_permission_mode',
() => query.setPermissionMode(mode),
options?.timeoutMs
).then(() => {}),
applyFlagSettings: (settings, options) =>
runClaudeControl(
'apply_flag_settings',
() => query.applyFlagSettings(settings),
options?.timeoutMs
).then(() => {}),
supportedModels: (options) =>
runClaudeControl('list_models', () => query.supportedModels(), options?.timeoutMs),
initializationResult: (options) =>
runClaudeControl('initialize', () => query.initializationResult(), options?.timeoutMs),
getSettings: (options) => {
const read = claudeQuerySettingsReader(query)
return read
? runClaudeControl('get_settings', read, options?.timeoutMs)
: Promise.reject(
new ClaudeControlRequestError(
'get_settings',
'this SDK exposes no get_settings request'
)
)
}
}
}
@@ -0,0 +1,104 @@
import { describe, expect, it, vi } from 'vitest'
import type { DescendantSnapshot } from '../pty-descendant-termination'
import type { WindowsDescendantSnapshot } from '../windows-descendant-exit-verification'
import { collectDescendantRows } from '../pty-descendant-termination'
import { createClaudeChildTreeReaper } from './claude-agent-sdk-exit-proof'
import { mergeClaudeCapturedTrees } from './claude-child-tree-snapshot'
function posixSnapshot(capturedAtMs: number): DescendantSnapshot {
return {
root: { pid: 100, startedAt: 'Mon Jan 1 00:00:00 2026' },
rootPgid: 100,
descendants: [{ pid: 200, ppid: 100, pgid: 100, startedAt: 'Mon Jan 1 00:00:01 2026' }],
capturedAtMs
}
}
function windowsSnapshot(): WindowsDescendantSnapshot {
return {
root: { pid: 100, creationTimeMs: 5 },
descendants: [{ pid: 200, creationTimeMs: 7 }],
unidentifiedCount: 0,
capturedAtMs: 1
}
}
describe('Claude child root identity', () => {
it('keeps a retained row boundary when a refresh observes no new descendants', () => {
const previous = posixSnapshot(1_700_000_000_900)
const next = posixSnapshot(1_700_000_002_100)
expect(
mergeClaudeCapturedTrees(
{ platform: 'posix', tree: previous },
{ platform: 'posix', tree: next }
)
).toEqual({
platform: 'posix',
tree: { ...next, capturedAtMsByPid: { '200': previous.capturedAtMs } }
})
})
it('keeps the descendant verdict when a POSIX root probe is unavailable', async () => {
const child = { pid: 100, kill: vi.fn(() => true) }
const terminateDescendants = vi.fn(async () => 'exited' as const)
const tree = createClaudeChildTreeReaper(child, {
platform: 'linux',
captureDescendants: vi.fn(async () => posixSnapshot(1)),
terminateDescendants,
verifyRootIdentity: vi.fn(async () => false)
})
// POSIX runs no bare-pid root operation, so a declined probe withholds
// nothing: the handle kill still lands and the verification still speaks.
await expect(tree.reap()).resolves.toBe('exited')
expect(terminateDescendants).toHaveBeenCalled()
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
})
it('rejects mixed old and recycled root rows instead of making the tree killable', async () => {
const child = { pid: 100, kill: vi.fn(() => true) }
const terminateDescendants = vi.fn(async () => 'exited' as const)
const tree = createClaudeChildTreeReaper(child, {
platform: 'linux',
captureDescendants: vi.fn(async () =>
collectDescendantRows(
100,
[
{ pid: 100, ppid: 1, pgid: 100, startedAt: 'Mon Jan 1 00:00:00 2026' },
{ pid: 100, ppid: 1, pgid: 101, startedAt: 'Mon Jan 1 00:00:01 2026' },
{ pid: 200, ppid: 100, pgid: 200, startedAt: 'Mon Jan 1 00:00:00 2026' }
],
1
)
),
terminateDescendants,
verifyRootIdentity: vi.fn(async () => true)
})
await expect(tree.reap()).resolves.toBe('unverifiable')
// No admissible snapshot means no row may be signalled from its number, but
// the root still leaves through the handle Node owns.
expect(terminateDescendants).not.toHaveBeenCalled()
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
})
it('fails closed when Windows root identity revalidation is unavailable', async () => {
const child = { pid: 100, kill: vi.fn(() => true) }
const terminateWindowsTree = vi.fn(async () => {})
const terminateWindowsDescendants = vi.fn(async () => 'exited' as const)
const tree = createClaudeChildTreeReaper(child, {
platform: 'win32',
captureWindowsDescendants: vi.fn(async () => windowsSnapshot()),
terminateWindowsTree,
terminateWindowsDescendants,
verifyRootIdentity: vi.fn(async () => false)
})
await expect(tree.reap()).resolves.toBe('unverifiable')
// taskkill /T /F addresses a bare pid and stays gated; the handle does not.
expect(terminateWindowsTree).not.toHaveBeenCalled()
expect(terminateWindowsDescendants).not.toHaveBeenCalled()
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
})
})
@@ -0,0 +1,934 @@
import { execFileSync } from 'node:child_process'
import { EventEmitter } from 'node:events'
import { PassThrough } from 'node:stream'
import { describe, expect, it, vi } from 'vitest'
import { spawnProcess, type SpawnedProcess } from '../../shared/child-process/run-process'
import type { DescendantTreeVerdict } from '../pty-descendant-exit-verification'
import type { DescendantSnapshot } from '../pty-descendant-termination'
import type { WindowsDescendantSnapshot } from '../windows-descendant-exit-verification'
import {
createClaudeChildTreeReaper as createClaudeChildTreeReaperImpl,
proveClaudeChildExit,
type ClaudeChildTreeReaper
} from './claude-agent-sdk-exit-proof'
// The descendant models an MCP server: it either cooperates or, when it traps
// SIGTERM, only a forced, verified sweep can reach it. The root either traps
// SIGTERM too, or leaves promptly on stdin end the way a healthy CLI does —
// which is the path that used to skip descendant proof entirely.
function childWithDescendantScript(input: {
rootTrapsSigterm: boolean
descendantTrapsSigterm: boolean
}): string {
const descendantScript = `${input.descendantTrapsSigterm ? 'process.on("SIGTERM", () => {}); ' : ''}setInterval(() => {}, 1000000)`
const rootBehaviour = input.rootTrapsSigterm
? `process.on('SIGTERM', () => {})
process.on('SIGINT', () => {})
setInterval(() => {}, 1000000)`
: `process.stdin.on('end', () => process.exit(0))
process.stdin.resume()`
return `
const descendant = require('node:child_process').spawn(
process.execPath,
['-e', ${JSON.stringify(descendantScript)}],
{ stdio: 'ignore' }
)
descendant.unref()
process.stdout.write(JSON.stringify({ descendantPid: descendant.pid }) + '\\n')
${rootBehaviour}
`
}
const COOPERATIVE_CHILD = `
process.stdin.on('end', () => process.exit(0))
process.stdin.resume()
process.stdout.write('ready\\n')
`
/**
* Sampled synchronously so it reads the exact moment the close boundary is
* crossed. A zombie has exited (its parent just has not reaped it yet), so a
* kill(pid, 0) probe would misreport it as running.
*/
function descendantState(pid: number): 'running' | 'exited' {
let state: string
try {
state = execFileSync('ps', ['-o', 'state=', '-p', String(pid)], {
encoding: 'utf8',
env: { ...process.env, LANG: 'C', LC_ALL: 'C' }
}).trim()
} catch (error) {
// ps exits 1 when no process matches; anything else is a failed probe, not an answer.
if ((error as { status?: number }).status !== 1) {
throw error
}
return 'exited'
}
return state.startsWith('Z') ? 'exited' : 'running'
}
/**
* ps lstart is second-resolution, so the identity-safe sweep only SIGKILLs a row
* born strictly before the second the snapshot was captured in. The snapshot is
* armed the moment close begins, so a descendant born in that same second can
* only be asked, never forced — the same bound an MCP server spawned within a
* second of the user closing the chat would hit.
*/
function ageDescendantPastTheCaptureSecond(): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, 1_000 - (Date.now() % 1_000) + 20))
}
/**
* The close ladder as production drives it: `closeProcessRegistry` retries an
* unproven close, and each retry re-verifies the retained snapshot. A loaded
* host can spend one attempt's whole window inside `ps`, and reporting false
* there is the honest verdict — the requirement is that TRUE never outruns the
* observation, which the caller asserts at whichever boundary returns it.
*/
async function proveExitWithRetries(
input: Parameters<typeof proveClaudeChildExit>[0],
attempts = 3
): Promise<boolean> {
for (let attempt = 1; attempt < attempts; attempt += 1) {
if (await proveClaudeChildExit(input)) {
return true
}
}
return proveClaudeChildExit(input)
}
function spawnScript(script: string): ReturnType<typeof spawnProcess> {
return spawnProcess({
program: process.execPath,
args: ['-e', script],
stdio: ['pipe', 'pipe', 'pipe']
})
}
function firstStdoutLine(child: ReturnType<typeof spawnProcess>): Promise<string> {
return new Promise((resolve) => {
child.stdout.setEncoding('utf8').once('data', (chunk: string) => resolve(chunk.trim()))
})
}
function observeExit(child: EventEmitter): { exitPromise: Promise<void>; exited: () => boolean } {
let exited = false
const exitPromise = new Promise<void>((resolve) => {
child.once('exit', () => {
exited = true
resolve()
})
})
return { exitPromise, exited: () => exited }
}
/** `null` models a spawn that failed before a pid existed. */
function mockChild(
pid: number | null = 424242
): EventEmitter &
Pick<SpawnedProcess, 'pid' | 'kill' | 'stdin'> & { kill: ReturnType<typeof vi.fn> } {
const child = new EventEmitter()
return Object.assign(child, {
pid: pid ?? undefined,
stdin: new PassThrough(),
kill: vi.fn(() => true)
}) as never
}
/** A tree whose verdict is scripted per reap, recording when it was armed. */
function mockTree(verdicts: DescendantTreeVerdict[]): ClaudeChildTreeReaper & {
capture: ReturnType<typeof vi.fn>
reap: ReturnType<typeof vi.fn>
} {
let treeVerdict: DescendantTreeVerdict = 'unverifiable'
return {
capture: vi.fn(async () => {}),
reap: vi.fn(async () => {
treeVerdict = verdicts.shift() ?? treeVerdict
return treeVerdict
}),
get treeVerdict() {
return treeVerdict
}
}
}
function windowsSnapshotOf(descendantPid: number): WindowsDescendantSnapshot {
return {
root: { pid: 424242, creationTimeMs: 1_700_000_000_001 },
descendants: [{ pid: descendantPid, creationTimeMs: 1_700_000_000_000 }],
unidentifiedCount: 0,
capturedAtMs: 1
}
}
function snapshotOf(descendantPid: number): DescendantSnapshot {
return {
root: { pid: 424242, startedAt: 'Mon Jan 1 00:00:00 2026' },
rootPgid: 1,
descendants: [
{ pid: descendantPid, ppid: 424242, pgid: 1, startedAt: 'Mon Jan 1 00:00:00 2026' }
],
capturedAtMs: 1
}
}
// Unit tests use synthetic process ids; production always supplies the fresh
// identity probe, so the harness explicitly models a matching probe.
function createClaudeChildTreeReaper(
child: Parameters<typeof createClaudeChildTreeReaperImpl>[0],
deps: Parameters<typeof createClaudeChildTreeReaperImpl>[1] = {}
): ReturnType<typeof createClaudeChildTreeReaperImpl> {
return createClaudeChildTreeReaperImpl(child, {
verifyRootIdentity: async () => true,
...deps
})
}
describe('claude child exit proof', () => {
it.runIf(process.platform !== 'win32')(
'reports a proven exit only once a SIGTERM-resistant descendant is gone at the close boundary',
async () => {
const child = spawnScript(
childWithDescendantScript({ rootTrapsSigterm: true, descendantTrapsSigterm: true })
)
const { descendantPid } = JSON.parse(await firstStdoutLine(child)) as {
descendantPid: number
}
expect(descendantState(descendantPid)).toBe('running')
await ageDescendantPastTheCaptureSecond()
try {
const proven = await proveExitWithRetries({ child, ...observeExit(child) })
// Evaluated AT the boundary, not by polling until a deferred sweep timer
// wins: true releases the lease, so a descendant still running here is
// exactly the orphan the proof exists to prevent. False would be the
// honest verdict for a tree that outlived the bounded ladder.
expect({ proven, descendant: descendantState(descendantPid) }).toEqual({
proven: true,
descendant: 'exited'
})
} finally {
// Failure-safe only: the assertion above owns the requirement, this just
// stops a failing run from leaking a process.
try {
process.kill(descendantPid, 'SIGKILL')
} catch {
// Already gone.
}
}
},
20_000
)
it.runIf(process.platform !== 'win32')(
'proves a promptly exiting root only once its stubborn descendant is gone too',
async () => {
// The ordinary healthy close: the root leaves on stdin end within the graceful
// window. Its descendant must still be proven gone, not assumed gone with it.
const child = spawnScript(
childWithDescendantScript({ rootTrapsSigterm: false, descendantTrapsSigterm: true })
)
const { descendantPid } = JSON.parse(await firstStdoutLine(child)) as {
descendantPid: number
}
expect(descendantState(descendantPid)).toBe('running')
await ageDescendantPastTheCaptureSecond()
try {
const proven = await proveExitWithRetries({ child, ...observeExit(child) })
expect({ proven, descendant: descendantState(descendantPid) }).toEqual({
proven: true,
descendant: 'exited'
})
} finally {
try {
process.kill(descendantPid, 'SIGKILL')
} catch {
// Already gone.
}
}
},
20_000
)
it.runIf(process.platform !== 'win32')(
'still proves a stubborn child whose descendant honours SIGTERM',
async () => {
const child = spawnScript(
childWithDescendantScript({ rootTrapsSigterm: true, descendantTrapsSigterm: false })
)
const { descendantPid } = JSON.parse(await firstStdoutLine(child)) as {
descendantPid: number
}
try {
const proven = await proveExitWithRetries({ child, ...observeExit(child) })
expect({ proven, descendant: descendantState(descendantPid) }).toEqual({
proven: true,
descendant: 'exited'
})
} finally {
try {
process.kill(descendantPid, 'SIGKILL')
} catch {
// Already gone.
}
}
},
20_000
)
it('arms the snapshot before stdin closes and verifies it after a clean exit', async () => {
const child = spawnScript(COOPERATIVE_CHILD)
expect(await firstStdoutLine(child)).toBe('ready')
const exit = observeExit(child)
const tree = mockTree(['exited'])
let exitedWhenArmed: boolean | null = null
tree.capture.mockImplementation(async () => {
exitedWhenArmed = exit.exited()
})
await expect(proveClaudeChildExit({ child, ...exit, tree })).resolves.toBe(true)
// The snapshot is the only proof that survives the root: taken while it lived,
// verified once it left. A reap before the exit would have been the forced ladder.
expect(exitedWhenArmed).toBe(false)
expect(tree.reap).toHaveBeenCalledTimes(1)
expect(exit.exited()).toBe(true)
}, 20_000)
it('proves a clean close of a childless root with one snapshot and no signal', async () => {
const child = spawnScript(COOPERATIVE_CHILD)
expect(await firstStdoutLine(child)).toBe('ready')
await expect(proveClaudeChildExit({ child, ...observeExit(child) })).resolves.toBe(true)
}, 20_000)
it('reports an unprovable exit as false rather than assuming the child died', async () => {
const child = mockChild()
const tree = mockTree(['exited'])
await expect(
proveClaudeChildExit({
child,
exitPromise: new Promise<void>(() => {}),
exited: () => false,
tree
})
).resolves.toBe(false)
expect(tree.reap).toHaveBeenCalledTimes(1)
}, 20_000)
it('reports false when the root exit was observed but a descendant was seen alive', async () => {
const child = mockChild()
const exit = observeExit(child)
const tree = mockTree(['live'])
tree.reap.mockImplementation(async () => {
child.emit('exit', null, 'SIGKILL')
return 'live'
})
await expect(proveClaudeChildExit({ child, ...exit, tree })).resolves.toBe(false)
expect(exit.exited()).toBe(true)
// One verification per attempt: the retried close re-verifies, this one does not.
expect(tree.reap).toHaveBeenCalledTimes(1)
}, 20_000)
it('re-verifies an unproven tree on a retried close instead of trusting the dead root', async () => {
const child = mockChild()
const tree = mockTree(['exited'])
await expect(
proveClaudeChildExit({ child, exitPromise: Promise.resolve(), exited: () => true, tree })
).resolves.toBe(true)
expect(tree.reap).toHaveBeenCalledTimes(1)
})
it('stays unproven for a root that left before any snapshot could be armed', async () => {
const child = mockChild()
const captureDescendants = vi.fn(async () => snapshotOf(4243))
const terminateDescendants = vi.fn()
const tree = createClaudeChildTreeReaper(child, {
platform: 'darwin',
exited: () => true,
captureDescendants,
terminateDescendants
})
await expect(
proveClaudeChildExit({ child, exitPromise: Promise.resolve(), exited: () => true, tree })
).resolves.toBe(false)
// A dead root's descendants have reparented: walking its pid now could only
// sweep a stranger, so no walk is attempted and nothing is proven.
expect(captureDescendants).not.toHaveBeenCalled()
expect(terminateDescendants).not.toHaveBeenCalled()
expect(tree.treeVerdict).toBe('unverifiable')
})
})
describe('claude child tree reaper', () => {
it('kills the root while verification runs and never stops it first', async () => {
const child = mockChild()
const release = Promise.withResolvers<DescendantTreeVerdict>()
const terminateDescendants = vi.fn(() => release.promise)
const captureDescendants = vi.fn(async () => snapshotOf(4243))
const tree = createClaudeChildTreeReaper(child, {
platform: 'darwin',
captureDescendants,
terminateDescendants
})
const first = tree.reap()
const second = tree.reap()
await vi.waitFor(() => expect(terminateDescendants).toHaveBeenCalledTimes(1))
// A stopped root cannot verify: its killed children stay zombie rows in ps.
expect(child.kill.mock.calls).toEqual([['SIGKILL']])
expect(tree.treeVerdict).toBe('unverifiable')
release.resolve('exited')
await expect(Promise.all([first, second])).resolves.toEqual(['exited', 'exited'])
expect(captureDescendants).toHaveBeenCalledTimes(1)
expect(tree.treeVerdict).toBe('exited')
})
it('re-verifies the retained snapshot on a later reap rather than re-walking a dead root', async () => {
const child = mockChild()
const captureDescendants = vi.fn(async () => snapshotOf(4243))
const terminateDescendants = vi
.fn()
.mockResolvedValueOnce('live')
.mockResolvedValueOnce('exited')
const tree = createClaudeChildTreeReaper(child, {
platform: 'linux',
captureDescendants,
terminateDescendants
})
await expect(tree.reap()).resolves.toBe('live')
expect(tree.treeVerdict).toBe('live')
await expect(tree.reap()).resolves.toBe('exited')
expect(captureDescendants).toHaveBeenCalledTimes(1)
expect(terminateDescendants).toHaveBeenNthCalledWith(2, snapshotOf(4243))
expect(tree.treeVerdict).toBe('exited')
})
it('keeps an observed exit when a later re-read cannot see the table', async () => {
const child = mockChild()
const terminateDescendants = vi
.fn()
.mockResolvedValueOnce('exited')
.mockResolvedValueOnce('unverifiable')
const tree = createClaudeChildTreeReaper(child, {
platform: 'linux',
captureDescendants: vi.fn(async () => snapshotOf(4243)),
terminateDescendants
})
await expect(tree.reap()).resolves.toBe('exited')
await expect(tree.reap()).resolves.toBe('unverifiable')
expect(tree.treeVerdict).toBe('exited')
})
it('keeps an observed live descendant when a later re-read cannot see the table', async () => {
const child = mockChild()
// Reap #1 completed and saw a descendant alive at its deadline; the root then
// left on its own and the re-verification on a loaded host could not read the
// table. "Could not look" must not erase "was seen alive": the lease release
// gate is exactly the pair this distinguishes.
const terminateDescendants = vi
.fn()
.mockResolvedValueOnce('live')
.mockResolvedValueOnce('unverifiable')
const tree = createClaudeChildTreeReaper(child, {
platform: 'linux',
captureDescendants: vi.fn(async () => snapshotOf(4243)),
terminateDescendants
})
await expect(tree.reap()).resolves.toBe('live')
await expect(tree.reap()).resolves.toBe('unverifiable')
expect(tree.treeVerdict).toBe('live')
})
it('treats an unreadable process table as unproven and re-walks the live root', async () => {
const child = mockChild()
// A loaded host can miss the table's deadline; while the root still lives
// that is a retryable read, not evidence that it has no descendants.
const captureDescendants = vi
.fn()
.mockResolvedValueOnce(null)
.mockResolvedValueOnce(snapshotOf(4243))
const terminateDescendants = vi.fn(async () => 'exited' as const)
const tree = createClaudeChildTreeReaper(child, {
platform: 'linux',
captureDescendants,
terminateDescendants
})
await expect(tree.reap()).resolves.toBe('unverifiable')
expect(terminateDescendants).not.toHaveBeenCalled()
await expect(tree.reap()).resolves.toBe('exited')
expect(captureDescendants).toHaveBeenCalledTimes(2)
})
it('does not latch a missing root while it is still live', async () => {
const child = mockChild()
const captureDescendants = vi
.fn()
.mockResolvedValueOnce({ rootPgid: null, descendants: [], capturedAtMs: 1 })
.mockResolvedValueOnce(snapshotOf(4243))
const terminateDescendants = vi.fn(async () => 'exited' as const)
const tree = createClaudeChildTreeReaper(child, {
platform: 'linux',
captureDescendants,
terminateDescendants
})
await tree.capture()
await expect(tree.reap()).resolves.toBe('exited')
expect(captureDescendants).toHaveBeenCalledTimes(2)
expect(terminateDescendants).toHaveBeenCalledWith(snapshotOf(4243))
})
it('refreshes the live snapshot at close time so late descendants are included', async () => {
const child = mockChild()
const first = snapshotOf(4243)
const second = {
...first,
descendants: [...first.descendants, { ...first.descendants[0], pid: 4244 }]
}
const captureDescendants = vi.fn().mockResolvedValueOnce(first).mockResolvedValueOnce(second)
const terminateDescendants = vi.fn(async () => 'exited' as const)
const tree = createClaudeChildTreeReaper(child, {
platform: 'linux',
captureDescendants,
terminateDescendants
})
await tree.capture()
await tree.refresh?.()
await tree.reap()
expect(captureDescendants).toHaveBeenCalledTimes(2)
expect(terminateDescendants).toHaveBeenCalledWith(second)
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
})
it('keeps the original capture boundary for retained POSIX rows', async () => {
const child = mockChild()
const first = {
...snapshotOf(4243),
capturedAtMs: 1_700_000_000_900
}
const refreshed = {
...first,
capturedAtMs: 1_700_000_002_100,
descendants: [
...first.descendants,
{
pid: 4244,
ppid: 424242,
pgid: 1,
startedAt: 'Tue Jan 2 00:00:00 2026'
}
]
}
const captureDescendants = vi.fn().mockResolvedValueOnce(first).mockResolvedValueOnce(refreshed)
const terminateDescendants = vi.fn(async () => 'exited' as const)
const tree = createClaudeChildTreeReaper(child, {
platform: 'linux',
captureDescendants,
terminateDescendants
})
await tree.capture()
await tree.refresh?.()
await tree.reap()
expect(terminateDescendants).toHaveBeenCalledWith({
...refreshed,
// The retained 4243 row was first observed in the earlier displayed
// second. Its per-row boundary must not advance with the refresh.
capturedAtMsByPid: {
'4243': first.capturedAtMs,
'4244': refreshed.capturedAtMs
}
})
})
it('fails closed when a POSIX refresh reuses a PID with a new identity', async () => {
const child = mockChild()
const first = snapshotOf(4243)
const replacement = {
...first,
descendants: [
{
...first.descendants[0],
pgid: 9,
startedAt: 'Tue Jan 2 00:00:00 2026'
}
]
}
const captureDescendants = vi
.fn()
.mockResolvedValueOnce(first)
.mockResolvedValueOnce(replacement)
const terminateDescendants = vi.fn(async () => 'exited' as const)
const tree = createClaudeChildTreeReaper(child, {
platform: 'linux',
captureDescendants,
terminateDescendants
})
await tree.capture()
await tree.refresh?.()
await expect(tree.reap()).resolves.toBe('unverifiable')
// The descendant evidence is discarded; the root's identity never was in doubt.
expect(terminateDescendants).not.toHaveBeenCalled()
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
})
it('fails closed when a Windows refresh reuses a PID with a new creation time', async () => {
const child = mockChild()
const first = windowsSnapshotOf(4243)
const replacement = {
...first,
descendants: [{ pid: 4243, creationTimeMs: first.descendants[0].creationTimeMs + 1 }]
}
const captureWindowsDescendants = vi
.fn()
.mockResolvedValueOnce(first)
.mockResolvedValueOnce(replacement)
const terminateWindowsTree = vi.fn(async () => {})
const terminateWindowsDescendants = vi.fn(async () => 'exited' as const)
const tree = createClaudeChildTreeReaper(child, {
platform: 'win32',
captureWindowsDescendants,
terminateWindowsTree,
terminateWindowsDescendants
})
await tree.capture()
await tree.refresh?.()
await expect(tree.reap()).resolves.toBe('unverifiable')
expect(terminateWindowsTree).not.toHaveBeenCalled()
expect(terminateWindowsDescendants).not.toHaveBeenCalled()
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
})
it('queues a fresh boundary behind an output-triggered capture already in flight', async () => {
const child = mockChild()
const firstDone = Promise.withResolvers<void>()
const first = snapshotOf(4243)
const second = {
...first,
descendants: [...first.descendants, { ...first.descendants[0], pid: 4244 }]
}
const captureDescendants = vi
.fn()
.mockImplementationOnce(async () => {
await firstDone.promise
return first
})
.mockResolvedValueOnce(second)
const terminateDescendants = vi.fn(async () => 'exited' as const)
const tree = createClaudeChildTreeReaper(child, {
platform: 'linux',
captureDescendants,
terminateDescendants
})
const outputCapture = tree.refresh!()
await vi.waitFor(() => expect(captureDescendants).toHaveBeenCalledTimes(1))
const closeCapture = tree.refresh!()
await Promise.resolve()
expect(captureDescendants).toHaveBeenCalledTimes(1)
firstDone.resolve()
await closeCapture
await tree.reap()
expect(captureDescendants).toHaveBeenCalledTimes(2)
expect(terminateDescendants).toHaveBeenCalledWith(second)
await outputCapture
})
it('retains a replacement descendant when the prior identity exited', async () => {
const child = mockChild()
const first = snapshotOf(4243)
const replacement = snapshotOf(4244)
const captureDescendants = vi
.fn()
.mockResolvedValueOnce(first)
.mockResolvedValueOnce(replacement)
const terminateDescendants = vi.fn(async (snapshot: DescendantSnapshot) =>
snapshot.descendants.some((row) => row.pid === 4244) ? ('live' as const) : ('exited' as const)
)
const tree = createClaudeChildTreeReaper(child, {
platform: 'linux',
captureDescendants,
terminateDescendants
})
await tree.capture()
await tree.refresh?.()
await expect(tree.reap()).resolves.toBe('live')
expect(terminateDescendants).toHaveBeenCalledWith({
...replacement,
descendants: [...first.descendants, ...replacement.descendants]
})
})
it('retains a Windows replacement descendant while preserving unidentified rows', async () => {
const child = mockChild()
const first = windowsSnapshotOf(4243)
const replacement = {
...windowsSnapshotOf(4244),
unidentifiedCount: 0
}
const captureWindowsDescendants = vi
.fn()
.mockResolvedValueOnce({ ...first, unidentifiedCount: 1 })
.mockResolvedValueOnce(replacement)
const terminateWindowsTree = vi.fn(async () => {})
const terminateWindowsDescendants = vi.fn(async (snapshot: WindowsDescendantSnapshot) =>
snapshot.descendants.some((row) => row.pid === 4244) ? ('live' as const) : ('exited' as const)
)
const tree = createClaudeChildTreeReaper(child, {
platform: 'win32',
captureWindowsDescendants,
terminateWindowsTree,
terminateWindowsDescendants
})
await tree.capture()
await tree.refresh?.()
await expect(tree.reap()).resolves.toBe('live')
expect(terminateWindowsDescendants).toHaveBeenCalledWith({
...replacement,
descendants: [...first.descendants, ...replacement.descendants],
unidentifiedCount: 1
})
})
it('retains the prior identity-safe snapshot when a refresh is partial', async () => {
const child = mockChild()
const first = {
...snapshotOf(4243),
descendants: [
...snapshotOf(4243).descendants,
{ ...snapshotOf(4243).descendants[0], pid: 4244 }
]
}
const captureDescendants = vi
.fn()
.mockResolvedValueOnce(first)
.mockResolvedValueOnce({
...first,
descendants: first.descendants.slice(0, 1)
})
const terminateDescendants = vi.fn(async () => 'exited' as const)
const tree = createClaudeChildTreeReaper(child, {
platform: 'linux',
captureDescendants,
terminateDescendants
})
await tree.capture()
await tree.refresh?.()
await tree.reap()
expect(terminateDescendants).toHaveBeenCalledWith(first)
})
it('stops re-walking once the root is gone, however the table behaved', async () => {
const child = mockChild()
let exited = false
const captureDescendants = vi.fn(async () => null)
const tree = createClaudeChildTreeReaper(child, {
platform: 'linux',
exited: () => exited,
captureDescendants,
terminateDescendants: vi.fn()
})
await expect(tree.reap()).resolves.toBe('unverifiable')
// An unreadable table costs the snapshot, never the kill on the live root.
expect(child.kill).toHaveBeenCalledTimes(1)
exited = true
await expect(tree.reap()).resolves.toBe('unverifiable')
expect(captureDescendants).toHaveBeenCalledTimes(1)
// The second attempt observes a dead root: Node has dropped the handle, so
// there is nothing left to signal and no recycled pid to reach.
expect(child.kill).toHaveBeenCalledTimes(1)
})
it('discards a walk that found no root instead of proving an empty tree', async () => {
const child = mockChild()
const captureDescendants = vi.fn(async () => ({
rootPgid: null,
descendants: [],
capturedAtMs: 1
}))
const tree = createClaudeChildTreeReaper(child, {
platform: 'linux',
captureDescendants,
terminateDescendants: vi.fn()
})
await tree.capture()
await expect(tree.reap()).resolves.toBe('unverifiable')
// A vacuous walk remains retryable while the root is live; no empty-tree
// verdict is latched from a missing root row.
expect(captureDescendants).toHaveBeenCalledTimes(2)
})
it('discards a walk that raced the root exit instead of proving an empty tree', async () => {
const child = mockChild()
let exited = false
const captureDescendants = vi.fn(async () => {
exited = true
return { rootPgid: 1, descendants: [], capturedAtMs: 1 }
})
const tree = createClaudeChildTreeReaper(child, {
platform: 'linux',
exited: () => exited,
captureDescendants,
terminateDescendants: vi.fn()
})
await tree.capture()
await expect(tree.reap()).resolves.toBe('unverifiable')
})
it('proves a childless snapshot without signalling anything', async () => {
const child = mockChild()
const terminateDescendants = vi.fn()
const tree = createClaudeChildTreeReaper(child, {
platform: 'linux',
captureDescendants: vi.fn(async () => ({
root: { pid: 424242, startedAt: 'Mon Jan 1 00:00:00 2026' },
rootPgid: 1,
descendants: [],
capturedAtMs: 1
})),
terminateDescendants
})
await expect(tree.reap()).resolves.toBe('exited')
expect(terminateDescendants).not.toHaveBeenCalled()
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
})
it('waits for the Windows tree kill before releasing the root', async () => {
const child = mockChild()
const release = Promise.withResolvers<void>()
const terminateWindowsTree = vi.fn(() => release.promise)
const captureDescendants = vi.fn()
const terminateWindowsDescendants = vi.fn(async () => 'exited' as const)
const tree = createClaudeChildTreeReaper(child, {
platform: 'win32',
captureDescendants,
captureWindowsDescendants: vi.fn(async () => windowsSnapshotOf(4243)),
terminateWindowsTree,
terminateWindowsDescendants
})
const reap = tree.reap()
await vi.waitFor(() =>
expect(terminateWindowsTree).toHaveBeenCalledWith({
pid: 424242,
creationTimeMs: 1_700_000_000_001
})
)
expect(child.kill).not.toHaveBeenCalled()
expect(terminateWindowsDescendants).not.toHaveBeenCalled()
release.resolve()
await expect(reap).resolves.toBe('exited')
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
expect(terminateWindowsDescendants).toHaveBeenCalledWith(windowsSnapshotOf(4243))
expect(captureDescendants).not.toHaveBeenCalled()
})
it('stays unproven on Windows when taskkill fails and a descendant is still observed', async () => {
const child = mockChild()
const tree = createClaudeChildTreeReaper(child, {
platform: 'win32',
captureWindowsDescendants: vi.fn(async () => windowsSnapshotOf(4243)),
terminateWindowsTree: vi.fn(async () => {
throw new Error('taskkill: access denied')
}),
terminateWindowsDescendants: vi.fn(async () => 'live' as const)
})
// taskkill's own outcome is not the proof; the table read after it is.
await expect(tree.reap()).resolves.toBe('live')
expect(tree.treeVerdict).toBe('live')
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
})
it('stays unproven on Windows when taskkill resolves but a descendant survives it', async () => {
const child = mockChild()
const terminateWindowsTree = vi.fn(async () => {})
const tree = createClaudeChildTreeReaper(child, {
platform: 'win32',
captureWindowsDescendants: vi.fn(async () => windowsSnapshotOf(4243)),
terminateWindowsTree,
terminateWindowsDescendants: vi.fn(async () => 'live' as const)
})
await expect(tree.reap()).resolves.toBe('live')
expect(terminateWindowsTree).toHaveBeenCalledTimes(1)
expect(tree.treeVerdict).toBe('live')
})
it('never taskkills a Windows root that already exited, but still verifies its snapshot', async () => {
const child = mockChild()
let exited = false
const terminateWindowsTree = vi.fn(async () => {})
const terminateWindowsDescendants = vi.fn(async () => 'exited' as const)
const tree = createClaudeChildTreeReaper(child, {
platform: 'win32',
exited: () => exited,
captureWindowsDescendants: vi.fn(async () => windowsSnapshotOf(4243)),
terminateWindowsTree,
terminateWindowsDescendants
})
await tree.capture()
exited = true
await expect(tree.reap()).resolves.toBe('exited')
// A dead root's pid may already belong to a stranger: taskkill /T /F on it
// would take down an unrelated tree.
expect(terminateWindowsTree).not.toHaveBeenCalled()
expect(terminateWindowsDescendants).toHaveBeenCalledWith(windowsSnapshotOf(4243))
})
it('treats an unreadable Windows table as unproven', async () => {
const child = mockChild()
const terminateWindowsDescendants = vi.fn()
const tree = createClaudeChildTreeReaper(child, {
platform: 'win32',
captureWindowsDescendants: vi.fn(async () => null),
terminateWindowsTree: vi.fn(async () => {}),
terminateWindowsDescendants
})
await expect(tree.reap()).resolves.toBe('unverifiable')
expect(terminateWindowsDescendants).not.toHaveBeenCalled()
// A host that cannot supply creation times blocks taskkill, not the root kill.
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
})
it('has nothing to reap for a child that never spawned', async () => {
const child = mockChild(null)
const captureDescendants = vi.fn()
const tree = createClaudeChildTreeReaper(child, { platform: 'linux', captureDescendants })
await expect(tree.reap()).resolves.toBe('exited')
expect(captureDescendants).not.toHaveBeenCalled()
})
})
@@ -0,0 +1,366 @@
import type { SpawnedProcess } from '../../shared/child-process/run-process'
import {
terminateDescendantSnapshotWithVerdict,
type DescendantTreeVerdict
} from '../pty-descendant-exit-verification'
import {
captureDescendantSnapshot,
type DescendantSnapshot,
type PosixProcessIdentity
} from '../pty-descendant-termination'
import {
captureWindowsDescendantSnapshot,
terminateIdentifiedWindowsProcessTree,
verifyWindowsDescendantSnapshotExit,
verifyWindowsProcessIdentity,
type WindowsDescendantSnapshot,
type WindowsProcessIdentity
} from '../windows-descendant-exit-verification'
import { mergeClaudeCapturedTrees, type ClaudeCapturedTree } from './claude-child-tree-snapshot'
import { terminateClaudeRoot, terminateClaudeWindowsRoot } from './claude-child-root-termination'
import {
proveClaudeChildExitWithReaper,
type ClaudeChildExitProofInput
} from './claude-child-exit-proof-ladder'
/**
* A later reap may only raise the latched verdict. An observed exit is final, and
* a descendant seen alive at a deadline is never forgotten by a later look that
* could not read the table: the lease gate discriminates on exactly that pair.
*/
const TREE_VERDICT_TRUST: Record<DescendantTreeVerdict, number> = {
unverifiable: 0,
live: 1,
exited: 2
}
type ReapableChild = Pick<SpawnedProcess, 'pid' | 'kill'>
/**
* A walk is only admissible while the root it walked was alive. A POSIX walk
* that found no root says so with a null pgid; either platform's walk can also
* have raced the root's death. Both can only have missed descendants that
* already reparented away, so neither is evidence about the tree.
*/
function admissibleTree(
captured: DescendantSnapshot | WindowsDescendantSnapshot | null,
platform: NodeJS.Platform,
exited: boolean
): ClaudeCapturedTree | null {
if (!captured || exited) {
return null
}
if (platform === 'win32') {
return { platform: 'win32', tree: captured as WindowsDescendantSnapshot }
}
const tree = captured as DescendantSnapshot
return tree.rootPgid === null ? null : { platform: 'posix', tree }
}
export type ClaudeChildTreeReaperDeps = {
platform?: NodeJS.Platform
/** Whether the root's exit has been observed; only a live root can be walked. */
exited?: () => boolean
captureDescendants?: (rootPid: number) => Promise<DescendantSnapshot | null>
terminateDescendants?: (snapshot: DescendantSnapshot) => Promise<DescendantTreeVerdict>
terminateWindowsTree?: (root: WindowsProcessIdentity) => Promise<void>
captureWindowsDescendants?: (rootPid: number) => Promise<WindowsDescendantSnapshot | null>
terminateWindowsDescendants?: (
snapshot: WindowsDescendantSnapshot
) => Promise<DescendantTreeVerdict>
/** Identity probe for the bare-pid tree kill; only Windows has one to gate. */
verifyRootIdentity?: (root: PosixProcessIdentity | WindowsProcessIdentity) => Promise<boolean>
}
export type ClaudeChildTreeReaper = {
/**
* Snapshot the root's live descendants. The moment the root dies they reparent
* and no table walk can find them again, so this has to run before anything
* gives the root a reason to leave. Held once; later calls are no-ops.
*/
capture(): Promise<void>
/** Refresh a live root's snapshot at the close boundary; a failed refresh keeps the prior proof. */
refresh?: () => Promise<void>
/**
* Kill the child's whole tree and report what the bounded verification
* observed. Concurrent calls share one reap, and a later call re-verifies the
* same snapshot rather than trusting a root that has since died on its own.
*/
reap(): Promise<DescendantTreeVerdict>
/**
* `unverifiable` until a reap observes otherwise. `exited` is the only verdict
* that lets a close release the lease; `live` names a descendant that was seen
* still running, which no later caller may collapse into "unknown".
*/
readonly treeVerdict: DescendantTreeVerdict
}
/**
* The same shared primitives the Codex structured provider composes: a raw
* pipe child owns no PTY job, so there is nothing for the PTY job sweep to
* terminate on Windows and no unref'd timer is allowed to outlive the proof.
*
* The proof is unproven by default. `treeVerdict` is assigned in exactly one
* place, from the verdict of `judgeTree`, so a code path that never reaches a
* verification cannot report the tree gone by omission.
*/
export function createClaudeChildTreeReaper(
child: ReapableChild,
deps: ClaudeChildTreeReaperDeps = {}
): ClaudeChildTreeReaper {
const platform = deps.platform ?? process.platform
const exited = deps.exited ?? (() => false)
// Undefined until captured; null when no admissible snapshot exists — the root
// was already gone, or the table could not be read while it was alive — which
// no later read can make up for.
let snapshot: ClaudeCapturedTree | null | undefined
let capturing: Promise<void> | null = null
let refreshing: Promise<void> | null = null
let queuedRefresh: Promise<void> | null = null
let inFlight: Promise<DescendantTreeVerdict> | null = null
let treeVerdict: DescendantTreeVerdict = 'unverifiable'
// Consulted only on win32: POSIX signals descendants by revalidated identity
// and reaches the root solely through Node's handle, so neither needs a probe.
const verifyRoot =
deps.verifyRootIdentity ??
((root: PosixProcessIdentity | WindowsProcessIdentity) =>
verifyWindowsProcessIdentity(root as WindowsProcessIdentity))
function captureOnce(): Promise<void> {
if (refreshing) {
const pending = refreshing
return pending.then(() => queuedRefresh ?? undefined)
}
if (snapshot !== undefined) {
return Promise.resolve()
}
if (capturing) {
const pending = capturing
return pending.then(() => queuedRefresh ?? undefined)
}
const rootPid = child.pid
if (!rootPid || exited()) {
// Only the root's death makes a missing snapshot final: its descendants
// have reparented, and no later walk can reach them.
snapshot = exited() ? null : snapshot
return Promise.resolve()
}
const capture =
platform === 'win32'
? (deps.captureWindowsDescendants ?? captureWindowsDescendantSnapshot)
: (deps.captureDescendants ?? captureDescendantSnapshot)
capturing = capture(rootPid)
.catch(() => null)
.then((captured) => {
// A walk that found no root, or that raced the root's death, can only
// have missed descendants that already reparented away. A table that
// could not be read in time is not an answer at all: while the root
// still lives the walk is simply retried, rather than latching a failed
// read as proof that there was nothing to find.
const rootExited = exited()
const tree = admissibleTree(captured, platform, rootExited)
if (tree) {
snapshot = tree
} else if (rootExited) {
// Once the root has exited its descendants may have reparented; no
// later table read can make an absent snapshot safe to signal.
snapshot = null
} else {
// A failed read or a walk that did not observe the live root is
// retryable while the root remains alive. Never latch a vacuous null.
snapshot = undefined
}
})
.finally(() => {
capturing = null
})
return capturing
}
function startRefresh(): Promise<void> {
if (exited()) {
return Promise.resolve()
}
const rootPid = child.pid
if (!rootPid) {
return Promise.resolve()
}
const capture =
platform === 'win32'
? (deps.captureWindowsDescendants ?? captureWindowsDescendantSnapshot)
: (deps.captureDescendants ?? captureDescendantSnapshot)
const operation = (async () => {
const captured = await capture(rootPid).catch(() => null)
if (exited()) {
return
}
const tree = admissibleTree(captured, platform, false)
if (!tree) {
return
}
if (snapshot === undefined) {
snapshot = tree
return
}
if (snapshot !== null) {
// A merge that returns null saw a same-PID identity change: a
// recycle/replace decision, not an absent descendant, so no row here may
// be signalled from its number. Only the descendant evidence is lost —
// the root still leaves through the handle no recycled pid can reach.
snapshot = mergeClaudeCapturedTrees(snapshot, tree)
}
// Keep an earlier admissible snapshot when this close-boundary read fails;
// it remains the only identity-safe evidence after root exit.
})()
refreshing = operation
const clearRefreshing = (): void => {
if (refreshing === operation) {
refreshing = null
}
}
void operation.then(clearRefreshing, clearRefreshing)
return operation
}
function queueRefreshAfter(pending: Promise<void>): Promise<void> {
if (queuedRefresh) {
return queuedRefresh
}
const operation = pending.then(() => {
if (exited()) {
return
}
return startRefresh()
})
queuedRefresh = operation
const clearQueuedRefresh = (): void => {
if (queuedRefresh === operation) {
queuedRefresh = null
}
}
void operation.then(clearQueuedRefresh, clearQueuedRefresh)
return operation
}
async function refresh(): Promise<void> {
const pending = capturing ?? refreshing
if (pending) {
await queueRefreshAfter(pending)
return
}
if (queuedRefresh) {
await queuedRefresh
return
}
try {
await startRefresh()
} catch {
// A refresh is advisory; capture failures leave the prior proof intact.
}
}
/** The only source of a tree verdict: every `exited` here is an observation. */
async function judgeTree(): Promise<DescendantTreeVerdict> {
const killRoot = (): boolean => terminateClaudeRoot({ child, exited })
const rootPid = child.pid
if (!rootPid) {
// Never spawned, so the OS never created a tree to orphan.
return 'exited'
}
await captureOnce()
if (platform === 'win32') {
// Why taskkill's own outcome is never the verdict: it resolves identically
// on a timeout, an access denial, a recycled root and a real kill.
const { rootVerified } = await terminateClaudeWindowsRoot({
snapshot: snapshot?.platform === 'win32' ? snapshot.tree : null,
exited,
verifyRoot: (root) => verifyRoot(root),
terminateTree: (root) =>
deps.terminateWindowsTree
? deps.terminateWindowsTree(root)
: terminateIdentifiedWindowsProcessTree(root, {
ownsRoot: () => !exited()
}).then(() => undefined),
killRoot
})
if (!rootVerified && !exited()) {
return 'unverifiable'
}
return snapshot?.platform === 'win32'
? await (deps.terminateWindowsDescendants ?? verifyWindowsDescendantSnapshotExit)(
snapshot.tree
)
: 'unverifiable'
}
if (snapshot?.platform !== 'posix') {
killRoot()
return 'unverifiable'
}
if (snapshot.tree.descendants.length === 0) {
// Read while the root was alive and childless: a later table read has no
// row it could match, so it would add nothing to this observation.
killRoot()
return 'exited'
}
// Why the root is killed while verification is already running, and never
// SIGSTOPped first the way the Codex non-group path does: measured on macOS, a
// killed child of a stopped parent stays a zombie row in ps with its lstart
// and pgid intact, so verification cannot pass until the root is dead. The
// descendants are signalled by the verifier as soon as it revalidates their
// identities; the root's death then reparents any zombies to init, which
// reaps them. After a root exit the kill is a no-op: Node drops the handle
// on exit and never signals a possibly recycled pid.
const verdictPromise = deps.terminateDescendants
? deps.terminateDescendants(snapshot.tree)
: terminateDescendantSnapshotWithVerdict(snapshot.tree, {
requireIdentityBeforeSignal: true
})
killRoot()
// What the verification observed is the verdict: a kill that reports no
// signal means the handle was already gone, never that the tree survived.
return verdictPromise
}
return {
capture: captureOnce,
refresh,
reap() {
if (inFlight) {
return inFlight
}
const attempt = judgeTree()
.catch((): DescendantTreeVerdict => 'unverifiable')
.then((verdict) => {
treeVerdict =
TREE_VERDICT_TRUST[verdict] > TREE_VERDICT_TRUST[treeVerdict] ? verdict : treeVerdict
return verdict
})
inFlight = attempt
void attempt.finally(() => {
if (inFlight === attempt) {
inFlight = null
}
})
return attempt
},
get treeVerdict() {
return treeVerdict
}
}
}
/**
* Orca's own shutdown ladder on the child it spawned, kept because the SDK's
* close path returns no proof and Orca never releases a lease on an assumed exit.
*
* Resolves true only after the child actually emitted exit and its snapshotted
* descendants were observed gone; false is unproven. A root that left on its
* own before a snapshot could be armed stays unproven: its descendants had
* already reparented out of reach when the ladder first looked.
*/
export function proveClaudeChildExit(input: ClaudeChildExitProofInput): Promise<boolean> {
return proveClaudeChildExitWithReaper(input, () =>
createClaudeChildTreeReaper(input.child, { exited: input.exited })
)
}
@@ -0,0 +1,154 @@
import { existsSync, readFileSync, statSync } from 'node:fs'
import { dirname, join, relative, resolve } from 'node:path'
import { describe, expect, it } from 'vitest'
import { spawnProcess } from '../../shared/child-process/run-process'
/**
* Keep the agent SDK on the structured-Claude side of the toggle.
*
* A user who never leaves the terminal/TUI Claude path must not pay for the SDK:
* importing it evaluates a package that rewrites
* `process.env.NoDefaultCurrentDirectoryInExePath`, changing how Windows resolves
* executables for every later subprocess, and a missing or incompatible install
* would take normal runtime startup down with it. The ordinary
* `OrcaRuntimeService` graph reaches the Claude transport module, so only a
* deferred import keeps that boundary — and only a walk of the real import graph
* keeps the next static import from quietly restoring it.
*/
const SDK_PACKAGE = '@anthropic-ai/claude-agent-sdk'
const REPO_ROOT = resolve(__dirname, '..', '..', '..')
/** The Electron main entry: everything the app loads before any session exists. */
const ROOT = 'src/main/index.ts'
/** Proof the walk goes all the way into the Claude transport rather than stopping short. */
const TRANSPORT_MODULE = 'src/main/claude/claude-stream-json-connection.ts'
/**
* Static, value-carrying specifiers only, read statement by statement so a
* multi-line `import { ... } from '...'` counts. `import type` is erased before
* the module ever loads and a bare `import(...)` is the deferral this guards, so
* neither is an edge the runtime traverses at load time.
*/
const STATEMENT_START = /^\s*(?:import|export)\b/
const TYPE_ONLY = /^\s*(?:import|export)\s+type\b/
const FROM_SPECIFIER = /(?:^|\s)from\s*['"]([^'"]+)['"]/
const SIDE_EFFECT_IMPORT = /^\s*import\s*['"]([^'"]+)['"]/
/** An import statement never spans more lines than its longest specifier list. */
const MAX_STATEMENT_LINES = 60
function readSpecifiers(source: string): string[] {
const lines = source.split('\n')
const found: string[] = []
for (let index = 0; index < lines.length; index += 1) {
const first = lines[index] as string
if (!STATEMENT_START.test(first) || TYPE_ONLY.test(first)) {
continue
}
const sideEffect = SIDE_EFFECT_IMPORT.exec(first)
if (sideEffect) {
found.push(sideEffect[1] as string)
continue
}
for (let scan = index; scan < Math.min(lines.length, index + MAX_STATEMENT_LINES); scan += 1) {
if (scan > index && STATEMENT_START.test(lines[scan] as string)) {
break
}
const specifier = FROM_SPECIFIER.exec(lines[scan] as string)
if (specifier) {
found.push(specifier[1] as string)
break
}
}
}
return found
}
/** Resolve a relative specifier the way the bundler does; unresolvable means not a module. */
function resolveRelative(fromFile: string, specifier: string): string | null {
const base = join(dirname(fromFile), specifier)
for (const candidate of [base, `${base}.ts`, `${base}.tsx`, join(base, 'index.ts')]) {
if (existsSync(candidate) && statSync(candidate).isFile()) {
return candidate
}
}
return null
}
function walkStaticImports(rootFile: string): { visited: Set<string>; sdkImporters: string[] } {
const visited = new Set<string>()
const sdkImporters: string[] = []
const queue = [resolve(REPO_ROOT, rootFile)]
while (queue.length > 0) {
const file = queue.pop() as string
const key = relative(REPO_ROOT, file).split('\\').join('/')
if (visited.has(key)) {
continue
}
visited.add(key)
for (const specifier of readSpecifiers(readFileSync(file, 'utf8'))) {
if (specifier === SDK_PACKAGE || specifier.startsWith(`${SDK_PACKAGE}/`)) {
sdkImporters.push(key)
continue
}
if (!specifier.startsWith('.')) {
continue
}
const target = resolveRelative(file, specifier)
if (target) {
queue.push(target)
}
}
}
return { visited, sdkImporters }
}
describe('claude agent SDK import boundary', () => {
const walk = walkStaticImports(ROOT)
it('walks a graph deep enough to reach the Claude transport', () => {
// Without this the guard passes for the wrong reason the moment the walk breaks.
expect(walk.visited.size).toBeGreaterThan(500)
expect([...walk.visited]).toContain(TRANSPORT_MODULE)
})
it('never reaches the SDK through a static import from the main entry', () => {
expect(
walk.sdkImporters,
`${SDK_PACKAGE} must stay behind the structured-Claude boundary. Load it with a deferred import inside the session path instead.`
).toEqual([])
})
it('leaves the Windows executable-search environment alone when the runtime loads', async () => {
// A vitest file runs in its own fork, so this is a clean process; the ambient
// value is cleared first because the developer's own shell may carry one.
delete process.env.NoDefaultCurrentDirectoryInExePath
await import('../runtime/structured-agent-session-runtime')
expect(process.env.NoDefaultCurrentDirectoryInExePath).toBeUndefined()
})
it('still lets the SDK set it, so the guard above is not measuring nothing', async () => {
// A separate process, not this fork: the assertion has to be about a first
// evaluation of the package, which a cached module registry cannot give.
const { NoDefaultCurrentDirectoryInExePath: _cleared, ...env } = process.env
const probe = spawnProcess({
program: process.execPath,
args: [
'-e',
`import(${JSON.stringify(SDK_PACKAGE)}).then(() => console.log(String(process.env.NoDefaultCurrentDirectoryInExePath)))`
],
cwd: REPO_ROOT,
env: env as Record<string, string>,
stdio: ['ignore', 'pipe', 'ignore']
})
const observed = await new Promise<string>((settle) => {
let output = ''
probe.stdout?.setEncoding('utf8').on('data', (chunk: string) => {
output += chunk
})
probe.once('close', () => settle(output.trim()))
})
expect(observed).toBe('1')
})
})
@@ -0,0 +1,107 @@
import { EventEmitter } from 'node:events'
import { PassThrough } from 'node:stream'
import { describe, expect, it, vi } from 'vitest'
import type { SpawnOptions as SdkSpawnOptions } from '@anthropic-ai/claude-agent-sdk'
import { resolveSpawn, type spawnProcess } from '../../shared/child-process/run-process'
import type { ProcessSpec } from '../../shared/child-process/process-spec'
import { createClaudeCodeProcessSpawn } from './claude-agent-sdk-process-spawn'
type FakeChild = EventEmitter & {
pid: number
stdin: PassThrough
stdout: PassThrough
stderr: PassThrough
kill: ReturnType<typeof vi.fn>
}
function fakeSpawn() {
const child = new EventEmitter() as FakeChild
child.pid = 4321
child.stdin = new PassThrough()
child.stdout = new PassThrough()
child.stderr = new PassThrough()
child.kill = vi.fn(() => true)
const specs: ProcessSpec[] = []
const spawnImpl = ((spec: ProcessSpec) => {
specs.push(spec)
return child
}) as unknown as typeof spawnProcess
return { child, spawnImpl, specs }
}
function sdkOptions(overrides: Partial<SdkSpawnOptions> = {}): SdkSpawnOptions {
return {
command: '/usr/local/bin/claude',
args: ['--output-format', 'stream-json'],
cwd: '/work/repo',
env: { PATH: '/usr/bin', CLAUDE_CONFIG_DIR: '/accounts/one', UNSET: undefined },
signal: new AbortController().signal,
...overrides
}
}
describe('claude agent SDK process spawn', () => {
it('routes the SDK spawn through Orca and retains the pid the lease adjudicates on', () => {
const process = fakeSpawn()
const spawn = createClaudeCodeProcessSpawn(process.spawnImpl)
expect(spawn.pid).toBeUndefined()
expect(spawn.child).toBeNull()
const child = spawn.spawn(sdkOptions())
expect(child).toBe(process.child)
expect(spawn.child).toBe(process.child)
expect(spawn.pid).toBe(4321)
expect(process.specs[0]).toEqual({
program: '/usr/local/bin/claude',
args: ['--output-format', 'stream-json'],
cwd: '/work/repo',
env: { PATH: '/usr/bin', CLAUDE_CONFIG_DIR: '/accounts/one' },
stdio: ['pipe', 'pipe', 'pipe']
})
})
it('keeps the child out of the SDK abort path so exit proof stays Orca-owned', () => {
const process = fakeSpawn()
const controller = new AbortController()
createClaudeCodeProcessSpawn(process.spawnImpl).spawn(sdkOptions({ signal: controller.signal }))
// Node's spawn({signal}) kills the child on abort; Orca's ladder must be the
// only thing that can end this process, or close() would report an assumed exit.
expect(process.specs[0]).not.toHaveProperty('signal')
})
it('drains stderr into a bounded tail so an exit error still carries it', async () => {
const process = fakeSpawn()
const spawn = createClaudeCodeProcessSpawn(process.spawnImpl)
spawn.spawn(sdkOptions())
process.child.stderr.write('x'.repeat(9000))
process.child.stderr.write('claude: not signed in')
await new Promise((resolve) => setImmediate(resolve))
expect(spawn.stderrTail).toMatch(/claude: not signed in$/)
expect(spawn.stderrTail.length).toBe(8192)
})
it('hands a Windows .cmd shim to Orca\u2019s argument encoder', () => {
const process = fakeSpawn()
createClaudeCodeProcessSpawn(process.spawnImpl).spawn(
sdkOptions({
command: 'C:\\Users\\dev\\AppData\\npm\\claude.cmd',
args: ['--setting-sources=user,project,local', '--session-id', 'a b&c']
})
)
// The spec the spawner builds is what Orca's Windows branch encodes; the SDK's
// own spawn would hand `.cmd` straight to Node and mangle the argument.
const resolved = resolveSpawn(process.specs[0] as ProcessSpec, 'win32')
expect(resolved.file.toLowerCase()).toContain('cmd.exe')
expect(resolved.options.windowsVerbatimArguments).toBe(true)
expect(resolved.args).toHaveLength(1)
// `/v:off` plus the quoted argument is what keeps `&` from splitting the line.
expect(resolved.args[0]).toContain('/v:off')
expect(resolved.args[0]).toContain('"a b&c"')
expect(resolved.args[0]).toContain('"--setting-sources=user,project,local"')
})
})
@@ -0,0 +1,69 @@
import type { SpawnOptions as ClaudeAgentSdkSpawnOptions } from '@anthropic-ai/claude-agent-sdk'
import { spawnProcess } from '../../shared/child-process/run-process'
/** Derived rather than imported: only src/shared/child-process may name node:child_process. */
type ClaudeCodeChild = ReturnType<typeof spawnProcess>
const STDERR_TAIL_MAX_BYTES = 8192
export type ClaudeCodeProcessSpawn = {
/** Pass as the SDK's `spawnClaudeCodeProcess`; the SDK never learns the pid because it never owns it. */
spawn: (options: ClaudeAgentSdkSpawnOptions) => ClaudeCodeChild
/** The retained child, so Orca keeps its own tree-kill and exit-proof ladder. Null until the SDK spawns. */
readonly child: ClaudeCodeChild | null
/** Ownership proof: the durable lease adjudicates on this pid plus start time plus the spawn token. */
readonly pid: number | undefined
readonly stderrTail: string
}
function definedEnv(env: Record<string, string | undefined>): Record<string, string> {
const next: Record<string, string> = {}
for (const [key, value] of Object.entries(env)) {
if (value !== undefined) {
next[key] = value
}
}
return next
}
/**
* Orca supplies the Claude Code child rather than letting the SDK spawn it.
*
* Two independent reasons: the SDK's `SpawnedProcess` has no pid, and Orca's
* spawner is the only path that encodes `.cmd` arguments safely on Windows.
*/
export function createClaudeCodeProcessSpawn(
spawnImpl: typeof spawnProcess = spawnProcess
): ClaudeCodeProcessSpawn {
let child: ClaudeCodeChild | null = null
let stderrTail = ''
return {
spawn: (options) => {
// Why `options.signal` is dropped: it would let the SDK kill the child outside
// Orca's ladder, and close() may never report an exit it did not observe.
const spawned = spawnImpl({
program: options.command,
args: [...options.args],
...(options.cwd === undefined ? {} : { cwd: options.cwd }),
env: definedEnv(options.env),
stdio: ['pipe', 'pipe', 'pipe']
})
child = spawned
// The SDK drains stderr only for its own local spawn, so a custom spawner must:
// otherwise the child blocks on a full pipe and exit errors lose their tail.
spawned.stderr.setEncoding('utf8').on('data', (chunk: string) => {
stderrTail = (stderrTail + chunk).slice(-STDERR_TAIL_MAX_BYTES)
})
return spawned
},
get child() {
return child
},
get pid() {
return child?.pid
},
get stderrTail() {
return stderrTail
}
}
}
@@ -0,0 +1,190 @@
import { EventEmitter } from 'node:events'
import { PassThrough } from 'node:stream'
import { describe, expect, it, vi } from 'vitest'
import type { SpawnedProcess } from '../../shared/child-process/run-process'
import type { DescendantSnapshot } from '../pty-descendant-termination'
import type { WindowsDescendantSnapshot } from '../windows-descendant-exit-verification'
import { createClaudeChildTreeReaper } from './claude-agent-sdk-exit-proof'
import { mergeClaudeCapturedTrees } from './claude-child-tree-snapshot'
const ROOT_PID = 424242
const ROOT_STARTED_AT = 'Mon Jan 1 00:00:00 2026'
const ROOT_FORK_MS = Date.parse(ROOT_STARTED_AT)
function mockChild(): EventEmitter &
Pick<SpawnedProcess, 'pid' | 'kill' | 'stdin'> & { kill: ReturnType<typeof vi.fn> } {
return Object.assign(new EventEmitter(), {
pid: ROOT_PID,
stdin: new PassThrough(),
kill: vi.fn(() => true)
}) as never
}
function posixSnapshot(input: {
capturedAtMs: number
descendants?: DescendantSnapshot['descendants']
}): DescendantSnapshot {
return {
root: { pid: ROOT_PID, startedAt: ROOT_STARTED_AT },
rootPgid: ROOT_PID,
descendants: input.descendants ?? [],
capturedAtMs: input.capturedAtMs
}
}
function windowsSnapshot(capturedAtMs = 1): WindowsDescendantSnapshot {
return {
root: { pid: ROOT_PID, creationTimeMs: 1_700_000_000_001 },
descendants: [{ pid: 4243, creationTimeMs: 1_700_000_000_000 }],
unidentifiedCount: 0,
capturedAtMs
}
}
describe('Claude root kill fallback', () => {
it('kills the root when the first capture landed in the fork second', async () => {
// The production POSIX verifier declines a root born in its capture second,
// and that verdict must not cost the tree the kill on Node's own handle.
const child = mockChild()
const tree = createClaudeChildTreeReaper(child, {
platform: 'linux',
exited: () => false,
captureDescendants: vi.fn(async () => posixSnapshot({ capturedAtMs: ROOT_FORK_MS + 300 }))
})
await expect(tree.reap()).resolves.toBe('exited')
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
})
it('kills the root after a recycled descendant pid voided the snapshot', async () => {
const child = mockChild()
const captureDescendants = vi
.fn()
.mockResolvedValueOnce(
posixSnapshot({
capturedAtMs: ROOT_FORK_MS + 5_000,
descendants: [{ pid: 100, ppid: ROOT_PID, pgid: ROOT_PID, startedAt: ROOT_STARTED_AT }]
})
)
.mockResolvedValueOnce(
posixSnapshot({
capturedAtMs: ROOT_FORK_MS + 6_000,
descendants: [
{ pid: 100, ppid: ROOT_PID, pgid: ROOT_PID, startedAt: 'Mon Jan 1 00:00:30 2026' }
]
})
)
const tree = createClaudeChildTreeReaper(child, {
platform: 'linux',
exited: () => false,
captureDescendants,
terminateDescendants: vi.fn(async () => 'exited' as const),
verifyRootIdentity: vi.fn(async () => true)
})
await tree.capture()
await tree.refresh?.()
// The descendant evidence is rightly discarded; the root's never was in doubt.
await expect(tree.reap()).resolves.toBe('unverifiable')
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
})
it('keeps an observed live descendant when the root identity probe declined', async () => {
const child = mockChild()
const tree = createClaudeChildTreeReaper(child, {
platform: 'linux',
exited: () => false,
captureDescendants: vi.fn(async () =>
posixSnapshot({
capturedAtMs: ROOT_FORK_MS + 5_000,
descendants: [{ pid: 100, ppid: ROOT_PID, pgid: ROOT_PID, startedAt: ROOT_STARTED_AT }]
})
),
terminateDescendants: vi.fn(async () => 'live' as const),
verifyRootIdentity: vi.fn(async () => false)
})
await expect(tree.reap()).resolves.toBe('live')
expect(tree.treeVerdict).toBe('live')
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
})
it('reports a Windows taskkill that worked as exited, not unverifiable', async () => {
const child = mockChild()
// Probe 1 gates taskkill; a later probe correctly finds the root already dead.
const verifyRootIdentity = vi.fn().mockResolvedValueOnce(true).mockResolvedValue(false)
const tree = createClaudeChildTreeReaper(child, {
platform: 'win32',
exited: () => false,
captureWindowsDescendants: vi.fn(async () => windowsSnapshot()),
terminateWindowsTree: vi.fn(async () => {}),
terminateWindowsDescendants: vi.fn(async () => 'exited' as const),
verifyRootIdentity
})
await expect(tree.reap()).resolves.toBe('exited')
})
it('kills the root when no POSIX snapshot could be read', async () => {
const child = mockChild()
const tree = createClaudeChildTreeReaper(child, {
platform: 'linux',
exited: () => false,
captureDescendants: vi.fn(async () => null),
terminateDescendants: vi.fn()
})
await expect(tree.reap()).resolves.toBe('unverifiable')
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
})
it('kills the root when the Windows process table is unreadable', async () => {
const child = mockChild()
const terminateWindowsTree = vi.fn(async () => {})
const tree = createClaudeChildTreeReaper(child, {
platform: 'win32',
exited: () => false,
captureWindowsDescendants: vi.fn(async () => null),
terminateWindowsTree,
terminateWindowsDescendants: vi.fn()
})
await expect(tree.reap()).resolves.toBe('unverifiable')
// No identity means no bare-pid tree kill, but the owned handle is still ours.
expect(terminateWindowsTree).not.toHaveBeenCalled()
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
})
it('never signals a root the reaper already saw exit', async () => {
const child = mockChild()
const tree = createClaudeChildTreeReaper(child, {
platform: 'linux',
exited: () => true,
captureDescendants: vi.fn(async () => null)
})
await expect(tree.reap()).resolves.toBe('unverifiable')
expect(child.kill).not.toHaveBeenCalled()
})
it('chains per-pid Windows boundaries across a second merge', async () => {
const first = windowsSnapshot(1_000)
const second: WindowsDescendantSnapshot = {
...windowsSnapshot(2_000),
descendants: [
{ pid: 4243, creationTimeMs: 1_700_000_000_000 },
{ pid: 4244, creationTimeMs: 1_700_000_000_002 }
]
}
const third: WindowsDescendantSnapshot = { ...second, capturedAtMs: 3_000 }
const merged = mergeClaudeCapturedTrees(
{ platform: 'win32', tree: first },
{ platform: 'win32', tree: second }
)
expect(merged?.tree.capturedAtMsByPid).toEqual({ '4243': 1_000, '4244': 2_000 })
const rechained = mergeClaudeCapturedTrees(merged!, { platform: 'win32', tree: third })
expect(rechained?.tree.capturedAtMsByPid).toEqual({ '4243': 1_000, '4244': 2_000 })
})
})
@@ -0,0 +1,65 @@
import type { SDKUserMessage } from '@anthropic-ai/claude-agent-sdk'
import { describe, expect, it } from 'vitest'
import { createClaudeUserMessageQueue } from './claude-agent-sdk-user-message-queue'
/**
* The SDK's input pump is `for await (const frame of prompt) { await transport.write(frame) }`.
* A rejected write — or an abort — ends that loop abruptly, which calls the
* generator's `return()`. Everything below drives that exact shape, because the
* frame the pump already pulled is the one nothing else can reach.
*/
const frame = (text: string): SDKUserMessage =>
({
type: 'user',
message: { role: 'user', content: [{ type: 'text', text }] }
}) as unknown as SDKUserMessage
const settled = (promise: Promise<void>): Promise<'settled' | 'pending'> =>
Promise.race([
promise.then(
() => 'settled' as const,
() => 'settled' as const
),
new Promise<'pending'>((resolve) => setTimeout(() => resolve('pending'), 100))
])
describe('claude user message queue', () => {
it('rejects the frame the SDK pulled but abandoned without writing', async () => {
const queue = createClaudeUserMessageQueue()
const pump = queue.messages[Symbol.asyncIterator]()
const sent = queue.push(frame('hello'))
await pump.next()
await pump.return?.(undefined)
await expect(settled(sent)).resolves.toBe('settled')
await expect(sent).rejects.toThrow(
'claude stream-json input ended before the frame was written'
)
})
it('rejects an in-flight frame from fail() when the SDK never resumes the pump', async () => {
const queue = createClaudeUserMessageQueue()
const pump = queue.messages[Symbol.asyncIterator]()
const sent = queue.push(frame('hello'))
await pump.next()
queue.fail(new Error('claude stream-json exited: child died'))
await expect(settled(sent)).resolves.toBe('settled')
await expect(sent).rejects.toThrow('claude stream-json exited: child died')
})
it('still settles a written frame only once the pump asks for the next one', async () => {
const queue = createClaudeUserMessageQueue()
const pump = queue.messages[Symbol.asyncIterator]()
const sent = queue.push(frame('hello'))
const pulled = await pump.next()
expect(pulled.value).toMatchObject({ type: 'user' })
// The write proof is the pump coming back for more, exactly as before.
await expect(settled(sent)).resolves.toBe('pending')
void pump.next()
await expect(sent).resolves.toBeUndefined()
})
})
@@ -0,0 +1,100 @@
import type { SDKUserMessage } from '@anthropic-ai/claude-agent-sdk'
type QueuedMessage = {
message: SDKUserMessage
resolve: () => void
reject: (error: Error) => void
}
export type ClaudeUserMessageQueue = {
/** The SDK's streaming-input prompt; it stays open until `end`. */
messages: AsyncIterable<SDKUserMessage>
/** Resolves once the SDK has finished writing the frame to the child. */
push: (message: SDKUserMessage) => Promise<void>
/** Reject every unwritten frame, in-flight included; a caller waiting on a send must not hang past the exit. */
fail: (error: Error) => void
end: () => void
}
/** The rejection an abandoned frame carries when nothing else has named a cause yet. */
const UNWRITTEN_FRAME_MESSAGE = 'claude stream-json input ended before the frame was written'
export function createClaudeUserMessageQueue(): ClaudeUserMessageQueue {
const queued: QueuedMessage[] = []
// The frame the SDK has taken but not yet acknowledged. It is out of `queued`,
// so it is unreachable from anywhere else and would otherwise never settle.
let inFlight: QueuedMessage | null = null
let wake: (() => void) | null = null
let ended = false
let failure: Error | null = null
const notify = (): void => {
wake?.()
wake = null
}
const rejectInFlight = (error: Error): void => {
const abandoned = inFlight
inFlight = null
abandoned?.reject(error)
}
async function* drain(): AsyncGenerator<SDKUserMessage> {
for (;;) {
const next = queued.shift()
if (next) {
inFlight = next
let written = false
try {
yield next.message
written = true
} finally {
// The SDK's input pump abandons this iterator when its
// `await transport.write(...)` rejects or the query aborts, and the code
// after a `yield` never runs on that path. Settling here is the only
// place a frame it already took can be reached.
if (written) {
inFlight = null
// Resumed only after the SDK's `await transport.write(...)` settled, so this
// is the same "the frame reached the child" proof the hand-rolled write gave.
next.resolve()
} else {
rejectInFlight(failure ?? new Error(UNWRITTEN_FRAME_MESSAGE))
}
}
continue
}
if (ended || failure) {
return
}
await new Promise<void>((resolve) => {
wake = resolve
})
}
}
return {
messages: drain(),
push: (message) =>
new Promise<void>((resolve, reject) => {
if (failure) {
reject(failure)
return
}
queued.push({ message, resolve, reject })
notify()
}),
fail: (error) => {
failure ??= error
for (const entry of queued.splice(0)) {
entry.reject(error)
}
// A pump that never resumes cannot run the generator's cleanup, so the
// exit path has to reach the in-flight frame itself.
rejectInFlight(error)
notify()
},
end: () => {
ended = true
notify()
}
}
}
@@ -0,0 +1,41 @@
import type { SpawnedProcess } from '../../shared/child-process/run-process'
import { waitForProcessExitUntil } from '../codex/codex-process-exit-deadline'
import type { ClaudeChildTreeReaper } from './claude-agent-sdk-exit-proof'
const GRACEFUL_EXIT_MS = 1_500
const FORCED_EXIT_MS = 1_000
export type ClaudeChildExitProofInput = {
child: Pick<SpawnedProcess, 'pid' | 'kill' | 'stdin'>
exitPromise: Promise<void>
exited: () => boolean
tree?: ClaudeChildTreeReaper
}
export async function proveClaudeChildExitWithReaper(
input: ClaudeChildExitProofInput,
createTree: () => ClaudeChildTreeReaper
): Promise<boolean> {
const tree = input.tree ?? createTree()
// Arm before stdin closes: only a live root can identify its descendants.
await tree.capture()
try {
input.child.stdin?.end()
} catch {
// The reap below still owns the process.
}
let reaped = false
if (!input.exited()) {
await waitForProcessExitUntil(input.exitPromise, GRACEFUL_EXIT_MS)
if (!input.exited()) {
reaped = true
await tree.refresh?.()
await tree.reap()
await waitForProcessExitUntil(input.exitPromise, FORCED_EXIT_MS)
}
}
if (!reaped && input.exited() && tree.treeVerdict !== 'exited') {
await tree.reap()
}
return input.exited() && tree.treeVerdict === 'exited'
}
@@ -0,0 +1,63 @@
import { describe, expect, it } from 'vitest'
import { applyClaudeEnvPatch } from '../claude-accounts/environment'
import { buildClaudeChildProcessEnv } from './claude-child-process-environment'
describe('Claude child process environment', () => {
it('strips case-insensitive auth headers through the shared env patch on Windows', () => {
expect(
applyClaudeEnvPatch(
{
anthropic_api_key: 'inherited-key',
Anthropic_Custom_Headers: 'Authorization: inherited',
SAFE_VALUE: 'preserved'
},
{},
{ stripAuthEnv: true, platform: 'win32' }
)
).toEqual({ SAFE_VALUE: 'preserved' })
})
it('strips case-insensitive inherited auth and session stamps on Windows', () => {
const env = buildClaudeChildProcessEnv(
{
ANTHROPIC_AUTH_TOKEN: 'configured-token',
Claude_Code_Session_Id: 'configured-session'
},
{
platform: 'win32',
inheritedEnv: {
anthropic_api_key: 'inherited-key',
Anthropic_Custom_Headers: 'Authorization: inherited',
claude_code_child_session: '1',
CLAUDE_CODE_SESSION_ID: 'inherited-session',
SAFE_VALUE: 'preserved'
}
}
)
expect(env).toEqual({
ANTHROPIC_AUTH_TOKEN: 'configured-token',
Claude_Code_Session_Id: 'configured-session',
SAFE_VALUE: 'preserved'
})
})
it('can strip child-session stamps reintroduced by a full SDK launch overlay', () => {
expect(
buildClaudeChildProcessEnv(
{
CLAUDE_CODE_CHILD_SESSION: 'configured-child-session',
CLAUDE_CODE_SESSION_ID: 'configured-session',
CLAUDE_CODE_BRIDGE_SESSION_ID: 'configured-bridge-session'
},
{
scrubConfiguredChildSessionStamps: true,
inheritedEnv: {
CLAUDE_CODE_CHILD_SESSION: 'inherited-child-session',
SAFE_VALUE: 'preserved'
}
}
)
).toEqual({ SAFE_VALUE: 'preserved' })
})
})
@@ -0,0 +1,69 @@
import { CLAUDE_AUTH_ENV_VARS, applyClaudeEnvPatch } from '../claude-accounts/environment'
const CLAUDE_CHILD_SESSION_STAMP_ENV_KEYS = [
'CLAUDE_CODE_CHILD_SESSION',
'CLAUDE_CODE_SESSION_ID',
'CLAUDE_CODE_BRIDGE_SESSION_ID'
] as const
function cloneProcessEnv(source: NodeJS.ProcessEnv): Record<string, string> {
const env: Record<string, string> = {}
for (const [key, value] of Object.entries(source)) {
if (value !== undefined) {
env[key] = value
}
}
return env
}
function stripClaudeChildSessionStamps(
env: Record<string, string>,
platform: NodeJS.Platform
): Record<string, string> {
for (const key of CLAUDE_CHILD_SESSION_STAMP_ENV_KEYS) {
for (const envKey of Object.keys(env)) {
if (envKey === key || (platform === 'win32' && envKey.toUpperCase() === key)) {
delete env[envKey]
}
}
}
return env
}
export function buildClaudeChildProcessEnv(
configuredEnv: Record<string, string> = {},
options: {
inheritedEnv?: NodeJS.ProcessEnv
platform?: NodeJS.Platform
scrubConfiguredChildSessionStamps?: boolean
} = {}
): Record<string, string> {
const inheritedEnv = options.inheritedEnv ?? process.env
const platform = options.platform ?? process.platform
const env = applyClaudeEnvPatch(
cloneProcessEnv(inheritedEnv),
{},
{
stripAuthEnv: true,
platform
}
)
if (platform === 'win32') {
const authKeys = new Set(CLAUDE_AUTH_ENV_VARS.map((key) => key.toUpperCase()))
for (const [key, value] of Object.entries(env)) {
const normalized = key.toUpperCase()
if (
authKeys.has(normalized) ||
(normalized === 'ANTHROPIC_CUSTOM_HEADERS' &&
/authorization|x-api-key|api-key|bearer/i.test(value))
) {
delete env[key]
}
}
}
if (options.scrubConfiguredChildSessionStamps) {
return stripClaudeChildSessionStamps({ ...env, ...configuredEnv }, platform)
}
stripClaudeChildSessionStamps(env, platform)
return { ...env, ...configuredEnv }
}
@@ -0,0 +1,54 @@
import type { SpawnedProcess } from '../../shared/child-process/run-process'
import type { PosixProcessIdentity } from '../pty-descendant-termination'
import type {
WindowsDescendantSnapshot,
WindowsProcessIdentity
} from '../windows-descendant-exit-verification'
export type ClaudeRootIdentity = PosixProcessIdentity | WindowsProcessIdentity
type RootTerminationInput = {
child: Pick<SpawnedProcess, 'kill'>
exited: () => boolean
}
/**
* Kills the root through the handle Node owns rather than through its pid, which
* is why no identity probe gates it: libuv drops that handle in the same turn it
* reaps, so the signal either reaches the process Orca spawned or reaches
* nothing. A probe here could only let an unreadable process table cost the tree
* the one fallback that still works once every table read has failed.
*
* False means no signal was sent, because the root had already left.
*/
export function terminateClaudeRoot(input: RootTerminationInput): boolean {
return input.exited() ? false : input.child.kill('SIGKILL')
}
type WindowsRootTerminationInput = {
snapshot: WindowsDescendantSnapshot | null
exited: () => boolean
verifyRoot: (root: WindowsProcessIdentity) => Promise<boolean>
terminateTree: (root: WindowsProcessIdentity) => Promise<void>
killRoot: () => boolean
}
/**
* `taskkill /T /F` addresses a bare pid, so a dead root's pid may already belong
* to a stranger whose whole tree it would take down: that one is identity-gated.
* The direct root kill after it runs however the probe decided.
*/
export async function terminateClaudeWindowsRoot(
input: WindowsRootTerminationInput
): Promise<{ rootVerified: boolean }> {
const { snapshot, exited, verifyRoot, terminateTree, killRoot } = input
let rootVerified = false
if (!exited() && snapshot) {
rootVerified = await verifyRoot(snapshot.root).catch(() => false)
if (rootVerified && !exited()) {
await terminateTree(snapshot.root).catch(() => {})
}
}
killRoot()
return { rootVerified }
}
@@ -0,0 +1,128 @@
import type { DescendantSnapshot } from '../pty-descendant-termination'
import type { WindowsDescendantSnapshot } from '../windows-descendant-exit-verification'
/** One platform's descendant tree, tagged so neither verifier can be handed the other's rows. */
export type ClaudeCapturedTree =
| { platform: 'posix'; tree: DescendantSnapshot }
| { platform: 'win32'; tree: WindowsDescendantSnapshot }
/**
* Process-table reads are not atomic: a refresh can omit a still-live row, but
* it can also observe a new process after the old row exited. Retain rows absent
* from the refresh, but reject a PID whose identity changed between reads.
*/
function mergeRowsByPid<Row extends { pid: number }>(
previous: readonly Row[],
next: readonly Row[],
sameIdentity: (previous: Row, next: Row) => boolean,
previousBoundary: (row: Row) => number,
nextBoundary: (row: Row) => number,
refreshBoundary: number
): { rows: Row[]; capturedAtMsByPid?: Readonly<Record<string, number>> } | null {
const merged = new Map<number, Row>()
const capturedAtMsByPid: Record<string, number> = {}
for (const row of previous) {
const prior = merged.get(row.pid)
if (prior && !sameIdentity(prior, row)) {
return null
}
merged.set(row.pid, row)
capturedAtMsByPid[String(row.pid)] = previousBoundary(row)
}
for (const row of next) {
const prior = merged.get(row.pid)
if (prior && !sameIdentity(prior, row)) {
return null
}
if (!prior) {
capturedAtMsByPid[String(row.pid)] = nextBoundary(row)
}
merged.set(row.pid, row)
}
const boundaries = Object.values(capturedAtMsByPid)
const needsBoundaryMap =
new Set(boundaries).size > 1 || boundaries.some((boundary) => boundary !== refreshBoundary)
return {
rows: [...merged.values()],
...(needsBoundaryMap ? { capturedAtMsByPid } : {})
}
}
export function mergeClaudeCapturedTrees(
previous: ClaudeCapturedTree,
next: ClaudeCapturedTree
): ClaudeCapturedTree | null {
if (previous.platform !== next.platform) {
return null
}
if (previous.platform === 'posix' && next.platform === 'posix') {
if (previous.tree.rootPgid !== next.tree.rootPgid) {
return null
}
// A refresh cannot repair an earlier capture that lacked root identity;
// retaining those rows would permit a later numeric-pid kill without proof.
if (!previous.tree.root || !next.tree.root) {
return null
}
if (
previous.tree.root.pid !== next.tree.root.pid ||
previous.tree.root.startedAt !== next.tree.root.startedAt
) {
return null
}
const descendants = mergeRowsByPid(
previous.tree.descendants,
next.tree.descendants,
(left, right) => left.pgid === right.pgid && left.startedAt === right.startedAt,
(row) => previous.tree.capturedAtMsByPid?.[String(row.pid)] ?? previous.tree.capturedAtMs,
(row) => next.tree.capturedAtMsByPid?.[String(row.pid)] ?? next.tree.capturedAtMs,
next.tree.capturedAtMs
)
if (!descendants) {
return null
}
return {
platform: 'posix',
tree: {
...next.tree,
// Retained rows keep their earlier boundary; new rows use the refresh
// boundary. The scalar remains the latest scan for legacy consumers.
descendants: descendants.rows,
...(descendants.capturedAtMsByPid
? { capturedAtMsByPid: descendants.capturedAtMsByPid }
: {})
}
}
}
if (previous.platform === 'win32' && next.platform === 'win32') {
if (
previous.tree.root.pid !== next.tree.root.pid ||
previous.tree.root.creationTimeMs !== next.tree.root.creationTimeMs
) {
return null
}
const descendants = mergeRowsByPid(
previous.tree.descendants,
next.tree.descendants,
(left, right) => left.creationTimeMs === right.creationTimeMs,
(row) => previous.tree.capturedAtMsByPid?.[String(row.pid)] ?? previous.tree.capturedAtMs,
(row) => next.tree.capturedAtMsByPid?.[String(row.pid)] ?? next.tree.capturedAtMs,
next.tree.capturedAtMs
)
if (!descendants) {
return null
}
return {
platform: 'win32',
tree: {
...next.tree,
descendants: descendants.rows,
...(descendants.capturedAtMsByPid
? { capturedAtMsByPid: descendants.capturedAtMsByPid }
: {}),
unidentifiedCount: Math.max(previous.tree.unidentifiedCount, next.tree.unidentifiedCount)
}
}
}
return null
}
@@ -0,0 +1,115 @@
import { describe, expect, it, vi } from 'vitest'
import type {
AgentJournalItemBody,
AgentJournalItemIdentity
} from '../../shared/agent-session-journal-types'
import type { StructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink'
import { createClaudeJournalTranslator } from './claude-structured-journal-translation'
function sinkState() {
const items: { identity: AgentJournalItemIdentity; body: AgentJournalItemBody }[] = []
const sink: StructuredAgentSessionEventSink = {
appendItem: (identity, body) => items.push({ identity, body }),
appendTombstone: () => {},
publish: vi.fn()
}
return { sink, items }
}
function providerFrameKinds(items: { body: AgentJournalItemBody }[]): string[] {
return items.flatMap((item) =>
item.body.kind === 'status' && item.body.providerFrame ? [item.body.providerFrame.kind] : []
)
}
/**
* The queue-bookkeeping frame Claude Code 2.1.258 emits for every uuid-stamped
* command: `command_uuid` plus a state, and no content of its own. Shape and
* states taken from the CLI's own emission sites.
*/
function commandLifecycle(state: 'started' | 'completed' | 'cancelled', uuid: string) {
return {
type: 'message' as const,
sessionId: 'orca-session',
message: {
type: 'command_lifecycle',
command_uuid: 'command-1',
state,
uuid,
session_id: 'claude-session'
}
}
}
function userTurn(uuid: string, text: string) {
return {
type: 'message' as const,
sessionId: 'orca-session',
startsTurn: true as const,
message: {
type: 'user',
uuid,
session_id: 'claude-session',
parent_tool_use_id: null,
isReplay: true,
message: { role: 'user', content: text }
}
}
}
function assistantReply(uuid: string, text: string) {
return {
type: 'message' as const,
sessionId: 'orca-session',
message: {
type: 'assistant',
uuid,
session_id: 'claude-session',
parent_tool_use_id: null,
message: { role: 'assistant', content: [{ type: 'text', text }] }
}
}
}
describe('Claude command_lifecycle frames', () => {
it('keeps queue bookkeeping off the transcript for a whole turn', () => {
const state = sinkState()
const translator = createClaudeJournalTranslator({ sink: state.sink })
translator.handle(userTurn('user-1', 'Reply with exactly PROBE_OK and nothing else.'))
translator.handle(commandLifecycle('started', 'lifecycle-1'))
translator.handle(assistantReply('assistant-1', 'PROBE_OK'))
translator.handle(commandLifecycle('completed', 'lifecycle-2'))
translator.handle(commandLifecycle('completed', 'lifecycle-3'))
translator.handle({
type: 'message',
sessionId: 'orca-session',
message: {
type: 'result',
subtype: 'success',
uuid: 'result-1',
session_id: 'claude-session',
is_error: false,
result: 'PROBE_OK',
terminal_reason: 'completed'
}
})
expect(providerFrameKinds(state.items)).toEqual([])
// The turn's real content is untouched.
expect(
state.items.flatMap((item) =>
item.body.kind === 'message' && item.body.role === 'assistant' ? [item.body.blocks] : []
)
).toEqual([[{ type: 'text', text: 'PROBE_OK' }]])
})
it('keeps a cancelled command off the transcript too', () => {
const state = sinkState()
const translator = createClaudeJournalTranslator({ sink: state.sink })
translator.handle(commandLifecycle('cancelled', 'lifecycle-4'))
expect(providerFrameKinds(state.items)).toEqual([])
})
})
@@ -0,0 +1,34 @@
import { homedir } from 'node:os'
import { join } from 'node:path'
import { describe, expect, it } from 'vitest'
import { claudeConfigDirEnvPatch, defaultClaudeConfigDir } from './claude-config-dir-pin'
describe('claude config dir pin', () => {
it('does not pin the CLI default home, so the macOS Keychain stays reachable', () => {
expect(claudeConfigDirEnvPatch(join(homedir(), '.claude'), { env: {} })).toEqual({})
expect(claudeConfigDirEnvPatch(`${join(homedir(), '.claude')}/`, { env: {} })).toEqual({})
expect(claudeConfigDirEnvPatch(' ', { env: {} })).toEqual({})
})
it('pins a managed account home the CLI would not find on its own', () => {
expect(claudeConfigDirEnvPatch('/accounts/claude/managed', { env: {} })).toEqual({
CLAUDE_CONFIG_DIR: '/accounts/claude/managed'
})
})
it('treats an inherited CLAUDE_CONFIG_DIR as the default the CLI already resolves', () => {
const env = { CLAUDE_CONFIG_DIR: '/inherited/home' }
expect(defaultClaudeConfigDir(env)).toBe('/inherited/home')
expect(claudeConfigDirEnvPatch('/inherited/home', { env })).toEqual({})
expect(claudeConfigDirEnvPatch('/other/home', { env })).toEqual({
CLAUDE_CONFIG_DIR: '/other/home'
})
})
it('compares Windows homes case-insensitively', () => {
const env = { CLAUDE_CONFIG_DIR: 'C:\\Users\\Work\\.claude' }
expect(claudeConfigDirEnvPatch('c:\\users\\work\\.claude', { env, platform: 'win32' })).toEqual(
{}
)
})
})
+37
View File
@@ -0,0 +1,37 @@
import { homedir } from 'node:os'
import { join, resolve } from 'node:path'
/** The config dir the Claude CLI resolves for itself when nothing pins one. */
export function defaultClaudeConfigDir(env: NodeJS.ProcessEnv = process.env): string {
return env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude')
}
function samePath(a: string, b: string, platform: NodeJS.Platform): boolean {
const left = resolve(a)
const right = resolve(b)
return platform === 'win32' ? left.toLowerCase() === right.toLowerCase() : left === right
}
/**
* An explicit CLAUDE_CONFIG_DIR moves the Claude CLI off the default Keychain item onto
* one derived from the pinned path, so a claude.ai OAuth login stops working even when
* the pin names the CLI's own default. Pin only a home the CLI would not find on its
* own — the same rule the legacy PTY path applies via `ClaudeRuntimePathResolver`.
*
* The pinned value is the account home verbatim: the CLI keys its credential lookup on
* the literal string, so re-spelling an equivalent path (absolute vs `~`, trailing
* separator) selects a different identity. Normalization here is for the equality test
* only and must never reach the env.
*/
export function claudeConfigDirEnvPatch(
accountHome: string,
options: { env?: NodeJS.ProcessEnv; platform?: NodeJS.Platform } = {}
): { CLAUDE_CONFIG_DIR?: string } {
const env = options.env ?? process.env
const platform = options.platform ?? process.platform
const resolved = accountHome.trim()
if (!resolved || samePath(resolved, defaultClaudeConfigDir(env), platform)) {
return {}
}
return { CLAUDE_CONFIG_DIR: resolved }
}
@@ -0,0 +1,124 @@
import { describe, expect, it, vi } from 'vitest'
import { terminateDescendantSnapshotWithVerdict } from '../pty-descendant-exit-verification'
import {
collectDescendantRows,
type DescendantSnapshot,
type ProcessTableRow
} from '../pty-descendant-termination'
import { createClaudeChildTreeReaper } from './claude-agent-sdk-exit-proof'
const ROOT_PID = 500
const ORCA_PGID = 400
const ROOT_STARTED_AT = 'Thu Sep 3 18:04:50 2026'
/** The second both close-time walks land in. */
const WALK_SECOND = 'Thu Sep 3 18:05:04 2026'
const WALK_MS = Date.parse(WALK_SECOND)
const EARLIER_SECOND = 'Thu Sep 3 18:05:03 2026'
/** The measured split: `s20` at :03.946 died, `s21` at :04.042 leaked. */
const EARLIER_BORN = [700, 701, 702]
const WALK_SECOND_BORN = [721, 722, 723, 724]
type Cohort = { pids: number[]; startedAt: string }
const LIVE_TREE: Cohort[] = [
{ pids: EARLIER_BORN, startedAt: EARLIER_SECOND },
{ pids: WALK_SECOND_BORN, startedAt: WALK_SECOND }
]
function rowsFor(cohorts: Cohort[]): ProcessTableRow[] {
return [
{ pid: ROOT_PID, ppid: 1, pgid: ORCA_PGID, startedAt: ROOT_STARTED_AT },
...cohorts.flatMap((cohort) =>
cohort.pids.map((pid) => ({
pid,
ppid: ROOT_PID,
pgid: ORCA_PGID,
startedAt: cohort.startedAt
}))
)
]
}
/** A real ppid walk from the root, exactly as production captures one. */
function walk(capturedAtMs: number, cohorts: Cohort[] = LIVE_TREE): DescendantSnapshot {
return collectDescendantRows(ROOT_PID, rowsFor(cohorts), capturedAtMs)
}
function killedPids(calls: [number, NodeJS.Signals][]): number[] {
return calls.flatMap(([pid, signal]) => (signal === 'SIGKILL' ? [pid] : [])).sort((a, b) => a - b)
}
function signalledPids(calls: [number, NodeJS.Signals][]): number[] {
return calls.flatMap(([pid, signal]) => (signal === 'SIGTERM' ? [pid] : [])).sort((a, b) => a - b)
}
/**
* Drives the real reaper and the real verifier against a process table where
* every descendant traps SIGTERM, so only a forced sweep can end them. The root
* is alive for both walks and gone by the sweep, which is the measured teardown.
*/
async function sweep(
captures: DescendantSnapshot[],
liveTree: Cohort[] = LIVE_TREE
): Promise<[number, NodeJS.Signals][]> {
const calls: [number, NodeJS.Signals][] = []
const captureDescendants = vi.fn()
for (const capture of captures) {
captureDescendants.mockResolvedValueOnce(capture)
}
const tree = createClaudeChildTreeReaper(
{ pid: ROOT_PID, kill: vi.fn(() => true) },
{
platform: 'linux',
exited: () => false,
captureDescendants,
terminateDescendants: (snapshot) =>
terminateDescendantSnapshotWithVerdict(snapshot, {
requireIdentityBeforeSignal: true,
graceMs: 0,
verifyMs: 120,
sendSignal: (pid, signal) => calls.push([pid, signal]),
readTable: async () => ({ rows: rowsFor(liveTree), capturedAtMs: Date.now() })
})
}
)
// The close ladder's shape: arm, then re-walk the live root at the boundary.
await tree.capture()
await tree.refresh?.()
await tree.reap()
return calls
}
describe('Claude descendant forced-sweep fence', () => {
it('escalates a descendant forked in the same second as both close walks', async () => {
// Both walks land inside second :04, one ps duration apart, and the root is
// gone before a third could run. A descendant born at :04.042 is no less
// ours than its sibling born 96ms earlier at :03.946.
const calls = await sweep([walk(WALK_MS + 42), walk(WALK_MS + 140)])
expect(signalledPids(calls)).toEqual([...EARLIER_BORN, ...WALK_SECOND_BORN])
expect(killedPids(calls)).toEqual([...EARLIER_BORN, ...WALK_SECOND_BORN])
})
it('still escalates descendants born before the walk that first saw them', async () => {
const onlyEarlier = [{ pids: EARLIER_BORN, startedAt: EARLIER_SECOND }]
const calls = await sweep([walk(WALK_MS + 42, onlyEarlier)], onlyEarlier)
expect(killedPids(calls)).toEqual(EARLIER_BORN)
})
it('withholds the sweep from a row no walk re-derived, on its start second alone', async () => {
// 900 was seen once, in its own birth second, and the refresh did not find
// it. The merge retains the row, but nothing re-proved it belongs to us, so
// the second-resolution fence is all there is and it still says no.
const retained = { pids: [900], startedAt: WALK_SECOND }
const firstWalk = walk(WALK_MS + 42, [...LIVE_TREE, retained])
const refresh = walk(WALK_MS + 140)
const calls = await sweep([firstWalk, refresh], [...LIVE_TREE, retained])
expect(signalledPids(calls)).toEqual([...EARLIER_BORN, ...WALK_SECOND_BORN, 900])
expect(killedPids(calls)).toEqual([...EARLIER_BORN, ...WALK_SECOND_BORN])
})
})
@@ -0,0 +1,126 @@
import { EventEmitter } from 'node:events'
import { PassThrough } from 'node:stream'
import type { ChildProcessWithoutNullStreams } from 'node:child_process'
import { describe, expect, it, vi } from 'vitest'
import type { query } from '@anthropic-ai/claude-agent-sdk'
import {
openClaudeStreamJsonConnection,
type ClaudeStreamJsonLaunch
} from './claude-stream-json-connection'
const mocks = vi.hoisted(() => {
const refresh = vi.fn()
const proveClaudeChildExit = vi.fn()
const tree = {
capture: vi.fn(async () => {}),
refresh: (...args: unknown[]) => refresh(...args),
reap: vi.fn(async () => 'exited' as const),
treeVerdict: 'unverifiable' as const
}
return { proveClaudeChildExit, refresh, tree }
})
vi.mock('./claude-agent-sdk-exit-proof', () => ({
createClaudeChildTreeReaper: vi.fn(() => mocks.tree),
proveClaudeChildExit: (...args: unknown[]) => mocks.proveClaudeChildExit(...args)
}))
function fakeChild(): ChildProcessWithoutNullStreams {
const child = new EventEmitter()
return Object.assign(child, {
pid: 424242,
stdin: new PassThrough(),
stdout: new PassThrough(),
stderr: new PassThrough(),
kill: vi.fn()
}) as unknown as ChildProcessWithoutNullStreams
}
describe('Claude stream-json close ordering', () => {
it('waits for the live tree refresh before ending stdin', async () => {
const refreshDone = Promise.withResolvers<void>()
mocks.refresh.mockReturnValueOnce(refreshDone.promise)
mocks.proveClaudeChildExit.mockResolvedValueOnce(true)
const child = fakeChild()
const launch: ClaudeStreamJsonLaunch = {
pathToClaudeCodeExecutable: 'claude',
options: {},
cwd: '/work/repo'
}
const queryImpl = ((params: Parameters<typeof query>[0]) => {
if (!params.options) {
throw new Error('missing SDK options')
}
params.options.spawnClaudeCodeProcess?.({
command: 'claude',
args: [],
env: {},
signal: new AbortController().signal
})
void (async () => {
for await (const _message of params.prompt) {
// The SDK owns the transport write; the close test only needs its EOF boundary.
}
child.stdin.end()
})()
return (async function* () {})()
}) as typeof query
const connection = await openClaudeStreamJsonConnection(launch, {}, () => child, queryImpl)
const closing = connection.close()
await new Promise((resolve) => setImmediate(resolve))
expect(child.stdin.writableEnded).toBe(false)
refreshDone.resolve()
await expect(closing).resolves.toBe(true)
expect(child.stdin.writableEnded).toBe(true)
})
it('requests a fresh close boundary after an output capture starts', async () => {
mocks.refresh.mockReset()
mocks.proveClaudeChildExit.mockReset()
const outputCapture = Promise.withResolvers<void>()
const closeCapture = Promise.withResolvers<void>()
mocks.refresh
.mockReturnValueOnce(outputCapture.promise)
.mockReturnValueOnce(closeCapture.promise)
mocks.proveClaudeChildExit.mockResolvedValueOnce(true)
const child = fakeChild()
const launch: ClaudeStreamJsonLaunch = {
pathToClaudeCodeExecutable: 'claude',
options: {},
cwd: '/work/repo'
}
const queryImpl = ((params: Parameters<typeof query>[0]) => {
params.options?.spawnClaudeCodeProcess?.({
command: 'claude',
args: [],
env: {},
signal: new AbortController().signal
})
void (async () => {
for await (const _message of params.prompt) {
// The SDK owns the transport write; the close test only needs its EOF boundary.
}
child.stdin.end()
})()
return (async function* () {})()
}) as typeof query
const connection = await openClaudeStreamJsonConnection(launch, {}, () => child, queryImpl)
child.stderr.emit('data', 'output')
await vi.waitFor(() => expect(mocks.refresh).toHaveBeenCalledTimes(1))
const closing = connection.close()
await Promise.resolve()
expect(mocks.refresh).toHaveBeenCalledTimes(2)
expect(child.stdin.writableEnded).toBe(false)
outputCapture.resolve()
await Promise.resolve()
expect(child.stdin.writableEnded).toBe(false)
closeCapture.resolve()
await expect(closing).resolves.toBe(true)
expect(child.stdin.writableEnded).toBe(true)
})
})
@@ -0,0 +1,768 @@
import { execFileSync } from 'node:child_process'
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { spawnProcess, type SpawnedProcess } from '../../shared/child-process/run-process'
import { hasLiveClaudePtys } from '../claude-accounts/live-pty-gate'
import type { ProcessSpec } from '../../shared/child-process/process-spec'
import { query, type CanUseTool, type Options } from '@anthropic-ai/claude-agent-sdk'
import {
openClaudeStreamJsonConnection,
type ClaudeStreamJsonConnection,
type ClaudeStreamJsonLaunch
} from './claude-stream-json-connection'
import { openAgentSessionJournal } from '../native-chat/agent-session-journal/journal-store-factory'
import { createDeferredStructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink'
import { claudeAuthDiagnostic } from './claude-structured-init-proof'
import { createClaudeJournalTranslator } from './claude-structured-journal-translation'
import { readClaudeStructuredSessionOptions } from './claude-structured-session-options'
import type { ClaudeSession } from './claude-structured-session-state'
import { CLAUDE_STRUCTURED_BASE_OPTIONS } from './claude-structured-launch-resolution'
// These drive the real SDK against the scripted fake CLI, so every assertion is
// about the environment, argv and frames a real child actually saw.
const FAKE_CLI = join(__dirname, '__fixtures__', 'claude-agent-sdk-scripted-cli.mjs')
const SESSION_ID = '5348c19f-6a54-4c2e-9c68-9c2b1a3d4e5f'
const HOLD_OPEN = { delayMs: 10_000 }
type ScriptedCliReport = {
argv: string[]
controlRequests: { request_id: string; request: { subtype: string } }[]
controlResponses: { response: { request_id: string; response?: unknown } }[]
userMessages: Record<string, unknown>[]
descendantPid: number | null
}
const scratchDirs: string[] = []
const openConnections: ClaudeStreamJsonConnection[] = []
afterEach(async () => {
for (const connection of openConnections.splice(0)) {
await connection.close()
}
for (const dir of scratchDirs.splice(0)) {
rmSync(dir, { recursive: true, force: true })
}
spawned.splice(0)
spawnedChildren.splice(0)
vi.unstubAllEnvs()
})
function scriptScenario(
steps: Record<string, unknown>[],
controlResponses: Record<string, unknown> = {}
) {
const dir = mkdtempSync(join(tmpdir(), 'claude-sdk-connection-'))
scratchDirs.push(dir)
const scenarioPath = join(dir, 'scenario.json')
const reportPath = join(dir, 'report.json')
writeFileSync(scenarioPath, JSON.stringify({ steps, controlResponses }))
return {
cwd: dir,
env: {
PATH: process.env.PATH ?? '',
ORCA_SDK_CONTRACT_SCENARIO_PATH: scenarioPath,
ORCA_SDK_CONTRACT_REPORT_PATH: reportPath
},
readReport: () => JSON.parse(readFileSync(reportPath, 'utf8')) as ScriptedCliReport
}
}
function launchFor(
scenario: { cwd: string; env: Record<string, string> },
env: Record<string, string> = {}
): ClaudeStreamJsonLaunch {
return {
pathToClaudeCodeExecutable: FAKE_CLI,
options: { ...CLAUDE_STRUCTURED_BASE_OPTIONS, sessionId: SESSION_ID },
cwd: scenario.cwd,
env: { ...scenario.env, ...env }
}
}
/** The derived child environment, captured where Orca actually hands it to the OS. */
const spawned: ProcessSpec[] = []
/** The retained child, so a test can end it the way a crashing CLI would. */
const spawnedChildren: SpawnedProcess[] = []
async function open(
launch: ClaudeStreamJsonLaunch,
handlers: Parameters<typeof openClaudeStreamJsonConnection>[1] = {},
queryImpl?: typeof query
): Promise<ClaudeStreamJsonConnection> {
const connection = await openClaudeStreamJsonConnection(
launch,
handlers,
(spec) => {
spawned.push(spec)
const child = spawnProcess(spec)
spawnedChildren.push(child)
return child
},
queryImpl
)
openConnections.push(connection)
return connection
}
function childEnv(): Record<string, string | undefined> {
return (spawned.at(-1)?.env ?? {}) as Record<string, string | undefined>
}
async function until<T>(read: () => T | null | undefined, label: string): Promise<T> {
for (let attempt = 0; attempt < 400; attempt++) {
const value = read()
if (value !== null && value !== undefined) {
return value
}
await new Promise((resolve) => setTimeout(resolve, 25))
}
throw new Error(`timed out waiting for ${label}`)
}
function readReportSafely(scenario: { readReport: () => ScriptedCliReport }) {
try {
return scenario.readReport()
} catch {
return null
}
}
function processState(pid: number): 'running' | 'exited' {
try {
const state = execFileSync('ps', ['-o', 'state=', '-p', String(pid)], {
encoding: 'utf8',
env: { ...process.env, LANG: 'C', LC_ALL: 'C' }
}).trim()
return state.startsWith('Z') ? 'exited' : 'running'
} catch (error) {
if ((error as { status?: number }).status === 1) {
return 'exited'
}
throw error
}
}
describe('Claude stream-json connection', () => {
it('passes the Claude Code system-prompt preset through to SDK query', async () => {
const scenario = scriptScenario([HOLD_OPEN])
let captured: Options | undefined
await open(launchFor(scenario), {}, (params) => {
captured = params.options
return query(params)
})
expect(captured?.systemPrompt).toEqual({ type: 'preset', preset: 'claude_code' })
})
it('hands the child a derived environment, the resolved CLI path, and keeps the pid', async () => {
vi.stubEnv('ANTHROPIC_API_KEY', 'sk-ant-SHELL-LEAK')
vi.stubEnv('CLAUDE_CODE_CHILD_SESSION', '1')
vi.stubEnv('NODE_OPTIONS', '--require=/tmp/inject.js')
// An inherited value wins over the SDK's default, so clear it to pin the default.
vi.stubEnv('CLAUDE_CODE_ENTRYPOINT', undefined)
vi.stubEnv('ORCA_CONNECTION_MARKER', 'inherited')
const scenario = scriptScenario([HOLD_OPEN])
const connection = await open(
launchFor(scenario, {
CLAUDE_CONFIG_DIR: '/accounts/managed/home',
ANTHROPIC_AUTH_TOKEN: 'configured-token',
ORCA_AGENT_SESSION_SPAWN_TOKEN: 'spawn-9',
CLAUDE_CODE_CHILD_SESSION: 'configured-child-session',
CLAUDE_CODE_SESSION_ID: 'configured-session',
CLAUDE_CODE_BRIDGE_SESSION_ID: 'configured-bridge-session'
})
)
// Ownership proof: the pid is a real live process, not a value the SDK reported.
expect(connection.pid).toEqual(expect.any(Number))
expect(() => process.kill(connection.pid as number, 0)).not.toThrow()
const env = childEnv()
// The managed home is pinned verbatim: the CLI keys credential lookup on the literal string.
expect(env.CLAUDE_CONFIG_DIR).toBe('/accounts/managed/home')
expect(env.ANTHROPIC_AUTH_TOKEN).toBe('configured-token')
expect(env.ORCA_AGENT_SESSION_SPAWN_TOKEN).toBe('spawn-9')
expect(env.ORCA_CONNECTION_MARKER).toBe('inherited')
expect(env.ANTHROPIC_API_KEY).toBeUndefined()
expect(env.CLAUDE_CODE_CHILD_SESSION).toBeUndefined()
expect(env.CLAUDE_CODE_SESSION_ID).toBeUndefined()
expect(env.CLAUDE_CODE_BRIDGE_SESSION_ID).toBeUndefined()
// Two SDK mutations of the child env, pinned so a bump cannot change them unseen.
expect(env.CLAUDE_CODE_ENTRYPOINT).toBe('sdk-ts')
expect(env.NODE_OPTIONS).toBeUndefined()
// The bundled binary is excluded from the install, so the resolved path is mandatory.
const report = await until(() => readReportSafely(scenario), 'the scripted CLI report')
expect(report.argv[0]).toBe(FAKE_CLI)
// The .mjs fixture makes the SDK run it under node; a real CLI path is the program
// itself. Either way the resolved path is what Orca's spawner is asked to execute.
expect([spawned.at(-1)?.program, ...(spawned.at(-1)?.args ?? [])]).toContain(FAKE_CLI)
expect(report.argv).toContain('--replay-user-messages')
expect(report.argv).toContain(`--session-id=${SESSION_ID}`)
})
it('leaves the default CLI home unpinned so macOS Keychain OAuth keeps working', async () => {
const scenario = scriptScenario([HOLD_OPEN])
await open(launchFor(scenario))
await until(() => readReportSafely(scenario), 'the scripted CLI report')
expect(childEnv().CLAUDE_CONFIG_DIR).toBeUndefined()
})
it('settles a send only once the frame reached the child, and replays reach onMessage', async () => {
const replay = {
type: 'user',
message: { role: 'user', content: [{ type: 'text', text: 'hello' }] },
parent_tool_use_id: null,
isReplay: true,
session_id: SESSION_ID,
uuid: 'uuid-replay-1'
}
const scenario = scriptScenario([{ awaitUserMessage: true }, { emit: replay }, HOLD_OPEN])
const messages: Record<string, unknown>[] = []
const connection = await open(launchFor(scenario), {
onMessage: (message) => messages.push(message)
})
await connection.send({
type: 'user',
message: { role: 'user', content: [{ type: 'text', text: 'hello' }] },
parent_tool_use_id: null,
session_id: SESSION_ID
})
// The report exists from the child's first line of work, so poll for the frame
// itself: `send` settles on the SDK's completed write, and the child still has
// to read that line before it can record it.
const report = await until(
() => (readReportSafely(scenario)?.userMessages.length ? readReportSafely(scenario) : null),
'the user frame recorded by the child'
)
expect(report.userMessages).toHaveLength(1)
await until(() => messages.find((message) => message.uuid === 'uuid-replay-1'), 'the replay')
// The replay is delivered verbatim, so the dispatch acknowledgement still binds on it.
expect(messages.find((message) => message.uuid === 'uuid-replay-1')).toEqual(replay)
})
it('rejects a send the SDK pulled but could not write to a terminated child', async () => {
const scenario = scriptScenario([{ awaitUserMessage: true }, HOLD_OPEN])
const connection = await open(launchFor(scenario))
const child = spawnedChildren.at(-1)
// Same tick as the send, so the liveness guard still passes and the frame
// reaches the SDK's input pump: its `transport.write` is what fails, which is
// the window a child crashing mid-send actually opens.
child?.kill('SIGKILL')
const sent = connection.send({
type: 'user',
message: { role: 'user', content: [{ type: 'text', text: 'hello' }] },
parent_tool_use_id: null,
session_id: SESSION_ID
})
await expect(sent).rejects.toThrow()
expect(readReportSafely(scenario)?.userMessages ?? []).toHaveLength(0)
})
it('delivers an unmodeled frame verbatim so the provider-fallback row survives', async () => {
const unknown = {
type: 'frame_kind_from_the_future',
session_id: SESSION_ID,
uuid: 'uuid-unknown-1',
payload: { nested: { flags: ['a', 'b'] } }
}
const scenario = scriptScenario([{ emit: unknown }, HOLD_OPEN])
const messages: Record<string, unknown>[] = []
await open(launchFor(scenario), { onMessage: (message) => messages.push(message) })
await until(() => messages.find((message) => message.uuid === 'uuid-unknown-1'), 'the frame')
expect(messages.find((message) => message.uuid === 'uuid-unknown-1')).toEqual(unknown)
})
it('commits the real partial-message cadence as one assistant item through the translator', async () => {
// The frame order and per-frame uuids are the ones Claude Code 2.1.258 emits
// under --include-partial-messages: every stream_event and the block's final
// assistant frame each carry their own uuid; only message.id ties them.
const stream = (uuid: string, event: Record<string, unknown>) => ({
type: 'stream_event',
uuid,
session_id: SESSION_ID,
parent_tool_use_id: null,
event
})
const frames = [
stream('uuid-message-start', {
type: 'message_start',
message: { id: 'msg_01', role: 'assistant', content: [] }
}),
stream('uuid-block-start', {
type: 'content_block_start',
index: 0,
content_block: { type: 'text', text: '' }
}),
stream('uuid-delta-1', {
type: 'content_block_delta',
index: 0,
delta: { type: 'text_delta', text: 'ST' }
}),
stream('uuid-delta-2', {
type: 'content_block_delta',
index: 0,
delta: { type: 'text_delta', text: 'REAMOK_ELEC_64E632' }
}),
{
type: 'assistant',
uuid: 'uuid-assistant-final',
session_id: SESSION_ID,
parent_tool_use_id: null,
message: {
id: 'msg_01',
role: 'assistant',
content: [{ type: 'text', text: 'STREAMOK_ELEC_64E632' }],
stop_reason: null
}
},
stream('uuid-block-stop', { type: 'content_block_stop', index: 0 }),
stream('uuid-message-delta', { type: 'message_delta', delta: { stop_reason: 'end_turn' } }),
stream('uuid-message-stop', { type: 'message_stop' }),
{
type: 'result',
subtype: 'success',
is_error: false,
duration_ms: 1,
duration_api_ms: 1,
num_turns: 1,
result: 'STREAMOK_ELEC_64E632',
stop_reason: 'end_turn',
session_id: SESSION_ID,
uuid: 'uuid-result'
}
]
const scenario = scriptScenario([...frames.map((frame) => ({ emit: frame })), HOLD_OPEN])
const journal = await openAgentSessionJournal({
identity: {
sessionId: 'session-1',
workspaceId: 'workspace-1',
hostId: 'host-1',
agent: 'claude',
providerHandle: { kind: 'claude', sessionId: SESSION_ID, leafUuid: 'leaf-1' }
},
journalDir: join(scenario.cwd, 'journal'),
now: () => 1_700_000_000_000,
mintEpoch: () => 'epoch-1'
})
const deferred = createDeferredStructuredAgentSessionEventSink()
deferred.bind({ journal, fence: 1, publish: vi.fn() })
const translator = createClaudeJournalTranslator({ sink: deferred.sink })
let settled = false
await open(launchFor(scenario), {
onMessage: (message) => {
translator.handle({ type: 'message', sessionId: 'session-1', message })
settled ||= message.type === 'result'
}
})
await until(() => (settled ? true : null), 'the result frame')
await deferred.drained()
const items = journal.snapshot().items
const assistant = items.filter(
(item) => item.body.kind === 'message' && item.body.role === 'assistant'
)
expect(assistant.map((item) => item.body)).toEqual([
{
kind: 'message',
role: 'assistant',
blocks: [{ type: 'text', text: 'STREAMOK_ELEC_64E632' }]
}
])
expect(assistant.map((item) => item.itemId)).toEqual([`claude:${SESSION_ID}:uuid-block-start`])
expect(
items.flatMap((item) =>
item.body.kind === 'status' && item.body.providerFrame ? [item.body.providerFrame.kind] : []
)
).toEqual([])
// The journal owns a SQLite connection now; afterEach removes this temp root and an open
// handle blocks that on Windows.
await journal.close()
})
it('feeds an inbound permission request to canUseTool and writes its answer back on the same id', async () => {
const scenario = scriptScenario([
{
emit: {
type: 'control_request',
request_id: 'perm-421',
request: {
subtype: 'can_use_tool',
tool_name: 'Bash',
input: { command: 'ls' },
tool_use_id: 'toolu_1',
permission_suggestions: [{ type: 'addRules' }]
}
}
},
{ awaitControlResponse: 'perm-421' },
HOLD_OPEN
])
const seen: { toolName: string; requestId: string; toolUseID: string; suggestions: unknown }[] =
[]
const canUseTool: CanUseTool = (toolName, _input, options) => {
seen.push({
toolName,
requestId: options.requestId,
toolUseID: options.toolUseID,
suggestions: options.suggestions
})
return Promise.resolve({ behavior: 'deny', message: 'No', toolUseID: options.toolUseID })
}
await open(launchFor(scenario), { canUseTool })
await until(() => (seen.length > 0 ? seen : null), 'the inbound permission request')
expect(seen).toEqual([
{
toolName: 'Bash',
requestId: 'perm-421',
toolUseID: 'toolu_1',
suggestions: [{ type: 'addRules' }]
}
])
const written = await until(
() =>
readReportSafely(scenario)?.controlResponses.find(
(frame) => frame.response.request_id === 'perm-421'
),
'the permission answer'
)
expect(written.response.response).toMatchObject({ behavior: 'deny', message: 'No' })
})
it('drives Orca control methods onto the SDK and times out with the init proof message', async () => {
const scenario = scriptScenario([HOLD_OPEN], {
initialize: { models: [{ value: 'sonnet' }], account: { tokenSource: 'oauth' } },
get_settings: { env: { ANTHROPIC_BASE_URL: 'https://settings.example.test' } }
})
const connection = await open(launchFor(scenario))
await expect(connection.initializationResult()).resolves.toMatchObject({
models: [{ value: 'sonnet' }]
})
await expect(connection.getSettings()).resolves.toEqual({
env: { ANTHROPIC_BASE_URL: 'https://settings.example.test' }
})
await expect(connection.setModel('opus')).resolves.toBeUndefined()
const requests = await until(
() =>
readReportSafely(scenario)?.controlRequests.find(
(frame) => frame.request.subtype === 'set_model'
),
'the set_model control request'
)
expect(requests.request.subtype).toBe('set_model')
})
it('reads supportedModels from the catalog the running CLI reported', async () => {
const scenario = scriptScenario([HOLD_OPEN], {
initialize: {
models: [
{ value: 'default', resolvedModel: 'claude-opus-5' },
{
value: 'opus',
displayName: 'Opus 5',
description: 'The live row, not the seed',
resolvedModel: 'claude-opus-5',
supportsEffort: true,
supportedEffortLevels: ['low', 'high']
}
]
}
})
const connection = await open(launchFor(scenario))
await expect(connection.supportedModels()).resolves.toMatchObject([
{ value: 'default', resolvedModel: 'claude-opus-5' },
{ value: 'opus', displayName: 'Opus 5', supportedEffortLevels: ['low', 'high'] }
])
})
it('serves the picker the live catalog rather than falling back to the static seed', async () => {
const scenario = scriptScenario([HOLD_OPEN], {
initialize: {
models: [
{ value: 'default', resolvedModel: 'claude-opus-5' },
{
value: 'opus',
displayName: 'Opus 5',
description: 'The live row, not the seed',
resolvedModel: 'claude-opus-5',
supportsEffort: true,
supportedEffortLevels: ['low', 'high']
}
]
}
})
const connection = await open(launchFor(scenario))
const session = {
connection,
options: new Map<string, string>(),
reportedOptions: {}
} as unknown as ClaudeSession
const options = await readClaudeStructuredSessionOptions(session, 5_000)
// The seed carries neither this description nor a two-level effort list, so
// both can only have come from the child.
expect(options.models).toContainEqual({
id: 'opus',
label: 'Opus 5',
description: 'The live row, not the seed',
isDefault: true,
efforts: [
{ value: 'low', label: 'Low' },
{ value: 'high', label: 'High' }
]
})
expect(options.current.model).toBe('opus')
})
it('feeds the auth diagnostic from the settings the running child reports', async () => {
for (const key of ['ANTHROPIC_BASE_URL', 'ANTHROPIC_AUTH_TOKEN', 'ANTHROPIC_API_KEY']) {
vi.stubEnv(key, undefined)
}
const scenario = scriptScenario([HOLD_OPEN], {
get_settings: {
env: {
ANTHROPIC_BASE_URL: 'https://settings.example.test',
ANTHROPIC_AUTH_TOKEN: 'secret'
}
}
})
const connection = await open(launchFor(scenario))
const init = { providerSessionId: SESSION_ID, uuid: null, model: null, message: {} }
// With no ambient auth, every true below can only have come from the CLI's settings.
expect(claudeAuthDiagnostic(init, null)).toMatchObject({
baseUrlConfigured: false,
authTokenConfigured: false
})
const diagnostic = claudeAuthDiagnostic(init, await connection.getSettings())
expect(diagnostic).toMatchObject({
baseUrlConfigured: true,
authTokenConfigured: true,
apiKeyConfigured: false
})
expect(JSON.stringify(diagnostic)).not.toContain('secret')
})
it('reports an unauthenticated start through the init deadline instead of hanging', async () => {
// The scripted CLI never answers, which is the shape of a silently unauthenticated CLI.
const scenario = scriptScenario([HOLD_OPEN])
const connection = await open({
...launchFor(scenario),
env: { ...launchFor(scenario).env, ORCA_SDK_CONTRACT_IGNORE_CONTROL_REQUESTS: '1' }
})
await expect(connection.initializationResult({ timeoutMs: 200 })).rejects.toThrow(
'claude initialize request timed out'
)
})
it('reports a self-exit with its status and stderr, and leaves its tree unverifiable', async () => {
const scenario = scriptScenario([{ stderr: 'claude: not signed in\n' }, { exit: 1 }])
let exit: Error | null = null
const connection = await open(launchFor(scenario), {
onExit: (error) => {
exit = error
}
})
await until(() => exit, 'the exit error')
// The status and stderr are the only diagnostic a refused start leaves behind.
expect((exit as unknown as Error).message).toMatch(/exited \(code 1\): claude: not signed in/)
expect(connection.closed).toBe(true)
// The root's exit is first-hand, but it left before a descendant snapshot
// could be armed, so close() has no tree proof to offer and says so.
await expect(connection.close()).resolves.toBe(false)
expect(connection.exitVerdict).toEqual({ root: 'exited', tree: 'unverifiable' })
})
it.runIf(process.platform !== 'win32')(
'proves a natural SDK exit and cleans up its descendant before recovery',
async () => {
const scenario = scriptScenario([
{ stderr: 'claude: natural exit\n' },
{ delayMs: 500 },
{ exit: 1 }
])
let exit: Error | null = null
const connection = await open(
{
...launchFor(scenario),
env: { ...launchFor(scenario).env, ORCA_SDK_CONTRACT_DESCENDANT: '1' }
},
{ onExit: (error) => (exit = error) }
)
const report = await until(() => {
const current = readReportSafely(scenario)
return current?.descendantPid ? current : null
}, 'the descendant report')
await until(() => exit, 'the natural exit error')
try {
await expect(connection.close()).resolves.toBe(true)
expect(connection.exitVerdict).toEqual({ root: 'exited', tree: 'exited' })
expect(processState(report.descendantPid as number)).toBe('exited')
} finally {
try {
process.kill(report.descendantPid as number, 'SIGKILL')
} catch {
// Already gone.
}
}
},
20_000
)
it('settles a spawn error followed by close as processless and closes idempotently', async () => {
const scenario = scriptScenario([HOLD_OPEN])
const missingCli = join(scenario.cwd, 'claude-that-does-not-exist')
let fault: Error | null = null
let exit: Error | null = null
const connection = await open(
{ ...launchFor(scenario), pathToClaudeCodeExecutable: missingCli },
{
onFault: (error) => {
fault = error
},
onExit: (error) => {
exit = error
}
}
)
await until(
() => (connection.exitVerdict.root === 'processless' ? connection.exitVerdict : null),
'the processless spawn settlement'
)
expect(connection.pid).toBeUndefined()
expect(fault).toBeInstanceOf(Error)
expect(exit).toBeNull()
await expect(Promise.all([connection.close(), connection.close()])).resolves.toEqual([
true,
true
])
await expect(connection.close()).resolves.toBe(true)
expect(connection.exitVerdict).toEqual({ root: 'processless', tree: 'exited' })
})
it('does not treat a child error event as first-hand root exit proof', async () => {
const scenario = scriptScenario([HOLD_OPEN])
let exit: Error | null = null
const connection = await open(launchFor(scenario), {
onExit: (error) => {
exit = error
}
})
const child = spawnedChildren.at(-1)
expect(child).toBeDefined()
child?.emit('error', new Error('child transport fault'))
expect(exit).toBeNull()
expect(connection.exitVerdict.root).toBe('live')
await until(() => exit, 'the distinct child exit')
expect(connection.exitVerdict.root).toBe('exited')
})
it('proves the exit of a child that ignores a graceful shutdown', async () => {
const scenario = scriptScenario([HOLD_OPEN])
const connection = await open({
...launchFor(scenario),
env: { ...launchFor(scenario).env, ORCA_SDK_CONTRACT_IGNORE_SIGTERM: '1' }
})
// Keep the lstart capture boundary outside the child's displayed start second.
await new Promise((resolve) => setTimeout(resolve, 1_100))
await expect(connection.close()).resolves.toBe(true)
}, 20_000)
})
// A structured Claude child owns the account's credentials while it runs, exactly as
// a Claude PTY does. The gate is what makes runtime-auth-sync defer the managed OAuth
// refresh instead of rotating the single-use token out from under a live session, and
// structured sessions used to be invisible to it.
describe('the managed-auth live gate', () => {
it('holds while a structured child runs and releases when it ends', async () => {
// The gate is a process-wide singleton and a sibling test's release lands on its
// child's 'close' event, which can settle after that test's close() resolved.
await until(() => (hasLiveClaudePtys() ? null : true), 'a drained auth gate')
const scenario = scriptScenario([
{ emit: { type: 'system', subtype: 'init', session_id: SESSION_ID, uuid: 'init-1' } },
{ wait: HOLD_OPEN }
])
const connection = await open(launchFor(scenario))
expect(hasLiveClaudePtys()).toBe(true)
await connection.close()
await until(() => (hasLiveClaudePtys() ? null : true), 'the auth gate to drain')
expect(hasLiveClaudePtys()).toBe(false)
}, 30_000)
it('releases when the child dies on its own rather than through close()', async () => {
await until(() => (hasLiveClaudePtys() ? null : true), 'a drained auth gate')
const scenario = scriptScenario([
{ emit: { type: 'system', subtype: 'init', session_id: SESSION_ID, uuid: 'init-1' } },
{ wait: HOLD_OPEN }
])
await open(launchFor(scenario))
expect(hasLiveClaudePtys()).toBe(true)
spawnedChildren.at(-1)?.kill('SIGKILL')
await until(() => (hasLiveClaudePtys() ? null : true), 'the auth gate to drain')
expect(hasLiveClaudePtys()).toBe(false)
}, 30_000)
// The gate entry is deliberately unpersisted, so confirmSeededClaudeLivePtys can never
// reconcile a stray one: a leak here defers the managed OAuth refresh for the life of
// the process. Entering the gate only after the release handlers are attached makes
// that unreachable regardless of what the setup in between does.
it('leaks no gate entry when setup throws between spawn and handler attachment', async () => {
await until(() => (hasLiveClaudePtys() ? null : true), 'a drained auth gate')
const scenario = scriptScenario([
{ emit: { type: 'system', subtype: 'init', session_id: SESSION_ID, uuid: 'init-1' } },
{ wait: HOLD_OPEN }
])
let started: SpawnedProcess | null = null
try {
await expect(
openClaudeStreamJsonConnection(launchFor(scenario), {}, (spec) => {
const child = spawnProcess(spec)
started = child
const attach = child.stderr.on.bind(child.stderr)
// Measured attach order: the SDK binds stderr 'data' from inside query(),
// before the child is even assigned. The SECOND bind is this connection's own
// armTreeOnOutput — the first statement that runs after the child exists and
// before its 'exit'/'close' release handlers. Throwing on the first is
// vacuous: it escapes before any gate entry could have happened.
let dataAttaches = 0
child.stderr.on = ((event: string, listener: (...args: unknown[]) => void) => {
if (event === 'data') {
dataAttaches += 1
if (dataAttaches === 2) {
throw new Error('stderr listener attach failed')
}
}
return attach(event, listener)
}) as typeof child.stderr.on
return child
})
).rejects.toThrow('stderr listener attach failed')
expect(hasLiveClaudePtys()).toBe(false)
} finally {
;(started as SpawnedProcess | null)?.kill('SIGKILL')
}
}, 30_000)
})
@@ -0,0 +1,283 @@
import { randomUUID } from 'node:crypto'
import type * as ClaudeAgentSdk from '@anthropic-ai/claude-agent-sdk'
import type { CanUseTool, OnUserDialog, SDKUserMessage } from '@anthropic-ai/claude-agent-sdk'
import { spawnProcess } from '../../shared/child-process/run-process'
import {
markClaudeStructuredChildExited,
markClaudeStructuredChildSpawned
} from '../claude-accounts/live-pty-gate'
import { buildClaudeChildProcessEnv } from './claude-child-process-environment'
import {
ClaudeControlRequestError,
createClaudeControlSurface,
type ClaudeControlSurface
} from './claude-agent-sdk-control-requests'
import { createClaudeChildTreeReaper, proveClaudeChildExit } from './claude-agent-sdk-exit-proof'
import type { DescendantTreeVerdict } from '../pty-descendant-exit-verification'
import { createClaudeCodeProcessSpawn } from './claude-agent-sdk-process-spawn'
import { createClaudeUserMessageQueue } from './claude-agent-sdk-user-message-queue'
import type { ClaudeStructuredSdkOptions } from './claude-structured-launch-resolution'
export { ClaudeControlRequestError }
/**
* The SDK is loaded at the structured-Claude boundary rather than by this module's
* import. The ordinary runtime's class graph statically reaches this file, and the
* SDK sets `process.env.NoDefaultCurrentDirectoryInExePath` at import time — a
* Windows executable-search change that a user who never leaves the terminal/TUI
* path never opted into, and a missing SDK would fail runtime startup. Memoized,
* so a session pays the import once per process rather than once per connection.
*/
let claudeAgentSdk: Promise<typeof ClaudeAgentSdk> | null = null
function loadClaudeAgentSdk(): Promise<typeof ClaudeAgentSdk> {
claudeAgentSdk ??= import('@anthropic-ai/claude-agent-sdk')
return claudeAgentSdk
}
export type ClaudeStreamJsonLaunch = {
/** Orca's resolved user CLI; the SDK falls back to a bundled binary that is not installed. */
pathToClaudeCodeExecutable: string
options: ClaudeStructuredSdkOptions
cwd: string
env?: Record<string, string>
}
export type ClaudeStreamJsonConnectionHandlers = {
onMessage?: (message: Record<string, unknown>) => void
/**
* The SDK owns inbound permission control: it hands `can_use_tool` to this callback with
* a stable requestId and an abort signal, dedups duplicate delivery, and matches the
* response by request_id itself. Setting it makes the SDK pass `--permission-prompt-tool
* stdio` automatically; it must not be paired with `permissionPromptToolName`.
*/
canUseTool?: CanUseTool
/** `request_user_dialog` control; the CLI only emits kinds declared in `supportedDialogKinds`. */
onUserDialog?: OnUserDialog
/** A transport/process fault that is not itself first-hand root exit proof. */
onFault?: (error: Error) => void
onExit?: (error: Error) => void
}
/**
* Two questions with their own evidence. The root's verdict is first-hand: Orca's
* own child handle reported exit, or reported error then close before it ever had
* a pid. The tree's comes from bounded descendant verification, and `unverifiable`
* is never collapsed into either neighbour.
*/
export type ClaudeChildExitVerdict = {
root: 'exited' | 'live' | 'processless'
tree: DescendantTreeVerdict
}
export type ClaudeStreamJsonConnection = ClaudeControlSurface & {
readonly pid: number | undefined
readonly closed: boolean
/** What the ladder has observed so far; read after a `close()` that returned false. */
readonly exitVerdict: ClaudeChildExitVerdict
send: (message: Record<string, unknown>) => Promise<void>
/** Resolves true after processless settlement, or root exit plus observed tree exit. */
close: () => Promise<boolean>
}
type ExitStatus = { code: number | null; signal: NodeJS.Signals | null }
function exitError(stderrTail: string, status: ExitStatus | null, cause?: Error): Error {
const detail = stderrTail.trim()
// The status is the diagnostic a signed-out or refused start leaves behind;
// it has to survive every wrapper between here and the user.
const how =
status?.signal !== null && status?.signal !== undefined
? ` (signal ${status.signal})`
: status?.code !== null && status?.code !== undefined
? ` (code ${status.code})`
: ''
const message = `claude stream-json exited${how}${detail ? `: ${detail}` : ''}`
return cause ? new Error(message, { cause }) : new Error(message)
}
export async function openClaudeStreamJsonConnection(
launch: ClaudeStreamJsonLaunch,
handlers: ClaudeStreamJsonConnectionHandlers = {},
spawnImpl: typeof spawnProcess = spawnProcess,
queryImpl?: typeof ClaudeAgentSdk.query
): Promise<ClaudeStreamJsonConnection> {
const { query } = await loadClaudeAgentSdk()
const spawner = createClaudeCodeProcessSpawn(spawnImpl)
const inbox = createClaudeUserMessageQueue()
const session = (queryImpl ?? query)({
prompt: inbox.messages,
options: {
...launch.options,
cwd: launch.cwd,
// Why env is never omitted: the SDK inherits process.env when it is, which is
// exactly the ambient ANTHROPIC_* auth leak this lane already shipped once.
env: buildClaudeChildProcessEnv(launch.env, { scrubConfiguredChildSessionStamps: true }),
pathToClaudeCodeExecutable: launch.pathToClaudeCodeExecutable,
spawnClaudeCodeProcess: spawner.spawn,
...(handlers.canUseTool ? { canUseTool: handlers.canUseTool } : {}),
...(handlers.onUserDialog ? { onUserDialog: handlers.onUserDialog } : {})
}
})
const child = spawner.child
if (!child) {
throw new Error('the claude agent SDK returned without spawning a child')
}
// This child owns the account's credentials for as long as it runs, exactly as a
// Claude PTY does — hold the OAuth-refresh gate so a managed refresh cannot rotate
// the single-use token out from under it mid-turn. Entered below, once a release
// path exists.
const authGateKey = randomUUID()
const releaseAuthGate = (): void => markClaudeStructuredChildExited(authGateKey)
let exited = false
let exitStatus: ExitStatus | null = null
let closing = false
let processless = false
let prePidSpawnError = false
let terminalError: Error | null = null
let faultReported = false
let exitReported = false
let closePromise: Promise<boolean> | null = null
// One reaper per child: every close attempt and error-path reap shares its proof.
const rootSettled = (): boolean => exited || processless
const tree = createClaudeChildTreeReaper(child, { exited: rootSettled })
// Arm lazily on actual child output instead of issuing a process-table scan for
// every session at startup. A natural SDK exit can race a later close, while
// output-triggered observation still catches the usual live-child window.
let outputObservationArmed = false
const armTreeOnOutput = (): void => {
if (outputObservationArmed) {
return
}
outputObservationArmed = true
void (tree.refresh?.() ?? tree.capture())
}
child.stderr.on('data', armTreeOnOutput)
// The SDK may synchronously spawn the CLI and consume an early stderr chunk
// before this connection can attach its listener; the bounded tail preserves
// that observation for the same lazy arm.
if (spawner.stderrTail.length > 0) {
armTreeOnOutput()
}
let settleExit = (): void => {}
const exitPromise = new Promise<void>((resolve) => {
settleExit = resolve
})
const markExited = (): void => {
exited = true
releaseAuthGate()
settleExit()
}
child.on('exit', (code, signal) => {
exitStatus = { code, signal }
markExited()
handleUnexpectedEnd()
})
const handleUnexpectedEnd = (cause?: Error): void => {
terminalError ??= exitError(spawner.stderrTail, exitStatus, cause)
inbox.fail(terminalError)
if (!closing && !faultReported) {
faultReported = true
handlers.onFault?.(terminalError)
}
if (!closing && exited && !exitReported) {
exitReported = true
handlers.onExit?.(terminalError)
}
}
void (async () => {
for await (const message of session) {
handlers.onMessage?.(message as unknown as Record<string, unknown>)
}
})().catch((error: unknown) => {
// The SDK ends its generator in error when the child dies or the transport
// fails; a transport failure with a live child still has to reap the tree.
if (!closing && !exited) {
void tree.reap()
}
handleUnexpectedEnd(error instanceof Error ? error : new Error(String(error)))
})
child.on('error', (error) => {
if (spawner.pid === undefined) {
prePidSpawnError = true
}
if (!closing && !exited) {
void tree.reap()
}
handleUnexpectedEnd(error)
})
child.on('close', () => {
// Covers the spawn-failure path too, where no 'exit' ever arrives.
releaseAuthGate()
if (prePidSpawnError && spawner.pid === undefined) {
processless = true
settleExit()
}
handleUnexpectedEnd()
})
child.stdin.on('error', (error) => {
if (!closing) {
void tree.reap()
handleUnexpectedEnd(error)
}
})
// Why here and not at spawn: a structured gate entry is deliberately unpersisted, so
// confirmSeededClaudeLivePtys can never reconcile a stray one and a leak defers the
// managed OAuth refresh for the life of the process. Entering only after 'exit' and
// 'close' are attached makes that unreachable — any later throw still leaves a
// listener that releases. Nothing between spawn and here can yield, so the child
// cannot end before the gate is entered.
markClaudeStructuredChildSpawned(authGateKey)
const send = (message: Record<string, unknown>): Promise<void> => {
if (closing || exited || terminalError || child.stdin.destroyed || !child.stdin.writable) {
return Promise.reject(terminalError ?? new Error('claude stream-json connection is closed'))
}
return inbox.push(message as unknown as SDKUserMessage)
}
const close = (): Promise<boolean> => {
closePromise ??= (async () => {
closing = true
// Arm the descendant proof before ending stdin. The SDK may exit the root
// immediately; a post-exit walk cannot recover descendants that reparented.
await (tree.refresh?.() ?? tree.capture())
inbox.end()
const proven = await proveClaudeChildExit({
child,
exitPromise,
exited: rootSettled,
tree
})
inbox.fail(new Error('claude stream-json connection closed'))
if (!proven) {
closePromise = null
}
return proven
})()
return closePromise
}
return {
...createClaudeControlSurface(session),
get pid() {
return spawner.pid
},
get closed() {
return closing || exited || terminalError !== null
},
get exitVerdict() {
return {
root: processless ? 'processless' : exited ? 'exited' : 'live',
tree: tree.treeVerdict
} as const
},
send,
close
}
}
@@ -0,0 +1,110 @@
import type { AgentJournalItemIdentity } from '../../shared/agent-session-journal-types'
import { claudeRecord, claudeText } from './claude-structured-item-translation'
// Under --include-partial-messages every stream_event frame carries its own
// uuid, and the block's final `assistant` frame carries yet another; only
// `message.id` ties them together. The block's first stream frame mints the
// journal identity, and the final frame lands on it in block order instead of
// appending a duplicate under its own uuid.
export type ClaudeStreamedTextDelta = { identity: AgentJournalItemIdentity; text: string }
type StreamedMessage = {
messageId: string | null
blocks: Map<number, AgentJournalItemIdentity>
/** Streamed text blocks whose final assistant frame has not arrived, in block order. */
awaitingFinal: AgentJournalItemIdentity[]
}
export type ClaudeStreamedBlockRegistry = {
/** Text a stream_event frame appends to its block, or null when it carries none. */
observe: (frame: Record<string, unknown>) => ClaudeStreamedTextDelta | null
/** The streamed identity a final assistant frame reconciles onto, if its block streamed. */
reconcile: (frame: {
sessionId: string
parentToolUseId: string | null
messageId: string | null
}) => AgentJournalItemIdentity | null
clear: () => void
}
function scopeKey(sessionId: string, parentToolUseId: string | null): string {
return `${sessionId}/${parentToolUseId ?? ''}`
}
export function createClaudeStreamedBlockRegistry(): ClaudeStreamedBlockRegistry {
const messages = new Map<string, StreamedMessage>()
const messageFor = (scope: string): StreamedMessage => {
let streamed = messages.get(scope)
if (!streamed) {
streamed = { messageId: null, blocks: new Map(), awaitingFinal: [] }
messages.set(scope, streamed)
}
return streamed
}
const mint = (
streamed: StreamedMessage,
sessionId: string,
index: number,
uuid: string
): AgentJournalItemIdentity => {
const identity: AgentJournalItemIdentity = { provider: 'claude', sessionId, uuid }
streamed.blocks.set(index, identity)
streamed.awaitingFinal.push(identity)
return identity
}
return {
observe: (frame) => {
const event = claudeRecord(frame.event)
const sessionId = claudeText(frame.session_id)
const uuid = claudeText(frame.uuid)
if (frame.type !== 'stream_event' || !event || !sessionId || !uuid) {
return null
}
const scope = scopeKey(sessionId, claudeText(frame.parent_tool_use_id))
if (event.type === 'message_start') {
messages.set(scope, {
messageId: claudeText(claudeRecord(event.message)?.id),
blocks: new Map(),
awaitingFinal: []
})
return null
}
const index = typeof event.index === 'number' ? event.index : 0
if (event.type === 'content_block_start') {
const block = claudeRecord(event.content_block)
if (block?.type !== 'text') {
return null
}
const identity = mint(messageFor(scope), sessionId, index, uuid)
const text = claudeText(block.text)
return text ? { identity, text } : null
}
if (event.type !== 'content_block_delta') {
return null
}
const delta = claudeRecord(event.delta)
const text = delta?.type === 'text_delta' ? claudeText(delta.text) : null
if (!text) {
return null
}
const streamed = messageFor(scope)
const identity = streamed.blocks.get(index) ?? mint(streamed, sessionId, index, uuid)
return { identity, text }
},
reconcile: (frame) => {
const streamed = messages.get(scopeKey(frame.sessionId, frame.parentToolUseId))
if (
!streamed ||
(frame.messageId && streamed.messageId && frame.messageId !== streamed.messageId)
) {
return null
}
return streamed.awaitingFinal.shift() ?? null
},
clear: () => messages.clear()
}
}
@@ -0,0 +1,93 @@
import { describe, expect, it } from 'vitest'
import type { AgentJournalItemIdentity } from '../../shared/agent-session-journal-types'
import { createClaudeStreamedTextCheckpoints } from './claude-streamed-text-checkpoints'
function identityOf(uuid: string): AgentJournalItemIdentity {
return { provider: 'claude', sessionId: 'claude-session', uuid }
}
function checkpoints() {
const rows: { uuid: string; text: string }[] = []
let scheduled: (() => void) | null = null
const store = createClaudeStreamedTextCheckpoints({
persist: (identity, text) => {
rows.push({ uuid: 'uuid' in identity ? identity.uuid : '', text })
},
schedule: (run) => {
scheduled = run
return () => {
scheduled = null
}
}
})
return {
store,
rows,
runWindow: () => {
const run = scheduled as (() => void) | null
run?.()
}
}
}
describe('claude streamed text checkpoints', () => {
it('rewrites a block row with the full text accumulated so far', () => {
const { store, rows, runWindow } = checkpoints()
store.append(identityOf('block-1'), 'hel')
store.append(identityOf('block-1'), 'lo')
runWindow()
expect(rows).toEqual([{ uuid: 'block-1', text: 'hello' }])
expect(store.pending).toBe(1)
})
it('drops every block still awaiting its final frame at settlement', () => {
const { store, rows, runWindow } = checkpoints()
store.append(identityOf('block-1'), 'partial answer')
runWindow()
store.settle()
expect(store.pending).toBe(0)
// The row written before settlement stays; nothing is rewritten afterwards.
store.flush()
expect(rows).toEqual([{ uuid: 'block-1', text: 'partial answer' }])
})
it('keeps a block whose final frame arrived out of the settlement sweep', () => {
const { store } = checkpoints()
store.append(identityOf('block-1'), 'one')
store.append(identityOf('block-2'), 'two')
store.forget('claude:claude-session:block-1')
expect(store.pending).toBe(1)
store.settle()
expect(store.pending).toBe(0)
})
it('flushes text the widening checkpoint interval has not written yet', () => {
const { store, rows } = checkpoints()
store.append(identityOf('block-1'), 'x')
store.flush()
expect(rows).toEqual([{ uuid: 'block-1', text: 'x' }])
// Already at the row's length: a second flush has nothing to write.
store.flush()
expect(rows).toHaveLength(1)
})
it('stops persisting once disposed', () => {
const { store, rows, runWindow } = checkpoints()
store.append(identityOf('block-1'), 'text')
store.dispose()
runWindow()
store.flush()
expect(rows).toEqual([])
expect(store.pending).toBe(0)
})
})
@@ -0,0 +1,105 @@
import type { AgentJournalItemIdentity } from '../../shared/agent-session-journal-types'
import { agentJournalItemKey } from '../../shared/agent-session-journal-item-key'
import {
createAgentSessionDeltaCoalescer,
type AgentSessionDeltaCoalescerDeps
} from '../native-chat/agent-session-wire/agent-session-delta-coalescer'
export type ClaudeStreamedTextCheckpointDeps = {
/** Rewrites the block's journal row with the text accumulated so far. */
persist: (identity: AgentJournalItemIdentity, text: string) => void
coalesceMs?: number
schedule?: AgentSessionDeltaCoalescerDeps['schedule']
}
export type ClaudeStreamedTextCheckpoints = {
/** Accumulate a delta; the row is rewritten on the coalescer's own cadence. */
append: (identity: AgentJournalItemIdentity, text: string) => void
/** Write every block whose row is behind the text received for it. */
flush: () => void
/** Drop one block's state, for a block whose final frame has now landed. */
forget: (key: string) => void
/**
* Drop every block still awaiting its final frame, at turn settlement. Their
* text is already journaled by the flush that precedes settlement; keeping it
* live would grow with every interrupted turn for the life of the session.
*/
settle: () => void
/** Blocks still awaiting a final frame. A settled turn must leave none. */
readonly pending: number
dispose: () => void
}
/**
* Growth of a streamed block's row between its deltas and its final frame.
*
* The row is rewritten on a widening interval rather than per delta: a 200-line
* reply would otherwise rewrite the same journal row once per token.
*/
export function createClaudeStreamedTextCheckpoints(
deps: ClaudeStreamedTextCheckpointDeps
): ClaudeStreamedTextCheckpoints {
const identities = new Map<string, AgentJournalItemIdentity>()
const latestText = new Map<string, string>()
const checkpointLengths = new Map<string, number>()
const persist = (key: string, text: string, force: boolean): void => {
latestText.set(key, text)
const checkpointLength = checkpointLengths.get(key) ?? 0
const nextLength = Math.max(checkpointLength + 32, Math.ceil(checkpointLength * 1.125))
if (!force && checkpointLength > 0 && text.length < nextLength) {
return
}
const identity = identities.get(key)
if (!identity) {
return
}
checkpointLengths.set(key, text.length)
deps.persist(identity, text)
}
const coalescer = createAgentSessionDeltaCoalescer({
...(deps.coalesceMs === undefined ? {} : { windowMs: deps.coalesceMs }),
...(deps.schedule ? { schedule: deps.schedule } : {}),
emit: (key, text) => persist(key, text, false)
})
const drop = (key: string): void => {
coalescer.forget(key)
identities.delete(key)
latestText.delete(key)
checkpointLengths.delete(key)
}
return {
append: (identity, text) => {
const key = agentJournalItemKey(identity)
identities.set(key, identity)
coalescer.append(key, text)
},
flush: () => {
coalescer.flushAll()
for (const [key, text] of latestText) {
if (checkpointLengths.get(key) !== text.length) {
persist(key, text, true)
}
}
},
forget: drop,
settle: () => {
// Map iteration tolerates deletion of the entry just visited.
for (const key of identities.keys()) {
drop(key)
}
},
get pending() {
return identities.size
},
dispose: () => {
coalescer.dispose()
identities.clear()
latestText.clear()
checkpointLengths.clear()
}
}
}
@@ -0,0 +1,43 @@
import {
closeClaudeSession,
claudeAcquisitionCleanupError
} from './claude-structured-session-close'
import type {
ClaudeAcquisitionRegistry,
ClaudeSession,
ClaudeSessionExit,
ClaudeStructuredSessionAdapterDeps
} from './claude-structured-session-state'
/**
* Cleanup for an acquisition the host could not commit or prove. A session that
* a first-hand exit already removed is not an absence to report as proven: the
* ladder on its connection still answers, and that answer is classified exactly
* as a start-time failure would be.
*/
export async function releaseClaudeAcquisition(input: {
sessionId: string
sessions: Map<string, ClaudeSession>
acquisitions: ClaudeAcquisitionRegistry
exits: Map<string, ClaudeSessionExit>
onExitProven?: (sessionId: string, exit: ClaudeSessionExit) => Promise<void>
persistHandle?: ClaudeStructuredSessionAdapterDeps['persistHandle']
onEvent?: ClaudeStructuredSessionAdapterDeps['onEvent']
}): Promise<boolean> {
const exit = input.exits.get(input.sessionId)
if (!exit || input.sessions.has(input.sessionId) || input.acquisitions.get(input.sessionId)) {
return closeClaudeSession(input)
}
const firstProof = exit.closePromise ? await exit.closePromise : false
// A failed exit-path proof is retained as evidence, not as a terminal result;
// a release retry must drive a fresh tree verification on the same connection.
const retriedProof = firstProof || (await exit.connection.close())
if (retriedProof) {
await input.onExitProven?.(input.sessionId, exit)
// Keep the first-hand exit evidence indexed until the tree proof succeeds;
// a failed close must be retryable and cannot look like an absent session.
input.exits.delete(input.sessionId)
return true
}
throw claudeAcquisitionCleanupError(exit.connection, exit.error)
}

Some files were not shown because too many files have changed in this diff Show More