mirror of
https://github.com/stablyai/orca.git
synced 2026-09-29 16:02:50 +00:00
Merge remote-tracking branch 'origin/main' into brennanb2025/chatui-2-monitor-status
# Conflicts: # src/renderer/src/components/native-chat/NativeChatResolvedView.tsx # src/renderer/src/components/native-chat/NativeChatStructuredSession.tsx # src/renderer/src/i18n/locales/en.json
This commit is contained in:
@@ -91,7 +91,11 @@ jobs:
|
||||
test -n "${CAPACITY_SERVICE_ACCOUNT}"
|
||||
test -n "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}"
|
||||
|
||||
# Full history: the monitor evidence this job verifies is sealed at an ancestor commit,
|
||||
# and the provenance check fails closed on a commit a shallow clone left out.
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: pnpm/action-setup@v4
|
||||
with: { package_json_file: cloud/package.json }
|
||||
|
||||
@@ -87,13 +87,18 @@ jobs:
|
||||
gate:
|
||||
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }}
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 10
|
||||
# Headroom for the full-history checkout the canary provenance check needs.
|
||||
timeout-minutes: 15
|
||||
environment: production
|
||||
outputs:
|
||||
cells: ${{ steps.wave.outputs.cells }}
|
||||
job-mode: ${{ steps.wave.outputs.job-mode }}
|
||||
steps:
|
||||
# Full history: the canary authority a batch verifies is sealed at an ancestor commit, and
|
||||
# the provenance check fails closed on a commit a shallow clone left out.
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with: { node-version: 24 }
|
||||
|
||||
@@ -95,7 +95,11 @@ jobs:
|
||||
;;
|
||||
esac
|
||||
|
||||
# Full history: the monitor evidence this job verifies is sealed at an ancestor commit,
|
||||
# and the provenance check fails closed on a commit a shallow clone left out.
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
|
||||
@@ -807,6 +807,7 @@ jobs:
|
||||
src/main/agent-hooks/windows-hook-payload-delivery.test.ts
|
||||
src/main/windows/windows-pty-job.win32.test.ts
|
||||
src/main/windows/windows-host-job.win32.test.ts
|
||||
src/main/windows-live-tree-kill.win32.test.ts
|
||||
src/main/wsl/wsl-runner.test.ts
|
||||
src/main/wsl/wsl-guest-environment.test.ts
|
||||
src/main/wsl/wsl-invocation-boundary.test.ts
|
||||
|
||||
@@ -13,6 +13,41 @@ const runService = {
|
||||
latestReadyRevision: 'projects/project/revisions/revision-one'
|
||||
}
|
||||
|
||||
const sleepingStagingGcloud: GcloudClient = { accessToken: async () => 'a'.repeat(40) }
|
||||
|
||||
// Staging's Cloud SQL is stopped, so this inventory reads REST only and probes no endpoint.
|
||||
type MigOutcome = 'ok' | 'throw' | 'missing'
|
||||
const sleepingStagingFetch = (migOutcome: (migName: string) => MigOutcome): typeof fetch =>
|
||||
async (input) => {
|
||||
const url = new URL(String(input))
|
||||
if (url.hostname === 'run.googleapis.com') return Response.json(runService)
|
||||
if (url.hostname === 'sqladmin.googleapis.com') return Response.json({
|
||||
state: 'STOPPED',
|
||||
databaseVersion: 'POSTGRES_17',
|
||||
settings: { activationPolicy: 'NEVER', availabilityType: 'ZONAL', tier: 'db-custom-1-3840' }
|
||||
})
|
||||
if (url.hostname === 'certificatemanager.googleapis.com') return Response.json({
|
||||
managed: { domains: ['*.relay-staging.onorca.dev'], state: 'ACTIVE' }
|
||||
})
|
||||
if (url.pathname.includes('/instanceGroupManagers/')) {
|
||||
const name = url.pathname.split('/').at(-1)!
|
||||
const outcome = migOutcome(name)
|
||||
if (outcome === 'throw') throw new TypeError('fetch failed')
|
||||
if (outcome === 'missing') return new Response(null, { status: 404 })
|
||||
return Response.json({
|
||||
name,
|
||||
targetSize: 0,
|
||||
size: '0',
|
||||
instanceGroup: `projects/project/zones/zone/instanceGroups/${name}`,
|
||||
instanceTemplate: `projects/project/global/instanceTemplates/template-${name}`,
|
||||
status: { isStable: true }
|
||||
})
|
||||
}
|
||||
if (url.pathname.includes('/instanceTemplates/')) return Response.json({ properties: {} })
|
||||
if (url.pathname.endsWith('/getHealth')) return Response.json([])
|
||||
throw new Error(`Unexpected request to ${url.hostname}${url.pathname}`)
|
||||
}
|
||||
|
||||
describe('readResourceInventory', () => {
|
||||
it('does not delay a healthy endpoint sample', async () => {
|
||||
let calls = 0
|
||||
@@ -23,8 +58,10 @@ describe('readResourceInventory', () => {
|
||||
calls += 1
|
||||
return new Response(null, { status: 200 })
|
||||
},
|
||||
async () => {
|
||||
waits += 1
|
||||
{
|
||||
wait: async () => {
|
||||
waits += 1
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
@@ -45,8 +82,10 @@ describe('readResourceInventory', () => {
|
||||
calls.set(path, call)
|
||||
return new Response(null, { status: path === '/ready' && call === 1 ? 503 : 200 })
|
||||
},
|
||||
async (ms) => {
|
||||
waits.push(ms)
|
||||
{
|
||||
wait: async (ms) => {
|
||||
waits.push(ms)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
@@ -65,17 +104,130 @@ describe('readResourceInventory', () => {
|
||||
calls += 1
|
||||
return new Response(null, { status: 503 })
|
||||
},
|
||||
async (ms) => {
|
||||
waits.push(ms)
|
||||
{
|
||||
wait: async (ms) => {
|
||||
waits.push(ms)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
expect(result.health).toBe(false)
|
||||
expect(result.ready).toBe(false)
|
||||
expect(calls).toBe(4)
|
||||
// A refusing endpoint is a reading, so only the independent retry runs.
|
||||
expect(waits).toEqual([11_000])
|
||||
})
|
||||
|
||||
it('treats a thrown fetch as no reading and re-asks that path once', async () => {
|
||||
const calls: string[] = []
|
||||
const waits: number[] = []
|
||||
const result = await probeEndpointHealth(
|
||||
'https://c9.relay.onorca.dev',
|
||||
async (input) => {
|
||||
const path = new URL(String(input)).pathname
|
||||
calls.push(path)
|
||||
if (path === '/health' && calls.filter((call) => call === '/health').length === 1) {
|
||||
throw new TypeError('fetch failed')
|
||||
}
|
||||
return new Response(null, { status: 200 })
|
||||
},
|
||||
{
|
||||
wait: async (ms) => {
|
||||
waits.push(ms)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
expect(result.health).toBe(true)
|
||||
expect(result.ready).toBe(true)
|
||||
expect(calls.filter((call) => call === '/health')).toEqual(['/health', '/health'])
|
||||
expect(waits).toEqual([1_000])
|
||||
})
|
||||
|
||||
it('fails closed when both attempts of a path throw', async () => {
|
||||
const calls: string[] = []
|
||||
const waits: number[] = []
|
||||
const result = await probeEndpointHealth(
|
||||
'https://c9.relay.onorca.dev',
|
||||
async (input) => {
|
||||
const path = new URL(String(input)).pathname
|
||||
calls.push(path)
|
||||
if (path === '/health') throw new TypeError('fetch failed')
|
||||
return new Response(null, { status: 200 })
|
||||
},
|
||||
{
|
||||
wait: async (ms) => {
|
||||
waits.push(ms)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
expect(result.health).toBe(false)
|
||||
expect(calls.filter((call) => call === '/health')).toHaveLength(4)
|
||||
expect(waits).toEqual([1_000, 11_000, 1_000])
|
||||
})
|
||||
|
||||
it('accepts an auth-shaped endpoint that serves no readiness path', async () => {
|
||||
const calls: string[] = []
|
||||
const waits: number[] = []
|
||||
const result = await probeEndpointHealth(
|
||||
'https://login.onorca.dev',
|
||||
async (input) => {
|
||||
const path = new URL(String(input)).pathname
|
||||
calls.push(path)
|
||||
return new Response(null, { status: path === '/ready' ? 404 : 200 })
|
||||
},
|
||||
{
|
||||
requiresReady: false,
|
||||
wait: async (ms) => {
|
||||
waits.push(ms)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
expect(result.health).toBe(true)
|
||||
expect(result.ready).toBeNull()
|
||||
expect(calls).toEqual(['/health'])
|
||||
expect(waits).toEqual([])
|
||||
})
|
||||
|
||||
it('still requires readiness for the director and cells', async () => {
|
||||
const waits: number[] = []
|
||||
const result = await probeEndpointHealth(
|
||||
'https://relay.onorca.dev',
|
||||
async (input) => new Response(null, {
|
||||
status: new URL(String(input)).pathname === '/ready' ? 503 : 200
|
||||
}),
|
||||
{
|
||||
wait: async (ms) => {
|
||||
waits.push(ms)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
expect(result.health).toBe(true)
|
||||
expect(result.ready).toBe(false)
|
||||
expect(waits).toEqual([11_000])
|
||||
})
|
||||
|
||||
it('measures latency as the answering round trip, not the retry delay', async () => {
|
||||
let healthCalls = 0
|
||||
const result = await probeEndpointHealth(
|
||||
'https://c9.relay.onorca.dev',
|
||||
async (input) => {
|
||||
if (new URL(String(input)).pathname !== '/health') return new Response(null, { status: 200 })
|
||||
healthCalls += 1
|
||||
if (healthCalls === 1) throw new TypeError('fetch failed')
|
||||
return new Response(null, { status: 200 })
|
||||
},
|
||||
{ wait: async (ms) => await new Promise((resolve) => setTimeout(resolve, Math.min(ms, 60))) }
|
||||
)
|
||||
|
||||
expect(result.health).toBe(true)
|
||||
expect(result.latencyMs).not.toBeNull()
|
||||
expect(result.latencyMs!).toBeLessThan(60)
|
||||
})
|
||||
|
||||
it('uses aggregate REST inventory without probing sleeping staging endpoints', async () => {
|
||||
const gcloud: GcloudClient = { accessToken: async () => 'a'.repeat(40) }
|
||||
let publicProbeCalls = 0
|
||||
@@ -132,6 +284,74 @@ describe('readResourceInventory', () => {
|
||||
expect(JSON.stringify(result)).not.toContain('SECRET_TEXT')
|
||||
})
|
||||
|
||||
it('re-asks a MIG read that failed once before calling a cell powered-unknown', async () => {
|
||||
const parkedCell = RELAY_OPS_ENVIRONMENTS.staging.cells[0]!
|
||||
const waits: number[] = []
|
||||
let parkedMigCalls = 0
|
||||
const result = await readResourceInventory(
|
||||
RELAY_OPS_ENVIRONMENTS.staging,
|
||||
sleepingStagingGcloud,
|
||||
sleepingStagingFetch((migName) => {
|
||||
if (!migName.endsWith(parkedCell.hostname)) return 'ok'
|
||||
parkedMigCalls += 1
|
||||
return parkedMigCalls === 1 ? 'throw' : 'ok'
|
||||
}),
|
||||
{ wait: async (ms) => { waits.push(ms) } }
|
||||
)
|
||||
|
||||
const parked = result.cells.find((cell) => cell.cellId === parkedCell.cellId)!
|
||||
// The MIG was fine and parked at zero; one transient read must not erase that reading.
|
||||
expect(parked.targetSize).toBe(0)
|
||||
expect(parkedMigCalls).toBe(2)
|
||||
expect(waits).toEqual([1_000])
|
||||
expect(result.warnings).toEqual([])
|
||||
})
|
||||
|
||||
it('reports a MIG unavailable only when the retry fails too', async () => {
|
||||
const parkedCell = RELAY_OPS_ENVIRONMENTS.staging.cells[0]!
|
||||
const waits: number[] = []
|
||||
let parkedMigCalls = 0
|
||||
const result = await readResourceInventory(
|
||||
RELAY_OPS_ENVIRONMENTS.staging,
|
||||
sleepingStagingGcloud,
|
||||
sleepingStagingFetch((migName) => {
|
||||
if (!migName.endsWith(parkedCell.hostname)) return 'ok'
|
||||
parkedMigCalls += 1
|
||||
return 'throw'
|
||||
}),
|
||||
{ wait: async (ms) => { waits.push(ms) } }
|
||||
)
|
||||
|
||||
const parked = result.cells.find((cell) => cell.cellId === parkedCell.cellId)!
|
||||
expect(parked.targetSize).toBeNull()
|
||||
expect(parked.backendHealth).toBe('unknown')
|
||||
expect(parkedMigCalls).toBe(2)
|
||||
expect(waits).toEqual([1_000])
|
||||
expect(result.warnings).toEqual([
|
||||
`${parkedCell.hostname.toUpperCase()} MIG inventory is unavailable.`
|
||||
])
|
||||
})
|
||||
|
||||
it('does not re-ask a MIG read the API answered with 404', async () => {
|
||||
const missingCell = RELAY_OPS_ENVIRONMENTS.staging.cells[0]!
|
||||
const waits: number[] = []
|
||||
let missingMigCalls = 0
|
||||
const result = await readResourceInventory(
|
||||
RELAY_OPS_ENVIRONMENTS.staging,
|
||||
sleepingStagingGcloud,
|
||||
sleepingStagingFetch((migName) => {
|
||||
if (!migName.endsWith(missingCell.hostname)) return 'ok'
|
||||
missingMigCalls += 1
|
||||
return 'missing'
|
||||
}),
|
||||
{ wait: async (ms) => { waits.push(ms) } }
|
||||
)
|
||||
|
||||
expect(result.cells.find((cell) => cell.cellId === missingCell.cellId)!.targetSize).toBeNull()
|
||||
expect(missingMigCalls).toBe(1)
|
||||
expect(waits).toEqual([])
|
||||
})
|
||||
|
||||
it('represents missing credentials as unknown inventory, never sleeping', async () => {
|
||||
const gcloud: GcloudClient = {
|
||||
accessToken: async () => { throw new Error('sensitive context') }
|
||||
|
||||
@@ -102,6 +102,9 @@ export type ResourceInventory = {
|
||||
|
||||
const unavailableEndpoint = (): EndpointHealth => ({ health: null, ready: null, latencyMs: null })
|
||||
const independentEndpointRetryDelayMs = 11_000
|
||||
const transientProbeRetryDelayMs = 1_000
|
||||
const sleep = async (ms: number): Promise<void> =>
|
||||
await new Promise((resolvePromise) => setTimeout(resolvePromise, ms))
|
||||
|
||||
function finalSegment(value: string): string {
|
||||
return value.split('/').at(-1) ?? value
|
||||
@@ -120,6 +123,12 @@ function parseService(value: unknown): ServiceInventory {
|
||||
}
|
||||
}
|
||||
|
||||
class GoogleApiError extends Error {
|
||||
constructor(readonly status: number) {
|
||||
super(`Google API returned ${status}`)
|
||||
}
|
||||
}
|
||||
|
||||
async function googleRequest(
|
||||
fetchImpl: typeof fetch,
|
||||
token: string,
|
||||
@@ -134,45 +143,97 @@ async function googleRequest(
|
||||
},
|
||||
signal: AbortSignal.timeout(30_000)
|
||||
})
|
||||
if (!response.ok) throw new Error(`Google API returned ${response.status}`)
|
||||
if (!response.ok) throw new GoogleApiError(response.status)
|
||||
return await response.json()
|
||||
}
|
||||
|
||||
async function endpointProbe(origin: string, fetchImpl: typeof fetch): Promise<EndpointHealth> {
|
||||
const startedAt = performance.now()
|
||||
const check = async (path: '/health' | '/ready'): Promise<boolean> => {
|
||||
// A 404 is the API's answer about the resource; anything else is the absence of a reading, so re-ask.
|
||||
async function readOnceMore(
|
||||
read: () => Promise<unknown>,
|
||||
wait: (ms: number) => Promise<void>
|
||||
): Promise<unknown> {
|
||||
try {
|
||||
return await read()
|
||||
} catch (error) {
|
||||
if (error instanceof GoogleApiError && error.status === 404) throw error
|
||||
await wait(transientProbeRetryDelayMs)
|
||||
return await read()
|
||||
}
|
||||
}
|
||||
|
||||
// A reading the endpoint actually produced: ok is its answer, latencyMs is that answer's round trip.
|
||||
type PathReading = { ok: boolean; latencyMs: number | null }
|
||||
|
||||
async function probePath(
|
||||
origin: string,
|
||||
path: '/health' | '/ready',
|
||||
fetchImpl: typeof fetch,
|
||||
wait: (ms: number) => Promise<void>
|
||||
): Promise<PathReading> {
|
||||
// null means the request never produced an answer (DNS/TCP/TLS failure or the 8s abort).
|
||||
const attempt = async (): Promise<PathReading | null> => {
|
||||
const startedAt = performance.now()
|
||||
try {
|
||||
const response = await fetchImpl(`${origin}${path}`, {
|
||||
redirect: 'error',
|
||||
signal: AbortSignal.timeout(8_000)
|
||||
})
|
||||
return response.ok
|
||||
return { ok: response.ok, latencyMs: Math.round(performance.now() - startedAt) }
|
||||
} catch {
|
||||
return false
|
||||
return null
|
||||
}
|
||||
}
|
||||
const [health, ready] = await Promise.all([check('/health'), check('/ready')])
|
||||
return { health, ready, latencyMs: Math.round(performance.now() - startedAt) }
|
||||
const first = await attempt()
|
||||
if (first) return first
|
||||
// A thrown fetch is the absence of a reading, not an unhealthy answer, so re-ask before concluding.
|
||||
await wait(transientProbeRetryDelayMs)
|
||||
return (await attempt()) ?? { ok: false, latencyMs: null }
|
||||
}
|
||||
|
||||
async function endpointProbe(
|
||||
origin: string,
|
||||
fetchImpl: typeof fetch,
|
||||
requiresReady: boolean,
|
||||
wait: (ms: number) => Promise<void>
|
||||
): Promise<EndpointHealth> {
|
||||
const [health, ready] = await Promise.all([
|
||||
probePath(origin, '/health', fetchImpl, wait),
|
||||
requiresReady ? probePath(origin, '/ready', fetchImpl, wait) : null
|
||||
])
|
||||
// Latency is the slowest answering round trip in this probe; retry delays are not serving latency.
|
||||
const latencies = [health.latencyMs, ready?.latencyMs ?? null].filter(
|
||||
(value): value is number => value !== null
|
||||
)
|
||||
return {
|
||||
health: health.ok,
|
||||
ready: ready ? ready.ok : null,
|
||||
latencyMs: latencies.length > 0 ? Math.max(...latencies) : null
|
||||
}
|
||||
}
|
||||
|
||||
export type EndpointProbeOptions = {
|
||||
// Auth serves no /ready by design, so it is judged on /health and latency alone.
|
||||
requiresReady?: boolean
|
||||
wait?: (ms: number) => Promise<void>
|
||||
}
|
||||
|
||||
export async function probeEndpointHealth(
|
||||
origin: string,
|
||||
fetchImpl: typeof fetch,
|
||||
wait: (ms: number) => Promise<void> = async (ms) =>
|
||||
await new Promise((resolvePromise) => setTimeout(resolvePromise, ms))
|
||||
options: EndpointProbeOptions = {}
|
||||
): Promise<EndpointHealth> {
|
||||
const first = await endpointProbe(origin, fetchImpl)
|
||||
if (
|
||||
first.health &&
|
||||
first.ready &&
|
||||
first.latencyMs !== null &&
|
||||
first.latencyMs <= INCIDENT_MONITOR_THRESHOLDS.endpointLatencyMs
|
||||
) {
|
||||
return first
|
||||
}
|
||||
const requiresReady = options.requiresReady ?? true
|
||||
const wait = options.wait ?? sleep
|
||||
const accepted = (probe: EndpointHealth): boolean =>
|
||||
probe.health === true &&
|
||||
(!requiresReady || probe.ready === true) &&
|
||||
probe.latencyMs !== null &&
|
||||
probe.latencyMs <= INCIDENT_MONITOR_THRESHOLDS.endpointLatencyMs
|
||||
const first = await endpointProbe(origin, fetchImpl, requiresReady, wait)
|
||||
if (accepted(first)) return first
|
||||
// Outwait Relay's ten-second readiness cache before treating the retry as independent.
|
||||
await wait(independentEndpointRetryDelayMs)
|
||||
return await endpointProbe(origin, fetchImpl)
|
||||
return await endpointProbe(origin, fetchImpl, requiresReady, wait)
|
||||
}
|
||||
|
||||
function imageDigest(template: z.infer<typeof TemplateSchema>): string | null {
|
||||
@@ -285,11 +346,17 @@ function unavailableInventory(environment: RelayOpsEnvironment, warning: string)
|
||||
}
|
||||
}
|
||||
|
||||
export type ResourceInventoryOptions = {
|
||||
wait?: (ms: number) => Promise<void>
|
||||
}
|
||||
|
||||
export async function readResourceInventory(
|
||||
environment: RelayOpsEnvironment,
|
||||
gcloud: GcloudClient,
|
||||
fetchImpl: typeof fetch = fetch
|
||||
fetchImpl: typeof fetch = fetch,
|
||||
options: ResourceInventoryOptions = {}
|
||||
): Promise<ResourceInventory> {
|
||||
const wait = options.wait ?? sleep
|
||||
let token: string
|
||||
try {
|
||||
token = await gcloud.accessToken()
|
||||
@@ -316,7 +383,10 @@ export async function readResourceInventory(
|
||||
token,
|
||||
`https://certificatemanager.googleapis.com/v1/projects/${environment.project}/locations/global/certificates/${environment.certificateName}`
|
||||
),
|
||||
...environment.cells.map((cell) => googleRequest(fetchImpl, token, migUrl(cell)))
|
||||
// One transient Compute read must never become a verdict on a cell's power state.
|
||||
...environment.cells.map((cell) =>
|
||||
readOnceMore(async () => await googleRequest(fetchImpl, token, migUrl(cell)), wait)
|
||||
)
|
||||
])
|
||||
const warnings: string[] = []
|
||||
const directorValue = parsed(settled[0]!, RunServiceSchema, 'Director service inventory is unavailable.', warnings)
|
||||
@@ -338,7 +408,8 @@ export async function readResourceInventory(
|
||||
? [unavailableEndpoint(), unavailableEndpoint()]
|
||||
: await Promise.all([
|
||||
probeEndpointHealth(environment.directorOrigin, fetchImpl),
|
||||
probeEndpointHealth(environment.authOrigin, fetchImpl)
|
||||
// The auth service exposes no /ready, so requiring it would fail every first probe.
|
||||
probeEndpointHealth(environment.authOrigin, fetchImpl, { requiresReady: false })
|
||||
])
|
||||
const cells = await Promise.all(environment.cells.map((cell, index) =>
|
||||
readCell(environment, cell, migValues[index] ?? null, token, fetchImpl)
|
||||
|
||||
@@ -645,9 +645,17 @@ export class RelayAssignmentStore {
|
||||
): Promise<RelayAssignment | null> {
|
||||
const now = this.now()
|
||||
return await this.database.transaction(async (transaction) => {
|
||||
const lockedCells = inventoryFirst
|
||||
? await this.lockCellInventory(transaction, lockMode)
|
||||
// Why: the retry exists to take a cell row before the assignment row, the
|
||||
// order placement uses. It only ever needs the one cell this host is
|
||||
// pinned to, so read the pin unlocked and lock that row alone; taking all
|
||||
// 23 queued every sticky refresh in the fleet behind every other one.
|
||||
const pinnedCellId = inventoryFirst
|
||||
? await this.pinnedCellId(transaction, identity)
|
||||
: undefined
|
||||
const lockedCells =
|
||||
pinnedCellId === undefined
|
||||
? undefined
|
||||
: await this.lockCellRows(transaction, [pinnedCellId], lockMode)
|
||||
const existing = await this.assignmentRow(transaction, identity, inventoryFirst)
|
||||
if (!existing) return null
|
||||
const activityLeases = await this.lockAssignmentActivities(transaction, identity, true)
|
||||
@@ -661,6 +669,11 @@ export class RelayAssignmentStore {
|
||||
}
|
||||
|
||||
const currentCellId = text(existing, 'cell_id')
|
||||
// The pin moved between the unlocked read and the assignment lock, so the
|
||||
// row held is the wrong one. Same recovery as losing the lock: retry.
|
||||
if (pinnedCellId !== undefined && pinnedCellId !== currentCellId) {
|
||||
throw new Error('database_lock_unavailable')
|
||||
}
|
||||
const hadControl = holdsControlLease(
|
||||
activityLeases,
|
||||
currentCellId,
|
||||
@@ -701,14 +714,9 @@ export class RelayAssignmentStore {
|
||||
if (hadControl) {
|
||||
await this.touchAssignment(transaction, identity, leaseExpiresAt, now)
|
||||
} else {
|
||||
const nextReservation = integer(currentRow, 'reserved_requests') + 1
|
||||
if (nextReservation > integer(currentRow, 'capacity_requests')) {
|
||||
throw new Error('relay_capacity_exhausted')
|
||||
}
|
||||
await transaction.query(
|
||||
`UPDATE relay_cells SET reserved_requests = ?, updated_at = ? WHERE cell_id = ?`,
|
||||
[nextReservation, now, currentCellId]
|
||||
)
|
||||
// Delta, not the value read from the snapshot: an absolute write here
|
||||
// would clobber any concurrent movement of the same counter.
|
||||
await this.adjustCellReservationAtomically(transaction, currentCellId, 1)
|
||||
await this.adjustActivityCount(transaction, identity, 'control', 1, leaseExpiresAt, now)
|
||||
await this.insertPendingControlLease(
|
||||
transaction,
|
||||
@@ -6864,13 +6872,24 @@ export class RelayAssignmentStore {
|
||||
targetCellId
|
||||
]
|
||||
)
|
||||
const cells = await this.lockCellInventory(transaction, 'pool-default')
|
||||
// Only the two cells this repairs need holding. The id set below is an
|
||||
// existence check against a table that only reconcileCells writes, so it
|
||||
// reads unlocked instead of dragging the other 21 rows into the section.
|
||||
const cellIds = new Set(
|
||||
(await transaction.query(`SELECT cell_id FROM relay_cells`)).map((row) =>
|
||||
text(row, 'cell_id')
|
||||
)
|
||||
)
|
||||
const cells = await this.lockCellRows(
|
||||
transaction,
|
||||
[sourceCellId, targetCellId],
|
||||
'pool-default'
|
||||
)
|
||||
const assignmentKeys = new Set(
|
||||
assignments.map((row) =>
|
||||
assignmentKey(text(row, 'user_id'), text(row, 'relay_host_id'))
|
||||
)
|
||||
)
|
||||
const cellIds = new Set(cells.map((row) => text(row, 'cell_id')))
|
||||
const assignmentCounts = new Map<
|
||||
string,
|
||||
{ counts: Record<AssignmentActivityKind, number>; leaseExpiresAt: number }
|
||||
@@ -6924,9 +6943,7 @@ export class RelayAssignmentStore {
|
||||
)
|
||||
}
|
||||
|
||||
for (const row of cells.filter((cell) =>
|
||||
[sourceCellId, targetCellId].includes(text(cell, 'cell_id'))
|
||||
)) {
|
||||
for (const row of cells) {
|
||||
const cellId = text(row, 'cell_id')
|
||||
const expected = cellUnits.get(cellId) ?? 0
|
||||
if (expected > integer(row, 'capacity_requests')) {
|
||||
@@ -6958,16 +6975,40 @@ export class RelayAssignmentStore {
|
||||
// Per-connection paths touch one or two cells. Locking exactly those rows,
|
||||
// in the same ascending order the inventory lock uses (ORDER BY fixes the
|
||||
// row-lock order), keeps them off the fleet-wide lock without a cycle.
|
||||
private async lockCellRows(database: RelayDatabase, cellIds: string[]): Promise<SqlRow[]> {
|
||||
// The wait policy follows the caller for the same reason the inventory lock's
|
||||
// does: a sweep must not fail terminally on ordinary contention. Hold time is
|
||||
// deliberately not sampled here — the metric tracks the fleet-wide lock these
|
||||
// rows replace, and mixing in short single-row holds would flatter it.
|
||||
private async lockCellRows(
|
||||
database: RelayDatabase,
|
||||
cellIds: string[],
|
||||
mode: CellInventoryLockMode = 'request'
|
||||
): Promise<SqlRow[]> {
|
||||
const distinct = [...new Set(cellIds)]
|
||||
const { measureHoldMs: _sampled, ...wait } = cellInventoryLockOptions(mode)
|
||||
return await database.queryLocked(
|
||||
`SELECT * FROM relay_cells WHERE cell_id IN (${distinct.map(() => '?').join(', ')})
|
||||
ORDER BY cell_id ASC`,
|
||||
distinct,
|
||||
{ lockTimeoutMs: CELL_INVENTORY_LOCK_TIMEOUT_MS }
|
||||
wait
|
||||
)
|
||||
}
|
||||
|
||||
// Unlocked on purpose: this only names the row to lock next, and the caller
|
||||
// re-checks the pin once the assignment row is held.
|
||||
private async pinnedCellId(
|
||||
database: RelayDatabase,
|
||||
identity: AssignmentIdentity
|
||||
): Promise<string | undefined> {
|
||||
const row = (
|
||||
await database.query(
|
||||
`SELECT cell_id FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[identity.userId, identity.relayHostId]
|
||||
)
|
||||
)[0]
|
||||
return row ? text(row, 'cell_id') : undefined
|
||||
}
|
||||
|
||||
private async lockGeneralCellInventory(
|
||||
database: RelayDatabase,
|
||||
mode: CellInventoryLockMode
|
||||
@@ -6986,10 +7027,11 @@ export class RelayAssignmentStore {
|
||||
|
||||
private async leastLoadedCell(
|
||||
database: RelayDatabase,
|
||||
lockedCells: SqlRow[] | undefined,
|
||||
// Required: the one caller has already locked the inventory it selects from,
|
||||
// and an optional parameter left a second fleet-wide lock reachable here.
|
||||
rows: SqlRow[],
|
||||
preferredRegion: RelayRegion
|
||||
): Promise<CellRow | null> {
|
||||
const rows = lockedCells ?? (await this.lockCellInventory(database, 'pool-default'))
|
||||
const regions = new Map(
|
||||
(await database.query(`SELECT cell_id, region FROM relay_cell_regions`)).map((row) => [
|
||||
text(row, 'cell_id'),
|
||||
|
||||
@@ -18,7 +18,9 @@ type CensusEntry = { method: string; mode: CensusMode; reach: Reachability }
|
||||
// assignment-store.ts, in source order. A new site fails this test until it is
|
||||
// classified here, which is the point.
|
||||
const CENSUS: CensusEntry[] = [
|
||||
{ method: 'assignStickyOnce', mode: 'caller', reach: 'both' },
|
||||
// assignStickyOnce is gone from this list: its retry now locks only the row
|
||||
// the host is pinned to (lockCellRows), which is what a sticky refresh
|
||||
// touches. Placement below is the one genuinely fleet-wide decision left.
|
||||
{ method: 'assignOnce', mode: 'caller', reach: 'both' },
|
||||
{ method: 'assignOnce', mode: 'caller', reach: 'both' },
|
||||
{ method: 'assignOnce', mode: 'nowait', reach: 'both' },
|
||||
@@ -49,8 +51,9 @@ const CENSUS: CensusEntry[] = [
|
||||
{ method: 'abortExpiredEvacuations', mode: 'nowait', reach: 'sweep' },
|
||||
{ method: 'releaseExpiredActivityLeases', mode: 'nowait', reach: 'sweep' },
|
||||
{ method: 'releaseExpiredActivity', mode: 'nowait', reach: 'sweep' },
|
||||
{ method: 'reconcileReservationAccounting', mode: 'pool-default', reach: 'both' },
|
||||
{ method: 'leastLoadedCell', mode: 'pool-default', reach: 'both' }
|
||||
// reconcileReservationAccounting and leastLoadedCell are gone too: the first
|
||||
// repairs exactly two cells' counters and now holds only those rows, and the
|
||||
// second selects from the inventory its single caller has already locked.
|
||||
]
|
||||
|
||||
// Every inline `FROM relay_cells ... FOR UPDATE` outside the named lock helpers,
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
|
||||
import { RelayAssignmentStore } from './assignment-store.js'
|
||||
import { openRelayDatabase, type RelayDatabase } from './database.js'
|
||||
|
||||
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
const describePostgres = databaseUrl ? describe : describe.skip
|
||||
|
||||
// Sorted ascending, and the host is pinned to the LAST id on purpose: the
|
||||
// fleet-wide lock is one ordered scan, so it holds every earlier row while it
|
||||
// waits on the pinned one. Pinning to the first id would make the two locking
|
||||
// models indistinguishable.
|
||||
const cells = ['a', 'b', 'c'].map((suffix) => ({
|
||||
id: `percell-postgres-${suffix}`,
|
||||
url: `https://percell-postgres-${suffix}.example.com`,
|
||||
capacityRequests: 1_000,
|
||||
connectionHardCap: 600 as const,
|
||||
connectionUnobservedBound: 50
|
||||
}))
|
||||
const [cellA, cellB, cellC] = cells as [(typeof cells)[0], (typeof cells)[0], (typeof cells)[0]]
|
||||
const identity = { userId: 'percell-postgres-user', relayHostId: 'percellhost00001' }
|
||||
|
||||
function heartbeat(cell: (typeof cells)[number]) {
|
||||
return {
|
||||
cellId: cell.id,
|
||||
cellUrl: cell.url,
|
||||
cellIncarnation: '11111111-1111-4111-8111-111111111111',
|
||||
startedAt: 50,
|
||||
ready: true,
|
||||
observedRequests: 0,
|
||||
totalConnections: 0,
|
||||
inFlightConnections: 0,
|
||||
reservedConnectionUnits: 0,
|
||||
enforcedConnectionUnits: 0,
|
||||
connectionInclusionWatermark: 1,
|
||||
connectionHardCap: 600 as const,
|
||||
connectionUnobservedBound: 50
|
||||
}
|
||||
}
|
||||
|
||||
describePostgres('PostgreSQL per-cell inventory locking', () => {
|
||||
const databases: RelayDatabase[] = []
|
||||
|
||||
beforeAll(async () => {
|
||||
for (let index = 0; index < 3; index++) {
|
||||
databases.push(await openRelayDatabase({ databaseUrl, dataDir: '' }))
|
||||
}
|
||||
})
|
||||
|
||||
async function removeTestRows(database: RelayDatabase): Promise<void> {
|
||||
await database.query(
|
||||
`DELETE FROM relay_control_connection_reservations WHERE user_id LIKE 'percell-postgres-%'`
|
||||
)
|
||||
for (const table of [
|
||||
'relay_assignment_activity_leases',
|
||||
'relay_post_drain_migration_pins',
|
||||
'relay_assignment_migration_incarnations',
|
||||
'relay_assignment_migrations',
|
||||
'relay_assignment_region_preferences',
|
||||
'relay_assignments'
|
||||
]) {
|
||||
await database.query(`DELETE FROM ${table} WHERE user_id LIKE 'percell-postgres-%'`)
|
||||
}
|
||||
for (const cell of cells) {
|
||||
for (const table of [
|
||||
'relay_cell_connection_snapshots',
|
||||
'relay_cell_connection_runtime',
|
||||
'relay_cell_connection_limits',
|
||||
'relay_cell_runtime',
|
||||
'relay_cells'
|
||||
]) {
|
||||
await database.query(`DELETE FROM ${table} WHERE cell_id = ?`, [cell.id])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
afterAll(async () => {
|
||||
if (databases[0]) await removeTestRows(databases[0])
|
||||
for (const connection of databases) await connection.close()
|
||||
})
|
||||
|
||||
async function pinHostToLastCell(store: RelayAssignmentStore): Promise<void> {
|
||||
await store.reconcileCells(cells)
|
||||
for (const cell of cells) await store.recordCellHeartbeat(heartbeat(cell))
|
||||
await store.setCellEnabled(cellA.id, false)
|
||||
await store.setCellEnabled(cellB.id, false)
|
||||
const assignment = await store.assign(identity)
|
||||
expect(assignment.cellId).toBe(cellC.id)
|
||||
await store.setCellEnabled(cellA.id, true)
|
||||
await store.setCellEnabled(cellB.id, true)
|
||||
}
|
||||
|
||||
async function lockWaiterAppeared(database: RelayDatabase): Promise<boolean> {
|
||||
const deadline = Date.now() + 4_000
|
||||
while (Date.now() < deadline) {
|
||||
const rows = await database.query(
|
||||
`SELECT count(*) AS waiting FROM pg_stat_activity
|
||||
WHERE datname = current_database() AND wait_event_type = 'Lock'`
|
||||
)
|
||||
if (Number(rows[0]!.waiting) > 0) return true
|
||||
await new Promise((resolve) => setTimeout(resolve, 10))
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Why: a sticky refresh whose first NOWAIT probe loses retries by taking a
|
||||
// cell row before the assignment row. That retry used to take the whole
|
||||
// inventory, so one busy cell stalled every other cell's reconnects.
|
||||
it('waits only on the pinned cell row while refreshing a sticky assignment', async () => {
|
||||
await removeTestRows(databases[0]!)
|
||||
const store = new RelayAssignmentStore(databases[0]!, () => 100)
|
||||
await pinHostToLastCell(store)
|
||||
// A host whose control lease was already reaped still holds its pin; that
|
||||
// is the shape that reaches the cell-row probe instead of touchAssignment.
|
||||
await databases[0]!.query(
|
||||
`DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`,
|
||||
[identity.userId]
|
||||
)
|
||||
|
||||
let releaseRow!: () => void
|
||||
const rowReleased = new Promise<void>((resolve) => {
|
||||
releaseRow = resolve
|
||||
})
|
||||
let rowHeld!: () => void
|
||||
const rowHeldPromise = new Promise<void>((resolve) => {
|
||||
rowHeld = resolve
|
||||
})
|
||||
const holder = databases[1]!.transaction(async (transaction) => {
|
||||
await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cellC.id])
|
||||
rowHeld()
|
||||
await rowReleased
|
||||
})
|
||||
await rowHeldPromise
|
||||
|
||||
const refresh = store.assign(identity)
|
||||
expect(await lockWaiterAppeared(databases[2]!)).toBe(true)
|
||||
// The refresh is blocked on cell C. Every earlier row must still be free:
|
||||
// the ordered fleet-wide scan would be holding both of them by now.
|
||||
const heldWhileRefreshWaits: string[] = []
|
||||
await databases[2]!.transaction(async (transaction) => {
|
||||
for (const cell of [cellA, cellB]) {
|
||||
try {
|
||||
await transaction.queryLocked(
|
||||
`SELECT * FROM relay_cells WHERE cell_id = ?`,
|
||||
[cell.id],
|
||||
{ failIfUnavailable: true }
|
||||
)
|
||||
} catch {
|
||||
heldWhileRefreshWaits.push(cell.id)
|
||||
}
|
||||
}
|
||||
})
|
||||
releaseRow()
|
||||
await holder
|
||||
|
||||
expect(heldWhileRefreshWaits).toEqual([])
|
||||
expect((await refresh).cellId).toBe(cellC.id)
|
||||
}, 15_000)
|
||||
|
||||
// Why: the counter moves by a delta now instead of an absolute value read
|
||||
// from a snapshot, so concurrent movement on the same cell must still sum.
|
||||
it('keeps a cell reservation exact under concurrent same-cell activity', async () => {
|
||||
await removeTestRows(databases[0]!)
|
||||
const store = new RelayAssignmentStore(databases[0]!, () => 100)
|
||||
await store.reconcileCells(cells)
|
||||
for (const cell of cells) await store.recordCellHeartbeat(heartbeat(cell))
|
||||
await store.setCellEnabled(cellA.id, false)
|
||||
await store.setCellEnabled(cellB.id, false)
|
||||
|
||||
const hosts = Array.from({ length: 6 }, (_, index) => ({
|
||||
userId: `percell-postgres-user-${index}`,
|
||||
relayHostId: `percellhost0000${index}`
|
||||
}))
|
||||
const stores = databases.map((database) => new RelayAssignmentStore(database, () => 100))
|
||||
await Promise.all(hosts.map((host, index) => stores[index % stores.length]!.assign(host)))
|
||||
|
||||
// One splice each (2 units) on the same cell, from three connections at once.
|
||||
await Promise.all(
|
||||
hosts.map((host, index) =>
|
||||
stores[index % stores.length]!.acquireActivity(host, {
|
||||
activityId: `splice:percell-${index}`,
|
||||
kind: 'splice',
|
||||
cellId: cellC.id
|
||||
})
|
||||
)
|
||||
)
|
||||
const afterAcquire = await databases[0]!.query(
|
||||
`SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`,
|
||||
[cellC.id]
|
||||
)
|
||||
// 6 pending control grants + 6 splices at 2 units each.
|
||||
expect(Number(afterAcquire[0]!.reserved_requests)).toBe(6 + 12)
|
||||
|
||||
await Promise.all(
|
||||
hosts.map((host, index) =>
|
||||
stores[index % stores.length]!.releaseActivity(host, `splice:percell-${index}`)
|
||||
)
|
||||
)
|
||||
const afterRelease = await databases[0]!.query(
|
||||
`SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`,
|
||||
[cellC.id]
|
||||
)
|
||||
expect(Number(afterRelease[0]!.reserved_requests)).toBe(6)
|
||||
await store.setCellEnabled(cellA.id, true)
|
||||
await store.setCellEnabled(cellB.id, true)
|
||||
}, 15_000)
|
||||
})
|
||||
@@ -2,7 +2,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const fakes = vi.hoisted(() => ({
|
||||
configs: [] as Array<Record<string, unknown>>,
|
||||
query: vi.fn(async () => ({ rows: [], rowCount: 0 })),
|
||||
// Pool construction and pool shutdown interleaved, so "the schema pool is
|
||||
// gone before the serving pool opens" is checkable rather than assumed.
|
||||
lifecycle: [] as string[],
|
||||
query: vi.fn(async (_sql: string) => ({ rows: [], rowCount: 0 })),
|
||||
release: vi.fn(),
|
||||
end: vi.fn(async () => undefined)
|
||||
}))
|
||||
@@ -13,20 +16,37 @@ vi.mock('pg', () => ({
|
||||
totalCount = 1
|
||||
idleCount = 1
|
||||
waitingCount = 0
|
||||
end = fakes.end
|
||||
on = vi.fn()
|
||||
connect = vi.fn(async () => ({ query: fakes.query, release: fakes.release }))
|
||||
private readonly label: string
|
||||
|
||||
constructor(config: Record<string, unknown>) {
|
||||
fakes.configs.push(config)
|
||||
this.label = `max=${String(config.max)} statement_timeout=${String(config.statement_timeout)}`
|
||||
fakes.lifecycle.push(`open ${this.label}`)
|
||||
}
|
||||
|
||||
async end(): Promise<void> {
|
||||
fakes.lifecycle.push(`end ${this.label}`)
|
||||
await fakes.end()
|
||||
}
|
||||
}
|
||||
}
|
||||
}))
|
||||
|
||||
import { openRelayDatabase } from './database.js'
|
||||
import { openRelayDatabase, relayPostgresStatementTimeoutMs } from './database.js'
|
||||
import { applyPostgresSchema } from './postgres-schema-startup.js'
|
||||
|
||||
const SCHEMA_POOL = {
|
||||
max: 1,
|
||||
application_name: 'orca-relay/director/director/schema',
|
||||
connectionTimeoutMillis: 2_000,
|
||||
// Why: DDL must not inherit the request deadline.
|
||||
statement_timeout: 0,
|
||||
lock_timeout: 1_000,
|
||||
idle_in_transaction_session_timeout: 5_000
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
@@ -34,9 +54,11 @@ afterEach(() => {
|
||||
describe('PostgreSQL relay deadlines', () => {
|
||||
beforeEach(() => {
|
||||
fakes.configs.length = 0
|
||||
fakes.lifecycle.length = 0
|
||||
fakes.query.mockClear()
|
||||
fakes.release.mockClear()
|
||||
fakes.end.mockClear()
|
||||
delete process.env.ORCA_RELAY_POSTGRES_STATEMENT_TIMEOUT_MS
|
||||
})
|
||||
|
||||
it('bounds pool acquisition, statements, locks, and abandoned transactions', async () => {
|
||||
@@ -48,6 +70,7 @@ describe('PostgreSQL relay deadlines', () => {
|
||||
})
|
||||
|
||||
expect(fakes.configs).toEqual([
|
||||
expect.objectContaining(SCHEMA_POOL),
|
||||
expect.objectContaining({
|
||||
max: 3,
|
||||
application_name: 'orca-relay/director/director',
|
||||
@@ -59,6 +82,110 @@ describe('PostgreSQL relay deadlines', () => {
|
||||
])
|
||||
await database.close()
|
||||
})
|
||||
|
||||
// Why: an untimed session left open would be a standing way for request work
|
||||
// to escape the deadline this whole pool config exists to enforce.
|
||||
it('closes the untimed schema pool before the serving pool opens', async () => {
|
||||
const database = await openRelayDatabase({
|
||||
databaseUrl: 'postgresql://relay:secret@127.0.0.1:5432/relay',
|
||||
dataDir: './unused',
|
||||
poolMax: 3,
|
||||
applicationName: 'orca-relay/director/director'
|
||||
})
|
||||
|
||||
expect(fakes.lifecycle).toEqual([
|
||||
'open max=1 statement_timeout=0',
|
||||
'end max=1 statement_timeout=0',
|
||||
'open max=3 statement_timeout=5000'
|
||||
])
|
||||
await database.close()
|
||||
})
|
||||
|
||||
it('applies the schema on the untimed pool, never on the serving one', async () => {
|
||||
fakes.query.mockClear()
|
||||
const ddl: string[] = []
|
||||
fakes.query.mockImplementation(async (sql: string) => {
|
||||
// Every statement issued before the serving pool exists is schema work.
|
||||
if (fakes.lifecycle.length === 1) ddl.push(sql)
|
||||
return { rows: [], rowCount: 0 }
|
||||
})
|
||||
const database = await openRelayDatabase({
|
||||
databaseUrl: 'postgresql://relay:secret@127.0.0.1:5432/relay',
|
||||
dataDir: './unused'
|
||||
})
|
||||
|
||||
expect(ddl.length).toBeGreaterThan(0)
|
||||
// Statements can open with a leading `--` rationale comment.
|
||||
const body = (statement: string): string =>
|
||||
statement.replace(/^(?:\s*--[^\n]*\n)*\s*/, '')
|
||||
expect(ddl.every((statement) => /^CREATE\b/i.test(body(statement)))).toBe(true)
|
||||
// The backfill is DML, so it stays on the deadline-bearing serving pool.
|
||||
expect(ddl.some((statement) => statement.includes('INSERT INTO'))).toBe(false)
|
||||
await database.close()
|
||||
})
|
||||
|
||||
it('takes the serving statement deadline from the environment', async () => {
|
||||
process.env.ORCA_RELAY_POSTGRES_STATEMENT_TIMEOUT_MS = '2500'
|
||||
|
||||
const database = await openRelayDatabase({
|
||||
databaseUrl: 'postgresql://relay:secret@127.0.0.1:5432/relay',
|
||||
dataDir: './unused'
|
||||
})
|
||||
|
||||
expect(fakes.configs).toEqual([
|
||||
expect.objectContaining({ statement_timeout: 0 }),
|
||||
expect.objectContaining({ statement_timeout: 2_500 })
|
||||
])
|
||||
await database.close()
|
||||
})
|
||||
|
||||
it.each(['0', '-1', '2.5', 'soon', ' '])(
|
||||
'refuses %s as a statement deadline instead of running unbounded',
|
||||
(value) => {
|
||||
expect(() =>
|
||||
relayPostgresStatementTimeoutMs({ ORCA_RELAY_POSTGRES_STATEMENT_TIMEOUT_MS: value })
|
||||
).toThrow('invalid_statement_timeout')
|
||||
}
|
||||
)
|
||||
|
||||
it.each([undefined, ''])('defaults to 5s when the environment says %s', (value) => {
|
||||
expect(
|
||||
relayPostgresStatementTimeoutMs(
|
||||
value === undefined ? {} : { ORCA_RELAY_POSTGRES_STATEMENT_TIMEOUT_MS: value }
|
||||
)
|
||||
).toBe(5_000)
|
||||
})
|
||||
|
||||
// Why: a statement deadline that reaches the caller as a crash converts a
|
||||
// transient stall into a failed assignment. It aborts the transaction exactly
|
||||
// as a lock timeout does, so it belongs on the same bounded retry.
|
||||
it('retries a statement timeout on a fresh client', async () => {
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
const database = await openRelayDatabase({
|
||||
databaseUrl: 'postgresql://relay:secret@127.0.0.1:5432/relay',
|
||||
dataDir: './unused'
|
||||
})
|
||||
let attempts = 0
|
||||
|
||||
const result = await database.transaction(async (transaction) => {
|
||||
attempts += 1
|
||||
if (attempts === 1) {
|
||||
await transaction.query('SELECT 1')
|
||||
throw Object.assign(new Error('canceling statement due to statement timeout'), {
|
||||
code: '57014'
|
||||
})
|
||||
}
|
||||
return 'committed'
|
||||
})
|
||||
|
||||
expect(result).toBe('committed')
|
||||
expect(attempts).toBe(2)
|
||||
expect(console.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining('"event":"orca_relay_postgres_transaction_retry"')
|
||||
)
|
||||
expect(console.warn).toHaveBeenCalledWith(expect.stringContaining('"code":"57014"'))
|
||||
await database.close()
|
||||
})
|
||||
})
|
||||
|
||||
describe('PostgreSQL schema startup', () => {
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
|
||||
import { openRelayDatabase, type RelayDatabase } from './database.js'
|
||||
|
||||
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
const describePostgres = databaseUrl ? describe : describe.skip
|
||||
const applicationName = 'orca-relay/statement-timeout-postgres'
|
||||
|
||||
describePostgres('PostgreSQL statement deadline', () => {
|
||||
const databases: RelayDatabase[] = []
|
||||
|
||||
beforeAll(async () => {
|
||||
databases.push(await openRelayDatabase({ databaseUrl, dataDir: '' }))
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
for (const database of databases) await database.close()
|
||||
})
|
||||
|
||||
it('serves requests under the configured deadline', async () => {
|
||||
const database = await openRelayDatabase({ databaseUrl, dataDir: '', statementTimeoutMs: 300 })
|
||||
databases.push(database)
|
||||
|
||||
expect(await database.query(`SELECT current_setting('statement_timeout') AS statement_timeout`)).toEqual([
|
||||
{ statement_timeout: '300ms' }
|
||||
])
|
||||
})
|
||||
|
||||
// Why: a real 57014 aborts the transaction exactly as a lock timeout does. If
|
||||
// it escapes the bounded retry it becomes a failed assignment instead of a
|
||||
// slow one.
|
||||
it('retries a real statement timeout on a fresh client', async () => {
|
||||
const database = await openRelayDatabase({ databaseUrl, dataDir: '', statementTimeoutMs: 300 })
|
||||
databases.push(database)
|
||||
let attempts = 0
|
||||
|
||||
const result = await database.transaction(async (transaction) => {
|
||||
attempts += 1
|
||||
if (attempts === 1) await transaction.query(`SELECT pg_sleep(2)`)
|
||||
return attempts
|
||||
})
|
||||
|
||||
expect(result).toBe(2)
|
||||
}, 15_000)
|
||||
|
||||
// Why: DDL runs on its own untimed connection. relay_invites carries a
|
||||
// CREATE INDEX IF NOT EXISTS, which (unlike CREATE TABLE IF NOT EXISTS)
|
||||
// really does queue behind an ACCESS EXCLUSIVE lock on the table.
|
||||
it('applies the schema behind a held ACCESS EXCLUSIVE lock', async () => {
|
||||
let releaseTable!: () => void
|
||||
const tableReleased = new Promise<void>((resolve) => {
|
||||
releaseTable = resolve
|
||||
})
|
||||
let tableHeld!: () => void
|
||||
const tableHeldPromise = new Promise<void>((resolve) => {
|
||||
tableHeld = resolve
|
||||
})
|
||||
const holder = databases[0]!.transaction(async (transaction) => {
|
||||
await transaction.query(`LOCK TABLE relay_invites IN ACCESS EXCLUSIVE MODE`)
|
||||
tableHeld()
|
||||
await tableReleased
|
||||
})
|
||||
await tableHeldPromise
|
||||
|
||||
const opening = openRelayDatabase({
|
||||
databaseUrl,
|
||||
dataDir: '',
|
||||
applicationName,
|
||||
// Far too short for a blocked DDL; the serving pool wears it, the schema
|
||||
// connection must not.
|
||||
statementTimeoutMs: 200
|
||||
})
|
||||
const blockedOnSchemaConnection = async (): Promise<boolean> => {
|
||||
const deadline = Date.now() + 4_000
|
||||
while (Date.now() < deadline) {
|
||||
const rows = await databases[0]!.query(
|
||||
`SELECT count(*) AS waiting FROM pg_stat_activity
|
||||
WHERE datname = current_database() AND wait_event_type = 'Lock'
|
||||
AND application_name = ?`,
|
||||
[`${applicationName}/schema`]
|
||||
)
|
||||
if (Number(rows[0]!.waiting) > 0) return true
|
||||
await new Promise((resolve) => setTimeout(resolve, 10))
|
||||
}
|
||||
return false
|
||||
}
|
||||
const blocked = await blockedOnSchemaConnection()
|
||||
releaseTable()
|
||||
await holder
|
||||
|
||||
const database = await opening
|
||||
databases.push(database)
|
||||
expect(blocked).toBe(true)
|
||||
// The serving pool still carries the short deadline it was opened with.
|
||||
expect(await database.query(`SELECT current_setting('statement_timeout') AS statement_timeout`)).toEqual([
|
||||
{ statement_timeout: '200ms' }
|
||||
])
|
||||
}, 15_000)
|
||||
})
|
||||
@@ -796,12 +796,34 @@ class PostgresTransaction implements RelayDatabase {
|
||||
const POSTGRES_TRANSACTION_ATTEMPTS = 3
|
||||
const POSTGRES_RETRY_MAX_DELAY_MS = 25
|
||||
const POSTGRES_CONNECTION_TIMEOUT_MS = 2_000
|
||||
const POSTGRES_STATEMENT_TIMEOUT_MS = 5_000
|
||||
// Derivation: a control renewal must land inside its own 30s tick
|
||||
// (RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2), and a transaction gets
|
||||
// POSTGRES_TRANSACTION_ATTEMPTS tries, so the worst case a renewal can spend in
|
||||
// Postgres is attempts * timeout. 5s keeps that at 15s, half the tick, and still
|
||||
// leaves room for the connect timeout above.
|
||||
export const POSTGRES_STATEMENT_TIMEOUT_MS = 5_000
|
||||
const POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS = 5_000
|
||||
|
||||
export function relayPostgresStatementTimeoutMs(
|
||||
env: NodeJS.ProcessEnv = process.env
|
||||
): number {
|
||||
const configured = env.ORCA_RELAY_POSTGRES_STATEMENT_TIMEOUT_MS
|
||||
if (configured === undefined || configured === '') return POSTGRES_STATEMENT_TIMEOUT_MS
|
||||
const milliseconds = Number(configured)
|
||||
// 0 is PostgreSQL's "no timeout"; refusing it keeps the deadline this exists
|
||||
// to enforce from being disabled by a typo in an environment variable.
|
||||
if (!Number.isInteger(milliseconds) || milliseconds < 1) {
|
||||
throw new Error('invalid_statement_timeout')
|
||||
}
|
||||
return milliseconds
|
||||
}
|
||||
|
||||
function retryablePostgresTransactionError(error: unknown): boolean {
|
||||
const code = String((error as { code?: unknown }).code)
|
||||
return code === '40P01' || code === '40001' || code === '55P03'
|
||||
// 57014 is the pool statement_timeout firing. It aborts the transaction the
|
||||
// same way a lock timeout does, so it belongs on the bounded retry path
|
||||
// rather than surfacing as a terminal failure to the caller.
|
||||
return code === '40P01' || code === '40001' || code === '55P03' || code === '57014'
|
||||
}
|
||||
|
||||
export function isRelayDatabaseTransientError(error: unknown): boolean {
|
||||
@@ -963,11 +985,36 @@ async function applySchema(database: RelayDatabase): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
async function applySchemaWithPostgresRetries(database: RelayDatabase): Promise<void> {
|
||||
await applyPostgresSchema(
|
||||
SCHEMA.split(';').filter((statement) => statement.trim()),
|
||||
async (statement) => await database.query(statement)
|
||||
)
|
||||
// Why: DDL is not a request. A CREATE INDEX on a grown table legitimately runs
|
||||
// longer than the request statement_timeout, and inheriting that timeout would
|
||||
// make every startup fail at the same statement instead of finishing once. One
|
||||
// short-lived connection of its own, ended before the serving pool opens, keeps
|
||||
// the untimed session off the request path entirely.
|
||||
async function applySchemaOnUntimedPool(
|
||||
databaseUrl: string,
|
||||
applicationName: string | undefined
|
||||
): Promise<void> {
|
||||
const pool = new pg.Pool({
|
||||
connectionString: databaseUrl,
|
||||
max: 1,
|
||||
application_name: applicationName ? `${applicationName}/schema` : undefined,
|
||||
connectionTimeoutMillis: POSTGRES_CONNECTION_TIMEOUT_MS,
|
||||
statement_timeout: 0,
|
||||
// Kept: a DDL blocked behind another director's ACCESS EXCLUSIVE lock must
|
||||
// yield to the bounded schema retry instead of holding the connection.
|
||||
lock_timeout: POSTGRES_LOCK_TIMEOUT_MS,
|
||||
idle_in_transaction_session_timeout: POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS
|
||||
})
|
||||
absorbPostgresIdleClientErrors(pool)
|
||||
const database = new PostgresDatabase(pool)
|
||||
try {
|
||||
await applyPostgresSchema(
|
||||
SCHEMA.split(';').filter((statement) => statement.trim()),
|
||||
async (statement) => await database.query(statement)
|
||||
)
|
||||
} finally {
|
||||
await database.close().catch(() => undefined)
|
||||
}
|
||||
}
|
||||
|
||||
async function backfillRelayCellRegions(database: RelayDatabase): Promise<void> {
|
||||
@@ -983,15 +1030,17 @@ export async function openRelayDatabase(input: {
|
||||
dataDir: string
|
||||
poolMax?: number
|
||||
applicationName?: string
|
||||
statementTimeoutMs?: number
|
||||
}): Promise<RelayDatabase> {
|
||||
let database: RelayDatabase
|
||||
if (input.databaseUrl) {
|
||||
await applySchemaOnUntimedPool(input.databaseUrl, input.applicationName)
|
||||
const pool = new pg.Pool({
|
||||
connectionString: input.databaseUrl,
|
||||
max: input.poolMax ?? 10,
|
||||
application_name: input.applicationName,
|
||||
connectionTimeoutMillis: POSTGRES_CONNECTION_TIMEOUT_MS,
|
||||
statement_timeout: POSTGRES_STATEMENT_TIMEOUT_MS,
|
||||
statement_timeout: input.statementTimeoutMs ?? relayPostgresStatementTimeoutMs(),
|
||||
lock_timeout: POSTGRES_LOCK_TIMEOUT_MS,
|
||||
idle_in_transaction_session_timeout: POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS
|
||||
})
|
||||
@@ -1004,8 +1053,7 @@ export async function openRelayDatabase(input: {
|
||||
database = new SqliteDatabase(sqlite)
|
||||
}
|
||||
try {
|
||||
if (input.databaseUrl) await applySchemaWithPostgresRetries(database)
|
||||
else await applySchema(database)
|
||||
if (!input.databaseUrl) await applySchema(database)
|
||||
await backfillRelayCellRegions(database)
|
||||
return database
|
||||
} catch (error) {
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
import { ASSIGNMENT_LIMITS, RELAY_HOST_CLOSE_REASON } from '@orca-cloud/relay-contract'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { HostCloseReasonMemory } from './host-close-reason-memory.js'
|
||||
|
||||
function memoryAt(clock: { now: number }): HostCloseReasonMemory {
|
||||
return new HostCloseReasonMemory(() => clock.now)
|
||||
}
|
||||
|
||||
describe('HostCloseReasonMemory', () => {
|
||||
it('remembers only reasons it knows', () => {
|
||||
const clock = { now: 1_000 }
|
||||
const memory = memoryAt(clock)
|
||||
|
||||
memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
memory.record('b', 'quitting')
|
||||
memory.record('c', Buffer.alloc(0))
|
||||
memory.record('d', undefined)
|
||||
|
||||
expect(memory.read('a')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
expect(memory.read('b')).toBeNull()
|
||||
expect(memory.read('c')).toBeNull()
|
||||
expect(memory.read('d')).toBeNull()
|
||||
})
|
||||
|
||||
it('accepts the reason as the Buffer a ws close delivers', () => {
|
||||
const clock = { now: 1_000 }
|
||||
const memory = memoryAt(clock)
|
||||
|
||||
memory.record('a', Buffer.from(RELAY_HOST_CLOSE_REASON.SIGNED_OUT))
|
||||
|
||||
expect(memory.read('a')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
})
|
||||
|
||||
it('expires an entry once its host may have been rebalanced away', () => {
|
||||
const clock = { now: 1_000 }
|
||||
const memory = memoryAt(clock)
|
||||
memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
|
||||
clock.now += ASSIGNMENT_LIMITS.dormantTtlMs - 1
|
||||
expect(memory.read('a')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
|
||||
clock.now += 1
|
||||
expect(memory.read('a')).toBeNull()
|
||||
expect(memory.size()).toBe(0)
|
||||
})
|
||||
|
||||
it('forgets on demand', () => {
|
||||
const clock = { now: 1_000 }
|
||||
const memory = memoryAt(clock)
|
||||
memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
|
||||
memory.forget('a')
|
||||
|
||||
expect(memory.read('a')).toBeNull()
|
||||
})
|
||||
|
||||
it('drops the oldest survivors rather than growing without bound', () => {
|
||||
const clock = { now: 1_000 }
|
||||
const memory = memoryAt(clock)
|
||||
for (let index = 0; index < 50_050; index++) {
|
||||
memory.record(`host-${index}`, RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
}
|
||||
|
||||
expect(memory.size()).toBe(50_000)
|
||||
expect(memory.read('host-0')).toBeNull()
|
||||
expect(memory.read('host-50049')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
})
|
||||
|
||||
it('re-recording refreshes recency so a live host is not evicted first', () => {
|
||||
const clock = { now: 1_000 }
|
||||
const memory = memoryAt(clock)
|
||||
memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
memory.record('b', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
|
||||
expect([...['a', 'b'].map((key) => memory.read(key))]).toEqual([
|
||||
RELAY_HOST_CLOSE_REASON.SIGNED_OUT,
|
||||
RELAY_HOST_CLOSE_REASON.SIGNED_OUT
|
||||
])
|
||||
expect(memory.size()).toBe(2)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,72 @@
|
||||
import {
|
||||
ASSIGNMENT_LIMITS,
|
||||
relayHostCloseReasonFrom,
|
||||
type RelayHostCloseReason
|
||||
} from '@orca-cloud/relay-contract'
|
||||
|
||||
// Retention matches the dormant assignment TTL: past it the host may have been
|
||||
// rebalanced onto another cell, so this cell is no longer the one a phone asks.
|
||||
const RETENTION_MS = ASSIGNMENT_LIMITS.dormantTtlMs
|
||||
// A fleet-wide auth outage signs out every host at once; the cap bounds that
|
||||
// burst well above any single cell's host count without becoming a leak.
|
||||
const MAX_ENTRIES = 50_000
|
||||
|
||||
// Why in-memory and not Postgres: a phone reaches the cell its host's assignment
|
||||
// row already names, which is the same cell that watched the control socket
|
||||
// close. Losing this on a cell restart degrades to the pre-existing generic
|
||||
// verdict, so the failure mode is the old behaviour rather than a wrong one.
|
||||
export class HostCloseReasonMemory {
|
||||
private readonly entries = new Map<string, { reason: RelayHostCloseReason; expiresAt: number }>()
|
||||
|
||||
constructor(private readonly now: () => number = Date.now) {}
|
||||
|
||||
// Silently ignores anything that is not a known reason, which is every close
|
||||
// from a host that predates the field and every abrupt 1006.
|
||||
record(key: string, reason: unknown): void {
|
||||
const parsed = relayHostCloseReasonFrom(reason)
|
||||
if (!parsed) {
|
||||
return
|
||||
}
|
||||
this.entries.delete(key)
|
||||
this.entries.set(key, { reason: parsed, expiresAt: this.now() + RETENTION_MS })
|
||||
this.evict()
|
||||
}
|
||||
|
||||
forget(key: string): void {
|
||||
this.entries.delete(key)
|
||||
}
|
||||
|
||||
read(key: string): RelayHostCloseReason | null {
|
||||
const entry = this.entries.get(key)
|
||||
if (!entry) {
|
||||
return null
|
||||
}
|
||||
if (entry.expiresAt <= this.now()) {
|
||||
this.entries.delete(key)
|
||||
return null
|
||||
}
|
||||
return entry.reason
|
||||
}
|
||||
|
||||
size(): number {
|
||||
return this.entries.size
|
||||
}
|
||||
|
||||
private evict(): void {
|
||||
const now = this.now()
|
||||
for (const [key, entry] of this.entries) {
|
||||
if (entry.expiresAt > now) {
|
||||
break
|
||||
}
|
||||
this.entries.delete(key)
|
||||
}
|
||||
// Insertion order is recency order (record deletes before setting), so the
|
||||
// head is always the oldest survivor.
|
||||
for (const key of this.entries.keys()) {
|
||||
if (this.entries.size <= MAX_ENTRIES) {
|
||||
break
|
||||
}
|
||||
this.entries.delete(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -14,7 +14,8 @@ import {
|
||||
HostHelloSchema,
|
||||
InviteCreateSchema,
|
||||
RELAY_PROTOCOL_LIMITS,
|
||||
RELAY_CLOSE_CODE
|
||||
RELAY_CLOSE_CODE,
|
||||
type RelayHostCloseReason
|
||||
} from '@orca-cloud/relay-contract'
|
||||
import nacl from 'tweetnacl'
|
||||
import type WebSocket from 'ws'
|
||||
@@ -25,6 +26,7 @@ import {
|
||||
RelayCredentialStore,
|
||||
type CredentialReservation
|
||||
} from './credential-store.js'
|
||||
import { HostCloseReasonMemory } from './host-close-reason-memory.js'
|
||||
import { relayHostLogDigest } from './relay-host-log-digest.js'
|
||||
import type { RelayTokenClaims } from './relay-token-verifier.js'
|
||||
import type { RelayRuntimeObserver } from './relay-observability.js'
|
||||
@@ -130,6 +132,10 @@ const ACTIVATION_QUEUE_WAIT_MS = 30_000
|
||||
export class HostSessionRegistry {
|
||||
private readonly sessions = new Map<string, HostSession>()
|
||||
private readonly activationQueues = new Map<string, Promise<void>>()
|
||||
// Why it outlives `sessions`: the orphan grace deletes the session within 30s,
|
||||
// but a signed-out desktop never comes back, so the phone that asks minutes
|
||||
// later would otherwise find nothing to explain its rejection with.
|
||||
private readonly hostCloseReasons = new HostCloseReasonMemory(() => this.now())
|
||||
private draining = false
|
||||
|
||||
constructor(
|
||||
@@ -175,7 +181,8 @@ export class HostSessionRegistry {
|
||||
return
|
||||
}
|
||||
this.observer.recordAuth(true)
|
||||
const session = this.sessions.get(this.key(reservation.userId, hostId))
|
||||
const sessionKey = this.key(reservation.userId, hostId)
|
||||
const session = this.sessions.get(sessionKey)
|
||||
if (
|
||||
!session ||
|
||||
session.state !== 'active' ||
|
||||
@@ -184,7 +191,13 @@ export class HostSessionRegistry {
|
||||
) {
|
||||
capacityReservation?.release()
|
||||
await this.store.failReservation(reservation)
|
||||
this.rejectClient(socket, RELAY_CLOSE_CODE.HOST_OFFLINE)
|
||||
// The only rejection that can name a cause: the host is genuinely absent.
|
||||
// The attach-deadline 4404 below fires while control is still connected.
|
||||
this.rejectClient(
|
||||
socket,
|
||||
RELAY_CLOSE_CODE.HOST_OFFLINE,
|
||||
this.hostCloseReasons.read(sessionKey)
|
||||
)
|
||||
return
|
||||
}
|
||||
if (session.activeConnIds.size + session.pendingConns.size >= 8) {
|
||||
@@ -793,7 +806,10 @@ export class HostSessionRegistry {
|
||||
regionalDrainTimer: null,
|
||||
regionalDrainExpiresAt: null
|
||||
}
|
||||
this.sessions.set(this.key(identity.sub, identity.relayHostId), session)
|
||||
const sessionKey = this.key(identity.sub, identity.relayHostId)
|
||||
// A host that proved itself again is not signed out, whatever it said last.
|
||||
this.hostCloseReasons.forget(sessionKey)
|
||||
this.sessions.set(sessionKey, session)
|
||||
this.wireActiveControl(session)
|
||||
this.sendHelloAck(session)
|
||||
}
|
||||
@@ -813,6 +829,11 @@ export class HostSessionRegistry {
|
||||
})
|
||||
socket.once('close', (code, reason) => {
|
||||
this.observer.recordControlClose?.(code)
|
||||
// Guarded on identity: a predecessor retired by a rebind must not stamp a
|
||||
// cause onto the live session that replaced it.
|
||||
if (session.socket === socket) {
|
||||
this.hostCloseReasons.record(this.key(session.identity.sub, session.relayHostId), reason)
|
||||
}
|
||||
// One line per control close makes reconnect churners attributable by
|
||||
// host digest without exposing the raw relay host id.
|
||||
console.warn(
|
||||
@@ -1187,9 +1208,16 @@ export class HostSessionRegistry {
|
||||
if (session.socket) send(session.socket, 'control-error', { ...(reqId ? { reqId } : {}), code })
|
||||
}
|
||||
|
||||
private rejectClient(socket: WebSocket, code: number): void {
|
||||
// hostCloseReason rides the WebSocket close reason, never relay-hello: every
|
||||
// shipped phone parses relay-hello with a strict schema that rejects an
|
||||
// unknown key, and none of them read the close reason at all.
|
||||
private rejectClient(
|
||||
socket: WebSocket,
|
||||
code: number,
|
||||
hostCloseReason?: RelayHostCloseReason | null
|
||||
): void {
|
||||
send(socket, 'relay-hello', { ok: false, code })
|
||||
closeRelayWebSocket(socket, code, 'relay connection rejected')
|
||||
closeRelayWebSocket(socket, code, hostCloseReason ?? 'relay connection rejected')
|
||||
}
|
||||
|
||||
private releaseControlActivity(session: HostSession): void {
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
import { EventEmitter } from 'node:events'
|
||||
import {
|
||||
CONTROL_CONTINUITY_LIMITS,
|
||||
RELAY_CLOSE_CODE,
|
||||
RELAY_HOST_CLOSE_REASON
|
||||
} from '@orca-cloud/relay-contract'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type WebSocket from 'ws'
|
||||
import type { RelayAssignmentStore } from './assignment-store.js'
|
||||
import type { RelayConfig } from './config.js'
|
||||
import type { RelayCredentialStore } from './credential-store.js'
|
||||
import { HostSessionRegistry } from './host-session-registry.js'
|
||||
import type { RelayRuntimeObserver } from './relay-observability.js'
|
||||
import type { RelayTokenClaims } from './relay-token-verifier.js'
|
||||
import { ProcessQueuedByteBudget } from './splice-forwarder.js'
|
||||
|
||||
class FakeSocket extends EventEmitter {
|
||||
readonly OPEN = 1
|
||||
readonly CLOSED = 3
|
||||
readyState = this.OPEN
|
||||
readonly send = vi.fn()
|
||||
readonly close = vi.fn((code?: number, reason?: string) => {
|
||||
this.readyState = this.CLOSED
|
||||
this.emit('close', code, Buffer.from(reason ?? ''))
|
||||
})
|
||||
readonly terminate = vi.fn(() => {
|
||||
this.readyState = this.CLOSED
|
||||
this.emit('close', 1006, Buffer.alloc(0))
|
||||
})
|
||||
}
|
||||
|
||||
const config = {
|
||||
port: 8080,
|
||||
publicUrl: 'https://relay-c3.example.com',
|
||||
cellUrl: 'https://relay-c3.example.com',
|
||||
authIssuer: 'https://auth.example.com',
|
||||
authAudience: 'orca-relay',
|
||||
jwksUrl: 'https://auth.example.com/jwks',
|
||||
assignmentSigningKey: new Uint8Array(32),
|
||||
role: 'cell',
|
||||
cellId: 'production-gce-c3',
|
||||
cells: []
|
||||
} as unknown as RelayConfig
|
||||
|
||||
const identity = {
|
||||
sub: 'user-1',
|
||||
prof: 'profile-1',
|
||||
org: 'org-1',
|
||||
relayHostId: 'AbCdEf0123_-xyZ9'
|
||||
} as unknown as RelayTokenClaims
|
||||
|
||||
const reservation = {
|
||||
userId: identity.sub,
|
||||
relayHostId: identity.relayHostId,
|
||||
credentialKind: 'resume',
|
||||
relayDeviceId: 'device-1',
|
||||
leaseExpiresAt: Date.now() + 60_000
|
||||
}
|
||||
|
||||
function createRegistry() {
|
||||
const store = {
|
||||
resolveResume: vi.fn().mockResolvedValue({ userId: identity.sub }),
|
||||
reserveCredential: vi.fn().mockResolvedValue(reservation),
|
||||
failReservation: vi.fn().mockResolvedValue(undefined)
|
||||
}
|
||||
const assignments = {
|
||||
activateControl: vi.fn().mockResolvedValue('control:production-gce-c3:1'),
|
||||
markMigrationTargetRegistered: vi.fn().mockResolvedValue(undefined),
|
||||
resolve: vi.fn().mockResolvedValue({ cellId: config.cellId }),
|
||||
acquireActivity: vi.fn().mockResolvedValue(undefined),
|
||||
renewControlActivity: vi.fn().mockResolvedValue(undefined),
|
||||
releaseActivity: vi.fn().mockResolvedValue(true)
|
||||
} as unknown as RelayAssignmentStore
|
||||
const observer = {
|
||||
recordAuth: vi.fn(),
|
||||
recordForwardedBytes: vi.fn(),
|
||||
recordHttp: vi.fn(),
|
||||
recordReconnect: vi.fn(),
|
||||
recordSql: vi.fn(),
|
||||
recordControlClose: vi.fn(),
|
||||
recordSpliceClose: vi.fn()
|
||||
} satisfies RelayRuntimeObserver
|
||||
const registry = new HostSessionRegistry(
|
||||
config,
|
||||
vi.fn(),
|
||||
store as unknown as RelayCredentialStore,
|
||||
assignments,
|
||||
new ProcessQueuedByteBudget(),
|
||||
observer
|
||||
)
|
||||
const activate = (socket: WebSocket, generation: number): Promise<void> =>
|
||||
(
|
||||
registry as unknown as {
|
||||
activate: (
|
||||
socket: WebSocket,
|
||||
identity: RelayTokenClaims,
|
||||
existing: null,
|
||||
generation: number,
|
||||
rebind: boolean,
|
||||
assignmentEpoch: number,
|
||||
appVersion: string
|
||||
) => Promise<void>
|
||||
}
|
||||
).activate(socket, identity, null, generation, false, 1, '1.4.173')
|
||||
return { registry, activate }
|
||||
}
|
||||
|
||||
async function dialPhone(registry: HostSessionRegistry): Promise<FakeSocket> {
|
||||
const phone = new FakeSocket()
|
||||
await registry.acceptClient(phone as unknown as WebSocket, identity.relayHostId, 'credential')
|
||||
return phone
|
||||
}
|
||||
|
||||
// The 4404 hello body is unchanged: every shipped phone parses it with a strict
|
||||
// schema, so the cause has to ride the close frame instead.
|
||||
const HOST_OFFLINE_HELLO = JSON.stringify({
|
||||
type: 'relay-hello',
|
||||
ok: false,
|
||||
code: RELAY_CLOSE_CODE.HOST_OFFLINE
|
||||
})
|
||||
|
||||
describe('host sign-out reason on phone rejection', () => {
|
||||
beforeEach(() => vi.useFakeTimers())
|
||||
afterEach(() => {
|
||||
vi.clearAllTimers()
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('names the sign-out to a phone that arrives after the host is gone', async () => {
|
||||
const { registry, activate } = createRegistry()
|
||||
const control = new FakeSocket()
|
||||
await activate(control as unknown as WebSocket, 1)
|
||||
|
||||
control.close(1000, RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
vi.advanceTimersByTime(CONTROL_CONTINUITY_LIMITS.orphanGraceMs + 1)
|
||||
|
||||
const phone = await dialPhone(registry)
|
||||
expect(phone.send).toHaveBeenCalledWith(HOST_OFFLINE_HELLO)
|
||||
expect(phone.close).toHaveBeenCalledWith(
|
||||
RELAY_CLOSE_CODE.HOST_OFFLINE,
|
||||
RELAY_HOST_CLOSE_REASON.SIGNED_OUT
|
||||
)
|
||||
})
|
||||
|
||||
it('says nothing when the host died without naming a cause', async () => {
|
||||
const { registry, activate } = createRegistry()
|
||||
const control = new FakeSocket()
|
||||
await activate(control as unknown as WebSocket, 1)
|
||||
|
||||
control.terminate()
|
||||
vi.advanceTimersByTime(CONTROL_CONTINUITY_LIMITS.orphanGraceMs + 1)
|
||||
|
||||
const phone = await dialPhone(registry)
|
||||
expect(phone.close).toHaveBeenCalledWith(
|
||||
RELAY_CLOSE_CODE.HOST_OFFLINE,
|
||||
'relay connection rejected'
|
||||
)
|
||||
})
|
||||
|
||||
it('ignores a close reason the host invented', async () => {
|
||||
const { registry, activate } = createRegistry()
|
||||
const control = new FakeSocket()
|
||||
await activate(control as unknown as WebSocket, 1)
|
||||
|
||||
control.close(1000, 'signed-out-ish')
|
||||
vi.advanceTimersByTime(CONTROL_CONTINUITY_LIMITS.orphanGraceMs + 1)
|
||||
|
||||
const phone = await dialPhone(registry)
|
||||
expect(phone.close).toHaveBeenCalledWith(
|
||||
RELAY_CLOSE_CODE.HOST_OFFLINE,
|
||||
'relay connection rejected'
|
||||
)
|
||||
})
|
||||
|
||||
it('forgets the sign-out once the host proves itself again', async () => {
|
||||
const { registry, activate } = createRegistry()
|
||||
const control = new FakeSocket()
|
||||
await activate(control as unknown as WebSocket, 1)
|
||||
control.close(1000, RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
vi.advanceTimersByTime(CONTROL_CONTINUITY_LIMITS.orphanGraceMs + 1)
|
||||
|
||||
const reconnected = new FakeSocket()
|
||||
await activate(reconnected as unknown as WebSocket, 2)
|
||||
// Drop it abruptly, as a network death would, so only the stale memory
|
||||
// could still name a cause.
|
||||
reconnected.terminate()
|
||||
vi.advanceTimersByTime(CONTROL_CONTINUITY_LIMITS.orphanGraceMs + 1)
|
||||
|
||||
const phone = await dialPhone(registry)
|
||||
expect(phone.close).toHaveBeenCalledWith(
|
||||
RELAY_CLOSE_CODE.HOST_OFFLINE,
|
||||
'relay connection rejected'
|
||||
)
|
||||
})
|
||||
|
||||
// A live host is present: the 4404 there is an attach deadline, not absence.
|
||||
it('never names a cause while the host control is connected', async () => {
|
||||
const { registry, activate } = createRegistry()
|
||||
const control = new FakeSocket()
|
||||
await activate(control as unknown as WebSocket, 1)
|
||||
|
||||
const phone = await dialPhone(registry)
|
||||
expect(phone.close).not.toHaveBeenCalled()
|
||||
expect(control.send).toHaveBeenCalledWith(expect.stringContaining('"type":"conn-open"'))
|
||||
})
|
||||
})
|
||||
@@ -503,12 +503,19 @@ describePostgres('PostgreSQL transaction recovery', () => {
|
||||
const directorLockOrder: string[] = []
|
||||
const assignmentDatabase = new TransactionProbeDatabase(database, async (phase, sql) => {
|
||||
if (phase === 'before') {
|
||||
if (sql.includes('FROM relay_assignments WHERE user_id = ?')) {
|
||||
// Only locked statements reach this hook, so classifying the pin read
|
||||
// is what proves it stays unlocked: if it ever grows a FOR UPDATE it
|
||||
// shows up in the order below instead of silently joining the queue.
|
||||
if (sql.includes('SELECT cell_id FROM relay_assignments')) {
|
||||
directorLockOrder.push('pin-read')
|
||||
} else if (sql.includes('FROM relay_assignments WHERE user_id = ?')) {
|
||||
directorLockOrder.push('assignment')
|
||||
} else if (sql.includes('FROM relay_assignment_activity_leases')) {
|
||||
directorLockOrder.push('activity')
|
||||
} else if (sql.includes('FROM relay_cells ORDER BY')) {
|
||||
directorLockOrder.push('cell-inventory')
|
||||
} else if (sql.includes('FROM relay_cells WHERE cell_id IN')) {
|
||||
directorLockOrder.push('cell-rows')
|
||||
} else if (sql.includes('FROM relay_cells WHERE cell_id = ?')) {
|
||||
directorLockOrder.push('cell')
|
||||
}
|
||||
@@ -530,11 +537,14 @@ describePostgres('PostgreSQL transaction recovery', () => {
|
||||
})
|
||||
await expect(legacyTransaction).resolves.toBeUndefined()
|
||||
expect(assignmentDatabase.attempts).toBe(2)
|
||||
// The retry still takes a cell row before the assignment row — the order
|
||||
// that avoids the legacy cycle — but only the pinned row, never the
|
||||
// inventory.
|
||||
expect(directorLockOrder).toEqual([
|
||||
'assignment',
|
||||
'activity',
|
||||
'cell',
|
||||
'cell-inventory',
|
||||
'cell-rows',
|
||||
'assignment',
|
||||
'activity'
|
||||
])
|
||||
|
||||
@@ -133,10 +133,15 @@
|
||||
"google_logging_metric.relay_snapshot",
|
||||
"google_monitoring_alert_policy.relay_assignment_5xx",
|
||||
"google_monitoring_alert_policy.relay_assignment_edge_429",
|
||||
"google_monitoring_alert_policy.relay_cell_process_exit",
|
||||
"google_monitoring_alert_policy.relay_cloud_nat_port_drops",
|
||||
"google_monitoring_alert_policy.relay_cloud_sql_backends",
|
||||
"google_monitoring_alert_policy.relay_cloud_sql_checkpoint_loop",
|
||||
"google_monitoring_alert_policy.relay_cloud_sql_disk",
|
||||
"google_monitoring_alert_policy.relay_custom",
|
||||
"google_monitoring_alert_policy.relay_gce_connection_headroom",
|
||||
"google_monitoring_alert_policy.relay_postgres_retry_exhausted",
|
||||
"google_monitoring_dashboard.relay_incident",
|
||||
"google_project_iam_custom_role.github_production_relay_capacity_mutation",
|
||||
"google_project_iam_custom_role.github_relay_asia_topology_mutation",
|
||||
"google_project_iam_custom_role.github_relay_asia_topology_read",
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import {
|
||||
RELAY_REPOSITORY_ROOT,
|
||||
relayTreePath,
|
||||
relayWorkflowPath
|
||||
} from './relay-repository.mjs'
|
||||
|
||||
const SHA = /^[a-f0-9]{40}$/
|
||||
|
||||
// Every file that decides how relay evidence is produced, sealed, verified, and then spent against
|
||||
// production; identical content across two commits is what makes the older commit's verdict binding.
|
||||
export const TRUSTED_EVIDENCE_CODE_PATHS = [
|
||||
// Produces and seals the 15-minute dry-run evidence.
|
||||
relayWorkflowPath('monitor-relay-production.yml'),
|
||||
relayWorkflowPath('monitor-relay-production-job.yml'),
|
||||
// Download it, verify its authority, and mutate production on it.
|
||||
relayWorkflowPath('deploy-relay-production-same-cap.yml'),
|
||||
relayWorkflowPath('deploy-relay-production-same-cap-job.yml'),
|
||||
relayWorkflowPath('operate-relay-production-rehome.yml'),
|
||||
relayWorkflowPath('operate-relay-production-rehome-job.yml'),
|
||||
// Sealing, verification, the wave/canary authority, and the path constants below.
|
||||
relayTreePath('dev/scripts/relay-evidence-code-provenance.mjs'),
|
||||
relayTreePath('dev/scripts/relay-monitor-evidence.mjs'),
|
||||
relayTreePath('dev/scripts/relay-production-same-cap-wave.mjs'),
|
||||
relayTreePath('dev/scripts/relay-repository.mjs'),
|
||||
// Every other script those jobs run against live production.
|
||||
relayTreePath('dev/scripts/infra.mjs'),
|
||||
relayTreePath('dev/scripts/operate-relay-regional-rehome.mjs'),
|
||||
relayTreePath('dev/scripts/prepare-relay-production-capacity-canary.mjs'),
|
||||
relayTreePath('dev/scripts/probe-relay-rehome-trust.mjs'),
|
||||
relayTreePath('dev/scripts/validate-relay-capacity-plan.mjs'),
|
||||
relayTreePath('dev/scripts/verify-relay-capacity-transition.mjs'),
|
||||
// The monitor itself and the live preflight recheck, plus anything that changes their behaviour.
|
||||
relayTreePath('apps/relay-ops'),
|
||||
relayTreePath('package.json'),
|
||||
relayTreePath('pnpm-lock.yaml'),
|
||||
relayTreePath('pnpm-workspace.yaml'),
|
||||
// The Cloud SQL rollout lease every mutation job takes and releases.
|
||||
'.github/actions/cloud-sql-rollout-lease'
|
||||
]
|
||||
|
||||
function git(root, args) {
|
||||
const result = spawnSync('git', ['-C', root, ...args], { encoding: 'utf8' })
|
||||
if (result.error) throw new Error('relay evidence provenance cannot run git')
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Accepts evidence sealed at a different commit only when the current commit descends from it and
|
||||
* every trusted path is byte-identical, so the verdict provably came from this exact code. Anything
|
||||
* git cannot answer (no checkout, unknown commit, shallow clone) fails closed.
|
||||
*/
|
||||
export function requireSameEvidenceCode({
|
||||
sealedSha,
|
||||
currentSha,
|
||||
label,
|
||||
repositoryRoot = fileURLToPath(RELAY_REPOSITORY_ROOT)
|
||||
}) {
|
||||
if (!SHA.test(sealedSha ?? '') || !SHA.test(currentSha ?? '')) {
|
||||
throw new Error(`${label} commit is invalid`)
|
||||
}
|
||||
if (sealedSha === currentSha) return
|
||||
if (git(repositoryRoot, ['rev-parse', '--git-dir']).status !== 0) {
|
||||
throw new Error(`${label} commit cannot be compared without a git checkout`)
|
||||
}
|
||||
for (const sha of [sealedSha, currentSha]) {
|
||||
if (git(repositoryRoot, ['rev-parse', '--verify', '--quiet', `${sha}^{commit}`]).status !== 0) {
|
||||
throw new Error(
|
||||
`${label} commit ${sha} is unknown to this checkout; check out with fetch-depth: 0`
|
||||
)
|
||||
}
|
||||
}
|
||||
const ancestry = git(repositoryRoot, ['merge-base', '--is-ancestor', sealedSha, currentSha])
|
||||
if (ancestry.status === 1) {
|
||||
throw new Error(`${label} commit ${sealedSha} is not an ancestor of ${currentSha}`)
|
||||
}
|
||||
if (ancestry.status !== 0) {
|
||||
throw new Error(`${label} commit ancestry could not be determined`)
|
||||
}
|
||||
const diff = git(repositoryRoot, [
|
||||
'diff',
|
||||
'--name-only',
|
||||
sealedSha,
|
||||
currentSha,
|
||||
'--',
|
||||
...TRUSTED_EVIDENCE_CODE_PATHS
|
||||
])
|
||||
if (diff.status !== 0) throw new Error(`${label} commit comparison failed`)
|
||||
const changed = diff.stdout.split('\n').filter(Boolean)
|
||||
if (changed.length > 0) {
|
||||
throw new Error(`${label} code changed after it was sealed: ${changed.join(',')}`)
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import { createHash } from 'node:crypto'
|
||||
import { chmod, readFile, readdir, stat, writeFile } from 'node:fs/promises'
|
||||
import { basename, join, resolve } from 'node:path'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { requireSameEvidenceCode } from './relay-evidence-code-provenance.mjs'
|
||||
|
||||
const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{1,127}$/
|
||||
const SHA = /^[a-f0-9]{40}$/
|
||||
@@ -102,7 +103,7 @@ export async function createEvidenceManifest(argv) {
|
||||
return manifest
|
||||
}
|
||||
|
||||
async function readAndVerifyManifest(directory, expected) {
|
||||
async function readAndVerifyManifest(directory, expected, sameCodeCommit) {
|
||||
const manifest = JSON.parse(
|
||||
await readFile(join(directory, 'evidence-manifest.json'), 'utf8')
|
||||
)
|
||||
@@ -111,11 +112,23 @@ async function readAndVerifyManifest(directory, expected) {
|
||||
manifest.incidentId !== expected.incidentId ||
|
||||
manifest.runId !== expected.runId ||
|
||||
manifest.runAttempt !== expected.runAttempt ||
|
||||
manifest.commitSha !== expected.commitSha ||
|
||||
manifest.mode !== expected.mode
|
||||
!SHA.test(manifest.commitSha ?? '') ||
|
||||
manifest.mode !== expected.mode ||
|
||||
(!sameCodeCommit && manifest.commitSha !== expected.commitSha)
|
||||
) {
|
||||
throw new Error('relay monitor evidence provenance does not match')
|
||||
}
|
||||
// Unrelated merges land on main every few minutes, so the deployer resolves a newer commit than
|
||||
// the monitor it must trust; identical monitor and mutation code is the property the SHA stood in
|
||||
// for. Restore and mutation keep the exact-SHA bind: both run at the commit that sealed them.
|
||||
if (sameCodeCommit) {
|
||||
requireSameEvidenceCode({
|
||||
sealedSha: manifest.commitSha,
|
||||
currentSha: expected.commitSha,
|
||||
label: 'relay monitor evidence',
|
||||
...sameCodeCommit
|
||||
})
|
||||
}
|
||||
const names = Object.keys(manifest.files ?? {})
|
||||
if (!names.includes(`${expected.incidentId}.state.json`)) {
|
||||
throw new Error('relay monitor evidence has no durable state')
|
||||
@@ -209,12 +222,12 @@ function validCompletedDryRunState(state, expected, nowMs, maxAgeMs) {
|
||||
)
|
||||
}
|
||||
|
||||
export async function verifyDryRunAuthority(argv, now = Date.now) {
|
||||
export async function verifyDryRunAuthority(argv, now = Date.now, repositoryRoot) {
|
||||
const values = argumentsByName(argv)
|
||||
const directory = resolve(values.directory ?? '')
|
||||
const expected = provenance(values)
|
||||
if (expected.mode !== 'dry-run') throw new Error('relay mutation requires dry-run evidence')
|
||||
const manifest = await readAndVerifyManifest(directory, expected)
|
||||
const manifest = await readAndVerifyManifest(directory, expected, { repositoryRoot })
|
||||
const state = JSON.parse(
|
||||
await readFile(join(directory, `${expected.incidentId}.state.json`), 'utf8')
|
||||
)
|
||||
|
||||
@@ -1,9 +1,15 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { dirname, join } from 'node:path'
|
||||
import test from 'node:test'
|
||||
import { relayWorkflowPath, relayWorkflowUrl } from './relay-repository.mjs'
|
||||
import { TRUSTED_EVIDENCE_CODE_PATHS } from './relay-evidence-code-provenance.mjs'
|
||||
import {
|
||||
RELAY_REPOSITORY_ROOT,
|
||||
relayWorkflowPath,
|
||||
relayWorkflowUrl
|
||||
} from './relay-repository.mjs'
|
||||
import {
|
||||
createEvidenceManifest,
|
||||
verifyDryRunAuthority,
|
||||
@@ -12,7 +18,7 @@ import {
|
||||
} from './relay-monitor-evidence.mjs'
|
||||
|
||||
const now = Date.parse('2026-07-28T12:00:00.000Z')
|
||||
const provenance = [
|
||||
const provenanceFor = (commitSha) => [
|
||||
'--incident-id',
|
||||
'relay-123',
|
||||
'--run-id',
|
||||
@@ -20,10 +26,11 @@ const provenance = [
|
||||
'--run-attempt',
|
||||
'1',
|
||||
'--commit-sha',
|
||||
'a'.repeat(40),
|
||||
commitSha,
|
||||
'--mode',
|
||||
'dry-run'
|
||||
]
|
||||
const provenance = provenanceFor('a'.repeat(40))
|
||||
const selector = {
|
||||
generation: 2,
|
||||
membership: {
|
||||
@@ -513,3 +520,157 @@ test('monitor uses a reusable job so exact job_workflow_ref is present', async (
|
||||
assert.match(job, /workflow_call:/)
|
||||
assert.match(job, /environment: production/)
|
||||
})
|
||||
|
||||
function gitIn(root, ...args) {
|
||||
return execFileSync('git', ['-C', root, ...args], { encoding: 'utf8' }).trim()
|
||||
}
|
||||
|
||||
// A real repository shaped like main under unrelated merge traffic: one sealed commit, a
|
||||
// descendant that only touched untrusted files, a descendant that touched the monitor, and a
|
||||
// sibling that never descended from the seal.
|
||||
async function trustedCodeRepository() {
|
||||
const root = await mkdtemp(join(tmpdir(), 'relay-evidence-repository-'))
|
||||
gitIn(root, 'init', '--quiet')
|
||||
gitIn(root, 'config', 'user.email', 'relay@example.test')
|
||||
gitIn(root, 'config', 'user.name', 'Relay Evidence Test')
|
||||
gitIn(root, 'config', 'commit.gpgsign', 'false')
|
||||
const commit = async (path, body, message) => {
|
||||
await mkdir(dirname(join(root, path)), { recursive: true })
|
||||
await writeFile(join(root, path), body)
|
||||
gitIn(root, 'add', '--all')
|
||||
gitIn(root, 'commit', '--quiet', '--no-verify', '--message', message)
|
||||
return gitIn(root, 'rev-parse', 'HEAD')
|
||||
}
|
||||
const base = await commit(
|
||||
'cloud/apps/relay-ops/src/incident-monitor.ts',
|
||||
'export const v = 1\n',
|
||||
'monitor'
|
||||
)
|
||||
const sealed = await commit('README.md', 'base\n', 'base')
|
||||
const sameCode = await commit('README.md', 'an unrelated merge\n', 'unrelated')
|
||||
const changedCode = await commit(
|
||||
'cloud/apps/relay-ops/src/incident-monitor.ts',
|
||||
'export const v = 2\n',
|
||||
'monitor change'
|
||||
)
|
||||
// Branches before the seal, so the seal is not in its history even though its code matches.
|
||||
gitIn(root, 'checkout', '--quiet', '--detach', base)
|
||||
const sibling = await commit('README.md', 'a divergent line\n', 'divergent')
|
||||
return { root, sealed, sameCode, changedCode, sibling }
|
||||
}
|
||||
|
||||
const authorityAt = (directory, commitSha, repositoryRoot) => verifyDryRunAuthority(
|
||||
[
|
||||
'--directory',
|
||||
directory,
|
||||
...provenanceFor(commitSha),
|
||||
'--required-migration-policy',
|
||||
'strict'
|
||||
],
|
||||
() => now,
|
||||
repositoryRoot
|
||||
)
|
||||
|
||||
test('accepts dry-run evidence sealed by identical code at an ancestor commit', async () => {
|
||||
const repository = await trustedCodeRepository()
|
||||
const directory = await evidenceDirectory()
|
||||
try {
|
||||
await createEvidenceManifest([
|
||||
'--directory',
|
||||
directory,
|
||||
...provenanceFor(repository.sealed)
|
||||
])
|
||||
// An exact match never consults git: a root with no checkout at all still verifies.
|
||||
await assert.doesNotReject(authorityAt(directory, repository.sealed, directory))
|
||||
await assert.doesNotReject(authorityAt(directory, repository.sameCode, repository.root))
|
||||
} finally {
|
||||
await rm(repository.root, { recursive: true, force: true })
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('rejects dry-run evidence whose monitor code or lineage differs', async () => {
|
||||
const repository = await trustedCodeRepository()
|
||||
const directory = await evidenceDirectory()
|
||||
try {
|
||||
await createEvidenceManifest([
|
||||
'--directory',
|
||||
directory,
|
||||
...provenanceFor(repository.sealed)
|
||||
])
|
||||
await assert.rejects(
|
||||
authorityAt(directory, repository.changedCode, repository.root),
|
||||
/code changed after it was sealed: cloud\/apps\/relay-ops\/src\/incident-monitor\.ts/
|
||||
)
|
||||
await assert.rejects(
|
||||
authorityAt(directory, repository.sibling, repository.root),
|
||||
/is not an ancestor of/
|
||||
)
|
||||
// Fails closed: a shallow clone that never fetched the sealed commit proves nothing.
|
||||
await assert.rejects(
|
||||
authorityAt(directory, 'f'.repeat(40), repository.root),
|
||||
/unknown to this checkout/
|
||||
)
|
||||
// Fails closed: no checkout to compare against.
|
||||
await assert.rejects(
|
||||
authorityAt(directory, repository.sameCode, directory),
|
||||
/cannot be compared without a git checkout/
|
||||
)
|
||||
} finally {
|
||||
await rm(repository.root, { recursive: true, force: true })
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test('keeps restore and mutation bound to the exact sealing commit', async () => {
|
||||
const repository = await trustedCodeRepository()
|
||||
const directory = await evidenceDirectory()
|
||||
try {
|
||||
await createEvidenceManifest([
|
||||
'--directory',
|
||||
directory,
|
||||
...provenanceFor(repository.sealed)
|
||||
])
|
||||
await assert.rejects(
|
||||
verifyRestoredEvidence([
|
||||
'--directory',
|
||||
directory,
|
||||
...provenanceFor(repository.sameCode)
|
||||
]),
|
||||
/provenance does not match/
|
||||
)
|
||||
await assert.rejects(
|
||||
verifyMutationEvidence(
|
||||
[
|
||||
'--directory',
|
||||
directory,
|
||||
...provenanceFor(repository.sameCode),
|
||||
'--mutation-mode',
|
||||
'execute',
|
||||
'--source-cell-id',
|
||||
'c1',
|
||||
'--director-origin',
|
||||
'https://relay.example'
|
||||
],
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' },
|
||||
async () => Response.json({ selector }),
|
||||
() => now
|
||||
),
|
||||
/provenance does not match/
|
||||
)
|
||||
} finally {
|
||||
await rm(repository.root, { recursive: true, force: true })
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
// A trusted path that no longer exists silently stops being compared, so the same-code rule would
|
||||
// pass over code it was written to pin.
|
||||
test('every trusted provenance path exists in this checkout', async () => {
|
||||
for (const path of TRUSTED_EVIDENCE_CODE_PATHS) {
|
||||
await assert.doesNotReject(
|
||||
stat(new URL(path, RELAY_REPOSITORY_ROOT)),
|
||||
`${path} is missing`
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { requireSameEvidenceCode } from './relay-evidence-code-provenance.mjs'
|
||||
|
||||
export const SAME_CAP_CELLS = [
|
||||
'production-gce-c7', 'production-gce-c8', 'production-gce-c9', 'production-gce-c10',
|
||||
@@ -85,10 +86,10 @@ export function canaryAuthority(input) {
|
||||
}
|
||||
}
|
||||
|
||||
export function verifyCanaryAuthority(authority, expected) {
|
||||
export function verifyCanaryAuthority(authority, expected, repositoryRoot) {
|
||||
if (
|
||||
authority?.v !== 1 ||
|
||||
authority.commitSha !== expected.commitSha ||
|
||||
!/^[0-9a-f]{40}$/.test(authority.commitSha ?? '') ||
|
||||
authority.runId !== expected.runId ||
|
||||
authority.targetDigest !== expected.targetDigest ||
|
||||
authority.rollbackDigest !== expected.rollbackDigest ||
|
||||
@@ -96,6 +97,14 @@ export function verifyCanaryAuthority(authority, expected) {
|
||||
authority.rehomeGeneration !== Number(expected.rehomeGeneration) ||
|
||||
!SAME_CAP_CELLS.includes(authority.cellId)
|
||||
) throw new Error('canary authority does not match this batch')
|
||||
// The batch dispatch resolves main after the canary sealed, so bind to the same code, not the
|
||||
// same SHA; every field above still pins this batch to that exact canary.
|
||||
requireSameEvidenceCode({
|
||||
sealedSha: authority.commitSha,
|
||||
currentSha: expected.commitSha,
|
||||
label: 'relay same-cap canary authority',
|
||||
repositoryRoot
|
||||
})
|
||||
return authority
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { test } from 'node:test'
|
||||
import {
|
||||
canaryAuthority,
|
||||
@@ -104,3 +108,66 @@ test('seals and verifies canary authority for later batches', () => {
|
||||
rehomeGeneration: '4'
|
||||
}), /does not match/)
|
||||
})
|
||||
|
||||
function gitIn(root, ...args) {
|
||||
return execFileSync('git', ['-C', root, ...args], { encoding: 'utf8' }).trim()
|
||||
}
|
||||
|
||||
async function canaryRepository() {
|
||||
const root = await mkdtemp(join(tmpdir(), 'relay-same-cap-canary-'))
|
||||
gitIn(root, 'init', '--quiet')
|
||||
gitIn(root, 'config', 'user.email', 'relay@example.test')
|
||||
gitIn(root, 'config', 'user.name', 'Relay Wave Test')
|
||||
gitIn(root, 'config', 'commit.gpgsign', 'false')
|
||||
const commit = async (path, body, message) => {
|
||||
await mkdir(dirname(join(root, path)), { recursive: true })
|
||||
await writeFile(join(root, path), body)
|
||||
gitIn(root, 'add', '--all')
|
||||
gitIn(root, 'commit', '--quiet', '--no-verify', '--message', message)
|
||||
return gitIn(root, 'rev-parse', 'HEAD')
|
||||
}
|
||||
const sealed = await commit(
|
||||
'cloud/dev/scripts/relay-production-same-cap-wave.mjs',
|
||||
'export const v = 1\n',
|
||||
'wave'
|
||||
)
|
||||
const sameCode = await commit('README.md', 'an unrelated merge\n', 'unrelated')
|
||||
const changedCode = await commit(
|
||||
'cloud/dev/scripts/relay-production-same-cap-wave.mjs',
|
||||
'export const v = 2\n',
|
||||
'wave change'
|
||||
)
|
||||
return { root, sealed, sameCode, changedCode }
|
||||
}
|
||||
|
||||
test('a batch trusts a canary sealed by identical code at an ancestor commit', async () => {
|
||||
const repository = await canaryRepository()
|
||||
try {
|
||||
const authority = canaryAuthority({
|
||||
cellIds: 'production-gce-c7',
|
||||
targetDigest,
|
||||
rollbackDigest,
|
||||
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7`,
|
||||
commitSha: repository.sealed,
|
||||
runId: '42',
|
||||
selectorGeneration: '11',
|
||||
rehomeGeneration: '4'
|
||||
})
|
||||
const verifyAt = (commitSha, repositoryRoot) => verifyCanaryAuthority(authority, {
|
||||
commitSha,
|
||||
runId: '42',
|
||||
targetDigest,
|
||||
rollbackDigest,
|
||||
selectorGeneration: '13',
|
||||
rehomeGeneration: '4'
|
||||
}, repositoryRoot)
|
||||
assert.equal(verifyAt(repository.sameCode, repository.root).cellId, 'production-gce-c7')
|
||||
assert.throws(
|
||||
() => verifyAt(repository.changedCode, repository.root),
|
||||
/code changed after it was sealed/
|
||||
)
|
||||
assert.throws(() => verifyAt('f'.repeat(40), repository.root), /unknown to this checkout/)
|
||||
} finally {
|
||||
await rm(repository.root, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { relative } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
// Single place naming the repository the Relay workflows live in and where their files sit. The
|
||||
// public-repo copy moves this tree under cloud/, prefixes every workflow filename, and changes the
|
||||
@@ -11,6 +13,19 @@ export const RELAY_WORKFLOW_FILE_PREFIX = 'cloud-'
|
||||
// this tree moves under cloud/, so the depth changes at the copy even though the layout does not.
|
||||
export const RELAY_WORKFLOW_DIRECTORY = new URL('../../../.github/workflows/', import.meta.url)
|
||||
|
||||
// Repository root, derived from the one directory above that already tracks the copy's depth.
|
||||
export const RELAY_REPOSITORY_ROOT = new URL('../../', RELAY_WORKFLOW_DIRECTORY)
|
||||
|
||||
// Repository-relative path for a file in this tree. The prefix is 'cloud/' here and empty where
|
||||
// the tree is the repository root, so callers naming git paths never restate the layout.
|
||||
export function relayTreePath(suffix) {
|
||||
const prefix = relative(
|
||||
fileURLToPath(RELAY_REPOSITORY_ROOT),
|
||||
fileURLToPath(new URL('../../', import.meta.url))
|
||||
).split(/[\\/]/).filter(Boolean)
|
||||
return [...prefix, suffix].join('/')
|
||||
}
|
||||
|
||||
export function relayWorkflowFile(name) {
|
||||
return `${RELAY_WORKFLOW_FILE_PREFIX}${name}`
|
||||
}
|
||||
|
||||
@@ -242,6 +242,7 @@ resource "google_compute_instance_template" "relay_gce_cell" {
|
||||
artifact_registry_host = "${var.region}-docker.pkg.dev"
|
||||
relay_image = each.value.image
|
||||
cloud_sql_proxy_image = var.relay_gce_cloud_sql_proxy_image
|
||||
cloud_sql_private_ip = var.relay_cloud_sql_private_ip
|
||||
# Keep cell-only plans independent from unrelated database configuration drift.
|
||||
cloud_sql_connection_name = local.relay_database_connection_name
|
||||
})
|
||||
|
||||
@@ -42,6 +42,12 @@ resource "google_compute_router_nat" "relay_gce" {
|
||||
router = google_compute_router.relay_gce[0].name
|
||||
nat_ip_allocate_option = "AUTO_ONLY"
|
||||
source_subnetwork_ip_ranges_to_nat = "LIST_OF_SUBNETWORKS"
|
||||
# Cells reach Cloud SQL's public IP through this NAT. The static default of 64 ports per VM
|
||||
# filled during the 2026-09-04 incident and every cell's proxy dial timed out at once.
|
||||
enable_dynamic_port_allocation = true
|
||||
enable_endpoint_independent_mapping = false
|
||||
min_ports_per_vm = 64
|
||||
max_ports_per_vm = 4096
|
||||
|
||||
subnetwork {
|
||||
name = google_compute_subnetwork.relay_gce[0].id
|
||||
@@ -85,6 +91,12 @@ resource "google_compute_router_nat" "relay_gce_additional" {
|
||||
router = google_compute_router.relay_gce_additional[each.key].name
|
||||
nat_ip_allocate_option = "AUTO_ONLY"
|
||||
source_subnetwork_ip_ranges_to_nat = "LIST_OF_SUBNETWORKS"
|
||||
# Cells reach Cloud SQL's public IP through this NAT. The static default of 64 ports per VM
|
||||
# filled during the 2026-09-04 incident and every cell's proxy dial timed out at once.
|
||||
enable_dynamic_port_allocation = true
|
||||
enable_endpoint_independent_mapping = false
|
||||
min_ports_per_vm = 64
|
||||
max_ports_per_vm = 4096
|
||||
|
||||
subnetwork {
|
||||
name = google_compute_subnetwork.relay_gce_additional[each.key].id
|
||||
|
||||
@@ -109,6 +109,9 @@ docker run --detach \
|
||||
--user 0:0 \
|
||||
--volume "$${cloudsql_dir}:/cloudsql" \
|
||||
'${cloud_sql_proxy_image}' \
|
||||
%{ if cloud_sql_private_ip ~}
|
||||
--private-ip \
|
||||
%{ endif ~}
|
||||
--unix-socket=/cloudsql \
|
||||
'${cloud_sql_connection_name}'
|
||||
|
||||
|
||||
@@ -37,6 +37,16 @@ locals {
|
||||
description = "Relay PostgreSQL transactions that exhausted bounded retry."
|
||||
filter = "((resource.type=\"cloud_run_revision\" AND (${local.relay_service_log_filter})) OR resource.type=\"gce_instance\") AND jsonPayload.event=\"orca_relay_postgres_transaction_exhausted\""
|
||||
}
|
||||
cell_process_exit = {
|
||||
# The docker event stream is the only per-exit line: the relay's own crash footer only
|
||||
# appears for unhandled rejections, and `container start` also counts healthy first boots.
|
||||
description = "Relay cell container exits, one Docker `container die` event per process exit."
|
||||
filter = "resource.type=\"gce_instance\" AND logName=\"projects/${var.project_id}/logs/cos_system\" AND jsonPayload.SYSLOG_IDENTIFIER=\"docker\" AND jsonPayload.MESSAGE:\"container die\" AND jsonPayload.MESSAGE:\"name=orca-relay)\""
|
||||
}
|
||||
cloud_sql_wal_checkpoint = {
|
||||
description = "Cloud SQL checkpoints triggered by WAL volume instead of the timed schedule; a sustained run is the fsync loop that stalled every relay process at once on 2026-09-04."
|
||||
filter = "resource.type=\"cloudsql_database\" AND resource.labels.database_id=\"${var.project_id}:${local.relay_database_instance_name}\" AND textPayload:\"checkpoint starting: wal\""
|
||||
}
|
||||
}
|
||||
|
||||
relay_runtime_metrics = {
|
||||
@@ -201,7 +211,8 @@ resource "google_logging_metric" "relay_snapshot" {
|
||||
label_extractors = {
|
||||
role = "EXTRACT(jsonPayload.role)"
|
||||
cell_id = "EXTRACT(jsonPayload.cellId)"
|
||||
region = "EXTRACT(jsonPayload.region)"
|
||||
# No region label: adding one replaces all 21 live metrics (label change = delete+create),
|
||||
# which resets history and blanks the relay alert policies during the swap.
|
||||
}
|
||||
|
||||
metric_descriptor {
|
||||
@@ -220,12 +231,6 @@ resource "google_logging_metric" "relay_snapshot" {
|
||||
value_type = "STRING"
|
||||
description = "Durable relay cell identifier."
|
||||
}
|
||||
|
||||
labels {
|
||||
key = "region"
|
||||
value_type = "STRING"
|
||||
description = "Coarse Relay region."
|
||||
}
|
||||
}
|
||||
|
||||
bucket_options {
|
||||
@@ -523,3 +528,280 @@ resource "google_monitoring_alert_policy" "relay_cloud_sql_backends" {
|
||||
mime_type = "text/markdown"
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_monitoring_alert_policy" "relay_cloud_sql_checkpoint_loop" {
|
||||
project = var.project_id
|
||||
display_name = "Orca Relay: Cloud SQL checkpoint loop"
|
||||
combiner = "OR"
|
||||
enabled = true
|
||||
notification_channels = var.relay_alert_notification_channels
|
||||
|
||||
conditions {
|
||||
display_name = "WAL-triggered checkpoints above 3 in 5 minutes"
|
||||
|
||||
condition_threshold {
|
||||
filter = "resource.type=\"cloudsql_database\" AND metric.type=\"logging.googleapis.com/user/orca_relay_cloud_sql_wal_checkpoint\""
|
||||
comparison = "COMPARISON_GT"
|
||||
threshold_value = 3
|
||||
duration = "300s"
|
||||
|
||||
aggregations {
|
||||
alignment_period = "300s"
|
||||
per_series_aligner = "ALIGN_SUM"
|
||||
cross_series_reducer = "REDUCE_SUM"
|
||||
}
|
||||
|
||||
trigger {
|
||||
count = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
documentation {
|
||||
content = "Healthy operation is one timed checkpoint every 5 minutes. Repeated `checkpoint starting: wal` lines mean WAL is outrunning `max_wal_size` and every checkpoint fsync stalls all relay SQL for seconds. Check `checkpoint complete` sync= times and disk write throughput against the PD-SSD ceiling; the fix is disk size and `max_wal_size` in the Terraform root that owns the instance (orca-cloud `infra/terraform-foundation`)."
|
||||
mime_type = "text/markdown"
|
||||
}
|
||||
|
||||
depends_on = [google_logging_metric.relay_incident]
|
||||
}
|
||||
|
||||
resource "google_monitoring_alert_policy" "relay_cloud_sql_disk" {
|
||||
project = var.project_id
|
||||
display_name = "Orca Relay: Cloud SQL disk utilization"
|
||||
combiner = "OR"
|
||||
enabled = true
|
||||
notification_channels = var.relay_alert_notification_channels
|
||||
|
||||
conditions {
|
||||
display_name = "Cloud SQL disk above 70%"
|
||||
|
||||
condition_threshold {
|
||||
filter = "resource.type=\"cloudsql_database\" AND resource.label.\"database_id\"=\"${var.project_id}:${local.relay_database_instance_name}\" AND metric.type=\"cloudsql.googleapis.com/database/disk/utilization\""
|
||||
comparison = "COMPARISON_GT"
|
||||
threshold_value = 0.7
|
||||
duration = "600s"
|
||||
|
||||
aggregations {
|
||||
alignment_period = "300s"
|
||||
per_series_aligner = "ALIGN_MAX"
|
||||
}
|
||||
|
||||
trigger {
|
||||
count = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
documentation {
|
||||
content = "The shared auth/relay Cloud SQL disk is filling. `refresh_tokens` is the largest table and grows without pruning; grow the disk (IOPS scale with size) before it reaches the WAL checkpoint loop, and prune revoked token rows."
|
||||
mime_type = "text/markdown"
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_monitoring_alert_policy" "relay_cloud_nat_port_drops" {
|
||||
count = local.relay_gce_configured ? 1 : 0
|
||||
|
||||
project = var.project_id
|
||||
display_name = "Orca Relay: Cloud NAT port exhaustion"
|
||||
combiner = "OR"
|
||||
enabled = true
|
||||
notification_channels = var.relay_alert_notification_channels
|
||||
|
||||
conditions {
|
||||
display_name = "NAT packets dropped for lack of ports"
|
||||
|
||||
condition_threshold {
|
||||
filter = "resource.type=\"nat_gateway\" AND resource.label.\"gateway_name\"=monitoring.regex.full_match(\"${local.relay_gce_name}(-.*)?\") AND metric.type=\"router.googleapis.com/nat/dropped_sent_packets_count\" AND metric.label.\"reason\"=\"OUT_OF_RESOURCES\""
|
||||
comparison = "COMPARISON_GT"
|
||||
threshold_value = 0
|
||||
duration = "120s"
|
||||
|
||||
aggregations {
|
||||
alignment_period = "60s"
|
||||
per_series_aligner = "ALIGN_SUM"
|
||||
cross_series_reducer = "REDUCE_SUM"
|
||||
group_by_fields = ["resource.label.\"gateway_name\""]
|
||||
}
|
||||
|
||||
trigger {
|
||||
count = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
documentation {
|
||||
content = "Relay cells reach Cloud SQL's public IP through this NAT. Port exhaustion makes every cell's Cloud SQL Auth Proxy dial time out at once, which reads as a fleet-wide SQL stall with a healthy database. Check `nat/port_usage` per VM and raise `max_ports_per_vm` in `relay-gce-foundation.tf`, or move the database to a private IP."
|
||||
mime_type = "text/markdown"
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_monitoring_alert_policy" "relay_cell_process_exit" {
|
||||
project = var.project_id
|
||||
display_name = "Orca Relay: cell process exits"
|
||||
combiner = "OR"
|
||||
enabled = true
|
||||
notification_channels = var.relay_alert_notification_channels
|
||||
|
||||
conditions {
|
||||
display_name = "Cell container exits above 3 in 15 minutes"
|
||||
|
||||
condition_threshold {
|
||||
filter = "resource.type=\"gce_instance\" AND metric.type=\"logging.googleapis.com/user/orca_relay_cell_process_exit\""
|
||||
comparison = "COMPARISON_GT"
|
||||
threshold_value = 3
|
||||
duration = "0s"
|
||||
|
||||
aggregations {
|
||||
alignment_period = "900s"
|
||||
per_series_aligner = "ALIGN_SUM"
|
||||
cross_series_reducer = "REDUCE_SUM"
|
||||
group_by_fields = ["resource.label.\"instance_id\""]
|
||||
}
|
||||
|
||||
trigger {
|
||||
count = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
documentation {
|
||||
content = "A Relay GCE cell restarted its container more than three times in 15 minutes. Each exit drops every host and phone on that cell, and 201 exits went unpaged over 48 h on 2026-09-04. The instance hostname is `relay-<cell>-<suffix>`; read `jsonPayload.MESSAGE` on `cos_system` for the exit code and the container's own stderr for the stack before blaming MIG autoheal or load. A same-capacity roll is the remedy when the running image is behind."
|
||||
mime_type = "text/markdown"
|
||||
}
|
||||
|
||||
depends_on = [google_logging_metric.relay_incident]
|
||||
}
|
||||
|
||||
# Why: the four signals that had to be assembled by hand during the 2026-09-04 incident.
|
||||
resource "google_monitoring_dashboard" "relay_incident" {
|
||||
project = var.project_id
|
||||
|
||||
dashboard_json = jsonencode({
|
||||
displayName = "Orca Relay: incident overview"
|
||||
mosaicLayout = {
|
||||
columns = 12
|
||||
tiles = [
|
||||
{
|
||||
xPos = 0
|
||||
yPos = 0
|
||||
width = 3
|
||||
height = 4
|
||||
widget = {
|
||||
title = "Cloud SQL WAL checkpoints"
|
||||
xyChart = {
|
||||
dataSets = [{
|
||||
plotType = "LINE"
|
||||
targetAxis = "Y1"
|
||||
timeSeriesQuery = {
|
||||
timeSeriesFilter = {
|
||||
filter = "metric.type=\"logging.googleapis.com/user/orca_relay_cloud_sql_wal_checkpoint\" AND resource.type=\"cloudsql_database\""
|
||||
aggregation = {
|
||||
alignmentPeriod = "300s"
|
||||
perSeriesAligner = "ALIGN_SUM"
|
||||
crossSeriesReducer = "REDUCE_SUM"
|
||||
}
|
||||
}
|
||||
}
|
||||
}]
|
||||
yAxis = {
|
||||
label = "checkpoints"
|
||||
scale = "LINEAR"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
xPos = 3
|
||||
yPos = 0
|
||||
width = 3
|
||||
height = 4
|
||||
widget = {
|
||||
title = "Cloud NAT dropped packets"
|
||||
xyChart = {
|
||||
dataSets = [{
|
||||
plotType = "LINE"
|
||||
targetAxis = "Y1"
|
||||
timeSeriesQuery = {
|
||||
timeSeriesFilter = {
|
||||
filter = "metric.type=\"router.googleapis.com/nat/dropped_sent_packets_count\" AND resource.type=\"nat_gateway\" AND resource.label.\"gateway_name\"=monitoring.regex.full_match(\"${local.relay_gce_name}(-.*)?\")"
|
||||
aggregation = {
|
||||
alignmentPeriod = "60s"
|
||||
perSeriesAligner = "ALIGN_SUM"
|
||||
crossSeriesReducer = "REDUCE_SUM"
|
||||
groupByFields = ["resource.label.\"gateway_name\"", "metric.label.\"reason\""]
|
||||
}
|
||||
}
|
||||
}
|
||||
}]
|
||||
yAxis = {
|
||||
label = "packets"
|
||||
scale = "LINEAR"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
xPos = 6
|
||||
yPos = 0
|
||||
width = 3
|
||||
height = 4
|
||||
widget = {
|
||||
title = "Auth refresh 401s"
|
||||
xyChart = {
|
||||
dataSets = [{
|
||||
plotType = "LINE"
|
||||
targetAxis = "Y1"
|
||||
timeSeriesQuery = {
|
||||
timeSeriesFilter = {
|
||||
filter = "metric.type=\"logging.googleapis.com/user/orca_auth_refresh_401\""
|
||||
aggregation = {
|
||||
alignmentPeriod = "300s"
|
||||
perSeriesAligner = "ALIGN_SUM"
|
||||
crossSeriesReducer = "REDUCE_SUM"
|
||||
}
|
||||
}
|
||||
}
|
||||
}]
|
||||
yAxis = {
|
||||
label = "rejections"
|
||||
scale = "LINEAR"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
xPos = 9
|
||||
yPos = 0
|
||||
width = 3
|
||||
height = 4
|
||||
widget = {
|
||||
title = "Standing desktop controls (fleet sum)"
|
||||
xyChart = {
|
||||
dataSets = [{
|
||||
plotType = "LINE"
|
||||
targetAxis = "Y1"
|
||||
timeSeriesQuery = {
|
||||
timeSeriesFilter = {
|
||||
# ALIGN_MEAN, not ALIGN_SUM: each process reports its standing control count once per interval.
|
||||
filter = "metric.type=\"logging.googleapis.com/user/orca_relay_controls\""
|
||||
aggregation = {
|
||||
alignmentPeriod = "300s"
|
||||
perSeriesAligner = "ALIGN_MEAN"
|
||||
crossSeriesReducer = "REDUCE_SUM"
|
||||
}
|
||||
}
|
||||
}
|
||||
}]
|
||||
yAxis = {
|
||||
label = "controls"
|
||||
scale = "LINEAR"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
})
|
||||
|
||||
depends_on = [google_logging_metric.relay_incident, google_logging_metric.relay_snapshot]
|
||||
}
|
||||
|
||||
@@ -468,6 +468,12 @@ variable "relay_gce_fenced_cells" {
|
||||
default = []
|
||||
}
|
||||
|
||||
variable "relay_cloud_sql_private_ip" {
|
||||
type = bool
|
||||
description = "Dial Cloud SQL over its private IP inside this VPC instead of its public IP through Cloud NAT. Requires the foundation root's private services access peering to be applied first; a cell that cannot reach the private IP never becomes ready."
|
||||
default = false
|
||||
}
|
||||
|
||||
variable "relay_gce_cloud_sql_proxy_image" {
|
||||
type = string
|
||||
description = "Digest-pinned Cloud SQL Auth Proxy image used by private relay workers."
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
// Mirror of src/shared/relay-host-close-reason.ts in the Orca app repo half.
|
||||
// A host control socket may close with one of these as its WebSocket close
|
||||
// reason; the cell records it so a later phone rejection can name the cause.
|
||||
// Anything else (including the empty reason of an abrupt 1006) means "unknown",
|
||||
// which is what every peer that predates this file sends.
|
||||
export const RELAY_HOST_CLOSE_REASON = {
|
||||
SIGNED_OUT: 'signed-out'
|
||||
} as const
|
||||
|
||||
export type RelayHostCloseReason =
|
||||
(typeof RELAY_HOST_CLOSE_REASON)[keyof typeof RELAY_HOST_CLOSE_REASON]
|
||||
|
||||
const REASONS: readonly string[] = Object.values(RELAY_HOST_CLOSE_REASON)
|
||||
|
||||
export function relayHostCloseReasonFrom(value: unknown): RelayHostCloseReason | null {
|
||||
const text = typeof value === 'string' ? value : (value?.toString() ?? '')
|
||||
return REASONS.includes(text) ? (text as RelayHostCloseReason) : null
|
||||
}
|
||||
@@ -5,6 +5,7 @@ export * from './control-messages.js'
|
||||
export * from './control-continuity.js'
|
||||
export * from './credential-messages.js'
|
||||
export * from './director-messages.js'
|
||||
export * from './host-close-reason.js'
|
||||
export * from './host-proof-transcript.js'
|
||||
export * from './persistence-invariants.js'
|
||||
export * from './protocol-limits.js'
|
||||
|
||||
@@ -219,6 +219,7 @@ const WINDOWS_PACKAGE_TESTS = [
|
||||
'src/main/agent-hooks/windows-hook-payload-delivery.test.ts',
|
||||
'src/main/windows/windows-pty-job.win32.test.ts',
|
||||
'src/main/windows/windows-host-job.win32.test.ts',
|
||||
'src/main/windows-live-tree-kill.win32.test.ts',
|
||||
'src/main/wsl/wsl-runner.test.ts',
|
||||
'src/main/wsl/wsl-guest-environment.test.ts',
|
||||
'src/main/wsl/wsl-invocation-boundary.test.ts',
|
||||
|
||||
@@ -11,7 +11,12 @@ import { repairTranslatedValue } from './locale-translation-policy.mjs'
|
||||
|
||||
const SOURCE_EXTENSIONS = new Set(['.ts', '.tsx', '.js', '.jsx', '.mts', '.cts'])
|
||||
const SKIP_PATH_PARTS = new Set(['.git', 'dist', 'node_modules', 'out', '__snapshots__', 'assets'])
|
||||
const LOCALIZATION_FUNCTION_NAMES = new Set(['t', 'translate', 'translateMain', 'translateSearchKeyword'])
|
||||
const LOCALIZATION_FUNCTION_NAMES = new Set([
|
||||
't',
|
||||
'translate',
|
||||
'translateMain',
|
||||
'translateSearchKeyword'
|
||||
])
|
||||
const PLACEHOLDER_RE = /\{\{[^}]+\}\}/g
|
||||
const LOCALES_RELATIVE_DIR = path.join('src', 'renderer', 'src', 'i18n', 'locales')
|
||||
export const LOCALIZATION_SOURCE_ROOTS = [
|
||||
|
||||
@@ -19,6 +19,7 @@ type MobileHomeHostListProps = {
|
||||
hostAttempts: Record<string, number>
|
||||
hostLastConnected: Record<string, number | null>
|
||||
hostPairingRejected: Record<string, boolean>
|
||||
hostSignedOut: Record<string, boolean>
|
||||
hostPaths: Record<string, MobileConnectionPath>
|
||||
hostPendingPaths: Record<string, MobileConnectionPath | null>
|
||||
hosts: HostCatalogEntry[]
|
||||
@@ -40,6 +41,7 @@ export function MobileHomeHostList(props: MobileHomeHostListProps) {
|
||||
hostAttempts={props.hostAttempts}
|
||||
hostLastConnected={props.hostLastConnected}
|
||||
hostPairingRejected={props.hostPairingRejected}
|
||||
hostSignedOut={props.hostSignedOut}
|
||||
hostPaths={props.hostPaths}
|
||||
hostPendingPaths={props.hostPendingPaths}
|
||||
hostStates={props.hostStates}
|
||||
@@ -54,6 +56,7 @@ export function MobileHomeHostList(props: MobileHomeHostListProps) {
|
||||
props.hostAttempts,
|
||||
props.hostLastConnected,
|
||||
props.hostPairingRejected,
|
||||
props.hostSignedOut,
|
||||
props.hostPaths,
|
||||
props.hostPendingPaths,
|
||||
props.hostStates,
|
||||
@@ -91,6 +94,7 @@ type MobileHomeHostRowProps = Pick<
|
||||
| 'hostAttempts'
|
||||
| 'hostLastConnected'
|
||||
| 'hostPairingRejected'
|
||||
| 'hostSignedOut'
|
||||
| 'hostPaths'
|
||||
| 'hostPendingPaths'
|
||||
| 'hostStates'
|
||||
@@ -113,7 +117,8 @@ const MobileHomeHostRow = memo(function MobileHomeHostRow(props: MobileHomeHostR
|
||||
lastConnectedAt: props.hostLastConnected[item.id] ?? null,
|
||||
endpoint: item.endpoint,
|
||||
pendingPath: props.hostPendingPaths[item.id] ?? null,
|
||||
pairingRejected: props.hostPairingRejected[item.id] ?? false
|
||||
pairingRejected: props.hostPairingRejected[item.id] ?? false,
|
||||
hostSignedOut: props.hostSignedOut[item.id] ?? false
|
||||
})
|
||||
const open = useCallback(() => onOpen(item), [item, onOpen])
|
||||
const longPress = useCallback(() => onLongPress(item), [item, onLongPress])
|
||||
|
||||
@@ -143,6 +143,7 @@ export function MobileHomeScreen() {
|
||||
hostAttempts={data.hostAttempts}
|
||||
hostLastConnected={data.hostLastConnected}
|
||||
hostPairingRejected={data.hostPairingRejected}
|
||||
hostSignedOut={data.hostSignedOut}
|
||||
hostPaths={data.hostPaths}
|
||||
hostPendingPaths={data.hostPendingPaths}
|
||||
hosts={data.sortedHostCatalog}
|
||||
|
||||
@@ -5,12 +5,14 @@ export type HomeHostConnectionProjectionEntry = {
|
||||
path: MobileConnectionPath
|
||||
pendingPath: MobileConnectionPath | null
|
||||
pairingRejected: boolean
|
||||
hostSignedOut: boolean
|
||||
}
|
||||
|
||||
export type HomeHostConnectionProjection = {
|
||||
hostPaths: Record<string, MobileConnectionPath>
|
||||
hostPendingPaths: Record<string, MobileConnectionPath | null>
|
||||
hostPairingRejected: Record<string, boolean>
|
||||
hostSignedOut: Record<string, boolean>
|
||||
}
|
||||
|
||||
/** Build all host lookup maps while reading each connection entry once. */
|
||||
@@ -22,16 +24,19 @@ export function projectHomeHostConnections(
|
||||
const hostPaths = Object.create(null) as Record<string, MobileConnectionPath>
|
||||
const hostPendingPaths = Object.create(null) as Record<string, MobileConnectionPath | null>
|
||||
const hostPairingRejected = Object.create(null) as Record<string, boolean>
|
||||
const hostSignedOut = Object.create(null) as Record<string, boolean>
|
||||
|
||||
for (const { hostId, path, pendingPath, pairingRejected } of entries) {
|
||||
for (const { hostId, path, pendingPath, pairingRejected, hostSignedOut: signedOut } of entries) {
|
||||
hostPaths[hostId] = path
|
||||
hostPendingPaths[hostId] = pendingPath
|
||||
hostPairingRejected[hostId] = pairingRejected
|
||||
hostSignedOut[hostId] = signedOut
|
||||
}
|
||||
|
||||
Object.setPrototypeOf(hostPaths, Object.prototype)
|
||||
Object.setPrototypeOf(hostPendingPaths, Object.prototype)
|
||||
Object.setPrototypeOf(hostPairingRejected, Object.prototype)
|
||||
Object.setPrototypeOf(hostSignedOut, Object.prototype)
|
||||
|
||||
return { hostPaths, hostPendingPaths, hostPairingRejected }
|
||||
return { hostPaths, hostPendingPaths, hostPairingRejected, hostSignedOut }
|
||||
}
|
||||
|
||||
@@ -179,6 +179,7 @@ export function useMobileHomeData() {
|
||||
connectedHosts,
|
||||
hostCatalog,
|
||||
hostPairingRejected: hostConnectionProjection.hostPairingRejected,
|
||||
hostSignedOut: hostConnectionProjection.hostSignedOut,
|
||||
hostPaths: hostConnectionProjection.hostPaths,
|
||||
hostPendingPaths: hostConnectionProjection.hostPendingPaths,
|
||||
primaryHost,
|
||||
|
||||
@@ -41,6 +41,7 @@ const MODEL_DESCRIPTOR: SessionOptionDescriptor = {
|
||||
]
|
||||
},
|
||||
valueSource: 'reported',
|
||||
transport: 'catalog',
|
||||
settable: true
|
||||
}
|
||||
|
||||
@@ -57,6 +58,7 @@ const EFFORT_DESCRIPTOR: SessionOptionDescriptor = {
|
||||
]
|
||||
},
|
||||
valueSource: 'dispatched',
|
||||
transport: 'catalog',
|
||||
settable: true
|
||||
}
|
||||
|
||||
@@ -66,6 +68,7 @@ const FAST_MODE_DESCRIPTOR: SessionOptionDescriptor = {
|
||||
category: 'mode',
|
||||
kind: { type: 'boolean', currentValue: false },
|
||||
valueSource: 'reported',
|
||||
transport: 'catalog',
|
||||
settable: true
|
||||
}
|
||||
|
||||
@@ -227,6 +230,7 @@ describe('MobileNativeChatSessionOptionPickers', () => {
|
||||
...MODEL_DESCRIPTOR,
|
||||
kind: { type: 'select', choices: [] },
|
||||
valueSource: 'unknown',
|
||||
transport: 'catalog',
|
||||
action: { type: 'agent-picker' }
|
||||
}
|
||||
])
|
||||
@@ -236,6 +240,46 @@ describe('MobileNativeChatSessionOptionPickers', () => {
|
||||
expect(invokeAction).toHaveBeenCalledWith('model')
|
||||
})
|
||||
|
||||
// The terminal transport can only learn the outcome by parsing the screen back,
|
||||
// so the sheet admits the value is unconfirmed; the structured transport reports
|
||||
// it every turn, which makes the same caption noise there.
|
||||
it.each([
|
||||
{ transport: 'catalog' as const, caption: true },
|
||||
{ transport: 'agent-session' as const, caption: false }
|
||||
])('captions a dispatched value only on the terminal transport', async (scenario) => {
|
||||
mount([
|
||||
MODEL_DESCRIPTOR,
|
||||
{ ...EFFORT_DESCRIPTOR, valueSource: 'dispatched', transport: scenario.transport }
|
||||
])
|
||||
await act(async () => pill('Model').props.onPress())
|
||||
await act(async () => rowByText('Effort').props.onPress())
|
||||
const captions = renderer!.root
|
||||
.findAll((node) => node.type === 'Text')
|
||||
.filter(
|
||||
(node) =>
|
||||
(node.props as { children?: unknown }).children === 'Sent to the agent — not confirmed'
|
||||
)
|
||||
expect(captions.length > 0).toBe(scenario.caption)
|
||||
})
|
||||
|
||||
it.each(['catalog', 'agent-session'] as const)(
|
||||
'does not caption a reported value on the %s transport',
|
||||
async (transport) => {
|
||||
mount([MODEL_DESCRIPTOR, { ...EFFORT_DESCRIPTOR, valueSource: 'reported', transport }])
|
||||
await act(async () => pill('Model').props.onPress())
|
||||
await act(async () => rowByText('Effort').props.onPress())
|
||||
expect(
|
||||
renderer!.root
|
||||
.findAll((node) => node.type === 'Text')
|
||||
.some(
|
||||
(node) =>
|
||||
(node.props as { children?: unknown }).children ===
|
||||
'Sent to the agent — not confirmed'
|
||||
)
|
||||
).toBe(false)
|
||||
}
|
||||
)
|
||||
|
||||
it('locks the pills while the agent is working', () => {
|
||||
mount([MODEL_DESCRIPTOR, EFFORT_DESCRIPTOR], true)
|
||||
expect(pill('Model').props).toMatchObject({ disabled: true })
|
||||
|
||||
@@ -3,9 +3,10 @@ import { ActivityIndicator, Keyboard, Pressable, StyleSheet, Text, View } from '
|
||||
import { ChevronLeft, X } from 'lucide-react-native'
|
||||
import { BottomDrawer } from '../components/BottomDrawer'
|
||||
import { colors, radii, spacing, typography } from '../theme/mobile-theme'
|
||||
import type {
|
||||
SessionOptionDescriptor,
|
||||
SessionOptionValue
|
||||
import {
|
||||
sessionOptionDispatchUnconfirmed,
|
||||
type SessionOptionDescriptor,
|
||||
type SessionOptionValue
|
||||
} from '../../../src/shared/native-chat-session-options'
|
||||
import {
|
||||
mobileModelPillLabel,
|
||||
@@ -119,7 +120,7 @@ export function MobileNativeChatSessionOptionPickers({
|
||||
) : null}
|
||||
</View>
|
||||
</View>
|
||||
{activeDescriptor.valueSource === 'dispatched' ? (
|
||||
{sessionOptionDispatchUnconfirmed(activeDescriptor) ? (
|
||||
<SessionOptionCaption>Sent to the agent — not confirmed</SessionOptionCaption>
|
||||
) : null}
|
||||
{reason ? <SessionOptionCaption>{reason}</SessionOptionCaption> : null}
|
||||
|
||||
@@ -168,7 +168,8 @@ export function useMobileNativeChatSessionOptions(args: {
|
||||
models: activeModels(catalog, record),
|
||||
record,
|
||||
mode: 'live',
|
||||
modelLabel: 'Model'
|
||||
modelLabel: 'Model',
|
||||
liveTransport: 'catalog'
|
||||
})
|
||||
}, [agent, catalog, scopeKey, version])
|
||||
|
||||
|
||||
@@ -30,14 +30,16 @@ export function useConnectionPathStatus(hostId: string | undefined): {
|
||||
export function useRelayRecoveryStatus(hostId: string | undefined): {
|
||||
pendingPath: MobileConnectionPath | null
|
||||
pairingRejected: boolean
|
||||
hostSignedOut: boolean
|
||||
} {
|
||||
return useHostMetric(
|
||||
hostId,
|
||||
(context, id) => ({
|
||||
pendingPath: context.getPendingPath(id),
|
||||
pairingRejected: context.isPairingRejected(id)
|
||||
pairingRejected: context.isPairingRejected(id),
|
||||
hostSignedOut: context.isHostSignedOut(id)
|
||||
}),
|
||||
{ pendingPath: null, pairingRejected: false }
|
||||
{ pendingPath: null, pairingRejected: false, hostSignedOut: false }
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -533,12 +533,12 @@ describe('useAllHostClients', () => {
|
||||
await Promise.resolve()
|
||||
})
|
||||
act(() => client.emitPendingPath('relay'))
|
||||
expect(status).toEqual({ pendingPath: 'relay', pairingRejected: false })
|
||||
expect(status).toEqual({ pendingPath: 'relay', pairingRejected: false, hostSignedOut: false })
|
||||
|
||||
// Why: the desktop refusing the credential is a status-only change — no
|
||||
// transport state moves, so only the connection-path signal can carry it.
|
||||
act(() => client.emitPairingRejected(true))
|
||||
expect(status).toEqual({ pendingPath: 'relay', pairingRejected: true })
|
||||
expect(status).toEqual({ pendingPath: 'relay', pairingRejected: true, hostSignedOut: false })
|
||||
|
||||
act(() => renderer.unmount())
|
||||
})
|
||||
|
||||
@@ -29,6 +29,10 @@ const STALE_SINCE_LAST_CONNECT_MS = 60_000
|
||||
// instead of leaving the user staring at a generic "Can't connect".
|
||||
const TAILSCALE_HINT = 'check Tailscale'
|
||||
|
||||
// No hint field: the remedy is the label, and appending "— check Tailscale" to
|
||||
// it would be wrong advice for a desktop that is reachable but signed out.
|
||||
const SIGNED_OUT_LABEL = 'Desktop signed out — sign in to Orca on your desktop to reconnect'
|
||||
|
||||
export type ConnectionVerdict =
|
||||
| { kind: 'normal'; label: string }
|
||||
| { kind: 'warning'; label: string; hint?: string } // "Can't connect"
|
||||
@@ -54,6 +58,10 @@ export function classifyConnection(args: {
|
||||
// The desktop has repeatedly refused this device's relay credential — retrying
|
||||
// cannot fix it, so it outranks any "still connecting" reading (STA-4681).
|
||||
pairingRejected?: boolean
|
||||
// The relay says the desktop's last control close named its own Orca Cloud
|
||||
// sign-out. Retrying is still correct and still happens on the same cadence,
|
||||
// but only the desktop's owner can end it, so the label has to say so.
|
||||
hostSignedOut?: boolean
|
||||
nowMs?: number
|
||||
}): ConnectionVerdict {
|
||||
const { state, reconnectAttempts, lastConnectedAt } = args
|
||||
@@ -70,6 +78,17 @@ export function classifyConnection(args: {
|
||||
return { kind: 'normal', label: 'Connected' }
|
||||
}
|
||||
|
||||
// Ahead of the attempt thresholds: this is evidence, not an inference from a
|
||||
// failure streak, and waiting twelve dials to show it wastes the whole point.
|
||||
// Below auth-failed because a revoked pairing cannot be fixed by signing in.
|
||||
if (args.hostSignedOut) {
|
||||
return {
|
||||
kind: 'unreachable',
|
||||
label: SIGNED_OUT_LABEL,
|
||||
reason: lastConnectedAt == null ? 'never-connected' : 'stale'
|
||||
}
|
||||
}
|
||||
|
||||
// A disconnected pending path can survive a cleared retry timer during a
|
||||
// lifecycle race. Only narrate Relay while dialing or after a retry has
|
||||
// recorded progress; otherwise the idle transport must read Disconnected.
|
||||
|
||||
@@ -89,10 +89,16 @@ export function createHostClientSelectors(
|
||||
getPendingPath: (hostId: string): MobileConnectionPath | null =>
|
||||
clientPendingPath(entries.get(hostId)?.client),
|
||||
isPairingRejected: (hostId: string): boolean =>
|
||||
clientPairingRejected(entries.get(hostId)?.client)
|
||||
clientPairingRejected(entries.get(hostId)?.client),
|
||||
isHostSignedOut: (hostId: string): boolean => clientHostSignedOut(entries.get(hostId)?.client)
|
||||
}
|
||||
}
|
||||
|
||||
export function clientHostSignedOut(client: RpcClient | undefined): boolean {
|
||||
const logical = client as Partial<StableLogicalRpcClient> | undefined
|
||||
return logical?.isHostSignedOut?.() ?? false
|
||||
}
|
||||
|
||||
export function clientPairingRejected(client: RpcClient | undefined): boolean {
|
||||
const logical = client as Partial<StableLogicalRpcClient> | undefined
|
||||
return logical?.isPairingRejected?.() ?? false
|
||||
|
||||
@@ -5,6 +5,7 @@ export class LogicalClientConnectionPath {
|
||||
private recovery: MobileConnectionPath | null = null
|
||||
private recoveryAttempt = 0
|
||||
private pairingRejected = false
|
||||
private hostSignedOut = false
|
||||
private readonly listeners = new Set<() => void>()
|
||||
|
||||
constructor(private readonly isConnected: () => boolean) {}
|
||||
@@ -35,12 +36,23 @@ export class LogicalClientConnectionPath {
|
||||
})
|
||||
}
|
||||
|
||||
isHostSignedOut(): boolean {
|
||||
return this.hostSignedOut
|
||||
}
|
||||
|
||||
setHostSignedOut(signedOut: boolean): void {
|
||||
this.update(() => {
|
||||
this.hostSignedOut = signedOut
|
||||
})
|
||||
}
|
||||
|
||||
clearAfterConnected(): void {
|
||||
this.migration = null
|
||||
this.recovery = null
|
||||
this.recoveryAttempt = 0
|
||||
// Why: an authenticated session is the desktop accepting this device.
|
||||
this.pairingRejected = false
|
||||
this.hostSignedOut = false
|
||||
}
|
||||
|
||||
setRecovery(path: MobileConnectionPath | null, attempt?: number): void {
|
||||
@@ -69,11 +81,13 @@ export class LogicalClientConnectionPath {
|
||||
const previousPath = this.pending()
|
||||
const previousAttempt = this.reconnectAttempt(0)
|
||||
const previousRejected = this.pairingRejected
|
||||
const previousSignedOut = this.hostSignedOut
|
||||
apply()
|
||||
if (
|
||||
previousPath === this.pending() &&
|
||||
previousAttempt === this.reconnectAttempt(0) &&
|
||||
previousRejected === this.pairingRejected
|
||||
previousRejected === this.pairingRejected &&
|
||||
previousSignedOut === this.hostSignedOut
|
||||
) {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -86,7 +86,7 @@ function createSupervisor(
|
||||
): MobileEndpointSupervisor {
|
||||
return new MobileEndpointSupervisor(logical, host, {
|
||||
openDirect: (endpoint) => connect(endpoint, host.deviceToken, host.publicKeyB64, { onLog }),
|
||||
openRelay: (relay, credential, confirmReqId) =>
|
||||
openRelay: (relay, credential, confirmReqId, onHostCloseReason) =>
|
||||
connectMobileRelayRpcSession({
|
||||
relay,
|
||||
resumeToken: credential.token,
|
||||
@@ -94,6 +94,7 @@ function createSupervisor(
|
||||
resumeConfirmReqId: confirmReqId,
|
||||
deviceToken: host.deviceToken,
|
||||
desktopPublicKeyB64: host.publicKeyB64,
|
||||
onHostCloseReason,
|
||||
onLog
|
||||
}),
|
||||
resolveRelay: resolveMobileRelayEndpoint,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { MobileRelayEndpoint } from '../../../src/shared/mobile-relay-credential-contract'
|
||||
import type { RelayHostCloseReason } from '../../../src/shared/relay-host-close-reason'
|
||||
import type { MobileRelayCredentialBundle } from './mobile-relay-credential-bundle'
|
||||
import type { MobileRelayRpcSession } from './mobile-relay-rpc-session'
|
||||
import type { resolveMobileRelayEndpoint } from './mobile-relay-resume-director'
|
||||
@@ -10,7 +11,8 @@ export type MobileEndpointSupervisorDependencies = {
|
||||
openRelay: (
|
||||
relay: MobileRelayEndpoint,
|
||||
credential: { token: string; version: number },
|
||||
confirmReqId: string
|
||||
confirmReqId: string,
|
||||
onHostCloseReason?: (reason: RelayHostCloseReason) => void
|
||||
) => MobileRelayRpcSession
|
||||
resolveRelay: typeof resolveMobileRelayEndpoint
|
||||
readBundle: (hostId: string) => Promise<MobileRelayCredentialBundle | null>
|
||||
|
||||
@@ -134,10 +134,22 @@ export class FakeLogicalClient extends FakeSession implements StableLogicalRpcCl
|
||||
}
|
||||
})
|
||||
isPairingRejected = () => this.pairingRejected
|
||||
private hostSignedOut = false
|
||||
setHostSignedOut = vi.fn((signedOut: boolean) => {
|
||||
if (this.hostSignedOut === signedOut) {
|
||||
return
|
||||
}
|
||||
this.hostSignedOut = signedOut
|
||||
for (const listener of this.pathListeners) {
|
||||
listener()
|
||||
}
|
||||
})
|
||||
isHostSignedOut = () => this.hostSignedOut
|
||||
// Mirrors LogicalClientConnectionPath.clearAfterConnected.
|
||||
publishState(state: ConnectionState): void {
|
||||
if (state === 'connected') {
|
||||
this.pairingRejected = false
|
||||
this.hostSignedOut = false
|
||||
}
|
||||
super.publishState(state)
|
||||
}
|
||||
|
||||
@@ -185,7 +185,12 @@ describe('mobile endpoint supervisor', () => {
|
||||
await supervisor.start()
|
||||
|
||||
expect(deps.resolveRelay).toHaveBeenCalledOnce()
|
||||
expect(openRelay).toHaveBeenLastCalledWith(resolved, expect.any(Object), expect.any(String))
|
||||
expect(openRelay).toHaveBeenLastCalledWith(
|
||||
resolved,
|
||||
expect.any(Object),
|
||||
expect.any(String),
|
||||
expect.any(Function)
|
||||
)
|
||||
expect(deps.saveHost).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ relay: resolved, endpoint: host.endpoint })
|
||||
)
|
||||
@@ -556,7 +561,8 @@ describe('mobile endpoint supervisor', () => {
|
||||
expect(openRelay).toHaveBeenLastCalledWith(
|
||||
relay,
|
||||
expect.objectContaining({ version: 3 }),
|
||||
expect.any(String)
|
||||
expect.any(String),
|
||||
expect.any(Function)
|
||||
)
|
||||
supervisor.stop()
|
||||
})
|
||||
@@ -603,7 +609,8 @@ describe('mobile endpoint supervisor', () => {
|
||||
expect(openRelay).toHaveBeenLastCalledWith(
|
||||
relay,
|
||||
expect.objectContaining({ version: 3 }),
|
||||
expect.any(String)
|
||||
expect.any(String),
|
||||
expect.any(Function)
|
||||
)
|
||||
supervisor.stop()
|
||||
})
|
||||
|
||||
@@ -2,6 +2,10 @@ import {
|
||||
RelayPhoneHelloSchema,
|
||||
type RelayPhoneHello
|
||||
} from '../../../src/shared/mobile-relay-phone-protocol'
|
||||
import {
|
||||
relayHostCloseReasonFrom,
|
||||
type RelayHostCloseReason
|
||||
} from '../../../src/shared/relay-host-close-reason'
|
||||
import { MobileE2EEV2ClientSession } from './mobile-e2ee-v2-client-session'
|
||||
import { MobileE2EEV2PhysicalChannel } from './mobile-e2ee-v2-physical-channel'
|
||||
import { websocketPayloadToUint8 } from './websocket-payload-bytes'
|
||||
@@ -26,6 +30,12 @@ type MobileRelayE2eeLinkOptions = {
|
||||
onText: (plaintext: string) => void
|
||||
onBinary: (plaintext: Uint8Array) => void
|
||||
onHello?: (hello: Extract<RelayPhoneHello, { ok: true }>) => void
|
||||
// The cell's account of why the desktop is absent, read off the close frame.
|
||||
// Reported separately from onError because a rejection is delivered as both a
|
||||
// relay-hello and a close, and which one the runtime dispatches first is not
|
||||
// ordered — only the close carries the reason, and it must not be lost to
|
||||
// that race.
|
||||
onHostCloseReason?: (reason: RelayHostCloseReason) => void
|
||||
// Fired once relay-auth is on the wire: from here the cell owns the wait.
|
||||
onOpen?: () => void
|
||||
onError: (error: Error) => void
|
||||
@@ -129,6 +139,11 @@ export class MobileRelayE2eeLink {
|
||||
clearTimeout(this.transportErrorTimer)
|
||||
this.transportErrorTimer = null
|
||||
}
|
||||
// Ahead of fail(), which no-ops once the hello already reported this close.
|
||||
const hostCloseReason = relayHostCloseReasonFrom(event.reason)
|
||||
if (hostCloseReason) {
|
||||
this.options.onHostCloseReason?.(hostCloseReason)
|
||||
}
|
||||
this.fail(new RelayOuterError(event.code || 1006))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import { RelayDialStageTracker, type RelayDialStageSource } from './relay-dial-s
|
||||
import { RelayPendingRequests } from './relay-pending-requests'
|
||||
import { RpcSessionLivenessWatchdog } from './rpc-session-liveness-watchdog'
|
||||
import { settleMobileRuntimeCapabilities } from './mobile-runtime-capability-negotiation'
|
||||
import type { RelayHostCloseReason } from '../../../src/shared/relay-host-close-reason'
|
||||
import type { RpcClient } from './rpc-client'
|
||||
import type { ConnectionLogSink, ConnectionState, RpcResponse } from './types'
|
||||
|
||||
@@ -40,6 +41,7 @@ export function connectMobileRelayRpcSession(args: {
|
||||
desktopPublicKeyB64: string
|
||||
requestTimeoutMs?: number
|
||||
createSocket?: (url: string) => WebSocket
|
||||
onHostCloseReason?: (reason: RelayHostCloseReason) => void
|
||||
onLog?: ConnectionLogSink
|
||||
}): MobileRelayRpcSession {
|
||||
const requestTimeoutMs = args.requestTimeoutMs ?? 30_000
|
||||
@@ -69,6 +71,7 @@ export function connectMobileRelayRpcSession(args: {
|
||||
deviceToken: args.deviceToken,
|
||||
desktopPublicKeyB64: args.desktopPublicKeyB64,
|
||||
createSocket: args.createSocket,
|
||||
onHostCloseReason: args.onHostCloseReason,
|
||||
onOpen: () => dialStage.advance('awaiting-hello'),
|
||||
onHello: (hello) => {
|
||||
if (
|
||||
|
||||
@@ -145,10 +145,22 @@ class FakeLogicalClient extends FakeSession implements StableLogicalRpcClient {
|
||||
}
|
||||
})
|
||||
isPairingRejected = () => this.pairingRejected
|
||||
private hostSignedOut = false
|
||||
setHostSignedOut = vi.fn((signedOut: boolean) => {
|
||||
if (this.hostSignedOut === signedOut) {
|
||||
return
|
||||
}
|
||||
this.hostSignedOut = signedOut
|
||||
for (const listener of this.pathListeners) {
|
||||
listener()
|
||||
}
|
||||
})
|
||||
isHostSignedOut = () => this.hostSignedOut
|
||||
// Mirrors LogicalClientConnectionPath.clearAfterConnected.
|
||||
publishState(state: ConnectionState): void {
|
||||
if (state === 'connected') {
|
||||
this.pairingRejected = false
|
||||
this.hostSignedOut = false
|
||||
}
|
||||
super.publishState(state)
|
||||
}
|
||||
@@ -264,7 +276,8 @@ describe('relay runtime recovery without direct connectivity', () => {
|
||||
expect(openRelay).toHaveBeenLastCalledWith(
|
||||
relay,
|
||||
expect.objectContaining({ version: 3 }),
|
||||
expect.any(String)
|
||||
expect.any(String),
|
||||
expect.any(Function)
|
||||
)
|
||||
expect(logical.getActivePath()).toBe('relay')
|
||||
supervisor.stop()
|
||||
@@ -353,7 +366,8 @@ describe('relay runtime recovery without direct connectivity', () => {
|
||||
expect(deps.openRelay).toHaveBeenLastCalledWith(
|
||||
relay,
|
||||
expect.objectContaining({ version: 2 }),
|
||||
expect.any(String)
|
||||
expect.any(String),
|
||||
expect.any(Function)
|
||||
)
|
||||
expect(logical.getActivePath()).toBe('relay')
|
||||
supervisor.stop()
|
||||
@@ -382,7 +396,8 @@ describe('relay runtime recovery without direct connectivity', () => {
|
||||
expect(openRelay).toHaveBeenLastCalledWith(
|
||||
relay,
|
||||
expect.objectContaining({ version: 1 }),
|
||||
expect.any(String)
|
||||
expect.any(String),
|
||||
expect.any(Function)
|
||||
)
|
||||
expect(logical.getActivePath()).toBe('relay')
|
||||
supervisor.stop()
|
||||
|
||||
@@ -10,6 +10,7 @@ import type { MobileRelayCredentialBundle } from './mobile-relay-credential-bund
|
||||
import type { RelayReconnectController } from './mobile-relay-reconnect-controller'
|
||||
import type { StableLogicalRpcClient } from './stable-logical-rpc-client'
|
||||
import type { MobileRelayEndpoint } from '../../../src/shared/mobile-relay-credential-contract'
|
||||
import { RELAY_HOST_CLOSE_REASON } from '../../../src/shared/relay-host-close-reason'
|
||||
import type { HostProfile } from './types'
|
||||
|
||||
type EstablishResult = { ok: true } | { ok: false; error: Error }
|
||||
@@ -100,7 +101,16 @@ export class MobileRelaySessionEstablisher {
|
||||
const session = args.openRelay(
|
||||
relay,
|
||||
credential,
|
||||
`confirm-${encodeBase64Url(args.randomBytes(16))}`
|
||||
`confirm-${encodeBase64Url(args.randomBytes(16))}`,
|
||||
// Latched on the logical client, not on the dial result: the close that
|
||||
// carries the reason can land after this dial has already reported its
|
||||
// failure. Clearing is clearAfterConnected's job, so any path that
|
||||
// reaches connected retires it.
|
||||
(reason) => {
|
||||
if (reason === RELAY_HOST_CLOSE_REASON.SIGNED_OUT) {
|
||||
args.logical.setHostSignedOut(true)
|
||||
}
|
||||
}
|
||||
)
|
||||
try {
|
||||
// Why: backgrounding or a direct winner withdraws this dial before cutover.
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { MOBILE_RELAY_CLOSE_CODE } from '../../../src/shared/mobile-relay-close-codes'
|
||||
import { RELAY_HOST_CLOSE_REASON } from '../../../src/shared/relay-host-close-reason'
|
||||
import { RelayOuterError } from './mobile-relay-e2ee-link'
|
||||
import {
|
||||
dependencies,
|
||||
FakeLogicalClient,
|
||||
FakeRelaySession,
|
||||
host
|
||||
} from './mobile-endpoint-supervisor-test-fakes'
|
||||
import { MobileEndpointSupervisor } from './mobile-endpoint-supervisor'
|
||||
|
||||
vi.mock('react-native', () => ({ Platform: { OS: 'ios' } }))
|
||||
vi.mock('expo-secure-store', () => ({ WHEN_UNLOCKED_THIS_DEVICE_ONLY: 'when-unlocked' }))
|
||||
vi.mock('expo-crypto', () => ({ getRandomBytes: (length: number) => new Uint8Array(length) }))
|
||||
|
||||
// The reason travels from the cell's close frame to the screens. This covers
|
||||
// the production wiring between them: the supervisor's own openRelay callback.
|
||||
describe('a signed-out desktop reaches the phone verdict', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
vi.setSystemTime(new Date('2026-07-13T12:00:00Z'))
|
||||
})
|
||||
afterEach(() => vi.useRealTimers())
|
||||
|
||||
function supervisorOver(closeReason: string | null) {
|
||||
const logical = new FakeLogicalClient('disconnected', 'lan')
|
||||
const deps = dependencies({
|
||||
openDirect: vi.fn(() => new FakeRelaySession('disconnected')),
|
||||
openRelay: vi.fn((_relay, _credential, _confirmReqId, onHostCloseReason) => {
|
||||
if (closeReason) {
|
||||
onHostCloseReason?.(closeReason as never)
|
||||
}
|
||||
return new FakeRelaySession(
|
||||
'disconnected',
|
||||
new RelayOuterError(MOBILE_RELAY_CLOSE_CODE.HOST_OFFLINE)
|
||||
)
|
||||
})
|
||||
})
|
||||
return { logical, supervisor: new MobileEndpointSupervisor(logical, host, deps) }
|
||||
}
|
||||
|
||||
it('latches the sign-out the cell reported', async () => {
|
||||
const { logical, supervisor } = supervisorOver(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
|
||||
await supervisor.start()
|
||||
await vi.waitFor(() => expect(logical.isHostSignedOut()).toBe(true))
|
||||
|
||||
supervisor.stop()
|
||||
})
|
||||
|
||||
it('stays quiet for an ordinary host-offline rejection', async () => {
|
||||
const { logical, supervisor } = supervisorOver(null)
|
||||
|
||||
await supervisor.start()
|
||||
|
||||
expect(logical.isHostSignedOut()).toBe(false)
|
||||
supervisor.stop()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,216 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
vi.mock('./mobile-e2ee-v2-client-session', () => ({
|
||||
MobileE2EEV2ClientSession: { create: () => ({}) }
|
||||
}))
|
||||
|
||||
vi.mock('./mobile-e2ee-v2-physical-channel', () => ({
|
||||
MobileE2EEAuthenticationError: class extends Error {},
|
||||
MobileE2EEV2PhysicalChannel: class {
|
||||
start = vi.fn()
|
||||
handleMessage = vi.fn(async () => {})
|
||||
sendText = vi.fn(() => true)
|
||||
sendBinary = vi.fn(() => true)
|
||||
dispose = vi.fn()
|
||||
}
|
||||
}))
|
||||
|
||||
import { RELAY_HOST_CLOSE_REASON } from '../../../src/shared/relay-host-close-reason'
|
||||
import { MOBILE_RELAY_CLOSE_CODE } from '../../../src/shared/mobile-relay-close-codes'
|
||||
import { classifyConnection, verdictDisplayLabel } from './connection-health'
|
||||
import { MobileRelayE2eeLink, RelayOuterError } from './mobile-relay-e2ee-link'
|
||||
import { LogicalClientConnectionPath } from './logical-client-connection-path'
|
||||
import { RelayReconnectController } from './mobile-relay-reconnect-controller'
|
||||
|
||||
const SIGNED_OUT_LABEL = 'Desktop signed out — sign in to Orca on your desktop to reconnect'
|
||||
|
||||
class FakeSocket {
|
||||
static readonly OPEN = 1
|
||||
readonly OPEN = FakeSocket.OPEN
|
||||
readyState = FakeSocket.OPEN
|
||||
bufferedAmount = 0
|
||||
onopen: (() => void) | null = null
|
||||
onmessage: ((event: { data: unknown }) => void) | null = null
|
||||
onerror: (() => void) | null = null
|
||||
onclose: ((event: { code: number; reason: string }) => void) | null = null
|
||||
send = vi.fn()
|
||||
close = vi.fn()
|
||||
}
|
||||
|
||||
function linkOver(
|
||||
socket: FakeSocket,
|
||||
onHostCloseReason: (reason: string) => void,
|
||||
onError: (error: Error) => void
|
||||
): MobileRelayE2eeLink {
|
||||
return new MobileRelayE2eeLink({
|
||||
endpoint: { cellUrl: 'https://relay-c1.onorca.dev', relayHostId: 'AbCdEf0123_-xyZ9' },
|
||||
credential: 'credential',
|
||||
expectedCredentialKind: 'resume',
|
||||
deviceToken: 'device-token',
|
||||
desktopPublicKeyB64: 'desktop-key',
|
||||
onAuthenticated: vi.fn(),
|
||||
onText: vi.fn(),
|
||||
onBinary: vi.fn(),
|
||||
onHostCloseReason,
|
||||
onError,
|
||||
createSocket: () => socket as unknown as WebSocket
|
||||
})
|
||||
}
|
||||
|
||||
describe('relay close reason on the phone', () => {
|
||||
it('reports the cell close reason and still fails with 4404', () => {
|
||||
const socket = new FakeSocket()
|
||||
const onHostCloseReason = vi.fn()
|
||||
const onError = vi.fn()
|
||||
linkOver(socket, onHostCloseReason, onError)
|
||||
|
||||
socket.onclose?.({
|
||||
code: MOBILE_RELAY_CLOSE_CODE.HOST_OFFLINE,
|
||||
reason: RELAY_HOST_CLOSE_REASON.SIGNED_OUT
|
||||
})
|
||||
|
||||
expect(onHostCloseReason).toHaveBeenCalledWith(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
expect(onError).toHaveBeenCalledWith(new RelayOuterError(MOBILE_RELAY_CLOSE_CODE.HOST_OFFLINE))
|
||||
})
|
||||
|
||||
// An old cell sends its constant, and every other close sends nothing.
|
||||
it('reports nothing for a reason it does not know', () => {
|
||||
const socket = new FakeSocket()
|
||||
const onHostCloseReason = vi.fn()
|
||||
linkOver(socket, onHostCloseReason, vi.fn())
|
||||
|
||||
socket.onclose?.({
|
||||
code: MOBILE_RELAY_CLOSE_CODE.HOST_OFFLINE,
|
||||
reason: 'relay connection rejected'
|
||||
})
|
||||
|
||||
expect(onHostCloseReason).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
// The rejection arrives as a relay-hello AND a close, in an unordered pair.
|
||||
// Whichever lands first, the reason must survive.
|
||||
it('still reports the reason when the hello already failed the link', async () => {
|
||||
const socket = new FakeSocket()
|
||||
const onHostCloseReason = vi.fn()
|
||||
linkOver(socket, onHostCloseReason, vi.fn())
|
||||
|
||||
socket.onmessage?.({
|
||||
data: JSON.stringify({
|
||||
type: 'relay-hello',
|
||||
ok: false,
|
||||
code: MOBILE_RELAY_CLOSE_CODE.HOST_OFFLINE
|
||||
})
|
||||
})
|
||||
await Promise.resolve()
|
||||
await Promise.resolve()
|
||||
socket.onclose?.({
|
||||
code: MOBILE_RELAY_CLOSE_CODE.HOST_OFFLINE,
|
||||
reason: RELAY_HOST_CLOSE_REASON.SIGNED_OUT
|
||||
})
|
||||
|
||||
expect(onHostCloseReason).toHaveBeenCalledWith(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
|
||||
})
|
||||
})
|
||||
|
||||
describe('the signed-out signal on the logical client', () => {
|
||||
it('publishes on change and retires when any path reaches connected', () => {
|
||||
const path = new LogicalClientConnectionPath(() => false)
|
||||
const changes = vi.fn()
|
||||
path.subscribe(changes)
|
||||
|
||||
path.setHostSignedOut(true)
|
||||
path.setHostSignedOut(true)
|
||||
expect(path.isHostSignedOut()).toBe(true)
|
||||
expect(changes).toHaveBeenCalledTimes(1)
|
||||
|
||||
path.clearAfterConnected()
|
||||
expect(path.isHostSignedOut()).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('RelayReconnectController cadence', () => {
|
||||
// The reason changes no recovery decision; 4404 keeps the host-offline
|
||||
// backoff it has always had, so a phone on this build retries exactly as
|
||||
// often as one that never hears the reason.
|
||||
it('keeps the host-offline retry delay for a 4404', () => {
|
||||
const delays: number[] = []
|
||||
const controller = new RelayReconnectController(
|
||||
{
|
||||
now: () => 0,
|
||||
randomBytes: () => new Uint8Array([0, 0]),
|
||||
setTimer: ((callback: () => void, delay: number) => {
|
||||
delays.push(delay)
|
||||
return 1 as unknown as ReturnType<typeof setTimeout>
|
||||
}) as unknown as typeof setTimeout,
|
||||
clearTimer: (() => {}) as unknown as typeof clearTimeout
|
||||
},
|
||||
vi.fn()
|
||||
)
|
||||
|
||||
controller.registerFailure(new RelayOuterError(MOBILE_RELAY_CLOSE_CODE.HOST_OFFLINE))
|
||||
|
||||
// hostOfflineDelayMs' 5s floor, not the 250ms transport-backoff floor.
|
||||
expect(delays.at(-1)).toBe(5_000)
|
||||
})
|
||||
})
|
||||
|
||||
describe('classifyConnection with a signed-out desktop', () => {
|
||||
const base = { reconnectAttempts: 0, lastConnectedAt: null, hostSignedOut: true }
|
||||
|
||||
it('says so from the first failed dial instead of "Connecting via Relay…"', () => {
|
||||
const verdict = classifyConnection({
|
||||
...base,
|
||||
state: 'connecting',
|
||||
pendingPath: 'relay'
|
||||
})
|
||||
|
||||
expect(verdict).toEqual({
|
||||
kind: 'unreachable',
|
||||
label: SIGNED_OUT_LABEL,
|
||||
reason: 'never-connected'
|
||||
})
|
||||
expect(verdictDisplayLabel(verdict)).toBe(SIGNED_OUT_LABEL)
|
||||
})
|
||||
|
||||
it('replaces "Can\'t reach desktop" on the direct path too', () => {
|
||||
expect(
|
||||
classifyConnection({ ...base, state: 'reconnecting', reconnectAttempts: 20 }).label
|
||||
).toBe(SIGNED_OUT_LABEL)
|
||||
})
|
||||
|
||||
it('reads as stale once this session had been connected', () => {
|
||||
expect(
|
||||
classifyConnection({ ...base, state: 'reconnecting', lastConnectedAt: 1, nowMs: 2 }).reason
|
||||
).toBe('stale')
|
||||
})
|
||||
|
||||
// A Tailscale endpoint cannot make "sign in on your desktop" better advice.
|
||||
it('never appends the Tailscale hint', () => {
|
||||
expect(
|
||||
classifyConnection({ ...base, state: 'reconnecting', endpoint: '100.64.0.1' })
|
||||
).not.toHaveProperty('hint')
|
||||
})
|
||||
|
||||
it('never outranks a connected session', () => {
|
||||
expect(classifyConnection({ ...base, state: 'connected' }).label).toBe('Connected')
|
||||
})
|
||||
|
||||
// Re-pairing, not signing in, is the remedy when the pairing itself is dead.
|
||||
it('never outranks a revoked pairing', () => {
|
||||
expect(classifyConnection({ ...base, state: 'reconnecting', pairingRejected: true }).kind).toBe(
|
||||
'auth-failed'
|
||||
)
|
||||
})
|
||||
|
||||
it('leaves every other verdict alone when the desktop is not signed out', () => {
|
||||
expect(
|
||||
classifyConnection({
|
||||
state: 'connecting',
|
||||
reconnectAttempts: 0,
|
||||
lastConnectedAt: null,
|
||||
pendingPath: 'relay',
|
||||
hostSignedOut: false
|
||||
}).label
|
||||
).toBe('Connecting via Relay…')
|
||||
})
|
||||
})
|
||||
@@ -24,6 +24,7 @@ export type RpcClientContextValue = {
|
||||
getActivePath: (hostId: string) => MobileConnectionPath
|
||||
getPendingPath: (hostId: string) => MobileConnectionPath | null
|
||||
isPairingRejected: (hostId: string) => boolean
|
||||
isHostSignedOut: (hostId: string) => boolean
|
||||
subscribeHostState: (hostId: string, listener: (state: ConnectionState) => void) => () => void
|
||||
getAllClients: () => { hostId: string; client: RpcClient }[]
|
||||
subscribeAllHosts: (listener: () => void) => () => void
|
||||
|
||||
@@ -55,6 +55,9 @@ export type StableLogicalRpcClient = RpcClient & {
|
||||
// Latched when the desktop has repeatedly refused this device's relay credential.
|
||||
setPairingRejected(rejected: boolean): void
|
||||
isPairingRejected(): boolean
|
||||
// Latched when the relay named the desktop's own sign-out as the reason it is absent.
|
||||
setHostSignedOut(signedOut: boolean): void
|
||||
isHostSignedOut(): boolean
|
||||
// Recovery attempts share this signal so status-only changes rerender.
|
||||
onConnectionPathChange(listener: () => void): () => void
|
||||
getGeneration(): number
|
||||
@@ -282,6 +285,8 @@ export function createStableLogicalRpcClient(
|
||||
setRecoveryAttempt: (attempt) => connectionPath.setRecoveryAttempt(attempt),
|
||||
setPairingRejected: (rejected) => connectionPath.setPairingRejected(rejected),
|
||||
isPairingRejected: () => connectionPath.isPairingRejected(),
|
||||
setHostSignedOut: (signedOut) => connectionPath.setHostSignedOut(signedOut),
|
||||
isHostSignedOut: () => connectionPath.isHostSignedOut(),
|
||||
onConnectionPathChange: (listener) => connectionPath.subscribe(listener),
|
||||
getGeneration: () => generation
|
||||
}
|
||||
|
||||
@@ -138,6 +138,7 @@ export function useAllHostClients(hostIds: string[], options?: UseAllHostClients
|
||||
path: MobileConnectionPath
|
||||
pendingPath: MobileConnectionPath | null
|
||||
pairingRejected: boolean
|
||||
hostSignedOut: boolean
|
||||
}>((hostId) => {
|
||||
const client = clientsByHostId.get(hostId)
|
||||
return client
|
||||
@@ -148,7 +149,8 @@ export function useAllHostClients(hostIds: string[], options?: UseAllHostClients
|
||||
state: ctx.getState(hostId),
|
||||
path: ctx.getActivePath(hostId),
|
||||
pendingPath: ctx.getPendingPath(hostId),
|
||||
pairingRejected: ctx.isPairingRejected(hostId)
|
||||
pairingRejected: ctx.isPairingRejected(hostId),
|
||||
hostSignedOut: ctx.isHostSignedOut(hostId)
|
||||
}
|
||||
]
|
||||
: []
|
||||
|
||||
@@ -153,6 +153,7 @@
|
||||
"repro:live-remote-realistic-freeze": "node config/scripts/live-remote-realistic-freeze-repro.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"@anthropic-ai/claude-agent-sdk": "0.3.251",
|
||||
"@electron-toolkit/preload": "^3.0.2",
|
||||
"@electron-toolkit/utils": "^4.0.0",
|
||||
"@floating-ui/dom": "1.7.6",
|
||||
|
||||
Generated
+117
-16
@@ -122,6 +122,9 @@ importers:
|
||||
|
||||
.:
|
||||
dependencies:
|
||||
'@anthropic-ai/claude-agent-sdk':
|
||||
specifier: 0.3.251
|
||||
version: 0.3.251(@anthropic-ai/sdk@0.122.0(zod@4.5.4))(@modelcontextprotocol/sdk@1.30.0(supports-color@7.2.0)(zod@4.5.4))(zod@4.5.4)
|
||||
'@electron-toolkit/preload':
|
||||
specifier: ^3.0.2
|
||||
version: 3.0.2(electron@43.4.1(supports-color@7.2.0))
|
||||
@@ -535,6 +538,23 @@ packages:
|
||||
'@antfu/install-pkg@1.1.0':
|
||||
resolution: {integrity: sha512-MGQsmw10ZyI+EJo45CdSER4zEb+p31LpDAFp2Z3gkSd1yqVZGi0Ebx++YTEMonJy4oChEMLsxZ64j8FH6sSqtQ==}
|
||||
|
||||
'@anthropic-ai/claude-agent-sdk@0.3.251':
|
||||
resolution: {integrity: sha512-DqSi8mH2tQYRlVV0G+lJnQ/WbjJZ/a+8cJ3vPuYoqh8esIIvXHm1ZOXV1UPGsFYRnbBytEoiSGitguEXd+sQ+Q==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
peerDependencies:
|
||||
'@anthropic-ai/sdk': '>=0.93.0'
|
||||
'@modelcontextprotocol/sdk': ^1.29.0
|
||||
zod: ^4.0.0
|
||||
|
||||
'@anthropic-ai/sdk@0.122.0':
|
||||
resolution: {integrity: sha512-GGPNftt0caaz9MDlmNQGHX8855Ojaduyy5pm9Sm1h7HalCn0cWNb5/bweadJF+4yzbal+QL6ztBa09WAAOzLmQ==}
|
||||
hasBin: true
|
||||
peerDependencies:
|
||||
zod: ^3.25.0 || ^4.0.0
|
||||
peerDependenciesMeta:
|
||||
zod:
|
||||
optional: true
|
||||
|
||||
'@babel/code-frame@7.29.7':
|
||||
resolution: {integrity: sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==}
|
||||
engines: {node: '>=6.9.0'}
|
||||
@@ -2628,6 +2648,9 @@ packages:
|
||||
resolution: {integrity: sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
'@stablelib/base64@1.0.1':
|
||||
resolution: {integrity: sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==}
|
||||
|
||||
'@stablyai/playwright-base@2.1.14':
|
||||
resolution: {integrity: sha512-/iAgMW5tC0ETDo3mFyTzszRrD7rGFIT4fgDgtZxqa9vPhiTLix/1+GeOOBNY0uS+XRLFY0Uc/irsC3XProL47g==}
|
||||
engines: {node: '>=18'}
|
||||
@@ -4493,6 +4516,9 @@ packages:
|
||||
resolution: {integrity: sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==}
|
||||
engines: {node: '>=8.6.0'}
|
||||
|
||||
fast-sha256@1.3.0:
|
||||
resolution: {integrity: sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==}
|
||||
|
||||
fast-string-truncated-width@3.0.3:
|
||||
resolution: {integrity: sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g==}
|
||||
|
||||
@@ -5039,6 +5065,10 @@ packages:
|
||||
json-parse-even-better-errors@2.3.1:
|
||||
resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==}
|
||||
|
||||
json-schema-to-ts@3.1.1:
|
||||
resolution: {integrity: sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==}
|
||||
engines: {node: '>=16'}
|
||||
|
||||
json-schema-traverse@1.0.0:
|
||||
resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==}
|
||||
|
||||
@@ -6425,6 +6455,9 @@ packages:
|
||||
stackback@0.0.2:
|
||||
resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==}
|
||||
|
||||
standardwebhooks@1.1.1:
|
||||
resolution: {integrity: sha512-bCbX9ZEyFkWPsRz7Bl3NuQUJohmwGSev/yhr7vhaGPlc4AfIrspIRa6cPTBuI1ItmrTDJ4d/S2hCsfe4+vQGnQ==}
|
||||
|
||||
stat-mode@1.0.0:
|
||||
resolution: {integrity: sha512-jH9EhtKIjuXZ2cWxmXS8ZP80XyC3iasQxMDV8jzhNJpfDb7VbQLVW4Wvsxz9QZvzV+G4YoSfBUVKDOyxLzi/sg==}
|
||||
engines: {node: '>= 6'}
|
||||
@@ -6608,6 +6641,9 @@ packages:
|
||||
truncate-utf8-bytes@1.0.2:
|
||||
resolution: {integrity: sha512-95Pu1QXQvruGEhv62XCMO3Mm90GscOCClvrIUwCM0PYOXK3kaF3l3sIHxx71ThJfcbM2O5Au6SO3AWCSEfW4mQ==}
|
||||
|
||||
ts-algebra@2.0.0:
|
||||
resolution: {integrity: sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==}
|
||||
|
||||
ts-dedent@2.2.0:
|
||||
resolution: {integrity: sha512-q5W7tVM71e2xjHZTlgfTDoPF/SmqKG5hddq9SzR49CH2hayqRKJtQ4mtRlSxKaJlR/+9rEM+mnBHf7I2/BQcpQ==}
|
||||
engines: {node: '>=6.10'}
|
||||
@@ -7007,6 +7043,16 @@ packages:
|
||||
zwitch@2.0.4:
|
||||
resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==}
|
||||
|
||||
ignoredOptionalDependencies:
|
||||
- '@anthropic-ai/claude-agent-sdk-darwin-arm64'
|
||||
- '@anthropic-ai/claude-agent-sdk-darwin-x64'
|
||||
- '@anthropic-ai/claude-agent-sdk-linux-arm64'
|
||||
- '@anthropic-ai/claude-agent-sdk-linux-arm64-musl'
|
||||
- '@anthropic-ai/claude-agent-sdk-linux-x64'
|
||||
- '@anthropic-ai/claude-agent-sdk-linux-x64-musl'
|
||||
- '@anthropic-ai/claude-agent-sdk-win32-arm64'
|
||||
- '@anthropic-ai/claude-agent-sdk-win32-x64'
|
||||
|
||||
snapshots:
|
||||
|
||||
'@adobe/css-tools@4.5.0': {}
|
||||
@@ -7016,6 +7062,19 @@ snapshots:
|
||||
package-manager-detector: 1.6.0
|
||||
tinyexec: 1.1.2
|
||||
|
||||
'@anthropic-ai/claude-agent-sdk@0.3.251(@anthropic-ai/sdk@0.122.0(zod@4.5.4))(@modelcontextprotocol/sdk@1.30.0(supports-color@7.2.0)(zod@4.5.4))(zod@4.5.4)':
|
||||
dependencies:
|
||||
'@anthropic-ai/sdk': 0.122.0(zod@4.5.4)
|
||||
'@modelcontextprotocol/sdk': 1.30.0(supports-color@7.2.0)(zod@4.5.4)
|
||||
zod: 4.5.4
|
||||
|
||||
'@anthropic-ai/sdk@0.122.0(zod@4.5.4)':
|
||||
dependencies:
|
||||
json-schema-to-ts: 3.1.1
|
||||
standardwebhooks: 1.1.1
|
||||
optionalDependencies:
|
||||
zod: 4.5.4
|
||||
|
||||
'@babel/code-frame@7.29.7':
|
||||
dependencies:
|
||||
'@babel/helper-validator-identifier': 7.29.7
|
||||
@@ -7669,6 +7728,28 @@ snapshots:
|
||||
dependencies:
|
||||
'@chevrotain/types': 11.1.2
|
||||
|
||||
'@modelcontextprotocol/sdk@1.30.0(supports-color@7.2.0)(zod@4.5.4)':
|
||||
dependencies:
|
||||
'@hono/node-server': 2.1.0(hono@4.13.0)
|
||||
ajv: 8.20.0
|
||||
ajv-formats: 3.0.1(ajv@8.20.0)
|
||||
content-type: 1.0.5
|
||||
cors: 2.8.6
|
||||
cross-spawn: 7.0.6
|
||||
eventsource: 3.0.7
|
||||
eventsource-parser: 3.0.8
|
||||
express: 5.2.1(supports-color@7.2.0)
|
||||
express-rate-limit: 8.5.2(express@5.2.1(supports-color@7.2.0))
|
||||
hono: 4.13.0
|
||||
jose: 6.2.3
|
||||
json-schema-typed: 8.0.2
|
||||
pkce-challenge: 5.0.1
|
||||
raw-body: 3.0.2
|
||||
zod: 4.5.4
|
||||
zod-to-json-schema: 3.25.2(zod@4.5.4)
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
'@modelcontextprotocol/sdk@1.30.0(zod@3.25.76)':
|
||||
dependencies:
|
||||
'@hono/node-server': 2.1.0(hono@4.13.0)
|
||||
@@ -7679,8 +7760,8 @@ snapshots:
|
||||
cross-spawn: 7.0.6
|
||||
eventsource: 3.0.7
|
||||
eventsource-parser: 3.0.8
|
||||
express: 5.2.1
|
||||
express-rate-limit: 8.5.2(express@5.2.1)
|
||||
express: 5.2.1(supports-color@7.2.0)
|
||||
express-rate-limit: 8.5.2(express@5.2.1(supports-color@7.2.0))
|
||||
hono: 4.13.0
|
||||
jose: 6.2.3
|
||||
json-schema-typed: 8.0.2
|
||||
@@ -8917,6 +8998,8 @@ snapshots:
|
||||
|
||||
'@sindresorhus/merge-streams@4.0.0': {}
|
||||
|
||||
'@stablelib/base64@1.0.1': {}
|
||||
|
||||
'@stablyai/playwright-base@2.1.14(@playwright/test@1.59.1)(zod@4.5.4)':
|
||||
dependencies:
|
||||
'@playwright/test': 1.59.1
|
||||
@@ -9923,7 +10006,7 @@ snapshots:
|
||||
|
||||
bluebird@3.7.2: {}
|
||||
|
||||
body-parser@2.3.0:
|
||||
body-parser@2.3.0(supports-color@7.2.0):
|
||||
dependencies:
|
||||
bytes: 3.1.2
|
||||
content-type: 2.0.0
|
||||
@@ -10779,15 +10862,15 @@ snapshots:
|
||||
|
||||
exponential-backoff@3.1.3: {}
|
||||
|
||||
express-rate-limit@8.5.2(express@5.2.1):
|
||||
express-rate-limit@8.5.2(express@5.2.1(supports-color@7.2.0)):
|
||||
dependencies:
|
||||
express: 5.2.1
|
||||
express: 5.2.1(supports-color@7.2.0)
|
||||
ip-address: 10.4.0
|
||||
|
||||
express@5.2.1:
|
||||
express@5.2.1(supports-color@7.2.0):
|
||||
dependencies:
|
||||
accepts: 2.0.0
|
||||
body-parser: 2.3.0
|
||||
body-parser: 2.3.0(supports-color@7.2.0)
|
||||
content-disposition: 1.1.0
|
||||
content-type: 1.0.5
|
||||
cookie: 0.7.2
|
||||
@@ -10797,7 +10880,7 @@ snapshots:
|
||||
encodeurl: 2.0.0
|
||||
escape-html: 1.0.3
|
||||
etag: 1.8.1
|
||||
finalhandler: 2.1.1
|
||||
finalhandler: 2.1.1(supports-color@7.2.0)
|
||||
fresh: 2.0.0
|
||||
http-errors: 2.0.1
|
||||
merge-descriptors: 2.0.0
|
||||
@@ -10808,9 +10891,9 @@ snapshots:
|
||||
proxy-addr: 2.0.7
|
||||
qs: 6.15.2
|
||||
range-parser: 1.2.1
|
||||
router: 2.2.0
|
||||
send: 1.2.1
|
||||
serve-static: 2.2.1
|
||||
router: 2.2.0(supports-color@7.2.0)
|
||||
send: 1.2.1(supports-color@7.2.0)
|
||||
serve-static: 2.2.1(supports-color@7.2.0)
|
||||
statuses: 2.0.2
|
||||
type-is: 2.1.0
|
||||
vary: 1.1.2
|
||||
@@ -10831,6 +10914,8 @@ snapshots:
|
||||
merge2: 1.4.1
|
||||
micromatch: 4.0.8
|
||||
|
||||
fast-sha256@1.3.0: {}
|
||||
|
||||
fast-string-truncated-width@3.0.3: {}
|
||||
|
||||
fast-string-width@3.0.2:
|
||||
@@ -10871,7 +10956,7 @@ snapshots:
|
||||
dependencies:
|
||||
to-regex-range: 5.0.1
|
||||
|
||||
finalhandler@2.1.1:
|
||||
finalhandler@2.1.1(supports-color@7.2.0):
|
||||
dependencies:
|
||||
debug: 4.4.3(supports-color@7.2.0)
|
||||
encodeurl: 2.0.0
|
||||
@@ -11445,6 +11530,11 @@ snapshots:
|
||||
|
||||
json-parse-even-better-errors@2.3.1: {}
|
||||
|
||||
json-schema-to-ts@3.1.1:
|
||||
dependencies:
|
||||
'@babel/runtime': 7.29.7
|
||||
ts-algebra: 2.0.0
|
||||
|
||||
json-schema-traverse@1.0.0: {}
|
||||
|
||||
json-schema-typed@8.0.2: {}
|
||||
@@ -13037,7 +13127,7 @@ snapshots:
|
||||
points-on-curve: 0.2.0
|
||||
points-on-path: 0.2.1
|
||||
|
||||
router@2.2.0:
|
||||
router@2.2.0(supports-color@7.2.0):
|
||||
dependencies:
|
||||
debug: 4.4.3(supports-color@7.2.0)
|
||||
depd: 2.0.0
|
||||
@@ -13080,7 +13170,7 @@ snapshots:
|
||||
|
||||
semver@7.8.1: {}
|
||||
|
||||
send@1.2.1:
|
||||
send@1.2.1(supports-color@7.2.0):
|
||||
dependencies:
|
||||
debug: 4.4.3(supports-color@7.2.0)
|
||||
encodeurl: 2.0.0
|
||||
@@ -13107,12 +13197,12 @@ snapshots:
|
||||
transitivePeerDependencies:
|
||||
- typescript
|
||||
|
||||
serve-static@2.2.1:
|
||||
serve-static@2.2.1(supports-color@7.2.0):
|
||||
dependencies:
|
||||
encodeurl: 2.0.0
|
||||
escape-html: 1.0.3
|
||||
parseurl: 1.3.3
|
||||
send: 1.2.1
|
||||
send: 1.2.1(supports-color@7.2.0)
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
@@ -13267,6 +13357,11 @@ snapshots:
|
||||
|
||||
stackback@0.0.2: {}
|
||||
|
||||
standardwebhooks@1.1.1:
|
||||
dependencies:
|
||||
'@stablelib/base64': 1.0.1
|
||||
fast-sha256: 1.3.0
|
||||
|
||||
stat-mode@1.0.0: {}
|
||||
|
||||
state-local@1.0.7: {}
|
||||
@@ -13437,6 +13532,8 @@ snapshots:
|
||||
dependencies:
|
||||
utf8-byte-length: 1.0.5
|
||||
|
||||
ts-algebra@2.0.0: {}
|
||||
|
||||
ts-dedent@2.2.0: {}
|
||||
|
||||
ts-morph@26.0.0:
|
||||
@@ -13786,6 +13883,10 @@ snapshots:
|
||||
dependencies:
|
||||
zod: 3.25.76
|
||||
|
||||
zod-to-json-schema@3.25.2(zod@4.5.4):
|
||||
dependencies:
|
||||
zod: 4.5.4
|
||||
|
||||
zod@3.25.76: {}
|
||||
|
||||
zod@4.5.4: {}
|
||||
|
||||
@@ -12,6 +12,20 @@ minimumReleaseAgeExclude:
|
||||
- zod@4.5.4
|
||||
shamefullyHoist: true
|
||||
|
||||
# Orca always launches the user's own resolved Claude CLI via
|
||||
# pathToClaudeCodeExecutable, so the SDK's bundled ~95 MB-per-platform CLI
|
||||
# binaries must never be installed. Excluding them is what makes the path
|
||||
# override mandatory rather than merely preferred.
|
||||
ignoredOptionalDependencies:
|
||||
- '@anthropic-ai/claude-agent-sdk-darwin-arm64'
|
||||
- '@anthropic-ai/claude-agent-sdk-darwin-x64'
|
||||
- '@anthropic-ai/claude-agent-sdk-linux-arm64'
|
||||
- '@anthropic-ai/claude-agent-sdk-linux-arm64-musl'
|
||||
- '@anthropic-ai/claude-agent-sdk-linux-x64'
|
||||
- '@anthropic-ai/claude-agent-sdk-linux-x64-musl'
|
||||
- '@anthropic-ai/claude-agent-sdk-win32-arm64'
|
||||
- '@anthropic-ai/claude-agent-sdk-win32-x64'
|
||||
|
||||
supportedArchitectures:
|
||||
os:
|
||||
- current
|
||||
|
||||
@@ -127,10 +127,6 @@ __orca_osc133_precmd() {
|
||||
unset __orca_in_command
|
||||
fi
|
||||
printf "\033]133;A\007"
|
||||
# Why: emit the shell-ready marker here (not a trailing PROMPT_COMMAND entry)
|
||||
# so a framework that must be last in PROMPT_COMMAND — bash-preexec — is not
|
||||
# displaced by one of Orca's own hooks.
|
||||
[[ -n "$__orca_ready_marker" ]] && printf "\033]777;orca-shell-ready\007"
|
||||
return "$exit_code"
|
||||
}
|
||||
__orca_osc133_preexec() {
|
||||
@@ -188,6 +184,11 @@ __orca_osc133_epilogue() {
|
||||
unset __orca_in_prompt_command
|
||||
__orca_adopt_outer_debug_trap
|
||||
trap '__orca_osc133_preexec' DEBUG
|
||||
# Readline renders PS1 after entering raw mode; prompt hooks still run in cooked mode.
|
||||
if [[ -n "$__orca_ready_marker" ]]; then
|
||||
PS1="${PS1-}"'\[\e]777;orca-shell-ready\a\]'
|
||||
__orca_ready_marker=""
|
||||
fi
|
||||
}
|
||||
__orca_normalize_prompt_command_part() {
|
||||
local __orca_value="$1" __orca_output_name="$2" __orca_character __orca_chunk
|
||||
|
||||
@@ -22,6 +22,20 @@ function workingStatus(): AgentAwakeStatus {
|
||||
}
|
||||
}
|
||||
|
||||
describe('AgentAwakeService status array ownership', () => {
|
||||
it('does not observe rows appended to the caller array after setStatuses', () => {
|
||||
const service = new AgentAwakeService()
|
||||
service.setMode('auto')
|
||||
const statuses: AgentAwakeStatus[] = [workingStatus()]
|
||||
|
||||
service.setStatuses(statuses)
|
||||
const before = service.getWorkingAgentCount()
|
||||
statuses.push(workingStatus(), workingStatus())
|
||||
|
||||
expect(service.getWorkingAgentCount()).toBe(before)
|
||||
})
|
||||
})
|
||||
|
||||
function createBlocker() {
|
||||
const startedIds = new Set<number>()
|
||||
let nextId = 1
|
||||
|
||||
@@ -105,7 +105,8 @@ export class AgentAwakeService {
|
||||
}
|
||||
|
||||
setStatuses(statuses: AgentAwakeStatus[]): void {
|
||||
this.statuses = statuses.map((status) => ({ ...status }))
|
||||
// Copy the array, not every row: the hook server allocates each row fresh per event.
|
||||
this.statuses = [...statuses]
|
||||
this.refresh('status-change')
|
||||
}
|
||||
|
||||
@@ -171,7 +172,8 @@ export class AgentAwakeService {
|
||||
|
||||
private getEligibleRunningStatusCount(): number {
|
||||
const now = this.now()
|
||||
return this.statuses.filter((status) => this.isWakeEligible(status, now)).length
|
||||
// Counted in place: the filtered array was only ever measured, and this runs per hook event.
|
||||
return this.statuses.reduce((count, s) => count + (this.isWakeEligible(s, now) ? 1 : 0), 0)
|
||||
}
|
||||
|
||||
private isWakeEligible(status: AgentAwakeStatus, now: number): boolean {
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { AutomationPrecheck, AutomationPrecheckResult } from '../../shared/
|
||||
import { MAX_AUTOMATION_PRECHECK_OUTPUT_CHARS } from '../../shared/automation-precheck'
|
||||
import { getSshConnectionManager } from '../ipc/ssh'
|
||||
import { shellEscape } from '../ssh/ssh-connection-utils'
|
||||
import { admitSelfInitiatedTreeKill } from '../own-chromium-tree-kill-guard'
|
||||
|
||||
type AutomationPrecheckExecutionTarget =
|
||||
| {
|
||||
@@ -73,7 +74,10 @@ function failedPrecheckResult(
|
||||
})
|
||||
}
|
||||
|
||||
function killLocalPrecheckProcessTree(child: ChildProcess): ReturnType<typeof setTimeout> | null {
|
||||
/** Exported for the refusal-fallback test; the timeout path is otherwise unreachable. */
|
||||
export function killLocalPrecheckProcessTree(
|
||||
child: ChildProcess
|
||||
): ReturnType<typeof setTimeout> | null {
|
||||
const pid = child.pid
|
||||
if (!pid) {
|
||||
child.kill()
|
||||
@@ -81,6 +85,18 @@ function killLocalPrecheckProcessTree(child: ChildProcess): ReturnType<typeof se
|
||||
}
|
||||
|
||||
if (process.platform === 'win32') {
|
||||
if (
|
||||
!admitSelfInitiatedTreeKill({
|
||||
pid,
|
||||
site: 'automation-precheck-timeout',
|
||||
scope: 'win-taskkill-tree'
|
||||
})
|
||||
) {
|
||||
// Refusal blocks the tree walk, not the termination: killing the root by
|
||||
// handle cannot reach a recycled pid, and a timed-out precheck must stop.
|
||||
child.kill()
|
||||
return null
|
||||
}
|
||||
try {
|
||||
// Why: shell prechecks can launch child processes; taskkill walks the
|
||||
// Windows process tree so timeout means the command is actually stopped.
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import type { GlobalSettings } from '../../shared/global-settings-types'
|
||||
import type { ClaudeManagedAccount } from '../../shared/managed-account-types'
|
||||
import {
|
||||
CLAUDE_AUTH_ENV_VARS,
|
||||
hasClaudeAuthEnvConflict,
|
||||
shouldStripClaudeAuthEnvForAccount
|
||||
} from './environment'
|
||||
import {
|
||||
normalizeTuiAgentEnvRecord,
|
||||
resolveTuiAgentLaunchEnv
|
||||
} from '../../shared/tui-agent-launch-defaults'
|
||||
import { claudeStructuredAuthPolicyForSettings } from './claude-structured-auth-policy'
|
||||
|
||||
const HOST_ACCOUNT = { id: 'host-a', managedAuthRuntime: 'host' } as ClaudeManagedAccount
|
||||
const WSL_ACCOUNT = { id: 'wsl-b', managedAuthRuntime: 'wsl' } as ClaudeManagedAccount
|
||||
const LEGACY_ACCOUNT = { id: 'legacy-c' } as ClaudeManagedAccount
|
||||
|
||||
function settings(
|
||||
overrides: Partial<
|
||||
Pick<
|
||||
GlobalSettings,
|
||||
| 'claudeManagedAccounts'
|
||||
| 'activeClaudeManagedAccountId'
|
||||
| 'activeClaudeManagedAccountIdsByRuntime'
|
||||
>
|
||||
>
|
||||
): Parameters<typeof claudeStructuredAuthPolicyForSettings>[0] {
|
||||
return {
|
||||
claudeManagedAccounts: [HOST_ACCOUNT, WSL_ACCOUNT, LEGACY_ACCOUNT],
|
||||
activeClaudeManagedAccountId: null,
|
||||
...overrides
|
||||
} as Parameters<typeof claudeStructuredAuthPolicyForSettings>[0]
|
||||
}
|
||||
|
||||
// The predicate now backs BOTH transports (runtime-auth-preparation.ts and the
|
||||
// structured wiring), so it needs a test of its own: forcing it to a constant used
|
||||
// to leave ~1000 tests green.
|
||||
describe('shouldStripClaudeAuthEnvForAccount', () => {
|
||||
it('does not strip when no managed account is selected', () => {
|
||||
expect(shouldStripClaudeAuthEnvForAccount([HOST_ACCOUNT], null)).toBe(false)
|
||||
expect(shouldStripClaudeAuthEnvForAccount([HOST_ACCOUNT], undefined)).toBe(false)
|
||||
expect(shouldStripClaudeAuthEnvForAccount([HOST_ACCOUNT], '')).toBe(false)
|
||||
})
|
||||
|
||||
it('strips for a host-managed account', () => {
|
||||
expect(shouldStripClaudeAuthEnvForAccount([HOST_ACCOUNT, WSL_ACCOUNT], 'host-a')).toBe(true)
|
||||
})
|
||||
|
||||
it('strips for an account with no explicit runtime (the legacy host shape)', () => {
|
||||
expect(shouldStripClaudeAuthEnvForAccount([LEGACY_ACCOUNT], 'legacy-c')).toBe(true)
|
||||
})
|
||||
|
||||
it('does not strip for a WSL-managed account, matching runtime-auth-preparation', () => {
|
||||
expect(shouldStripClaudeAuthEnvForAccount([HOST_ACCOUNT, WSL_ACCOUNT], 'wsl-b')).toBe(false)
|
||||
})
|
||||
|
||||
it('strips for a selected id no account list explains', () => {
|
||||
// Fail-safe: an id we cannot resolve is treated as a pinned account, never as
|
||||
// "no account", so an unreadable settings blob cannot open the strip.
|
||||
expect(shouldStripClaudeAuthEnvForAccount([HOST_ACCOUNT], 'deleted-d')).toBe(true)
|
||||
expect(shouldStripClaudeAuthEnvForAccount(undefined, 'deleted-d')).toBe(true)
|
||||
expect(shouldStripClaudeAuthEnvForAccount([], 'deleted-d')).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('claudeStructuredAuthPolicyForSettings', () => {
|
||||
it('reads the host runtime selection, not the legacy flat field alone', () => {
|
||||
expect(
|
||||
claudeStructuredAuthPolicyForSettings(
|
||||
settings({
|
||||
activeClaudeManagedAccountId: 'host-a',
|
||||
activeClaudeManagedAccountIdsByRuntime: { host: null, wsl: {} }
|
||||
})
|
||||
)
|
||||
).toEqual({ stripAuthEnv: true })
|
||||
})
|
||||
|
||||
it('strips when a host account is pinned by runtime selection', () => {
|
||||
expect(
|
||||
claudeStructuredAuthPolicyForSettings(
|
||||
settings({ activeClaudeManagedAccountIdsByRuntime: { host: 'host-a', wsl: {} } })
|
||||
)
|
||||
).toEqual({ stripAuthEnv: true })
|
||||
})
|
||||
|
||||
it('does not strip for system auth, so an API-key-only user keeps their sign-in', () => {
|
||||
expect(claudeStructuredAuthPolicyForSettings(settings({}))).toEqual({ stripAuthEnv: false })
|
||||
})
|
||||
|
||||
it('ignores a WSL-only selection: the structured child is always a native host process', () => {
|
||||
expect(
|
||||
claudeStructuredAuthPolicyForSettings(
|
||||
settings({
|
||||
activeClaudeManagedAccountIdsByRuntime: { host: null, wsl: { Ubuntu: 'wsl-b' } }
|
||||
})
|
||||
)
|
||||
).toEqual({ stripAuthEnv: false })
|
||||
})
|
||||
})
|
||||
|
||||
describe('the strip vocabulary the policy governs', () => {
|
||||
it('covers every Anthropic auth variable the terminal path knows about', () => {
|
||||
// A new auth var added to the list without a matching refusal/strip path is the
|
||||
// shape of the leak this lane already shipped once.
|
||||
expect([...CLAUDE_AUTH_ENV_VARS]).toEqual([
|
||||
'ANTHROPIC_API_KEY',
|
||||
'ANTHROPIC_AUTH_TOKEN',
|
||||
'CLAUDE_CODE_OAUTH_TOKEN',
|
||||
'AWS_BEARER_TOKEN_BEDROCK'
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
// The refusal has to cover exactly what the strip removes. Anything narrower lets an
|
||||
// override reach the child that applyClaudeEnvPatch would have deleted.
|
||||
describe('hasClaudeAuthEnvConflict matches the strip it guards', () => {
|
||||
it('refuses each Anthropic auth variable', () => {
|
||||
for (const key of CLAUDE_AUTH_ENV_VARS) {
|
||||
expect(hasClaudeAuthEnvConflict({ [key]: 'v' }, 'linux')).toBe(true)
|
||||
}
|
||||
})
|
||||
|
||||
// `ANTHROPIC_API_KEY=` in the agent env box is how a user blanks a variable, and the
|
||||
// settings pipeline preserves the empty value (agent-default-env-draft.ts assigns
|
||||
// everything after the `=`; normalizeTuiAgentEnvRecord drops empty KEYS only). An
|
||||
// empty value cannot beat the pinned account and the strip removes the name anyway,
|
||||
// so refusing it would break a terminal launch that works today for no security gain.
|
||||
it('admits an override whose value is empty, the documented way to blank a variable', () => {
|
||||
expect(hasClaudeAuthEnvConflict({ ANTHROPIC_API_KEY: '' }, 'linux')).toBe(false)
|
||||
expect(hasClaudeAuthEnvConflict({ anthropic_api_key: '' }, 'win32')).toBe(false)
|
||||
expect(hasClaudeAuthEnvConflict({ ANTHROPIC_CUSTOM_HEADERS: '' }, 'linux')).toBe(false)
|
||||
})
|
||||
|
||||
it('still refuses the same names once they carry a value', () => {
|
||||
expect(hasClaudeAuthEnvConflict({ ANTHROPIC_API_KEY: 'sk-ant' }, 'linux')).toBe(true)
|
||||
})
|
||||
|
||||
// The end-to-end shape the regression actually took: settings text -> normalized
|
||||
// record -> launch env -> the predicate the terminal preflight gates on.
|
||||
it('admits a blanked variable all the way from the settings record', () => {
|
||||
const configured = normalizeTuiAgentEnvRecord({ claude: { ANTHROPIC_API_KEY: '' } })
|
||||
const launchEnv = resolveTuiAgentLaunchEnv('claude', configured)
|
||||
|
||||
expect(launchEnv).toEqual({ ANTHROPIC_API_KEY: '' })
|
||||
expect(hasClaudeAuthEnvConflict(launchEnv, 'linux')).toBe(false)
|
||||
})
|
||||
|
||||
it('folds case on win32, where the OS does', () => {
|
||||
expect(hasClaudeAuthEnvConflict({ anthropic_api_key: 'sk-lower' }, 'win32')).toBe(true)
|
||||
expect(hasClaudeAuthEnvConflict({ Anthropic_Custom_Headers: 'x-api-key: v' }, 'win32')).toBe(
|
||||
true
|
||||
)
|
||||
})
|
||||
|
||||
it('keeps env names case-sensitive off win32', () => {
|
||||
expect(hasClaudeAuthEnvConflict({ anthropic_api_key: 'sk-lower' }, 'linux')).toBe(false)
|
||||
})
|
||||
|
||||
it('admits non-auth Anthropic settings on both platforms', () => {
|
||||
expect(hasClaudeAuthEnvConflict({ ANTHROPIC_BASE_URL: 'https://gw.test' }, 'linux')).toBe(false)
|
||||
expect(hasClaudeAuthEnvConflict({ ANTHROPIC_BASE_URL: 'https://gw.test' }, 'win32')).toBe(false)
|
||||
expect(hasClaudeAuthEnvConflict({ ANTHROPIC_CUSTOM_HEADERS: 'X-Trace: 1' }, 'linux')).toBe(
|
||||
false
|
||||
)
|
||||
expect(hasClaudeAuthEnvConflict(undefined, 'linux')).toBe(false)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,37 @@
|
||||
import type { GlobalSettings } from '../../shared/global-settings-types'
|
||||
import { shouldStripClaudeAuthEnvForAccount } from './environment'
|
||||
import { getSelectedClaudeAccountIdForTarget } from './runtime-selection'
|
||||
|
||||
/** The structured mirror of the terminal preflight's `prepareClaudeAuth` result:
|
||||
* the one field a launch resolution needs from the managed-account state. */
|
||||
export type ClaudeStructuredAuthPolicy = {
|
||||
stripAuthEnv: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* The only supported way to build a structured launch's auth policy.
|
||||
*
|
||||
* It exists as a named function rather than an inline object at the wiring site so
|
||||
* that the settings-to-policy mapping is testable on its own: the one production
|
||||
* wiring lives in a `@ts-nocheck` file, where neither the compiler nor a type test
|
||||
* can see a dropped field.
|
||||
*
|
||||
* Structured Claude always spawns a native local-host child — the launch resolver
|
||||
* refuses any record with a remote execution host or a WSL distro — so the host
|
||||
* selection, not the platform default target, owns its auth.
|
||||
*/
|
||||
export function claudeStructuredAuthPolicyForSettings(
|
||||
settings: Pick<
|
||||
GlobalSettings,
|
||||
| 'claudeManagedAccounts'
|
||||
| 'activeClaudeManagedAccountId'
|
||||
| 'activeClaudeManagedAccountIdsByRuntime'
|
||||
>
|
||||
): ClaudeStructuredAuthPolicy {
|
||||
return {
|
||||
stripAuthEnv: shouldStripClaudeAuthEnvForAccount(
|
||||
settings.claudeManagedAccounts,
|
||||
getSelectedClaudeAccountIdForTarget(settings, { runtime: 'host' })
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
import type { ClaudeManagedAccount } from '../../shared/managed-account-types'
|
||||
|
||||
export const CLAUDE_AUTH_ENV_VARS = [
|
||||
'ANTHROPIC_API_KEY',
|
||||
'ANTHROPIC_AUTH_TOKEN',
|
||||
@@ -13,14 +15,21 @@ export type ClaudeEnvPatch = {
|
||||
export function applyClaudeEnvPatch(
|
||||
baseEnv: Record<string, string>,
|
||||
patch: ClaudeEnvPatch,
|
||||
options?: { stripAuthEnv?: boolean }
|
||||
options?: { stripAuthEnv?: boolean; platform?: NodeJS.Platform }
|
||||
): Record<string, string> {
|
||||
if (options?.stripAuthEnv) {
|
||||
for (const key of CLAUDE_AUTH_ENV_VARS) {
|
||||
delete baseEnv[key]
|
||||
}
|
||||
if (isAuthLikeCustomHeaders(baseEnv.ANTHROPIC_CUSTOM_HEADERS)) {
|
||||
delete baseEnv.ANTHROPIC_CUSTOM_HEADERS
|
||||
const platform = options.platform ?? process.platform
|
||||
for (const key of Object.keys(baseEnv)) {
|
||||
const normalized = platform === 'win32' ? key.toUpperCase() : key
|
||||
if (
|
||||
(platform === 'win32' && CLAUDE_AUTH_ENV_VARS.some((authKey) => authKey === normalized)) ||
|
||||
(normalized === 'ANTHROPIC_CUSTOM_HEADERS' && isAuthLikeCustomHeaders(baseEnv[key]))
|
||||
) {
|
||||
delete baseEnv[key]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,16 +43,94 @@ export function applyClaudeEnvPatch(
|
||||
return baseEnv
|
||||
}
|
||||
|
||||
export function hasClaudeAuthEnvConflict(env: Record<string, string> | undefined): boolean {
|
||||
if (!env) {
|
||||
/** One string for every transport, so a terminal launch and a structured launch
|
||||
* cannot drift into telling the user two different things about one refusal. */
|
||||
export const CLAUDE_AUTH_ENV_CONFLICT_MESSAGE =
|
||||
'This Claude launch defines explicit Anthropic auth environment variables. Remove those overrides before using a managed Claude account.'
|
||||
|
||||
export const CLAUDE_AUTH_SWITCH_IN_PROGRESS_MESSAGE =
|
||||
'A Claude account switch is in progress. Try again after it finishes.'
|
||||
|
||||
/**
|
||||
* Whether a launch on the host runtime must drop inherited Anthropic auth.
|
||||
*
|
||||
* Only a pinned host-managed account owns the credential, so only it may strip:
|
||||
* with no managed account the user's own `ANTHROPIC_*` is their sign-in, and
|
||||
* removing it signs them out of a CLI that would otherwise have worked.
|
||||
*/
|
||||
export function shouldStripClaudeAuthEnvForAccount(
|
||||
accounts: readonly ClaudeManagedAccount[] | undefined,
|
||||
activeAccountId: string | null | undefined
|
||||
): boolean {
|
||||
if (!activeAccountId) {
|
||||
return false
|
||||
}
|
||||
return (
|
||||
CLAUDE_AUTH_ENV_VARS.some((key) => Boolean(env[key])) ||
|
||||
isAuthLikeCustomHeaders(env.ANTHROPIC_CUSTOM_HEADERS)
|
||||
(accounts ?? []).find((account) => account.id === activeAccountId)?.managedAuthRuntime !== 'wsl'
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a launch's explicit env carries Anthropic auth a managed account must own.
|
||||
*
|
||||
* The key comparison mirrors applyClaudeEnvPatch's strip exactly: case-insensitive on
|
||||
* win32, where the OS folds env names so `anthropic_api_key` is an effective
|
||||
* `ANTHROPIC_API_KEY`, and case-sensitive elsewhere. A refusal narrower than the strip
|
||||
* lets an override through that the strip would have removed.
|
||||
*
|
||||
* A non-empty value is what makes it a conflict. `ANTHROPIC_API_KEY=` in the agent env
|
||||
* box is how a user blanks a variable — the settings pipeline preserves that empty value
|
||||
* (normalizeTuiAgentEnvRecord drops empty KEYS only) — and an empty override can neither
|
||||
* authenticate nor beat the pinned account, while the strip removes the name regardless.
|
||||
* Refusing it would break a terminal launch that works today for no security gain.
|
||||
*/
|
||||
/**
|
||||
* The inherited Anthropic auth a non-stripping launch has to carry forward explicitly.
|
||||
*
|
||||
* applyClaudeEnvPatch always strips the inherited half of a child env, and the
|
||||
* configured half is what overrides it — so a system-auth user's own key only survives
|
||||
* if the caller puts it back deliberately. Returns the exact keys present, so a
|
||||
* win32 `anthropic_api_key` is carried under the name the OS actually has.
|
||||
*/
|
||||
export function claudeAuthEnvCarriedForward(
|
||||
inherited: NodeJS.ProcessEnv,
|
||||
platform: NodeJS.Platform = process.platform
|
||||
): Record<string, string> {
|
||||
const carried: Record<string, string> = {}
|
||||
for (const [key, value] of Object.entries(inherited)) {
|
||||
if (value === undefined) {
|
||||
continue
|
||||
}
|
||||
const normalized = platform === 'win32' ? key.toUpperCase() : key
|
||||
if (
|
||||
CLAUDE_AUTH_ENV_VARS.some((authKey) => authKey === normalized) ||
|
||||
(normalized === 'ANTHROPIC_CUSTOM_HEADERS' && isAuthLikeCustomHeaders(value))
|
||||
) {
|
||||
carried[key] = value
|
||||
}
|
||||
}
|
||||
return carried
|
||||
}
|
||||
|
||||
export function hasClaudeAuthEnvConflict(
|
||||
env: Record<string, string> | undefined,
|
||||
platform: NodeJS.Platform = process.platform
|
||||
): boolean {
|
||||
if (!env) {
|
||||
return false
|
||||
}
|
||||
for (const [key, value] of Object.entries(env)) {
|
||||
const normalized = platform === 'win32' ? key.toUpperCase() : key
|
||||
if (value && CLAUDE_AUTH_ENV_VARS.some((authKey) => authKey === normalized)) {
|
||||
return true
|
||||
}
|
||||
if (normalized === 'ANTHROPIC_CUSTOM_HEADERS' && isAuthLikeCustomHeaders(value)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
function isAuthLikeCustomHeaders(value: string | undefined): boolean {
|
||||
if (!value) {
|
||||
return false
|
||||
|
||||
@@ -5,6 +5,13 @@ const liveClaudePtyIds = new Set<string>()
|
||||
// survived the app restart inside the daemon.
|
||||
const seededUnconfirmedPtyIds = new Set<string>()
|
||||
let switchInProgress = false
|
||||
// Woken by endClaudeAuthSwitch so a caller past the point of no return can wait the
|
||||
// swap out instead of refusing. See whenClaudeAuthSwitchSettles.
|
||||
const switchSettledListeners = new Set<() => void>()
|
||||
|
||||
/** A managed account swap is a credential-file rewrite, not a network round trip;
|
||||
* anything past this is a wedged switch, and refusing beats waiting forever. */
|
||||
export const CLAUDE_AUTH_SWITCH_SETTLE_TIMEOUT_MS = 15_000
|
||||
|
||||
export type ClaudeLivePtyPersistence = {
|
||||
addClaudeLivePtySessionId(sessionId: string): void
|
||||
@@ -81,6 +88,35 @@ export function markClaudePtyExited(ptyId: string): void {
|
||||
notifyDrainedOnTransition(hadLivePtys)
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a structured Claude child with the same gate the terminal path uses.
|
||||
*
|
||||
* The gate is what makes the managed OAuth refresh defer instead of rotating a
|
||||
* single-use refresh token out from under a running Claude (runtime-auth-sync.ts).
|
||||
* A structured session's child is as much a live Claude as a PTY's is, so it has to
|
||||
* hold the gate too — otherwise a refresh mid-turn breaks its next API call while an
|
||||
* identical terminal session is protected.
|
||||
*
|
||||
* Deliberately not persisted, unlike markClaudePtySpawned: these children are direct
|
||||
* children of this process and cannot survive a restart, so seeding them back on the
|
||||
* next launch would hold the gate closed for a process that is provably gone.
|
||||
*/
|
||||
export function markClaudeStructuredChildSpawned(childKey: string): void {
|
||||
liveClaudePtyIds.add(structuredChildGateId(childKey))
|
||||
}
|
||||
|
||||
export function markClaudeStructuredChildExited(childKey: string): void {
|
||||
const hadLivePtys = liveClaudePtyIds.size > 0
|
||||
liveClaudePtyIds.delete(structuredChildGateId(childKey))
|
||||
notifyDrainedOnTransition(hadLivePtys)
|
||||
}
|
||||
|
||||
// Namespaced so a structured child can never collide with a daemon PTY session id,
|
||||
// which confirmSeededClaudeLivePtys reconciles against the daemon's own list.
|
||||
function structuredChildGateId(childKey: string): string {
|
||||
return `claude-structured:${childKey}`
|
||||
}
|
||||
|
||||
export function hasLiveClaudePtys(): boolean {
|
||||
return liveClaudePtyIds.size > 0
|
||||
}
|
||||
@@ -93,7 +129,44 @@ export function beginClaudeAuthSwitch(): void {
|
||||
}
|
||||
|
||||
export function endClaudeAuthSwitch(): void {
|
||||
const wasInProgress = switchInProgress
|
||||
switchInProgress = false
|
||||
if (!wasInProgress) {
|
||||
return
|
||||
}
|
||||
// Each listener removes itself as it settles; Set iteration is defined over that.
|
||||
for (const listener of switchSettledListeners) {
|
||||
listener()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves `true` once no account switch is running, `false` if one is still running
|
||||
* at the deadline.
|
||||
*
|
||||
* Exists for callers that have already done irreversible work — a structured acquire
|
||||
* has closed the old child by the time it resolves its launch, so turning a switch
|
||||
* into a refusal there strands the user with a dead session and no replacement.
|
||||
* Waiting for the swap and then launching against it is the recoverable answer;
|
||||
* refusing is only correct when nothing has been torn down yet.
|
||||
*/
|
||||
export function whenClaudeAuthSwitchSettles(
|
||||
timeoutMs = CLAUDE_AUTH_SWITCH_SETTLE_TIMEOUT_MS
|
||||
): Promise<boolean> {
|
||||
if (!switchInProgress) {
|
||||
return Promise.resolve(true)
|
||||
}
|
||||
return new Promise<boolean>((resolve) => {
|
||||
const settle = (settled: boolean): void => {
|
||||
switchSettledListeners.delete(listener)
|
||||
clearTimeout(timer)
|
||||
resolve(settled)
|
||||
}
|
||||
const listener = (): void => settle(true)
|
||||
switchSettledListeners.add(listener)
|
||||
const timer = setTimeout(() => settle(false), timeoutMs)
|
||||
timer.unref?.()
|
||||
})
|
||||
}
|
||||
|
||||
export function isClaudeAuthSwitchInProgress(): boolean {
|
||||
|
||||
@@ -2,6 +2,7 @@ import { join } from 'node:path'
|
||||
import type { ClaudeManagedAccount } from '../../../shared/managed-account-types'
|
||||
import { resolveLocalAccountRuntimeTarget } from '../../../shared/local-account-runtime'
|
||||
import { parseWslUncPath } from '../../../shared/wsl-paths'
|
||||
import { shouldStripClaudeAuthEnvForAccount } from '../environment'
|
||||
import { getDefaultWslDistro, getWslHome } from '../../wsl'
|
||||
import {
|
||||
getSelectedClaudeAccountIdForTarget,
|
||||
@@ -69,7 +70,10 @@ export class ClaudeRuntimeAuthPreparationService extends ClaudeRuntimeAuthSnapsh
|
||||
wslDistro: null,
|
||||
wslLinuxConfigDir: null,
|
||||
envPatch: paths.envPatch,
|
||||
stripAuthEnv: Boolean(activeAccountId && activeAccount?.managedAuthRuntime !== 'wsl'),
|
||||
stripAuthEnv: shouldStripClaudeAuthEnvForAccount(
|
||||
settings.claudeManagedAccounts,
|
||||
activeAccountId
|
||||
),
|
||||
managedRefreshDeferredByLivePty: Boolean(
|
||||
activeAccountId &&
|
||||
activeAccount?.managedAuthRuntime !== 'wsl' &&
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { parseClaudeUsageRecord } from './transcript-record-parser'
|
||||
|
||||
function assistantLine(overrides: Record<string, unknown> = {}): string {
|
||||
return JSON.stringify({
|
||||
type: 'assistant',
|
||||
sessionId: 'session-1',
|
||||
timestamp: '2026-01-01T00:00:00.000Z',
|
||||
message: { usage: { input_tokens: 3, output_tokens: 5 } },
|
||||
...overrides
|
||||
})
|
||||
}
|
||||
|
||||
describe('assistant-record prefilter', () => {
|
||||
it('still parses an ordinary assistant record', () => {
|
||||
expect(parseClaudeUsageRecord(assistantLine())?.inputTokens).toBe(3)
|
||||
})
|
||||
|
||||
it('rejects a user record that never mentions assistant', () => {
|
||||
const userLine = JSON.stringify({
|
||||
type: 'user',
|
||||
sessionId: 'session-1',
|
||||
timestamp: '2026-01-01T00:00:00.000Z',
|
||||
message: { content: 'x'.repeat(200) }
|
||||
})
|
||||
|
||||
expect(parseClaudeUsageRecord(userLine)).toBeNull()
|
||||
})
|
||||
|
||||
it('rejects a non-assistant record that happens to contain the word assistant', () => {
|
||||
const userLine = JSON.stringify({
|
||||
type: 'user',
|
||||
sessionId: 'session-1',
|
||||
timestamp: '2026-01-01T00:00:00.000Z',
|
||||
message: { content: 'ask the assistant about this' }
|
||||
})
|
||||
|
||||
expect(parseClaudeUsageRecord(userLine)).toBeNull()
|
||||
})
|
||||
})
|
||||
@@ -84,10 +84,30 @@ function dedupeClaudeUsageTurns(
|
||||
return deduped
|
||||
}
|
||||
|
||||
/**
|
||||
* Necessary condition for `JSON.parse(line).type === 'assistant'`, checked before the parse.
|
||||
*
|
||||
* Sound for any transcript written by a standard JSON serializer: `JSON.stringify` (which writes
|
||||
* these files) escapes only quotes, backslashes and control characters, never ASCII letters, so
|
||||
* the decoded value can only be `assistant` if the line spells it literally. The gate over-admits
|
||||
* freely — the `parsed.type` check below stays authoritative.
|
||||
*
|
||||
* A `\u`-escape fallback was measured and rejected: it costs a second full-line scan and made
|
||||
* transcripts whose tool results contain control characters 1.43x slower overall.
|
||||
*/
|
||||
function mayEncodeAssistantType(line: string): boolean {
|
||||
return line.includes('assistant')
|
||||
}
|
||||
|
||||
function parseClaudeUsageSourceRecord(
|
||||
line: string,
|
||||
fallbackSessionId: string | null = null
|
||||
): ClaudeUsageParsedSourceTurn | null {
|
||||
// Only assistant records carry usage, but transcripts interleave user/tool-result lines that
|
||||
// routinely embed whole files. Reject those before paying for a full parse.
|
||||
if (!mayEncodeAssistantType(line)) {
|
||||
return null
|
||||
}
|
||||
let parsed: ClaudeUsageSourceRecord
|
||||
try {
|
||||
parsed = JSON.parse(line) as ClaudeUsageSourceRecord
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
// Scripted stand-in for the Claude Code CLI, driven by the SDK contract-pin
|
||||
// tests. It speaks just enough stream-json to satisfy the SDK: it answers every
|
||||
// inbound control_request with a success control_response, records everything it
|
||||
// observes to a report file, and plays back the steps listed in a scenario file.
|
||||
//
|
||||
// Env contract (set by the test):
|
||||
// ORCA_SDK_CONTRACT_SCENARIO_PATH — JSON file
|
||||
// { steps: Step[], controlResponses?: { [subtype]: <response> } } where a Step is
|
||||
// { emit: <frame> } | { awaitUserMessage: true } | { stderr: <text> } |
|
||||
// { awaitControlResponse: <request_id> } | { delayMs: <n> } | { exit: <code> }
|
||||
// ORCA_SDK_CONTRACT_REPORT_PATH — where argv/env observations are written
|
||||
// ORCA_SDK_CONTRACT_IGNORE_SIGTERM — trap SIGTERM/SIGINT and outlive stdin close
|
||||
// ORCA_SDK_CONTRACT_IGNORE_CONTROL_REQUESTS — record control requests but never answer
|
||||
// ORCA_SDK_CONTRACT_DESCENDANT — fork an idle grandchild and report its pid
|
||||
import { spawn } from 'node:child_process'
|
||||
import { readFileSync, writeFileSync } from 'node:fs'
|
||||
import { createInterface } from 'node:readline'
|
||||
|
||||
const scenarioPath = process.env.ORCA_SDK_CONTRACT_SCENARIO_PATH
|
||||
const reportPath = process.env.ORCA_SDK_CONTRACT_REPORT_PATH
|
||||
|
||||
const report = {
|
||||
argv: process.argv.slice(1),
|
||||
execPath: process.execPath,
|
||||
controlRequests: [],
|
||||
controlResponses: [],
|
||||
userMessages: [],
|
||||
descendantPid: null
|
||||
}
|
||||
const writeReport = () => {
|
||||
if (reportPath) {
|
||||
writeFileSync(reportPath, JSON.stringify(report))
|
||||
}
|
||||
}
|
||||
// Written immediately so a test can prove which script the SDK executed even if
|
||||
// the session dies before the scenario completes.
|
||||
writeReport()
|
||||
|
||||
const scenario = scenarioPath ? JSON.parse(readFileSync(scenarioPath, 'utf8')) : { steps: [] }
|
||||
|
||||
if (process.env.ORCA_SDK_CONTRACT_IGNORE_SIGTERM) {
|
||||
process.on('SIGTERM', () => {})
|
||||
process.on('SIGINT', () => {})
|
||||
setInterval(() => {}, 1_000_000)
|
||||
}
|
||||
if (process.env.ORCA_SDK_CONTRACT_DESCENDANT) {
|
||||
const descendant = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000000)'], {
|
||||
stdio: 'ignore'
|
||||
})
|
||||
descendant.unref()
|
||||
report.descendantPid = descendant.pid ?? null
|
||||
writeReport()
|
||||
}
|
||||
|
||||
const emit = (frame) => process.stdout.write(`${JSON.stringify(frame)}\n`)
|
||||
|
||||
const waiters = []
|
||||
const settle = (kind, requestId) => {
|
||||
for (let i = waiters.length - 1; i >= 0; i--) {
|
||||
const waiter = waiters[i]
|
||||
if (
|
||||
waiter.kind === kind &&
|
||||
(waiter.requestId === undefined || waiter.requestId === requestId)
|
||||
) {
|
||||
waiters.splice(i, 1)
|
||||
waiter.resolve()
|
||||
}
|
||||
}
|
||||
}
|
||||
const waitFor = (kind, requestId) => {
|
||||
if (kind === 'user' && report.userMessages.length > 0) {
|
||||
return Promise.resolve()
|
||||
}
|
||||
if (
|
||||
kind === 'control_response' &&
|
||||
report.controlResponses.some((frame) => frame.response?.request_id === requestId)
|
||||
) {
|
||||
return Promise.resolve()
|
||||
}
|
||||
return new Promise((resolve) => waiters.push({ kind, requestId, resolve }))
|
||||
}
|
||||
|
||||
createInterface({ input: process.stdin }).on('line', (line) => {
|
||||
let frame
|
||||
try {
|
||||
frame = JSON.parse(line)
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
if (frame.type === 'control_request') {
|
||||
report.controlRequests.push(frame)
|
||||
writeReport()
|
||||
if (process.env.ORCA_SDK_CONTRACT_IGNORE_CONTROL_REQUESTS) {
|
||||
return
|
||||
}
|
||||
emit({
|
||||
type: 'control_response',
|
||||
response: {
|
||||
subtype: 'success',
|
||||
request_id: frame.request_id,
|
||||
response: scenario.controlResponses?.[frame.request?.subtype] ?? {
|
||||
commands: [],
|
||||
models: []
|
||||
}
|
||||
}
|
||||
})
|
||||
return
|
||||
}
|
||||
if (frame.type === 'control_response') {
|
||||
report.controlResponses.push(frame)
|
||||
writeReport()
|
||||
settle('control_response', frame.response?.request_id)
|
||||
return
|
||||
}
|
||||
if (frame.type === 'user') {
|
||||
report.userMessages.push(frame)
|
||||
writeReport()
|
||||
settle('user')
|
||||
}
|
||||
})
|
||||
|
||||
// Never outlive a wedged test: the readline subscription would otherwise hold
|
||||
// this process open forever if the SDK side stops driving the scenario.
|
||||
setTimeout(() => process.exit(3), 20_000).unref()
|
||||
|
||||
for (const step of scenario.steps) {
|
||||
if (step.emit) {
|
||||
emit(step.emit)
|
||||
} else if (step.stderr !== undefined) {
|
||||
process.stderr.write(step.stderr)
|
||||
} else if (step.awaitUserMessage) {
|
||||
await waitFor('user')
|
||||
} else if (step.awaitControlResponse !== undefined) {
|
||||
await waitFor('control_response', step.awaitControlResponse)
|
||||
} else if (step.delayMs) {
|
||||
await new Promise((resolve) => setTimeout(resolve, step.delayMs))
|
||||
} else if (step.exit !== undefined) {
|
||||
// A CLI that refuses to start: leave with its own status, stderr already written.
|
||||
writeReport()
|
||||
process.exit(step.exit)
|
||||
}
|
||||
}
|
||||
writeReport()
|
||||
process.exit(0)
|
||||
@@ -0,0 +1,519 @@
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { createRequire } from 'node:module'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { dirname, join } from 'node:path'
|
||||
import {
|
||||
query,
|
||||
type CanUseTool,
|
||||
type Options,
|
||||
type SDKUserMessage,
|
||||
type SpawnedProcess as SdkSpawnedProcess,
|
||||
type SpawnOptions as SdkSpawnOptions
|
||||
} from '@anthropic-ai/claude-agent-sdk'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { spawnProcess } from '../../shared/child-process/run-process'
|
||||
import type { AgentSessionRecord } from '../../shared/agent-session-record'
|
||||
import { LOCAL_EXECUTION_HOST_ID } from '../../shared/execution-host'
|
||||
import type { AgentSessionRecordStore } from '../runtime/agent-session-record-store'
|
||||
import { claudeQuerySettingsReader } from './claude-agent-sdk-control-requests'
|
||||
import { createClaudeStructuredLaunchResolver } from './claude-structured-launch-resolution'
|
||||
|
||||
// Contract pins for @anthropic-ai/claude-agent-sdk, run against the real SDK
|
||||
// driving a scripted fake CLI (never the real Claude binary). These tests exist
|
||||
// to catch a future SDK version drifting under Orca: unknown-frame pass-through,
|
||||
// spawner env fidelity, argument parity with the pre-SDK argv,
|
||||
// permission-callback semantics, and executable-path override.
|
||||
|
||||
const FAKE_CLI = join(__dirname, '__fixtures__', 'claude-agent-sdk-scripted-cli.mjs')
|
||||
const SESSION_ID = '5348c19f-6a54-4c2e-9c68-9c2b1a3d4e5f'
|
||||
const LEAF_UUID = 'ad0f7c9e-1b2c-4d3e-8f90-abc123def456'
|
||||
const PINNED_SDK_VERSION = '0.3.251'
|
||||
const SDK_PLATFORM_PACKAGE_BASENAMES = [
|
||||
'claude-agent-sdk-darwin-arm64',
|
||||
'claude-agent-sdk-darwin-x64',
|
||||
'claude-agent-sdk-linux-arm64',
|
||||
'claude-agent-sdk-linux-arm64-musl',
|
||||
'claude-agent-sdk-linux-x64',
|
||||
'claude-agent-sdk-linux-x64-musl',
|
||||
'claude-agent-sdk-win32-arm64',
|
||||
'claude-agent-sdk-win32-x64'
|
||||
]
|
||||
|
||||
/**
|
||||
* The exact argv the hand-rolled transport built before the SDK swap. Frozen here
|
||||
* as the parity oracle: CLAUDE_STRUCTURED_BASE_OPTIONS has to keep producing it.
|
||||
*/
|
||||
const PRE_SDK_ARGV = [
|
||||
'-p',
|
||||
'--input-format',
|
||||
'stream-json',
|
||||
'--output-format',
|
||||
'stream-json',
|
||||
'--include-partial-messages',
|
||||
'--verbose',
|
||||
'--replay-user-messages',
|
||||
'--permission-prompt-tool',
|
||||
'stdio',
|
||||
'--setting-sources',
|
||||
'user,project,local'
|
||||
]
|
||||
|
||||
const RESULT_FRAME = {
|
||||
type: 'result',
|
||||
subtype: 'success',
|
||||
is_error: false,
|
||||
duration_ms: 1,
|
||||
duration_api_ms: 1,
|
||||
num_turns: 1,
|
||||
result: 'ok',
|
||||
session_id: SESSION_ID,
|
||||
total_cost_usd: 0,
|
||||
usage: { input_tokens: 1, output_tokens: 1 },
|
||||
uuid: 'uuid-result-1'
|
||||
}
|
||||
|
||||
type ScenarioStep = Record<string, unknown>
|
||||
type SpawnSeen = {
|
||||
command: string
|
||||
args: string[]
|
||||
cwd: string | undefined
|
||||
env: Record<string, string | undefined>
|
||||
}
|
||||
type ScriptedCliReport = {
|
||||
argv: string[]
|
||||
execPath: string
|
||||
controlRequests: { request_id: string; request: { subtype: string } }[]
|
||||
controlResponses: { response: { request_id: string; response?: Record<string, unknown> } }[]
|
||||
userMessages: Record<string, unknown>[]
|
||||
}
|
||||
|
||||
const scratchDirs: string[] = []
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs()
|
||||
for (const dir of scratchDirs.splice(0)) {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
function scriptScenario(
|
||||
steps: ScenarioStep[],
|
||||
controlResponses: Record<string, unknown> = {}
|
||||
): {
|
||||
scenarioPath: string
|
||||
reportPath: string
|
||||
cwd: string
|
||||
readReport: () => ScriptedCliReport
|
||||
} {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'claude-sdk-contract-'))
|
||||
scratchDirs.push(dir)
|
||||
const scenarioPath = join(dir, 'scenario.json')
|
||||
const reportPath = join(dir, 'report.json')
|
||||
writeFileSync(scenarioPath, JSON.stringify({ steps, controlResponses }))
|
||||
return {
|
||||
scenarioPath,
|
||||
reportPath,
|
||||
cwd: dir,
|
||||
readReport: () => JSON.parse(readFileSync(reportPath, 'utf8')) as ScriptedCliReport
|
||||
}
|
||||
}
|
||||
|
||||
function scenarioEnv(scenario: { scenarioPath: string; reportPath: string }) {
|
||||
return {
|
||||
PATH: process.env.PATH,
|
||||
ORCA_SDK_CONTRACT_SCENARIO_PATH: scenario.scenarioPath,
|
||||
ORCA_SDK_CONTRACT_REPORT_PATH: scenario.reportPath
|
||||
}
|
||||
}
|
||||
|
||||
function recordingSpawner(spawns: SpawnSeen[]) {
|
||||
return (opts: SdkSpawnOptions): SdkSpawnedProcess => {
|
||||
spawns.push({
|
||||
command: opts.command,
|
||||
args: [...opts.args],
|
||||
cwd: opts.cwd,
|
||||
env: { ...opts.env }
|
||||
})
|
||||
return spawnProcess({
|
||||
program: opts.command,
|
||||
args: opts.args,
|
||||
cwd: opts.cwd,
|
||||
env: opts.env as NodeJS.ProcessEnv,
|
||||
signal: opts.signal
|
||||
}) as unknown as SdkSpawnedProcess
|
||||
}
|
||||
}
|
||||
|
||||
function resolvedLaunch(launchArgs: string[]) {
|
||||
const record = {
|
||||
sessionId: 'contract-pin-session',
|
||||
provider: 'claude',
|
||||
location: {
|
||||
executionHostId: LOCAL_EXECUTION_HOST_ID,
|
||||
wslDistro: null,
|
||||
workspaceId: 'workspace-1',
|
||||
workspaceKind: 'folder'
|
||||
},
|
||||
accountHome: { variable: 'CLAUDE_CONFIG_DIR', path: '/home/work/.claude' },
|
||||
providerHandleChain: [],
|
||||
launchArgs
|
||||
} as unknown as AgentSessionRecord
|
||||
return createClaudeStructuredLaunchResolver({
|
||||
store: { getRecord: () => record } as unknown as AgentSessionRecordStore,
|
||||
resolveWorkspacePath: async () => '/repos/workspace-1',
|
||||
resolveCommand: () => FAKE_CLI,
|
||||
resolveAuthPolicy: () => ({ stripAuthEnv: true })
|
||||
})({ identity: { sessionId: record.sessionId } as never })
|
||||
}
|
||||
|
||||
function singleUserTurn(): AsyncIterable<SDKUserMessage> {
|
||||
return (async function* () {
|
||||
yield {
|
||||
type: 'user',
|
||||
message: { role: 'user', content: [{ type: 'text', text: 'hello' }] },
|
||||
parent_tool_use_id: null,
|
||||
session_id: SESSION_ID
|
||||
} as SDKUserMessage
|
||||
// Hold input open; the stream ends when the scripted CLI exits, and an
|
||||
// unresolved bare promise does not keep the event loop alive.
|
||||
await new Promise<void>(() => {})
|
||||
})()
|
||||
}
|
||||
|
||||
async function drainQuery(options: Options): Promise<Record<string, unknown>[]> {
|
||||
const messages: Record<string, unknown>[] = []
|
||||
for await (const message of query({ prompt: singleUserTurn(), options })) {
|
||||
messages.push(message as unknown as Record<string, unknown>)
|
||||
}
|
||||
return messages
|
||||
}
|
||||
|
||||
/** Expand `--flag=value` argv entries so both SDK spellings compare equal. */
|
||||
function normalizeArgv(args: string[]): string[] {
|
||||
return args.flatMap((arg) => {
|
||||
if (!arg.startsWith('--')) {
|
||||
return [arg]
|
||||
}
|
||||
const eq = arg.indexOf('=')
|
||||
return eq === -1 ? [arg] : [arg.slice(0, eq), arg.slice(eq + 1)]
|
||||
})
|
||||
}
|
||||
|
||||
/** Group the pre-SDK argv into flag/value pairs. */
|
||||
function flagTable(args: readonly string[]): { flag: string; value: string | null }[] {
|
||||
const table: { flag: string; value: string | null }[] = []
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const flag = args[i]!
|
||||
const next = args[i + 1]
|
||||
if (next !== undefined && !next.startsWith('-')) {
|
||||
table.push({ flag, value: next })
|
||||
i++
|
||||
} else {
|
||||
table.push({ flag, value: null })
|
||||
}
|
||||
}
|
||||
return table
|
||||
}
|
||||
|
||||
describe('Claude Agent SDK contract pins', () => {
|
||||
it('yields unknown types, unknown fields and unknown content blocks verbatim, and consumes keep_alive', async () => {
|
||||
const unknownTopLevel = {
|
||||
type: 'message_kind_from_the_future',
|
||||
session_id: SESSION_ID,
|
||||
uuid: 'uuid-unknown-1',
|
||||
payload: { alpha: 1, nested: { flags: ['a', 'b'] } }
|
||||
}
|
||||
const assistantWithUnknowns = {
|
||||
type: 'assistant',
|
||||
message: {
|
||||
id: 'msg-1',
|
||||
type: 'message',
|
||||
role: 'assistant',
|
||||
model: 'claude-x',
|
||||
content: [
|
||||
{ type: 'text', text: 'hello back' },
|
||||
{ type: 'content_block_from_the_future', payload: { depth: 3 } }
|
||||
],
|
||||
stop_reason: null,
|
||||
stop_sequence: null,
|
||||
usage: { input_tokens: 1, output_tokens: 2 }
|
||||
},
|
||||
parent_tool_use_id: null,
|
||||
uuid: 'uuid-assistant-1',
|
||||
session_id: SESSION_ID,
|
||||
field_from_the_future: 'preserved'
|
||||
}
|
||||
const scenario = scriptScenario([
|
||||
{ awaitUserMessage: true },
|
||||
{ emit: { type: 'keep_alive' } },
|
||||
{ emit: unknownTopLevel },
|
||||
{ emit: assistantWithUnknowns },
|
||||
{ emit: RESULT_FRAME }
|
||||
])
|
||||
const spawns: SpawnSeen[] = []
|
||||
const messages = await drainQuery({
|
||||
pathToClaudeCodeExecutable: FAKE_CLI,
|
||||
cwd: scenario.cwd,
|
||||
env: scenarioEnv(scenario),
|
||||
spawnClaudeCodeProcess: recordingSpawner(spawns)
|
||||
})
|
||||
|
||||
expect(messages.find((m) => m.uuid === 'uuid-unknown-1')).toEqual(unknownTopLevel)
|
||||
expect(messages.find((m) => m.uuid === 'uuid-assistant-1')).toEqual(assistantWithUnknowns)
|
||||
// The SDK intercepts keep_alive internally — a liveness signal must never
|
||||
// be derived from it reaching the consumer, because it does not.
|
||||
expect(messages.some((m) => m.type === 'keep_alive')).toBe(false)
|
||||
expect(messages.some((m) => m.type === 'result')).toBe(true)
|
||||
})
|
||||
|
||||
it('hands the custom spawner exactly the caller-supplied env, plus the two pinned SDK mutations', async () => {
|
||||
vi.stubEnv('ANTHROPIC_API_KEY', 'ambient-key-must-not-leak')
|
||||
const scenario = scriptScenario([{ awaitUserMessage: true }, { emit: RESULT_FRAME }])
|
||||
const spawns: SpawnSeen[] = []
|
||||
await drainQuery({
|
||||
pathToClaudeCodeExecutable: FAKE_CLI,
|
||||
cwd: scenario.cwd,
|
||||
env: {
|
||||
...scenarioEnv(scenario),
|
||||
CLAUDE_CONFIG_DIR: '/pinned/claude-config',
|
||||
ORCA_AGENT_SESSION_SPAWN_TOKEN: 'spawn-token-1',
|
||||
NODE_OPTIONS: '--max-old-space-size=64'
|
||||
},
|
||||
spawnClaudeCodeProcess: recordingSpawner(spawns)
|
||||
})
|
||||
|
||||
const env = spawns[0]!.env
|
||||
// Supplied values arrive verbatim: the config-dir pin and spawn token are
|
||||
// observable at this boundary, so Orca's auth scrubbing stays assertable.
|
||||
expect(env.CLAUDE_CONFIG_DIR).toBe('/pinned/claude-config')
|
||||
expect(env.ORCA_AGENT_SESSION_SPAWN_TOKEN).toBe('spawn-token-1')
|
||||
// Ambient process.env is NOT merged in when env is supplied.
|
||||
expect(env.ANTHROPIC_API_KEY).toBeUndefined()
|
||||
// The SDK's two documented mutations, pinned so a change is noticed.
|
||||
expect(env.CLAUDE_CODE_ENTRYPOINT).toBe('sdk-ts')
|
||||
expect('NODE_OPTIONS' in env).toBe(false)
|
||||
})
|
||||
|
||||
it('inherits process.env into the child when env is omitted — the ambient-auth sharp edge', async () => {
|
||||
const scenario = scriptScenario([{ awaitUserMessage: true }, { emit: RESULT_FRAME }])
|
||||
vi.stubEnv('ORCA_SDK_CONTRACT_SCENARIO_PATH', scenario.scenarioPath)
|
||||
vi.stubEnv('ORCA_SDK_CONTRACT_REPORT_PATH', scenario.reportPath)
|
||||
vi.stubEnv('ORCA_SDK_CONTRACT_AMBIENT_CANARY', 'inherited-from-process-env')
|
||||
const spawns: SpawnSeen[] = []
|
||||
await drainQuery({
|
||||
pathToClaudeCodeExecutable: FAKE_CLI,
|
||||
cwd: scenario.cwd,
|
||||
spawnClaudeCodeProcess: recordingSpawner(spawns)
|
||||
})
|
||||
|
||||
// Omitting env reproduces the ambient-auth-leak failure mode: the child
|
||||
// sees everything in process.env. Orca must therefore always pass an
|
||||
// explicit, fully-constructed env.
|
||||
expect(spawns[0]!.env.ORCA_SDK_CONTRACT_AMBIENT_CANARY).toBe('inherited-from-process-env')
|
||||
})
|
||||
|
||||
it('emits --replay-user-messages only through extraArgs, never on its own', async () => {
|
||||
const scenario = scriptScenario([{ awaitUserMessage: true }, { emit: RESULT_FRAME }])
|
||||
const bareSpawns: SpawnSeen[] = []
|
||||
await drainQuery({
|
||||
pathToClaudeCodeExecutable: FAKE_CLI,
|
||||
cwd: scenario.cwd,
|
||||
env: scenarioEnv(scenario),
|
||||
spawnClaudeCodeProcess: recordingSpawner(bareSpawns)
|
||||
})
|
||||
expect(bareSpawns[0]!.args).not.toContain('--replay-user-messages')
|
||||
|
||||
const replayScenario = scriptScenario([{ awaitUserMessage: true }, { emit: RESULT_FRAME }])
|
||||
const replaySpawns: SpawnSeen[] = []
|
||||
await drainQuery({
|
||||
pathToClaudeCodeExecutable: FAKE_CLI,
|
||||
cwd: replayScenario.cwd,
|
||||
env: scenarioEnv(replayScenario),
|
||||
extraArgs: { 'replay-user-messages': null },
|
||||
spawnClaudeCodeProcess: recordingSpawner(replaySpawns)
|
||||
})
|
||||
const replayArgs = replaySpawns[0]!.args
|
||||
expect(replayArgs.filter((arg) => arg === '--replay-user-messages')).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('produces a matching CLI flag for every pre-SDK argv entry', async () => {
|
||||
const scenario = scriptScenario([{ awaitUserMessage: true }, { emit: RESULT_FRAME }])
|
||||
const spawns: SpawnSeen[] = []
|
||||
// Driven by the real resolver, so the argv walk covers the durable-launchArgs
|
||||
// translation and its merge order, not a hand-written options literal.
|
||||
const launch = await resolvedLaunch(['--model', 'claude-sonnet-4-5', '--effort', 'high'])
|
||||
await drainQuery({
|
||||
...launch.options,
|
||||
pathToClaudeCodeExecutable: FAKE_CLI,
|
||||
cwd: scenario.cwd,
|
||||
env: scenarioEnv(scenario),
|
||||
canUseTool: (async () => ({ behavior: 'deny', message: 'unused' })) as CanUseTool,
|
||||
spawnClaudeCodeProcess: recordingSpawner(spawns)
|
||||
})
|
||||
|
||||
expect(spawns).toHaveLength(1)
|
||||
const argv = normalizeArgv(spawns[0]!.args)
|
||||
// Typed-first translation must not also spell the flag through extraArgs.
|
||||
for (const flag of ['--model', '--effort']) {
|
||||
expect(
|
||||
argv.filter((arg) => arg === flag),
|
||||
`${flag} occurrences`
|
||||
).toHaveLength(1)
|
||||
}
|
||||
expect(argv[argv.indexOf('--model') + 1]).toBe('claude-sonnet-4-5')
|
||||
expect(argv[argv.indexOf('--effort') + 1]).toBe('high')
|
||||
// Headless print mode is the SDK's only mode; `query()` never passes `-p`,
|
||||
// and if the SDK ever started passing it this pin would notice.
|
||||
const impliedByHeadlessQuery = new Set(['-p'])
|
||||
for (const entry of flagTable(PRE_SDK_ARGV)) {
|
||||
if (impliedByHeadlessQuery.has(entry.flag)) {
|
||||
expect(argv, `${entry.flag} is implied, never spelled`).not.toContain(entry.flag)
|
||||
continue
|
||||
}
|
||||
const at = argv.indexOf(entry.flag)
|
||||
expect(at, `SDK argv is missing ${entry.flag}`).toBeGreaterThanOrEqual(0)
|
||||
if (entry.value !== null) {
|
||||
expect(argv[at + 1], `value of ${entry.flag}`).toBe(entry.value)
|
||||
}
|
||||
}
|
||||
// The launch resolver always carries one of --session-id / --resume.
|
||||
const sessionAt = argv.indexOf('--session-id')
|
||||
expect(sessionAt).toBeGreaterThanOrEqual(0)
|
||||
expect(argv[sessionAt + 1]).toBe(launch.providerSessionId)
|
||||
})
|
||||
|
||||
it('still exposes the runtime get_settings reader the auth diagnostic depends on', async () => {
|
||||
// 0.3.251 ships getSettings() but redacts it from the Query declaration. This pin
|
||||
// is the drift alarm: if a bump drops or reshapes it, the diagnostic degrades and
|
||||
// this test says so instead of the degradation shipping silently.
|
||||
const settings = { env: { ANTHROPIC_BASE_URL: 'https://settings.example.test' } }
|
||||
const scenario = scriptScenario([{ delayMs: 3_000 }], { get_settings: settings })
|
||||
const session = query({
|
||||
prompt: singleUserTurn(),
|
||||
options: {
|
||||
pathToClaudeCodeExecutable: FAKE_CLI,
|
||||
cwd: scenario.cwd,
|
||||
env: scenarioEnv(scenario)
|
||||
}
|
||||
})
|
||||
try {
|
||||
const read = claudeQuerySettingsReader(session)
|
||||
expect(read, 'the SDK no longer exposes get_settings at runtime').not.toBeNull()
|
||||
await expect(read?.()).resolves.toEqual(settings)
|
||||
} finally {
|
||||
await session.return(undefined)
|
||||
}
|
||||
})
|
||||
|
||||
it('maps resume identity to --resume and --resume-session-at', async () => {
|
||||
const scenario = scriptScenario([{ awaitUserMessage: true }, { emit: RESULT_FRAME }])
|
||||
const spawns: SpawnSeen[] = []
|
||||
await drainQuery({
|
||||
pathToClaudeCodeExecutable: FAKE_CLI,
|
||||
cwd: scenario.cwd,
|
||||
env: scenarioEnv(scenario),
|
||||
resume: SESSION_ID,
|
||||
resumeSessionAt: LEAF_UUID,
|
||||
spawnClaudeCodeProcess: recordingSpawner(spawns)
|
||||
})
|
||||
|
||||
const argv = normalizeArgv(spawns[0]!.args)
|
||||
const resumeAt = argv.indexOf('--resume')
|
||||
expect(resumeAt).toBeGreaterThanOrEqual(0)
|
||||
expect(argv[resumeAt + 1]).toBe(SESSION_ID)
|
||||
const leafAt = argv.indexOf('--resume-session-at')
|
||||
expect(leafAt).toBeGreaterThanOrEqual(0)
|
||||
expect(argv[leafAt + 1]).toBe(LEAF_UUID)
|
||||
})
|
||||
|
||||
it('gives canUseTool the wire request_id and fires its abort signal on control_cancel_request', async () => {
|
||||
const scenario = scriptScenario([
|
||||
{ awaitUserMessage: true },
|
||||
{
|
||||
emit: {
|
||||
type: 'control_request',
|
||||
request_id: 'perm-421',
|
||||
request: {
|
||||
subtype: 'can_use_tool',
|
||||
tool_name: 'Bash',
|
||||
input: { command: 'echo hi' },
|
||||
tool_use_id: 'tool-use-9'
|
||||
}
|
||||
}
|
||||
},
|
||||
{ delayMs: 120 },
|
||||
{ emit: { type: 'control_cancel_request', request_id: 'perm-421' } },
|
||||
{ awaitControlResponse: 'perm-421' },
|
||||
{ emit: RESULT_FRAME }
|
||||
])
|
||||
const seen: { toolName: string; requestId: string; toolUseID: string }[] = []
|
||||
let abortFired = false
|
||||
const canUseTool: CanUseTool = (toolName, _input, { signal, requestId, toolUseID }) => {
|
||||
seen.push({ toolName, requestId, toolUseID })
|
||||
return new Promise((resolve) => {
|
||||
signal.addEventListener('abort', () => {
|
||||
abortFired = true
|
||||
resolve({ behavior: 'deny', message: 'cancelled by test' })
|
||||
})
|
||||
})
|
||||
}
|
||||
const spawns: SpawnSeen[] = []
|
||||
await drainQuery({
|
||||
pathToClaudeCodeExecutable: FAKE_CLI,
|
||||
cwd: scenario.cwd,
|
||||
env: scenarioEnv(scenario),
|
||||
canUseTool,
|
||||
spawnClaudeCodeProcess: recordingSpawner(spawns)
|
||||
})
|
||||
|
||||
expect(seen).toEqual([{ toolName: 'Bash', requestId: 'perm-421', toolUseID: 'tool-use-9' }])
|
||||
expect(abortFired).toBe(true)
|
||||
// The callback's settlement is written back onto the wire against the same id.
|
||||
const settled = scenario
|
||||
.readReport()
|
||||
.controlResponses.find((frame) => frame.response.request_id === 'perm-421')
|
||||
expect(settled?.response.response?.behavior).toBe('deny')
|
||||
// Exactly one process spawn per query, control traffic included.
|
||||
expect(spawns).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('runs the executable given via pathToClaudeCodeExecutable under the default spawner', async () => {
|
||||
const scenario = scriptScenario([{ awaitUserMessage: true }, { emit: RESULT_FRAME }])
|
||||
const messages = await drainQuery({
|
||||
pathToClaudeCodeExecutable: FAKE_CLI,
|
||||
cwd: scenario.cwd,
|
||||
env: scenarioEnv(scenario)
|
||||
})
|
||||
|
||||
expect(messages.some((m) => m.type === 'result')).toBe(true)
|
||||
const report = scenario.readReport()
|
||||
// The SDK executed exactly the script we pointed it at — no bundled binary.
|
||||
expect(report.argv[0]).toBe(FAKE_CLI)
|
||||
expect(report.execPath).toContain('node')
|
||||
// And the streaming handshake went to it: the SDK sent its initialize
|
||||
// control request to our script.
|
||||
expect(report.controlRequests.some((frame) => frame.request.subtype === 'initialize')).toBe(
|
||||
true
|
||||
)
|
||||
})
|
||||
|
||||
it('pins the SDK version the contract was verified against', () => {
|
||||
const sdkEntry = createRequire(__filename).resolve('@anthropic-ai/claude-agent-sdk')
|
||||
const manifest = JSON.parse(readFileSync(join(dirname(sdkEntry), 'package.json'), 'utf8')) as {
|
||||
version: string
|
||||
}
|
||||
expect(manifest.version).toBe(PINNED_SDK_VERSION)
|
||||
})
|
||||
|
||||
it('keeps the eight bundled CLI platform binaries out of the install', () => {
|
||||
const sdkEntry = createRequire(__filename).resolve('@anthropic-ai/claude-agent-sdk')
|
||||
// The SDK's own scoped directory is where pnpm would link its optional
|
||||
// platform packages; ignoredOptionalDependencies must keep them all absent.
|
||||
const scopeDir = dirname(dirname(sdkEntry))
|
||||
for (const basename of SDK_PLATFORM_PACKAGE_BASENAMES) {
|
||||
expect(
|
||||
existsSync(join(scopeDir, basename, 'package.json')),
|
||||
`${basename} must not be installed`
|
||||
).toBe(false)
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,154 @@
|
||||
import type {
|
||||
PermissionMode,
|
||||
Query,
|
||||
SDKControlInterruptResponse
|
||||
} from '@anthropic-ai/claude-agent-sdk'
|
||||
|
||||
export class ClaudeControlRequestError extends Error {
|
||||
constructor(
|
||||
readonly subtype: string,
|
||||
message: string
|
||||
) {
|
||||
super(message)
|
||||
this.name = 'ClaudeControlRequestError'
|
||||
}
|
||||
}
|
||||
|
||||
export const CLAUDE_DEFAULT_REQUEST_TIMEOUT_MS = 30_000
|
||||
|
||||
/** The SDK closes a query out from under an in-flight control request with this exact message. */
|
||||
const QUERY_CLOSED_MESSAGE = 'Query closed before response received'
|
||||
|
||||
/** 0.3.251 ships getSettings() but redacts it from the Query declaration; the typeof guard below is its degradation path. */
|
||||
type ClaudeQuerySettingsReader = { getSettings?: () => Promise<unknown> }
|
||||
|
||||
export function claudeQuerySettingsReader(query: Query): (() => Promise<unknown>) | null {
|
||||
const reader = (query as unknown as ClaudeQuerySettingsReader).getSettings
|
||||
return typeof reader === 'function' ? reader.bind(query) : null
|
||||
}
|
||||
|
||||
/**
|
||||
* cancel_async_message is a runtime Query method the shipped 0.3.251 declaration omits;
|
||||
* it withdraws a single still-queued async user message by uuid so an interrupted turn
|
||||
* cannot spawn a later unexpected turn. The typeof guard is its degradation path.
|
||||
*/
|
||||
type ClaudeQueryAsyncCanceller = { cancelAsyncMessage?: (uuid: string) => Promise<unknown> }
|
||||
|
||||
export function claudeQueryAsyncCanceller(
|
||||
query: Query
|
||||
): ((uuid: string) => Promise<unknown>) | null {
|
||||
const cancel = (query as unknown as ClaudeQueryAsyncCanceller).cancelAsyncMessage
|
||||
return typeof cancel === 'function' ? cancel.bind(query) : null
|
||||
}
|
||||
|
||||
export type ClaudeControlOptions = { timeoutMs?: number }
|
||||
|
||||
/**
|
||||
* Run one native Query control method under Orca's deadline and error classification.
|
||||
*
|
||||
* The SDK owns correlation but applies no deadline, so the timeout stays here — and its
|
||||
* message is load-bearing: the init proof matches on `claude initialize request timed out`.
|
||||
* A closed query is a transport failure, not the CLI rejecting the request, so only the
|
||||
* latter is re-thrown as a `ClaudeControlRequestError` a caller may surface as a rejection.
|
||||
*/
|
||||
export function runClaudeControl<T>(
|
||||
subtype: string,
|
||||
run: () => Promise<T>,
|
||||
timeoutMs: number = CLAUDE_DEFAULT_REQUEST_TIMEOUT_MS
|
||||
): Promise<T> {
|
||||
let timer: ReturnType<typeof setTimeout> | null = null
|
||||
const deadline = new Promise<never>((_resolve, reject) => {
|
||||
timer = setTimeout(() => reject(new Error(`claude ${subtype} request timed out`)), timeoutMs)
|
||||
timer.unref?.()
|
||||
})
|
||||
return Promise.race([
|
||||
Promise.resolve()
|
||||
.then(run)
|
||||
.catch((error: unknown) => {
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
if (error instanceof ClaudeControlRequestError || message === QUERY_CLOSED_MESSAGE) {
|
||||
throw error
|
||||
}
|
||||
throw new ClaudeControlRequestError(subtype, message)
|
||||
}),
|
||||
deadline
|
||||
]).finally(() => {
|
||||
if (timer) {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/** The native control surface Orca drives, one method per Query control request. */
|
||||
export type ClaudeControlSurface = {
|
||||
interrupt: (
|
||||
options?: ClaudeControlOptions & { cancelQueued?: boolean }
|
||||
) => Promise<SDKControlInterruptResponse | undefined>
|
||||
cancelAsyncMessage: (uuid: string, options?: ClaudeControlOptions) => Promise<void>
|
||||
setModel: (model: string | undefined, options?: ClaudeControlOptions) => Promise<void>
|
||||
setPermissionMode: (mode: PermissionMode, options?: ClaudeControlOptions) => Promise<void>
|
||||
applyFlagSettings: (
|
||||
settings: Parameters<Query['applyFlagSettings']>[0],
|
||||
options?: ClaudeControlOptions
|
||||
) => Promise<void>
|
||||
supportedModels: (options?: ClaudeControlOptions) => Promise<unknown[]>
|
||||
initializationResult: (options?: ClaudeControlOptions) => Promise<unknown>
|
||||
getSettings: (options?: ClaudeControlOptions) => Promise<unknown>
|
||||
}
|
||||
|
||||
type InterruptingQuery = {
|
||||
interrupt: (options?: {
|
||||
cancelQueued?: boolean
|
||||
}) => Promise<SDKControlInterruptResponse | undefined>
|
||||
}
|
||||
|
||||
export function createClaudeControlSurface(query: Query): ClaudeControlSurface {
|
||||
return {
|
||||
interrupt: (options) =>
|
||||
runClaudeControl(
|
||||
'interrupt',
|
||||
() =>
|
||||
(query as unknown as InterruptingQuery).interrupt(
|
||||
options?.cancelQueued ? { cancelQueued: true } : undefined
|
||||
),
|
||||
options?.timeoutMs
|
||||
),
|
||||
cancelAsyncMessage: (uuid, options) => {
|
||||
const cancel = claudeQueryAsyncCanceller(query)
|
||||
return cancel
|
||||
? runClaudeControl('cancel_async_message', () => cancel(uuid), options?.timeoutMs).then(
|
||||
() => {}
|
||||
)
|
||||
: Promise.resolve()
|
||||
},
|
||||
setModel: (model, options) =>
|
||||
runClaudeControl('set_model', () => query.setModel(model), options?.timeoutMs).then(() => {}),
|
||||
setPermissionMode: (mode, options) =>
|
||||
runClaudeControl(
|
||||
'set_permission_mode',
|
||||
() => query.setPermissionMode(mode),
|
||||
options?.timeoutMs
|
||||
).then(() => {}),
|
||||
applyFlagSettings: (settings, options) =>
|
||||
runClaudeControl(
|
||||
'apply_flag_settings',
|
||||
() => query.applyFlagSettings(settings),
|
||||
options?.timeoutMs
|
||||
).then(() => {}),
|
||||
supportedModels: (options) =>
|
||||
runClaudeControl('list_models', () => query.supportedModels(), options?.timeoutMs),
|
||||
initializationResult: (options) =>
|
||||
runClaudeControl('initialize', () => query.initializationResult(), options?.timeoutMs),
|
||||
getSettings: (options) => {
|
||||
const read = claudeQuerySettingsReader(query)
|
||||
return read
|
||||
? runClaudeControl('get_settings', read, options?.timeoutMs)
|
||||
: Promise.reject(
|
||||
new ClaudeControlRequestError(
|
||||
'get_settings',
|
||||
'this SDK exposes no get_settings request'
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { DescendantSnapshot } from '../pty-descendant-termination'
|
||||
import type { WindowsDescendantSnapshot } from '../windows-descendant-exit-verification'
|
||||
import { collectDescendantRows } from '../pty-descendant-termination'
|
||||
import { createClaudeChildTreeReaper } from './claude-agent-sdk-exit-proof'
|
||||
import { mergeClaudeCapturedTrees } from './claude-child-tree-snapshot'
|
||||
|
||||
function posixSnapshot(capturedAtMs: number): DescendantSnapshot {
|
||||
return {
|
||||
root: { pid: 100, startedAt: 'Mon Jan 1 00:00:00 2026' },
|
||||
rootPgid: 100,
|
||||
descendants: [{ pid: 200, ppid: 100, pgid: 100, startedAt: 'Mon Jan 1 00:00:01 2026' }],
|
||||
capturedAtMs
|
||||
}
|
||||
}
|
||||
|
||||
function windowsSnapshot(): WindowsDescendantSnapshot {
|
||||
return {
|
||||
root: { pid: 100, creationTimeMs: 5 },
|
||||
descendants: [{ pid: 200, creationTimeMs: 7 }],
|
||||
unidentifiedCount: 0,
|
||||
capturedAtMs: 1
|
||||
}
|
||||
}
|
||||
|
||||
describe('Claude child root identity', () => {
|
||||
it('keeps a retained row boundary when a refresh observes no new descendants', () => {
|
||||
const previous = posixSnapshot(1_700_000_000_900)
|
||||
const next = posixSnapshot(1_700_000_002_100)
|
||||
|
||||
expect(
|
||||
mergeClaudeCapturedTrees(
|
||||
{ platform: 'posix', tree: previous },
|
||||
{ platform: 'posix', tree: next }
|
||||
)
|
||||
).toEqual({
|
||||
platform: 'posix',
|
||||
tree: { ...next, capturedAtMsByPid: { '200': previous.capturedAtMs } }
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps the descendant verdict when a POSIX root probe is unavailable', async () => {
|
||||
const child = { pid: 100, kill: vi.fn(() => true) }
|
||||
const terminateDescendants = vi.fn(async () => 'exited' as const)
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'linux',
|
||||
captureDescendants: vi.fn(async () => posixSnapshot(1)),
|
||||
terminateDescendants,
|
||||
verifyRootIdentity: vi.fn(async () => false)
|
||||
})
|
||||
|
||||
// POSIX runs no bare-pid root operation, so a declined probe withholds
|
||||
// nothing: the handle kill still lands and the verification still speaks.
|
||||
await expect(tree.reap()).resolves.toBe('exited')
|
||||
expect(terminateDescendants).toHaveBeenCalled()
|
||||
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
|
||||
})
|
||||
|
||||
it('rejects mixed old and recycled root rows instead of making the tree killable', async () => {
|
||||
const child = { pid: 100, kill: vi.fn(() => true) }
|
||||
const terminateDescendants = vi.fn(async () => 'exited' as const)
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'linux',
|
||||
captureDescendants: vi.fn(async () =>
|
||||
collectDescendantRows(
|
||||
100,
|
||||
[
|
||||
{ pid: 100, ppid: 1, pgid: 100, startedAt: 'Mon Jan 1 00:00:00 2026' },
|
||||
{ pid: 100, ppid: 1, pgid: 101, startedAt: 'Mon Jan 1 00:00:01 2026' },
|
||||
{ pid: 200, ppid: 100, pgid: 200, startedAt: 'Mon Jan 1 00:00:00 2026' }
|
||||
],
|
||||
1
|
||||
)
|
||||
),
|
||||
terminateDescendants,
|
||||
verifyRootIdentity: vi.fn(async () => true)
|
||||
})
|
||||
|
||||
await expect(tree.reap()).resolves.toBe('unverifiable')
|
||||
// No admissible snapshot means no row may be signalled from its number, but
|
||||
// the root still leaves through the handle Node owns.
|
||||
expect(terminateDescendants).not.toHaveBeenCalled()
|
||||
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
|
||||
})
|
||||
|
||||
it('fails closed when Windows root identity revalidation is unavailable', async () => {
|
||||
const child = { pid: 100, kill: vi.fn(() => true) }
|
||||
const terminateWindowsTree = vi.fn(async () => {})
|
||||
const terminateWindowsDescendants = vi.fn(async () => 'exited' as const)
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'win32',
|
||||
captureWindowsDescendants: vi.fn(async () => windowsSnapshot()),
|
||||
terminateWindowsTree,
|
||||
terminateWindowsDescendants,
|
||||
verifyRootIdentity: vi.fn(async () => false)
|
||||
})
|
||||
|
||||
await expect(tree.reap()).resolves.toBe('unverifiable')
|
||||
// taskkill /T /F addresses a bare pid and stays gated; the handle does not.
|
||||
expect(terminateWindowsTree).not.toHaveBeenCalled()
|
||||
expect(terminateWindowsDescendants).not.toHaveBeenCalled()
|
||||
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,934 @@
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { PassThrough } from 'node:stream'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { spawnProcess, type SpawnedProcess } from '../../shared/child-process/run-process'
|
||||
import type { DescendantTreeVerdict } from '../pty-descendant-exit-verification'
|
||||
import type { DescendantSnapshot } from '../pty-descendant-termination'
|
||||
import type { WindowsDescendantSnapshot } from '../windows-descendant-exit-verification'
|
||||
import {
|
||||
createClaudeChildTreeReaper as createClaudeChildTreeReaperImpl,
|
||||
proveClaudeChildExit,
|
||||
type ClaudeChildTreeReaper
|
||||
} from './claude-agent-sdk-exit-proof'
|
||||
|
||||
// The descendant models an MCP server: it either cooperates or, when it traps
|
||||
// SIGTERM, only a forced, verified sweep can reach it. The root either traps
|
||||
// SIGTERM too, or leaves promptly on stdin end the way a healthy CLI does —
|
||||
// which is the path that used to skip descendant proof entirely.
|
||||
function childWithDescendantScript(input: {
|
||||
rootTrapsSigterm: boolean
|
||||
descendantTrapsSigterm: boolean
|
||||
}): string {
|
||||
const descendantScript = `${input.descendantTrapsSigterm ? 'process.on("SIGTERM", () => {}); ' : ''}setInterval(() => {}, 1000000)`
|
||||
const rootBehaviour = input.rootTrapsSigterm
|
||||
? `process.on('SIGTERM', () => {})
|
||||
process.on('SIGINT', () => {})
|
||||
setInterval(() => {}, 1000000)`
|
||||
: `process.stdin.on('end', () => process.exit(0))
|
||||
process.stdin.resume()`
|
||||
return `
|
||||
const descendant = require('node:child_process').spawn(
|
||||
process.execPath,
|
||||
['-e', ${JSON.stringify(descendantScript)}],
|
||||
{ stdio: 'ignore' }
|
||||
)
|
||||
descendant.unref()
|
||||
process.stdout.write(JSON.stringify({ descendantPid: descendant.pid }) + '\\n')
|
||||
${rootBehaviour}
|
||||
`
|
||||
}
|
||||
|
||||
const COOPERATIVE_CHILD = `
|
||||
process.stdin.on('end', () => process.exit(0))
|
||||
process.stdin.resume()
|
||||
process.stdout.write('ready\\n')
|
||||
`
|
||||
|
||||
/**
|
||||
* Sampled synchronously so it reads the exact moment the close boundary is
|
||||
* crossed. A zombie has exited (its parent just has not reaped it yet), so a
|
||||
* kill(pid, 0) probe would misreport it as running.
|
||||
*/
|
||||
function descendantState(pid: number): 'running' | 'exited' {
|
||||
let state: string
|
||||
try {
|
||||
state = execFileSync('ps', ['-o', 'state=', '-p', String(pid)], {
|
||||
encoding: 'utf8',
|
||||
env: { ...process.env, LANG: 'C', LC_ALL: 'C' }
|
||||
}).trim()
|
||||
} catch (error) {
|
||||
// ps exits 1 when no process matches; anything else is a failed probe, not an answer.
|
||||
if ((error as { status?: number }).status !== 1) {
|
||||
throw error
|
||||
}
|
||||
return 'exited'
|
||||
}
|
||||
return state.startsWith('Z') ? 'exited' : 'running'
|
||||
}
|
||||
|
||||
/**
|
||||
* ps lstart is second-resolution, so the identity-safe sweep only SIGKILLs a row
|
||||
* born strictly before the second the snapshot was captured in. The snapshot is
|
||||
* armed the moment close begins, so a descendant born in that same second can
|
||||
* only be asked, never forced — the same bound an MCP server spawned within a
|
||||
* second of the user closing the chat would hit.
|
||||
*/
|
||||
function ageDescendantPastTheCaptureSecond(): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, 1_000 - (Date.now() % 1_000) + 20))
|
||||
}
|
||||
|
||||
/**
|
||||
* The close ladder as production drives it: `closeProcessRegistry` retries an
|
||||
* unproven close, and each retry re-verifies the retained snapshot. A loaded
|
||||
* host can spend one attempt's whole window inside `ps`, and reporting false
|
||||
* there is the honest verdict — the requirement is that TRUE never outruns the
|
||||
* observation, which the caller asserts at whichever boundary returns it.
|
||||
*/
|
||||
async function proveExitWithRetries(
|
||||
input: Parameters<typeof proveClaudeChildExit>[0],
|
||||
attempts = 3
|
||||
): Promise<boolean> {
|
||||
for (let attempt = 1; attempt < attempts; attempt += 1) {
|
||||
if (await proveClaudeChildExit(input)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return proveClaudeChildExit(input)
|
||||
}
|
||||
|
||||
function spawnScript(script: string): ReturnType<typeof spawnProcess> {
|
||||
return spawnProcess({
|
||||
program: process.execPath,
|
||||
args: ['-e', script],
|
||||
stdio: ['pipe', 'pipe', 'pipe']
|
||||
})
|
||||
}
|
||||
|
||||
function firstStdoutLine(child: ReturnType<typeof spawnProcess>): Promise<string> {
|
||||
return new Promise((resolve) => {
|
||||
child.stdout.setEncoding('utf8').once('data', (chunk: string) => resolve(chunk.trim()))
|
||||
})
|
||||
}
|
||||
|
||||
function observeExit(child: EventEmitter): { exitPromise: Promise<void>; exited: () => boolean } {
|
||||
let exited = false
|
||||
const exitPromise = new Promise<void>((resolve) => {
|
||||
child.once('exit', () => {
|
||||
exited = true
|
||||
resolve()
|
||||
})
|
||||
})
|
||||
return { exitPromise, exited: () => exited }
|
||||
}
|
||||
|
||||
/** `null` models a spawn that failed before a pid existed. */
|
||||
function mockChild(
|
||||
pid: number | null = 424242
|
||||
): EventEmitter &
|
||||
Pick<SpawnedProcess, 'pid' | 'kill' | 'stdin'> & { kill: ReturnType<typeof vi.fn> } {
|
||||
const child = new EventEmitter()
|
||||
return Object.assign(child, {
|
||||
pid: pid ?? undefined,
|
||||
stdin: new PassThrough(),
|
||||
kill: vi.fn(() => true)
|
||||
}) as never
|
||||
}
|
||||
|
||||
/** A tree whose verdict is scripted per reap, recording when it was armed. */
|
||||
function mockTree(verdicts: DescendantTreeVerdict[]): ClaudeChildTreeReaper & {
|
||||
capture: ReturnType<typeof vi.fn>
|
||||
reap: ReturnType<typeof vi.fn>
|
||||
} {
|
||||
let treeVerdict: DescendantTreeVerdict = 'unverifiable'
|
||||
return {
|
||||
capture: vi.fn(async () => {}),
|
||||
reap: vi.fn(async () => {
|
||||
treeVerdict = verdicts.shift() ?? treeVerdict
|
||||
return treeVerdict
|
||||
}),
|
||||
get treeVerdict() {
|
||||
return treeVerdict
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function windowsSnapshotOf(descendantPid: number): WindowsDescendantSnapshot {
|
||||
return {
|
||||
root: { pid: 424242, creationTimeMs: 1_700_000_000_001 },
|
||||
descendants: [{ pid: descendantPid, creationTimeMs: 1_700_000_000_000 }],
|
||||
unidentifiedCount: 0,
|
||||
capturedAtMs: 1
|
||||
}
|
||||
}
|
||||
|
||||
function snapshotOf(descendantPid: number): DescendantSnapshot {
|
||||
return {
|
||||
root: { pid: 424242, startedAt: 'Mon Jan 1 00:00:00 2026' },
|
||||
rootPgid: 1,
|
||||
descendants: [
|
||||
{ pid: descendantPid, ppid: 424242, pgid: 1, startedAt: 'Mon Jan 1 00:00:00 2026' }
|
||||
],
|
||||
capturedAtMs: 1
|
||||
}
|
||||
}
|
||||
|
||||
// Unit tests use synthetic process ids; production always supplies the fresh
|
||||
// identity probe, so the harness explicitly models a matching probe.
|
||||
function createClaudeChildTreeReaper(
|
||||
child: Parameters<typeof createClaudeChildTreeReaperImpl>[0],
|
||||
deps: Parameters<typeof createClaudeChildTreeReaperImpl>[1] = {}
|
||||
): ReturnType<typeof createClaudeChildTreeReaperImpl> {
|
||||
return createClaudeChildTreeReaperImpl(child, {
|
||||
verifyRootIdentity: async () => true,
|
||||
...deps
|
||||
})
|
||||
}
|
||||
|
||||
describe('claude child exit proof', () => {
|
||||
it.runIf(process.platform !== 'win32')(
|
||||
'reports a proven exit only once a SIGTERM-resistant descendant is gone at the close boundary',
|
||||
async () => {
|
||||
const child = spawnScript(
|
||||
childWithDescendantScript({ rootTrapsSigterm: true, descendantTrapsSigterm: true })
|
||||
)
|
||||
const { descendantPid } = JSON.parse(await firstStdoutLine(child)) as {
|
||||
descendantPid: number
|
||||
}
|
||||
expect(descendantState(descendantPid)).toBe('running')
|
||||
await ageDescendantPastTheCaptureSecond()
|
||||
|
||||
try {
|
||||
const proven = await proveExitWithRetries({ child, ...observeExit(child) })
|
||||
// Evaluated AT the boundary, not by polling until a deferred sweep timer
|
||||
// wins: true releases the lease, so a descendant still running here is
|
||||
// exactly the orphan the proof exists to prevent. False would be the
|
||||
// honest verdict for a tree that outlived the bounded ladder.
|
||||
expect({ proven, descendant: descendantState(descendantPid) }).toEqual({
|
||||
proven: true,
|
||||
descendant: 'exited'
|
||||
})
|
||||
} finally {
|
||||
// Failure-safe only: the assertion above owns the requirement, this just
|
||||
// stops a failing run from leaking a process.
|
||||
try {
|
||||
process.kill(descendantPid, 'SIGKILL')
|
||||
} catch {
|
||||
// Already gone.
|
||||
}
|
||||
}
|
||||
},
|
||||
20_000
|
||||
)
|
||||
|
||||
it.runIf(process.platform !== 'win32')(
|
||||
'proves a promptly exiting root only once its stubborn descendant is gone too',
|
||||
async () => {
|
||||
// The ordinary healthy close: the root leaves on stdin end within the graceful
|
||||
// window. Its descendant must still be proven gone, not assumed gone with it.
|
||||
const child = spawnScript(
|
||||
childWithDescendantScript({ rootTrapsSigterm: false, descendantTrapsSigterm: true })
|
||||
)
|
||||
const { descendantPid } = JSON.parse(await firstStdoutLine(child)) as {
|
||||
descendantPid: number
|
||||
}
|
||||
expect(descendantState(descendantPid)).toBe('running')
|
||||
await ageDescendantPastTheCaptureSecond()
|
||||
|
||||
try {
|
||||
const proven = await proveExitWithRetries({ child, ...observeExit(child) })
|
||||
expect({ proven, descendant: descendantState(descendantPid) }).toEqual({
|
||||
proven: true,
|
||||
descendant: 'exited'
|
||||
})
|
||||
} finally {
|
||||
try {
|
||||
process.kill(descendantPid, 'SIGKILL')
|
||||
} catch {
|
||||
// Already gone.
|
||||
}
|
||||
}
|
||||
},
|
||||
20_000
|
||||
)
|
||||
|
||||
it.runIf(process.platform !== 'win32')(
|
||||
'still proves a stubborn child whose descendant honours SIGTERM',
|
||||
async () => {
|
||||
const child = spawnScript(
|
||||
childWithDescendantScript({ rootTrapsSigterm: true, descendantTrapsSigterm: false })
|
||||
)
|
||||
const { descendantPid } = JSON.parse(await firstStdoutLine(child)) as {
|
||||
descendantPid: number
|
||||
}
|
||||
try {
|
||||
const proven = await proveExitWithRetries({ child, ...observeExit(child) })
|
||||
expect({ proven, descendant: descendantState(descendantPid) }).toEqual({
|
||||
proven: true,
|
||||
descendant: 'exited'
|
||||
})
|
||||
} finally {
|
||||
try {
|
||||
process.kill(descendantPid, 'SIGKILL')
|
||||
} catch {
|
||||
// Already gone.
|
||||
}
|
||||
}
|
||||
},
|
||||
20_000
|
||||
)
|
||||
|
||||
it('arms the snapshot before stdin closes and verifies it after a clean exit', async () => {
|
||||
const child = spawnScript(COOPERATIVE_CHILD)
|
||||
expect(await firstStdoutLine(child)).toBe('ready')
|
||||
const exit = observeExit(child)
|
||||
const tree = mockTree(['exited'])
|
||||
let exitedWhenArmed: boolean | null = null
|
||||
tree.capture.mockImplementation(async () => {
|
||||
exitedWhenArmed = exit.exited()
|
||||
})
|
||||
|
||||
await expect(proveClaudeChildExit({ child, ...exit, tree })).resolves.toBe(true)
|
||||
// The snapshot is the only proof that survives the root: taken while it lived,
|
||||
// verified once it left. A reap before the exit would have been the forced ladder.
|
||||
expect(exitedWhenArmed).toBe(false)
|
||||
expect(tree.reap).toHaveBeenCalledTimes(1)
|
||||
expect(exit.exited()).toBe(true)
|
||||
}, 20_000)
|
||||
|
||||
it('proves a clean close of a childless root with one snapshot and no signal', async () => {
|
||||
const child = spawnScript(COOPERATIVE_CHILD)
|
||||
expect(await firstStdoutLine(child)).toBe('ready')
|
||||
|
||||
await expect(proveClaudeChildExit({ child, ...observeExit(child) })).resolves.toBe(true)
|
||||
}, 20_000)
|
||||
|
||||
it('reports an unprovable exit as false rather than assuming the child died', async () => {
|
||||
const child = mockChild()
|
||||
const tree = mockTree(['exited'])
|
||||
|
||||
await expect(
|
||||
proveClaudeChildExit({
|
||||
child,
|
||||
exitPromise: new Promise<void>(() => {}),
|
||||
exited: () => false,
|
||||
tree
|
||||
})
|
||||
).resolves.toBe(false)
|
||||
expect(tree.reap).toHaveBeenCalledTimes(1)
|
||||
}, 20_000)
|
||||
|
||||
it('reports false when the root exit was observed but a descendant was seen alive', async () => {
|
||||
const child = mockChild()
|
||||
const exit = observeExit(child)
|
||||
const tree = mockTree(['live'])
|
||||
tree.reap.mockImplementation(async () => {
|
||||
child.emit('exit', null, 'SIGKILL')
|
||||
return 'live'
|
||||
})
|
||||
|
||||
await expect(proveClaudeChildExit({ child, ...exit, tree })).resolves.toBe(false)
|
||||
expect(exit.exited()).toBe(true)
|
||||
// One verification per attempt: the retried close re-verifies, this one does not.
|
||||
expect(tree.reap).toHaveBeenCalledTimes(1)
|
||||
}, 20_000)
|
||||
|
||||
it('re-verifies an unproven tree on a retried close instead of trusting the dead root', async () => {
|
||||
const child = mockChild()
|
||||
const tree = mockTree(['exited'])
|
||||
|
||||
await expect(
|
||||
proveClaudeChildExit({ child, exitPromise: Promise.resolve(), exited: () => true, tree })
|
||||
).resolves.toBe(true)
|
||||
expect(tree.reap).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('stays unproven for a root that left before any snapshot could be armed', async () => {
|
||||
const child = mockChild()
|
||||
const captureDescendants = vi.fn(async () => snapshotOf(4243))
|
||||
const terminateDescendants = vi.fn()
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'darwin',
|
||||
exited: () => true,
|
||||
captureDescendants,
|
||||
terminateDescendants
|
||||
})
|
||||
|
||||
await expect(
|
||||
proveClaudeChildExit({ child, exitPromise: Promise.resolve(), exited: () => true, tree })
|
||||
).resolves.toBe(false)
|
||||
// A dead root's descendants have reparented: walking its pid now could only
|
||||
// sweep a stranger, so no walk is attempted and nothing is proven.
|
||||
expect(captureDescendants).not.toHaveBeenCalled()
|
||||
expect(terminateDescendants).not.toHaveBeenCalled()
|
||||
expect(tree.treeVerdict).toBe('unverifiable')
|
||||
})
|
||||
})
|
||||
|
||||
describe('claude child tree reaper', () => {
|
||||
it('kills the root while verification runs and never stops it first', async () => {
|
||||
const child = mockChild()
|
||||
const release = Promise.withResolvers<DescendantTreeVerdict>()
|
||||
const terminateDescendants = vi.fn(() => release.promise)
|
||||
const captureDescendants = vi.fn(async () => snapshotOf(4243))
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'darwin',
|
||||
captureDescendants,
|
||||
terminateDescendants
|
||||
})
|
||||
|
||||
const first = tree.reap()
|
||||
const second = tree.reap()
|
||||
await vi.waitFor(() => expect(terminateDescendants).toHaveBeenCalledTimes(1))
|
||||
// A stopped root cannot verify: its killed children stay zombie rows in ps.
|
||||
expect(child.kill.mock.calls).toEqual([['SIGKILL']])
|
||||
expect(tree.treeVerdict).toBe('unverifiable')
|
||||
|
||||
release.resolve('exited')
|
||||
await expect(Promise.all([first, second])).resolves.toEqual(['exited', 'exited'])
|
||||
expect(captureDescendants).toHaveBeenCalledTimes(1)
|
||||
expect(tree.treeVerdict).toBe('exited')
|
||||
})
|
||||
|
||||
it('re-verifies the retained snapshot on a later reap rather than re-walking a dead root', async () => {
|
||||
const child = mockChild()
|
||||
const captureDescendants = vi.fn(async () => snapshotOf(4243))
|
||||
const terminateDescendants = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce('live')
|
||||
.mockResolvedValueOnce('exited')
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'linux',
|
||||
captureDescendants,
|
||||
terminateDescendants
|
||||
})
|
||||
|
||||
await expect(tree.reap()).resolves.toBe('live')
|
||||
expect(tree.treeVerdict).toBe('live')
|
||||
await expect(tree.reap()).resolves.toBe('exited')
|
||||
expect(captureDescendants).toHaveBeenCalledTimes(1)
|
||||
expect(terminateDescendants).toHaveBeenNthCalledWith(2, snapshotOf(4243))
|
||||
expect(tree.treeVerdict).toBe('exited')
|
||||
})
|
||||
|
||||
it('keeps an observed exit when a later re-read cannot see the table', async () => {
|
||||
const child = mockChild()
|
||||
const terminateDescendants = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce('exited')
|
||||
.mockResolvedValueOnce('unverifiable')
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'linux',
|
||||
captureDescendants: vi.fn(async () => snapshotOf(4243)),
|
||||
terminateDescendants
|
||||
})
|
||||
|
||||
await expect(tree.reap()).resolves.toBe('exited')
|
||||
await expect(tree.reap()).resolves.toBe('unverifiable')
|
||||
expect(tree.treeVerdict).toBe('exited')
|
||||
})
|
||||
|
||||
it('keeps an observed live descendant when a later re-read cannot see the table', async () => {
|
||||
const child = mockChild()
|
||||
// Reap #1 completed and saw a descendant alive at its deadline; the root then
|
||||
// left on its own and the re-verification on a loaded host could not read the
|
||||
// table. "Could not look" must not erase "was seen alive": the lease release
|
||||
// gate is exactly the pair this distinguishes.
|
||||
const terminateDescendants = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce('live')
|
||||
.mockResolvedValueOnce('unverifiable')
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'linux',
|
||||
captureDescendants: vi.fn(async () => snapshotOf(4243)),
|
||||
terminateDescendants
|
||||
})
|
||||
|
||||
await expect(tree.reap()).resolves.toBe('live')
|
||||
await expect(tree.reap()).resolves.toBe('unverifiable')
|
||||
expect(tree.treeVerdict).toBe('live')
|
||||
})
|
||||
|
||||
it('treats an unreadable process table as unproven and re-walks the live root', async () => {
|
||||
const child = mockChild()
|
||||
// A loaded host can miss the table's deadline; while the root still lives
|
||||
// that is a retryable read, not evidence that it has no descendants.
|
||||
const captureDescendants = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(null)
|
||||
.mockResolvedValueOnce(snapshotOf(4243))
|
||||
const terminateDescendants = vi.fn(async () => 'exited' as const)
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'linux',
|
||||
captureDescendants,
|
||||
terminateDescendants
|
||||
})
|
||||
|
||||
await expect(tree.reap()).resolves.toBe('unverifiable')
|
||||
expect(terminateDescendants).not.toHaveBeenCalled()
|
||||
await expect(tree.reap()).resolves.toBe('exited')
|
||||
expect(captureDescendants).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('does not latch a missing root while it is still live', async () => {
|
||||
const child = mockChild()
|
||||
const captureDescendants = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({ rootPgid: null, descendants: [], capturedAtMs: 1 })
|
||||
.mockResolvedValueOnce(snapshotOf(4243))
|
||||
const terminateDescendants = vi.fn(async () => 'exited' as const)
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'linux',
|
||||
captureDescendants,
|
||||
terminateDescendants
|
||||
})
|
||||
|
||||
await tree.capture()
|
||||
await expect(tree.reap()).resolves.toBe('exited')
|
||||
expect(captureDescendants).toHaveBeenCalledTimes(2)
|
||||
expect(terminateDescendants).toHaveBeenCalledWith(snapshotOf(4243))
|
||||
})
|
||||
|
||||
it('refreshes the live snapshot at close time so late descendants are included', async () => {
|
||||
const child = mockChild()
|
||||
const first = snapshotOf(4243)
|
||||
const second = {
|
||||
...first,
|
||||
descendants: [...first.descendants, { ...first.descendants[0], pid: 4244 }]
|
||||
}
|
||||
const captureDescendants = vi.fn().mockResolvedValueOnce(first).mockResolvedValueOnce(second)
|
||||
const terminateDescendants = vi.fn(async () => 'exited' as const)
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'linux',
|
||||
captureDescendants,
|
||||
terminateDescendants
|
||||
})
|
||||
|
||||
await tree.capture()
|
||||
await tree.refresh?.()
|
||||
await tree.reap()
|
||||
|
||||
expect(captureDescendants).toHaveBeenCalledTimes(2)
|
||||
expect(terminateDescendants).toHaveBeenCalledWith(second)
|
||||
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
|
||||
})
|
||||
|
||||
it('keeps the original capture boundary for retained POSIX rows', async () => {
|
||||
const child = mockChild()
|
||||
const first = {
|
||||
...snapshotOf(4243),
|
||||
capturedAtMs: 1_700_000_000_900
|
||||
}
|
||||
const refreshed = {
|
||||
...first,
|
||||
capturedAtMs: 1_700_000_002_100,
|
||||
descendants: [
|
||||
...first.descendants,
|
||||
{
|
||||
pid: 4244,
|
||||
ppid: 424242,
|
||||
pgid: 1,
|
||||
startedAt: 'Tue Jan 2 00:00:00 2026'
|
||||
}
|
||||
]
|
||||
}
|
||||
const captureDescendants = vi.fn().mockResolvedValueOnce(first).mockResolvedValueOnce(refreshed)
|
||||
const terminateDescendants = vi.fn(async () => 'exited' as const)
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'linux',
|
||||
captureDescendants,
|
||||
terminateDescendants
|
||||
})
|
||||
|
||||
await tree.capture()
|
||||
await tree.refresh?.()
|
||||
await tree.reap()
|
||||
|
||||
expect(terminateDescendants).toHaveBeenCalledWith({
|
||||
...refreshed,
|
||||
// The retained 4243 row was first observed in the earlier displayed
|
||||
// second. Its per-row boundary must not advance with the refresh.
|
||||
capturedAtMsByPid: {
|
||||
'4243': first.capturedAtMs,
|
||||
'4244': refreshed.capturedAtMs
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('fails closed when a POSIX refresh reuses a PID with a new identity', async () => {
|
||||
const child = mockChild()
|
||||
const first = snapshotOf(4243)
|
||||
const replacement = {
|
||||
...first,
|
||||
descendants: [
|
||||
{
|
||||
...first.descendants[0],
|
||||
pgid: 9,
|
||||
startedAt: 'Tue Jan 2 00:00:00 2026'
|
||||
}
|
||||
]
|
||||
}
|
||||
const captureDescendants = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(first)
|
||||
.mockResolvedValueOnce(replacement)
|
||||
const terminateDescendants = vi.fn(async () => 'exited' as const)
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'linux',
|
||||
captureDescendants,
|
||||
terminateDescendants
|
||||
})
|
||||
|
||||
await tree.capture()
|
||||
await tree.refresh?.()
|
||||
|
||||
await expect(tree.reap()).resolves.toBe('unverifiable')
|
||||
// The descendant evidence is discarded; the root's identity never was in doubt.
|
||||
expect(terminateDescendants).not.toHaveBeenCalled()
|
||||
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
|
||||
})
|
||||
|
||||
it('fails closed when a Windows refresh reuses a PID with a new creation time', async () => {
|
||||
const child = mockChild()
|
||||
const first = windowsSnapshotOf(4243)
|
||||
const replacement = {
|
||||
...first,
|
||||
descendants: [{ pid: 4243, creationTimeMs: first.descendants[0].creationTimeMs + 1 }]
|
||||
}
|
||||
const captureWindowsDescendants = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(first)
|
||||
.mockResolvedValueOnce(replacement)
|
||||
const terminateWindowsTree = vi.fn(async () => {})
|
||||
const terminateWindowsDescendants = vi.fn(async () => 'exited' as const)
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'win32',
|
||||
captureWindowsDescendants,
|
||||
terminateWindowsTree,
|
||||
terminateWindowsDescendants
|
||||
})
|
||||
|
||||
await tree.capture()
|
||||
await tree.refresh?.()
|
||||
|
||||
await expect(tree.reap()).resolves.toBe('unverifiable')
|
||||
expect(terminateWindowsTree).not.toHaveBeenCalled()
|
||||
expect(terminateWindowsDescendants).not.toHaveBeenCalled()
|
||||
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
|
||||
})
|
||||
|
||||
it('queues a fresh boundary behind an output-triggered capture already in flight', async () => {
|
||||
const child = mockChild()
|
||||
const firstDone = Promise.withResolvers<void>()
|
||||
const first = snapshotOf(4243)
|
||||
const second = {
|
||||
...first,
|
||||
descendants: [...first.descendants, { ...first.descendants[0], pid: 4244 }]
|
||||
}
|
||||
const captureDescendants = vi
|
||||
.fn()
|
||||
.mockImplementationOnce(async () => {
|
||||
await firstDone.promise
|
||||
return first
|
||||
})
|
||||
.mockResolvedValueOnce(second)
|
||||
const terminateDescendants = vi.fn(async () => 'exited' as const)
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'linux',
|
||||
captureDescendants,
|
||||
terminateDescendants
|
||||
})
|
||||
|
||||
const outputCapture = tree.refresh!()
|
||||
await vi.waitFor(() => expect(captureDescendants).toHaveBeenCalledTimes(1))
|
||||
const closeCapture = tree.refresh!()
|
||||
await Promise.resolve()
|
||||
expect(captureDescendants).toHaveBeenCalledTimes(1)
|
||||
|
||||
firstDone.resolve()
|
||||
await closeCapture
|
||||
await tree.reap()
|
||||
|
||||
expect(captureDescendants).toHaveBeenCalledTimes(2)
|
||||
expect(terminateDescendants).toHaveBeenCalledWith(second)
|
||||
await outputCapture
|
||||
})
|
||||
|
||||
it('retains a replacement descendant when the prior identity exited', async () => {
|
||||
const child = mockChild()
|
||||
const first = snapshotOf(4243)
|
||||
const replacement = snapshotOf(4244)
|
||||
const captureDescendants = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(first)
|
||||
.mockResolvedValueOnce(replacement)
|
||||
const terminateDescendants = vi.fn(async (snapshot: DescendantSnapshot) =>
|
||||
snapshot.descendants.some((row) => row.pid === 4244) ? ('live' as const) : ('exited' as const)
|
||||
)
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'linux',
|
||||
captureDescendants,
|
||||
terminateDescendants
|
||||
})
|
||||
|
||||
await tree.capture()
|
||||
await tree.refresh?.()
|
||||
await expect(tree.reap()).resolves.toBe('live')
|
||||
|
||||
expect(terminateDescendants).toHaveBeenCalledWith({
|
||||
...replacement,
|
||||
descendants: [...first.descendants, ...replacement.descendants]
|
||||
})
|
||||
})
|
||||
|
||||
it('retains a Windows replacement descendant while preserving unidentified rows', async () => {
|
||||
const child = mockChild()
|
||||
const first = windowsSnapshotOf(4243)
|
||||
const replacement = {
|
||||
...windowsSnapshotOf(4244),
|
||||
unidentifiedCount: 0
|
||||
}
|
||||
const captureWindowsDescendants = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({ ...first, unidentifiedCount: 1 })
|
||||
.mockResolvedValueOnce(replacement)
|
||||
const terminateWindowsTree = vi.fn(async () => {})
|
||||
const terminateWindowsDescendants = vi.fn(async (snapshot: WindowsDescendantSnapshot) =>
|
||||
snapshot.descendants.some((row) => row.pid === 4244) ? ('live' as const) : ('exited' as const)
|
||||
)
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'win32',
|
||||
captureWindowsDescendants,
|
||||
terminateWindowsTree,
|
||||
terminateWindowsDescendants
|
||||
})
|
||||
|
||||
await tree.capture()
|
||||
await tree.refresh?.()
|
||||
await expect(tree.reap()).resolves.toBe('live')
|
||||
|
||||
expect(terminateWindowsDescendants).toHaveBeenCalledWith({
|
||||
...replacement,
|
||||
descendants: [...first.descendants, ...replacement.descendants],
|
||||
unidentifiedCount: 1
|
||||
})
|
||||
})
|
||||
|
||||
it('retains the prior identity-safe snapshot when a refresh is partial', async () => {
|
||||
const child = mockChild()
|
||||
const first = {
|
||||
...snapshotOf(4243),
|
||||
descendants: [
|
||||
...snapshotOf(4243).descendants,
|
||||
{ ...snapshotOf(4243).descendants[0], pid: 4244 }
|
||||
]
|
||||
}
|
||||
const captureDescendants = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(first)
|
||||
.mockResolvedValueOnce({
|
||||
...first,
|
||||
descendants: first.descendants.slice(0, 1)
|
||||
})
|
||||
const terminateDescendants = vi.fn(async () => 'exited' as const)
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'linux',
|
||||
captureDescendants,
|
||||
terminateDescendants
|
||||
})
|
||||
|
||||
await tree.capture()
|
||||
await tree.refresh?.()
|
||||
await tree.reap()
|
||||
|
||||
expect(terminateDescendants).toHaveBeenCalledWith(first)
|
||||
})
|
||||
|
||||
it('stops re-walking once the root is gone, however the table behaved', async () => {
|
||||
const child = mockChild()
|
||||
let exited = false
|
||||
const captureDescendants = vi.fn(async () => null)
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'linux',
|
||||
exited: () => exited,
|
||||
captureDescendants,
|
||||
terminateDescendants: vi.fn()
|
||||
})
|
||||
|
||||
await expect(tree.reap()).resolves.toBe('unverifiable')
|
||||
// An unreadable table costs the snapshot, never the kill on the live root.
|
||||
expect(child.kill).toHaveBeenCalledTimes(1)
|
||||
exited = true
|
||||
await expect(tree.reap()).resolves.toBe('unverifiable')
|
||||
expect(captureDescendants).toHaveBeenCalledTimes(1)
|
||||
// The second attempt observes a dead root: Node has dropped the handle, so
|
||||
// there is nothing left to signal and no recycled pid to reach.
|
||||
expect(child.kill).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('discards a walk that found no root instead of proving an empty tree', async () => {
|
||||
const child = mockChild()
|
||||
const captureDescendants = vi.fn(async () => ({
|
||||
rootPgid: null,
|
||||
descendants: [],
|
||||
capturedAtMs: 1
|
||||
}))
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'linux',
|
||||
captureDescendants,
|
||||
terminateDescendants: vi.fn()
|
||||
})
|
||||
|
||||
await tree.capture()
|
||||
await expect(tree.reap()).resolves.toBe('unverifiable')
|
||||
// A vacuous walk remains retryable while the root is live; no empty-tree
|
||||
// verdict is latched from a missing root row.
|
||||
expect(captureDescendants).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('discards a walk that raced the root exit instead of proving an empty tree', async () => {
|
||||
const child = mockChild()
|
||||
let exited = false
|
||||
const captureDescendants = vi.fn(async () => {
|
||||
exited = true
|
||||
return { rootPgid: 1, descendants: [], capturedAtMs: 1 }
|
||||
})
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'linux',
|
||||
exited: () => exited,
|
||||
captureDescendants,
|
||||
terminateDescendants: vi.fn()
|
||||
})
|
||||
|
||||
await tree.capture()
|
||||
await expect(tree.reap()).resolves.toBe('unverifiable')
|
||||
})
|
||||
|
||||
it('proves a childless snapshot without signalling anything', async () => {
|
||||
const child = mockChild()
|
||||
const terminateDescendants = vi.fn()
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'linux',
|
||||
captureDescendants: vi.fn(async () => ({
|
||||
root: { pid: 424242, startedAt: 'Mon Jan 1 00:00:00 2026' },
|
||||
rootPgid: 1,
|
||||
descendants: [],
|
||||
capturedAtMs: 1
|
||||
})),
|
||||
terminateDescendants
|
||||
})
|
||||
|
||||
await expect(tree.reap()).resolves.toBe('exited')
|
||||
expect(terminateDescendants).not.toHaveBeenCalled()
|
||||
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
|
||||
})
|
||||
|
||||
it('waits for the Windows tree kill before releasing the root', async () => {
|
||||
const child = mockChild()
|
||||
const release = Promise.withResolvers<void>()
|
||||
const terminateWindowsTree = vi.fn(() => release.promise)
|
||||
const captureDescendants = vi.fn()
|
||||
const terminateWindowsDescendants = vi.fn(async () => 'exited' as const)
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'win32',
|
||||
captureDescendants,
|
||||
captureWindowsDescendants: vi.fn(async () => windowsSnapshotOf(4243)),
|
||||
terminateWindowsTree,
|
||||
terminateWindowsDescendants
|
||||
})
|
||||
|
||||
const reap = tree.reap()
|
||||
await vi.waitFor(() =>
|
||||
expect(terminateWindowsTree).toHaveBeenCalledWith({
|
||||
pid: 424242,
|
||||
creationTimeMs: 1_700_000_000_001
|
||||
})
|
||||
)
|
||||
expect(child.kill).not.toHaveBeenCalled()
|
||||
expect(terminateWindowsDescendants).not.toHaveBeenCalled()
|
||||
release.resolve()
|
||||
await expect(reap).resolves.toBe('exited')
|
||||
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
|
||||
expect(terminateWindowsDescendants).toHaveBeenCalledWith(windowsSnapshotOf(4243))
|
||||
expect(captureDescendants).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('stays unproven on Windows when taskkill fails and a descendant is still observed', async () => {
|
||||
const child = mockChild()
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'win32',
|
||||
captureWindowsDescendants: vi.fn(async () => windowsSnapshotOf(4243)),
|
||||
terminateWindowsTree: vi.fn(async () => {
|
||||
throw new Error('taskkill: access denied')
|
||||
}),
|
||||
terminateWindowsDescendants: vi.fn(async () => 'live' as const)
|
||||
})
|
||||
|
||||
// taskkill's own outcome is not the proof; the table read after it is.
|
||||
await expect(tree.reap()).resolves.toBe('live')
|
||||
expect(tree.treeVerdict).toBe('live')
|
||||
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
|
||||
})
|
||||
|
||||
it('stays unproven on Windows when taskkill resolves but a descendant survives it', async () => {
|
||||
const child = mockChild()
|
||||
const terminateWindowsTree = vi.fn(async () => {})
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'win32',
|
||||
captureWindowsDescendants: vi.fn(async () => windowsSnapshotOf(4243)),
|
||||
terminateWindowsTree,
|
||||
terminateWindowsDescendants: vi.fn(async () => 'live' as const)
|
||||
})
|
||||
|
||||
await expect(tree.reap()).resolves.toBe('live')
|
||||
expect(terminateWindowsTree).toHaveBeenCalledTimes(1)
|
||||
expect(tree.treeVerdict).toBe('live')
|
||||
})
|
||||
|
||||
it('never taskkills a Windows root that already exited, but still verifies its snapshot', async () => {
|
||||
const child = mockChild()
|
||||
let exited = false
|
||||
const terminateWindowsTree = vi.fn(async () => {})
|
||||
const terminateWindowsDescendants = vi.fn(async () => 'exited' as const)
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'win32',
|
||||
exited: () => exited,
|
||||
captureWindowsDescendants: vi.fn(async () => windowsSnapshotOf(4243)),
|
||||
terminateWindowsTree,
|
||||
terminateWindowsDescendants
|
||||
})
|
||||
|
||||
await tree.capture()
|
||||
exited = true
|
||||
await expect(tree.reap()).resolves.toBe('exited')
|
||||
// A dead root's pid may already belong to a stranger: taskkill /T /F on it
|
||||
// would take down an unrelated tree.
|
||||
expect(terminateWindowsTree).not.toHaveBeenCalled()
|
||||
expect(terminateWindowsDescendants).toHaveBeenCalledWith(windowsSnapshotOf(4243))
|
||||
})
|
||||
|
||||
it('treats an unreadable Windows table as unproven', async () => {
|
||||
const child = mockChild()
|
||||
const terminateWindowsDescendants = vi.fn()
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'win32',
|
||||
captureWindowsDescendants: vi.fn(async () => null),
|
||||
terminateWindowsTree: vi.fn(async () => {}),
|
||||
terminateWindowsDescendants
|
||||
})
|
||||
|
||||
await expect(tree.reap()).resolves.toBe('unverifiable')
|
||||
expect(terminateWindowsDescendants).not.toHaveBeenCalled()
|
||||
// A host that cannot supply creation times blocks taskkill, not the root kill.
|
||||
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
|
||||
})
|
||||
|
||||
it('has nothing to reap for a child that never spawned', async () => {
|
||||
const child = mockChild(null)
|
||||
const captureDescendants = vi.fn()
|
||||
const tree = createClaudeChildTreeReaper(child, { platform: 'linux', captureDescendants })
|
||||
|
||||
await expect(tree.reap()).resolves.toBe('exited')
|
||||
expect(captureDescendants).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,366 @@
|
||||
import type { SpawnedProcess } from '../../shared/child-process/run-process'
|
||||
import {
|
||||
terminateDescendantSnapshotWithVerdict,
|
||||
type DescendantTreeVerdict
|
||||
} from '../pty-descendant-exit-verification'
|
||||
import {
|
||||
captureDescendantSnapshot,
|
||||
type DescendantSnapshot,
|
||||
type PosixProcessIdentity
|
||||
} from '../pty-descendant-termination'
|
||||
import {
|
||||
captureWindowsDescendantSnapshot,
|
||||
terminateIdentifiedWindowsProcessTree,
|
||||
verifyWindowsDescendantSnapshotExit,
|
||||
verifyWindowsProcessIdentity,
|
||||
type WindowsDescendantSnapshot,
|
||||
type WindowsProcessIdentity
|
||||
} from '../windows-descendant-exit-verification'
|
||||
import { mergeClaudeCapturedTrees, type ClaudeCapturedTree } from './claude-child-tree-snapshot'
|
||||
import { terminateClaudeRoot, terminateClaudeWindowsRoot } from './claude-child-root-termination'
|
||||
import {
|
||||
proveClaudeChildExitWithReaper,
|
||||
type ClaudeChildExitProofInput
|
||||
} from './claude-child-exit-proof-ladder'
|
||||
|
||||
/**
|
||||
* A later reap may only raise the latched verdict. An observed exit is final, and
|
||||
* a descendant seen alive at a deadline is never forgotten by a later look that
|
||||
* could not read the table: the lease gate discriminates on exactly that pair.
|
||||
*/
|
||||
const TREE_VERDICT_TRUST: Record<DescendantTreeVerdict, number> = {
|
||||
unverifiable: 0,
|
||||
live: 1,
|
||||
exited: 2
|
||||
}
|
||||
|
||||
type ReapableChild = Pick<SpawnedProcess, 'pid' | 'kill'>
|
||||
|
||||
/**
|
||||
* A walk is only admissible while the root it walked was alive. A POSIX walk
|
||||
* that found no root says so with a null pgid; either platform's walk can also
|
||||
* have raced the root's death. Both can only have missed descendants that
|
||||
* already reparented away, so neither is evidence about the tree.
|
||||
*/
|
||||
function admissibleTree(
|
||||
captured: DescendantSnapshot | WindowsDescendantSnapshot | null,
|
||||
platform: NodeJS.Platform,
|
||||
exited: boolean
|
||||
): ClaudeCapturedTree | null {
|
||||
if (!captured || exited) {
|
||||
return null
|
||||
}
|
||||
if (platform === 'win32') {
|
||||
return { platform: 'win32', tree: captured as WindowsDescendantSnapshot }
|
||||
}
|
||||
const tree = captured as DescendantSnapshot
|
||||
return tree.rootPgid === null ? null : { platform: 'posix', tree }
|
||||
}
|
||||
|
||||
export type ClaudeChildTreeReaperDeps = {
|
||||
platform?: NodeJS.Platform
|
||||
/** Whether the root's exit has been observed; only a live root can be walked. */
|
||||
exited?: () => boolean
|
||||
captureDescendants?: (rootPid: number) => Promise<DescendantSnapshot | null>
|
||||
terminateDescendants?: (snapshot: DescendantSnapshot) => Promise<DescendantTreeVerdict>
|
||||
terminateWindowsTree?: (root: WindowsProcessIdentity) => Promise<void>
|
||||
captureWindowsDescendants?: (rootPid: number) => Promise<WindowsDescendantSnapshot | null>
|
||||
terminateWindowsDescendants?: (
|
||||
snapshot: WindowsDescendantSnapshot
|
||||
) => Promise<DescendantTreeVerdict>
|
||||
/** Identity probe for the bare-pid tree kill; only Windows has one to gate. */
|
||||
verifyRootIdentity?: (root: PosixProcessIdentity | WindowsProcessIdentity) => Promise<boolean>
|
||||
}
|
||||
|
||||
export type ClaudeChildTreeReaper = {
|
||||
/**
|
||||
* Snapshot the root's live descendants. The moment the root dies they reparent
|
||||
* and no table walk can find them again, so this has to run before anything
|
||||
* gives the root a reason to leave. Held once; later calls are no-ops.
|
||||
*/
|
||||
capture(): Promise<void>
|
||||
/** Refresh a live root's snapshot at the close boundary; a failed refresh keeps the prior proof. */
|
||||
refresh?: () => Promise<void>
|
||||
/**
|
||||
* Kill the child's whole tree and report what the bounded verification
|
||||
* observed. Concurrent calls share one reap, and a later call re-verifies the
|
||||
* same snapshot rather than trusting a root that has since died on its own.
|
||||
*/
|
||||
reap(): Promise<DescendantTreeVerdict>
|
||||
/**
|
||||
* `unverifiable` until a reap observes otherwise. `exited` is the only verdict
|
||||
* that lets a close release the lease; `live` names a descendant that was seen
|
||||
* still running, which no later caller may collapse into "unknown".
|
||||
*/
|
||||
readonly treeVerdict: DescendantTreeVerdict
|
||||
}
|
||||
|
||||
/**
|
||||
* The same shared primitives the Codex structured provider composes: a raw
|
||||
* pipe child owns no PTY job, so there is nothing for the PTY job sweep to
|
||||
* terminate on Windows and no unref'd timer is allowed to outlive the proof.
|
||||
*
|
||||
* The proof is unproven by default. `treeVerdict` is assigned in exactly one
|
||||
* place, from the verdict of `judgeTree`, so a code path that never reaches a
|
||||
* verification cannot report the tree gone by omission.
|
||||
*/
|
||||
export function createClaudeChildTreeReaper(
|
||||
child: ReapableChild,
|
||||
deps: ClaudeChildTreeReaperDeps = {}
|
||||
): ClaudeChildTreeReaper {
|
||||
const platform = deps.platform ?? process.platform
|
||||
const exited = deps.exited ?? (() => false)
|
||||
// Undefined until captured; null when no admissible snapshot exists — the root
|
||||
// was already gone, or the table could not be read while it was alive — which
|
||||
// no later read can make up for.
|
||||
let snapshot: ClaudeCapturedTree | null | undefined
|
||||
let capturing: Promise<void> | null = null
|
||||
let refreshing: Promise<void> | null = null
|
||||
let queuedRefresh: Promise<void> | null = null
|
||||
let inFlight: Promise<DescendantTreeVerdict> | null = null
|
||||
let treeVerdict: DescendantTreeVerdict = 'unverifiable'
|
||||
|
||||
// Consulted only on win32: POSIX signals descendants by revalidated identity
|
||||
// and reaches the root solely through Node's handle, so neither needs a probe.
|
||||
const verifyRoot =
|
||||
deps.verifyRootIdentity ??
|
||||
((root: PosixProcessIdentity | WindowsProcessIdentity) =>
|
||||
verifyWindowsProcessIdentity(root as WindowsProcessIdentity))
|
||||
|
||||
function captureOnce(): Promise<void> {
|
||||
if (refreshing) {
|
||||
const pending = refreshing
|
||||
return pending.then(() => queuedRefresh ?? undefined)
|
||||
}
|
||||
if (snapshot !== undefined) {
|
||||
return Promise.resolve()
|
||||
}
|
||||
if (capturing) {
|
||||
const pending = capturing
|
||||
return pending.then(() => queuedRefresh ?? undefined)
|
||||
}
|
||||
const rootPid = child.pid
|
||||
if (!rootPid || exited()) {
|
||||
// Only the root's death makes a missing snapshot final: its descendants
|
||||
// have reparented, and no later walk can reach them.
|
||||
snapshot = exited() ? null : snapshot
|
||||
return Promise.resolve()
|
||||
}
|
||||
const capture =
|
||||
platform === 'win32'
|
||||
? (deps.captureWindowsDescendants ?? captureWindowsDescendantSnapshot)
|
||||
: (deps.captureDescendants ?? captureDescendantSnapshot)
|
||||
capturing = capture(rootPid)
|
||||
.catch(() => null)
|
||||
.then((captured) => {
|
||||
// A walk that found no root, or that raced the root's death, can only
|
||||
// have missed descendants that already reparented away. A table that
|
||||
// could not be read in time is not an answer at all: while the root
|
||||
// still lives the walk is simply retried, rather than latching a failed
|
||||
// read as proof that there was nothing to find.
|
||||
const rootExited = exited()
|
||||
const tree = admissibleTree(captured, platform, rootExited)
|
||||
if (tree) {
|
||||
snapshot = tree
|
||||
} else if (rootExited) {
|
||||
// Once the root has exited its descendants may have reparented; no
|
||||
// later table read can make an absent snapshot safe to signal.
|
||||
snapshot = null
|
||||
} else {
|
||||
// A failed read or a walk that did not observe the live root is
|
||||
// retryable while the root remains alive. Never latch a vacuous null.
|
||||
snapshot = undefined
|
||||
}
|
||||
})
|
||||
.finally(() => {
|
||||
capturing = null
|
||||
})
|
||||
return capturing
|
||||
}
|
||||
|
||||
function startRefresh(): Promise<void> {
|
||||
if (exited()) {
|
||||
return Promise.resolve()
|
||||
}
|
||||
const rootPid = child.pid
|
||||
if (!rootPid) {
|
||||
return Promise.resolve()
|
||||
}
|
||||
const capture =
|
||||
platform === 'win32'
|
||||
? (deps.captureWindowsDescendants ?? captureWindowsDescendantSnapshot)
|
||||
: (deps.captureDescendants ?? captureDescendantSnapshot)
|
||||
const operation = (async () => {
|
||||
const captured = await capture(rootPid).catch(() => null)
|
||||
if (exited()) {
|
||||
return
|
||||
}
|
||||
const tree = admissibleTree(captured, platform, false)
|
||||
if (!tree) {
|
||||
return
|
||||
}
|
||||
if (snapshot === undefined) {
|
||||
snapshot = tree
|
||||
return
|
||||
}
|
||||
if (snapshot !== null) {
|
||||
// A merge that returns null saw a same-PID identity change: a
|
||||
// recycle/replace decision, not an absent descendant, so no row here may
|
||||
// be signalled from its number. Only the descendant evidence is lost —
|
||||
// the root still leaves through the handle no recycled pid can reach.
|
||||
snapshot = mergeClaudeCapturedTrees(snapshot, tree)
|
||||
}
|
||||
// Keep an earlier admissible snapshot when this close-boundary read fails;
|
||||
// it remains the only identity-safe evidence after root exit.
|
||||
})()
|
||||
refreshing = operation
|
||||
const clearRefreshing = (): void => {
|
||||
if (refreshing === operation) {
|
||||
refreshing = null
|
||||
}
|
||||
}
|
||||
void operation.then(clearRefreshing, clearRefreshing)
|
||||
return operation
|
||||
}
|
||||
|
||||
function queueRefreshAfter(pending: Promise<void>): Promise<void> {
|
||||
if (queuedRefresh) {
|
||||
return queuedRefresh
|
||||
}
|
||||
const operation = pending.then(() => {
|
||||
if (exited()) {
|
||||
return
|
||||
}
|
||||
return startRefresh()
|
||||
})
|
||||
queuedRefresh = operation
|
||||
const clearQueuedRefresh = (): void => {
|
||||
if (queuedRefresh === operation) {
|
||||
queuedRefresh = null
|
||||
}
|
||||
}
|
||||
void operation.then(clearQueuedRefresh, clearQueuedRefresh)
|
||||
return operation
|
||||
}
|
||||
|
||||
async function refresh(): Promise<void> {
|
||||
const pending = capturing ?? refreshing
|
||||
if (pending) {
|
||||
await queueRefreshAfter(pending)
|
||||
return
|
||||
}
|
||||
if (queuedRefresh) {
|
||||
await queuedRefresh
|
||||
return
|
||||
}
|
||||
try {
|
||||
await startRefresh()
|
||||
} catch {
|
||||
// A refresh is advisory; capture failures leave the prior proof intact.
|
||||
}
|
||||
}
|
||||
|
||||
/** The only source of a tree verdict: every `exited` here is an observation. */
|
||||
async function judgeTree(): Promise<DescendantTreeVerdict> {
|
||||
const killRoot = (): boolean => terminateClaudeRoot({ child, exited })
|
||||
const rootPid = child.pid
|
||||
if (!rootPid) {
|
||||
// Never spawned, so the OS never created a tree to orphan.
|
||||
return 'exited'
|
||||
}
|
||||
await captureOnce()
|
||||
if (platform === 'win32') {
|
||||
// Why taskkill's own outcome is never the verdict: it resolves identically
|
||||
// on a timeout, an access denial, a recycled root and a real kill.
|
||||
const { rootVerified } = await terminateClaudeWindowsRoot({
|
||||
snapshot: snapshot?.platform === 'win32' ? snapshot.tree : null,
|
||||
exited,
|
||||
verifyRoot: (root) => verifyRoot(root),
|
||||
terminateTree: (root) =>
|
||||
deps.terminateWindowsTree
|
||||
? deps.terminateWindowsTree(root)
|
||||
: terminateIdentifiedWindowsProcessTree(root, {
|
||||
ownsRoot: () => !exited()
|
||||
}).then(() => undefined),
|
||||
killRoot
|
||||
})
|
||||
if (!rootVerified && !exited()) {
|
||||
return 'unverifiable'
|
||||
}
|
||||
return snapshot?.platform === 'win32'
|
||||
? await (deps.terminateWindowsDescendants ?? verifyWindowsDescendantSnapshotExit)(
|
||||
snapshot.tree
|
||||
)
|
||||
: 'unverifiable'
|
||||
}
|
||||
if (snapshot?.platform !== 'posix') {
|
||||
killRoot()
|
||||
return 'unverifiable'
|
||||
}
|
||||
if (snapshot.tree.descendants.length === 0) {
|
||||
// Read while the root was alive and childless: a later table read has no
|
||||
// row it could match, so it would add nothing to this observation.
|
||||
killRoot()
|
||||
return 'exited'
|
||||
}
|
||||
// Why the root is killed while verification is already running, and never
|
||||
// SIGSTOPped first the way the Codex non-group path does: measured on macOS, a
|
||||
// killed child of a stopped parent stays a zombie row in ps with its lstart
|
||||
// and pgid intact, so verification cannot pass until the root is dead. The
|
||||
// descendants are signalled by the verifier as soon as it revalidates their
|
||||
// identities; the root's death then reparents any zombies to init, which
|
||||
// reaps them. After a root exit the kill is a no-op: Node drops the handle
|
||||
// on exit and never signals a possibly recycled pid.
|
||||
const verdictPromise = deps.terminateDescendants
|
||||
? deps.terminateDescendants(snapshot.tree)
|
||||
: terminateDescendantSnapshotWithVerdict(snapshot.tree, {
|
||||
requireIdentityBeforeSignal: true
|
||||
})
|
||||
killRoot()
|
||||
// What the verification observed is the verdict: a kill that reports no
|
||||
// signal means the handle was already gone, never that the tree survived.
|
||||
return verdictPromise
|
||||
}
|
||||
|
||||
return {
|
||||
capture: captureOnce,
|
||||
refresh,
|
||||
reap() {
|
||||
if (inFlight) {
|
||||
return inFlight
|
||||
}
|
||||
const attempt = judgeTree()
|
||||
.catch((): DescendantTreeVerdict => 'unverifiable')
|
||||
.then((verdict) => {
|
||||
treeVerdict =
|
||||
TREE_VERDICT_TRUST[verdict] > TREE_VERDICT_TRUST[treeVerdict] ? verdict : treeVerdict
|
||||
return verdict
|
||||
})
|
||||
inFlight = attempt
|
||||
void attempt.finally(() => {
|
||||
if (inFlight === attempt) {
|
||||
inFlight = null
|
||||
}
|
||||
})
|
||||
return attempt
|
||||
},
|
||||
get treeVerdict() {
|
||||
return treeVerdict
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Orca's own shutdown ladder on the child it spawned, kept because the SDK's
|
||||
* close path returns no proof and Orca never releases a lease on an assumed exit.
|
||||
*
|
||||
* Resolves true only after the child actually emitted exit and its snapshotted
|
||||
* descendants were observed gone; false is unproven. A root that left on its
|
||||
* own before a snapshot could be armed stays unproven: its descendants had
|
||||
* already reparented out of reach when the ladder first looked.
|
||||
*/
|
||||
export function proveClaudeChildExit(input: ClaudeChildExitProofInput): Promise<boolean> {
|
||||
return proveClaudeChildExitWithReaper(input, () =>
|
||||
createClaudeChildTreeReaper(input.child, { exited: input.exited })
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
import { existsSync, readFileSync, statSync } from 'node:fs'
|
||||
import { dirname, join, relative, resolve } from 'node:path'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { spawnProcess } from '../../shared/child-process/run-process'
|
||||
|
||||
/**
|
||||
* Keep the agent SDK on the structured-Claude side of the toggle.
|
||||
*
|
||||
* A user who never leaves the terminal/TUI Claude path must not pay for the SDK:
|
||||
* importing it evaluates a package that rewrites
|
||||
* `process.env.NoDefaultCurrentDirectoryInExePath`, changing how Windows resolves
|
||||
* executables for every later subprocess, and a missing or incompatible install
|
||||
* would take normal runtime startup down with it. The ordinary
|
||||
* `OrcaRuntimeService` graph reaches the Claude transport module, so only a
|
||||
* deferred import keeps that boundary — and only a walk of the real import graph
|
||||
* keeps the next static import from quietly restoring it.
|
||||
*/
|
||||
const SDK_PACKAGE = '@anthropic-ai/claude-agent-sdk'
|
||||
const REPO_ROOT = resolve(__dirname, '..', '..', '..')
|
||||
|
||||
/** The Electron main entry: everything the app loads before any session exists. */
|
||||
const ROOT = 'src/main/index.ts'
|
||||
/** Proof the walk goes all the way into the Claude transport rather than stopping short. */
|
||||
const TRANSPORT_MODULE = 'src/main/claude/claude-stream-json-connection.ts'
|
||||
|
||||
/**
|
||||
* Static, value-carrying specifiers only, read statement by statement so a
|
||||
* multi-line `import { ... } from '...'` counts. `import type` is erased before
|
||||
* the module ever loads and a bare `import(...)` is the deferral this guards, so
|
||||
* neither is an edge the runtime traverses at load time.
|
||||
*/
|
||||
const STATEMENT_START = /^\s*(?:import|export)\b/
|
||||
const TYPE_ONLY = /^\s*(?:import|export)\s+type\b/
|
||||
const FROM_SPECIFIER = /(?:^|\s)from\s*['"]([^'"]+)['"]/
|
||||
const SIDE_EFFECT_IMPORT = /^\s*import\s*['"]([^'"]+)['"]/
|
||||
/** An import statement never spans more lines than its longest specifier list. */
|
||||
const MAX_STATEMENT_LINES = 60
|
||||
|
||||
function readSpecifiers(source: string): string[] {
|
||||
const lines = source.split('\n')
|
||||
const found: string[] = []
|
||||
for (let index = 0; index < lines.length; index += 1) {
|
||||
const first = lines[index] as string
|
||||
if (!STATEMENT_START.test(first) || TYPE_ONLY.test(first)) {
|
||||
continue
|
||||
}
|
||||
const sideEffect = SIDE_EFFECT_IMPORT.exec(first)
|
||||
if (sideEffect) {
|
||||
found.push(sideEffect[1] as string)
|
||||
continue
|
||||
}
|
||||
for (let scan = index; scan < Math.min(lines.length, index + MAX_STATEMENT_LINES); scan += 1) {
|
||||
if (scan > index && STATEMENT_START.test(lines[scan] as string)) {
|
||||
break
|
||||
}
|
||||
const specifier = FROM_SPECIFIER.exec(lines[scan] as string)
|
||||
if (specifier) {
|
||||
found.push(specifier[1] as string)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
/** Resolve a relative specifier the way the bundler does; unresolvable means not a module. */
|
||||
function resolveRelative(fromFile: string, specifier: string): string | null {
|
||||
const base = join(dirname(fromFile), specifier)
|
||||
for (const candidate of [base, `${base}.ts`, `${base}.tsx`, join(base, 'index.ts')]) {
|
||||
if (existsSync(candidate) && statSync(candidate).isFile()) {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
function walkStaticImports(rootFile: string): { visited: Set<string>; sdkImporters: string[] } {
|
||||
const visited = new Set<string>()
|
||||
const sdkImporters: string[] = []
|
||||
const queue = [resolve(REPO_ROOT, rootFile)]
|
||||
while (queue.length > 0) {
|
||||
const file = queue.pop() as string
|
||||
const key = relative(REPO_ROOT, file).split('\\').join('/')
|
||||
if (visited.has(key)) {
|
||||
continue
|
||||
}
|
||||
visited.add(key)
|
||||
for (const specifier of readSpecifiers(readFileSync(file, 'utf8'))) {
|
||||
if (specifier === SDK_PACKAGE || specifier.startsWith(`${SDK_PACKAGE}/`)) {
|
||||
sdkImporters.push(key)
|
||||
continue
|
||||
}
|
||||
if (!specifier.startsWith('.')) {
|
||||
continue
|
||||
}
|
||||
const target = resolveRelative(file, specifier)
|
||||
if (target) {
|
||||
queue.push(target)
|
||||
}
|
||||
}
|
||||
}
|
||||
return { visited, sdkImporters }
|
||||
}
|
||||
|
||||
describe('claude agent SDK import boundary', () => {
|
||||
const walk = walkStaticImports(ROOT)
|
||||
|
||||
it('walks a graph deep enough to reach the Claude transport', () => {
|
||||
// Without this the guard passes for the wrong reason the moment the walk breaks.
|
||||
expect(walk.visited.size).toBeGreaterThan(500)
|
||||
expect([...walk.visited]).toContain(TRANSPORT_MODULE)
|
||||
})
|
||||
|
||||
it('never reaches the SDK through a static import from the main entry', () => {
|
||||
expect(
|
||||
walk.sdkImporters,
|
||||
`${SDK_PACKAGE} must stay behind the structured-Claude boundary. Load it with a deferred import inside the session path instead.`
|
||||
).toEqual([])
|
||||
})
|
||||
|
||||
it('leaves the Windows executable-search environment alone when the runtime loads', async () => {
|
||||
// A vitest file runs in its own fork, so this is a clean process; the ambient
|
||||
// value is cleared first because the developer's own shell may carry one.
|
||||
delete process.env.NoDefaultCurrentDirectoryInExePath
|
||||
await import('../runtime/structured-agent-session-runtime')
|
||||
|
||||
expect(process.env.NoDefaultCurrentDirectoryInExePath).toBeUndefined()
|
||||
})
|
||||
|
||||
it('still lets the SDK set it, so the guard above is not measuring nothing', async () => {
|
||||
// A separate process, not this fork: the assertion has to be about a first
|
||||
// evaluation of the package, which a cached module registry cannot give.
|
||||
const { NoDefaultCurrentDirectoryInExePath: _cleared, ...env } = process.env
|
||||
const probe = spawnProcess({
|
||||
program: process.execPath,
|
||||
args: [
|
||||
'-e',
|
||||
`import(${JSON.stringify(SDK_PACKAGE)}).then(() => console.log(String(process.env.NoDefaultCurrentDirectoryInExePath)))`
|
||||
],
|
||||
cwd: REPO_ROOT,
|
||||
env: env as Record<string, string>,
|
||||
stdio: ['ignore', 'pipe', 'ignore']
|
||||
})
|
||||
const observed = await new Promise<string>((settle) => {
|
||||
let output = ''
|
||||
probe.stdout?.setEncoding('utf8').on('data', (chunk: string) => {
|
||||
output += chunk
|
||||
})
|
||||
probe.once('close', () => settle(output.trim()))
|
||||
})
|
||||
|
||||
expect(observed).toBe('1')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,107 @@
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { PassThrough } from 'node:stream'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { SpawnOptions as SdkSpawnOptions } from '@anthropic-ai/claude-agent-sdk'
|
||||
import { resolveSpawn, type spawnProcess } from '../../shared/child-process/run-process'
|
||||
import type { ProcessSpec } from '../../shared/child-process/process-spec'
|
||||
import { createClaudeCodeProcessSpawn } from './claude-agent-sdk-process-spawn'
|
||||
|
||||
type FakeChild = EventEmitter & {
|
||||
pid: number
|
||||
stdin: PassThrough
|
||||
stdout: PassThrough
|
||||
stderr: PassThrough
|
||||
kill: ReturnType<typeof vi.fn>
|
||||
}
|
||||
|
||||
function fakeSpawn() {
|
||||
const child = new EventEmitter() as FakeChild
|
||||
child.pid = 4321
|
||||
child.stdin = new PassThrough()
|
||||
child.stdout = new PassThrough()
|
||||
child.stderr = new PassThrough()
|
||||
child.kill = vi.fn(() => true)
|
||||
const specs: ProcessSpec[] = []
|
||||
const spawnImpl = ((spec: ProcessSpec) => {
|
||||
specs.push(spec)
|
||||
return child
|
||||
}) as unknown as typeof spawnProcess
|
||||
return { child, spawnImpl, specs }
|
||||
}
|
||||
|
||||
function sdkOptions(overrides: Partial<SdkSpawnOptions> = {}): SdkSpawnOptions {
|
||||
return {
|
||||
command: '/usr/local/bin/claude',
|
||||
args: ['--output-format', 'stream-json'],
|
||||
cwd: '/work/repo',
|
||||
env: { PATH: '/usr/bin', CLAUDE_CONFIG_DIR: '/accounts/one', UNSET: undefined },
|
||||
signal: new AbortController().signal,
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
describe('claude agent SDK process spawn', () => {
|
||||
it('routes the SDK spawn through Orca and retains the pid the lease adjudicates on', () => {
|
||||
const process = fakeSpawn()
|
||||
const spawn = createClaudeCodeProcessSpawn(process.spawnImpl)
|
||||
|
||||
expect(spawn.pid).toBeUndefined()
|
||||
expect(spawn.child).toBeNull()
|
||||
const child = spawn.spawn(sdkOptions())
|
||||
|
||||
expect(child).toBe(process.child)
|
||||
expect(spawn.child).toBe(process.child)
|
||||
expect(spawn.pid).toBe(4321)
|
||||
expect(process.specs[0]).toEqual({
|
||||
program: '/usr/local/bin/claude',
|
||||
args: ['--output-format', 'stream-json'],
|
||||
cwd: '/work/repo',
|
||||
env: { PATH: '/usr/bin', CLAUDE_CONFIG_DIR: '/accounts/one' },
|
||||
stdio: ['pipe', 'pipe', 'pipe']
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps the child out of the SDK abort path so exit proof stays Orca-owned', () => {
|
||||
const process = fakeSpawn()
|
||||
const controller = new AbortController()
|
||||
createClaudeCodeProcessSpawn(process.spawnImpl).spawn(sdkOptions({ signal: controller.signal }))
|
||||
|
||||
// Node's spawn({signal}) kills the child on abort; Orca's ladder must be the
|
||||
// only thing that can end this process, or close() would report an assumed exit.
|
||||
expect(process.specs[0]).not.toHaveProperty('signal')
|
||||
})
|
||||
|
||||
it('drains stderr into a bounded tail so an exit error still carries it', async () => {
|
||||
const process = fakeSpawn()
|
||||
const spawn = createClaudeCodeProcessSpawn(process.spawnImpl)
|
||||
spawn.spawn(sdkOptions())
|
||||
|
||||
process.child.stderr.write('x'.repeat(9000))
|
||||
process.child.stderr.write('claude: not signed in')
|
||||
await new Promise((resolve) => setImmediate(resolve))
|
||||
|
||||
expect(spawn.stderrTail).toMatch(/claude: not signed in$/)
|
||||
expect(spawn.stderrTail.length).toBe(8192)
|
||||
})
|
||||
|
||||
it('hands a Windows .cmd shim to Orca\u2019s argument encoder', () => {
|
||||
const process = fakeSpawn()
|
||||
createClaudeCodeProcessSpawn(process.spawnImpl).spawn(
|
||||
sdkOptions({
|
||||
command: 'C:\\Users\\dev\\AppData\\npm\\claude.cmd',
|
||||
args: ['--setting-sources=user,project,local', '--session-id', 'a b&c']
|
||||
})
|
||||
)
|
||||
|
||||
// The spec the spawner builds is what Orca's Windows branch encodes; the SDK's
|
||||
// own spawn would hand `.cmd` straight to Node and mangle the argument.
|
||||
const resolved = resolveSpawn(process.specs[0] as ProcessSpec, 'win32')
|
||||
expect(resolved.file.toLowerCase()).toContain('cmd.exe')
|
||||
expect(resolved.options.windowsVerbatimArguments).toBe(true)
|
||||
expect(resolved.args).toHaveLength(1)
|
||||
// `/v:off` plus the quoted argument is what keeps `&` from splitting the line.
|
||||
expect(resolved.args[0]).toContain('/v:off')
|
||||
expect(resolved.args[0]).toContain('"a b&c"')
|
||||
expect(resolved.args[0]).toContain('"--setting-sources=user,project,local"')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,69 @@
|
||||
import type { SpawnOptions as ClaudeAgentSdkSpawnOptions } from '@anthropic-ai/claude-agent-sdk'
|
||||
import { spawnProcess } from '../../shared/child-process/run-process'
|
||||
|
||||
/** Derived rather than imported: only src/shared/child-process may name node:child_process. */
|
||||
type ClaudeCodeChild = ReturnType<typeof spawnProcess>
|
||||
|
||||
const STDERR_TAIL_MAX_BYTES = 8192
|
||||
|
||||
export type ClaudeCodeProcessSpawn = {
|
||||
/** Pass as the SDK's `spawnClaudeCodeProcess`; the SDK never learns the pid because it never owns it. */
|
||||
spawn: (options: ClaudeAgentSdkSpawnOptions) => ClaudeCodeChild
|
||||
/** The retained child, so Orca keeps its own tree-kill and exit-proof ladder. Null until the SDK spawns. */
|
||||
readonly child: ClaudeCodeChild | null
|
||||
/** Ownership proof: the durable lease adjudicates on this pid plus start time plus the spawn token. */
|
||||
readonly pid: number | undefined
|
||||
readonly stderrTail: string
|
||||
}
|
||||
|
||||
function definedEnv(env: Record<string, string | undefined>): Record<string, string> {
|
||||
const next: Record<string, string> = {}
|
||||
for (const [key, value] of Object.entries(env)) {
|
||||
if (value !== undefined) {
|
||||
next[key] = value
|
||||
}
|
||||
}
|
||||
return next
|
||||
}
|
||||
|
||||
/**
|
||||
* Orca supplies the Claude Code child rather than letting the SDK spawn it.
|
||||
*
|
||||
* Two independent reasons: the SDK's `SpawnedProcess` has no pid, and Orca's
|
||||
* spawner is the only path that encodes `.cmd` arguments safely on Windows.
|
||||
*/
|
||||
export function createClaudeCodeProcessSpawn(
|
||||
spawnImpl: typeof spawnProcess = spawnProcess
|
||||
): ClaudeCodeProcessSpawn {
|
||||
let child: ClaudeCodeChild | null = null
|
||||
let stderrTail = ''
|
||||
return {
|
||||
spawn: (options) => {
|
||||
// Why `options.signal` is dropped: it would let the SDK kill the child outside
|
||||
// Orca's ladder, and close() may never report an exit it did not observe.
|
||||
const spawned = spawnImpl({
|
||||
program: options.command,
|
||||
args: [...options.args],
|
||||
...(options.cwd === undefined ? {} : { cwd: options.cwd }),
|
||||
env: definedEnv(options.env),
|
||||
stdio: ['pipe', 'pipe', 'pipe']
|
||||
})
|
||||
child = spawned
|
||||
// The SDK drains stderr only for its own local spawn, so a custom spawner must:
|
||||
// otherwise the child blocks on a full pipe and exit errors lose their tail.
|
||||
spawned.stderr.setEncoding('utf8').on('data', (chunk: string) => {
|
||||
stderrTail = (stderrTail + chunk).slice(-STDERR_TAIL_MAX_BYTES)
|
||||
})
|
||||
return spawned
|
||||
},
|
||||
get child() {
|
||||
return child
|
||||
},
|
||||
get pid() {
|
||||
return child?.pid
|
||||
},
|
||||
get stderrTail() {
|
||||
return stderrTail
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { PassThrough } from 'node:stream'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { SpawnedProcess } from '../../shared/child-process/run-process'
|
||||
import type { DescendantSnapshot } from '../pty-descendant-termination'
|
||||
import type { WindowsDescendantSnapshot } from '../windows-descendant-exit-verification'
|
||||
import { createClaudeChildTreeReaper } from './claude-agent-sdk-exit-proof'
|
||||
import { mergeClaudeCapturedTrees } from './claude-child-tree-snapshot'
|
||||
|
||||
const ROOT_PID = 424242
|
||||
const ROOT_STARTED_AT = 'Mon Jan 1 00:00:00 2026'
|
||||
const ROOT_FORK_MS = Date.parse(ROOT_STARTED_AT)
|
||||
|
||||
function mockChild(): EventEmitter &
|
||||
Pick<SpawnedProcess, 'pid' | 'kill' | 'stdin'> & { kill: ReturnType<typeof vi.fn> } {
|
||||
return Object.assign(new EventEmitter(), {
|
||||
pid: ROOT_PID,
|
||||
stdin: new PassThrough(),
|
||||
kill: vi.fn(() => true)
|
||||
}) as never
|
||||
}
|
||||
|
||||
function posixSnapshot(input: {
|
||||
capturedAtMs: number
|
||||
descendants?: DescendantSnapshot['descendants']
|
||||
}): DescendantSnapshot {
|
||||
return {
|
||||
root: { pid: ROOT_PID, startedAt: ROOT_STARTED_AT },
|
||||
rootPgid: ROOT_PID,
|
||||
descendants: input.descendants ?? [],
|
||||
capturedAtMs: input.capturedAtMs
|
||||
}
|
||||
}
|
||||
|
||||
function windowsSnapshot(capturedAtMs = 1): WindowsDescendantSnapshot {
|
||||
return {
|
||||
root: { pid: ROOT_PID, creationTimeMs: 1_700_000_000_001 },
|
||||
descendants: [{ pid: 4243, creationTimeMs: 1_700_000_000_000 }],
|
||||
unidentifiedCount: 0,
|
||||
capturedAtMs
|
||||
}
|
||||
}
|
||||
|
||||
describe('Claude root kill fallback', () => {
|
||||
it('kills the root when the first capture landed in the fork second', async () => {
|
||||
// The production POSIX verifier declines a root born in its capture second,
|
||||
// and that verdict must not cost the tree the kill on Node's own handle.
|
||||
const child = mockChild()
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'linux',
|
||||
exited: () => false,
|
||||
captureDescendants: vi.fn(async () => posixSnapshot({ capturedAtMs: ROOT_FORK_MS + 300 }))
|
||||
})
|
||||
|
||||
await expect(tree.reap()).resolves.toBe('exited')
|
||||
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
|
||||
})
|
||||
|
||||
it('kills the root after a recycled descendant pid voided the snapshot', async () => {
|
||||
const child = mockChild()
|
||||
const captureDescendants = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
posixSnapshot({
|
||||
capturedAtMs: ROOT_FORK_MS + 5_000,
|
||||
descendants: [{ pid: 100, ppid: ROOT_PID, pgid: ROOT_PID, startedAt: ROOT_STARTED_AT }]
|
||||
})
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
posixSnapshot({
|
||||
capturedAtMs: ROOT_FORK_MS + 6_000,
|
||||
descendants: [
|
||||
{ pid: 100, ppid: ROOT_PID, pgid: ROOT_PID, startedAt: 'Mon Jan 1 00:00:30 2026' }
|
||||
]
|
||||
})
|
||||
)
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'linux',
|
||||
exited: () => false,
|
||||
captureDescendants,
|
||||
terminateDescendants: vi.fn(async () => 'exited' as const),
|
||||
verifyRootIdentity: vi.fn(async () => true)
|
||||
})
|
||||
|
||||
await tree.capture()
|
||||
await tree.refresh?.()
|
||||
// The descendant evidence is rightly discarded; the root's never was in doubt.
|
||||
await expect(tree.reap()).resolves.toBe('unverifiable')
|
||||
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
|
||||
})
|
||||
|
||||
it('keeps an observed live descendant when the root identity probe declined', async () => {
|
||||
const child = mockChild()
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'linux',
|
||||
exited: () => false,
|
||||
captureDescendants: vi.fn(async () =>
|
||||
posixSnapshot({
|
||||
capturedAtMs: ROOT_FORK_MS + 5_000,
|
||||
descendants: [{ pid: 100, ppid: ROOT_PID, pgid: ROOT_PID, startedAt: ROOT_STARTED_AT }]
|
||||
})
|
||||
),
|
||||
terminateDescendants: vi.fn(async () => 'live' as const),
|
||||
verifyRootIdentity: vi.fn(async () => false)
|
||||
})
|
||||
|
||||
await expect(tree.reap()).resolves.toBe('live')
|
||||
expect(tree.treeVerdict).toBe('live')
|
||||
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
|
||||
})
|
||||
|
||||
it('reports a Windows taskkill that worked as exited, not unverifiable', async () => {
|
||||
const child = mockChild()
|
||||
// Probe 1 gates taskkill; a later probe correctly finds the root already dead.
|
||||
const verifyRootIdentity = vi.fn().mockResolvedValueOnce(true).mockResolvedValue(false)
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'win32',
|
||||
exited: () => false,
|
||||
captureWindowsDescendants: vi.fn(async () => windowsSnapshot()),
|
||||
terminateWindowsTree: vi.fn(async () => {}),
|
||||
terminateWindowsDescendants: vi.fn(async () => 'exited' as const),
|
||||
verifyRootIdentity
|
||||
})
|
||||
|
||||
await expect(tree.reap()).resolves.toBe('exited')
|
||||
})
|
||||
|
||||
it('kills the root when no POSIX snapshot could be read', async () => {
|
||||
const child = mockChild()
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'linux',
|
||||
exited: () => false,
|
||||
captureDescendants: vi.fn(async () => null),
|
||||
terminateDescendants: vi.fn()
|
||||
})
|
||||
|
||||
await expect(tree.reap()).resolves.toBe('unverifiable')
|
||||
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
|
||||
})
|
||||
|
||||
it('kills the root when the Windows process table is unreadable', async () => {
|
||||
const child = mockChild()
|
||||
const terminateWindowsTree = vi.fn(async () => {})
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'win32',
|
||||
exited: () => false,
|
||||
captureWindowsDescendants: vi.fn(async () => null),
|
||||
terminateWindowsTree,
|
||||
terminateWindowsDescendants: vi.fn()
|
||||
})
|
||||
|
||||
await expect(tree.reap()).resolves.toBe('unverifiable')
|
||||
// No identity means no bare-pid tree kill, but the owned handle is still ours.
|
||||
expect(terminateWindowsTree).not.toHaveBeenCalled()
|
||||
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
|
||||
})
|
||||
|
||||
it('never signals a root the reaper already saw exit', async () => {
|
||||
const child = mockChild()
|
||||
const tree = createClaudeChildTreeReaper(child, {
|
||||
platform: 'linux',
|
||||
exited: () => true,
|
||||
captureDescendants: vi.fn(async () => null)
|
||||
})
|
||||
|
||||
await expect(tree.reap()).resolves.toBe('unverifiable')
|
||||
expect(child.kill).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('chains per-pid Windows boundaries across a second merge', async () => {
|
||||
const first = windowsSnapshot(1_000)
|
||||
const second: WindowsDescendantSnapshot = {
|
||||
...windowsSnapshot(2_000),
|
||||
descendants: [
|
||||
{ pid: 4243, creationTimeMs: 1_700_000_000_000 },
|
||||
{ pid: 4244, creationTimeMs: 1_700_000_000_002 }
|
||||
]
|
||||
}
|
||||
const third: WindowsDescendantSnapshot = { ...second, capturedAtMs: 3_000 }
|
||||
|
||||
const merged = mergeClaudeCapturedTrees(
|
||||
{ platform: 'win32', tree: first },
|
||||
{ platform: 'win32', tree: second }
|
||||
)
|
||||
expect(merged?.tree.capturedAtMsByPid).toEqual({ '4243': 1_000, '4244': 2_000 })
|
||||
const rechained = mergeClaudeCapturedTrees(merged!, { platform: 'win32', tree: third })
|
||||
|
||||
expect(rechained?.tree.capturedAtMsByPid).toEqual({ '4243': 1_000, '4244': 2_000 })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,65 @@
|
||||
import type { SDKUserMessage } from '@anthropic-ai/claude-agent-sdk'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { createClaudeUserMessageQueue } from './claude-agent-sdk-user-message-queue'
|
||||
|
||||
/**
|
||||
* The SDK's input pump is `for await (const frame of prompt) { await transport.write(frame) }`.
|
||||
* A rejected write — or an abort — ends that loop abruptly, which calls the
|
||||
* generator's `return()`. Everything below drives that exact shape, because the
|
||||
* frame the pump already pulled is the one nothing else can reach.
|
||||
*/
|
||||
const frame = (text: string): SDKUserMessage =>
|
||||
({
|
||||
type: 'user',
|
||||
message: { role: 'user', content: [{ type: 'text', text }] }
|
||||
}) as unknown as SDKUserMessage
|
||||
|
||||
const settled = (promise: Promise<void>): Promise<'settled' | 'pending'> =>
|
||||
Promise.race([
|
||||
promise.then(
|
||||
() => 'settled' as const,
|
||||
() => 'settled' as const
|
||||
),
|
||||
new Promise<'pending'>((resolve) => setTimeout(() => resolve('pending'), 100))
|
||||
])
|
||||
|
||||
describe('claude user message queue', () => {
|
||||
it('rejects the frame the SDK pulled but abandoned without writing', async () => {
|
||||
const queue = createClaudeUserMessageQueue()
|
||||
const pump = queue.messages[Symbol.asyncIterator]()
|
||||
const sent = queue.push(frame('hello'))
|
||||
|
||||
await pump.next()
|
||||
await pump.return?.(undefined)
|
||||
|
||||
await expect(settled(sent)).resolves.toBe('settled')
|
||||
await expect(sent).rejects.toThrow(
|
||||
'claude stream-json input ended before the frame was written'
|
||||
)
|
||||
})
|
||||
|
||||
it('rejects an in-flight frame from fail() when the SDK never resumes the pump', async () => {
|
||||
const queue = createClaudeUserMessageQueue()
|
||||
const pump = queue.messages[Symbol.asyncIterator]()
|
||||
const sent = queue.push(frame('hello'))
|
||||
|
||||
await pump.next()
|
||||
queue.fail(new Error('claude stream-json exited: child died'))
|
||||
|
||||
await expect(settled(sent)).resolves.toBe('settled')
|
||||
await expect(sent).rejects.toThrow('claude stream-json exited: child died')
|
||||
})
|
||||
|
||||
it('still settles a written frame only once the pump asks for the next one', async () => {
|
||||
const queue = createClaudeUserMessageQueue()
|
||||
const pump = queue.messages[Symbol.asyncIterator]()
|
||||
const sent = queue.push(frame('hello'))
|
||||
|
||||
const pulled = await pump.next()
|
||||
expect(pulled.value).toMatchObject({ type: 'user' })
|
||||
// The write proof is the pump coming back for more, exactly as before.
|
||||
await expect(settled(sent)).resolves.toBe('pending')
|
||||
void pump.next()
|
||||
await expect(sent).resolves.toBeUndefined()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,100 @@
|
||||
import type { SDKUserMessage } from '@anthropic-ai/claude-agent-sdk'
|
||||
|
||||
type QueuedMessage = {
|
||||
message: SDKUserMessage
|
||||
resolve: () => void
|
||||
reject: (error: Error) => void
|
||||
}
|
||||
|
||||
export type ClaudeUserMessageQueue = {
|
||||
/** The SDK's streaming-input prompt; it stays open until `end`. */
|
||||
messages: AsyncIterable<SDKUserMessage>
|
||||
/** Resolves once the SDK has finished writing the frame to the child. */
|
||||
push: (message: SDKUserMessage) => Promise<void>
|
||||
/** Reject every unwritten frame, in-flight included; a caller waiting on a send must not hang past the exit. */
|
||||
fail: (error: Error) => void
|
||||
end: () => void
|
||||
}
|
||||
|
||||
/** The rejection an abandoned frame carries when nothing else has named a cause yet. */
|
||||
const UNWRITTEN_FRAME_MESSAGE = 'claude stream-json input ended before the frame was written'
|
||||
|
||||
export function createClaudeUserMessageQueue(): ClaudeUserMessageQueue {
|
||||
const queued: QueuedMessage[] = []
|
||||
// The frame the SDK has taken but not yet acknowledged. It is out of `queued`,
|
||||
// so it is unreachable from anywhere else and would otherwise never settle.
|
||||
let inFlight: QueuedMessage | null = null
|
||||
let wake: (() => void) | null = null
|
||||
let ended = false
|
||||
let failure: Error | null = null
|
||||
const notify = (): void => {
|
||||
wake?.()
|
||||
wake = null
|
||||
}
|
||||
const rejectInFlight = (error: Error): void => {
|
||||
const abandoned = inFlight
|
||||
inFlight = null
|
||||
abandoned?.reject(error)
|
||||
}
|
||||
|
||||
async function* drain(): AsyncGenerator<SDKUserMessage> {
|
||||
for (;;) {
|
||||
const next = queued.shift()
|
||||
if (next) {
|
||||
inFlight = next
|
||||
let written = false
|
||||
try {
|
||||
yield next.message
|
||||
written = true
|
||||
} finally {
|
||||
// The SDK's input pump abandons this iterator when its
|
||||
// `await transport.write(...)` rejects or the query aborts, and the code
|
||||
// after a `yield` never runs on that path. Settling here is the only
|
||||
// place a frame it already took can be reached.
|
||||
if (written) {
|
||||
inFlight = null
|
||||
// Resumed only after the SDK's `await transport.write(...)` settled, so this
|
||||
// is the same "the frame reached the child" proof the hand-rolled write gave.
|
||||
next.resolve()
|
||||
} else {
|
||||
rejectInFlight(failure ?? new Error(UNWRITTEN_FRAME_MESSAGE))
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if (ended || failure) {
|
||||
return
|
||||
}
|
||||
await new Promise<void>((resolve) => {
|
||||
wake = resolve
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
messages: drain(),
|
||||
push: (message) =>
|
||||
new Promise<void>((resolve, reject) => {
|
||||
if (failure) {
|
||||
reject(failure)
|
||||
return
|
||||
}
|
||||
queued.push({ message, resolve, reject })
|
||||
notify()
|
||||
}),
|
||||
fail: (error) => {
|
||||
failure ??= error
|
||||
for (const entry of queued.splice(0)) {
|
||||
entry.reject(error)
|
||||
}
|
||||
// A pump that never resumes cannot run the generator's cleanup, so the
|
||||
// exit path has to reach the in-flight frame itself.
|
||||
rejectInFlight(error)
|
||||
notify()
|
||||
},
|
||||
end: () => {
|
||||
ended = true
|
||||
notify()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import type { SpawnedProcess } from '../../shared/child-process/run-process'
|
||||
import { waitForProcessExitUntil } from '../codex/codex-process-exit-deadline'
|
||||
import type { ClaudeChildTreeReaper } from './claude-agent-sdk-exit-proof'
|
||||
|
||||
const GRACEFUL_EXIT_MS = 1_500
|
||||
const FORCED_EXIT_MS = 1_000
|
||||
|
||||
export type ClaudeChildExitProofInput = {
|
||||
child: Pick<SpawnedProcess, 'pid' | 'kill' | 'stdin'>
|
||||
exitPromise: Promise<void>
|
||||
exited: () => boolean
|
||||
tree?: ClaudeChildTreeReaper
|
||||
}
|
||||
|
||||
export async function proveClaudeChildExitWithReaper(
|
||||
input: ClaudeChildExitProofInput,
|
||||
createTree: () => ClaudeChildTreeReaper
|
||||
): Promise<boolean> {
|
||||
const tree = input.tree ?? createTree()
|
||||
// Arm before stdin closes: only a live root can identify its descendants.
|
||||
await tree.capture()
|
||||
try {
|
||||
input.child.stdin?.end()
|
||||
} catch {
|
||||
// The reap below still owns the process.
|
||||
}
|
||||
let reaped = false
|
||||
if (!input.exited()) {
|
||||
await waitForProcessExitUntil(input.exitPromise, GRACEFUL_EXIT_MS)
|
||||
if (!input.exited()) {
|
||||
reaped = true
|
||||
await tree.refresh?.()
|
||||
await tree.reap()
|
||||
await waitForProcessExitUntil(input.exitPromise, FORCED_EXIT_MS)
|
||||
}
|
||||
}
|
||||
if (!reaped && input.exited() && tree.treeVerdict !== 'exited') {
|
||||
await tree.reap()
|
||||
}
|
||||
return input.exited() && tree.treeVerdict === 'exited'
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { applyClaudeEnvPatch } from '../claude-accounts/environment'
|
||||
import { buildClaudeChildProcessEnv } from './claude-child-process-environment'
|
||||
|
||||
describe('Claude child process environment', () => {
|
||||
it('strips case-insensitive auth headers through the shared env patch on Windows', () => {
|
||||
expect(
|
||||
applyClaudeEnvPatch(
|
||||
{
|
||||
anthropic_api_key: 'inherited-key',
|
||||
Anthropic_Custom_Headers: 'Authorization: inherited',
|
||||
SAFE_VALUE: 'preserved'
|
||||
},
|
||||
{},
|
||||
{ stripAuthEnv: true, platform: 'win32' }
|
||||
)
|
||||
).toEqual({ SAFE_VALUE: 'preserved' })
|
||||
})
|
||||
|
||||
it('strips case-insensitive inherited auth and session stamps on Windows', () => {
|
||||
const env = buildClaudeChildProcessEnv(
|
||||
{
|
||||
ANTHROPIC_AUTH_TOKEN: 'configured-token',
|
||||
Claude_Code_Session_Id: 'configured-session'
|
||||
},
|
||||
{
|
||||
platform: 'win32',
|
||||
inheritedEnv: {
|
||||
anthropic_api_key: 'inherited-key',
|
||||
Anthropic_Custom_Headers: 'Authorization: inherited',
|
||||
claude_code_child_session: '1',
|
||||
CLAUDE_CODE_SESSION_ID: 'inherited-session',
|
||||
SAFE_VALUE: 'preserved'
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
expect(env).toEqual({
|
||||
ANTHROPIC_AUTH_TOKEN: 'configured-token',
|
||||
Claude_Code_Session_Id: 'configured-session',
|
||||
SAFE_VALUE: 'preserved'
|
||||
})
|
||||
})
|
||||
|
||||
it('can strip child-session stamps reintroduced by a full SDK launch overlay', () => {
|
||||
expect(
|
||||
buildClaudeChildProcessEnv(
|
||||
{
|
||||
CLAUDE_CODE_CHILD_SESSION: 'configured-child-session',
|
||||
CLAUDE_CODE_SESSION_ID: 'configured-session',
|
||||
CLAUDE_CODE_BRIDGE_SESSION_ID: 'configured-bridge-session'
|
||||
},
|
||||
{
|
||||
scrubConfiguredChildSessionStamps: true,
|
||||
inheritedEnv: {
|
||||
CLAUDE_CODE_CHILD_SESSION: 'inherited-child-session',
|
||||
SAFE_VALUE: 'preserved'
|
||||
}
|
||||
}
|
||||
)
|
||||
).toEqual({ SAFE_VALUE: 'preserved' })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,69 @@
|
||||
import { CLAUDE_AUTH_ENV_VARS, applyClaudeEnvPatch } from '../claude-accounts/environment'
|
||||
|
||||
const CLAUDE_CHILD_SESSION_STAMP_ENV_KEYS = [
|
||||
'CLAUDE_CODE_CHILD_SESSION',
|
||||
'CLAUDE_CODE_SESSION_ID',
|
||||
'CLAUDE_CODE_BRIDGE_SESSION_ID'
|
||||
] as const
|
||||
|
||||
function cloneProcessEnv(source: NodeJS.ProcessEnv): Record<string, string> {
|
||||
const env: Record<string, string> = {}
|
||||
for (const [key, value] of Object.entries(source)) {
|
||||
if (value !== undefined) {
|
||||
env[key] = value
|
||||
}
|
||||
}
|
||||
return env
|
||||
}
|
||||
|
||||
function stripClaudeChildSessionStamps(
|
||||
env: Record<string, string>,
|
||||
platform: NodeJS.Platform
|
||||
): Record<string, string> {
|
||||
for (const key of CLAUDE_CHILD_SESSION_STAMP_ENV_KEYS) {
|
||||
for (const envKey of Object.keys(env)) {
|
||||
if (envKey === key || (platform === 'win32' && envKey.toUpperCase() === key)) {
|
||||
delete env[envKey]
|
||||
}
|
||||
}
|
||||
}
|
||||
return env
|
||||
}
|
||||
|
||||
export function buildClaudeChildProcessEnv(
|
||||
configuredEnv: Record<string, string> = {},
|
||||
options: {
|
||||
inheritedEnv?: NodeJS.ProcessEnv
|
||||
platform?: NodeJS.Platform
|
||||
scrubConfiguredChildSessionStamps?: boolean
|
||||
} = {}
|
||||
): Record<string, string> {
|
||||
const inheritedEnv = options.inheritedEnv ?? process.env
|
||||
const platform = options.platform ?? process.platform
|
||||
const env = applyClaudeEnvPatch(
|
||||
cloneProcessEnv(inheritedEnv),
|
||||
{},
|
||||
{
|
||||
stripAuthEnv: true,
|
||||
platform
|
||||
}
|
||||
)
|
||||
if (platform === 'win32') {
|
||||
const authKeys = new Set(CLAUDE_AUTH_ENV_VARS.map((key) => key.toUpperCase()))
|
||||
for (const [key, value] of Object.entries(env)) {
|
||||
const normalized = key.toUpperCase()
|
||||
if (
|
||||
authKeys.has(normalized) ||
|
||||
(normalized === 'ANTHROPIC_CUSTOM_HEADERS' &&
|
||||
/authorization|x-api-key|api-key|bearer/i.test(value))
|
||||
) {
|
||||
delete env[key]
|
||||
}
|
||||
}
|
||||
}
|
||||
if (options.scrubConfiguredChildSessionStamps) {
|
||||
return stripClaudeChildSessionStamps({ ...env, ...configuredEnv }, platform)
|
||||
}
|
||||
stripClaudeChildSessionStamps(env, platform)
|
||||
return { ...env, ...configuredEnv }
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import type { SpawnedProcess } from '../../shared/child-process/run-process'
|
||||
import type { PosixProcessIdentity } from '../pty-descendant-termination'
|
||||
import type {
|
||||
WindowsDescendantSnapshot,
|
||||
WindowsProcessIdentity
|
||||
} from '../windows-descendant-exit-verification'
|
||||
|
||||
export type ClaudeRootIdentity = PosixProcessIdentity | WindowsProcessIdentity
|
||||
|
||||
type RootTerminationInput = {
|
||||
child: Pick<SpawnedProcess, 'kill'>
|
||||
exited: () => boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Kills the root through the handle Node owns rather than through its pid, which
|
||||
* is why no identity probe gates it: libuv drops that handle in the same turn it
|
||||
* reaps, so the signal either reaches the process Orca spawned or reaches
|
||||
* nothing. A probe here could only let an unreadable process table cost the tree
|
||||
* the one fallback that still works once every table read has failed.
|
||||
*
|
||||
* False means no signal was sent, because the root had already left.
|
||||
*/
|
||||
export function terminateClaudeRoot(input: RootTerminationInput): boolean {
|
||||
return input.exited() ? false : input.child.kill('SIGKILL')
|
||||
}
|
||||
|
||||
type WindowsRootTerminationInput = {
|
||||
snapshot: WindowsDescendantSnapshot | null
|
||||
exited: () => boolean
|
||||
verifyRoot: (root: WindowsProcessIdentity) => Promise<boolean>
|
||||
terminateTree: (root: WindowsProcessIdentity) => Promise<void>
|
||||
killRoot: () => boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* `taskkill /T /F` addresses a bare pid, so a dead root's pid may already belong
|
||||
* to a stranger whose whole tree it would take down: that one is identity-gated.
|
||||
* The direct root kill after it runs however the probe decided.
|
||||
*/
|
||||
export async function terminateClaudeWindowsRoot(
|
||||
input: WindowsRootTerminationInput
|
||||
): Promise<{ rootVerified: boolean }> {
|
||||
const { snapshot, exited, verifyRoot, terminateTree, killRoot } = input
|
||||
let rootVerified = false
|
||||
if (!exited() && snapshot) {
|
||||
rootVerified = await verifyRoot(snapshot.root).catch(() => false)
|
||||
if (rootVerified && !exited()) {
|
||||
await terminateTree(snapshot.root).catch(() => {})
|
||||
}
|
||||
}
|
||||
killRoot()
|
||||
return { rootVerified }
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
import type { DescendantSnapshot } from '../pty-descendant-termination'
|
||||
import type { WindowsDescendantSnapshot } from '../windows-descendant-exit-verification'
|
||||
|
||||
/** One platform's descendant tree, tagged so neither verifier can be handed the other's rows. */
|
||||
export type ClaudeCapturedTree =
|
||||
| { platform: 'posix'; tree: DescendantSnapshot }
|
||||
| { platform: 'win32'; tree: WindowsDescendantSnapshot }
|
||||
|
||||
/**
|
||||
* Process-table reads are not atomic: a refresh can omit a still-live row, but
|
||||
* it can also observe a new process after the old row exited. Retain rows absent
|
||||
* from the refresh, but reject a PID whose identity changed between reads.
|
||||
*/
|
||||
function mergeRowsByPid<Row extends { pid: number }>(
|
||||
previous: readonly Row[],
|
||||
next: readonly Row[],
|
||||
sameIdentity: (previous: Row, next: Row) => boolean,
|
||||
previousBoundary: (row: Row) => number,
|
||||
nextBoundary: (row: Row) => number,
|
||||
refreshBoundary: number
|
||||
): { rows: Row[]; capturedAtMsByPid?: Readonly<Record<string, number>> } | null {
|
||||
const merged = new Map<number, Row>()
|
||||
const capturedAtMsByPid: Record<string, number> = {}
|
||||
for (const row of previous) {
|
||||
const prior = merged.get(row.pid)
|
||||
if (prior && !sameIdentity(prior, row)) {
|
||||
return null
|
||||
}
|
||||
merged.set(row.pid, row)
|
||||
capturedAtMsByPid[String(row.pid)] = previousBoundary(row)
|
||||
}
|
||||
for (const row of next) {
|
||||
const prior = merged.get(row.pid)
|
||||
if (prior && !sameIdentity(prior, row)) {
|
||||
return null
|
||||
}
|
||||
if (!prior) {
|
||||
capturedAtMsByPid[String(row.pid)] = nextBoundary(row)
|
||||
}
|
||||
merged.set(row.pid, row)
|
||||
}
|
||||
const boundaries = Object.values(capturedAtMsByPid)
|
||||
const needsBoundaryMap =
|
||||
new Set(boundaries).size > 1 || boundaries.some((boundary) => boundary !== refreshBoundary)
|
||||
return {
|
||||
rows: [...merged.values()],
|
||||
...(needsBoundaryMap ? { capturedAtMsByPid } : {})
|
||||
}
|
||||
}
|
||||
|
||||
export function mergeClaudeCapturedTrees(
|
||||
previous: ClaudeCapturedTree,
|
||||
next: ClaudeCapturedTree
|
||||
): ClaudeCapturedTree | null {
|
||||
if (previous.platform !== next.platform) {
|
||||
return null
|
||||
}
|
||||
if (previous.platform === 'posix' && next.platform === 'posix') {
|
||||
if (previous.tree.rootPgid !== next.tree.rootPgid) {
|
||||
return null
|
||||
}
|
||||
// A refresh cannot repair an earlier capture that lacked root identity;
|
||||
// retaining those rows would permit a later numeric-pid kill without proof.
|
||||
if (!previous.tree.root || !next.tree.root) {
|
||||
return null
|
||||
}
|
||||
if (
|
||||
previous.tree.root.pid !== next.tree.root.pid ||
|
||||
previous.tree.root.startedAt !== next.tree.root.startedAt
|
||||
) {
|
||||
return null
|
||||
}
|
||||
const descendants = mergeRowsByPid(
|
||||
previous.tree.descendants,
|
||||
next.tree.descendants,
|
||||
(left, right) => left.pgid === right.pgid && left.startedAt === right.startedAt,
|
||||
(row) => previous.tree.capturedAtMsByPid?.[String(row.pid)] ?? previous.tree.capturedAtMs,
|
||||
(row) => next.tree.capturedAtMsByPid?.[String(row.pid)] ?? next.tree.capturedAtMs,
|
||||
next.tree.capturedAtMs
|
||||
)
|
||||
if (!descendants) {
|
||||
return null
|
||||
}
|
||||
return {
|
||||
platform: 'posix',
|
||||
tree: {
|
||||
...next.tree,
|
||||
// Retained rows keep their earlier boundary; new rows use the refresh
|
||||
// boundary. The scalar remains the latest scan for legacy consumers.
|
||||
descendants: descendants.rows,
|
||||
...(descendants.capturedAtMsByPid
|
||||
? { capturedAtMsByPid: descendants.capturedAtMsByPid }
|
||||
: {})
|
||||
}
|
||||
}
|
||||
}
|
||||
if (previous.platform === 'win32' && next.platform === 'win32') {
|
||||
if (
|
||||
previous.tree.root.pid !== next.tree.root.pid ||
|
||||
previous.tree.root.creationTimeMs !== next.tree.root.creationTimeMs
|
||||
) {
|
||||
return null
|
||||
}
|
||||
const descendants = mergeRowsByPid(
|
||||
previous.tree.descendants,
|
||||
next.tree.descendants,
|
||||
(left, right) => left.creationTimeMs === right.creationTimeMs,
|
||||
(row) => previous.tree.capturedAtMsByPid?.[String(row.pid)] ?? previous.tree.capturedAtMs,
|
||||
(row) => next.tree.capturedAtMsByPid?.[String(row.pid)] ?? next.tree.capturedAtMs,
|
||||
next.tree.capturedAtMs
|
||||
)
|
||||
if (!descendants) {
|
||||
return null
|
||||
}
|
||||
return {
|
||||
platform: 'win32',
|
||||
tree: {
|
||||
...next.tree,
|
||||
descendants: descendants.rows,
|
||||
...(descendants.capturedAtMsByPid
|
||||
? { capturedAtMsByPid: descendants.capturedAtMsByPid }
|
||||
: {}),
|
||||
unidentifiedCount: Math.max(previous.tree.unidentifiedCount, next.tree.unidentifiedCount)
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type {
|
||||
AgentJournalItemBody,
|
||||
AgentJournalItemIdentity
|
||||
} from '../../shared/agent-session-journal-types'
|
||||
import type { StructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink'
|
||||
import { createClaudeJournalTranslator } from './claude-structured-journal-translation'
|
||||
|
||||
function sinkState() {
|
||||
const items: { identity: AgentJournalItemIdentity; body: AgentJournalItemBody }[] = []
|
||||
const sink: StructuredAgentSessionEventSink = {
|
||||
appendItem: (identity, body) => items.push({ identity, body }),
|
||||
appendTombstone: () => {},
|
||||
publish: vi.fn()
|
||||
}
|
||||
return { sink, items }
|
||||
}
|
||||
|
||||
function providerFrameKinds(items: { body: AgentJournalItemBody }[]): string[] {
|
||||
return items.flatMap((item) =>
|
||||
item.body.kind === 'status' && item.body.providerFrame ? [item.body.providerFrame.kind] : []
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* The queue-bookkeeping frame Claude Code 2.1.258 emits for every uuid-stamped
|
||||
* command: `command_uuid` plus a state, and no content of its own. Shape and
|
||||
* states taken from the CLI's own emission sites.
|
||||
*/
|
||||
function commandLifecycle(state: 'started' | 'completed' | 'cancelled', uuid: string) {
|
||||
return {
|
||||
type: 'message' as const,
|
||||
sessionId: 'orca-session',
|
||||
message: {
|
||||
type: 'command_lifecycle',
|
||||
command_uuid: 'command-1',
|
||||
state,
|
||||
uuid,
|
||||
session_id: 'claude-session'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function userTurn(uuid: string, text: string) {
|
||||
return {
|
||||
type: 'message' as const,
|
||||
sessionId: 'orca-session',
|
||||
startsTurn: true as const,
|
||||
message: {
|
||||
type: 'user',
|
||||
uuid,
|
||||
session_id: 'claude-session',
|
||||
parent_tool_use_id: null,
|
||||
isReplay: true,
|
||||
message: { role: 'user', content: text }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function assistantReply(uuid: string, text: string) {
|
||||
return {
|
||||
type: 'message' as const,
|
||||
sessionId: 'orca-session',
|
||||
message: {
|
||||
type: 'assistant',
|
||||
uuid,
|
||||
session_id: 'claude-session',
|
||||
parent_tool_use_id: null,
|
||||
message: { role: 'assistant', content: [{ type: 'text', text }] }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
describe('Claude command_lifecycle frames', () => {
|
||||
it('keeps queue bookkeeping off the transcript for a whole turn', () => {
|
||||
const state = sinkState()
|
||||
const translator = createClaudeJournalTranslator({ sink: state.sink })
|
||||
|
||||
translator.handle(userTurn('user-1', 'Reply with exactly PROBE_OK and nothing else.'))
|
||||
translator.handle(commandLifecycle('started', 'lifecycle-1'))
|
||||
translator.handle(assistantReply('assistant-1', 'PROBE_OK'))
|
||||
translator.handle(commandLifecycle('completed', 'lifecycle-2'))
|
||||
translator.handle(commandLifecycle('completed', 'lifecycle-3'))
|
||||
translator.handle({
|
||||
type: 'message',
|
||||
sessionId: 'orca-session',
|
||||
message: {
|
||||
type: 'result',
|
||||
subtype: 'success',
|
||||
uuid: 'result-1',
|
||||
session_id: 'claude-session',
|
||||
is_error: false,
|
||||
result: 'PROBE_OK',
|
||||
terminal_reason: 'completed'
|
||||
}
|
||||
})
|
||||
|
||||
expect(providerFrameKinds(state.items)).toEqual([])
|
||||
// The turn's real content is untouched.
|
||||
expect(
|
||||
state.items.flatMap((item) =>
|
||||
item.body.kind === 'message' && item.body.role === 'assistant' ? [item.body.blocks] : []
|
||||
)
|
||||
).toEqual([[{ type: 'text', text: 'PROBE_OK' }]])
|
||||
})
|
||||
|
||||
it('keeps a cancelled command off the transcript too', () => {
|
||||
const state = sinkState()
|
||||
const translator = createClaudeJournalTranslator({ sink: state.sink })
|
||||
|
||||
translator.handle(commandLifecycle('cancelled', 'lifecycle-4'))
|
||||
|
||||
expect(providerFrameKinds(state.items)).toEqual([])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,34 @@
|
||||
import { homedir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { claudeConfigDirEnvPatch, defaultClaudeConfigDir } from './claude-config-dir-pin'
|
||||
|
||||
describe('claude config dir pin', () => {
|
||||
it('does not pin the CLI default home, so the macOS Keychain stays reachable', () => {
|
||||
expect(claudeConfigDirEnvPatch(join(homedir(), '.claude'), { env: {} })).toEqual({})
|
||||
expect(claudeConfigDirEnvPatch(`${join(homedir(), '.claude')}/`, { env: {} })).toEqual({})
|
||||
expect(claudeConfigDirEnvPatch(' ', { env: {} })).toEqual({})
|
||||
})
|
||||
|
||||
it('pins a managed account home the CLI would not find on its own', () => {
|
||||
expect(claudeConfigDirEnvPatch('/accounts/claude/managed', { env: {} })).toEqual({
|
||||
CLAUDE_CONFIG_DIR: '/accounts/claude/managed'
|
||||
})
|
||||
})
|
||||
|
||||
it('treats an inherited CLAUDE_CONFIG_DIR as the default the CLI already resolves', () => {
|
||||
const env = { CLAUDE_CONFIG_DIR: '/inherited/home' }
|
||||
expect(defaultClaudeConfigDir(env)).toBe('/inherited/home')
|
||||
expect(claudeConfigDirEnvPatch('/inherited/home', { env })).toEqual({})
|
||||
expect(claudeConfigDirEnvPatch('/other/home', { env })).toEqual({
|
||||
CLAUDE_CONFIG_DIR: '/other/home'
|
||||
})
|
||||
})
|
||||
|
||||
it('compares Windows homes case-insensitively', () => {
|
||||
const env = { CLAUDE_CONFIG_DIR: 'C:\\Users\\Work\\.claude' }
|
||||
expect(claudeConfigDirEnvPatch('c:\\users\\work\\.claude', { env, platform: 'win32' })).toEqual(
|
||||
{}
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,37 @@
|
||||
import { homedir } from 'node:os'
|
||||
import { join, resolve } from 'node:path'
|
||||
|
||||
/** The config dir the Claude CLI resolves for itself when nothing pins one. */
|
||||
export function defaultClaudeConfigDir(env: NodeJS.ProcessEnv = process.env): string {
|
||||
return env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude')
|
||||
}
|
||||
|
||||
function samePath(a: string, b: string, platform: NodeJS.Platform): boolean {
|
||||
const left = resolve(a)
|
||||
const right = resolve(b)
|
||||
return platform === 'win32' ? left.toLowerCase() === right.toLowerCase() : left === right
|
||||
}
|
||||
|
||||
/**
|
||||
* An explicit CLAUDE_CONFIG_DIR moves the Claude CLI off the default Keychain item onto
|
||||
* one derived from the pinned path, so a claude.ai OAuth login stops working even when
|
||||
* the pin names the CLI's own default. Pin only a home the CLI would not find on its
|
||||
* own — the same rule the legacy PTY path applies via `ClaudeRuntimePathResolver`.
|
||||
*
|
||||
* The pinned value is the account home verbatim: the CLI keys its credential lookup on
|
||||
* the literal string, so re-spelling an equivalent path (absolute vs `~`, trailing
|
||||
* separator) selects a different identity. Normalization here is for the equality test
|
||||
* only and must never reach the env.
|
||||
*/
|
||||
export function claudeConfigDirEnvPatch(
|
||||
accountHome: string,
|
||||
options: { env?: NodeJS.ProcessEnv; platform?: NodeJS.Platform } = {}
|
||||
): { CLAUDE_CONFIG_DIR?: string } {
|
||||
const env = options.env ?? process.env
|
||||
const platform = options.platform ?? process.platform
|
||||
const resolved = accountHome.trim()
|
||||
if (!resolved || samePath(resolved, defaultClaudeConfigDir(env), platform)) {
|
||||
return {}
|
||||
}
|
||||
return { CLAUDE_CONFIG_DIR: resolved }
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { terminateDescendantSnapshotWithVerdict } from '../pty-descendant-exit-verification'
|
||||
import {
|
||||
collectDescendantRows,
|
||||
type DescendantSnapshot,
|
||||
type ProcessTableRow
|
||||
} from '../pty-descendant-termination'
|
||||
import { createClaudeChildTreeReaper } from './claude-agent-sdk-exit-proof'
|
||||
|
||||
const ROOT_PID = 500
|
||||
const ORCA_PGID = 400
|
||||
const ROOT_STARTED_AT = 'Thu Sep 3 18:04:50 2026'
|
||||
/** The second both close-time walks land in. */
|
||||
const WALK_SECOND = 'Thu Sep 3 18:05:04 2026'
|
||||
const WALK_MS = Date.parse(WALK_SECOND)
|
||||
const EARLIER_SECOND = 'Thu Sep 3 18:05:03 2026'
|
||||
|
||||
/** The measured split: `s20` at :03.946 died, `s21` at :04.042 leaked. */
|
||||
const EARLIER_BORN = [700, 701, 702]
|
||||
const WALK_SECOND_BORN = [721, 722, 723, 724]
|
||||
|
||||
type Cohort = { pids: number[]; startedAt: string }
|
||||
|
||||
const LIVE_TREE: Cohort[] = [
|
||||
{ pids: EARLIER_BORN, startedAt: EARLIER_SECOND },
|
||||
{ pids: WALK_SECOND_BORN, startedAt: WALK_SECOND }
|
||||
]
|
||||
|
||||
function rowsFor(cohorts: Cohort[]): ProcessTableRow[] {
|
||||
return [
|
||||
{ pid: ROOT_PID, ppid: 1, pgid: ORCA_PGID, startedAt: ROOT_STARTED_AT },
|
||||
...cohorts.flatMap((cohort) =>
|
||||
cohort.pids.map((pid) => ({
|
||||
pid,
|
||||
ppid: ROOT_PID,
|
||||
pgid: ORCA_PGID,
|
||||
startedAt: cohort.startedAt
|
||||
}))
|
||||
)
|
||||
]
|
||||
}
|
||||
|
||||
/** A real ppid walk from the root, exactly as production captures one. */
|
||||
function walk(capturedAtMs: number, cohorts: Cohort[] = LIVE_TREE): DescendantSnapshot {
|
||||
return collectDescendantRows(ROOT_PID, rowsFor(cohorts), capturedAtMs)
|
||||
}
|
||||
|
||||
function killedPids(calls: [number, NodeJS.Signals][]): number[] {
|
||||
return calls.flatMap(([pid, signal]) => (signal === 'SIGKILL' ? [pid] : [])).sort((a, b) => a - b)
|
||||
}
|
||||
|
||||
function signalledPids(calls: [number, NodeJS.Signals][]): number[] {
|
||||
return calls.flatMap(([pid, signal]) => (signal === 'SIGTERM' ? [pid] : [])).sort((a, b) => a - b)
|
||||
}
|
||||
|
||||
/**
|
||||
* Drives the real reaper and the real verifier against a process table where
|
||||
* every descendant traps SIGTERM, so only a forced sweep can end them. The root
|
||||
* is alive for both walks and gone by the sweep, which is the measured teardown.
|
||||
*/
|
||||
async function sweep(
|
||||
captures: DescendantSnapshot[],
|
||||
liveTree: Cohort[] = LIVE_TREE
|
||||
): Promise<[number, NodeJS.Signals][]> {
|
||||
const calls: [number, NodeJS.Signals][] = []
|
||||
const captureDescendants = vi.fn()
|
||||
for (const capture of captures) {
|
||||
captureDescendants.mockResolvedValueOnce(capture)
|
||||
}
|
||||
const tree = createClaudeChildTreeReaper(
|
||||
{ pid: ROOT_PID, kill: vi.fn(() => true) },
|
||||
{
|
||||
platform: 'linux',
|
||||
exited: () => false,
|
||||
captureDescendants,
|
||||
terminateDescendants: (snapshot) =>
|
||||
terminateDescendantSnapshotWithVerdict(snapshot, {
|
||||
requireIdentityBeforeSignal: true,
|
||||
graceMs: 0,
|
||||
verifyMs: 120,
|
||||
sendSignal: (pid, signal) => calls.push([pid, signal]),
|
||||
readTable: async () => ({ rows: rowsFor(liveTree), capturedAtMs: Date.now() })
|
||||
})
|
||||
}
|
||||
)
|
||||
// The close ladder's shape: arm, then re-walk the live root at the boundary.
|
||||
await tree.capture()
|
||||
await tree.refresh?.()
|
||||
await tree.reap()
|
||||
return calls
|
||||
}
|
||||
|
||||
describe('Claude descendant forced-sweep fence', () => {
|
||||
it('escalates a descendant forked in the same second as both close walks', async () => {
|
||||
// Both walks land inside second :04, one ps duration apart, and the root is
|
||||
// gone before a third could run. A descendant born at :04.042 is no less
|
||||
// ours than its sibling born 96ms earlier at :03.946.
|
||||
const calls = await sweep([walk(WALK_MS + 42), walk(WALK_MS + 140)])
|
||||
|
||||
expect(signalledPids(calls)).toEqual([...EARLIER_BORN, ...WALK_SECOND_BORN])
|
||||
expect(killedPids(calls)).toEqual([...EARLIER_BORN, ...WALK_SECOND_BORN])
|
||||
})
|
||||
|
||||
it('still escalates descendants born before the walk that first saw them', async () => {
|
||||
const onlyEarlier = [{ pids: EARLIER_BORN, startedAt: EARLIER_SECOND }]
|
||||
const calls = await sweep([walk(WALK_MS + 42, onlyEarlier)], onlyEarlier)
|
||||
|
||||
expect(killedPids(calls)).toEqual(EARLIER_BORN)
|
||||
})
|
||||
|
||||
it('withholds the sweep from a row no walk re-derived, on its start second alone', async () => {
|
||||
// 900 was seen once, in its own birth second, and the refresh did not find
|
||||
// it. The merge retains the row, but nothing re-proved it belongs to us, so
|
||||
// the second-resolution fence is all there is and it still says no.
|
||||
const retained = { pids: [900], startedAt: WALK_SECOND }
|
||||
const firstWalk = walk(WALK_MS + 42, [...LIVE_TREE, retained])
|
||||
const refresh = walk(WALK_MS + 140)
|
||||
|
||||
const calls = await sweep([firstWalk, refresh], [...LIVE_TREE, retained])
|
||||
|
||||
expect(signalledPids(calls)).toEqual([...EARLIER_BORN, ...WALK_SECOND_BORN, 900])
|
||||
expect(killedPids(calls)).toEqual([...EARLIER_BORN, ...WALK_SECOND_BORN])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,126 @@
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { PassThrough } from 'node:stream'
|
||||
import type { ChildProcessWithoutNullStreams } from 'node:child_process'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { query } from '@anthropic-ai/claude-agent-sdk'
|
||||
import {
|
||||
openClaudeStreamJsonConnection,
|
||||
type ClaudeStreamJsonLaunch
|
||||
} from './claude-stream-json-connection'
|
||||
|
||||
const mocks = vi.hoisted(() => {
|
||||
const refresh = vi.fn()
|
||||
const proveClaudeChildExit = vi.fn()
|
||||
const tree = {
|
||||
capture: vi.fn(async () => {}),
|
||||
refresh: (...args: unknown[]) => refresh(...args),
|
||||
reap: vi.fn(async () => 'exited' as const),
|
||||
treeVerdict: 'unverifiable' as const
|
||||
}
|
||||
return { proveClaudeChildExit, refresh, tree }
|
||||
})
|
||||
|
||||
vi.mock('./claude-agent-sdk-exit-proof', () => ({
|
||||
createClaudeChildTreeReaper: vi.fn(() => mocks.tree),
|
||||
proveClaudeChildExit: (...args: unknown[]) => mocks.proveClaudeChildExit(...args)
|
||||
}))
|
||||
|
||||
function fakeChild(): ChildProcessWithoutNullStreams {
|
||||
const child = new EventEmitter()
|
||||
return Object.assign(child, {
|
||||
pid: 424242,
|
||||
stdin: new PassThrough(),
|
||||
stdout: new PassThrough(),
|
||||
stderr: new PassThrough(),
|
||||
kill: vi.fn()
|
||||
}) as unknown as ChildProcessWithoutNullStreams
|
||||
}
|
||||
|
||||
describe('Claude stream-json close ordering', () => {
|
||||
it('waits for the live tree refresh before ending stdin', async () => {
|
||||
const refreshDone = Promise.withResolvers<void>()
|
||||
mocks.refresh.mockReturnValueOnce(refreshDone.promise)
|
||||
mocks.proveClaudeChildExit.mockResolvedValueOnce(true)
|
||||
const child = fakeChild()
|
||||
const launch: ClaudeStreamJsonLaunch = {
|
||||
pathToClaudeCodeExecutable: 'claude',
|
||||
options: {},
|
||||
cwd: '/work/repo'
|
||||
}
|
||||
const queryImpl = ((params: Parameters<typeof query>[0]) => {
|
||||
if (!params.options) {
|
||||
throw new Error('missing SDK options')
|
||||
}
|
||||
params.options.spawnClaudeCodeProcess?.({
|
||||
command: 'claude',
|
||||
args: [],
|
||||
env: {},
|
||||
signal: new AbortController().signal
|
||||
})
|
||||
void (async () => {
|
||||
for await (const _message of params.prompt) {
|
||||
// The SDK owns the transport write; the close test only needs its EOF boundary.
|
||||
}
|
||||
child.stdin.end()
|
||||
})()
|
||||
return (async function* () {})()
|
||||
}) as typeof query
|
||||
const connection = await openClaudeStreamJsonConnection(launch, {}, () => child, queryImpl)
|
||||
|
||||
const closing = connection.close()
|
||||
await new Promise((resolve) => setImmediate(resolve))
|
||||
expect(child.stdin.writableEnded).toBe(false)
|
||||
|
||||
refreshDone.resolve()
|
||||
await expect(closing).resolves.toBe(true)
|
||||
expect(child.stdin.writableEnded).toBe(true)
|
||||
})
|
||||
|
||||
it('requests a fresh close boundary after an output capture starts', async () => {
|
||||
mocks.refresh.mockReset()
|
||||
mocks.proveClaudeChildExit.mockReset()
|
||||
const outputCapture = Promise.withResolvers<void>()
|
||||
const closeCapture = Promise.withResolvers<void>()
|
||||
mocks.refresh
|
||||
.mockReturnValueOnce(outputCapture.promise)
|
||||
.mockReturnValueOnce(closeCapture.promise)
|
||||
mocks.proveClaudeChildExit.mockResolvedValueOnce(true)
|
||||
const child = fakeChild()
|
||||
const launch: ClaudeStreamJsonLaunch = {
|
||||
pathToClaudeCodeExecutable: 'claude',
|
||||
options: {},
|
||||
cwd: '/work/repo'
|
||||
}
|
||||
const queryImpl = ((params: Parameters<typeof query>[0]) => {
|
||||
params.options?.spawnClaudeCodeProcess?.({
|
||||
command: 'claude',
|
||||
args: [],
|
||||
env: {},
|
||||
signal: new AbortController().signal
|
||||
})
|
||||
void (async () => {
|
||||
for await (const _message of params.prompt) {
|
||||
// The SDK owns the transport write; the close test only needs its EOF boundary.
|
||||
}
|
||||
child.stdin.end()
|
||||
})()
|
||||
return (async function* () {})()
|
||||
}) as typeof query
|
||||
const connection = await openClaudeStreamJsonConnection(launch, {}, () => child, queryImpl)
|
||||
|
||||
child.stderr.emit('data', 'output')
|
||||
await vi.waitFor(() => expect(mocks.refresh).toHaveBeenCalledTimes(1))
|
||||
const closing = connection.close()
|
||||
await Promise.resolve()
|
||||
|
||||
expect(mocks.refresh).toHaveBeenCalledTimes(2)
|
||||
expect(child.stdin.writableEnded).toBe(false)
|
||||
|
||||
outputCapture.resolve()
|
||||
await Promise.resolve()
|
||||
expect(child.stdin.writableEnded).toBe(false)
|
||||
closeCapture.resolve()
|
||||
await expect(closing).resolves.toBe(true)
|
||||
expect(child.stdin.writableEnded).toBe(true)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,768 @@
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { spawnProcess, type SpawnedProcess } from '../../shared/child-process/run-process'
|
||||
import { hasLiveClaudePtys } from '../claude-accounts/live-pty-gate'
|
||||
import type { ProcessSpec } from '../../shared/child-process/process-spec'
|
||||
import { query, type CanUseTool, type Options } from '@anthropic-ai/claude-agent-sdk'
|
||||
import {
|
||||
openClaudeStreamJsonConnection,
|
||||
type ClaudeStreamJsonConnection,
|
||||
type ClaudeStreamJsonLaunch
|
||||
} from './claude-stream-json-connection'
|
||||
import { openAgentSessionJournal } from '../native-chat/agent-session-journal/journal-store-factory'
|
||||
import { createDeferredStructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink'
|
||||
import { claudeAuthDiagnostic } from './claude-structured-init-proof'
|
||||
import { createClaudeJournalTranslator } from './claude-structured-journal-translation'
|
||||
import { readClaudeStructuredSessionOptions } from './claude-structured-session-options'
|
||||
import type { ClaudeSession } from './claude-structured-session-state'
|
||||
import { CLAUDE_STRUCTURED_BASE_OPTIONS } from './claude-structured-launch-resolution'
|
||||
|
||||
// These drive the real SDK against the scripted fake CLI, so every assertion is
|
||||
// about the environment, argv and frames a real child actually saw.
|
||||
const FAKE_CLI = join(__dirname, '__fixtures__', 'claude-agent-sdk-scripted-cli.mjs')
|
||||
const SESSION_ID = '5348c19f-6a54-4c2e-9c68-9c2b1a3d4e5f'
|
||||
const HOLD_OPEN = { delayMs: 10_000 }
|
||||
|
||||
type ScriptedCliReport = {
|
||||
argv: string[]
|
||||
controlRequests: { request_id: string; request: { subtype: string } }[]
|
||||
controlResponses: { response: { request_id: string; response?: unknown } }[]
|
||||
userMessages: Record<string, unknown>[]
|
||||
descendantPid: number | null
|
||||
}
|
||||
|
||||
const scratchDirs: string[] = []
|
||||
const openConnections: ClaudeStreamJsonConnection[] = []
|
||||
|
||||
afterEach(async () => {
|
||||
for (const connection of openConnections.splice(0)) {
|
||||
await connection.close()
|
||||
}
|
||||
for (const dir of scratchDirs.splice(0)) {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
spawned.splice(0)
|
||||
spawnedChildren.splice(0)
|
||||
vi.unstubAllEnvs()
|
||||
})
|
||||
|
||||
function scriptScenario(
|
||||
steps: Record<string, unknown>[],
|
||||
controlResponses: Record<string, unknown> = {}
|
||||
) {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'claude-sdk-connection-'))
|
||||
scratchDirs.push(dir)
|
||||
const scenarioPath = join(dir, 'scenario.json')
|
||||
const reportPath = join(dir, 'report.json')
|
||||
writeFileSync(scenarioPath, JSON.stringify({ steps, controlResponses }))
|
||||
return {
|
||||
cwd: dir,
|
||||
env: {
|
||||
PATH: process.env.PATH ?? '',
|
||||
ORCA_SDK_CONTRACT_SCENARIO_PATH: scenarioPath,
|
||||
ORCA_SDK_CONTRACT_REPORT_PATH: reportPath
|
||||
},
|
||||
readReport: () => JSON.parse(readFileSync(reportPath, 'utf8')) as ScriptedCliReport
|
||||
}
|
||||
}
|
||||
|
||||
function launchFor(
|
||||
scenario: { cwd: string; env: Record<string, string> },
|
||||
env: Record<string, string> = {}
|
||||
): ClaudeStreamJsonLaunch {
|
||||
return {
|
||||
pathToClaudeCodeExecutable: FAKE_CLI,
|
||||
options: { ...CLAUDE_STRUCTURED_BASE_OPTIONS, sessionId: SESSION_ID },
|
||||
cwd: scenario.cwd,
|
||||
env: { ...scenario.env, ...env }
|
||||
}
|
||||
}
|
||||
|
||||
/** The derived child environment, captured where Orca actually hands it to the OS. */
|
||||
const spawned: ProcessSpec[] = []
|
||||
/** The retained child, so a test can end it the way a crashing CLI would. */
|
||||
const spawnedChildren: SpawnedProcess[] = []
|
||||
|
||||
async function open(
|
||||
launch: ClaudeStreamJsonLaunch,
|
||||
handlers: Parameters<typeof openClaudeStreamJsonConnection>[1] = {},
|
||||
queryImpl?: typeof query
|
||||
): Promise<ClaudeStreamJsonConnection> {
|
||||
const connection = await openClaudeStreamJsonConnection(
|
||||
launch,
|
||||
handlers,
|
||||
(spec) => {
|
||||
spawned.push(spec)
|
||||
const child = spawnProcess(spec)
|
||||
spawnedChildren.push(child)
|
||||
return child
|
||||
},
|
||||
queryImpl
|
||||
)
|
||||
openConnections.push(connection)
|
||||
return connection
|
||||
}
|
||||
|
||||
function childEnv(): Record<string, string | undefined> {
|
||||
return (spawned.at(-1)?.env ?? {}) as Record<string, string | undefined>
|
||||
}
|
||||
|
||||
async function until<T>(read: () => T | null | undefined, label: string): Promise<T> {
|
||||
for (let attempt = 0; attempt < 400; attempt++) {
|
||||
const value = read()
|
||||
if (value !== null && value !== undefined) {
|
||||
return value
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 25))
|
||||
}
|
||||
throw new Error(`timed out waiting for ${label}`)
|
||||
}
|
||||
|
||||
function readReportSafely(scenario: { readReport: () => ScriptedCliReport }) {
|
||||
try {
|
||||
return scenario.readReport()
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function processState(pid: number): 'running' | 'exited' {
|
||||
try {
|
||||
const state = execFileSync('ps', ['-o', 'state=', '-p', String(pid)], {
|
||||
encoding: 'utf8',
|
||||
env: { ...process.env, LANG: 'C', LC_ALL: 'C' }
|
||||
}).trim()
|
||||
return state.startsWith('Z') ? 'exited' : 'running'
|
||||
} catch (error) {
|
||||
if ((error as { status?: number }).status === 1) {
|
||||
return 'exited'
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
describe('Claude stream-json connection', () => {
|
||||
it('passes the Claude Code system-prompt preset through to SDK query', async () => {
|
||||
const scenario = scriptScenario([HOLD_OPEN])
|
||||
let captured: Options | undefined
|
||||
await open(launchFor(scenario), {}, (params) => {
|
||||
captured = params.options
|
||||
return query(params)
|
||||
})
|
||||
|
||||
expect(captured?.systemPrompt).toEqual({ type: 'preset', preset: 'claude_code' })
|
||||
})
|
||||
|
||||
it('hands the child a derived environment, the resolved CLI path, and keeps the pid', async () => {
|
||||
vi.stubEnv('ANTHROPIC_API_KEY', 'sk-ant-SHELL-LEAK')
|
||||
vi.stubEnv('CLAUDE_CODE_CHILD_SESSION', '1')
|
||||
vi.stubEnv('NODE_OPTIONS', '--require=/tmp/inject.js')
|
||||
// An inherited value wins over the SDK's default, so clear it to pin the default.
|
||||
vi.stubEnv('CLAUDE_CODE_ENTRYPOINT', undefined)
|
||||
vi.stubEnv('ORCA_CONNECTION_MARKER', 'inherited')
|
||||
const scenario = scriptScenario([HOLD_OPEN])
|
||||
const connection = await open(
|
||||
launchFor(scenario, {
|
||||
CLAUDE_CONFIG_DIR: '/accounts/managed/home',
|
||||
ANTHROPIC_AUTH_TOKEN: 'configured-token',
|
||||
ORCA_AGENT_SESSION_SPAWN_TOKEN: 'spawn-9',
|
||||
CLAUDE_CODE_CHILD_SESSION: 'configured-child-session',
|
||||
CLAUDE_CODE_SESSION_ID: 'configured-session',
|
||||
CLAUDE_CODE_BRIDGE_SESSION_ID: 'configured-bridge-session'
|
||||
})
|
||||
)
|
||||
|
||||
// Ownership proof: the pid is a real live process, not a value the SDK reported.
|
||||
expect(connection.pid).toEqual(expect.any(Number))
|
||||
expect(() => process.kill(connection.pid as number, 0)).not.toThrow()
|
||||
const env = childEnv()
|
||||
// The managed home is pinned verbatim: the CLI keys credential lookup on the literal string.
|
||||
expect(env.CLAUDE_CONFIG_DIR).toBe('/accounts/managed/home')
|
||||
expect(env.ANTHROPIC_AUTH_TOKEN).toBe('configured-token')
|
||||
expect(env.ORCA_AGENT_SESSION_SPAWN_TOKEN).toBe('spawn-9')
|
||||
expect(env.ORCA_CONNECTION_MARKER).toBe('inherited')
|
||||
expect(env.ANTHROPIC_API_KEY).toBeUndefined()
|
||||
expect(env.CLAUDE_CODE_CHILD_SESSION).toBeUndefined()
|
||||
expect(env.CLAUDE_CODE_SESSION_ID).toBeUndefined()
|
||||
expect(env.CLAUDE_CODE_BRIDGE_SESSION_ID).toBeUndefined()
|
||||
// Two SDK mutations of the child env, pinned so a bump cannot change them unseen.
|
||||
expect(env.CLAUDE_CODE_ENTRYPOINT).toBe('sdk-ts')
|
||||
expect(env.NODE_OPTIONS).toBeUndefined()
|
||||
// The bundled binary is excluded from the install, so the resolved path is mandatory.
|
||||
const report = await until(() => readReportSafely(scenario), 'the scripted CLI report')
|
||||
expect(report.argv[0]).toBe(FAKE_CLI)
|
||||
// The .mjs fixture makes the SDK run it under node; a real CLI path is the program
|
||||
// itself. Either way the resolved path is what Orca's spawner is asked to execute.
|
||||
expect([spawned.at(-1)?.program, ...(spawned.at(-1)?.args ?? [])]).toContain(FAKE_CLI)
|
||||
expect(report.argv).toContain('--replay-user-messages')
|
||||
expect(report.argv).toContain(`--session-id=${SESSION_ID}`)
|
||||
})
|
||||
|
||||
it('leaves the default CLI home unpinned so macOS Keychain OAuth keeps working', async () => {
|
||||
const scenario = scriptScenario([HOLD_OPEN])
|
||||
await open(launchFor(scenario))
|
||||
|
||||
await until(() => readReportSafely(scenario), 'the scripted CLI report')
|
||||
expect(childEnv().CLAUDE_CONFIG_DIR).toBeUndefined()
|
||||
})
|
||||
|
||||
it('settles a send only once the frame reached the child, and replays reach onMessage', async () => {
|
||||
const replay = {
|
||||
type: 'user',
|
||||
message: { role: 'user', content: [{ type: 'text', text: 'hello' }] },
|
||||
parent_tool_use_id: null,
|
||||
isReplay: true,
|
||||
session_id: SESSION_ID,
|
||||
uuid: 'uuid-replay-1'
|
||||
}
|
||||
const scenario = scriptScenario([{ awaitUserMessage: true }, { emit: replay }, HOLD_OPEN])
|
||||
const messages: Record<string, unknown>[] = []
|
||||
const connection = await open(launchFor(scenario), {
|
||||
onMessage: (message) => messages.push(message)
|
||||
})
|
||||
|
||||
await connection.send({
|
||||
type: 'user',
|
||||
message: { role: 'user', content: [{ type: 'text', text: 'hello' }] },
|
||||
parent_tool_use_id: null,
|
||||
session_id: SESSION_ID
|
||||
})
|
||||
// The report exists from the child's first line of work, so poll for the frame
|
||||
// itself: `send` settles on the SDK's completed write, and the child still has
|
||||
// to read that line before it can record it.
|
||||
const report = await until(
|
||||
() => (readReportSafely(scenario)?.userMessages.length ? readReportSafely(scenario) : null),
|
||||
'the user frame recorded by the child'
|
||||
)
|
||||
expect(report.userMessages).toHaveLength(1)
|
||||
|
||||
await until(() => messages.find((message) => message.uuid === 'uuid-replay-1'), 'the replay')
|
||||
// The replay is delivered verbatim, so the dispatch acknowledgement still binds on it.
|
||||
expect(messages.find((message) => message.uuid === 'uuid-replay-1')).toEqual(replay)
|
||||
})
|
||||
|
||||
it('rejects a send the SDK pulled but could not write to a terminated child', async () => {
|
||||
const scenario = scriptScenario([{ awaitUserMessage: true }, HOLD_OPEN])
|
||||
const connection = await open(launchFor(scenario))
|
||||
const child = spawnedChildren.at(-1)
|
||||
|
||||
// Same tick as the send, so the liveness guard still passes and the frame
|
||||
// reaches the SDK's input pump: its `transport.write` is what fails, which is
|
||||
// the window a child crashing mid-send actually opens.
|
||||
child?.kill('SIGKILL')
|
||||
const sent = connection.send({
|
||||
type: 'user',
|
||||
message: { role: 'user', content: [{ type: 'text', text: 'hello' }] },
|
||||
parent_tool_use_id: null,
|
||||
session_id: SESSION_ID
|
||||
})
|
||||
|
||||
await expect(sent).rejects.toThrow()
|
||||
expect(readReportSafely(scenario)?.userMessages ?? []).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('delivers an unmodeled frame verbatim so the provider-fallback row survives', async () => {
|
||||
const unknown = {
|
||||
type: 'frame_kind_from_the_future',
|
||||
session_id: SESSION_ID,
|
||||
uuid: 'uuid-unknown-1',
|
||||
payload: { nested: { flags: ['a', 'b'] } }
|
||||
}
|
||||
const scenario = scriptScenario([{ emit: unknown }, HOLD_OPEN])
|
||||
const messages: Record<string, unknown>[] = []
|
||||
await open(launchFor(scenario), { onMessage: (message) => messages.push(message) })
|
||||
|
||||
await until(() => messages.find((message) => message.uuid === 'uuid-unknown-1'), 'the frame')
|
||||
expect(messages.find((message) => message.uuid === 'uuid-unknown-1')).toEqual(unknown)
|
||||
})
|
||||
|
||||
it('commits the real partial-message cadence as one assistant item through the translator', async () => {
|
||||
// The frame order and per-frame uuids are the ones Claude Code 2.1.258 emits
|
||||
// under --include-partial-messages: every stream_event and the block's final
|
||||
// assistant frame each carry their own uuid; only message.id ties them.
|
||||
const stream = (uuid: string, event: Record<string, unknown>) => ({
|
||||
type: 'stream_event',
|
||||
uuid,
|
||||
session_id: SESSION_ID,
|
||||
parent_tool_use_id: null,
|
||||
event
|
||||
})
|
||||
const frames = [
|
||||
stream('uuid-message-start', {
|
||||
type: 'message_start',
|
||||
message: { id: 'msg_01', role: 'assistant', content: [] }
|
||||
}),
|
||||
stream('uuid-block-start', {
|
||||
type: 'content_block_start',
|
||||
index: 0,
|
||||
content_block: { type: 'text', text: '' }
|
||||
}),
|
||||
stream('uuid-delta-1', {
|
||||
type: 'content_block_delta',
|
||||
index: 0,
|
||||
delta: { type: 'text_delta', text: 'ST' }
|
||||
}),
|
||||
stream('uuid-delta-2', {
|
||||
type: 'content_block_delta',
|
||||
index: 0,
|
||||
delta: { type: 'text_delta', text: 'REAMOK_ELEC_64E632' }
|
||||
}),
|
||||
{
|
||||
type: 'assistant',
|
||||
uuid: 'uuid-assistant-final',
|
||||
session_id: SESSION_ID,
|
||||
parent_tool_use_id: null,
|
||||
message: {
|
||||
id: 'msg_01',
|
||||
role: 'assistant',
|
||||
content: [{ type: 'text', text: 'STREAMOK_ELEC_64E632' }],
|
||||
stop_reason: null
|
||||
}
|
||||
},
|
||||
stream('uuid-block-stop', { type: 'content_block_stop', index: 0 }),
|
||||
stream('uuid-message-delta', { type: 'message_delta', delta: { stop_reason: 'end_turn' } }),
|
||||
stream('uuid-message-stop', { type: 'message_stop' }),
|
||||
{
|
||||
type: 'result',
|
||||
subtype: 'success',
|
||||
is_error: false,
|
||||
duration_ms: 1,
|
||||
duration_api_ms: 1,
|
||||
num_turns: 1,
|
||||
result: 'STREAMOK_ELEC_64E632',
|
||||
stop_reason: 'end_turn',
|
||||
session_id: SESSION_ID,
|
||||
uuid: 'uuid-result'
|
||||
}
|
||||
]
|
||||
const scenario = scriptScenario([...frames.map((frame) => ({ emit: frame })), HOLD_OPEN])
|
||||
const journal = await openAgentSessionJournal({
|
||||
identity: {
|
||||
sessionId: 'session-1',
|
||||
workspaceId: 'workspace-1',
|
||||
hostId: 'host-1',
|
||||
agent: 'claude',
|
||||
providerHandle: { kind: 'claude', sessionId: SESSION_ID, leafUuid: 'leaf-1' }
|
||||
},
|
||||
journalDir: join(scenario.cwd, 'journal'),
|
||||
now: () => 1_700_000_000_000,
|
||||
mintEpoch: () => 'epoch-1'
|
||||
})
|
||||
const deferred = createDeferredStructuredAgentSessionEventSink()
|
||||
deferred.bind({ journal, fence: 1, publish: vi.fn() })
|
||||
const translator = createClaudeJournalTranslator({ sink: deferred.sink })
|
||||
let settled = false
|
||||
await open(launchFor(scenario), {
|
||||
onMessage: (message) => {
|
||||
translator.handle({ type: 'message', sessionId: 'session-1', message })
|
||||
settled ||= message.type === 'result'
|
||||
}
|
||||
})
|
||||
|
||||
await until(() => (settled ? true : null), 'the result frame')
|
||||
await deferred.drained()
|
||||
const items = journal.snapshot().items
|
||||
const assistant = items.filter(
|
||||
(item) => item.body.kind === 'message' && item.body.role === 'assistant'
|
||||
)
|
||||
expect(assistant.map((item) => item.body)).toEqual([
|
||||
{
|
||||
kind: 'message',
|
||||
role: 'assistant',
|
||||
blocks: [{ type: 'text', text: 'STREAMOK_ELEC_64E632' }]
|
||||
}
|
||||
])
|
||||
expect(assistant.map((item) => item.itemId)).toEqual([`claude:${SESSION_ID}:uuid-block-start`])
|
||||
expect(
|
||||
items.flatMap((item) =>
|
||||
item.body.kind === 'status' && item.body.providerFrame ? [item.body.providerFrame.kind] : []
|
||||
)
|
||||
).toEqual([])
|
||||
// The journal owns a SQLite connection now; afterEach removes this temp root and an open
|
||||
// handle blocks that on Windows.
|
||||
await journal.close()
|
||||
})
|
||||
|
||||
it('feeds an inbound permission request to canUseTool and writes its answer back on the same id', async () => {
|
||||
const scenario = scriptScenario([
|
||||
{
|
||||
emit: {
|
||||
type: 'control_request',
|
||||
request_id: 'perm-421',
|
||||
request: {
|
||||
subtype: 'can_use_tool',
|
||||
tool_name: 'Bash',
|
||||
input: { command: 'ls' },
|
||||
tool_use_id: 'toolu_1',
|
||||
permission_suggestions: [{ type: 'addRules' }]
|
||||
}
|
||||
}
|
||||
},
|
||||
{ awaitControlResponse: 'perm-421' },
|
||||
HOLD_OPEN
|
||||
])
|
||||
const seen: { toolName: string; requestId: string; toolUseID: string; suggestions: unknown }[] =
|
||||
[]
|
||||
const canUseTool: CanUseTool = (toolName, _input, options) => {
|
||||
seen.push({
|
||||
toolName,
|
||||
requestId: options.requestId,
|
||||
toolUseID: options.toolUseID,
|
||||
suggestions: options.suggestions
|
||||
})
|
||||
return Promise.resolve({ behavior: 'deny', message: 'No', toolUseID: options.toolUseID })
|
||||
}
|
||||
await open(launchFor(scenario), { canUseTool })
|
||||
|
||||
await until(() => (seen.length > 0 ? seen : null), 'the inbound permission request')
|
||||
expect(seen).toEqual([
|
||||
{
|
||||
toolName: 'Bash',
|
||||
requestId: 'perm-421',
|
||||
toolUseID: 'toolu_1',
|
||||
suggestions: [{ type: 'addRules' }]
|
||||
}
|
||||
])
|
||||
const written = await until(
|
||||
() =>
|
||||
readReportSafely(scenario)?.controlResponses.find(
|
||||
(frame) => frame.response.request_id === 'perm-421'
|
||||
),
|
||||
'the permission answer'
|
||||
)
|
||||
expect(written.response.response).toMatchObject({ behavior: 'deny', message: 'No' })
|
||||
})
|
||||
|
||||
it('drives Orca control methods onto the SDK and times out with the init proof message', async () => {
|
||||
const scenario = scriptScenario([HOLD_OPEN], {
|
||||
initialize: { models: [{ value: 'sonnet' }], account: { tokenSource: 'oauth' } },
|
||||
get_settings: { env: { ANTHROPIC_BASE_URL: 'https://settings.example.test' } }
|
||||
})
|
||||
const connection = await open(launchFor(scenario))
|
||||
|
||||
await expect(connection.initializationResult()).resolves.toMatchObject({
|
||||
models: [{ value: 'sonnet' }]
|
||||
})
|
||||
await expect(connection.getSettings()).resolves.toEqual({
|
||||
env: { ANTHROPIC_BASE_URL: 'https://settings.example.test' }
|
||||
})
|
||||
await expect(connection.setModel('opus')).resolves.toBeUndefined()
|
||||
const requests = await until(
|
||||
() =>
|
||||
readReportSafely(scenario)?.controlRequests.find(
|
||||
(frame) => frame.request.subtype === 'set_model'
|
||||
),
|
||||
'the set_model control request'
|
||||
)
|
||||
expect(requests.request.subtype).toBe('set_model')
|
||||
})
|
||||
|
||||
it('reads supportedModels from the catalog the running CLI reported', async () => {
|
||||
const scenario = scriptScenario([HOLD_OPEN], {
|
||||
initialize: {
|
||||
models: [
|
||||
{ value: 'default', resolvedModel: 'claude-opus-5' },
|
||||
{
|
||||
value: 'opus',
|
||||
displayName: 'Opus 5',
|
||||
description: 'The live row, not the seed',
|
||||
resolvedModel: 'claude-opus-5',
|
||||
supportsEffort: true,
|
||||
supportedEffortLevels: ['low', 'high']
|
||||
}
|
||||
]
|
||||
}
|
||||
})
|
||||
const connection = await open(launchFor(scenario))
|
||||
|
||||
await expect(connection.supportedModels()).resolves.toMatchObject([
|
||||
{ value: 'default', resolvedModel: 'claude-opus-5' },
|
||||
{ value: 'opus', displayName: 'Opus 5', supportedEffortLevels: ['low', 'high'] }
|
||||
])
|
||||
})
|
||||
|
||||
it('serves the picker the live catalog rather than falling back to the static seed', async () => {
|
||||
const scenario = scriptScenario([HOLD_OPEN], {
|
||||
initialize: {
|
||||
models: [
|
||||
{ value: 'default', resolvedModel: 'claude-opus-5' },
|
||||
{
|
||||
value: 'opus',
|
||||
displayName: 'Opus 5',
|
||||
description: 'The live row, not the seed',
|
||||
resolvedModel: 'claude-opus-5',
|
||||
supportsEffort: true,
|
||||
supportedEffortLevels: ['low', 'high']
|
||||
}
|
||||
]
|
||||
}
|
||||
})
|
||||
const connection = await open(launchFor(scenario))
|
||||
const session = {
|
||||
connection,
|
||||
options: new Map<string, string>(),
|
||||
reportedOptions: {}
|
||||
} as unknown as ClaudeSession
|
||||
|
||||
const options = await readClaudeStructuredSessionOptions(session, 5_000)
|
||||
|
||||
// The seed carries neither this description nor a two-level effort list, so
|
||||
// both can only have come from the child.
|
||||
expect(options.models).toContainEqual({
|
||||
id: 'opus',
|
||||
label: 'Opus 5',
|
||||
description: 'The live row, not the seed',
|
||||
isDefault: true,
|
||||
efforts: [
|
||||
{ value: 'low', label: 'Low' },
|
||||
{ value: 'high', label: 'High' }
|
||||
]
|
||||
})
|
||||
expect(options.current.model).toBe('opus')
|
||||
})
|
||||
|
||||
it('feeds the auth diagnostic from the settings the running child reports', async () => {
|
||||
for (const key of ['ANTHROPIC_BASE_URL', 'ANTHROPIC_AUTH_TOKEN', 'ANTHROPIC_API_KEY']) {
|
||||
vi.stubEnv(key, undefined)
|
||||
}
|
||||
const scenario = scriptScenario([HOLD_OPEN], {
|
||||
get_settings: {
|
||||
env: {
|
||||
ANTHROPIC_BASE_URL: 'https://settings.example.test',
|
||||
ANTHROPIC_AUTH_TOKEN: 'secret'
|
||||
}
|
||||
}
|
||||
})
|
||||
const connection = await open(launchFor(scenario))
|
||||
const init = { providerSessionId: SESSION_ID, uuid: null, model: null, message: {} }
|
||||
|
||||
// With no ambient auth, every true below can only have come from the CLI's settings.
|
||||
expect(claudeAuthDiagnostic(init, null)).toMatchObject({
|
||||
baseUrlConfigured: false,
|
||||
authTokenConfigured: false
|
||||
})
|
||||
const diagnostic = claudeAuthDiagnostic(init, await connection.getSettings())
|
||||
expect(diagnostic).toMatchObject({
|
||||
baseUrlConfigured: true,
|
||||
authTokenConfigured: true,
|
||||
apiKeyConfigured: false
|
||||
})
|
||||
expect(JSON.stringify(diagnostic)).not.toContain('secret')
|
||||
})
|
||||
|
||||
it('reports an unauthenticated start through the init deadline instead of hanging', async () => {
|
||||
// The scripted CLI never answers, which is the shape of a silently unauthenticated CLI.
|
||||
const scenario = scriptScenario([HOLD_OPEN])
|
||||
const connection = await open({
|
||||
...launchFor(scenario),
|
||||
env: { ...launchFor(scenario).env, ORCA_SDK_CONTRACT_IGNORE_CONTROL_REQUESTS: '1' }
|
||||
})
|
||||
|
||||
await expect(connection.initializationResult({ timeoutMs: 200 })).rejects.toThrow(
|
||||
'claude initialize request timed out'
|
||||
)
|
||||
})
|
||||
|
||||
it('reports a self-exit with its status and stderr, and leaves its tree unverifiable', async () => {
|
||||
const scenario = scriptScenario([{ stderr: 'claude: not signed in\n' }, { exit: 1 }])
|
||||
let exit: Error | null = null
|
||||
const connection = await open(launchFor(scenario), {
|
||||
onExit: (error) => {
|
||||
exit = error
|
||||
}
|
||||
})
|
||||
|
||||
await until(() => exit, 'the exit error')
|
||||
// The status and stderr are the only diagnostic a refused start leaves behind.
|
||||
expect((exit as unknown as Error).message).toMatch(/exited \(code 1\): claude: not signed in/)
|
||||
expect(connection.closed).toBe(true)
|
||||
// The root's exit is first-hand, but it left before a descendant snapshot
|
||||
// could be armed, so close() has no tree proof to offer and says so.
|
||||
await expect(connection.close()).resolves.toBe(false)
|
||||
expect(connection.exitVerdict).toEqual({ root: 'exited', tree: 'unverifiable' })
|
||||
})
|
||||
|
||||
it.runIf(process.platform !== 'win32')(
|
||||
'proves a natural SDK exit and cleans up its descendant before recovery',
|
||||
async () => {
|
||||
const scenario = scriptScenario([
|
||||
{ stderr: 'claude: natural exit\n' },
|
||||
{ delayMs: 500 },
|
||||
{ exit: 1 }
|
||||
])
|
||||
let exit: Error | null = null
|
||||
const connection = await open(
|
||||
{
|
||||
...launchFor(scenario),
|
||||
env: { ...launchFor(scenario).env, ORCA_SDK_CONTRACT_DESCENDANT: '1' }
|
||||
},
|
||||
{ onExit: (error) => (exit = error) }
|
||||
)
|
||||
const report = await until(() => {
|
||||
const current = readReportSafely(scenario)
|
||||
return current?.descendantPid ? current : null
|
||||
}, 'the descendant report')
|
||||
await until(() => exit, 'the natural exit error')
|
||||
try {
|
||||
await expect(connection.close()).resolves.toBe(true)
|
||||
expect(connection.exitVerdict).toEqual({ root: 'exited', tree: 'exited' })
|
||||
expect(processState(report.descendantPid as number)).toBe('exited')
|
||||
} finally {
|
||||
try {
|
||||
process.kill(report.descendantPid as number, 'SIGKILL')
|
||||
} catch {
|
||||
// Already gone.
|
||||
}
|
||||
}
|
||||
},
|
||||
20_000
|
||||
)
|
||||
|
||||
it('settles a spawn error followed by close as processless and closes idempotently', async () => {
|
||||
const scenario = scriptScenario([HOLD_OPEN])
|
||||
const missingCli = join(scenario.cwd, 'claude-that-does-not-exist')
|
||||
let fault: Error | null = null
|
||||
let exit: Error | null = null
|
||||
const connection = await open(
|
||||
{ ...launchFor(scenario), pathToClaudeCodeExecutable: missingCli },
|
||||
{
|
||||
onFault: (error) => {
|
||||
fault = error
|
||||
},
|
||||
onExit: (error) => {
|
||||
exit = error
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
await until(
|
||||
() => (connection.exitVerdict.root === 'processless' ? connection.exitVerdict : null),
|
||||
'the processless spawn settlement'
|
||||
)
|
||||
expect(connection.pid).toBeUndefined()
|
||||
expect(fault).toBeInstanceOf(Error)
|
||||
expect(exit).toBeNull()
|
||||
await expect(Promise.all([connection.close(), connection.close()])).resolves.toEqual([
|
||||
true,
|
||||
true
|
||||
])
|
||||
await expect(connection.close()).resolves.toBe(true)
|
||||
expect(connection.exitVerdict).toEqual({ root: 'processless', tree: 'exited' })
|
||||
})
|
||||
|
||||
it('does not treat a child error event as first-hand root exit proof', async () => {
|
||||
const scenario = scriptScenario([HOLD_OPEN])
|
||||
let exit: Error | null = null
|
||||
const connection = await open(launchFor(scenario), {
|
||||
onExit: (error) => {
|
||||
exit = error
|
||||
}
|
||||
})
|
||||
const child = spawnedChildren.at(-1)
|
||||
expect(child).toBeDefined()
|
||||
|
||||
child?.emit('error', new Error('child transport fault'))
|
||||
|
||||
expect(exit).toBeNull()
|
||||
expect(connection.exitVerdict.root).toBe('live')
|
||||
await until(() => exit, 'the distinct child exit')
|
||||
expect(connection.exitVerdict.root).toBe('exited')
|
||||
})
|
||||
|
||||
it('proves the exit of a child that ignores a graceful shutdown', async () => {
|
||||
const scenario = scriptScenario([HOLD_OPEN])
|
||||
const connection = await open({
|
||||
...launchFor(scenario),
|
||||
env: { ...launchFor(scenario).env, ORCA_SDK_CONTRACT_IGNORE_SIGTERM: '1' }
|
||||
})
|
||||
|
||||
// Keep the lstart capture boundary outside the child's displayed start second.
|
||||
await new Promise((resolve) => setTimeout(resolve, 1_100))
|
||||
await expect(connection.close()).resolves.toBe(true)
|
||||
}, 20_000)
|
||||
})
|
||||
|
||||
// A structured Claude child owns the account's credentials while it runs, exactly as
|
||||
// a Claude PTY does. The gate is what makes runtime-auth-sync defer the managed OAuth
|
||||
// refresh instead of rotating the single-use token out from under a live session, and
|
||||
// structured sessions used to be invisible to it.
|
||||
describe('the managed-auth live gate', () => {
|
||||
it('holds while a structured child runs and releases when it ends', async () => {
|
||||
// The gate is a process-wide singleton and a sibling test's release lands on its
|
||||
// child's 'close' event, which can settle after that test's close() resolved.
|
||||
await until(() => (hasLiveClaudePtys() ? null : true), 'a drained auth gate')
|
||||
const scenario = scriptScenario([
|
||||
{ emit: { type: 'system', subtype: 'init', session_id: SESSION_ID, uuid: 'init-1' } },
|
||||
{ wait: HOLD_OPEN }
|
||||
])
|
||||
const connection = await open(launchFor(scenario))
|
||||
|
||||
expect(hasLiveClaudePtys()).toBe(true)
|
||||
|
||||
await connection.close()
|
||||
|
||||
await until(() => (hasLiveClaudePtys() ? null : true), 'the auth gate to drain')
|
||||
expect(hasLiveClaudePtys()).toBe(false)
|
||||
}, 30_000)
|
||||
|
||||
it('releases when the child dies on its own rather than through close()', async () => {
|
||||
await until(() => (hasLiveClaudePtys() ? null : true), 'a drained auth gate')
|
||||
const scenario = scriptScenario([
|
||||
{ emit: { type: 'system', subtype: 'init', session_id: SESSION_ID, uuid: 'init-1' } },
|
||||
{ wait: HOLD_OPEN }
|
||||
])
|
||||
await open(launchFor(scenario))
|
||||
expect(hasLiveClaudePtys()).toBe(true)
|
||||
|
||||
spawnedChildren.at(-1)?.kill('SIGKILL')
|
||||
|
||||
await until(() => (hasLiveClaudePtys() ? null : true), 'the auth gate to drain')
|
||||
expect(hasLiveClaudePtys()).toBe(false)
|
||||
}, 30_000)
|
||||
|
||||
// The gate entry is deliberately unpersisted, so confirmSeededClaudeLivePtys can never
|
||||
// reconcile a stray one: a leak here defers the managed OAuth refresh for the life of
|
||||
// the process. Entering the gate only after the release handlers are attached makes
|
||||
// that unreachable regardless of what the setup in between does.
|
||||
it('leaks no gate entry when setup throws between spawn and handler attachment', async () => {
|
||||
await until(() => (hasLiveClaudePtys() ? null : true), 'a drained auth gate')
|
||||
const scenario = scriptScenario([
|
||||
{ emit: { type: 'system', subtype: 'init', session_id: SESSION_ID, uuid: 'init-1' } },
|
||||
{ wait: HOLD_OPEN }
|
||||
])
|
||||
let started: SpawnedProcess | null = null
|
||||
|
||||
try {
|
||||
await expect(
|
||||
openClaudeStreamJsonConnection(launchFor(scenario), {}, (spec) => {
|
||||
const child = spawnProcess(spec)
|
||||
started = child
|
||||
const attach = child.stderr.on.bind(child.stderr)
|
||||
// Measured attach order: the SDK binds stderr 'data' from inside query(),
|
||||
// before the child is even assigned. The SECOND bind is this connection's own
|
||||
// armTreeOnOutput — the first statement that runs after the child exists and
|
||||
// before its 'exit'/'close' release handlers. Throwing on the first is
|
||||
// vacuous: it escapes before any gate entry could have happened.
|
||||
let dataAttaches = 0
|
||||
child.stderr.on = ((event: string, listener: (...args: unknown[]) => void) => {
|
||||
if (event === 'data') {
|
||||
dataAttaches += 1
|
||||
if (dataAttaches === 2) {
|
||||
throw new Error('stderr listener attach failed')
|
||||
}
|
||||
}
|
||||
return attach(event, listener)
|
||||
}) as typeof child.stderr.on
|
||||
return child
|
||||
})
|
||||
).rejects.toThrow('stderr listener attach failed')
|
||||
|
||||
expect(hasLiveClaudePtys()).toBe(false)
|
||||
} finally {
|
||||
;(started as SpawnedProcess | null)?.kill('SIGKILL')
|
||||
}
|
||||
}, 30_000)
|
||||
})
|
||||
@@ -0,0 +1,283 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import type * as ClaudeAgentSdk from '@anthropic-ai/claude-agent-sdk'
|
||||
import type { CanUseTool, OnUserDialog, SDKUserMessage } from '@anthropic-ai/claude-agent-sdk'
|
||||
import { spawnProcess } from '../../shared/child-process/run-process'
|
||||
import {
|
||||
markClaudeStructuredChildExited,
|
||||
markClaudeStructuredChildSpawned
|
||||
} from '../claude-accounts/live-pty-gate'
|
||||
import { buildClaudeChildProcessEnv } from './claude-child-process-environment'
|
||||
import {
|
||||
ClaudeControlRequestError,
|
||||
createClaudeControlSurface,
|
||||
type ClaudeControlSurface
|
||||
} from './claude-agent-sdk-control-requests'
|
||||
import { createClaudeChildTreeReaper, proveClaudeChildExit } from './claude-agent-sdk-exit-proof'
|
||||
import type { DescendantTreeVerdict } from '../pty-descendant-exit-verification'
|
||||
import { createClaudeCodeProcessSpawn } from './claude-agent-sdk-process-spawn'
|
||||
import { createClaudeUserMessageQueue } from './claude-agent-sdk-user-message-queue'
|
||||
import type { ClaudeStructuredSdkOptions } from './claude-structured-launch-resolution'
|
||||
|
||||
export { ClaudeControlRequestError }
|
||||
|
||||
/**
|
||||
* The SDK is loaded at the structured-Claude boundary rather than by this module's
|
||||
* import. The ordinary runtime's class graph statically reaches this file, and the
|
||||
* SDK sets `process.env.NoDefaultCurrentDirectoryInExePath` at import time — a
|
||||
* Windows executable-search change that a user who never leaves the terminal/TUI
|
||||
* path never opted into, and a missing SDK would fail runtime startup. Memoized,
|
||||
* so a session pays the import once per process rather than once per connection.
|
||||
*/
|
||||
let claudeAgentSdk: Promise<typeof ClaudeAgentSdk> | null = null
|
||||
|
||||
function loadClaudeAgentSdk(): Promise<typeof ClaudeAgentSdk> {
|
||||
claudeAgentSdk ??= import('@anthropic-ai/claude-agent-sdk')
|
||||
return claudeAgentSdk
|
||||
}
|
||||
|
||||
export type ClaudeStreamJsonLaunch = {
|
||||
/** Orca's resolved user CLI; the SDK falls back to a bundled binary that is not installed. */
|
||||
pathToClaudeCodeExecutable: string
|
||||
options: ClaudeStructuredSdkOptions
|
||||
cwd: string
|
||||
env?: Record<string, string>
|
||||
}
|
||||
|
||||
export type ClaudeStreamJsonConnectionHandlers = {
|
||||
onMessage?: (message: Record<string, unknown>) => void
|
||||
/**
|
||||
* The SDK owns inbound permission control: it hands `can_use_tool` to this callback with
|
||||
* a stable requestId and an abort signal, dedups duplicate delivery, and matches the
|
||||
* response by request_id itself. Setting it makes the SDK pass `--permission-prompt-tool
|
||||
* stdio` automatically; it must not be paired with `permissionPromptToolName`.
|
||||
*/
|
||||
canUseTool?: CanUseTool
|
||||
/** `request_user_dialog` control; the CLI only emits kinds declared in `supportedDialogKinds`. */
|
||||
onUserDialog?: OnUserDialog
|
||||
/** A transport/process fault that is not itself first-hand root exit proof. */
|
||||
onFault?: (error: Error) => void
|
||||
onExit?: (error: Error) => void
|
||||
}
|
||||
|
||||
/**
|
||||
* Two questions with their own evidence. The root's verdict is first-hand: Orca's
|
||||
* own child handle reported exit, or reported error then close before it ever had
|
||||
* a pid. The tree's comes from bounded descendant verification, and `unverifiable`
|
||||
* is never collapsed into either neighbour.
|
||||
*/
|
||||
export type ClaudeChildExitVerdict = {
|
||||
root: 'exited' | 'live' | 'processless'
|
||||
tree: DescendantTreeVerdict
|
||||
}
|
||||
|
||||
export type ClaudeStreamJsonConnection = ClaudeControlSurface & {
|
||||
readonly pid: number | undefined
|
||||
readonly closed: boolean
|
||||
/** What the ladder has observed so far; read after a `close()` that returned false. */
|
||||
readonly exitVerdict: ClaudeChildExitVerdict
|
||||
send: (message: Record<string, unknown>) => Promise<void>
|
||||
/** Resolves true after processless settlement, or root exit plus observed tree exit. */
|
||||
close: () => Promise<boolean>
|
||||
}
|
||||
|
||||
type ExitStatus = { code: number | null; signal: NodeJS.Signals | null }
|
||||
|
||||
function exitError(stderrTail: string, status: ExitStatus | null, cause?: Error): Error {
|
||||
const detail = stderrTail.trim()
|
||||
// The status is the diagnostic a signed-out or refused start leaves behind;
|
||||
// it has to survive every wrapper between here and the user.
|
||||
const how =
|
||||
status?.signal !== null && status?.signal !== undefined
|
||||
? ` (signal ${status.signal})`
|
||||
: status?.code !== null && status?.code !== undefined
|
||||
? ` (code ${status.code})`
|
||||
: ''
|
||||
const message = `claude stream-json exited${how}${detail ? `: ${detail}` : ''}`
|
||||
return cause ? new Error(message, { cause }) : new Error(message)
|
||||
}
|
||||
|
||||
export async function openClaudeStreamJsonConnection(
|
||||
launch: ClaudeStreamJsonLaunch,
|
||||
handlers: ClaudeStreamJsonConnectionHandlers = {},
|
||||
spawnImpl: typeof spawnProcess = spawnProcess,
|
||||
queryImpl?: typeof ClaudeAgentSdk.query
|
||||
): Promise<ClaudeStreamJsonConnection> {
|
||||
const { query } = await loadClaudeAgentSdk()
|
||||
const spawner = createClaudeCodeProcessSpawn(spawnImpl)
|
||||
const inbox = createClaudeUserMessageQueue()
|
||||
const session = (queryImpl ?? query)({
|
||||
prompt: inbox.messages,
|
||||
options: {
|
||||
...launch.options,
|
||||
cwd: launch.cwd,
|
||||
// Why env is never omitted: the SDK inherits process.env when it is, which is
|
||||
// exactly the ambient ANTHROPIC_* auth leak this lane already shipped once.
|
||||
env: buildClaudeChildProcessEnv(launch.env, { scrubConfiguredChildSessionStamps: true }),
|
||||
pathToClaudeCodeExecutable: launch.pathToClaudeCodeExecutable,
|
||||
spawnClaudeCodeProcess: spawner.spawn,
|
||||
...(handlers.canUseTool ? { canUseTool: handlers.canUseTool } : {}),
|
||||
...(handlers.onUserDialog ? { onUserDialog: handlers.onUserDialog } : {})
|
||||
}
|
||||
})
|
||||
const child = spawner.child
|
||||
if (!child) {
|
||||
throw new Error('the claude agent SDK returned without spawning a child')
|
||||
}
|
||||
// This child owns the account's credentials for as long as it runs, exactly as a
|
||||
// Claude PTY does — hold the OAuth-refresh gate so a managed refresh cannot rotate
|
||||
// the single-use token out from under it mid-turn. Entered below, once a release
|
||||
// path exists.
|
||||
const authGateKey = randomUUID()
|
||||
const releaseAuthGate = (): void => markClaudeStructuredChildExited(authGateKey)
|
||||
let exited = false
|
||||
let exitStatus: ExitStatus | null = null
|
||||
let closing = false
|
||||
let processless = false
|
||||
let prePidSpawnError = false
|
||||
let terminalError: Error | null = null
|
||||
let faultReported = false
|
||||
let exitReported = false
|
||||
let closePromise: Promise<boolean> | null = null
|
||||
// One reaper per child: every close attempt and error-path reap shares its proof.
|
||||
const rootSettled = (): boolean => exited || processless
|
||||
const tree = createClaudeChildTreeReaper(child, { exited: rootSettled })
|
||||
|
||||
// Arm lazily on actual child output instead of issuing a process-table scan for
|
||||
// every session at startup. A natural SDK exit can race a later close, while
|
||||
// output-triggered observation still catches the usual live-child window.
|
||||
let outputObservationArmed = false
|
||||
const armTreeOnOutput = (): void => {
|
||||
if (outputObservationArmed) {
|
||||
return
|
||||
}
|
||||
outputObservationArmed = true
|
||||
void (tree.refresh?.() ?? tree.capture())
|
||||
}
|
||||
child.stderr.on('data', armTreeOnOutput)
|
||||
// The SDK may synchronously spawn the CLI and consume an early stderr chunk
|
||||
// before this connection can attach its listener; the bounded tail preserves
|
||||
// that observation for the same lazy arm.
|
||||
if (spawner.stderrTail.length > 0) {
|
||||
armTreeOnOutput()
|
||||
}
|
||||
|
||||
let settleExit = (): void => {}
|
||||
const exitPromise = new Promise<void>((resolve) => {
|
||||
settleExit = resolve
|
||||
})
|
||||
const markExited = (): void => {
|
||||
exited = true
|
||||
releaseAuthGate()
|
||||
settleExit()
|
||||
}
|
||||
child.on('exit', (code, signal) => {
|
||||
exitStatus = { code, signal }
|
||||
markExited()
|
||||
handleUnexpectedEnd()
|
||||
})
|
||||
|
||||
const handleUnexpectedEnd = (cause?: Error): void => {
|
||||
terminalError ??= exitError(spawner.stderrTail, exitStatus, cause)
|
||||
inbox.fail(terminalError)
|
||||
if (!closing && !faultReported) {
|
||||
faultReported = true
|
||||
handlers.onFault?.(terminalError)
|
||||
}
|
||||
if (!closing && exited && !exitReported) {
|
||||
exitReported = true
|
||||
handlers.onExit?.(terminalError)
|
||||
}
|
||||
}
|
||||
|
||||
void (async () => {
|
||||
for await (const message of session) {
|
||||
handlers.onMessage?.(message as unknown as Record<string, unknown>)
|
||||
}
|
||||
})().catch((error: unknown) => {
|
||||
// The SDK ends its generator in error when the child dies or the transport
|
||||
// fails; a transport failure with a live child still has to reap the tree.
|
||||
if (!closing && !exited) {
|
||||
void tree.reap()
|
||||
}
|
||||
handleUnexpectedEnd(error instanceof Error ? error : new Error(String(error)))
|
||||
})
|
||||
|
||||
child.on('error', (error) => {
|
||||
if (spawner.pid === undefined) {
|
||||
prePidSpawnError = true
|
||||
}
|
||||
if (!closing && !exited) {
|
||||
void tree.reap()
|
||||
}
|
||||
handleUnexpectedEnd(error)
|
||||
})
|
||||
child.on('close', () => {
|
||||
// Covers the spawn-failure path too, where no 'exit' ever arrives.
|
||||
releaseAuthGate()
|
||||
if (prePidSpawnError && spawner.pid === undefined) {
|
||||
processless = true
|
||||
settleExit()
|
||||
}
|
||||
handleUnexpectedEnd()
|
||||
})
|
||||
child.stdin.on('error', (error) => {
|
||||
if (!closing) {
|
||||
void tree.reap()
|
||||
handleUnexpectedEnd(error)
|
||||
}
|
||||
})
|
||||
// Why here and not at spawn: a structured gate entry is deliberately unpersisted, so
|
||||
// confirmSeededClaudeLivePtys can never reconcile a stray one and a leak defers the
|
||||
// managed OAuth refresh for the life of the process. Entering only after 'exit' and
|
||||
// 'close' are attached makes that unreachable — any later throw still leaves a
|
||||
// listener that releases. Nothing between spawn and here can yield, so the child
|
||||
// cannot end before the gate is entered.
|
||||
markClaudeStructuredChildSpawned(authGateKey)
|
||||
|
||||
const send = (message: Record<string, unknown>): Promise<void> => {
|
||||
if (closing || exited || terminalError || child.stdin.destroyed || !child.stdin.writable) {
|
||||
return Promise.reject(terminalError ?? new Error('claude stream-json connection is closed'))
|
||||
}
|
||||
return inbox.push(message as unknown as SDKUserMessage)
|
||||
}
|
||||
|
||||
const close = (): Promise<boolean> => {
|
||||
closePromise ??= (async () => {
|
||||
closing = true
|
||||
// Arm the descendant proof before ending stdin. The SDK may exit the root
|
||||
// immediately; a post-exit walk cannot recover descendants that reparented.
|
||||
await (tree.refresh?.() ?? tree.capture())
|
||||
inbox.end()
|
||||
const proven = await proveClaudeChildExit({
|
||||
child,
|
||||
exitPromise,
|
||||
exited: rootSettled,
|
||||
tree
|
||||
})
|
||||
inbox.fail(new Error('claude stream-json connection closed'))
|
||||
if (!proven) {
|
||||
closePromise = null
|
||||
}
|
||||
return proven
|
||||
})()
|
||||
return closePromise
|
||||
}
|
||||
|
||||
return {
|
||||
...createClaudeControlSurface(session),
|
||||
get pid() {
|
||||
return spawner.pid
|
||||
},
|
||||
get closed() {
|
||||
return closing || exited || terminalError !== null
|
||||
},
|
||||
get exitVerdict() {
|
||||
return {
|
||||
root: processless ? 'processless' : exited ? 'exited' : 'live',
|
||||
tree: tree.treeVerdict
|
||||
} as const
|
||||
},
|
||||
send,
|
||||
close
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
import type { AgentJournalItemIdentity } from '../../shared/agent-session-journal-types'
|
||||
import { claudeRecord, claudeText } from './claude-structured-item-translation'
|
||||
|
||||
// Under --include-partial-messages every stream_event frame carries its own
|
||||
// uuid, and the block's final `assistant` frame carries yet another; only
|
||||
// `message.id` ties them together. The block's first stream frame mints the
|
||||
// journal identity, and the final frame lands on it in block order instead of
|
||||
// appending a duplicate under its own uuid.
|
||||
|
||||
export type ClaudeStreamedTextDelta = { identity: AgentJournalItemIdentity; text: string }
|
||||
|
||||
type StreamedMessage = {
|
||||
messageId: string | null
|
||||
blocks: Map<number, AgentJournalItemIdentity>
|
||||
/** Streamed text blocks whose final assistant frame has not arrived, in block order. */
|
||||
awaitingFinal: AgentJournalItemIdentity[]
|
||||
}
|
||||
|
||||
export type ClaudeStreamedBlockRegistry = {
|
||||
/** Text a stream_event frame appends to its block, or null when it carries none. */
|
||||
observe: (frame: Record<string, unknown>) => ClaudeStreamedTextDelta | null
|
||||
/** The streamed identity a final assistant frame reconciles onto, if its block streamed. */
|
||||
reconcile: (frame: {
|
||||
sessionId: string
|
||||
parentToolUseId: string | null
|
||||
messageId: string | null
|
||||
}) => AgentJournalItemIdentity | null
|
||||
clear: () => void
|
||||
}
|
||||
|
||||
function scopeKey(sessionId: string, parentToolUseId: string | null): string {
|
||||
return `${sessionId}/${parentToolUseId ?? ''}`
|
||||
}
|
||||
|
||||
export function createClaudeStreamedBlockRegistry(): ClaudeStreamedBlockRegistry {
|
||||
const messages = new Map<string, StreamedMessage>()
|
||||
|
||||
const messageFor = (scope: string): StreamedMessage => {
|
||||
let streamed = messages.get(scope)
|
||||
if (!streamed) {
|
||||
streamed = { messageId: null, blocks: new Map(), awaitingFinal: [] }
|
||||
messages.set(scope, streamed)
|
||||
}
|
||||
return streamed
|
||||
}
|
||||
|
||||
const mint = (
|
||||
streamed: StreamedMessage,
|
||||
sessionId: string,
|
||||
index: number,
|
||||
uuid: string
|
||||
): AgentJournalItemIdentity => {
|
||||
const identity: AgentJournalItemIdentity = { provider: 'claude', sessionId, uuid }
|
||||
streamed.blocks.set(index, identity)
|
||||
streamed.awaitingFinal.push(identity)
|
||||
return identity
|
||||
}
|
||||
|
||||
return {
|
||||
observe: (frame) => {
|
||||
const event = claudeRecord(frame.event)
|
||||
const sessionId = claudeText(frame.session_id)
|
||||
const uuid = claudeText(frame.uuid)
|
||||
if (frame.type !== 'stream_event' || !event || !sessionId || !uuid) {
|
||||
return null
|
||||
}
|
||||
const scope = scopeKey(sessionId, claudeText(frame.parent_tool_use_id))
|
||||
if (event.type === 'message_start') {
|
||||
messages.set(scope, {
|
||||
messageId: claudeText(claudeRecord(event.message)?.id),
|
||||
blocks: new Map(),
|
||||
awaitingFinal: []
|
||||
})
|
||||
return null
|
||||
}
|
||||
const index = typeof event.index === 'number' ? event.index : 0
|
||||
if (event.type === 'content_block_start') {
|
||||
const block = claudeRecord(event.content_block)
|
||||
if (block?.type !== 'text') {
|
||||
return null
|
||||
}
|
||||
const identity = mint(messageFor(scope), sessionId, index, uuid)
|
||||
const text = claudeText(block.text)
|
||||
return text ? { identity, text } : null
|
||||
}
|
||||
if (event.type !== 'content_block_delta') {
|
||||
return null
|
||||
}
|
||||
const delta = claudeRecord(event.delta)
|
||||
const text = delta?.type === 'text_delta' ? claudeText(delta.text) : null
|
||||
if (!text) {
|
||||
return null
|
||||
}
|
||||
const streamed = messageFor(scope)
|
||||
const identity = streamed.blocks.get(index) ?? mint(streamed, sessionId, index, uuid)
|
||||
return { identity, text }
|
||||
},
|
||||
reconcile: (frame) => {
|
||||
const streamed = messages.get(scopeKey(frame.sessionId, frame.parentToolUseId))
|
||||
if (
|
||||
!streamed ||
|
||||
(frame.messageId && streamed.messageId && frame.messageId !== streamed.messageId)
|
||||
) {
|
||||
return null
|
||||
}
|
||||
return streamed.awaitingFinal.shift() ?? null
|
||||
},
|
||||
clear: () => messages.clear()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import type { AgentJournalItemIdentity } from '../../shared/agent-session-journal-types'
|
||||
import { createClaudeStreamedTextCheckpoints } from './claude-streamed-text-checkpoints'
|
||||
|
||||
function identityOf(uuid: string): AgentJournalItemIdentity {
|
||||
return { provider: 'claude', sessionId: 'claude-session', uuid }
|
||||
}
|
||||
|
||||
function checkpoints() {
|
||||
const rows: { uuid: string; text: string }[] = []
|
||||
let scheduled: (() => void) | null = null
|
||||
const store = createClaudeStreamedTextCheckpoints({
|
||||
persist: (identity, text) => {
|
||||
rows.push({ uuid: 'uuid' in identity ? identity.uuid : '', text })
|
||||
},
|
||||
schedule: (run) => {
|
||||
scheduled = run
|
||||
return () => {
|
||||
scheduled = null
|
||||
}
|
||||
}
|
||||
})
|
||||
return {
|
||||
store,
|
||||
rows,
|
||||
runWindow: () => {
|
||||
const run = scheduled as (() => void) | null
|
||||
run?.()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
describe('claude streamed text checkpoints', () => {
|
||||
it('rewrites a block row with the full text accumulated so far', () => {
|
||||
const { store, rows, runWindow } = checkpoints()
|
||||
|
||||
store.append(identityOf('block-1'), 'hel')
|
||||
store.append(identityOf('block-1'), 'lo')
|
||||
runWindow()
|
||||
|
||||
expect(rows).toEqual([{ uuid: 'block-1', text: 'hello' }])
|
||||
expect(store.pending).toBe(1)
|
||||
})
|
||||
|
||||
it('drops every block still awaiting its final frame at settlement', () => {
|
||||
const { store, rows, runWindow } = checkpoints()
|
||||
|
||||
store.append(identityOf('block-1'), 'partial answer')
|
||||
runWindow()
|
||||
store.settle()
|
||||
|
||||
expect(store.pending).toBe(0)
|
||||
// The row written before settlement stays; nothing is rewritten afterwards.
|
||||
store.flush()
|
||||
expect(rows).toEqual([{ uuid: 'block-1', text: 'partial answer' }])
|
||||
})
|
||||
|
||||
it('keeps a block whose final frame arrived out of the settlement sweep', () => {
|
||||
const { store } = checkpoints()
|
||||
|
||||
store.append(identityOf('block-1'), 'one')
|
||||
store.append(identityOf('block-2'), 'two')
|
||||
store.forget('claude:claude-session:block-1')
|
||||
|
||||
expect(store.pending).toBe(1)
|
||||
store.settle()
|
||||
expect(store.pending).toBe(0)
|
||||
})
|
||||
|
||||
it('flushes text the widening checkpoint interval has not written yet', () => {
|
||||
const { store, rows } = checkpoints()
|
||||
|
||||
store.append(identityOf('block-1'), 'x')
|
||||
store.flush()
|
||||
|
||||
expect(rows).toEqual([{ uuid: 'block-1', text: 'x' }])
|
||||
// Already at the row's length: a second flush has nothing to write.
|
||||
store.flush()
|
||||
expect(rows).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('stops persisting once disposed', () => {
|
||||
const { store, rows, runWindow } = checkpoints()
|
||||
|
||||
store.append(identityOf('block-1'), 'text')
|
||||
store.dispose()
|
||||
runWindow()
|
||||
store.flush()
|
||||
|
||||
expect(rows).toEqual([])
|
||||
expect(store.pending).toBe(0)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,105 @@
|
||||
import type { AgentJournalItemIdentity } from '../../shared/agent-session-journal-types'
|
||||
import { agentJournalItemKey } from '../../shared/agent-session-journal-item-key'
|
||||
import {
|
||||
createAgentSessionDeltaCoalescer,
|
||||
type AgentSessionDeltaCoalescerDeps
|
||||
} from '../native-chat/agent-session-wire/agent-session-delta-coalescer'
|
||||
|
||||
export type ClaudeStreamedTextCheckpointDeps = {
|
||||
/** Rewrites the block's journal row with the text accumulated so far. */
|
||||
persist: (identity: AgentJournalItemIdentity, text: string) => void
|
||||
coalesceMs?: number
|
||||
schedule?: AgentSessionDeltaCoalescerDeps['schedule']
|
||||
}
|
||||
|
||||
export type ClaudeStreamedTextCheckpoints = {
|
||||
/** Accumulate a delta; the row is rewritten on the coalescer's own cadence. */
|
||||
append: (identity: AgentJournalItemIdentity, text: string) => void
|
||||
/** Write every block whose row is behind the text received for it. */
|
||||
flush: () => void
|
||||
/** Drop one block's state, for a block whose final frame has now landed. */
|
||||
forget: (key: string) => void
|
||||
/**
|
||||
* Drop every block still awaiting its final frame, at turn settlement. Their
|
||||
* text is already journaled by the flush that precedes settlement; keeping it
|
||||
* live would grow with every interrupted turn for the life of the session.
|
||||
*/
|
||||
settle: () => void
|
||||
/** Blocks still awaiting a final frame. A settled turn must leave none. */
|
||||
readonly pending: number
|
||||
dispose: () => void
|
||||
}
|
||||
|
||||
/**
|
||||
* Growth of a streamed block's row between its deltas and its final frame.
|
||||
*
|
||||
* The row is rewritten on a widening interval rather than per delta: a 200-line
|
||||
* reply would otherwise rewrite the same journal row once per token.
|
||||
*/
|
||||
export function createClaudeStreamedTextCheckpoints(
|
||||
deps: ClaudeStreamedTextCheckpointDeps
|
||||
): ClaudeStreamedTextCheckpoints {
|
||||
const identities = new Map<string, AgentJournalItemIdentity>()
|
||||
const latestText = new Map<string, string>()
|
||||
const checkpointLengths = new Map<string, number>()
|
||||
|
||||
const persist = (key: string, text: string, force: boolean): void => {
|
||||
latestText.set(key, text)
|
||||
const checkpointLength = checkpointLengths.get(key) ?? 0
|
||||
const nextLength = Math.max(checkpointLength + 32, Math.ceil(checkpointLength * 1.125))
|
||||
if (!force && checkpointLength > 0 && text.length < nextLength) {
|
||||
return
|
||||
}
|
||||
const identity = identities.get(key)
|
||||
if (!identity) {
|
||||
return
|
||||
}
|
||||
checkpointLengths.set(key, text.length)
|
||||
deps.persist(identity, text)
|
||||
}
|
||||
|
||||
const coalescer = createAgentSessionDeltaCoalescer({
|
||||
...(deps.coalesceMs === undefined ? {} : { windowMs: deps.coalesceMs }),
|
||||
...(deps.schedule ? { schedule: deps.schedule } : {}),
|
||||
emit: (key, text) => persist(key, text, false)
|
||||
})
|
||||
|
||||
const drop = (key: string): void => {
|
||||
coalescer.forget(key)
|
||||
identities.delete(key)
|
||||
latestText.delete(key)
|
||||
checkpointLengths.delete(key)
|
||||
}
|
||||
|
||||
return {
|
||||
append: (identity, text) => {
|
||||
const key = agentJournalItemKey(identity)
|
||||
identities.set(key, identity)
|
||||
coalescer.append(key, text)
|
||||
},
|
||||
flush: () => {
|
||||
coalescer.flushAll()
|
||||
for (const [key, text] of latestText) {
|
||||
if (checkpointLengths.get(key) !== text.length) {
|
||||
persist(key, text, true)
|
||||
}
|
||||
}
|
||||
},
|
||||
forget: drop,
|
||||
settle: () => {
|
||||
// Map iteration tolerates deletion of the entry just visited.
|
||||
for (const key of identities.keys()) {
|
||||
drop(key)
|
||||
}
|
||||
},
|
||||
get pending() {
|
||||
return identities.size
|
||||
},
|
||||
dispose: () => {
|
||||
coalescer.dispose()
|
||||
identities.clear()
|
||||
latestText.clear()
|
||||
checkpointLengths.clear()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import {
|
||||
closeClaudeSession,
|
||||
claudeAcquisitionCleanupError
|
||||
} from './claude-structured-session-close'
|
||||
import type {
|
||||
ClaudeAcquisitionRegistry,
|
||||
ClaudeSession,
|
||||
ClaudeSessionExit,
|
||||
ClaudeStructuredSessionAdapterDeps
|
||||
} from './claude-structured-session-state'
|
||||
|
||||
/**
|
||||
* Cleanup for an acquisition the host could not commit or prove. A session that
|
||||
* a first-hand exit already removed is not an absence to report as proven: the
|
||||
* ladder on its connection still answers, and that answer is classified exactly
|
||||
* as a start-time failure would be.
|
||||
*/
|
||||
export async function releaseClaudeAcquisition(input: {
|
||||
sessionId: string
|
||||
sessions: Map<string, ClaudeSession>
|
||||
acquisitions: ClaudeAcquisitionRegistry
|
||||
exits: Map<string, ClaudeSessionExit>
|
||||
onExitProven?: (sessionId: string, exit: ClaudeSessionExit) => Promise<void>
|
||||
persistHandle?: ClaudeStructuredSessionAdapterDeps['persistHandle']
|
||||
onEvent?: ClaudeStructuredSessionAdapterDeps['onEvent']
|
||||
}): Promise<boolean> {
|
||||
const exit = input.exits.get(input.sessionId)
|
||||
if (!exit || input.sessions.has(input.sessionId) || input.acquisitions.get(input.sessionId)) {
|
||||
return closeClaudeSession(input)
|
||||
}
|
||||
const firstProof = exit.closePromise ? await exit.closePromise : false
|
||||
// A failed exit-path proof is retained as evidence, not as a terminal result;
|
||||
// a release retry must drive a fresh tree verification on the same connection.
|
||||
const retriedProof = firstProof || (await exit.connection.close())
|
||||
if (retriedProof) {
|
||||
await input.onExitProven?.(input.sessionId, exit)
|
||||
// Keep the first-hand exit evidence indexed until the tree proof succeeds;
|
||||
// a failed close must be retryable and cannot look like an absent session.
|
||||
input.exits.delete(input.sessionId)
|
||||
return true
|
||||
}
|
||||
throw claudeAcquisitionCleanupError(exit.connection, exit.error)
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user