Merge remote-tracking branch 'origin/main' into mobile-rearch

# Conflicts:
#	mobile/src/session/use-mobile-native-chat-image-attachments.ts
This commit is contained in:
Jinwoo-H
2026-09-04 18:41:52 -04:00
452 changed files with 19201 additions and 6538 deletions
+2 -2
View File
@@ -238,9 +238,9 @@ Pair with your desktop app to monitor and steer your agents from your phone.
- **Discord:** Join the community on **[Discord](https://discord.gg/fzjDKHxv8Q)**.
- **Twitter / X:** Follow **[@orca_build](https://x.com/orca_build)** for updates and announcements.
- **WeChat:** Scan to join the Orca community WeChat group 8.
- **WeChat:** Scan to join the Orca community WeChat group 8. Group 8 may be full; if so, scan the Group 9 QR code instead.
<img src="docs/assets/wechat-qr-group8.jpg" alt="WeChat group 8 QR code for the Orca community" width="160" />
<img src="docs/assets/wechat-qr-group8.jpg" alt="WeChat group 8 QR code for the Orca community" width="160" />&nbsp;&nbsp;<img src="docs/assets/wechat-qr-group9.jpg" alt="WeChat group 9 QR code for the Orca community" width="160" />
- **Feedback &amp; Ideas:** We ship fast. Missing something? [Request a new feature](https://github.com/stablyai/orca/issues).
- **Privacy:** See the [privacy &amp; telemetry docs](https://www.onorca.dev/docs/telemetry) for what anonymous usage data Orca collects and how to opt out.
@@ -111,14 +111,19 @@ describe('incident monitor evaluator', () => {
})
})
it('freezes when postgres retries exceed the recalibrated ceiling', () => {
const sample = healthySample()
sample.sources['relay-logs']!.signals['relay.postgres_retries'] =
signal(INCIDENT_MONITOR_THRESHOLDS.relayPostgresRetries + 1)
expect(evaluateIncidentSample(sample, startedAt)).toMatchObject({
// Why: the global relay_cells lock made retries a steady-state rate (24 h p99
// 1320/5min on 2026-09-04); the bar fences only unbounded growth beyond that.
it('tolerates the measured healthy retry rate and freezes above the bar', () => {
const healthy = healthySample()
healthy.sources['relay-logs']!.signals['relay.postgres_retries'] = signal(1504)
expect(evaluateIncidentSample(healthy, startedAt).status).toBe('green')
const incident = healthySample()
incident.sources['relay-logs']!.signals['relay.postgres_retries'] = signal(2001)
expect(evaluateIncidentSample(incident, startedAt)).toMatchObject({
status: 'freeze',
failures: [
expect.objectContaining({ signal: 'relay.postgres_retries', threshold: 300 })
expect.objectContaining({ signal: 'relay.postgres_retries', threshold: 2000 })
]
})
})
+15 -6
View File
@@ -32,11 +32,20 @@ export const INCIDENT_MONITOR_THRESHOLDS = {
relayPoolWaiting: 800,
relayPoolWaitMs: 2_500,
// Why: successful lock retries are the contention machinery working, not harm.
// Healthy 2026-08-26 baseline bursts to 234/5min (26% of windows crossed the old
// bar of 20, set unmeasured at the monitor's 2026-07-28 birth); the 2026-08-23
// incident ran ~2,200-3,000/5min. 300 clears healthy bursts with ~10x incident
// margin; relayPostgresRetryExhausted below bounds the terminally failed share.
relayPostgresRetries: 300,
// Recalibrated 2026-09-04 from 300, which was set 2026-08-26 when healthy bursts
// reached 234/5min. The global relay_cells FOR UPDATE lock has since become the
// fleet's steady state: measured fleet-wide (director + cells, summed per five
// minutes) 2026-09-03T05Z..2026-09-04T05Z p50 430 / p90 924 / p99 1320 / max
// 1504, with 55% of windows over 300 and only 22% of 15-minute gates clean, so
// the bar blocked the very cell roll that carries the 500 ms lock wait (#18521)
// and the beginProof crash guard to the cells. The 2026-08-23 lock incident on
// this same metric peaked at 1510 in one window and 646 in the next, so it is
// not separable from today's contention by retries alone; it is caught by
// relayPostgresRetryExhausted (467 at the peak vs a 300 bar), director
// concurrency, and the pool bars. 2000 passes every healthy 15-minute window
// measured in the last 24 h and still fences unbounded growth. Re-tighten once
// the fleet is on the 500 ms lock wait and the baseline is re-measured.
relayPostgresRetries: 2000,
// Why: 300 per five minutes, recalibrated 2026-09-04 from a bar of zero that no
// production window has cleared since #18521 shipped to the director. That
// change cut the request-path cell-inventory wait from the 1 s pool lock_timeout
@@ -48,7 +57,7 @@ export const INCIDENT_MONITOR_THRESHOLDS = {
// quiet hours p50 2 / max 36; pre-#18521 daytime p50 10 / p90 25 / max 87;
// post-#18521 p50 42 / p90 147 / max 220. The 2026-08-23 lock incident peaked
// at 467. 300 clears every measured healthy window and still sits below the
// incident shape; relayPostgresRetries above stays the ~10x discriminator.
// incident shape; retries above fence only unbounded growth.
// User-facing /v1/assign 503 share did not move with #18521 (13.9% old image
// vs 12.3% new, same evening), so exhaustion is not a proxy for user harm.
relayPostgresRetryExhausted: 300,
@@ -23,8 +23,10 @@ describe('readResourceInventory', () => {
calls += 1
return new Response(null, { status: 200 })
},
async () => {
waits += 1
{
wait: async () => {
waits += 1
}
}
)
@@ -45,8 +47,10 @@ describe('readResourceInventory', () => {
calls.set(path, call)
return new Response(null, { status: path === '/ready' && call === 1 ? 503 : 200 })
},
async (ms) => {
waits.push(ms)
{
wait: async (ms) => {
waits.push(ms)
}
}
)
@@ -65,17 +69,130 @@ describe('readResourceInventory', () => {
calls += 1
return new Response(null, { status: 503 })
},
async (ms) => {
waits.push(ms)
{
wait: async (ms) => {
waits.push(ms)
}
}
)
expect(result.health).toBe(false)
expect(result.ready).toBe(false)
expect(calls).toBe(4)
// A refusing endpoint is a reading, so only the independent retry runs.
expect(waits).toEqual([11_000])
})
it('treats a thrown fetch as no reading and re-asks that path once', async () => {
const calls: string[] = []
const waits: number[] = []
const result = await probeEndpointHealth(
'https://c9.relay.onorca.dev',
async (input) => {
const path = new URL(String(input)).pathname
calls.push(path)
if (path === '/health' && calls.filter((call) => call === '/health').length === 1) {
throw new TypeError('fetch failed')
}
return new Response(null, { status: 200 })
},
{
wait: async (ms) => {
waits.push(ms)
}
}
)
expect(result.health).toBe(true)
expect(result.ready).toBe(true)
expect(calls.filter((call) => call === '/health')).toEqual(['/health', '/health'])
expect(waits).toEqual([1_000])
})
it('fails closed when both attempts of a path throw', async () => {
const calls: string[] = []
const waits: number[] = []
const result = await probeEndpointHealth(
'https://c9.relay.onorca.dev',
async (input) => {
const path = new URL(String(input)).pathname
calls.push(path)
if (path === '/health') throw new TypeError('fetch failed')
return new Response(null, { status: 200 })
},
{
wait: async (ms) => {
waits.push(ms)
}
}
)
expect(result.health).toBe(false)
expect(calls.filter((call) => call === '/health')).toHaveLength(4)
expect(waits).toEqual([1_000, 11_000, 1_000])
})
it('accepts an auth-shaped endpoint that serves no readiness path', async () => {
const calls: string[] = []
const waits: number[] = []
const result = await probeEndpointHealth(
'https://login.onorca.dev',
async (input) => {
const path = new URL(String(input)).pathname
calls.push(path)
return new Response(null, { status: path === '/ready' ? 404 : 200 })
},
{
requiresReady: false,
wait: async (ms) => {
waits.push(ms)
}
}
)
expect(result.health).toBe(true)
expect(result.ready).toBeNull()
expect(calls).toEqual(['/health'])
expect(waits).toEqual([])
})
it('still requires readiness for the director and cells', async () => {
const waits: number[] = []
const result = await probeEndpointHealth(
'https://relay.onorca.dev',
async (input) => new Response(null, {
status: new URL(String(input)).pathname === '/ready' ? 503 : 200
}),
{
wait: async (ms) => {
waits.push(ms)
}
}
)
expect(result.health).toBe(true)
expect(result.ready).toBe(false)
expect(waits).toEqual([11_000])
})
it('measures latency as the answering round trip, not the retry delay', async () => {
let healthCalls = 0
const result = await probeEndpointHealth(
'https://c9.relay.onorca.dev',
async (input) => {
if (new URL(String(input)).pathname !== '/health') return new Response(null, { status: 200 })
healthCalls += 1
if (healthCalls === 1) throw new TypeError('fetch failed')
return new Response(null, { status: 200 })
},
{ wait: async (ms) => await new Promise((resolve) => setTimeout(resolve, Math.min(ms, 60))) }
)
expect(result.health).toBe(true)
expect(result.latencyMs).not.toBeNull()
expect(result.latencyMs!).toBeLessThan(60)
})
it('uses aggregate REST inventory without probing sleeping staging endpoints', async () => {
const gcloud: GcloudClient = { accessToken: async () => 'a'.repeat(40) }
let publicProbeCalls = 0
+61 -20
View File
@@ -102,6 +102,7 @@ export type ResourceInventory = {
const unavailableEndpoint = (): EndpointHealth => ({ health: null, ready: null, latencyMs: null })
const independentEndpointRetryDelayMs = 11_000
const transientProbeRetryDelayMs = 1_000
function finalSegment(value: string): string {
return value.split('/').at(-1) ?? value
@@ -138,41 +139,80 @@ async function googleRequest(
return await response.json()
}
async function endpointProbe(origin: string, fetchImpl: typeof fetch): Promise<EndpointHealth> {
const startedAt = performance.now()
const check = async (path: '/health' | '/ready'): Promise<boolean> => {
// A reading the endpoint actually produced: ok is its answer, latencyMs is that answer's round trip.
type PathReading = { ok: boolean; latencyMs: number | null }
async function probePath(
origin: string,
path: '/health' | '/ready',
fetchImpl: typeof fetch,
wait: (ms: number) => Promise<void>
): Promise<PathReading> {
// null means the request never produced an answer (DNS/TCP/TLS failure or the 8s abort).
const attempt = async (): Promise<PathReading | null> => {
const startedAt = performance.now()
try {
const response = await fetchImpl(`${origin}${path}`, {
redirect: 'error',
signal: AbortSignal.timeout(8_000)
})
return response.ok
return { ok: response.ok, latencyMs: Math.round(performance.now() - startedAt) }
} catch {
return false
return null
}
}
const [health, ready] = await Promise.all([check('/health'), check('/ready')])
return { health, ready, latencyMs: Math.round(performance.now() - startedAt) }
const first = await attempt()
if (first) return first
// A thrown fetch is the absence of a reading, not an unhealthy answer, so re-ask before concluding.
await wait(transientProbeRetryDelayMs)
return (await attempt()) ?? { ok: false, latencyMs: null }
}
async function endpointProbe(
origin: string,
fetchImpl: typeof fetch,
requiresReady: boolean,
wait: (ms: number) => Promise<void>
): Promise<EndpointHealth> {
const [health, ready] = await Promise.all([
probePath(origin, '/health', fetchImpl, wait),
requiresReady ? probePath(origin, '/ready', fetchImpl, wait) : null
])
// Latency is the slowest answering round trip in this probe; retry delays are not serving latency.
const latencies = [health.latencyMs, ready?.latencyMs ?? null].filter(
(value): value is number => value !== null
)
return {
health: health.ok,
ready: ready ? ready.ok : null,
latencyMs: latencies.length > 0 ? Math.max(...latencies) : null
}
}
export type EndpointProbeOptions = {
// Auth serves no /ready by design, so it is judged on /health and latency alone.
requiresReady?: boolean
wait?: (ms: number) => Promise<void>
}
export async function probeEndpointHealth(
origin: string,
fetchImpl: typeof fetch,
wait: (ms: number) => Promise<void> = async (ms) =>
await new Promise((resolvePromise) => setTimeout(resolvePromise, ms))
options: EndpointProbeOptions = {}
): Promise<EndpointHealth> {
const first = await endpointProbe(origin, fetchImpl)
if (
first.health &&
first.ready &&
first.latencyMs !== null &&
first.latencyMs <= INCIDENT_MONITOR_THRESHOLDS.endpointLatencyMs
) {
return first
}
const requiresReady = options.requiresReady ?? true
const wait = options.wait ??
(async (ms: number) => await new Promise((resolvePromise) => setTimeout(resolvePromise, ms)))
const accepted = (probe: EndpointHealth): boolean =>
probe.health === true &&
(!requiresReady || probe.ready === true) &&
probe.latencyMs !== null &&
probe.latencyMs <= INCIDENT_MONITOR_THRESHOLDS.endpointLatencyMs
const first = await endpointProbe(origin, fetchImpl, requiresReady, wait)
if (accepted(first)) return first
// Outwait Relay's ten-second readiness cache before treating the retry as independent.
await wait(independentEndpointRetryDelayMs)
return await endpointProbe(origin, fetchImpl)
return await endpointProbe(origin, fetchImpl, requiresReady, wait)
}
function imageDigest(template: z.infer<typeof TemplateSchema>): string | null {
@@ -338,7 +378,8 @@ export async function readResourceInventory(
? [unavailableEndpoint(), unavailableEndpoint()]
: await Promise.all([
probeEndpointHealth(environment.directorOrigin, fetchImpl),
probeEndpointHealth(environment.authOrigin, fetchImpl)
// The auth service exposes no /ready, so requiring it would fail every first probe.
probeEndpointHealth(environment.authOrigin, fetchImpl, { requiresReady: false })
])
const cells = await Promise.all(environment.cells.map((cell, index) =>
readCell(environment, cell, migValues[index] ?? null, token, fetchImpl)
@@ -44,6 +44,12 @@ describePostgres('PostgreSQL assignment connection headroom', () => {
`DELETE FROM relay_assignments
WHERE user_id LIKE 'connection-headroom-postgres-%'`
)
// A snapshot left by an aborted run rejects the replayed watermark
// with stale_connection_snapshot.
await database.query(
`DELETE FROM relay_cell_connection_snapshots WHERE cell_id = ?`,
[cell.id]
)
await database.query(
`DELETE FROM relay_cell_connection_runtime WHERE cell_id = ?`,
[cell.id]
@@ -38,6 +38,10 @@ describePostgres('PostgreSQL control supersession', () => {
[identity.userId]
)
await database.query(`DELETE FROM relay_assignments WHERE user_id = ?`, [identity.userId])
// A snapshot left by an aborted run rejects the replayed watermark with stale_connection_snapshot.
await database.query(`DELETE FROM relay_cell_connection_snapshots WHERE cell_id = ?`, [
cell.id
])
await database.query(`DELETE FROM relay_cell_connection_runtime WHERE cell_id = ?`, [cell.id])
await database.query(`DELETE FROM relay_cell_connection_limits WHERE cell_id = ?`, [cell.id])
await database.query(`DELETE FROM relay_cell_runtime WHERE cell_id = ?`, [cell.id])
+83 -25
View File
@@ -645,9 +645,17 @@ export class RelayAssignmentStore {
): Promise<RelayAssignment | null> {
const now = this.now()
return await this.database.transaction(async (transaction) => {
const lockedCells = inventoryFirst
? await this.lockCellInventory(transaction, lockMode)
// Why: the retry exists to take a cell row before the assignment row, the
// order placement uses. It only ever needs the one cell this host is
// pinned to, so read the pin unlocked and lock that row alone; taking all
// 23 queued every sticky refresh in the fleet behind every other one.
const pinnedCellId = inventoryFirst
? await this.pinnedCellId(transaction, identity)
: undefined
const lockedCells =
pinnedCellId === undefined
? undefined
: await this.lockCellRows(transaction, [pinnedCellId], lockMode)
const existing = await this.assignmentRow(transaction, identity, inventoryFirst)
if (!existing) return null
const activityLeases = await this.lockAssignmentActivities(transaction, identity, true)
@@ -661,6 +669,11 @@ export class RelayAssignmentStore {
}
const currentCellId = text(existing, 'cell_id')
// The pin moved between the unlocked read and the assignment lock, so the
// row held is the wrong one. Same recovery as losing the lock: retry.
if (pinnedCellId !== undefined && pinnedCellId !== currentCellId) {
throw new Error('database_lock_unavailable')
}
const hadControl = holdsControlLease(
activityLeases,
currentCellId,
@@ -701,14 +714,9 @@ export class RelayAssignmentStore {
if (hadControl) {
await this.touchAssignment(transaction, identity, leaseExpiresAt, now)
} else {
const nextReservation = integer(currentRow, 'reserved_requests') + 1
if (nextReservation > integer(currentRow, 'capacity_requests')) {
throw new Error('relay_capacity_exhausted')
}
await transaction.query(
`UPDATE relay_cells SET reserved_requests = ?, updated_at = ? WHERE cell_id = ?`,
[nextReservation, now, currentCellId]
)
// Delta, not the value read from the snapshot: an absolute write here
// would clobber any concurrent movement of the same counter.
await this.adjustCellReservationAtomically(transaction, currentCellId, 1)
await this.adjustActivityCount(transaction, identity, 'control', 1, leaseExpiresAt, now)
await this.insertPendingControlLease(
transaction,
@@ -3202,8 +3210,7 @@ export class RelayAssignmentStore {
)
const requestDelta = ACTIVITY_REQUEST_UNITS[kind] * (after - before)
if (requestDelta !== 0) {
await this.lockCellInventory(transaction, 'request')
await this.adjustCellReservation(transaction, text(row, 'cell_id'), requestDelta)
await this.adjustCellReservationAtomically(transaction, text(row, 'cell_id'), requestDelta)
}
})
})
@@ -3263,9 +3270,12 @@ export class RelayAssignmentStore {
}
const units = ACTIVITY_REQUEST_UNITS[input.kind]
if (existing) {
await this.lockCellInventory(transaction, 'request')
// Why: a client-chosen activity id can move between cells, so lock the
// one or two rows this path touches in cell_id order, the same order
// placement takes the inventory in, and no cycle can form.
await this.lockCellRows(transaction, [text(existing, 'cell_id'), input.cellId])
await this.removeActivityLease(transaction, identity, existing, now)
await this.adjustCellReservation(transaction, input.cellId, units)
await this.adjustCellReservationAtomically(transaction, input.cellId, units)
}
await this.adjustActivityCount(transaction, identity, input.kind, 1, expiresAt, now)
await transaction.query(
@@ -3580,8 +3590,7 @@ export class RelayAssignmentStore {
)
await this.touchAssignment(transaction, identity, expiresAt, now)
} else {
await this.lockCellInventory(transaction, 'request')
await this.adjustCellReservation(transaction, input.cellId, 1)
await this.adjustCellReservationAtomically(transaction, input.cellId, 1)
await this.adjustActivityCount(transaction, identity, 'control', 1, expiresAt, now)
await transaction.query(
`INSERT INTO relay_assignment_activity_leases
@@ -6863,13 +6872,24 @@ export class RelayAssignmentStore {
targetCellId
]
)
const cells = await this.lockCellInventory(transaction, 'pool-default')
// Only the two cells this repairs need holding. The id set below is an
// existence check against a table that only reconcileCells writes, so it
// reads unlocked instead of dragging the other 21 rows into the section.
const cellIds = new Set(
(await transaction.query(`SELECT cell_id FROM relay_cells`)).map((row) =>
text(row, 'cell_id')
)
)
const cells = await this.lockCellRows(
transaction,
[sourceCellId, targetCellId],
'pool-default'
)
const assignmentKeys = new Set(
assignments.map((row) =>
assignmentKey(text(row, 'user_id'), text(row, 'relay_host_id'))
)
)
const cellIds = new Set(cells.map((row) => text(row, 'cell_id')))
const assignmentCounts = new Map<
string,
{ counts: Record<AssignmentActivityKind, number>; leaseExpiresAt: number }
@@ -6923,9 +6943,7 @@ export class RelayAssignmentStore {
)
}
for (const row of cells.filter((cell) =>
[sourceCellId, targetCellId].includes(text(cell, 'cell_id'))
)) {
for (const row of cells) {
const cellId = text(row, 'cell_id')
const expected = cellUnits.get(cellId) ?? 0
if (expected > integer(row, 'capacity_requests')) {
@@ -6954,6 +6972,43 @@ export class RelayAssignmentStore {
return rows
}
// Per-connection paths touch one or two cells. Locking exactly those rows,
// in the same ascending order the inventory lock uses (ORDER BY fixes the
// row-lock order), keeps them off the fleet-wide lock without a cycle.
// The wait policy follows the caller for the same reason the inventory lock's
// does: a sweep must not fail terminally on ordinary contention. Hold time is
// deliberately not sampled here — the metric tracks the fleet-wide lock these
// rows replace, and mixing in short single-row holds would flatter it.
private async lockCellRows(
database: RelayDatabase,
cellIds: string[],
mode: CellInventoryLockMode = 'request'
): Promise<SqlRow[]> {
const distinct = [...new Set(cellIds)]
const { measureHoldMs: _sampled, ...wait } = cellInventoryLockOptions(mode)
return await database.queryLocked(
`SELECT * FROM relay_cells WHERE cell_id IN (${distinct.map(() => '?').join(', ')})
ORDER BY cell_id ASC`,
distinct,
wait
)
}
// Unlocked on purpose: this only names the row to lock next, and the caller
// re-checks the pin once the assignment row is held.
private async pinnedCellId(
database: RelayDatabase,
identity: AssignmentIdentity
): Promise<string | undefined> {
const row = (
await database.query(
`SELECT cell_id FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
)[0]
return row ? text(row, 'cell_id') : undefined
}
private async lockGeneralCellInventory(
database: RelayDatabase,
mode: CellInventoryLockMode
@@ -6972,10 +7027,11 @@ export class RelayAssignmentStore {
private async leastLoadedCell(
database: RelayDatabase,
lockedCells: SqlRow[] | undefined,
// Required: the one caller has already locked the inventory it selects from,
// and an optional parameter left a second fleet-wide lock reachable here.
rows: SqlRow[],
preferredRegion: RelayRegion
): Promise<CellRow | null> {
const rows = lockedCells ?? (await this.lockCellInventory(database, 'pool-default'))
const regions = new Map(
(await database.query(`SELECT cell_id, region FROM relay_cell_regions`)).map((row) => [
text(row, 'cell_id'),
@@ -7590,7 +7646,10 @@ export class RelayAssignmentStore {
) {
throw new Error('activity_lease_shape_mismatch')
}
const cells = await this.lockCellInventory(database, 'request')
// Why: this recomputes one cell's reservation from its leases, so only that
// row needs to be held; the 23-row inventory lock here serialised every
// desktop control rebind in the fleet behind every other one.
const cellRow = (await this.lockCellRows(database, [cellId]))[0]
await database.query(
`DELETE FROM relay_assignment_activity_leases
WHERE user_id = ? AND relay_host_id = ? AND activity_kind = 'control'
@@ -7611,7 +7670,6 @@ export class RelayAssignmentStore {
[cellId]
)
)[0]!
const cellRow = cells.find((cell) => text(cell, 'cell_id') === cellId)
const cellUnits = integer(cellUnitsRow, 'request_units')
if (!cellRow) throw new Error('assigned_cell_missing')
if (cellUnits > integer(cellRow, 'capacity_requests')) {
@@ -18,17 +18,21 @@ type CensusEntry = { method: string; mode: CensusMode; reach: Reachability }
// assignment-store.ts, in source order. A new site fails this test until it is
// classified here, which is the point.
const CENSUS: CensusEntry[] = [
{ method: 'assignStickyOnce', mode: 'caller', reach: 'both' },
// assignStickyOnce is gone from this list: its retry now locks only the row
// the host is pinned to (lockCellRows), which is what a sticky refresh
// touches. Placement below is the one genuinely fleet-wide decision left.
{ method: 'assignOnce', mode: 'caller', reach: 'both' },
{ method: 'assignOnce', mode: 'caller', reach: 'both' },
{ method: 'assignOnce', mode: 'nowait', reach: 'both' },
{ method: 'assignOnce', mode: 'nowait', reach: 'both' },
{ method: 'assignOnce', mode: 'nowait', reach: 'both' },
{ method: 'refreshDrainMigrationLeasesOnce', mode: 'request', reach: 'request' },
// Reachable from neither: changeActivity has no production callers, only tests.
{ method: 'changeActivity', mode: 'request', reach: 'orphan' },
{ method: 'acquireActivity', mode: 'request', reach: 'request' },
{ method: 'activateControl', mode: 'request', reach: 'request' },
// changeActivity, acquireActivity, activateControl and
// removeSupersededSameCellControls no longer take the inventory: they lock
// only the one or two cell rows they touch, in cell_id order (lockCellRows),
// so they cannot cycle with placement's ordered inventory lock, and the
// 23-row lock there had serialised every reconnect in the fleet behind every
// other one.
{ method: 'startEvacuation', mode: 'request', reach: 'request' },
{ method: 'completeEvacuationFromDeadSourceOnce', mode: 'request', reach: 'request' },
{ method: 'completeEvacuationFromDeadSourceOnce', mode: 'nowait', reach: 'request' },
@@ -47,9 +51,33 @@ const CENSUS: CensusEntry[] = [
{ method: 'abortExpiredEvacuations', mode: 'nowait', reach: 'sweep' },
{ method: 'releaseExpiredActivityLeases', mode: 'nowait', reach: 'sweep' },
{ method: 'releaseExpiredActivity', mode: 'nowait', reach: 'sweep' },
{ method: 'reconcileReservationAccounting', mode: 'pool-default', reach: 'both' },
{ method: 'leastLoadedCell', mode: 'pool-default', reach: 'both' },
{ method: 'removeSupersededSameCellControls', mode: 'request', reach: 'request' }
// reconcileReservationAccounting and leastLoadedCell are gone too: the first
// repairs exactly two cells' counters and now holds only those rows, and the
// second selects from the inventory its single caller has already locked.
]
// Every inline `FROM relay_cells ... FOR UPDATE` outside the named lock helpers,
// in source order: whole-table locks in reconciliation and sticky placement,
// and single-row locks for a cell the method is already scoped to (heartbeat,
// fence, drain generation, configuration, or a reservation adjust that runs
// under a lock its caller already holds). A new inline lock fails the census
// below until it is listed here; per-connection paths that touch more than one
// cell go through lockCellRows so the order is fixed.
const NAMED_LOCK_HELPERS = ['lockCellInventory', 'lockGeneralCellInventory', 'lockCellRows']
const INLINE_CELL_LOCK_SITES = [
'reconcileCellsWithOptions',
'assignStickyOnce',
'recordCellHeartbeat',
'attestCellFence',
'adoptLegacyCellFence',
'commitLegacyCellFenceAdoption',
'prepareCellFenceAttempt',
'attestCellFenceAttempt',
'attestCellFenceAttempt',
'configureCell',
'assertDrainCellGeneration',
'adjustCellReservation'
]
// The background sweeps, and nothing else. A method reachable from one of these
@@ -151,6 +179,42 @@ describe('cell inventory lock call-site census', () => {
)
})
// Why: the census only sees lockCellInventory calls, so a hand-written
// `relay_cells ... FOR UPDATE` would escape classification entirely.
it('routes every relay_cells row lock through a named lock helper', () => {
const lines = storeSource()
const rawSites: string[] = []
// Whole statements, not a fixed window: a wide column list or a raw
// FOR UPDATE inside query() must not slip past.
const source = lines.join('\n')
const bounds: { name: string; start: number }[] = []
lines.forEach((line, index) => {
const declaration = DECLARATION.exec(line)
if (declaration) bounds.push({ name: declaration[1]!, start: index })
})
const methodAt = (offset: number): string => {
const lineIndex = source.slice(0, offset).split('\n').length - 1
let name = '<module>'
for (const bound of bounds) if (bound.start <= lineIndex) name = bound.name
return name
}
const tick = String.fromCharCode(96)
const statementCall = new RegExp(
'\\.(queryLocked|query)\\(\\s*' + tick + '([^' + tick + ']*)' + tick,
'g'
)
for (const call of source.matchAll(statementCall)) {
const statement = call[2]!
if (!/\bFROM\s+relay_cells\b/.test(statement)) continue
const locks = call[1] === 'queryLocked' || /\bFOR\s+UPDATE\b/.test(statement)
if (!locks) continue
const method = methodAt(call.index)
if (NAMED_LOCK_HELPERS.includes(method)) continue
rawSites.push(method)
}
expect(rawSites).toEqual(INLINE_CELL_LOCK_SITES)
})
it('leaves no call site taking the inventory without naming a mode', () => {
const source = readFileSync(new URL('./assignment-store.ts', import.meta.url), 'utf8')
const unclassified = source
@@ -0,0 +1,206 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { openRelayDatabase, type RelayDatabase } from './database.js'
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
// Sorted ascending, and the host is pinned to the LAST id on purpose: the
// fleet-wide lock is one ordered scan, so it holds every earlier row while it
// waits on the pinned one. Pinning to the first id would make the two locking
// models indistinguishable.
const cells = ['a', 'b', 'c'].map((suffix) => ({
id: `percell-postgres-${suffix}`,
url: `https://percell-postgres-${suffix}.example.com`,
capacityRequests: 1_000,
connectionHardCap: 600 as const,
connectionUnobservedBound: 50
}))
const [cellA, cellB, cellC] = cells as [(typeof cells)[0], (typeof cells)[0], (typeof cells)[0]]
const identity = { userId: 'percell-postgres-user', relayHostId: 'percellhost00001' }
function heartbeat(cell: (typeof cells)[number]) {
return {
cellId: cell.id,
cellUrl: cell.url,
cellIncarnation: '11111111-1111-4111-8111-111111111111',
startedAt: 50,
ready: true,
observedRequests: 0,
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0,
connectionInclusionWatermark: 1,
connectionHardCap: 600 as const,
connectionUnobservedBound: 50
}
}
describePostgres('PostgreSQL per-cell inventory locking', () => {
const databases: RelayDatabase[] = []
beforeAll(async () => {
for (let index = 0; index < 3; index++) {
databases.push(await openRelayDatabase({ databaseUrl, dataDir: '' }))
}
})
async function removeTestRows(database: RelayDatabase): Promise<void> {
await database.query(
`DELETE FROM relay_control_connection_reservations WHERE user_id LIKE 'percell-postgres-%'`
)
for (const table of [
'relay_assignment_activity_leases',
'relay_post_drain_migration_pins',
'relay_assignment_migration_incarnations',
'relay_assignment_migrations',
'relay_assignment_region_preferences',
'relay_assignments'
]) {
await database.query(`DELETE FROM ${table} WHERE user_id LIKE 'percell-postgres-%'`)
}
for (const cell of cells) {
for (const table of [
'relay_cell_connection_snapshots',
'relay_cell_connection_runtime',
'relay_cell_connection_limits',
'relay_cell_runtime',
'relay_cells'
]) {
await database.query(`DELETE FROM ${table} WHERE cell_id = ?`, [cell.id])
}
}
}
afterAll(async () => {
if (databases[0]) await removeTestRows(databases[0])
for (const connection of databases) await connection.close()
})
async function pinHostToLastCell(store: RelayAssignmentStore): Promise<void> {
await store.reconcileCells(cells)
for (const cell of cells) await store.recordCellHeartbeat(heartbeat(cell))
await store.setCellEnabled(cellA.id, false)
await store.setCellEnabled(cellB.id, false)
const assignment = await store.assign(identity)
expect(assignment.cellId).toBe(cellC.id)
await store.setCellEnabled(cellA.id, true)
await store.setCellEnabled(cellB.id, true)
}
async function lockWaiterAppeared(database: RelayDatabase): Promise<boolean> {
const deadline = Date.now() + 4_000
while (Date.now() < deadline) {
const rows = await database.query(
`SELECT count(*) AS waiting FROM pg_stat_activity
WHERE datname = current_database() AND wait_event_type = 'Lock'`
)
if (Number(rows[0]!.waiting) > 0) return true
await new Promise((resolve) => setTimeout(resolve, 10))
}
return false
}
// Why: a sticky refresh whose first NOWAIT probe loses retries by taking a
// cell row before the assignment row. That retry used to take the whole
// inventory, so one busy cell stalled every other cell's reconnects.
it('waits only on the pinned cell row while refreshing a sticky assignment', async () => {
await removeTestRows(databases[0]!)
const store = new RelayAssignmentStore(databases[0]!, () => 100)
await pinHostToLastCell(store)
// A host whose control lease was already reaped still holds its pin; that
// is the shape that reaches the cell-row probe instead of touchAssignment.
await databases[0]!.query(
`DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`,
[identity.userId]
)
let releaseRow!: () => void
const rowReleased = new Promise<void>((resolve) => {
releaseRow = resolve
})
let rowHeld!: () => void
const rowHeldPromise = new Promise<void>((resolve) => {
rowHeld = resolve
})
const holder = databases[1]!.transaction(async (transaction) => {
await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cellC.id])
rowHeld()
await rowReleased
})
await rowHeldPromise
const refresh = store.assign(identity)
expect(await lockWaiterAppeared(databases[2]!)).toBe(true)
// The refresh is blocked on cell C. Every earlier row must still be free:
// the ordered fleet-wide scan would be holding both of them by now.
const heldWhileRefreshWaits: string[] = []
await databases[2]!.transaction(async (transaction) => {
for (const cell of [cellA, cellB]) {
try {
await transaction.queryLocked(
`SELECT * FROM relay_cells WHERE cell_id = ?`,
[cell.id],
{ failIfUnavailable: true }
)
} catch {
heldWhileRefreshWaits.push(cell.id)
}
}
})
releaseRow()
await holder
expect(heldWhileRefreshWaits).toEqual([])
expect((await refresh).cellId).toBe(cellC.id)
}, 15_000)
// Why: the counter moves by a delta now instead of an absolute value read
// from a snapshot, so concurrent movement on the same cell must still sum.
it('keeps a cell reservation exact under concurrent same-cell activity', async () => {
await removeTestRows(databases[0]!)
const store = new RelayAssignmentStore(databases[0]!, () => 100)
await store.reconcileCells(cells)
for (const cell of cells) await store.recordCellHeartbeat(heartbeat(cell))
await store.setCellEnabled(cellA.id, false)
await store.setCellEnabled(cellB.id, false)
const hosts = Array.from({ length: 6 }, (_, index) => ({
userId: `percell-postgres-user-${index}`,
relayHostId: `percellhost0000${index}`
}))
const stores = databases.map((database) => new RelayAssignmentStore(database, () => 100))
await Promise.all(hosts.map((host, index) => stores[index % stores.length]!.assign(host)))
// One splice each (2 units) on the same cell, from three connections at once.
await Promise.all(
hosts.map((host, index) =>
stores[index % stores.length]!.acquireActivity(host, {
activityId: `splice:percell-${index}`,
kind: 'splice',
cellId: cellC.id
})
)
)
const afterAcquire = await databases[0]!.query(
`SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`,
[cellC.id]
)
// 6 pending control grants + 6 splices at 2 units each.
expect(Number(afterAcquire[0]!.reserved_requests)).toBe(6 + 12)
await Promise.all(
hosts.map((host, index) =>
stores[index % stores.length]!.releaseActivity(host, `splice:percell-${index}`)
)
)
const afterRelease = await databases[0]!.query(
`SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`,
[cellC.id]
)
expect(Number(afterRelease[0]!.reserved_requests)).toBe(6)
await store.setCellEnabled(cellA.id, true)
await store.setCellEnabled(cellB.id, true)
}, 15_000)
})
@@ -0,0 +1,260 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { openRelayDatabase, type RelayDatabase } from './database.js'
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
// Three cells: the inventory lock covers more than the rows a move touches, and
// a high-to-low move exposes any lock taken out of cell_id order.
const cells = [
{
id: 'rebind-inventory-postgres-a',
url: 'https://rebind-inventory-postgres-a.example.com',
capacityRequests: 1_000,
connectionHardCap: 600 as const,
connectionUnobservedBound: 50
},
{
id: 'rebind-inventory-postgres-b',
url: 'https://rebind-inventory-postgres-b.example.com',
capacityRequests: 1_000,
connectionHardCap: 600 as const,
connectionUnobservedBound: 50
},
{
id: 'rebind-inventory-postgres-c',
url: 'https://rebind-inventory-postgres-c.example.com',
capacityRequests: 1_000,
connectionHardCap: 600 as const,
connectionUnobservedBound: 50
}
]
const identity = { userId: 'rebind-inventory-postgres-user', relayHostId: 'rebindinvhost001' }
function heartbeat(cell: (typeof cells)[number]) {
return {
cellId: cell.id,
cellUrl: cell.url,
cellIncarnation: '11111111-1111-4111-8111-111111111111',
startedAt: 50,
ready: true,
observedRequests: 0,
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0,
connectionInclusionWatermark: 1,
connectionHardCap: 600 as const,
connectionUnobservedBound: 50
}
}
// Why: every desktop control rebind used to take the fleet-wide relay_cells
// FOR UPDATE lock, so a rebind on one cell queued behind whatever held any
// other cell's row, until COMMIT (55P03 at the request bound). A rebind only
// touches its own cell row, so it must proceed while another cell's row is
// held elsewhere.
describePostgres('PostgreSQL control rebind under a held cell row', () => {
const databases: RelayDatabase[] = []
beforeAll(async () => {
databases.push(
await openRelayDatabase({ databaseUrl, dataDir: '' }),
await openRelayDatabase({ databaseUrl, dataDir: '' })
)
})
async function removeTestRows(database: RelayDatabase): Promise<void> {
await database.query(
`DELETE FROM relay_control_connection_reservations WHERE user_id = ?`,
[identity.userId]
)
for (const table of [
'relay_assignment_activity_leases',
'relay_post_drain_migration_pins',
'relay_assignment_migration_incarnations',
'relay_assignment_migrations',
'relay_assignments'
]) {
await database.query(`DELETE FROM ${table} WHERE user_id = ?`, [identity.userId])
}
for (const cell of cells) {
for (const table of [
'relay_cell_connection_snapshots',
'relay_cell_connection_runtime',
'relay_cell_connection_limits',
'relay_cell_runtime',
'relay_cells'
]) {
await database.query(`DELETE FROM ${table} WHERE cell_id = ?`, [cell.id])
}
}
}
afterAll(async () => {
if (databases[0]) await removeTestRows(databases[0])
for (const connection of databases) await connection.close()
})
it("rebinds and supersedes a control while another cell's row is held", async () => {
// A prior aborted run leaves connection snapshots that reject a replayed watermark.
await removeTestRows(databases[0]!)
const store = new RelayAssignmentStore(databases[0]!, () => 100)
await store.reconcileCells(cells)
for (const cell of cells) await store.recordCellHeartbeat(heartbeat(cell))
// Pin the host to cell A so placement is deterministic.
await store.setCellEnabled(cells[1]!.id, false)
await store.setCellEnabled(cells[2]!.id, false)
const assignment = await store.assign(identity)
expect(assignment.cellId).toBe(cells[0]!.id)
await store.setCellEnabled(cells[1]!.id, true)
await store.setCellEnabled(cells[2]!.id, true)
await store.activateControl(identity, {
cellId: cells[0]!.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 1,
connectionInclusionWatermark: 10
})
// Hold only cell B's row on a second connection, the way a rebind on B
// does, for longer than the request-path lock bound.
let releaseInventory!: () => void
const inventoryReleased = new Promise<void>((resolve) => {
releaseInventory = resolve
})
let inventoryHeld!: () => void
const inventoryHeldPromise = new Promise<void>((resolve) => {
inventoryHeld = resolve
})
const holder = databases[1]!.transaction(async (transaction) => {
await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cells[1]!.id])
inventoryHeld()
await inventoryReleased
})
await inventoryHeldPromise
// A generation-2 rebind on cell A supersedes generation 1. It must not
// wait on cell B's row.
const startedAt = Date.now()
const blockedStatement = async (): Promise<string> => {
const rows = await databases[1]!.query(
`SELECT left(query, 160) AS q FROM pg_stat_activity
WHERE datname = current_database() AND wait_event_type = 'Lock'`
)
return rows.map((row) => String(row.q)).join(' | ')
}
const timeout = new Promise<never>((_, reject) =>
setTimeout(
() =>
void blockedStatement().then((statement) =>
reject(new Error(`rebind on cell A blocked behind cell B's row: ${statement}`))
),
2_000
)
)
const rebound = await Promise.race([
store.activateControl(identity, {
cellId: cells[0]!.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 2,
connectionInclusionWatermark: 11
}),
timeout
])
const elapsedMs = Date.now() - startedAt
releaseInventory()
await holder
expect(rebound).toBe(`control:${cells[0]!.id}:2`)
expect(elapsedMs).toBeLessThan(2_000)
const controls = await databases[0]!.query(
`SELECT activity_id FROM relay_assignment_activity_leases
WHERE user_id = ? AND activity_kind = 'control' ORDER BY activity_id`,
[identity.userId]
)
expect(controls).toEqual([{ activity_id: `control:${cells[0]!.id}:2` }])
const reserved = await databases[0]!.query(
`SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`,
[cells[0]!.id]
)
expect(Number(reserved[0]!.reserved_requests)).toBe(1)
}, 15_000)
// Why: a phone's activity id is client-chosen and can follow the host across
// a migration, so acquireActivity may touch two cell rows. Moving from the
// higher cell to the lower one is where an unordered lock cycles with
// placement's ascending inventory lock (reproduced live before this fix).
it('moves an activity from a higher cell to a lower one in cell_id order', async () => {
await removeTestRows(databases[0]!)
const [cellA, cellB, cellC] = cells as [typeof cells[0], typeof cells[0], typeof cells[0]]
const store = new RelayAssignmentStore(databases[0]!, () => 100)
await store.reconcileCells(cells)
for (const cell of cells) await store.recordCellHeartbeat(heartbeat(cell))
await store.setCellEnabled(cellA.id, false)
await store.setCellEnabled(cellB.id, false)
const assignment = await store.assign(identity)
expect(assignment.cellId).toBe(cellC.id)
await store.setCellEnabled(cellA.id, true)
await store.setCellEnabled(cellB.id, true)
const activityId = 'splice:rebind-inventory-postgres'
await store.acquireActivity(identity, { activityId, kind: 'splice', cellId: cellC.id })
// The migration makes B authoritative; the lease still sits on C.
const migration = await store.startEvacuation(identity, cellB.id)
expect(migration.targetCellId).toBe(cellB.id)
// Hold B elsewhere. An ordered move locks B first and queues here holding
// nothing else. Locking C first (the old lease's row, as an unordered move
// does) or the whole inventory (which takes A) shows up as a held row.
let releaseRow!: () => void
const rowReleased = new Promise<void>((resolve) => {
releaseRow = resolve
})
let rowHeld!: () => void
const rowHeldPromise = new Promise<void>((resolve) => {
rowHeld = resolve
})
const heldWhileMoverWaits: string[] = []
const holder = databases[1]!.transaction(async (transaction) => {
await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cellB.id])
rowHeld()
await rowReleased
for (const cell of [cellA, cellC]) {
try {
await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cell.id], {
failIfUnavailable: true
})
} catch {
heldWhileMoverWaits.push(cell.id)
}
}
})
await rowHeldPromise
const move = store.acquireActivity(identity, { activityId, kind: 'splice', cellId: cellB.id })
let moved = false
void move.then(() => {
moved = true
})
await new Promise((resolve) => setTimeout(resolve, 250))
expect(moved).toBe(false)
releaseRow()
await holder
await move
expect(heldWhileMoverWaits).toEqual([])
const reservations = await databases[0]!.query(
`SELECT cell_id, reserved_requests FROM relay_cells
WHERE cell_id IN (?, ?, ?) ORDER BY cell_id ASC`,
[cellA.id, cellB.id, cellC.id]
)
const reserved = reservations.map((row) => [String(row.cell_id), Number(row.reserved_requests)])
expect(reserved).toEqual([
[cellA.id, 0],
// Migration grant plus the moved splice, as in the SQLite origin-scoped
// reservation case: the lock change did not alter accounting.
[cellB.id, 6],
// The sticky grant stays on the source until the migration completes.
[cellC.id, 1]
])
}, 15_000)
})
@@ -2,7 +2,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
const fakes = vi.hoisted(() => ({
configs: [] as Array<Record<string, unknown>>,
query: vi.fn(async () => ({ rows: [], rowCount: 0 })),
// Pool construction and pool shutdown interleaved, so "the schema pool is
// gone before the serving pool opens" is checkable rather than assumed.
lifecycle: [] as string[],
query: vi.fn(async (_sql: string) => ({ rows: [], rowCount: 0 })),
release: vi.fn(),
end: vi.fn(async () => undefined)
}))
@@ -13,20 +16,37 @@ vi.mock('pg', () => ({
totalCount = 1
idleCount = 1
waitingCount = 0
end = fakes.end
on = vi.fn()
connect = vi.fn(async () => ({ query: fakes.query, release: fakes.release }))
private readonly label: string
constructor(config: Record<string, unknown>) {
fakes.configs.push(config)
this.label = `max=${String(config.max)} statement_timeout=${String(config.statement_timeout)}`
fakes.lifecycle.push(`open ${this.label}`)
}
async end(): Promise<void> {
fakes.lifecycle.push(`end ${this.label}`)
await fakes.end()
}
}
}
}))
import { openRelayDatabase } from './database.js'
import { openRelayDatabase, relayPostgresStatementTimeoutMs } from './database.js'
import { applyPostgresSchema } from './postgres-schema-startup.js'
const SCHEMA_POOL = {
max: 1,
application_name: 'orca-relay/director/director/schema',
connectionTimeoutMillis: 2_000,
// Why: DDL must not inherit the request deadline.
statement_timeout: 0,
lock_timeout: 1_000,
idle_in_transaction_session_timeout: 5_000
}
afterEach(() => {
vi.restoreAllMocks()
})
@@ -34,9 +54,11 @@ afterEach(() => {
describe('PostgreSQL relay deadlines', () => {
beforeEach(() => {
fakes.configs.length = 0
fakes.lifecycle.length = 0
fakes.query.mockClear()
fakes.release.mockClear()
fakes.end.mockClear()
delete process.env.ORCA_RELAY_POSTGRES_STATEMENT_TIMEOUT_MS
})
it('bounds pool acquisition, statements, locks, and abandoned transactions', async () => {
@@ -48,6 +70,7 @@ describe('PostgreSQL relay deadlines', () => {
})
expect(fakes.configs).toEqual([
expect.objectContaining(SCHEMA_POOL),
expect.objectContaining({
max: 3,
application_name: 'orca-relay/director/director',
@@ -59,6 +82,110 @@ describe('PostgreSQL relay deadlines', () => {
])
await database.close()
})
// Why: an untimed session left open would be a standing way for request work
// to escape the deadline this whole pool config exists to enforce.
it('closes the untimed schema pool before the serving pool opens', async () => {
const database = await openRelayDatabase({
databaseUrl: 'postgresql://relay:secret@127.0.0.1:5432/relay',
dataDir: './unused',
poolMax: 3,
applicationName: 'orca-relay/director/director'
})
expect(fakes.lifecycle).toEqual([
'open max=1 statement_timeout=0',
'end max=1 statement_timeout=0',
'open max=3 statement_timeout=5000'
])
await database.close()
})
it('applies the schema on the untimed pool, never on the serving one', async () => {
fakes.query.mockClear()
const ddl: string[] = []
fakes.query.mockImplementation(async (sql: string) => {
// Every statement issued before the serving pool exists is schema work.
if (fakes.lifecycle.length === 1) ddl.push(sql)
return { rows: [], rowCount: 0 }
})
const database = await openRelayDatabase({
databaseUrl: 'postgresql://relay:secret@127.0.0.1:5432/relay',
dataDir: './unused'
})
expect(ddl.length).toBeGreaterThan(0)
// Statements can open with a leading `--` rationale comment.
const body = (statement: string): string =>
statement.replace(/^(?:\s*--[^\n]*\n)*\s*/, '')
expect(ddl.every((statement) => /^CREATE\b/i.test(body(statement)))).toBe(true)
// The backfill is DML, so it stays on the deadline-bearing serving pool.
expect(ddl.some((statement) => statement.includes('INSERT INTO'))).toBe(false)
await database.close()
})
it('takes the serving statement deadline from the environment', async () => {
process.env.ORCA_RELAY_POSTGRES_STATEMENT_TIMEOUT_MS = '2500'
const database = await openRelayDatabase({
databaseUrl: 'postgresql://relay:secret@127.0.0.1:5432/relay',
dataDir: './unused'
})
expect(fakes.configs).toEqual([
expect.objectContaining({ statement_timeout: 0 }),
expect.objectContaining({ statement_timeout: 2_500 })
])
await database.close()
})
it.each(['0', '-1', '2.5', 'soon', ' '])(
'refuses %s as a statement deadline instead of running unbounded',
(value) => {
expect(() =>
relayPostgresStatementTimeoutMs({ ORCA_RELAY_POSTGRES_STATEMENT_TIMEOUT_MS: value })
).toThrow('invalid_statement_timeout')
}
)
it.each([undefined, ''])('defaults to 5s when the environment says %s', (value) => {
expect(
relayPostgresStatementTimeoutMs(
value === undefined ? {} : { ORCA_RELAY_POSTGRES_STATEMENT_TIMEOUT_MS: value }
)
).toBe(5_000)
})
// Why: a statement deadline that reaches the caller as a crash converts a
// transient stall into a failed assignment. It aborts the transaction exactly
// as a lock timeout does, so it belongs on the same bounded retry.
it('retries a statement timeout on a fresh client', async () => {
vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const database = await openRelayDatabase({
databaseUrl: 'postgresql://relay:secret@127.0.0.1:5432/relay',
dataDir: './unused'
})
let attempts = 0
const result = await database.transaction(async (transaction) => {
attempts += 1
if (attempts === 1) {
await transaction.query('SELECT 1')
throw Object.assign(new Error('canceling statement due to statement timeout'), {
code: '57014'
})
}
return 'committed'
})
expect(result).toBe('committed')
expect(attempts).toBe(2)
expect(console.warn).toHaveBeenCalledWith(
expect.stringContaining('"event":"orca_relay_postgres_transaction_retry"')
)
expect(console.warn).toHaveBeenCalledWith(expect.stringContaining('"code":"57014"'))
await database.close()
})
})
describe('PostgreSQL schema startup', () => {
@@ -0,0 +1,98 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { openRelayDatabase, type RelayDatabase } from './database.js'
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
const applicationName = 'orca-relay/statement-timeout-postgres'
describePostgres('PostgreSQL statement deadline', () => {
const databases: RelayDatabase[] = []
beforeAll(async () => {
databases.push(await openRelayDatabase({ databaseUrl, dataDir: '' }))
})
afterAll(async () => {
for (const database of databases) await database.close()
})
it('serves requests under the configured deadline', async () => {
const database = await openRelayDatabase({ databaseUrl, dataDir: '', statementTimeoutMs: 300 })
databases.push(database)
expect(await database.query(`SELECT current_setting('statement_timeout') AS statement_timeout`)).toEqual([
{ statement_timeout: '300ms' }
])
})
// Why: a real 57014 aborts the transaction exactly as a lock timeout does. If
// it escapes the bounded retry it becomes a failed assignment instead of a
// slow one.
it('retries a real statement timeout on a fresh client', async () => {
const database = await openRelayDatabase({ databaseUrl, dataDir: '', statementTimeoutMs: 300 })
databases.push(database)
let attempts = 0
const result = await database.transaction(async (transaction) => {
attempts += 1
if (attempts === 1) await transaction.query(`SELECT pg_sleep(2)`)
return attempts
})
expect(result).toBe(2)
}, 15_000)
// Why: DDL runs on its own untimed connection. relay_invites carries a
// CREATE INDEX IF NOT EXISTS, which (unlike CREATE TABLE IF NOT EXISTS)
// really does queue behind an ACCESS EXCLUSIVE lock on the table.
it('applies the schema behind a held ACCESS EXCLUSIVE lock', async () => {
let releaseTable!: () => void
const tableReleased = new Promise<void>((resolve) => {
releaseTable = resolve
})
let tableHeld!: () => void
const tableHeldPromise = new Promise<void>((resolve) => {
tableHeld = resolve
})
const holder = databases[0]!.transaction(async (transaction) => {
await transaction.query(`LOCK TABLE relay_invites IN ACCESS EXCLUSIVE MODE`)
tableHeld()
await tableReleased
})
await tableHeldPromise
const opening = openRelayDatabase({
databaseUrl,
dataDir: '',
applicationName,
// Far too short for a blocked DDL; the serving pool wears it, the schema
// connection must not.
statementTimeoutMs: 200
})
const blockedOnSchemaConnection = async (): Promise<boolean> => {
const deadline = Date.now() + 4_000
while (Date.now() < deadline) {
const rows = await databases[0]!.query(
`SELECT count(*) AS waiting FROM pg_stat_activity
WHERE datname = current_database() AND wait_event_type = 'Lock'
AND application_name = ?`,
[`${applicationName}/schema`]
)
if (Number(rows[0]!.waiting) > 0) return true
await new Promise((resolve) => setTimeout(resolve, 10))
}
return false
}
const blocked = await blockedOnSchemaConnection()
releaseTable()
await holder
const database = await opening
databases.push(database)
expect(blocked).toBe(true)
// The serving pool still carries the short deadline it was opened with.
expect(await database.query(`SELECT current_setting('statement_timeout') AS statement_timeout`)).toEqual([
{ statement_timeout: '200ms' }
])
}, 15_000)
})
+58 -10
View File
@@ -796,12 +796,34 @@ class PostgresTransaction implements RelayDatabase {
const POSTGRES_TRANSACTION_ATTEMPTS = 3
const POSTGRES_RETRY_MAX_DELAY_MS = 25
const POSTGRES_CONNECTION_TIMEOUT_MS = 2_000
const POSTGRES_STATEMENT_TIMEOUT_MS = 5_000
// Derivation: a control renewal must land inside its own 30s tick
// (RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2), and a transaction gets
// POSTGRES_TRANSACTION_ATTEMPTS tries, so the worst case a renewal can spend in
// Postgres is attempts * timeout. 5s keeps that at 15s, half the tick, and still
// leaves room for the connect timeout above.
export const POSTGRES_STATEMENT_TIMEOUT_MS = 5_000
const POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS = 5_000
export function relayPostgresStatementTimeoutMs(
env: NodeJS.ProcessEnv = process.env
): number {
const configured = env.ORCA_RELAY_POSTGRES_STATEMENT_TIMEOUT_MS
if (configured === undefined || configured === '') return POSTGRES_STATEMENT_TIMEOUT_MS
const milliseconds = Number(configured)
// 0 is PostgreSQL's "no timeout"; refusing it keeps the deadline this exists
// to enforce from being disabled by a typo in an environment variable.
if (!Number.isInteger(milliseconds) || milliseconds < 1) {
throw new Error('invalid_statement_timeout')
}
return milliseconds
}
function retryablePostgresTransactionError(error: unknown): boolean {
const code = String((error as { code?: unknown }).code)
return code === '40P01' || code === '40001' || code === '55P03'
// 57014 is the pool statement_timeout firing. It aborts the transaction the
// same way a lock timeout does, so it belongs on the bounded retry path
// rather than surfacing as a terminal failure to the caller.
return code === '40P01' || code === '40001' || code === '55P03' || code === '57014'
}
export function isRelayDatabaseTransientError(error: unknown): boolean {
@@ -963,11 +985,36 @@ async function applySchema(database: RelayDatabase): Promise<void> {
}
}
async function applySchemaWithPostgresRetries(database: RelayDatabase): Promise<void> {
await applyPostgresSchema(
SCHEMA.split(';').filter((statement) => statement.trim()),
async (statement) => await database.query(statement)
)
// Why: DDL is not a request. A CREATE INDEX on a grown table legitimately runs
// longer than the request statement_timeout, and inheriting that timeout would
// make every startup fail at the same statement instead of finishing once. One
// short-lived connection of its own, ended before the serving pool opens, keeps
// the untimed session off the request path entirely.
async function applySchemaOnUntimedPool(
databaseUrl: string,
applicationName: string | undefined
): Promise<void> {
const pool = new pg.Pool({
connectionString: databaseUrl,
max: 1,
application_name: applicationName ? `${applicationName}/schema` : undefined,
connectionTimeoutMillis: POSTGRES_CONNECTION_TIMEOUT_MS,
statement_timeout: 0,
// Kept: a DDL blocked behind another director's ACCESS EXCLUSIVE lock must
// yield to the bounded schema retry instead of holding the connection.
lock_timeout: POSTGRES_LOCK_TIMEOUT_MS,
idle_in_transaction_session_timeout: POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS
})
absorbPostgresIdleClientErrors(pool)
const database = new PostgresDatabase(pool)
try {
await applyPostgresSchema(
SCHEMA.split(';').filter((statement) => statement.trim()),
async (statement) => await database.query(statement)
)
} finally {
await database.close().catch(() => undefined)
}
}
async function backfillRelayCellRegions(database: RelayDatabase): Promise<void> {
@@ -983,15 +1030,17 @@ export async function openRelayDatabase(input: {
dataDir: string
poolMax?: number
applicationName?: string
statementTimeoutMs?: number
}): Promise<RelayDatabase> {
let database: RelayDatabase
if (input.databaseUrl) {
await applySchemaOnUntimedPool(input.databaseUrl, input.applicationName)
const pool = new pg.Pool({
connectionString: input.databaseUrl,
max: input.poolMax ?? 10,
application_name: input.applicationName,
connectionTimeoutMillis: POSTGRES_CONNECTION_TIMEOUT_MS,
statement_timeout: POSTGRES_STATEMENT_TIMEOUT_MS,
statement_timeout: input.statementTimeoutMs ?? relayPostgresStatementTimeoutMs(),
lock_timeout: POSTGRES_LOCK_TIMEOUT_MS,
idle_in_transaction_session_timeout: POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS
})
@@ -1004,8 +1053,7 @@ export async function openRelayDatabase(input: {
database = new SqliteDatabase(sqlite)
}
try {
if (input.databaseUrl) await applySchemaWithPostgresRetries(database)
else await applySchema(database)
if (!input.databaseUrl) await applySchema(database)
await backfillRelayCellRegions(database)
return database
} catch (error) {
@@ -0,0 +1,82 @@
import { ASSIGNMENT_LIMITS, RELAY_HOST_CLOSE_REASON } from '@orca-cloud/relay-contract'
import { describe, expect, it } from 'vitest'
import { HostCloseReasonMemory } from './host-close-reason-memory.js'
function memoryAt(clock: { now: number }): HostCloseReasonMemory {
return new HostCloseReasonMemory(() => clock.now)
}
describe('HostCloseReasonMemory', () => {
it('remembers only reasons it knows', () => {
const clock = { now: 1_000 }
const memory = memoryAt(clock)
memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
memory.record('b', 'quitting')
memory.record('c', Buffer.alloc(0))
memory.record('d', undefined)
expect(memory.read('a')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
expect(memory.read('b')).toBeNull()
expect(memory.read('c')).toBeNull()
expect(memory.read('d')).toBeNull()
})
it('accepts the reason as the Buffer a ws close delivers', () => {
const clock = { now: 1_000 }
const memory = memoryAt(clock)
memory.record('a', Buffer.from(RELAY_HOST_CLOSE_REASON.SIGNED_OUT))
expect(memory.read('a')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
})
it('expires an entry once its host may have been rebalanced away', () => {
const clock = { now: 1_000 }
const memory = memoryAt(clock)
memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
clock.now += ASSIGNMENT_LIMITS.dormantTtlMs - 1
expect(memory.read('a')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
clock.now += 1
expect(memory.read('a')).toBeNull()
expect(memory.size()).toBe(0)
})
it('forgets on demand', () => {
const clock = { now: 1_000 }
const memory = memoryAt(clock)
memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
memory.forget('a')
expect(memory.read('a')).toBeNull()
})
it('drops the oldest survivors rather than growing without bound', () => {
const clock = { now: 1_000 }
const memory = memoryAt(clock)
for (let index = 0; index < 50_050; index++) {
memory.record(`host-${index}`, RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
}
expect(memory.size()).toBe(50_000)
expect(memory.read('host-0')).toBeNull()
expect(memory.read('host-50049')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
})
it('re-recording refreshes recency so a live host is not evicted first', () => {
const clock = { now: 1_000 }
const memory = memoryAt(clock)
memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
memory.record('b', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
expect([...['a', 'b'].map((key) => memory.read(key))]).toEqual([
RELAY_HOST_CLOSE_REASON.SIGNED_OUT,
RELAY_HOST_CLOSE_REASON.SIGNED_OUT
])
expect(memory.size()).toBe(2)
})
})
@@ -0,0 +1,72 @@
import {
ASSIGNMENT_LIMITS,
relayHostCloseReasonFrom,
type RelayHostCloseReason
} from '@orca-cloud/relay-contract'
// Retention matches the dormant assignment TTL: past it the host may have been
// rebalanced onto another cell, so this cell is no longer the one a phone asks.
const RETENTION_MS = ASSIGNMENT_LIMITS.dormantTtlMs
// A fleet-wide auth outage signs out every host at once; the cap bounds that
// burst well above any single cell's host count without becoming a leak.
const MAX_ENTRIES = 50_000
// Why in-memory and not Postgres: a phone reaches the cell its host's assignment
// row already names, which is the same cell that watched the control socket
// close. Losing this on a cell restart degrades to the pre-existing generic
// verdict, so the failure mode is the old behaviour rather than a wrong one.
export class HostCloseReasonMemory {
private readonly entries = new Map<string, { reason: RelayHostCloseReason; expiresAt: number }>()
constructor(private readonly now: () => number = Date.now) {}
// Silently ignores anything that is not a known reason, which is every close
// from a host that predates the field and every abrupt 1006.
record(key: string, reason: unknown): void {
const parsed = relayHostCloseReasonFrom(reason)
if (!parsed) {
return
}
this.entries.delete(key)
this.entries.set(key, { reason: parsed, expiresAt: this.now() + RETENTION_MS })
this.evict()
}
forget(key: string): void {
this.entries.delete(key)
}
read(key: string): RelayHostCloseReason | null {
const entry = this.entries.get(key)
if (!entry) {
return null
}
if (entry.expiresAt <= this.now()) {
this.entries.delete(key)
return null
}
return entry.reason
}
size(): number {
return this.entries.size
}
private evict(): void {
const now = this.now()
for (const [key, entry] of this.entries) {
if (entry.expiresAt > now) {
break
}
this.entries.delete(key)
}
// Insertion order is recency order (record deletes before setting), so the
// head is always the oldest survivor.
for (const key of this.entries.keys()) {
if (this.entries.size <= MAX_ENTRIES) {
break
}
this.entries.delete(key)
}
}
}
+34 -6
View File
@@ -14,7 +14,8 @@ import {
HostHelloSchema,
InviteCreateSchema,
RELAY_PROTOCOL_LIMITS,
RELAY_CLOSE_CODE
RELAY_CLOSE_CODE,
type RelayHostCloseReason
} from '@orca-cloud/relay-contract'
import nacl from 'tweetnacl'
import type WebSocket from 'ws'
@@ -25,6 +26,7 @@ import {
RelayCredentialStore,
type CredentialReservation
} from './credential-store.js'
import { HostCloseReasonMemory } from './host-close-reason-memory.js'
import { relayHostLogDigest } from './relay-host-log-digest.js'
import type { RelayTokenClaims } from './relay-token-verifier.js'
import type { RelayRuntimeObserver } from './relay-observability.js'
@@ -130,6 +132,10 @@ const ACTIVATION_QUEUE_WAIT_MS = 30_000
export class HostSessionRegistry {
private readonly sessions = new Map<string, HostSession>()
private readonly activationQueues = new Map<string, Promise<void>>()
// Why it outlives `sessions`: the orphan grace deletes the session within 30s,
// but a signed-out desktop never comes back, so the phone that asks minutes
// later would otherwise find nothing to explain its rejection with.
private readonly hostCloseReasons = new HostCloseReasonMemory(() => this.now())
private draining = false
constructor(
@@ -175,7 +181,8 @@ export class HostSessionRegistry {
return
}
this.observer.recordAuth(true)
const session = this.sessions.get(this.key(reservation.userId, hostId))
const sessionKey = this.key(reservation.userId, hostId)
const session = this.sessions.get(sessionKey)
if (
!session ||
session.state !== 'active' ||
@@ -184,7 +191,13 @@ export class HostSessionRegistry {
) {
capacityReservation?.release()
await this.store.failReservation(reservation)
this.rejectClient(socket, RELAY_CLOSE_CODE.HOST_OFFLINE)
// The only rejection that can name a cause: the host is genuinely absent.
// The attach-deadline 4404 below fires while control is still connected.
this.rejectClient(
socket,
RELAY_CLOSE_CODE.HOST_OFFLINE,
this.hostCloseReasons.read(sessionKey)
)
return
}
if (session.activeConnIds.size + session.pendingConns.size >= 8) {
@@ -793,7 +806,10 @@ export class HostSessionRegistry {
regionalDrainTimer: null,
regionalDrainExpiresAt: null
}
this.sessions.set(this.key(identity.sub, identity.relayHostId), session)
const sessionKey = this.key(identity.sub, identity.relayHostId)
// A host that proved itself again is not signed out, whatever it said last.
this.hostCloseReasons.forget(sessionKey)
this.sessions.set(sessionKey, session)
this.wireActiveControl(session)
this.sendHelloAck(session)
}
@@ -813,6 +829,11 @@ export class HostSessionRegistry {
})
socket.once('close', (code, reason) => {
this.observer.recordControlClose?.(code)
// Guarded on identity: a predecessor retired by a rebind must not stamp a
// cause onto the live session that replaced it.
if (session.socket === socket) {
this.hostCloseReasons.record(this.key(session.identity.sub, session.relayHostId), reason)
}
// One line per control close makes reconnect churners attributable by
// host digest without exposing the raw relay host id.
console.warn(
@@ -1187,9 +1208,16 @@ export class HostSessionRegistry {
if (session.socket) send(session.socket, 'control-error', { ...(reqId ? { reqId } : {}), code })
}
private rejectClient(socket: WebSocket, code: number): void {
// hostCloseReason rides the WebSocket close reason, never relay-hello: every
// shipped phone parses relay-hello with a strict schema that rejects an
// unknown key, and none of them read the close reason at all.
private rejectClient(
socket: WebSocket,
code: number,
hostCloseReason?: RelayHostCloseReason | null
): void {
send(socket, 'relay-hello', { ok: false, code })
closeRelayWebSocket(socket, code, 'relay connection rejected')
closeRelayWebSocket(socket, code, hostCloseReason ?? 'relay connection rejected')
}
private releaseControlActivity(session: HostSession): void {
@@ -0,0 +1,206 @@
import { EventEmitter } from 'node:events'
import {
CONTROL_CONTINUITY_LIMITS,
RELAY_CLOSE_CODE,
RELAY_HOST_CLOSE_REASON
} from '@orca-cloud/relay-contract'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type WebSocket from 'ws'
import type { RelayAssignmentStore } from './assignment-store.js'
import type { RelayConfig } from './config.js'
import type { RelayCredentialStore } from './credential-store.js'
import { HostSessionRegistry } from './host-session-registry.js'
import type { RelayRuntimeObserver } from './relay-observability.js'
import type { RelayTokenClaims } from './relay-token-verifier.js'
import { ProcessQueuedByteBudget } from './splice-forwarder.js'
class FakeSocket extends EventEmitter {
readonly OPEN = 1
readonly CLOSED = 3
readyState = this.OPEN
readonly send = vi.fn()
readonly close = vi.fn((code?: number, reason?: string) => {
this.readyState = this.CLOSED
this.emit('close', code, Buffer.from(reason ?? ''))
})
readonly terminate = vi.fn(() => {
this.readyState = this.CLOSED
this.emit('close', 1006, Buffer.alloc(0))
})
}
const config = {
port: 8080,
publicUrl: 'https://relay-c3.example.com',
cellUrl: 'https://relay-c3.example.com',
authIssuer: 'https://auth.example.com',
authAudience: 'orca-relay',
jwksUrl: 'https://auth.example.com/jwks',
assignmentSigningKey: new Uint8Array(32),
role: 'cell',
cellId: 'production-gce-c3',
cells: []
} as unknown as RelayConfig
const identity = {
sub: 'user-1',
prof: 'profile-1',
org: 'org-1',
relayHostId: 'AbCdEf0123_-xyZ9'
} as unknown as RelayTokenClaims
const reservation = {
userId: identity.sub,
relayHostId: identity.relayHostId,
credentialKind: 'resume',
relayDeviceId: 'device-1',
leaseExpiresAt: Date.now() + 60_000
}
function createRegistry() {
const store = {
resolveResume: vi.fn().mockResolvedValue({ userId: identity.sub }),
reserveCredential: vi.fn().mockResolvedValue(reservation),
failReservation: vi.fn().mockResolvedValue(undefined)
}
const assignments = {
activateControl: vi.fn().mockResolvedValue('control:production-gce-c3:1'),
markMigrationTargetRegistered: vi.fn().mockResolvedValue(undefined),
resolve: vi.fn().mockResolvedValue({ cellId: config.cellId }),
acquireActivity: vi.fn().mockResolvedValue(undefined),
renewControlActivity: vi.fn().mockResolvedValue(undefined),
releaseActivity: vi.fn().mockResolvedValue(true)
} as unknown as RelayAssignmentStore
const observer = {
recordAuth: vi.fn(),
recordForwardedBytes: vi.fn(),
recordHttp: vi.fn(),
recordReconnect: vi.fn(),
recordSql: vi.fn(),
recordControlClose: vi.fn(),
recordSpliceClose: vi.fn()
} satisfies RelayRuntimeObserver
const registry = new HostSessionRegistry(
config,
vi.fn(),
store as unknown as RelayCredentialStore,
assignments,
new ProcessQueuedByteBudget(),
observer
)
const activate = (socket: WebSocket, generation: number): Promise<void> =>
(
registry as unknown as {
activate: (
socket: WebSocket,
identity: RelayTokenClaims,
existing: null,
generation: number,
rebind: boolean,
assignmentEpoch: number,
appVersion: string
) => Promise<void>
}
).activate(socket, identity, null, generation, false, 1, '1.4.173')
return { registry, activate }
}
async function dialPhone(registry: HostSessionRegistry): Promise<FakeSocket> {
const phone = new FakeSocket()
await registry.acceptClient(phone as unknown as WebSocket, identity.relayHostId, 'credential')
return phone
}
// The 4404 hello body is unchanged: every shipped phone parses it with a strict
// schema, so the cause has to ride the close frame instead.
const HOST_OFFLINE_HELLO = JSON.stringify({
type: 'relay-hello',
ok: false,
code: RELAY_CLOSE_CODE.HOST_OFFLINE
})
describe('host sign-out reason on phone rejection', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
it('names the sign-out to a phone that arrives after the host is gone', async () => {
const { registry, activate } = createRegistry()
const control = new FakeSocket()
await activate(control as unknown as WebSocket, 1)
control.close(1000, RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
vi.advanceTimersByTime(CONTROL_CONTINUITY_LIMITS.orphanGraceMs + 1)
const phone = await dialPhone(registry)
expect(phone.send).toHaveBeenCalledWith(HOST_OFFLINE_HELLO)
expect(phone.close).toHaveBeenCalledWith(
RELAY_CLOSE_CODE.HOST_OFFLINE,
RELAY_HOST_CLOSE_REASON.SIGNED_OUT
)
})
it('says nothing when the host died without naming a cause', async () => {
const { registry, activate } = createRegistry()
const control = new FakeSocket()
await activate(control as unknown as WebSocket, 1)
control.terminate()
vi.advanceTimersByTime(CONTROL_CONTINUITY_LIMITS.orphanGraceMs + 1)
const phone = await dialPhone(registry)
expect(phone.close).toHaveBeenCalledWith(
RELAY_CLOSE_CODE.HOST_OFFLINE,
'relay connection rejected'
)
})
it('ignores a close reason the host invented', async () => {
const { registry, activate } = createRegistry()
const control = new FakeSocket()
await activate(control as unknown as WebSocket, 1)
control.close(1000, 'signed-out-ish')
vi.advanceTimersByTime(CONTROL_CONTINUITY_LIMITS.orphanGraceMs + 1)
const phone = await dialPhone(registry)
expect(phone.close).toHaveBeenCalledWith(
RELAY_CLOSE_CODE.HOST_OFFLINE,
'relay connection rejected'
)
})
it('forgets the sign-out once the host proves itself again', async () => {
const { registry, activate } = createRegistry()
const control = new FakeSocket()
await activate(control as unknown as WebSocket, 1)
control.close(1000, RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
vi.advanceTimersByTime(CONTROL_CONTINUITY_LIMITS.orphanGraceMs + 1)
const reconnected = new FakeSocket()
await activate(reconnected as unknown as WebSocket, 2)
// Drop it abruptly, as a network death would, so only the stale memory
// could still name a cause.
reconnected.terminate()
vi.advanceTimersByTime(CONTROL_CONTINUITY_LIMITS.orphanGraceMs + 1)
const phone = await dialPhone(registry)
expect(phone.close).toHaveBeenCalledWith(
RELAY_CLOSE_CODE.HOST_OFFLINE,
'relay connection rejected'
)
})
// A live host is present: the 4404 there is an attach deadline, not absence.
it('never names a cause while the host control is connected', async () => {
const { registry, activate } = createRegistry()
const control = new FakeSocket()
await activate(control as unknown as WebSocket, 1)
const phone = await dialPhone(registry)
expect(phone.close).not.toHaveBeenCalled()
expect(control.send).toHaveBeenCalledWith(expect.stringContaining('"type":"conn-open"'))
})
})
@@ -503,12 +503,19 @@ describePostgres('PostgreSQL transaction recovery', () => {
const directorLockOrder: string[] = []
const assignmentDatabase = new TransactionProbeDatabase(database, async (phase, sql) => {
if (phase === 'before') {
if (sql.includes('FROM relay_assignments WHERE user_id = ?')) {
// Only locked statements reach this hook, so classifying the pin read
// is what proves it stays unlocked: if it ever grows a FOR UPDATE it
// shows up in the order below instead of silently joining the queue.
if (sql.includes('SELECT cell_id FROM relay_assignments')) {
directorLockOrder.push('pin-read')
} else if (sql.includes('FROM relay_assignments WHERE user_id = ?')) {
directorLockOrder.push('assignment')
} else if (sql.includes('FROM relay_assignment_activity_leases')) {
directorLockOrder.push('activity')
} else if (sql.includes('FROM relay_cells ORDER BY')) {
directorLockOrder.push('cell-inventory')
} else if (sql.includes('FROM relay_cells WHERE cell_id IN')) {
directorLockOrder.push('cell-rows')
} else if (sql.includes('FROM relay_cells WHERE cell_id = ?')) {
directorLockOrder.push('cell')
}
@@ -530,11 +537,14 @@ describePostgres('PostgreSQL transaction recovery', () => {
})
await expect(legacyTransaction).resolves.toBeUndefined()
expect(assignmentDatabase.attempts).toBe(2)
// The retry still takes a cell row before the assignment row — the order
// that avoids the legacy cycle — but only the pinned row, never the
// inventory.
expect(directorLockOrder).toEqual([
'assignment',
'activity',
'cell',
'cell-inventory',
'cell-rows',
'assignment',
'activity'
])
@@ -133,10 +133,15 @@
"google_logging_metric.relay_snapshot",
"google_monitoring_alert_policy.relay_assignment_5xx",
"google_monitoring_alert_policy.relay_assignment_edge_429",
"google_monitoring_alert_policy.relay_cell_process_exit",
"google_monitoring_alert_policy.relay_cloud_nat_port_drops",
"google_monitoring_alert_policy.relay_cloud_sql_backends",
"google_monitoring_alert_policy.relay_cloud_sql_checkpoint_loop",
"google_monitoring_alert_policy.relay_cloud_sql_disk",
"google_monitoring_alert_policy.relay_custom",
"google_monitoring_alert_policy.relay_gce_connection_headroom",
"google_monitoring_alert_policy.relay_postgres_retry_exhausted",
"google_monitoring_dashboard.relay_incident",
"google_project_iam_custom_role.github_production_relay_capacity_mutation",
"google_project_iam_custom_role.github_relay_asia_topology_mutation",
"google_project_iam_custom_role.github_relay_asia_topology_read",
+18 -2
View File
@@ -99,7 +99,7 @@ durably marked consumed before mutation and cannot authorize another run.
| Cloud SQL deadlocks | over 0 |
| Relay pool waiters | over 800 |
| Relay pool wait | over 2,500 ms |
| PostgreSQL retries in five minutes | over 300 |
| PostgreSQL retries in five minutes | over 2,000 |
| Exhausted PostgreSQL retries in five minutes | over 300 |
| Director instances | outside 5–6 |
| Director CPU or memory | over 80% |
@@ -138,7 +138,23 @@ heartbeats, and matching live admission.
`jsonPayload.event="orca_relay_postgres_transaction_retry"` in production
logs: healthy-day bursts reach 234/5min with zero exhausted retries and 26%
of five-minute windows over 20, while the 2026-08-23 lock-contention
incident ran roughly 2,200–3,000/5min.
incident ran roughly 2,200–3,000/5min by raw log-line count (the gate's
own `orca_relay_postgres_retries` metric read 1,510 for that window; see the
2026-09-04 entry).
- Recalibrated the PostgreSQL-retry freeze from 300 to 2,000 per five minutes
(2026-09-04). Basis: the global `relay_cells FOR UPDATE` lock made
successful retries a steady-state rate. Measured fleet-wide (director +
cells, summed per five minutes from the `orca_relay_postgres_retries`
log metric) over 2026-09-03T05Z..2026-09-04T05Z: p50 430 / p90 924 /
p99 1,320 / max 1,504; 55% of windows over 300; only 22% of 15-minute gates
clean at 300 versus 100% at 2,000. Three read-only dry-runs on 2026-09-04
froze on this bar (runs 33836470590, 33838698725) or on a genuine six-cell
crash storm (33837160275), blocking the same-cap roll that carries #18521
and the `beginProof` crash guard to the 23 cells. The 2026-08-23 incident
on this metric peaked at 1,510 then 646, so retries alone no longer
separate it from today's baseline; the exhausted-retry bar (incident peak
467 vs bar 300), director concurrency, and the pool bars carry that role.
Re-tighten after the fleet is on the 500 ms lock wait.
- Recalibrated the exhausted-PostgreSQL-retry freeze from 0 to 300 per five
minutes (2026-09-04). Basis: #18521 cut the request-path cell-inventory
lock wait from the 1 s pool `lock_timeout` to 500 ms, so contended waiters
+1
View File
@@ -242,6 +242,7 @@ resource "google_compute_instance_template" "relay_gce_cell" {
artifact_registry_host = "${var.region}-docker.pkg.dev"
relay_image = each.value.image
cloud_sql_proxy_image = var.relay_gce_cloud_sql_proxy_image
cloud_sql_private_ip = var.relay_cloud_sql_private_ip
# Keep cell-only plans independent from unrelated database configuration drift.
cloud_sql_connection_name = local.relay_database_connection_name
})
@@ -42,6 +42,12 @@ resource "google_compute_router_nat" "relay_gce" {
router = google_compute_router.relay_gce[0].name
nat_ip_allocate_option = "AUTO_ONLY"
source_subnetwork_ip_ranges_to_nat = "LIST_OF_SUBNETWORKS"
# Cells reach Cloud SQL's public IP through this NAT. The static default of 64 ports per VM
# filled during the 2026-09-04 incident and every cell's proxy dial timed out at once.
enable_dynamic_port_allocation = true
enable_endpoint_independent_mapping = false
min_ports_per_vm = 64
max_ports_per_vm = 4096
subnetwork {
name = google_compute_subnetwork.relay_gce[0].id
@@ -85,6 +91,12 @@ resource "google_compute_router_nat" "relay_gce_additional" {
router = google_compute_router.relay_gce_additional[each.key].name
nat_ip_allocate_option = "AUTO_ONLY"
source_subnetwork_ip_ranges_to_nat = "LIST_OF_SUBNETWORKS"
# Cells reach Cloud SQL's public IP through this NAT. The static default of 64 ports per VM
# filled during the 2026-09-04 incident and every cell's proxy dial timed out at once.
enable_dynamic_port_allocation = true
enable_endpoint_independent_mapping = false
min_ports_per_vm = 64
max_ports_per_vm = 4096
subnetwork {
name = google_compute_subnetwork.relay_gce_additional[each.key].id
@@ -109,6 +109,9 @@ docker run --detach \
--user 0:0 \
--volume "$${cloudsql_dir}:/cloudsql" \
'${cloud_sql_proxy_image}' \
%{ if cloud_sql_private_ip ~}
--private-ip \
%{ endif ~}
--unix-socket=/cloudsql \
'${cloud_sql_connection_name}'
@@ -37,6 +37,16 @@ locals {
description = "Relay PostgreSQL transactions that exhausted bounded retry."
filter = "((resource.type=\"cloud_run_revision\" AND (${local.relay_service_log_filter})) OR resource.type=\"gce_instance\") AND jsonPayload.event=\"orca_relay_postgres_transaction_exhausted\""
}
cell_process_exit = {
# The docker event stream is the only per-exit line: the relay's own crash footer only
# appears for unhandled rejections, and `container start` also counts healthy first boots.
description = "Relay cell container exits, one Docker `container die` event per process exit."
filter = "resource.type=\"gce_instance\" AND logName=\"projects/${var.project_id}/logs/cos_system\" AND jsonPayload.SYSLOG_IDENTIFIER=\"docker\" AND jsonPayload.MESSAGE:\"container die\" AND jsonPayload.MESSAGE:\"name=orca-relay)\""
}
cloud_sql_wal_checkpoint = {
description = "Cloud SQL checkpoints triggered by WAL volume instead of the timed schedule; a sustained run is the fsync loop that stalled every relay process at once on 2026-09-04."
filter = "resource.type=\"cloudsql_database\" AND resource.labels.database_id=\"${var.project_id}:${local.relay_database_instance_name}\" AND textPayload:\"checkpoint starting: wal\""
}
}
relay_runtime_metrics = {
@@ -523,3 +533,280 @@ resource "google_monitoring_alert_policy" "relay_cloud_sql_backends" {
mime_type = "text/markdown"
}
}
resource "google_monitoring_alert_policy" "relay_cloud_sql_checkpoint_loop" {
project = var.project_id
display_name = "Orca Relay: Cloud SQL checkpoint loop"
combiner = "OR"
enabled = true
notification_channels = var.relay_alert_notification_channels
conditions {
display_name = "WAL-triggered checkpoints above 3 in 5 minutes"
condition_threshold {
filter = "resource.type=\"cloudsql_database\" AND metric.type=\"logging.googleapis.com/user/orca_relay_cloud_sql_wal_checkpoint\""
comparison = "COMPARISON_GT"
threshold_value = 3
duration = "300s"
aggregations {
alignment_period = "300s"
per_series_aligner = "ALIGN_SUM"
cross_series_reducer = "REDUCE_SUM"
}
trigger {
count = 1
}
}
}
documentation {
content = "Healthy operation is one timed checkpoint every 5 minutes. Repeated `checkpoint starting: wal` lines mean WAL is outrunning `max_wal_size` and every checkpoint fsync stalls all relay SQL for seconds. Check `checkpoint complete` sync= times and disk write throughput against the PD-SSD ceiling; the fix is disk size and `max_wal_size` in the Terraform root that owns the instance (orca-cloud `infra/terraform-foundation`)."
mime_type = "text/markdown"
}
depends_on = [google_logging_metric.relay_incident]
}
resource "google_monitoring_alert_policy" "relay_cloud_sql_disk" {
project = var.project_id
display_name = "Orca Relay: Cloud SQL disk utilization"
combiner = "OR"
enabled = true
notification_channels = var.relay_alert_notification_channels
conditions {
display_name = "Cloud SQL disk above 70%"
condition_threshold {
filter = "resource.type=\"cloudsql_database\" AND resource.label.\"database_id\"=\"${var.project_id}:${local.relay_database_instance_name}\" AND metric.type=\"cloudsql.googleapis.com/database/disk/utilization\""
comparison = "COMPARISON_GT"
threshold_value = 0.7
duration = "600s"
aggregations {
alignment_period = "300s"
per_series_aligner = "ALIGN_MAX"
}
trigger {
count = 1
}
}
}
documentation {
content = "The shared auth/relay Cloud SQL disk is filling. `refresh_tokens` is the largest table and grows without pruning; grow the disk (IOPS scale with size) before it reaches the WAL checkpoint loop, and prune revoked token rows."
mime_type = "text/markdown"
}
}
resource "google_monitoring_alert_policy" "relay_cloud_nat_port_drops" {
count = local.relay_gce_configured ? 1 : 0
project = var.project_id
display_name = "Orca Relay: Cloud NAT port exhaustion"
combiner = "OR"
enabled = true
notification_channels = var.relay_alert_notification_channels
conditions {
display_name = "NAT packets dropped for lack of ports"
condition_threshold {
filter = "resource.type=\"nat_gateway\" AND resource.label.\"gateway_name\"=monitoring.regex.full_match(\"${local.relay_gce_name}(-.*)?\") AND metric.type=\"router.googleapis.com/nat/dropped_sent_packets_count\" AND metric.label.\"reason\"=\"OUT_OF_RESOURCES\""
comparison = "COMPARISON_GT"
threshold_value = 0
duration = "120s"
aggregations {
alignment_period = "60s"
per_series_aligner = "ALIGN_SUM"
cross_series_reducer = "REDUCE_SUM"
group_by_fields = ["resource.label.\"gateway_name\""]
}
trigger {
count = 1
}
}
}
documentation {
content = "Relay cells reach Cloud SQL's public IP through this NAT. Port exhaustion makes every cell's Cloud SQL Auth Proxy dial time out at once, which reads as a fleet-wide SQL stall with a healthy database. Check `nat/port_usage` per VM and raise `max_ports_per_vm` in `relay-gce-foundation.tf`, or move the database to a private IP."
mime_type = "text/markdown"
}
}
resource "google_monitoring_alert_policy" "relay_cell_process_exit" {
project = var.project_id
display_name = "Orca Relay: cell process exits"
combiner = "OR"
enabled = true
notification_channels = var.relay_alert_notification_channels
conditions {
display_name = "Cell container exits above 3 in 15 minutes"
condition_threshold {
filter = "resource.type=\"gce_instance\" AND metric.type=\"logging.googleapis.com/user/orca_relay_cell_process_exit\""
comparison = "COMPARISON_GT"
threshold_value = 3
duration = "0s"
aggregations {
alignment_period = "900s"
per_series_aligner = "ALIGN_SUM"
cross_series_reducer = "REDUCE_SUM"
group_by_fields = ["resource.label.\"instance_id\""]
}
trigger {
count = 1
}
}
}
documentation {
content = "A Relay GCE cell restarted its container more than three times in 15 minutes. Each exit drops every host and phone on that cell, and 201 exits went unpaged over 48 h on 2026-09-04. The instance hostname is `relay-<cell>-<suffix>`; read `jsonPayload.MESSAGE` on `cos_system` for the exit code and the container's own stderr for the stack before blaming MIG autoheal or load. A same-capacity roll is the remedy when the running image is behind."
mime_type = "text/markdown"
}
depends_on = [google_logging_metric.relay_incident]
}
# Why: the four signals that had to be assembled by hand during the 2026-09-04 incident.
resource "google_monitoring_dashboard" "relay_incident" {
project = var.project_id
dashboard_json = jsonencode({
displayName = "Orca Relay: incident overview"
mosaicLayout = {
columns = 12
tiles = [
{
xPos = 0
yPos = 0
width = 3
height = 4
widget = {
title = "Cloud SQL WAL checkpoints"
xyChart = {
dataSets = [{
plotType = "LINE"
targetAxis = "Y1"
timeSeriesQuery = {
timeSeriesFilter = {
filter = "metric.type=\"logging.googleapis.com/user/orca_relay_cloud_sql_wal_checkpoint\" AND resource.type=\"cloudsql_database\""
aggregation = {
alignmentPeriod = "300s"
perSeriesAligner = "ALIGN_SUM"
crossSeriesReducer = "REDUCE_SUM"
}
}
}
}]
yAxis = {
label = "checkpoints"
scale = "LINEAR"
}
}
}
},
{
xPos = 3
yPos = 0
width = 3
height = 4
widget = {
title = "Cloud NAT dropped packets"
xyChart = {
dataSets = [{
plotType = "LINE"
targetAxis = "Y1"
timeSeriesQuery = {
timeSeriesFilter = {
filter = "metric.type=\"router.googleapis.com/nat/dropped_sent_packets_count\" AND resource.type=\"nat_gateway\" AND resource.label.\"gateway_name\"=monitoring.regex.full_match(\"${local.relay_gce_name}(-.*)?\")"
aggregation = {
alignmentPeriod = "60s"
perSeriesAligner = "ALIGN_SUM"
crossSeriesReducer = "REDUCE_SUM"
groupByFields = ["resource.label.\"gateway_name\"", "metric.label.\"reason\""]
}
}
}
}]
yAxis = {
label = "packets"
scale = "LINEAR"
}
}
}
},
{
xPos = 6
yPos = 0
width = 3
height = 4
widget = {
title = "Auth refresh 401s"
xyChart = {
dataSets = [{
plotType = "LINE"
targetAxis = "Y1"
timeSeriesQuery = {
timeSeriesFilter = {
filter = "metric.type=\"logging.googleapis.com/user/orca_auth_refresh_401\""
aggregation = {
alignmentPeriod = "300s"
perSeriesAligner = "ALIGN_SUM"
crossSeriesReducer = "REDUCE_SUM"
}
}
}
}]
yAxis = {
label = "rejections"
scale = "LINEAR"
}
}
}
},
{
xPos = 9
yPos = 0
width = 3
height = 4
widget = {
title = "Standing desktop controls (fleet sum)"
xyChart = {
dataSets = [{
plotType = "LINE"
targetAxis = "Y1"
timeSeriesQuery = {
timeSeriesFilter = {
# ALIGN_MEAN, not ALIGN_SUM: each process reports its standing control count once per interval.
filter = "metric.type=\"logging.googleapis.com/user/orca_relay_controls\""
aggregation = {
alignmentPeriod = "300s"
perSeriesAligner = "ALIGN_MEAN"
crossSeriesReducer = "REDUCE_SUM"
}
}
}
}]
yAxis = {
label = "controls"
scale = "LINEAR"
}
}
}
}
]
}
})
depends_on = [google_logging_metric.relay_incident, google_logging_metric.relay_snapshot]
}
+6
View File
@@ -468,6 +468,12 @@ variable "relay_gce_fenced_cells" {
default = []
}
variable "relay_cloud_sql_private_ip" {
type = bool
description = "Dial Cloud SQL over its private IP inside this VPC instead of its public IP through Cloud NAT. Requires the foundation root's private services access peering to be applied first; a cell that cannot reach the private IP never becomes ready."
default = false
}
variable "relay_gce_cloud_sql_proxy_image" {
type = string
description = "Digest-pinned Cloud SQL Auth Proxy image used by private relay workers."
@@ -0,0 +1,18 @@
// Mirror of src/shared/relay-host-close-reason.ts in the Orca app repo half.
// A host control socket may close with one of these as its WebSocket close
// reason; the cell records it so a later phone rejection can name the cause.
// Anything else (including the empty reason of an abrupt 1006) means "unknown",
// which is what every peer that predates this file sends.
export const RELAY_HOST_CLOSE_REASON = {
SIGNED_OUT: 'signed-out'
} as const
export type RelayHostCloseReason =
(typeof RELAY_HOST_CLOSE_REASON)[keyof typeof RELAY_HOST_CLOSE_REASON]
const REASONS: readonly string[] = Object.values(RELAY_HOST_CLOSE_REASON)
export function relayHostCloseReasonFrom(value: unknown): RelayHostCloseReason | null {
const text = typeof value === 'string' ? value : (value?.toString() ?? '')
return REASONS.includes(text) ? (text as RelayHostCloseReason) : null
}
@@ -5,6 +5,7 @@ export * from './control-messages.js'
export * from './control-continuity.js'
export * from './credential-messages.js'
export * from './director-messages.js'
export * from './host-close-reason.js'
export * from './host-proof-transcript.js'
export * from './persistence-invariants.js'
export * from './protocol-limits.js'
+149 -21
View File
@@ -603,7 +603,7 @@ index 7b4b9e1f990fbf95b51528bb56dc9717f5b87532..2ae787c5bd4f3eba470584dc658a01a5
}
#endif
diff --git a/src/win/conpty.cc b/src/win/conpty.cc
index 7b286d3d644c26141df516929703aa6e129df4b2..ec6bf3932c65b89c013ff133dc6bf46a6a4082ce 100644
index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c97209248e 100644
--- a/src/win/conpty.cc
+++ b/src/win/conpty.cc
@@ -18,6 +18,7 @@
@@ -614,7 +614,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..ec6bf3932c65b89c013ff133dc6bf46a
#include <vector>
#include <Windows.h>
#include <strsafe.h>
@@ -44,12 +45,29 @@ struct pty_baton {
@@ -44,12 +45,39 @@ struct pty_baton {
HANDLE hOut;
HPCON hpc;
@@ -630,22 +630,32 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..ec6bf3932c65b89c013ff133dc6bf46a
+ // refused to create or assign one (an outer job without breakaway rights),
+ // in which case callers fall back to their pre-job behaviour.
+ HANDLE hJob = nullptr;
+
+ // Orca: teardown needs BOTH the shell's death and an explicit kill() before
+ // the baton can be freed, so each side records that it has run. Whichever
+ // arrives second frees it. Freeing on the shell's death alone -- what this
+ // file did before -- destroyed the only record of `hpc` while
+ // ClosePseudoConsole was still owed, which is why a self-exiting shell
+ // leaked its pseudoconsole and the console host it reaps (#18601 / F24).
+ bool shellExited = false;
+ bool consoleClosed = false;
pty_baton(int _id, HANDLE _hIn, HANDLE _hOut, HPCON _hpc) : id(_id), hIn(_hIn), hOut(_hOut), hpc(_hpc) {};
};
static std::vector<std::unique_ptr<pty_baton>> ptyHandles;
+// Orca: guards the job accessors below against the exit watcher thread. It does
+// NOT make the whole table safe -- PtyResize/PtyClear/PtyKill read it unlocked,
+// as they always have -- but it closes the window this patch opened, where the
+// watcher can close hShell/hJob and free the baton between a lookup and its use.
+// Orca: guards the job accessors below, and PtyKill, against the exit watcher
+// thread. It does NOT make the whole table safe -- PtyResize and PtyClear still
+// read it unlocked, as they always have -- but it closes the window this patch
+// opened, where the watcher can close hShell/hJob and free the baton between a
+// lookup and its use.
+// Handle VALUES are recycled aggressively, so an unguarded read could pass the
+// shell-pid check against an unrelated process and terminate the wrong job.
+static std::mutex ptyJobMutex;
static volatile LONG ptyCounter;
static pty_baton* get_pty_baton(int id) {
@@ -102,8 +120,27 @@ void SetupExitCallback(Napi::Env env, Napi::Function cb, pty_baton* baton) {
@@ -102,8 +130,31 @@ void SetupExitCallback(Napi::Env env, Napi::Function cb, pty_baton* baton) {
// Get process exit code.
GetExitCodeProcess(baton->hShell, (LPDWORD)(&exit_event->exit_code));
// Clean up handles
@@ -665,9 +675,13 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..ec6bf3932c65b89c013ff133dc6bf46a
+ // Why inside the lock: erasing frees the baton the job accessors hold a
+ // pointer to. Note remove_pty_baton must not be an assert() argument --
+ // NDEBUG would compile the call away and leak every baton.
+ const bool removed = remove_pty_baton(baton->id);
+ assert(removed);
+ (void)removed;
+ baton->shellExited = true;
+ if (baton->consoleClosed) {
+ const bool removed = remove_pty_baton(baton->id);
+ assert(removed);
+ (void)removed;
+ }
+ // Else PtyKill has not run yet and still owns hpc. It frees the baton.
+ }
+ // Why the lock ends here: BlockingCall below waits on the JS thread, and the
+ // JS thread can be waiting on ptyJobMutex inside PtyTerminateJob. Holding
@@ -675,7 +689,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..ec6bf3932c65b89c013ff133dc6bf46a
auto status = tsfn.BlockingCall(exit_event, callback); // In main thread
switch (status) {
@@ -409,6 +446,15 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
@@ -409,6 +460,15 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
throw errorWithCode(info, "UpdateProcThreadAttribute failed");
}
@@ -691,7 +705,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..ec6bf3932c65b89c013ff133dc6bf46a
PROCESS_INFORMATION piClient{};
fSuccess = !!CreateProcessW(
nullptr,
@@ -416,7 +462,10 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
@@ -416,7 +476,10 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
nullptr, // lpProcessAttributes
nullptr, // lpThreadAttributes
false, // bInheritHandles VERY IMPORTANT that this is false
@@ -703,7 +717,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..ec6bf3932c65b89c013ff133dc6bf46a
envArg, // lpEnvironment
mutableCwd.get(), // lpCurrentDirectory
&siEx.StartupInfo, // lpStartupInfo
@@ -426,8 +475,47 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
@@ -426,8 +489,47 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
throw errorWithCode(info, "Cannot create process");
}
@@ -753,7 +767,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..ec6bf3932c65b89c013ff133dc6bf46a
if (useConptyDll && fLoadedDll)
{
PFNRELEASEPSEUDOCONSOLE const pfnReleasePseudoConsole = (PFNRELEASEPSEUDOCONSOLE)GetProcAddress(
@@ -440,6 +528,8 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
@@ -440,6 +542,8 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
// Update handle
handle->hShell = piClient.hProcess;
@@ -762,7 +776,91 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..ec6bf3932c65b89c013ff133dc6bf46a
// Close the thread handle to avoid resource leak
CloseHandle(piClient.hThread);
@@ -567,6 +657,143 @@ static Napi::Value PtyKill(const Napi::CallbackInfo& info) {
@@ -544,29 +648,215 @@ static Napi::Value PtyKill(const Napi::CallbackInfo& info) {
int id = info[0].As<Napi::Number>().Int32Value();
const bool useConptyDll = info[1].As<Napi::Boolean>().Value();
- const pty_baton* handle = get_pty_baton(id);
+ // Orca: resolve the DLL BEFORE touching any baton state, for the same reason
+ // PtyConnect does it before creating anything. LoadConptyDll throws when
+ // conpty.dll is missing, and a throw after consoleClosed was set would strand
+ // the pseudoconsole permanently: the retry would find the work already
+ // claimed and do nothing. Only the useConptyDll path can throw here; the
+ // other returns kernel32.
+ HANDLE hLibrary = LoadConptyDll(info, useConptyDll);
+ PFNCLOSEPSEUDOCONSOLE pfnClosePseudoConsole = nullptr;
+ if (hLibrary != nullptr) {
+ pfnClosePseudoConsole = (PFNCLOSEPSEUDOCONSOLE)GetProcAddress(
+ (HMODULE)hLibrary,
+ useConptyDll ? "ConptyClosePseudoConsole" : "ClosePseudoConsole");
+ }
- if (handle != nullptr) {
- HANDLE hLibrary = LoadConptyDll(info, useConptyDll);
- bool fLoadedDll = hLibrary != nullptr;
- if (fLoadedDll)
- {
- PFNCLOSEPSEUDOCONSOLE const pfnClosePseudoConsole = (PFNCLOSEPSEUDOCONSOLE)GetProcAddress(
- (HMODULE)hLibrary,
- useConptyDll ? "ConptyClosePseudoConsole" : "ClosePseudoConsole");
- if (pfnClosePseudoConsole)
- {
- pfnClosePseudoConsole(handle->hpc);
+ // Orca: the baton now outlives the shell, so this runs on a self-exited pty
+ // too -- that is the whole point. Take what we need under the lock: the
+ // watcher thread nulls hShell the moment the shell dies, and TerminateProcess
+ // on a handle it just closed is an invalid-handle operation. Duplicating
+ // rather than reordering keeps upstream's close-then-terminate sequence.
+ HPCON hpc = nullptr;
+ HANDLE hShellDup = nullptr;
+ bool owed = false;
+ {
+ std::lock_guard<std::mutex> guard(ptyJobMutex);
+ pty_baton* handle = get_pty_baton(id);
+ // Why the consoleClosed check: a second kill() would otherwise close the
+ // same pseudoconsole twice. Upstream relied on the baton being gone.
+ if (handle != nullptr && !handle->consoleClosed) {
+ hpc = handle->hpc;
+ owed = true;
+ handle->consoleClosed = true;
+ // Null hShell means a self-exited pty, where there is nothing to kill.
+ if (useConptyDll && handle->hShell != nullptr) {
+ if (!DuplicateHandle(GetCurrentProcess(), handle->hShell, GetCurrentProcess(),
+ &hShellDup, 0, FALSE, DUPLICATE_SAME_ACCESS)) {
+ // Why terminate here instead of skipping: a failed duplication leaves
+ // hShellDup null, which is indistinguishable from the self-exit case,
+ // and skipping would leave the shell RUNNING after its pane closed --
+ // a worse outcome than the leak this all exists to fix. hShell is
+ // valid under this lock and TerminateProcess does not block, so the
+ // only cost is that this rare path kills before the console closes.
+ hShellDup = nullptr;
+ TerminateProcess(handle->hShell, 1);
+ }
+ }
+ if (handle->shellExited) {
+ const bool removed = remove_pty_baton(id);
+ assert(removed);
+ (void)removed;
}
+ // Else the shell is still running and the watcher frees the baton.
}
- if (useConptyDll) {
- TerminateProcess(handle->hShell, 1);
+ }
+
+ // Why outside the lock: ClosePseudoConsole blocks until the conout side has
+ // drained, and the watcher must be able to take the lock while it does.
+ if (owed) {
+ if (pfnClosePseudoConsole)
+ {
+ pfnClosePseudoConsole(hpc);
+ }
+ if (hShellDup != nullptr) {
+ TerminateProcess(hShellDup, 1);
+ CloseHandle(hShellDup);
}
}
return env.Undefined();
}
@@ -808,9 +906,11 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..ec6bf3932c65b89c013ff133dc6bf46a
+ * Orca: the pids still alive in this pty's tree, straight from the kernel.
+ *
+ * Descendant liveness for a tree that is still tracked, including children that
+ * detached from the console. Once the shell exits the baton is gone, so this
+ * returns null rather than an empty list -- null means "no answer", never
+ * "they died". Also returns null when no job was assigned.
+ * detached from the console. Once the shell exits the watcher nulls hJob, which
+ * ownsShell rejects, so this returns null rather than an empty list -- null
+ * means "no answer", never "they died". (The baton itself now outlives the
+ * shell, until kill() runs; hJob is what makes the answer null.) Also returns
+ * null when no job was assigned.
+ *
+ * Does not include the ConPTY console host: CreatePseudoConsole spawns it
+ * before this job exists, so it is not a member and ClosePseudoConsole is what
@@ -906,7 +1006,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..ec6bf3932c65b89c013ff133dc6bf46a
/**
* Init
*/
@@ -577,6 +804,9 @@ Napi::Object init(Napi::Env env, Napi::Object exports) {
@@ -577,6 +867,9 @@ Napi::Object init(Napi::Env env, Napi::Object exports) {
exports.Set("resize", Napi::Function::New(env, PtyResize));
exports.Set("clear", Napi::Function::New(env, PtyClear));
exports.Set("kill", Napi::Function::New(env, PtyKill));
@@ -917,7 +1017,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..ec6bf3932c65b89c013ff133dc6bf46a
};
diff --git a/lib/windowsPtyAgent.js b/lib/windowsPtyAgent.js
index a358ffb..fb3a96f 100644
index a358ffb177357e177661033c1b092f9c9d0e5f5a..26c2a4c58799ce649f5113131e4c52f7ed2d87ad 100644
--- a/lib/windowsPtyAgent.js
+++ b/lib/windowsPtyAgent.js
@@ -136,6 +136,9 @@ var WindowsPtyAgent = /** @class */ (function () {
@@ -930,6 +1030,20 @@ index a358ffb..fb3a96f 100644
this._outSocket.readable = false;
this._getConsoleProcessList().then(function (consoleProcessList) {
consoleProcessList.forEach(function (pid) {
@@ -154,9 +157,10 @@ var WindowsPtyAgent = /** @class */ (function () {
// Close the input write handle to signal the end of session.
this._inSocket.destroy();
this._ptyNative.kill(this._pty, this._useConptyDll);
- this._outSocket.on('data', function () {
- _this._conoutSocketWorker.dispose();
- });
+ // Orca: dispose unconditionally, as the non-DLL branch above does.
+ // Waiting for another 'data' event leaks the conout worker on every
+ // self-exiting shell, because no more data ever arrives (F24).
+ this._conoutSocketWorker.dispose();
}
}
else {
diff --git a/lib/windowsTerminal.js b/lib/windowsTerminal.js
index 3c38f89..e20b3e6 100644
--- a/lib/windowsTerminal.js
@@ -1015,7 +1129,7 @@ index 3c38f89..e20b3e6 100644
\ No newline at end of file
+//# sourceMappingURL=windowsTerminal.js.map
diff --git a/src/windowsPtyAgent.ts b/src/windowsPtyAgent.ts
index d705444..ce611b8 100644
index d7054449516f0c9a62af351c2caa17331206d530..0c28a32e2e1db2b3f208ddde8443cd4e67bb1ad6 100644
--- a/src/windowsPtyAgent.ts
+++ b/src/windowsPtyAgent.ts
@@ -143,6 +143,9 @@ export class WindowsPtyAgent {
@@ -1028,6 +1142,20 @@ index d705444..ce611b8 100644
this._outSocket.readable = false;
this._getConsoleProcessList().then(consoleProcessList => {
consoleProcessList.forEach((pid: number) => {
@@ -159,9 +162,10 @@ export class WindowsPtyAgent {
// Close the input write handle to signal the end of session.
this._inSocket.destroy();
(this._ptyNative as IConptyNative).kill(this._pty, this._useConptyDll);
- this._outSocket.on('data', () => {
- this._conoutSocketWorker.dispose();
- });
+ // Orca: dispose unconditionally, as the non-DLL branch above does.
+ // Waiting for another 'data' event leaks the conout worker on every
+ // self-exiting shell, because no more data ever arrives (F24).
+ this._conoutSocketWorker.dispose();
}
} else {
// Because pty.kill closes the handle, it will kill most processes by itself.
diff --git a/src/windowsTerminal.ts b/src/windowsTerminal.ts
index 13f6c6d..eda63c8 100644
--- a/src/windowsTerminal.ts
@@ -0,0 +1,158 @@
const { createHash } = require('node:crypto')
const { readFileSync, renameSync, rmSync, writeFileSync } = require('node:fs')
const { join, resolve } = require('node:path')
/**
* Release the ConPTY teardown handles a relay's npm-installed node-pty never releases.
*
* Two files, and the ORDER of one of the edits is the whole fix.
*
* `windowsPtyAgent.js` -- `kill()` flips `readable` on both sockets and destroys neither.
* `_cleanUpProcess` destroys `_outSocket`, so the conout handle comes back; nothing ever destroys
* `_inSocket`, and it wraps a real Windows named-pipe handle from `fs.openSync(term.conin, 'w')`.
* Every terminal leaks one File handle for the life of the host process.
*
* The obvious fix -- and the one the desktop patch ships -- releases it at the TOP of the branch,
* before `_getConsoleProcessList()` forks and before the native kill. That is measurably worse than
* leaving the leak alone: teardown aborts partway, the forked console-list agent is never reaped,
* and both pipe handles stay alive instead of one. This asset releases it at the END of the branch
* instead, after the fork and the kill have already happened.
*
* Measured on a Windows SSH host, 20 spawn/kill cycles, handles bucketed by NT object type
* (identical numbers standalone and through a real relay):
*
* published node-pty File +1/terminal, Process flat
* desktop patch placement File +2/terminal, Process +1/terminal <-- 3x WORSE
* released last (here) File flat, Process flat
*
* `windowsTerminal.js` carries the desktop's error-listener hunks verbatim. The conin listener is
* what keeps a pipe error retiring one terminal instead of the host -- its own comment names the
* failure mode: "Without a listener, Node promotes errors such as write EAGAIN to uncaughtException".
* It is not what fixes the leak (adding it changed nothing on its own), but it is the guard that
* makes destroying conin safe at all.
*
* Why this ships as a relay asset rather than only in config/patches/node-pty@1.1.0.patch: pnpm
* patches do not cross the SSH boundary -- a relay host runs the tree `npm install` put there.
*
* DELIBERATE DIVERGENCE FROM THE DESKTOP: the desktop patch has the early placement and therefore
* the +2 File / +1 Process regression, measured against its exact installed tree. Correcting it
* there is a separate change with its own verification, so the two trees differ on this one hunk on
* purpose, and the test pins that so a future "sync the patches" does not copy the bug back.
*
* NOT ADDRESSED, AND A SEPARATE DEFECT THAT IS STILL OPEN: a terminal that exits on its own is
* still torn down through `kill()` -- both hosts call `destroy()` on natural exit and
* `WindowsTerminal.destroy()` is `kill()` -- but the shell is already gone by then, and the
* ordering this patch relies on does not hold. Measured over 20 self-exit cycles with that
* `destroy()` issued: published +3 File/+1 Process per terminal, desktop-patched +2/+1, this tree
* +2/+1. So this patch does not close it and the desktop patch does not either. It is reachable
* for every Windows user, local and relay, on every terminal closed by typing `exit`.
*/
const EXPECTED_NODE_PTY_VERSION = '1.1.0'
/** Each entry is one published file, its patched form, and the edits between them. */
const PATCH_TARGETS = [
{
relativePath: ['lib', 'windowsPtyAgent.js'],
originalSha256: '8636d16b38266112204061a22b135734177c242837982fd3a4055be726efa64a',
patchedSha256: '1e23ef480569e73706e3ab4f5482c7e553c76f51414ae8e7b0bdcc2fd75f7280',
replacements: [
[
' this._ptyNative.kill(this._pty, this._useConptyDll);\n this._conoutSocketWorker.dispose();\n',
' this._ptyNative.kill(this._pty, this._useConptyDll);\n this._conoutSocketWorker.dispose();\n // Orca: released AFTER the console-list fork and the native kill, not before them.\n // Destroying conin first aborts teardown partway -- measured on a Windows SSH relay\n // as +2 File and +1 Process handles per terminal, against +1 File unpatched.\n this._inSocket.destroy();\n'
]
]
},
{
relativePath: ['lib', 'windowsTerminal.js'],
originalSha256: 'c3a65716f53fed0135a8a633373d5f9c2ab092544d651f27ef0a67096dd3bcd9',
patchedSha256: '8247ecd69be8b18257050fb026b290024612c5ffc6d492ff1d46f81e613be2cf',
replacements: [
[
' _this._agent = new windowsPtyAgent_1.WindowsPtyAgent(file, args, parsedEnv, cwd, _this._cols, _this._rows, false, opt.useConpty, opt.useConptyDll, opt.conptyInheritCursor);\n _this._socket = _this._agent.outSocket;\n // Not available until `ready` event emitted.\n _this._pid = _this._agent.innerPid;',
" _this._agent = new windowsPtyAgent_1.WindowsPtyAgent(file, args, parsedEnv, cwd, _this._cols, _this._rows, false, opt.useConpty, opt.useConptyDll, opt.conptyInheritCursor);\n _this._socket = _this._agent.outSocket;\n // Attach before readiness so a broken ConPTY output pipe cannot be unhandled.\n _this._socket.on('error', function (err) {\n var code = err && err.code;\n // PTY output can report EPIPE before `_close()` wins the race.\n _this._close();\n if (code === 'EPIPE' || code === 'ERR_STREAM_PUSH_AFTER_EOF' || code === 'ERR_STREAM_DESTROYED') {\n return;\n }\n // EIO, happens when someone closes our child process: the only process\n // in the terminal.\n // node < 0.6.14: errno 5\n // node >= 0.6.14: read EIO\n if (typeof code === 'string') {\n if (~code.indexOf('errno 5') || ~code.indexOf('EIO'))\n return;\n }\n // Throw anything else.\n if (_this.listeners('error').length < 2) {\n throw err;\n }\n });\n // Not available until `ready` event emitted.\n _this._pid = _this._agent.innerPid;"
],
[
" }\n });\n // Shutdown if `error` event is emitted.\n _this._socket.on('error', function (err) {\n // Close terminal session.\n _this._close();\n // EIO, happens when someone closes our child process: the only process\n // in the terminal.\n // node < 0.6.14: errno 5\n // node >= 0.6.14: read EIO\n if (err.code) {\n if (~err.code.indexOf('errno 5') || ~err.code.indexOf('EIO'))\n return;\n }\n // Throw anything else.\n if (_this.listeners('error').length < 2) {\n throw err;\n }\n });\n // Cleanup after the socket is closed.\n _this._socket.on('close', function () {",
" }\n });\n // Cleanup after the socket is closed.\n _this._socket.on('close', function () {"
],
[
' _this._readable = true;\n _this._writable = true;\n _this._forwardEvents();\n return _this;',
" _this._readable = true;\n _this._writable = true;\n // A ConPTY input-pipe error must retire only this terminal. Without a listener, Node promotes\n // errors such as write EAGAIN to uncaughtException and kills every PTY in the daemon.\n _this._agent.inSocket.on('error', function () {\n if (!_this._writable) {\n return;\n }\n _this._close();\n try {\n _this._agent.kill();\n }\n catch (_a) {\n // The failing pipe may have raced process exit; the terminal is already unwritable.\n }\n });\n _this._forwardEvents();\n return _this;"
],
[
'exports.WindowsTerminal = WindowsTerminal;\n//# sourceMappingURL=windowsTerminal.js.map',
'exports.WindowsTerminal = WindowsTerminal;\n//# sourceMappingURL=windowsTerminal.js.map\n'
]
]
}
]
function inspectTarget(relayDir, target) {
const nodePtyDir = resolve(relayDir, 'node_modules', 'node-pty')
const packageJson = JSON.parse(readFileSync(join(nodePtyDir, 'package.json'), 'utf8'))
if (packageJson.version !== EXPECTED_NODE_PTY_VERSION) {
throw new Error(
`Refusing to patch node-pty ${packageJson.version}; expected ${EXPECTED_NODE_PTY_VERSION}`
)
}
const filePath = join(nodePtyDir, ...target.relativePath)
return { filePath, source: readFileSync(filePath, 'utf8') }
}
function assertPatchedNodePtyWindowsTeardown(relayDir = process.cwd()) {
for (const target of PATCH_TARGETS) {
const inspected = inspectTarget(relayDir, target)
if (sourceSha256(inspected.source) !== target.patchedSha256) {
throw new Error(
`node-pty ConPTY teardown release is not installed in ${target.relativePath.join('/')}`
)
}
}
}
function patchNodePtyWindowsTeardown(relayDir = process.cwd()) {
for (const target of PATCH_TARGETS) {
const inspected = inspectTarget(relayDir, target)
const sourceHash = sourceSha256(inspected.source)
if (sourceHash === target.patchedSha256) {
continue
}
if (sourceHash !== target.originalSha256) {
throw new Error(
`Refusing to patch unexpected node-pty source in ${target.relativePath.join('/')}`
)
}
let patchedSource = inspected.source
for (const [from, to] of target.replacements) {
// Why the count check: an anchor that matched twice would patch the wrong site silently, and
// the hash below would then reject a tree this script had already rewritten.
if (patchedSource.split(from).length - 1 !== 1) {
throw new Error(`Refusing to patch ${target.relativePath.join('/')}; anchor is not unique`)
}
patchedSource = patchedSource.replace(from, to)
}
const temporaryPath = `${inspected.filePath}.orca-patch-${process.pid}`
// Why: a terminated remote install must leave either known source version recoverable on reconnect.
try {
writeFileSync(temporaryPath, patchedSource)
renameSync(temporaryPath, inspected.filePath)
} finally {
rmSync(temporaryPath, { force: true })
}
}
assertPatchedNodePtyWindowsTeardown(relayDir)
}
function sourceSha256(source) {
return createHash('sha256').update(source).digest('hex')
}
if (require.main === module) {
patchNodePtyWindowsTeardown()
}
module.exports = {
assertPatchedNodePtyWindowsTeardown,
patchNodePtyWindowsTeardown
}
+11
View File
@@ -57,6 +57,13 @@ const NODE_PTY_CONSOLE_LIST_PATCH_SOURCE = join(
'relay-assets',
NODE_PTY_CONSOLE_LIST_PATCH_FILENAME
)
const NODE_PTY_WINDOWS_TEARDOWN_PATCH_FILENAME = 'node-pty-1.1.0-windows-pty-teardown-patch.cjs'
const NODE_PTY_WINDOWS_TEARDOWN_PATCH_SOURCE = join(
ROOT,
'config',
'relay-assets',
NODE_PTY_WINDOWS_TEARDOWN_PATCH_FILENAME
)
const NODE_PTY_MASTER_CLOEXEC_PATCH_FILENAME = 'node-pty-1.1.0-master-cloexec-patch.cjs'
const NODE_PTY_MASTER_CLOEXEC_PATCH_SOURCE = join(
ROOT,
@@ -132,6 +139,10 @@ for (const platform of RELAY_BUILD_PLATFORMS) {
NODE_PTY_CONSOLE_LIST_PATCH_SOURCE,
join(outDir, NODE_PTY_CONSOLE_LIST_PATCH_FILENAME)
)
copyFileSync(
NODE_PTY_WINDOWS_TEARDOWN_PATCH_SOURCE,
join(outDir, NODE_PTY_WINDOWS_TEARDOWN_PATCH_FILENAME)
)
}
copyFileSync(
NODE_PTY_MASTER_CLOEXEC_PATCH_SOURCE,
+1 -1
View File
@@ -492,7 +492,7 @@
"ko": "agent CLI를 찾지 못했습니다. 하나를 설치하거나 설정에서 기본 agent를 선택하세요."
},
"auto.components.Terminal.7958465754": {
"ko": "실행 중인 프로세스가 있는 로컬 terminals이 있습니다. 그래도 창을 닫으시겠습니까?"
"ko": "실행 중인 프로세스가 있는 terminals이 있습니다. 그래도 창을 닫으시겠습니까?"
},
"auto.components.Terminal.cdc9ac4b2d": {
"ko": "편집기"
@@ -0,0 +1,213 @@
// The relay's copy of the ConPTY teardown release, and the guard that keeps it in lockstep with the
// desktop's own node-pty patch. pnpm patches do not cross the SSH boundary, so a relay runs the tree
// `npm install` put there; the desktop had this fix and the relay did not, and every terminal on a
// Windows SSH host leaked one File handle for the life of the relay process.
//
// The ORDER of the conin release is the fix. Releasing it at the top of the branch -- what the
// desktop patch does -- was measured at 3x WORSE than shipping nothing (File +2/terminal and a new
// Process +1/terminal); releasing it after the console-list fork and the native kill is flat.
import { createRequire } from 'node:module'
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { join, resolve } from 'node:path'
import { afterEach, describe, expect, it } from 'vitest'
const require = createRequire(import.meta.url)
const {
assertPatchedNodePtyWindowsTeardown,
patchNodePtyWindowsTeardown
} = require('../relay-assets/node-pty-1.1.0-windows-pty-teardown-patch.cjs')
const projectDir = resolve(import.meta.dirname, '..', '..')
const cleanupDirs = []
const PATCHED_FILES = ['windowsPtyAgent.js', 'windowsTerminal.js']
/** The hunks config/patches/node-pty@1.1.0.patch adds to the installed desktop tree. */
const DESKTOP_HUNKS = {
'windowsPtyAgent.js': [
[
[
' this._inSocket.readable = false;',
' // The non-DLL path previously only flipped `readable`, leaving the',
' // conin PipeWrap alive until the host exited (#947).',
' this._inSocket.destroy();',
' this._outSocket.readable = false;',
''
].join('\n'),
[
' this._inSocket.readable = false;',
' this._outSocket.readable = false;',
''
].join('\n')
],
// The useConptyDll branch, which only the DESKTOP runs -- the relay takes the
// non-DLL branch above, where the dispose is already unconditional. Listed here
// so un-applying still yields published; the relay asset needs no counterpart.
[
[
' // Orca: dispose unconditionally, as the non-DLL branch above does.',
" // Waiting for another 'data' event leaks the conout worker on every",
' // self-exiting shell, because no more data ever arrives (F24).',
' this._conoutSocketWorker.dispose();',
''
].join('\n'),
[
" this._outSocket.on('data', function () {",
' _this._conoutSocketWorker.dispose();',
' });',
''
].join('\n')
]
],
'windowsTerminal.js': [
[
' // Attach before readiness so a broken ConPTY output pipe cannot be unhandled.',
null
],
[' // A ConPTY input-pipe error must retire only this terminal.', null]
]
}
function desktopPath(file) {
return join(projectDir, 'node_modules', 'node-pty', 'lib', file)
}
afterEach(() => {
for (const dir of cleanupDirs.splice(0)) {
rmSync(dir, { recursive: true, force: true })
}
})
describe('Windows SSH relay node-pty ConPTY teardown patch', () => {
// Why reconstruct rather than vendor upstream: the installed tree IS the published file plus the
// desktop's hunks, so un-applying them yields upstream exactly -- and pinning that against this
// asset's own hashes is what fails loudly if either side of the pair moves.
it('takes the desktop error listeners verbatim', () => {
const fixture = writeNodePtyFixture('1.1.0')
patchNodePtyWindowsTeardown(fixture.root)
expect(readFileSync(join(fixture.libDir, 'windowsTerminal.js'), 'utf8')).toBe(
readFileSync(desktopPath('windowsTerminal.js'), 'utf8')
)
})
// The one hunk that must NOT match the desktop, and the reason is measured, not stylistic:
// releasing conin before `_getConsoleProcessList()` forks aborts teardown partway.
it('releases conin after the console-list fork, not before it like the desktop patch', () => {
const fixture = writeNodePtyFixture('1.1.0')
patchNodePtyWindowsTeardown(fixture.root)
const patched = readFileSync(join(fixture.libDir, 'windowsPtyAgent.js'), 'utf8')
const branch = patched.slice(
patched.indexOf('if (!this._useConptyDll) {'),
patched.indexOf('else {', patched.indexOf('if (!this._useConptyDll) {'))
)
expect(branch).toContain('this._inSocket.destroy();')
expect(branch.indexOf('this._inSocket.destroy();')).toBeGreaterThan(
branch.indexOf('this._conoutSocketWorker.dispose();')
)
expect(branch.indexOf('this._inSocket.destroy();')).toBeGreaterThan(
branch.indexOf('this._getConsoleProcessList()')
)
// Pinned so a future "sync the relay asset to config/patches" cannot copy the regression back.
expect(patched).not.toBe(readFileSync(desktopPath('windowsPtyAgent.js'), 'utf8'))
})
it('installs and verifies idempotently', () => {
const fixture = writeNodePtyFixture('1.1.0')
patchNodePtyWindowsTeardown(fixture.root)
const once = PATCHED_FILES.map((file) => readFileSync(join(fixture.libDir, file), 'utf8'))
for (const file of PATCHED_FILES) {
expect(existsSync(`${join(fixture.libDir, file)}.orca-patch-${process.pid}`)).toBe(false)
}
expect(() => assertPatchedNodePtyWindowsTeardown(fixture.root)).not.toThrow()
patchNodePtyWindowsTeardown(fixture.root)
expect(PATCHED_FILES.map((file) => readFileSync(join(fixture.libDir, file), 'utf8'))).toEqual(
once
)
})
it('refuses a different package version or unexpected source', () => {
const wrongVersion = writeNodePtyFixture('1.2.0-beta.11')
expect(() => patchNodePtyWindowsTeardown(wrongVersion.root)).toThrow('expected 1.1.0')
for (const file of PATCHED_FILES) {
const drifted = writeNodePtyFixture('1.1.0')
const path = join(drifted.libDir, file)
writeFileSync(path, `${readFileSync(path, 'utf8')}\n// drift`)
expect(() => patchNodePtyWindowsTeardown(drifted.root)).toThrow('unexpected node-pty')
}
})
it('refuses a half-applied tree, so one file cannot pass for both', () => {
for (const file of PATCHED_FILES) {
const partial = writeNodePtyFixture('1.1.0')
const fixture = writeNodePtyFixture('1.1.0')
patchNodePtyWindowsTeardown(fixture.root)
writeFileSync(join(partial.libDir, file), readFileSync(join(fixture.libDir, file), 'utf8'))
expect(() => assertPatchedNodePtyWindowsTeardown(partial.root)).toThrow('is not installed')
}
})
})
/** A published node-pty tree, rebuilt by un-applying the desktop hunks from the installed one. */
function writeNodePtyFixture(version) {
const root = mkdtempSync(join(projectDir, '.node-pty-teardown-patch-test-'))
cleanupDirs.push(root)
const libDir = join(root, 'node_modules', 'node-pty', 'lib')
mkdirSync(libDir, { recursive: true })
writeFileSync(join(root, 'node_modules', 'node-pty', 'package.json'), JSON.stringify({ version }))
for (const file of PATCHED_FILES) {
const desktop = readFileSync(desktopPath(file), 'utf8')
for (const [marker] of DESKTOP_HUNKS[file]) {
expect(desktop).toContain(marker)
}
writeFileSync(join(libDir, file), unapplyDesktopHunks(file, desktop))
}
return { root, libDir }
}
/**
* Reverse of the published-to-desktop transform.
*
* `windowsTerminal.js` is taken verbatim from the desktop, so the asset's own replacement table is
* the transform and reversing it is exact. `windowsPtyAgent.js` deliberately diverges, so its
* published form is rebuilt from the desktop hunk instead -- which is also what makes this file the
* place that notices if the desktop hunk itself ever moves.
*/
function unapplyDesktopHunks(file, desktop) {
if (file === 'windowsPtyAgent.js') {
let published = desktop
for (const [patched, original] of DESKTOP_HUNKS[file]) {
expect(published.split(patched).length - 1).toBe(1)
published = published.replace(patched, original)
}
return published
}
const asset = readFileSync(
join(projectDir, 'config', 'relay-assets', 'node-pty-1.1.0-windows-pty-teardown-patch.cjs'),
'utf8'
)
const { PATCH_TARGETS } = loadPatchTargets(asset)
const target = PATCH_TARGETS.find((entry) => entry.relativePath.at(-1) === file)
expect(target).toBeDefined()
let published = desktop
for (const [from, to] of target.replacements.toReversed()) {
expect(published.split(to).length - 1).toBe(1)
published = published.replace(to, from)
}
return published
}
function loadPatchTargets(assetSource) {
const module = { exports: {} }
const factory = new Function(
'module',
'exports',
'require',
`${assetSource}\nmodule.exports.PATCH_TARGETS = PATCH_TARGETS`
)
factory(module, module.exports, require)
return module.exports
}
@@ -0,0 +1,39 @@
import { readdirSync, readFileSync } from 'node:fs'
import { join, resolve } from 'node:path'
import { describe, expect, it } from 'vitest'
import { parse } from 'yaml'
const skillsDir = resolve(import.meta.dirname, '../../skills')
// Why: the Agent Skills spec caps `description` at 1024 chars and conforming installers
// reject the whole skill (#17935); the frontmatter is what the installer parses, so check it.
const MAX_DESCRIPTION_LENGTH = 1024
function readDescription(skillName) {
const skillMarkdown = readFileSync(join(skillsDir, skillName, 'SKILL.md'), 'utf8')
const frontmatter = /^---\r?\n([\s\S]*?)\r?\n---\r?\n/u.exec(skillMarkdown)?.[1]
expect(frontmatter, `${skillName}: missing frontmatter`).toBeDefined()
return parse(frontmatter ?? '').description
}
describe('bundled skill descriptions', () => {
const skillNames = readdirSync(skillsDir, { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.map((entry) => entry.name)
it('discovers the bundled skills', () => {
expect(skillNames).toContain('orchestration')
})
it.each(skillNames)('%s keeps description within the Agent Skills spec limit', (name) => {
const description = readDescription(name)
expect(typeof description, `${name}: description must be a string`).toBe('string')
expect(description.trim().length, `${name}: description is empty`).toBeGreaterThan(0)
expect(
description.length,
`${name}: description is ${description.length} chars`
).toBeLessThanOrEqual(MAX_DESCRIPTION_LENGTH)
})
})
+4 -4
View File
@@ -1,5 +1,5 @@
<svg xmlns="http://www.w3.org/2000/svg" width="106" height="20" role="img" aria-label="downloads: 38m">
<title>downloads: 38m</title>
<svg xmlns="http://www.w3.org/2000/svg" width="106" height="20" role="img" aria-label="downloads: 39m">
<title>downloads: 39m</title>
<linearGradient id="s" x2="0" y2="100%">
<stop offset="0" stop-color="#bbb" stop-opacity=".1"/>
<stop offset="1" stop-opacity=".1"/>
@@ -15,7 +15,7 @@
<g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="11">
<text x="37" y="15" fill="#010101" fill-opacity=".3">downloads</text>
<text x="37" y="14">downloads</text>
<text x="90" y="15" fill="#010101" fill-opacity=".3">38m</text>
<text x="90" y="14">38m</text>
<text x="90" y="15" fill="#010101" fill-opacity=".3">39m</text>
<text x="90" y="14">39m</text>
</g>
</svg>

Before

Width:  |  Height:  |  Size: 935 B

After

Width:  |  Height:  |  Size: 935 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 386 KiB

+2 -2
View File
@@ -243,9 +243,9 @@ Associez-la à l'app de bureau pour surveiller et piloter vos agents depuis votr
- **Discord :** Rejoignez la communauté sur **[Discord](https://discord.gg/fzjDKHxv8Q)**.
- **Twitter / X :** Suivez **[@orca_build](https://x.com/orca_build)** pour les news et annonces.
- **WeChat :** Scannez pour rejoindre le groupe WeChat 8 de la communauté Orca.
- **WeChat :** Scannez pour rejoindre le groupe WeChat 8 de la communauté Orca. Le groupe 8 est peut-être complet ; dans ce cas, scannez plutôt le QR code du groupe 9.
<img src="../assets/wechat-qr-group8.jpg" alt="QR code WeChat groupe 8 de la communauté Orca" width="160" />
<img src="../assets/wechat-qr-group8.jpg" alt="QR code WeChat groupe 8 de la communauté Orca" width="160" />&nbsp;&nbsp;<img src="../assets/wechat-qr-group9.jpg" alt="QR code WeChat groupe 9 de la communauté Orca" width="160" />
- **Feedback &amp; idées :** On ship vite. Il manque quelque chose ? [Demandez une feature](https://github.com/stablyai/orca/issues).
- **Confidentialité :** Voir la [doc confidentialité &amp; télémétrie](https://www.onorca.dev/docs/telemetry) pour ce qu'Orca collecte en anonyme et comment désactiver la télémétrie.
+2 -2
View File
@@ -238,9 +238,9 @@ yay -S stably-orca-bin
- **Discord:** **[Discord](https://discord.gg/fzjDKHxv8Q)** 커뮤니티에 참여하세요.
- **Twitter / X:** 업데이트와 공지는 **[@orca_build](https://x.com/orca_build)** 를 팔로우하세요.
- **WeChat:** QR 코드를 스캔해 Orca 커뮤니티 WeChat 그룹 8에 참여하세요.
- **WeChat:** QR 코드를 스캔해 Orca 커뮤니티 WeChat 그룹 8에 참여하세요. 그룹 8이 가득 찼을 수 있으니, 그런 경우 그룹 9 QR 코드를 스캔하세요.
<img src="../assets/wechat-qr-group8.jpg" alt="Orca 커뮤니티 WeChat 그룹 8 QR 코드" width="160" />
<img src="../assets/wechat-qr-group8.jpg" alt="Orca 커뮤니티 WeChat 그룹 8 QR 코드" width="160" />&nbsp;&nbsp;<img src="../assets/wechat-qr-group9.jpg" alt="Orca 커뮤니티 WeChat 그룹 9 QR 코드" width="160" />
- **피드백과 아이디어:** 우리는 빠르게 출시합니다. 필요한 기능이 있나요? [새 기능을 요청](https://github.com/stablyai/orca/issues)하세요.
- **개인정보 보호:** Orca가 수집하는 익명 사용 데이터와 수집 거부 방법은 [개인정보 및 텔레메트리 문서](https://www.onorca.dev/docs/telemetry)를 참고하세요.
+2 -2
View File
@@ -235,9 +235,9 @@ yay -S stably-orca-bin
- **Discord:** 加入 **[Discord](https://discord.gg/fzjDKHxv8Q)** 社区。
- **Twitter / X:** 关注 **[@orca_build](https://x.com/orca_build)** 获取更新和公告。
- **微信:** 扫码加入 Orca 社区微信第 8 群。
- **微信:** 扫码加入 Orca 社区微信第 8 群。第 8 群可能已满,如遇这种情况请扫描第 9 群二维码。
<img src="../assets/wechat-qr-group8.jpg" alt="Orca 社区微信第 8 群二维码" width="160" />
<img src="../assets/wechat-qr-group8.jpg" alt="Orca 社区微信第 8 群二维码" width="160" />&nbsp;&nbsp;<img src="../assets/wechat-qr-group9.jpg" alt="Orca 社区微信第 9 群二维码" width="160" />
- **反馈与想法:** 我们发布很快。缺少什么功能?[提交功能请求](https://github.com/stablyai/orca/issues)。
- **隐私:** 查看[隐私与遥测文档](https://www.onorca.dev/docs/telemetry),了解 Orca 收集哪些匿名使用数据以及如何退出。
+6 -1
View File
@@ -19,6 +19,7 @@ type MobileHomeHostListProps = {
hostAttempts: Record<string, number>
hostLastConnected: Record<string, number | null>
hostPairingRejected: Record<string, boolean>
hostSignedOut: Record<string, boolean>
hostPaths: Record<string, MobileConnectionPath>
hostPendingPaths: Record<string, MobileConnectionPath | null>
hosts: HostCatalogEntry[]
@@ -40,6 +41,7 @@ export function MobileHomeHostList(props: MobileHomeHostListProps) {
hostAttempts={props.hostAttempts}
hostLastConnected={props.hostLastConnected}
hostPairingRejected={props.hostPairingRejected}
hostSignedOut={props.hostSignedOut}
hostPaths={props.hostPaths}
hostPendingPaths={props.hostPendingPaths}
hostStates={props.hostStates}
@@ -54,6 +56,7 @@ export function MobileHomeHostList(props: MobileHomeHostListProps) {
props.hostAttempts,
props.hostLastConnected,
props.hostPairingRejected,
props.hostSignedOut,
props.hostPaths,
props.hostPendingPaths,
props.hostStates,
@@ -91,6 +94,7 @@ type MobileHomeHostRowProps = Pick<
| 'hostAttempts'
| 'hostLastConnected'
| 'hostPairingRejected'
| 'hostSignedOut'
| 'hostPaths'
| 'hostPendingPaths'
| 'hostStates'
@@ -113,7 +117,8 @@ const MobileHomeHostRow = memo(function MobileHomeHostRow(props: MobileHomeHostR
lastConnectedAt: props.hostLastConnected[item.id] ?? null,
endpoint: item.endpoint,
pendingPath: props.hostPendingPaths[item.id] ?? null,
pairingRejected: props.hostPairingRejected[item.id] ?? false
pairingRejected: props.hostPairingRejected[item.id] ?? false,
hostSignedOut: props.hostSignedOut[item.id] ?? false
})
const open = useCallback(() => onOpen(item), [item, onOpen])
const longPress = useCallback(() => onLongPress(item), [item, onLongPress])
+1
View File
@@ -151,6 +151,7 @@ export function MobileHomeScreen() {
hostAttempts={data.hostAttempts}
hostLastConnected={data.hostLastConnected}
hostPairingRejected={data.hostPairingRejected}
hostSignedOut={data.hostSignedOut}
hostPaths={data.hostPaths}
hostPendingPaths={data.hostPendingPaths}
hosts={data.sortedHostCatalog}
@@ -5,12 +5,14 @@ export type HomeHostConnectionProjectionEntry = {
path: MobileConnectionPath
pendingPath: MobileConnectionPath | null
pairingRejected: boolean
hostSignedOut: boolean
}
export type HomeHostConnectionProjection = {
hostPaths: Record<string, MobileConnectionPath>
hostPendingPaths: Record<string, MobileConnectionPath | null>
hostPairingRejected: Record<string, boolean>
hostSignedOut: Record<string, boolean>
}
/** Build all host lookup maps while reading each connection entry once. */
@@ -22,16 +24,19 @@ export function projectHomeHostConnections(
const hostPaths = Object.create(null) as Record<string, MobileConnectionPath>
const hostPendingPaths = Object.create(null) as Record<string, MobileConnectionPath | null>
const hostPairingRejected = Object.create(null) as Record<string, boolean>
const hostSignedOut = Object.create(null) as Record<string, boolean>
for (const { hostId, path, pendingPath, pairingRejected } of entries) {
for (const { hostId, path, pendingPath, pairingRejected, hostSignedOut: signedOut } of entries) {
hostPaths[hostId] = path
hostPendingPaths[hostId] = pendingPath
hostPairingRejected[hostId] = pairingRejected
hostSignedOut[hostId] = signedOut
}
Object.setPrototypeOf(hostPaths, Object.prototype)
Object.setPrototypeOf(hostPendingPaths, Object.prototype)
Object.setPrototypeOf(hostPairingRejected, Object.prototype)
Object.setPrototypeOf(hostSignedOut, Object.prototype)
return { hostPaths, hostPendingPaths, hostPairingRejected }
return { hostPaths, hostPendingPaths, hostPairingRejected, hostSignedOut }
}
+1
View File
@@ -179,6 +179,7 @@ export function useMobileHomeData() {
connectedHosts,
hostCatalog,
hostPairingRejected: hostConnectionProjection.hostPairingRejected,
hostSignedOut: hostConnectionProjection.hostSignedOut,
hostPaths: hostConnectionProjection.hostPaths,
hostPendingPaths: hostConnectionProjection.hostPendingPaths,
primaryHost,
@@ -24,6 +24,7 @@ const value: RpcClientContextValue = {
getActivePath: () => 'lan',
getPendingPath: () => null,
isPairingRejected: () => false,
isHostSignedOut: () => false,
subscribeHostState: () => noop,
getAllClients: () => [],
subscribeAllHosts: () => noop,
@@ -86,6 +86,7 @@ export async function executeMobileWebNativeChatImageOperation(args: {
terminal: binding.hostTerminalId!,
deviceToken: args.terminalClientId,
imagePaths,
followedByText: payload.followedByText === true,
deadline: payload.deadline,
assertCurrent: () =>
assertCurrentMobileWebNativeChatPageBinding(
@@ -82,6 +82,7 @@ export async function executeMobileWebNativeChatTerminalOperation(args: {
terminal: binding.hostTerminalId!,
deviceToken: args.terminalClientId,
imagePaths: [],
followedByText: false,
deadline: payload.deadline,
assertCurrent: () => assertCurrentBinding(args, payload, binding)
})
@@ -69,7 +69,8 @@ export type HostSessionNativeChatOperations = {
pasteImages?(
target: HostSessionNativeChatTarget,
references: readonly string[],
deadline?: number
deadline?: number,
followedByText?: boolean
): Promise<boolean>
releaseImages?(target: HostSessionNativeChatTarget, references: readonly string[]): Promise<void>
searchFiles(target: HostSessionNativeChatTarget, query: string): Promise<string[]>
@@ -50,7 +50,9 @@ describe('attachMobileImageToTerminal', () => {
const sendCall = client.calls.find((c) => c.method === 'terminal.send')
expect(sendCall?.params).toEqual({
terminal: 'term-1',
text: '\x1b[200~/tmp/orca-attach.png\x1b[201~',
// Trailing space: the user types on this same line next, so a bare
// `…\x1b[201~` would arrive as `…pngadd` (STA-4847).
text: '\x1b[200~/tmp/orca-attach.png\x1b[201~ ',
enter: false,
client: { id: 'device-9', type: 'mobile' }
})
@@ -1,4 +1,5 @@
import type { RpcClient } from '../transport/rpc-client'
import { separateImagePasteFromFollowingText } from '../../../src/shared/image-paste-following-text'
import {
buildMobileImagePastePayload,
saveMobileClipboardImageAsTempFile
@@ -47,7 +48,10 @@ export async function attachMobileImageToTerminal(
})
// Why: a generated image path is terminal image injection, so it's always
// bracketed (matching desktop paste) regardless of terminal mode.
const payload = buildMobileImagePastePayload(imagePath)
// Always separated: attach-then-type is the whole interaction here, so the user's
// next keystroke would otherwise glue onto the path (`…pngadd`). Unlike native
// chat there is no batch to look ahead in, and a trailing space is inert.
const payload = separateImagePasteFromFollowingText(buildMobileImagePastePayload(imagePath), true)
if (beforeTerminalSend && !(await beforeTerminalSend(terminal))) {
return false
}
@@ -38,7 +38,8 @@ describe('pasteMobileNativeChatImagePaths', () => {
client,
terminal: 'term-1',
deviceToken: 'device-9',
imagePaths: ['/tmp/a.png', '/tmp/b.png', '/tmp/c.png']
imagePaths: ['/tmp/a.png', '/tmp/b.png', '/tmp/c.png'],
followedByText: true
})
expect(ok).toBe(true)
@@ -55,7 +56,7 @@ describe('pasteMobileNativeChatImagePaths', () => {
})
expect(client.calls[1]?.params.text).toBe('\x1b[200~/tmp/a.png\x1b[201~')
expect(client.calls[2]?.params.text).toBe('\x1b[200~/tmp/b.png\x1b[201~')
expect(client.calls[3]?.params.text).toBe('\x1b[200~/tmp/c.png\x1b[201~')
expect(client.calls[3]?.params.text).toBe('\x1b[200~/tmp/c.png\x1b[201~ ')
})
it('stops and reports failure as soon as a paste is rejected', async () => {
@@ -66,7 +67,8 @@ describe('pasteMobileNativeChatImagePaths', () => {
client,
terminal: 'term-1',
deviceToken: null,
imagePaths: ['/tmp/a.png', '/tmp/b.png']
imagePaths: ['/tmp/a.png', '/tmp/b.png'],
followedByText: true
})
expect(ok).toBe(false)
@@ -94,7 +96,8 @@ describe('pasteMobileNativeChatImagePaths', () => {
client,
terminal: 'term-1',
deviceToken: null,
imagePaths: ['/tmp/a.png', '/tmp/b.png']
imagePaths: ['/tmp/a.png', '/tmp/b.png'],
followedByText: true
})
expect(ok).toBe(false)
@@ -121,6 +124,7 @@ describe('clearing a parked multi-line launch draft before the image paste', ()
terminal: 'term-1',
deviceToken: null,
imagePaths: ['/tmp/a.png'],
followedByText: true,
clearInput
})
@@ -137,6 +141,7 @@ describe('clearing a parked multi-line launch draft before the image paste', ()
terminal: 'term-1',
deviceToken: null,
imagePaths: ['/tmp/a.png', '/tmp/b.png'],
followedByText: true,
clearInput
})
@@ -151,9 +156,27 @@ describe('clearing a parked multi-line launch draft before the image paste', ()
client,
terminal: 'term-1',
deviceToken: null,
imagePaths: ['/tmp/a.png']
imagePaths: ['/tmp/a.png'],
followedByText: true
})
expect(client.calls[0]?.params.text).toBe('\x15')
})
it('keeps image writes byte-clean when no text or submit follows', async () => {
const client = clientWithResponses([sendResult(true), sendResult(true), sendResult(true)])
await pasteMobileNativeChatImagePaths({
client,
terminal: 'term-1',
deviceToken: null,
imagePaths: ['/tmp/a.png', '/tmp/b.png'],
followedByText: false
})
expect(client.calls.slice(1).map((call) => call.params.text)).toEqual([
'\x1b[200~/tmp/a.png\x1b[201~',
'\x1b[200~/tmp/b.png\x1b[201~'
])
})
})
@@ -1,4 +1,5 @@
import type { RpcClient } from '../transport/rpc-client'
import { imagePasteWritesFollowedByText } from '../../../src/shared/image-paste-following-text'
import { buildMobileImagePastePayload } from './mobile-clipboard-image'
import {
MOBILE_NATIVE_CHAT_MIN_WRITE_TIMEOUT_MS,
@@ -23,6 +24,7 @@ type PasteImagesArgs = {
readonly terminal: string
readonly deviceToken: string | null
readonly imagePaths: readonly string[]
readonly followedByText: boolean
/** Budget shared with the rest of the user action (the text body that follows, or
* the send this is healing for). Omit to open a fresh one for this paste alone. */
readonly deadline?: number
@@ -41,6 +43,7 @@ export async function pasteMobileNativeChatImagePaths({
terminal,
deviceToken,
imagePaths,
followedByText,
deadline: sharedDeadline,
clearInput,
assertCurrent = () => {}
@@ -55,7 +58,7 @@ export async function pasteMobileNativeChatImagePaths({
const deadline = sharedDeadline ?? openMobileNativeChatSendBudget()
for (const text of [
clearInput ?? MOBILE_NATIVE_CHAT_CLEAR_UNSUBMITTED_INPUT,
...imagePaths.map(buildMobileImagePastePayload)
...imagePasteWritesFollowedByText(imagePaths.map(buildMobileImagePastePayload), followedByText)
]) {
const remainingMs = deadline - Date.now()
// Why: the budget is the whole sequence's — starting a write it can't fund would
@@ -65,6 +65,7 @@ export async function sendMobileNativeChatWithImages(args: {
}
try {
const references = args.pendingImages.map((attachment) => attachment.path)
const followedByText = args.text.trim().length > 0
const seededLaunchDraft = args.readSeededLaunchDraft()
const pasted = args.client
? await pasteMobileNativeChatImagePaths({
@@ -72,12 +73,13 @@ export async function sendMobileNativeChatWithImages(args: {
terminal: handle,
deviceToken: args.deviceTokenRef.current,
imagePaths: references,
followedByText,
deadline,
...(seededLaunchDraft
? { clearInput: buildAgentTuiClearInputForText(seededLaunchDraft) }
: {})
})
: await args.operations!.pasteImages!(target!, references, deadline)
: await args.operations!.pasteImages!(target!, references, deadline, followedByText)
if (!pasted) {
markMobileNativeChatInputStale(handle)
args.onError?.()
@@ -55,6 +55,7 @@ export async function healMobileNativeChatStaleInput(args: {
terminal: args.terminal,
deviceToken: args.deviceToken,
imagePaths: [],
followedByText: false,
...(args.deadline === undefined ? {} : { deadline: args.deadline })
})
} catch {
@@ -82,7 +82,7 @@ describe('hosted native-chat image attachments', () => {
await act(async () => {
expect(await hook!.sendNativeChat('inspect')).toBe(true)
})
expect(pasteImages).toHaveBeenCalledWith(TARGET, [REFERENCE], expect.any(Number))
expect(pasteImages).toHaveBeenCalledWith(TARGET, [REFERENCE], expect.any(Number), true)
expect(baseSend).toHaveBeenCalledWith('inspect', [PREVIEW], expect.any(Number))
expect(releaseImages).toHaveBeenCalledWith(TARGET, [REFERENCE])
expect(hook!.attachments).toEqual([])
@@ -184,9 +184,12 @@ describe('useMobileNativeChatImageAttachments', () => {
expect(sendCalls).toHaveLength(2)
expect(sendCalls[0]?.params).toMatchObject({ text: '\x15', enter: false })
expect(sendCalls[1]?.params).toMatchObject({
text: '\x1b[200~/tmp/a.png\x1b[201~',
text: '\x1b[200~/tmp/a.png\x1b[201~ ',
enter: false
})
const combined = String(sendCalls[1]?.params.text ?? '') + 'look at this'
expect(combined).toContain('.png\x1b[201~ look')
expect(combined).not.toContain('.png\x1b[201~look')
// Clear, then paste, then settle, then the text send — in that order.
expect(order).toEqual(['clear', 'paste', 'settle', 'text:look at this'])
// The local preview URI rides along so the sent bubble shows the photo.
@@ -269,7 +272,10 @@ describe('useMobileNativeChatImageAttachments', () => {
}
})
it('routes an attachments-only send through baseSend with empty text so the echo still shows the photo', async () => {
it.each([
['empty', ''],
['whitespace-only', ' ']
])('routes an attachments-only send through baseSend with %s text', async (_label, text) => {
pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }])
const client = makeClient([
methodNotFound('start'),
@@ -285,16 +291,17 @@ describe('useMobileNativeChatImageAttachments', () => {
})
let accepted = false
await act(async () => {
accepted = await hook!.sendNativeChat('')
accepted = await hook!.sendNativeChat(text)
})
expect(accepted).toBe(true)
// Empty text still goes through baseSend (which submits the bare Enter) so the
// Attachment-only text still goes through baseSend (which submits Enter) so the
// optimistic echo carries the preview URI.
expect(baseSend).toHaveBeenCalledWith('', ['file:///a.jpg'], expect.any(Number))
expect(baseSend).toHaveBeenCalledWith(text, ['file:///a.jpg'], expect.any(Number))
const sendCalls = client.calls.filter((c) => c.method === 'terminal.send')
// Only the clear + image paste hit the wire here; baseSend owns the submit.
expect(sendCalls).toHaveLength(2)
expect(sendCalls[1]?.params.text).toBe('\x1b[200~/tmp/a.png\x1b[201~')
expect(hook!.attachments).toEqual([])
})
@@ -80,6 +80,44 @@ describe('hosted native-chat deadlines', () => {
expect(pasteImages).not.toHaveBeenCalled()
})
it('forwards the following-text hint only when typed text follows the paste', async () => {
vi.useFakeTimers()
vi.setSystemTime(10_000)
const pasteImages = vi.fn().mockResolvedValue({ pasted: true })
const client = { nativeChat: { pasteImages } } as unknown as MobileWebBridgeClient
const operations = webHostSessionNativeChatOperations(client)
await expect(operations.pasteImages!(TARGET, ['opaque-image'], 20_000, true)).resolves.toBe(
true
)
await expect(operations.pasteImages!(TARGET, ['opaque-image'], 20_000, false)).resolves.toBe(
true
)
// Why: an older strict shell rejects unknown keys, so the flag rides only when it is true.
expect(pasteImages).toHaveBeenNthCalledWith(
1,
{
workspaceId: 'workspace',
sessionId: 'native_chat_session',
references: ['opaque-image'],
deadline: 20_000,
followedByText: true
},
{ timeoutMs: 10_000 }
)
expect(pasteImages).toHaveBeenNthCalledWith(
2,
{
workspaceId: 'workspace',
sessionId: 'native_chat_session',
references: ['opaque-image'],
deadline: 20_000
},
{ timeoutMs: 10_000 }
)
})
it('clears a stale hosted composer only after the shell accepts preparation', async () => {
vi.useFakeTimers()
vi.setSystemTime(10_000)
@@ -128,7 +128,7 @@ export function webHostSessionNativeChatOperations(
}
return { status: result.status }
},
async pasteImages(target, references, deadline) {
async pasteImages(target, references, deadline, followedByText) {
const budget = bridgeBudget(deadline)
if (!budget) {
return false
@@ -138,7 +138,8 @@ export function webHostSessionNativeChatOperations(
await client.nativeChat.pasteImages(
bridgeTarget(target, {
references: [...references],
deadline: budget.deadline
deadline: budget.deadline,
...(followedByText ? { followedByText } : {})
}),
{ timeoutMs: budget.timeoutMs }
)
@@ -30,14 +30,16 @@ export function useConnectionPathStatus(hostId: string | undefined): {
export function useRelayRecoveryStatus(hostId: string | undefined): {
pendingPath: MobileConnectionPath | null
pairingRejected: boolean
hostSignedOut: boolean
} {
return useHostMetric(
hostId,
(context, id) => ({
pendingPath: context.getPendingPath(id),
pairingRejected: context.isPairingRejected(id)
pairingRejected: context.isPairingRejected(id),
hostSignedOut: context.isHostSignedOut(id)
}),
{ pendingPath: null, pairingRejected: false }
{ pendingPath: null, pairingRejected: false, hostSignedOut: false }
)
}
+2 -2
View File
@@ -533,12 +533,12 @@ describe('useAllHostClients', () => {
await Promise.resolve()
})
act(() => client.emitPendingPath('relay'))
expect(status).toEqual({ pendingPath: 'relay', pairingRejected: false })
expect(status).toEqual({ pendingPath: 'relay', pairingRejected: false, hostSignedOut: false })
// Why: the desktop refusing the credential is a status-only change — no
// transport state moves, so only the connection-path signal can carry it.
act(() => client.emitPairingRejected(true))
expect(status).toEqual({ pendingPath: 'relay', pairingRejected: true })
expect(status).toEqual({ pendingPath: 'relay', pairingRejected: true, hostSignedOut: false })
act(() => renderer.unmount())
})
+19
View File
@@ -29,6 +29,10 @@ const STALE_SINCE_LAST_CONNECT_MS = 60_000
// instead of leaving the user staring at a generic "Can't connect".
const TAILSCALE_HINT = 'check Tailscale'
// No hint field: the remedy is the label, and appending "— check Tailscale" to
// it would be wrong advice for a desktop that is reachable but signed out.
const SIGNED_OUT_LABEL = 'Desktop signed out — sign in to Orca on your desktop to reconnect'
export type ConnectionVerdict =
| { kind: 'normal'; label: string }
| { kind: 'warning'; label: string; hint?: string } // "Can't connect"
@@ -54,6 +58,10 @@ export function classifyConnection(args: {
// The desktop has repeatedly refused this device's relay credential — retrying
// cannot fix it, so it outranks any "still connecting" reading (STA-4681).
pairingRejected?: boolean
// The relay says the desktop's last control close named its own Orca Cloud
// sign-out. Retrying is still correct and still happens on the same cadence,
// but only the desktop's owner can end it, so the label has to say so.
hostSignedOut?: boolean
nowMs?: number
}): ConnectionVerdict {
const { state, reconnectAttempts, lastConnectedAt } = args
@@ -70,6 +78,17 @@ export function classifyConnection(args: {
return { kind: 'normal', label: 'Connected' }
}
// Ahead of the attempt thresholds: this is evidence, not an inference from a
// failure streak, and waiting twelve dials to show it wastes the whole point.
// Below auth-failed because a revoked pairing cannot be fixed by signing in.
if (args.hostSignedOut) {
return {
kind: 'unreachable',
label: SIGNED_OUT_LABEL,
reason: lastConnectedAt == null ? 'never-connected' : 'stale'
}
}
// A disconnected pending path can survive a cleared retry timer during a
// lifecycle race. Only narrate Relay while dialing or after a retry has
// recorded progress; otherwise the idle transport must read Disconnected.
@@ -89,10 +89,16 @@ export function createHostClientSelectors(
getPendingPath: (hostId: string): MobileConnectionPath | null =>
clientPendingPath(entries.get(hostId)?.client),
isPairingRejected: (hostId: string): boolean =>
clientPairingRejected(entries.get(hostId)?.client)
clientPairingRejected(entries.get(hostId)?.client),
isHostSignedOut: (hostId: string): boolean => clientHostSignedOut(entries.get(hostId)?.client)
}
}
export function clientHostSignedOut(client: RpcClient | undefined): boolean {
const logical = client as Partial<StableLogicalRpcClient> | undefined
return logical?.isHostSignedOut?.() ?? false
}
export function clientPairingRejected(client: RpcClient | undefined): boolean {
const logical = client as Partial<StableLogicalRpcClient> | undefined
return logical?.isPairingRejected?.() ?? false
@@ -5,6 +5,7 @@ export class LogicalClientConnectionPath {
private recovery: MobileConnectionPath | null = null
private recoveryAttempt = 0
private pairingRejected = false
private hostSignedOut = false
private readonly listeners = new Set<() => void>()
constructor(private readonly isConnected: () => boolean) {}
@@ -35,12 +36,23 @@ export class LogicalClientConnectionPath {
})
}
isHostSignedOut(): boolean {
return this.hostSignedOut
}
setHostSignedOut(signedOut: boolean): void {
this.update(() => {
this.hostSignedOut = signedOut
})
}
clearAfterConnected(): void {
this.migration = null
this.recovery = null
this.recoveryAttempt = 0
// Why: an authenticated session is the desktop accepting this device.
this.pairingRejected = false
this.hostSignedOut = false
}
setRecovery(path: MobileConnectionPath | null, attempt?: number): void {
@@ -69,11 +81,13 @@ export class LogicalClientConnectionPath {
const previousPath = this.pending()
const previousAttempt = this.reconnectAttempt(0)
const previousRejected = this.pairingRejected
const previousSignedOut = this.hostSignedOut
apply()
if (
previousPath === this.pending() &&
previousAttempt === this.reconnectAttempt(0) &&
previousRejected === this.pairingRejected
previousRejected === this.pairingRejected &&
previousSignedOut === this.hostSignedOut
) {
return
}
@@ -86,7 +86,7 @@ function createSupervisor(
): MobileEndpointSupervisor {
return new MobileEndpointSupervisor(logical, host, {
openDirect: (endpoint) => connect(endpoint, host.deviceToken, host.publicKeyB64, { onLog }),
openRelay: (relay, credential, confirmReqId) =>
openRelay: (relay, credential, confirmReqId, onHostCloseReason) =>
connectMobileRelayRpcSession({
relay,
resumeToken: credential.token,
@@ -94,6 +94,7 @@ function createSupervisor(
resumeConfirmReqId: confirmReqId,
deviceToken: host.deviceToken,
desktopPublicKeyB64: host.publicKeyB64,
onHostCloseReason,
onLog
}),
resolveRelay: resolveMobileRelayEndpoint,
@@ -1,4 +1,5 @@
import type { MobileRelayEndpoint } from '../../../src/shared/mobile-relay-credential-contract'
import type { RelayHostCloseReason } from '../../../src/shared/relay-host-close-reason'
import type { MobileRelayCredentialBundle } from './mobile-relay-credential-bundle'
import type { MobileRelayRpcSession } from './mobile-relay-rpc-session'
import type { resolveMobileRelayEndpoint } from './mobile-relay-resume-director'
@@ -10,7 +11,8 @@ export type MobileEndpointSupervisorDependencies = {
openRelay: (
relay: MobileRelayEndpoint,
credential: { token: string; version: number },
confirmReqId: string
confirmReqId: string,
onHostCloseReason?: (reason: RelayHostCloseReason) => void
) => MobileRelayRpcSession
resolveRelay: typeof resolveMobileRelayEndpoint
readBundle: (hostId: string) => Promise<MobileRelayCredentialBundle | null>
@@ -135,10 +135,22 @@ export class FakeLogicalClient extends FakeSession implements StableLogicalRpcCl
}
})
isPairingRejected = () => this.pairingRejected
private hostSignedOut = false
setHostSignedOut = vi.fn((signedOut: boolean) => {
if (this.hostSignedOut === signedOut) {
return
}
this.hostSignedOut = signedOut
for (const listener of this.pathListeners) {
listener()
}
})
isHostSignedOut = () => this.hostSignedOut
// Mirrors LogicalClientConnectionPath.clearAfterConnected.
publishState(state: ConnectionState): void {
if (state === 'connected') {
this.pairingRejected = false
this.hostSignedOut = false
}
super.publishState(state)
}
@@ -185,7 +185,12 @@ describe('mobile endpoint supervisor', () => {
await supervisor.start()
expect(deps.resolveRelay).toHaveBeenCalledOnce()
expect(openRelay).toHaveBeenLastCalledWith(resolved, expect.any(Object), expect.any(String))
expect(openRelay).toHaveBeenLastCalledWith(
resolved,
expect.any(Object),
expect.any(String),
expect.any(Function)
)
expect(deps.saveHost).toHaveBeenCalledWith(
expect.objectContaining({ relay: resolved, endpoint: host.endpoint })
)
@@ -556,7 +561,8 @@ describe('mobile endpoint supervisor', () => {
expect(openRelay).toHaveBeenLastCalledWith(
relay,
expect.objectContaining({ version: 3 }),
expect.any(String)
expect.any(String),
expect.any(Function)
)
supervisor.stop()
})
@@ -603,7 +609,8 @@ describe('mobile endpoint supervisor', () => {
expect(openRelay).toHaveBeenLastCalledWith(
relay,
expect.objectContaining({ version: 3 }),
expect.any(String)
expect.any(String),
expect.any(Function)
)
supervisor.stop()
})
@@ -2,6 +2,10 @@ import {
RelayPhoneHelloSchema,
type RelayPhoneHello
} from '../../../src/shared/mobile-relay-phone-protocol'
import {
relayHostCloseReasonFrom,
type RelayHostCloseReason
} from '../../../src/shared/relay-host-close-reason'
import { MobileE2EEV2ClientSession } from './mobile-e2ee-v2-client-session'
import { MobileE2EEV2PhysicalChannel } from './mobile-e2ee-v2-physical-channel'
import { websocketPayloadToUint8 } from './websocket-payload-bytes'
@@ -26,6 +30,12 @@ type MobileRelayE2eeLinkOptions = {
onText: (plaintext: string) => void
onBinary: (plaintext: Uint8Array) => void
onHello?: (hello: Extract<RelayPhoneHello, { ok: true }>) => void
// The cell's account of why the desktop is absent, read off the close frame.
// Reported separately from onError because a rejection is delivered as both a
// relay-hello and a close, and which one the runtime dispatches first is not
// ordered — only the close carries the reason, and it must not be lost to
// that race.
onHostCloseReason?: (reason: RelayHostCloseReason) => void
// Fired once relay-auth is on the wire: from here the cell owns the wait.
onOpen?: () => void
onError: (error: Error) => void
@@ -129,6 +139,11 @@ export class MobileRelayE2eeLink {
clearTimeout(this.transportErrorTimer)
this.transportErrorTimer = null
}
// Ahead of fail(), which no-ops once the hello already reported this close.
const hostCloseReason = relayHostCloseReasonFrom(event.reason)
if (hostCloseReason) {
this.options.onHostCloseReason?.(hostCloseReason)
}
this.fail(new RelayOuterError(event.code || 1006))
}
}
@@ -12,6 +12,7 @@ import { isRpcResponse } from './rpc-response-shape'
import { RelayDialStageTracker, type RelayDialStageSource } from './relay-dial-stage'
import { RelayPendingRequests } from './relay-pending-requests'
import { settleMobileRuntimeCapabilities } from './mobile-runtime-capability-negotiation'
import type { RelayHostCloseReason } from '../../../src/shared/relay-host-close-reason'
import type { RpcClient } from './rpc-client'
import type { ConnectionLogSink, ConnectionState, RpcResponse } from './types'
import { encodeTerminalStreamFrame } from './terminal-stream-protocol'
@@ -36,6 +37,7 @@ export function connectMobileRelayRpcSession(args: {
desktopPublicKeyB64: string
requestTimeoutMs?: number
createSocket?: (url: string) => WebSocket
onHostCloseReason?: (reason: RelayHostCloseReason) => void
onLog?: ConnectionLogSink
}): MobileRelayRpcSession {
const requestTimeoutMs = args.requestTimeoutMs ?? 30_000
@@ -63,6 +65,7 @@ export function connectMobileRelayRpcSession(args: {
deviceToken: args.deviceToken,
desktopPublicKeyB64: args.desktopPublicKeyB64,
createSocket: args.createSocket,
onHostCloseReason: args.onHostCloseReason,
onOpen: () => dialStage.advance('awaiting-hello'),
onHello: (hello) => {
if (
@@ -145,10 +145,22 @@ class FakeLogicalClient extends FakeSession implements StableLogicalRpcClient {
}
})
isPairingRejected = () => this.pairingRejected
private hostSignedOut = false
setHostSignedOut = vi.fn((signedOut: boolean) => {
if (this.hostSignedOut === signedOut) {
return
}
this.hostSignedOut = signedOut
for (const listener of this.pathListeners) {
listener()
}
})
isHostSignedOut = () => this.hostSignedOut
// Mirrors LogicalClientConnectionPath.clearAfterConnected.
publishState(state: ConnectionState): void {
if (state === 'connected') {
this.pairingRejected = false
this.hostSignedOut = false
}
super.publishState(state)
}
@@ -264,7 +276,8 @@ describe('relay runtime recovery without direct connectivity', () => {
expect(openRelay).toHaveBeenLastCalledWith(
relay,
expect.objectContaining({ version: 3 }),
expect.any(String)
expect.any(String),
expect.any(Function)
)
expect(logical.getActivePath()).toBe('relay')
supervisor.stop()
@@ -353,7 +366,8 @@ describe('relay runtime recovery without direct connectivity', () => {
expect(deps.openRelay).toHaveBeenLastCalledWith(
relay,
expect.objectContaining({ version: 2 }),
expect.any(String)
expect.any(String),
expect.any(Function)
)
expect(logical.getActivePath()).toBe('relay')
supervisor.stop()
@@ -382,7 +396,8 @@ describe('relay runtime recovery without direct connectivity', () => {
expect(openRelay).toHaveBeenLastCalledWith(
relay,
expect.objectContaining({ version: 1 }),
expect.any(String)
expect.any(String),
expect.any(Function)
)
expect(logical.getActivePath()).toBe('relay')
supervisor.stop()
@@ -10,6 +10,7 @@ import type { MobileRelayCredentialBundle } from './mobile-relay-credential-bund
import type { RelayReconnectController } from './mobile-relay-reconnect-controller'
import type { StableLogicalRpcClient } from './stable-logical-rpc-client'
import type { MobileRelayEndpoint } from '../../../src/shared/mobile-relay-credential-contract'
import { RELAY_HOST_CLOSE_REASON } from '../../../src/shared/relay-host-close-reason'
import type { HostProfile } from './types'
type EstablishResult = { ok: true } | { ok: false; error: Error }
@@ -100,7 +101,16 @@ export class MobileRelaySessionEstablisher {
const session = args.openRelay(
relay,
credential,
`confirm-${encodeBase64Url(args.randomBytes(16))}`
`confirm-${encodeBase64Url(args.randomBytes(16))}`,
// Latched on the logical client, not on the dial result: the close that
// carries the reason can land after this dial has already reported its
// failure. Clearing is clearAfterConnected's job, so any path that
// reaches connected retires it.
(reason) => {
if (reason === RELAY_HOST_CLOSE_REASON.SIGNED_OUT) {
args.logical.setHostSignedOut(true)
}
}
)
try {
// Why: backgrounding or a direct winner withdraws this dial before cutover.
@@ -0,0 +1,60 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { MOBILE_RELAY_CLOSE_CODE } from '../../../src/shared/mobile-relay-close-codes'
import { RELAY_HOST_CLOSE_REASON } from '../../../src/shared/relay-host-close-reason'
import { RelayOuterError } from './mobile-relay-e2ee-link'
import {
dependencies,
FakeLogicalClient,
FakeRelaySession,
host
} from './mobile-endpoint-supervisor-test-fakes'
import { MobileEndpointSupervisor } from './mobile-endpoint-supervisor'
vi.mock('react-native', () => ({ Platform: { OS: 'ios' } }))
vi.mock('expo-secure-store', () => ({ WHEN_UNLOCKED_THIS_DEVICE_ONLY: 'when-unlocked' }))
vi.mock('expo-crypto', () => ({ getRandomBytes: (length: number) => new Uint8Array(length) }))
// The reason travels from the cell's close frame to the screens. This covers
// the production wiring between them: the supervisor's own openRelay callback.
describe('a signed-out desktop reaches the phone verdict', () => {
beforeEach(() => {
vi.useFakeTimers()
vi.setSystemTime(new Date('2026-07-13T12:00:00Z'))
})
afterEach(() => vi.useRealTimers())
function supervisorOver(closeReason: string | null) {
const logical = new FakeLogicalClient('disconnected', 'lan')
const deps = dependencies({
openDirect: vi.fn(() => new FakeRelaySession('disconnected')),
openRelay: vi.fn((_relay, _credential, _confirmReqId, onHostCloseReason) => {
if (closeReason) {
onHostCloseReason?.(closeReason as never)
}
return new FakeRelaySession(
'disconnected',
new RelayOuterError(MOBILE_RELAY_CLOSE_CODE.HOST_OFFLINE)
)
})
})
return { logical, supervisor: new MobileEndpointSupervisor(logical, host, deps) }
}
it('latches the sign-out the cell reported', async () => {
const { logical, supervisor } = supervisorOver(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
await supervisor.start()
await vi.waitFor(() => expect(logical.isHostSignedOut()).toBe(true))
supervisor.stop()
})
it('stays quiet for an ordinary host-offline rejection', async () => {
const { logical, supervisor } = supervisorOver(null)
await supervisor.start()
expect(logical.isHostSignedOut()).toBe(false)
supervisor.stop()
})
})
@@ -0,0 +1,216 @@
import { describe, expect, it, vi } from 'vitest'
vi.mock('./mobile-e2ee-v2-client-session', () => ({
MobileE2EEV2ClientSession: { create: () => ({}) }
}))
vi.mock('./mobile-e2ee-v2-physical-channel', () => ({
MobileE2EEAuthenticationError: class extends Error {},
MobileE2EEV2PhysicalChannel: class {
start = vi.fn()
handleMessage = vi.fn(async () => {})
sendText = vi.fn(() => true)
sendBinary = vi.fn(() => true)
dispose = vi.fn()
}
}))
import { RELAY_HOST_CLOSE_REASON } from '../../../src/shared/relay-host-close-reason'
import { MOBILE_RELAY_CLOSE_CODE } from '../../../src/shared/mobile-relay-close-codes'
import { classifyConnection, verdictDisplayLabel } from './connection-health'
import { MobileRelayE2eeLink, RelayOuterError } from './mobile-relay-e2ee-link'
import { LogicalClientConnectionPath } from './logical-client-connection-path'
import { RelayReconnectController } from './mobile-relay-reconnect-controller'
const SIGNED_OUT_LABEL = 'Desktop signed out — sign in to Orca on your desktop to reconnect'
class FakeSocket {
static readonly OPEN = 1
readonly OPEN = FakeSocket.OPEN
readyState = FakeSocket.OPEN
bufferedAmount = 0
onopen: (() => void) | null = null
onmessage: ((event: { data: unknown }) => void) | null = null
onerror: (() => void) | null = null
onclose: ((event: { code: number; reason: string }) => void) | null = null
send = vi.fn()
close = vi.fn()
}
function linkOver(
socket: FakeSocket,
onHostCloseReason: (reason: string) => void,
onError: (error: Error) => void
): MobileRelayE2eeLink {
return new MobileRelayE2eeLink({
endpoint: { cellUrl: 'https://relay-c1.onorca.dev', relayHostId: 'AbCdEf0123_-xyZ9' },
credential: 'credential',
expectedCredentialKind: 'resume',
deviceToken: 'device-token',
desktopPublicKeyB64: 'desktop-key',
onAuthenticated: vi.fn(),
onText: vi.fn(),
onBinary: vi.fn(),
onHostCloseReason,
onError,
createSocket: () => socket as unknown as WebSocket
})
}
describe('relay close reason on the phone', () => {
it('reports the cell close reason and still fails with 4404', () => {
const socket = new FakeSocket()
const onHostCloseReason = vi.fn()
const onError = vi.fn()
linkOver(socket, onHostCloseReason, onError)
socket.onclose?.({
code: MOBILE_RELAY_CLOSE_CODE.HOST_OFFLINE,
reason: RELAY_HOST_CLOSE_REASON.SIGNED_OUT
})
expect(onHostCloseReason).toHaveBeenCalledWith(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
expect(onError).toHaveBeenCalledWith(new RelayOuterError(MOBILE_RELAY_CLOSE_CODE.HOST_OFFLINE))
})
// An old cell sends its constant, and every other close sends nothing.
it('reports nothing for a reason it does not know', () => {
const socket = new FakeSocket()
const onHostCloseReason = vi.fn()
linkOver(socket, onHostCloseReason, vi.fn())
socket.onclose?.({
code: MOBILE_RELAY_CLOSE_CODE.HOST_OFFLINE,
reason: 'relay connection rejected'
})
expect(onHostCloseReason).not.toHaveBeenCalled()
})
// The rejection arrives as a relay-hello AND a close, in an unordered pair.
// Whichever lands first, the reason must survive.
it('still reports the reason when the hello already failed the link', async () => {
const socket = new FakeSocket()
const onHostCloseReason = vi.fn()
linkOver(socket, onHostCloseReason, vi.fn())
socket.onmessage?.({
data: JSON.stringify({
type: 'relay-hello',
ok: false,
code: MOBILE_RELAY_CLOSE_CODE.HOST_OFFLINE
})
})
await Promise.resolve()
await Promise.resolve()
socket.onclose?.({
code: MOBILE_RELAY_CLOSE_CODE.HOST_OFFLINE,
reason: RELAY_HOST_CLOSE_REASON.SIGNED_OUT
})
expect(onHostCloseReason).toHaveBeenCalledWith(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
})
})
describe('the signed-out signal on the logical client', () => {
it('publishes on change and retires when any path reaches connected', () => {
const path = new LogicalClientConnectionPath(() => false)
const changes = vi.fn()
path.subscribe(changes)
path.setHostSignedOut(true)
path.setHostSignedOut(true)
expect(path.isHostSignedOut()).toBe(true)
expect(changes).toHaveBeenCalledTimes(1)
path.clearAfterConnected()
expect(path.isHostSignedOut()).toBe(false)
})
})
describe('RelayReconnectController cadence', () => {
// The reason changes no recovery decision; 4404 keeps the host-offline
// backoff it has always had, so a phone on this build retries exactly as
// often as one that never hears the reason.
it('keeps the host-offline retry delay for a 4404', () => {
const delays: number[] = []
const controller = new RelayReconnectController(
{
now: () => 0,
randomBytes: () => new Uint8Array([0, 0]),
setTimer: ((callback: () => void, delay: number) => {
delays.push(delay)
return 1 as unknown as ReturnType<typeof setTimeout>
}) as unknown as typeof setTimeout,
clearTimer: (() => {}) as unknown as typeof clearTimeout
},
vi.fn()
)
controller.registerFailure(new RelayOuterError(MOBILE_RELAY_CLOSE_CODE.HOST_OFFLINE))
// hostOfflineDelayMs' 5s floor, not the 250ms transport-backoff floor.
expect(delays.at(-1)).toBe(5_000)
})
})
describe('classifyConnection with a signed-out desktop', () => {
const base = { reconnectAttempts: 0, lastConnectedAt: null, hostSignedOut: true }
it('says so from the first failed dial instead of "Connecting via Relay…"', () => {
const verdict = classifyConnection({
...base,
state: 'connecting',
pendingPath: 'relay'
})
expect(verdict).toEqual({
kind: 'unreachable',
label: SIGNED_OUT_LABEL,
reason: 'never-connected'
})
expect(verdictDisplayLabel(verdict)).toBe(SIGNED_OUT_LABEL)
})
it('replaces "Can\'t reach desktop" on the direct path too', () => {
expect(
classifyConnection({ ...base, state: 'reconnecting', reconnectAttempts: 20 }).label
).toBe(SIGNED_OUT_LABEL)
})
it('reads as stale once this session had been connected', () => {
expect(
classifyConnection({ ...base, state: 'reconnecting', lastConnectedAt: 1, nowMs: 2 }).reason
).toBe('stale')
})
// A Tailscale endpoint cannot make "sign in on your desktop" better advice.
it('never appends the Tailscale hint', () => {
expect(
classifyConnection({ ...base, state: 'reconnecting', endpoint: '100.64.0.1' })
).not.toHaveProperty('hint')
})
it('never outranks a connected session', () => {
expect(classifyConnection({ ...base, state: 'connected' }).label).toBe('Connected')
})
// Re-pairing, not signing in, is the remedy when the pairing itself is dead.
it('never outranks a revoked pairing', () => {
expect(classifyConnection({ ...base, state: 'reconnecting', pairingRejected: true }).kind).toBe(
'auth-failed'
)
})
it('leaves every other verdict alone when the desktop is not signed out', () => {
expect(
classifyConnection({
state: 'connecting',
reconnectAttempts: 0,
lastConnectedAt: null,
pendingPath: 'relay',
hostSignedOut: false
}).label
).toBe('Connecting via Relay…')
})
})
@@ -24,6 +24,7 @@ export type RpcClientContextValue = {
getActivePath: (hostId: string) => MobileConnectionPath
getPendingPath: (hostId: string) => MobileConnectionPath | null
isPairingRejected: (hostId: string) => boolean
isHostSignedOut: (hostId: string) => boolean
subscribeHostState: (hostId: string, listener: (state: ConnectionState) => void) => () => void
getAllClients: () => { hostId: string; client: RpcClient }[]
subscribeAllHosts: (listener: () => void) => () => void
@@ -55,6 +55,9 @@ export type StableLogicalRpcClient = RpcClient & {
// Latched when the desktop has repeatedly refused this device's relay credential.
setPairingRejected(rejected: boolean): void
isPairingRejected(): boolean
// Latched when the relay named the desktop's own sign-out as the reason it is absent.
setHostSignedOut(signedOut: boolean): void
isHostSignedOut(): boolean
// Recovery attempts share this signal so status-only changes rerender.
onConnectionPathChange(listener: () => void): () => void
getGeneration(): number
@@ -286,6 +289,8 @@ export function createStableLogicalRpcClient(
setRecoveryAttempt: (attempt) => connectionPath.setRecoveryAttempt(attempt),
setPairingRejected: (rejected) => connectionPath.setPairingRejected(rejected),
isPairingRejected: () => connectionPath.isPairingRejected(),
setHostSignedOut: (signedOut) => connectionPath.setHostSignedOut(signedOut),
isHostSignedOut: () => connectionPath.isHostSignedOut(),
onConnectionPathChange: (listener) => connectionPath.subscribe(listener),
getGeneration: () => generation
}
+3 -1
View File
@@ -138,6 +138,7 @@ export function useAllHostClients(hostIds: string[], options?: UseAllHostClients
path: MobileConnectionPath
pendingPath: MobileConnectionPath | null
pairingRejected: boolean
hostSignedOut: boolean
}>((hostId) => {
const client = clientsByHostId.get(hostId)
return client
@@ -148,7 +149,8 @@ export function useAllHostClients(hostIds: string[], options?: UseAllHostClients
state: ctx.getState(hostId),
path: ctx.getActivePath(hostId),
pendingPath: ctx.getPendingPath(hostId),
pairingRejected: ctx.isPairingRejected(hostId)
pairingRejected: ctx.isPairingRejected(hostId),
hostSignedOut: ctx.isHostSignedOut(hostId)
}
]
: []
+3 -3
View File
@@ -116,7 +116,7 @@ patchedDependencies:
'@xterm/addon-webgl@0.20.0-beta.299': 94687e89a0115e6e6aa102837f986debdc029c091527ee5eb4a4e17ceaf9473e
'@xterm/xterm@6.1.0-beta.303': 98756bcedc402bcdb7c6ab7b015d2e59cd18e97b03a2c06a27e95bb3ba429d9d
lint-staged@16.4.0: 7333b3837f80a7fbd045964db6d76ba4fc118e49134bdbabb00585b6b7b60673
node-pty@1.1.0: e262847f57a1d4d3f2287a843822f7dcf3c9d8655892b07a69eba464e1317eaa
node-pty@1.1.0: 7cc9d45f3d2c38f142490d0805e75db55f0eef5174ad41c4b52abc5fbe079ad1
importers:
@@ -157,7 +157,7 @@ importers:
version: 3.3.1
node-pty:
specifier: ^1.1.0
version: 1.1.0(patch_hash=e262847f57a1d4d3f2287a843822f7dcf3c9d8655892b07a69eba464e1317eaa)
version: 1.1.0(patch_hash=7cc9d45f3d2c38f142490d0805e75db55f0eef5174ad41c4b52abc5fbe079ad1)
posthog-node:
specifier: ^5.33.3
version: 5.33.3
@@ -12214,7 +12214,7 @@ snapshots:
node-int64@0.4.0: {}
node-pty@1.1.0(patch_hash=e262847f57a1d4d3f2287a843822f7dcf3c9d8655892b07a69eba464e1317eaa):
node-pty@1.1.0(patch_hash=7cc9d45f3d2c38f142490d0805e75db55f0eef5174ad41c4b52abc5fbe079ad1):
dependencies:
node-addon-api: 7.1.1
+6 -6
View File
@@ -131,17 +131,17 @@
"name": "orchestration",
"sourcePath": "skills/orchestration",
"releaseRevision": 29,
"packageDigest": "7a386ce558ba54abe02b4a0de5d71fe3d63c944ef0888ddde130c729b37f7cc8",
"gitTreeSha": "4199ec6988801dd491631706cba62631b4bed8fb",
"packageDigest": "689e31d84256aded123c801eaa87413474943a9a30d96bff9a19d0a321aefb54",
"gitTreeSha": "902cc33dd65730b32ac234dd0ae7166d75498b46",
"files": [
{
"path": "SKILL.md",
"size": 4451,
"size": 4398,
"executable": false,
"classification": "text",
"exactSha256": "a7e3350f037698ebbce36b2818d8383ec6e96c2a4825537caab39a83b4fb4b7f",
"textNormalizedSha256": "a7e3350f037698ebbce36b2818d8383ec6e96c2a4825537caab39a83b4fb4b7f",
"identitySha256": "a7e3350f037698ebbce36b2818d8383ec6e96c2a4825537caab39a83b4fb4b7f"
"exactSha256": "19ffdc1fe0d2c97dae845e8d636edb16781453ce2ec26f65a323c492ef90da18",
"textNormalizedSha256": "19ffdc1fe0d2c97dae845e8d636edb16781453ce2ec26f65a323c492ef90da18",
"identitySha256": "19ffdc1fe0d2c97dae845e8d636edb16781453ce2ec26f65a323c492ef90da18"
}
]
}
+6 -6
View File
@@ -1046,17 +1046,17 @@
},
{
"releaseRevision": 29,
"packageDigest": "7a386ce558ba54abe02b4a0de5d71fe3d63c944ef0888ddde130c729b37f7cc8",
"gitTreeSha": "4199ec6988801dd491631706cba62631b4bed8fb",
"packageDigest": "689e31d84256aded123c801eaa87413474943a9a30d96bff9a19d0a321aefb54",
"gitTreeSha": "902cc33dd65730b32ac234dd0ae7166d75498b46",
"files": [
{
"path": "SKILL.md",
"size": 4451,
"size": 4398,
"executable": false,
"classification": "text",
"exactSha256": "a7e3350f037698ebbce36b2818d8383ec6e96c2a4825537caab39a83b4fb4b7f",
"textNormalizedSha256": "a7e3350f037698ebbce36b2818d8383ec6e96c2a4825537caab39a83b4fb4b7f",
"identitySha256": "a7e3350f037698ebbce36b2818d8383ec6e96c2a4825537caab39a83b4fb4b7f"
"exactSha256": "19ffdc1fe0d2c97dae845e8d636edb16781453ce2ec26f65a323c492ef90da18",
"textNormalizedSha256": "19ffdc1fe0d2c97dae845e8d636edb16781453ce2ec26f65a323c492ef90da18",
"identitySha256": "19ffdc1fe0d2c97dae845e8d636edb16781453ce2ec26f65a323c492ef90da18"
}
]
}
+12 -12
View File
@@ -3,18 +3,18 @@ name: orchestration
description: >-
Use Orca orchestration for structured multi-agent coordination: threaded
messages, blocking ask/reply flows, task dispatch, worker_done/escalation
waits, task DAGs, decision gates, coordinator loops, or decomposing work
across agents. Use `orca-cli` instead for full ownership handoffs, including
requests phrased as "hand off", "handoff", "handover", "give this to another
agent", or "another worktree" when the user did not explicitly ask to
supervise, monitor, wait for results, or coordinate a DAG. Use `orca-cli` for
terminal control, lightweight terminal prompts, shell commands, Orca
worktree management, reading or waiting on terminals, and automation of the
browser embedded inside Orca. Use Computer Use for external browser windows,
webviews, Orca app UI, or desktop UI outside Orca's embedded browser only when
the task requires OS/window-level control such as focus, menus, dialogs,
coordinates, or screenshots. Use `orca-cli` for Orca's embedded pages and a
page-automation tool such as Playwright or CDP for external pages.
waits, task DAGs, decision gates, or coordinator loops. Use `orca-cli`
instead for full ownership handoffs, including requests phrased as "hand
off", "handoff", "handover", "give this to another agent", or "another
worktree" when the user did not explicitly ask to supervise, monitor, wait
for results, or coordinate a DAG. Use `orca-cli` for terminal control,
lightweight terminal prompts, shell commands, Orca worktree management,
reading or waiting on terminals, and the Orca embedded browser. Use Computer
Use for external browser windows, webviews, Orca app UI, or desktop UI
outside Orca's embedded browser only when the task requires OS/window-level
control such as focus, menus, dialogs, coordinates, or screenshots. Use
`orca-cli` for Orca's embedded pages and a page-automation tool such as
Playwright or CDP for external pages.
---
# Orca Inter-Agent Orchestration
+12 -12
View File
@@ -3,18 +3,18 @@ name: orchestration
description: >-
Use Orca orchestration for structured multi-agent coordination: threaded
messages, blocking ask/reply flows, task dispatch, worker_done/escalation
waits, task DAGs, decision gates, coordinator loops, or decomposing work
across agents. Use `orca-cli` instead for full ownership handoffs, including
requests phrased as "hand off", "handoff", "handover", "give this to another
agent", or "another worktree" when the user did not explicitly ask to
supervise, monitor, wait for results, or coordinate a DAG. Use `orca-cli` for
terminal control, lightweight terminal prompts, shell commands, Orca
worktree management, reading or waiting on terminals, and automation of the
browser embedded inside Orca. Use Computer Use for external browser windows,
webviews, Orca app UI, or desktop UI outside Orca's embedded browser only when
the task requires OS/window-level control such as focus, menus, dialogs,
coordinates, or screenshots. Use `orca-cli` for Orca's embedded pages and a
page-automation tool such as Playwright or CDP for external pages.
waits, task DAGs, decision gates, or coordinator loops. Use `orca-cli`
instead for full ownership handoffs, including requests phrased as "hand
off", "handoff", "handover", "give this to another agent", or "another
worktree" when the user did not explicitly ask to supervise, monitor, wait
for results, or coordinate a DAG. Use `orca-cli` for terminal control,
lightweight terminal prompts, shell commands, Orca worktree management,
reading or waiting on terminals, and the Orca embedded browser. Use Computer
Use for external browser windows, webviews, Orca app UI, or desktop UI
outside Orca's embedded browser only when the task requires OS/window-level
control such as focus, menus, dialogs, coordinates, or screenshots. Use
`orca-cli` for Orca's embedded pages and a page-automation tool such as
Playwright or CDP for external pages.
---
# Orca Orchestration
File diff suppressed because one or more lines are too long
@@ -22,6 +22,20 @@ function workingStatus(): AgentAwakeStatus {
}
}
describe('AgentAwakeService status array ownership', () => {
it('does not observe rows appended to the caller array after setStatuses', () => {
const service = new AgentAwakeService()
service.setMode('auto')
const statuses: AgentAwakeStatus[] = [workingStatus()]
service.setStatuses(statuses)
const before = service.getWorkingAgentCount()
statuses.push(workingStatus(), workingStatus())
expect(service.getWorkingAgentCount()).toBe(before)
})
})
function createBlocker() {
const startedIds = new Set<number>()
let nextId = 1
+4 -2
View File
@@ -105,7 +105,8 @@ export class AgentAwakeService {
}
setStatuses(statuses: AgentAwakeStatus[]): void {
this.statuses = statuses.map((status) => ({ ...status }))
// Copy the array, not every row: the hook server allocates each row fresh per event.
this.statuses = [...statuses]
this.refresh('status-change')
}
@@ -171,7 +172,8 @@ export class AgentAwakeService {
private getEligibleRunningStatusCount(): number {
const now = this.now()
return this.statuses.filter((status) => this.isWakeEligible(status, now)).length
// Counted in place: the filtered array was only ever measured, and this runs per hook event.
return this.statuses.reduce((count, s) => count + (this.isWakeEligible(s, now) ? 1 : 0), 0)
}
private isWakeEligible(status: AgentAwakeStatus, now: number): boolean {
@@ -0,0 +1,40 @@
import { describe, expect, it } from 'vitest'
import { parseClaudeUsageRecord } from './transcript-record-parser'
function assistantLine(overrides: Record<string, unknown> = {}): string {
return JSON.stringify({
type: 'assistant',
sessionId: 'session-1',
timestamp: '2026-01-01T00:00:00.000Z',
message: { usage: { input_tokens: 3, output_tokens: 5 } },
...overrides
})
}
describe('assistant-record prefilter', () => {
it('still parses an ordinary assistant record', () => {
expect(parseClaudeUsageRecord(assistantLine())?.inputTokens).toBe(3)
})
it('rejects a user record that never mentions assistant', () => {
const userLine = JSON.stringify({
type: 'user',
sessionId: 'session-1',
timestamp: '2026-01-01T00:00:00.000Z',
message: { content: 'x'.repeat(200) }
})
expect(parseClaudeUsageRecord(userLine)).toBeNull()
})
it('rejects a non-assistant record that happens to contain the word assistant', () => {
const userLine = JSON.stringify({
type: 'user',
sessionId: 'session-1',
timestamp: '2026-01-01T00:00:00.000Z',
message: { content: 'ask the assistant about this' }
})
expect(parseClaudeUsageRecord(userLine)).toBeNull()
})
})
@@ -84,10 +84,30 @@ function dedupeClaudeUsageTurns(
return deduped
}
/**
* Necessary condition for `JSON.parse(line).type === 'assistant'`, checked before the parse.
*
* Sound for any transcript written by a standard JSON serializer: `JSON.stringify` (which writes
* these files) escapes only quotes, backslashes and control characters, never ASCII letters, so
* the decoded value can only be `assistant` if the line spells it literally. The gate over-admits
* freely — the `parsed.type` check below stays authoritative.
*
* A `\u`-escape fallback was measured and rejected: it costs a second full-line scan and made
* transcripts whose tool results contain control characters 1.43x slower overall.
*/
function mayEncodeAssistantType(line: string): boolean {
return line.includes('assistant')
}
function parseClaudeUsageSourceRecord(
line: string,
fallbackSessionId: string | null = null
): ClaudeUsageParsedSourceTurn | null {
// Only assistant records carry usage, but transcripts interleave user/tool-result lines that
// routinely embed whole files. Reject those before paying for a full parse.
if (!mayEncodeAssistantType(line)) {
return null
}
let parsed: ClaudeUsageSourceRecord
try {
parsed = JSON.parse(line) as ClaudeUsageSourceRecord
@@ -1,7 +1,14 @@
import type { ChildProcess } from 'node:child_process'
import { describe, expect, it, vi } from 'vitest'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import {
findSelfInitiatedTreeKills,
resetSelfInitiatedTreeKillLogForTest
} from '../crash-reporting/self-initiated-tree-kill-log'
import { terminateCodexAppServerProcessTree } from './codex-app-server-process-teardown'
/** Above pid_max on every supported POSIX host, so the group signal is a real ESRCH. */
const UNREACHABLE_PGID = 2_147_483_647
function child() {
return {
pid: 1234,
@@ -10,6 +17,10 @@ function child() {
}
describe('terminateCodexAppServerProcessTree', () => {
beforeEach(() => {
resetSelfInitiatedTreeKillLogForTest()
})
it('waits for the Windows tree kill before releasing the wrapper', async () => {
const target = child()
const release = Promise.withResolvers<void>()
@@ -120,6 +131,55 @@ describe('terminateCodexAppServerProcessTree', () => {
expect(target.kill).not.toHaveBeenCalled()
})
/**
* `selfInitiatedTreeKillCount` decides whether a `render-process-gone` was
* ours. A group that had already exited was killed by nobody, so crediting it
* puts a suspect in the five-second window that Orca never issued. Exercised
* through the real `process.kill(-pgid)` because the swallow being tested
* lives in the production default, not in an injectable seam.
*/
it('does not claim a snapshot group that was already gone', async () => {
const target = { pid: UNREACHABLE_PGID, kill: vi.fn(() => true) as ChildProcess['kill'] }
await expect(
terminateCodexAppServerProcessTree(target, undefined, {
platform: 'darwin',
captureDescendants: async () => ({
rootPgid: UNREACHABLE_PGID,
descendants: [],
capturedAtMs: 1
}),
terminateDescendants: async () => true
})
).resolves.toBe(true)
expect(target.kill).toHaveBeenLastCalledWith('SIGKILL')
expect(findSelfInitiatedTreeKills(Date.now())).toEqual([])
})
it('claims a snapshot group the signal actually reached', async () => {
const target = child()
const signalProcessGroup = vi.fn()
await expect(
terminateCodexAppServerProcessTree(target, undefined, {
platform: 'darwin',
captureDescendants: async () => ({ rootPgid: 1234, descendants: [], capturedAtMs: 1 }),
terminateDescendants: async () => true,
signalProcessGroup
})
).resolves.toBe(true)
expect(signalProcessGroup).toHaveBeenCalledWith(1234, 'SIGKILL')
expect(findSelfInitiatedTreeKills(Date.now())).toEqual([
expect.objectContaining({
pid: 1234,
site: 'codex-app-server-teardown',
scope: 'posix-process-group'
})
])
})
it('tears down 40 dedicated groups without process-table scans or cross-group fanout', async () => {
const killMocks = Array.from({ length: 40 }, () => vi.fn(() => true))
const targets = killMocks.map((kill, index) => ({
@@ -128,19 +128,24 @@ async function terminatePosixTree(
if (descendantsExited && snapshot.rootPgid === rootPid) {
const signalGroup =
deps.signalProcessGroup ??
((pgid: number, signal: NodeJS.Signals) => {
try {
process.kill(-pgid, signal)
} catch {
// Group already exited.
}
((pgid: number, signal: NodeJS.Signals) => process.kill(-pgid, signal))
let groupSignalled = false
try {
signalGroup(snapshot.rootPgid, 'SIGKILL')
groupSignalled = true
} catch {
// Already-gone is still the desired outcome, but nothing here killed it,
// and a crumb for a kill we never landed is a false render-process-gone suspect.
}
if (groupSignalled) {
// Outside the try, as in terminateDedicatedPosixGroup: that catch is the
// already-gone contract, not a breadcrumb handler.
recordSelfInitiatedTreeKill({
pid: snapshot.rootPgid,
site: 'codex-app-server-teardown',
scope: 'posix-process-group'
})
signalGroup(snapshot.rootPgid, 'SIGKILL')
recordSelfInitiatedTreeKill({
pid: snapshot.rootPgid,
site: 'codex-app-server-teardown',
scope: 'posix-process-group'
})
}
}
if (!descendantsExited) {
child.kill('SIGCONT')
@@ -20,10 +20,7 @@ export function codexJournalPromptIdPart(value: string): string {
}
const suffix = `#${digestPayload(value).slice(0, 32)}`
const bounded = boundPayload(value, {
inlineHeadBytes: CODEX_JOURNAL_PROMPT_ID_COMPONENT_MAX_BYTES - suffix.length,
maxSessionBytes: Number.MAX_SAFE_INTEGER,
maxAppendsPerWindow: Number.MAX_SAFE_INTEGER,
appendWindowMs: Number.MAX_SAFE_INTEGER
inlineHeadBytes: CODEX_JOURNAL_PROMPT_ID_COMPONENT_MAX_BYTES - suffix.length
})
return `${bounded.head}${suffix}`
}
@@ -17,14 +17,8 @@ export function codexStructuredItemKey(threadId: string, itemId: string): string
return `${key.slice(0, 960)}:${(hash >>> 0).toString(16)}`
}
export function pendingPatchBytes(pending: {
body: unknown
blobs: readonly { payload: string }[]
}): number {
return (
Buffer.byteLength(JSON.stringify(pending.body), 'utf8') +
pending.blobs.reduce((total, blob) => total + Buffer.byteLength(blob.payload, 'utf8'), 0)
)
export function pendingPatchBytes(pending: { body: unknown }): number {
return Buffer.byteLength(JSON.stringify(pending.body), 'utf8')
}
export function boundStreamItem(item: Record<string, unknown>): Record<string, unknown> {
@@ -24,7 +24,6 @@ export type CodexItemStreamState = {
export type CodexPendingItemPatch = {
identity: AgentJournalItemIdentity
body: NonNullable<ReturnType<typeof codexJournalItem>['body']>
blobs: ReturnType<typeof codexJournalItem>['blobs']
}
export type CodexStructuredItemStreamAdmission =
@@ -103,8 +103,8 @@ export function createCodexStructuredItemStreams(
}
const options = { coalescingKey: `checkpoint:${agentJournalItemKey(state.identity)}` }
const admission = deps.sink.tryAppendItem
? deps.sink.tryAppendItem(state.identity, translated.body, translated.blobs, options)
: (deps.sink.appendItem(state.identity, translated.body, translated.blobs, options),
? deps.sink.tryAppendItem(state.identity, translated.body, options)
: (deps.sink.appendItem(state.identity, translated.body, options),
{ accepted: true as const })
if (!admission.accepted) {
return false
@@ -167,9 +167,8 @@ export function createCodexStructuredItemStreams(
}
for (const [key, pending] of pendingPatches) {
const admission = deps.sink.tryAppendItem
? deps.sink.tryAppendItem(pending.identity, pending.body, pending.blobs)
: (deps.sink.appendItem(pending.identity, pending.body, pending.blobs),
{ accepted: true as const })
? deps.sink.tryAppendItem(pending.identity, pending.body)
: (deps.sink.appendItem(pending.identity, pending.body), { accepted: true as const })
if (!admission.accepted) {
flushed = false
continue
@@ -193,9 +192,8 @@ export function createCodexStructuredItemStreams(
return { accepted: true }
}
const admission = deps.sink.tryAppendItem
? deps.sink.tryAppendItem(pending.identity, pending.body, pending.blobs)
: (deps.sink.appendItem(pending.identity, pending.body, pending.blobs),
{ accepted: true as const })
? deps.sink.tryAppendItem(pending.identity, pending.body)
: (deps.sink.appendItem(pending.identity, pending.body), { accepted: true as const })
if (!admission.accepted) {
return admission
}
@@ -237,8 +235,7 @@ export function createCodexStructuredItemStreams(
if (translated.body) {
const nextPending: CodexPendingItemPatch = {
identity: state.identity,
body: translated.body,
blobs: translated.blobs
body: translated.body
}
const previous = pendingPatches.get(key)
const previousBytes = previous ? pendingPatchBytes(previous) : 0
@@ -220,12 +220,6 @@ describe('codex item bodies', () => {
throw new Error('expected bounded command output')
}
expect(body.output.head.length).toBeLessThan(20_000)
expect(translated.blobs).toEqual([
{
digest: body.output.digest,
payload: output
}
])
})
it('continues to accept camel-case command completion output', () => {
@@ -172,7 +172,6 @@ function commandState(item: CodexThreadItem): 'running' | 'completed' | 'failed'
export type CodexJournalItem = {
body: AgentJournalItemBody | null
blobs: { digest: string; payload: string }[]
handled: boolean
}
@@ -190,10 +189,6 @@ function commandItem(item: CodexThreadItem): CodexJournalItem {
state: commandState(item),
...(bounded === null ? {} : { output: bounded.bounded })
},
blobs:
output !== null && bounded?.bounded.truncated
? [{ digest: bounded.bounded.digest, payload: output }]
: [],
handled: true
}
}
@@ -215,7 +210,6 @@ function fileChangeItem(item: CodexThreadItem): CodexJournalItem {
input: boundToolInput({ changes: item.changes ?? null }, DEFAULT_JOURNAL_PAYLOAD_LIMITS),
state: commandState(item)
},
blobs: [],
handled: true
}
}
@@ -227,7 +221,6 @@ function fileChangeItem(item: CodexThreadItem): CodexJournalItem {
path: changes.length === 1 ? changes[0]!.path : `${changes.length} files`,
patch: bounded
},
blobs: bounded.truncated ? [{ digest: bounded.digest, payload: patch }] : [],
handled: true
}
}
@@ -246,7 +239,6 @@ export function codexJournalItem(item: CodexThreadItem): CodexJournalItem {
blocks.length === 0
? null
: { kind: 'message', role: item.type === 'userMessage' ? 'user' : 'assistant', blocks },
blobs: [],
handled: true
}
}
@@ -266,14 +258,11 @@ export function codexJournalItem(item: CodexThreadItem): CodexJournalItem {
text === null
? null
: { kind: 'status', text: boundInlineText(text, DEFAULT_JOURNAL_PAYLOAD_LIMITS).text },
blobs: [],
handled: true
}
}
const unhandled = unhandledProviderFrameJournalItem('codex', `item:${item.type}`, item)
return unhandled
? { body: unhandled.body, blobs: unhandled.blobs, handled: false }
: { body: null, blobs: [], handled: true }
return unhandled ? { body: unhandled.body, handled: false } : { body: null, handled: true }
}
export function codexItemBody(item: CodexThreadItem): AgentJournalItemBody | null {
@@ -292,7 +281,7 @@ export function codexStreamingMessageBody(text: string): AgentJournalItemBody {
/** Snapshot body for any item-level stream, keyed onto its parent item. */
export function codexStreamingJournalItem(item: CodexThreadItem, text: string): CodexJournalItem {
if (item.type === 'agentMessage') {
return { body: codexStreamingMessageBody(text), blobs: [], handled: true }
return { body: codexStreamingMessageBody(text), handled: true }
}
if (item.type === 'commandExecution') {
return commandItem({ ...item, aggregatedOutput: text })
@@ -304,10 +293,9 @@ export function codexStreamingJournalItem(item: CodexThreadItem, text: string):
const bounded = boundInlineText(text, DEFAULT_JOURNAL_PAYLOAD_LIMITS).bounded
return {
body: { kind: 'diff', path: path ?? 'pending patch', patch: bounded },
blobs: bounded.truncated ? [{ digest: bounded.digest, payload: text }] : [],
handled: true
}
}
const bounded = boundInlineText(text, DEFAULT_JOURNAL_PAYLOAD_LIMITS)
return { body: { kind: 'status', text: bounded.text }, blobs: [], handled: true }
return { body: { kind: 'status', text: bounded.text }, handled: true }
}
@@ -91,13 +91,11 @@ export class CodexJournalGenericFrames {
const admission = this.deps.sink.tryAppendItem
? this.deps.sink.tryAppendItem(
{ provider: 'orca', clientMessageId: `provider-frame:codex:${this.fallbackSequence}` },
translated.body,
translated.blobs
translated.body
)
: (this.deps.sink.appendItem(
{ provider: 'orca', clientMessageId: `provider-frame:codex:${this.fallbackSequence}` },
translated.body,
translated.blobs
translated.body
),
CODEX_JOURNAL_ADMITTED)
if (!admission.accepted) {
@@ -136,13 +134,11 @@ export class CodexJournalGenericFrames {
kind: 'status',
text
},
[],
{ coalescingKey: `provider-frame-suppressed:codex:${bucket}` }
)
: (this.deps.sink.appendItem(
{ provider: 'orca', clientMessageId: `provider-frame-suppressed:codex:${bucket}` },
{ kind: 'status', text },
[],
{ coalescingKey: `provider-frame-suppressed:codex:${bucket}` }
),
CODEX_JOURNAL_ADMITTED)
@@ -2,7 +2,7 @@ import type {
AgentJournalItemBody,
AgentJournalItemIdentity
} from '../../shared/agent-session-journal-types'
import { requiresTerminalSettlement } from '../native-chat/agent-session-journal/journal-lifecycle-capacity'
import { requiresTerminalSettlement } from '../native-chat/agent-session-journal/journal-terminal-settlement'
import {
codexItemIdentity,
codexJournalItem,
@@ -126,19 +126,13 @@ export class CodexJournalItems {
return CODEX_JOURNAL_ADMITTED
}
if (method === 'item/completed') {
const admission = appendCodexLifecycleItem(
this.deps.sink,
identity,
translated.body,
translated.blobs
)
const admission = appendCodexLifecycleItem(this.deps.sink, identity, translated.body)
return admission.accepted ? publishCodexLifecycle(this.deps.sink) : admission
}
const options = requiresTerminalSettlement(translated.body) ? { lifecycle: true } : {}
const admission = this.deps.sink.tryAppendItem
? this.deps.sink.tryAppendItem(identity, translated.body, translated.blobs, options)
: (this.deps.sink.appendItem(identity, translated.body, translated.blobs),
CODEX_JOURNAL_ADMITTED)
? this.deps.sink.tryAppendItem(identity, translated.body, options)
: (this.deps.sink.appendItem(identity, translated.body), CODEX_JOURNAL_ADMITTED)
if (!admission.accepted) {
return admission
}
@@ -243,14 +243,12 @@ function appendLifecycleMutations(
for (const mutation of chunk) {
if (mutation.kind === 'item') {
if (sink.tryAppendItem) {
admission = sink.tryAppendItem(mutation.identity, mutation.body, [], {
lifecycle: true
})
admission = sink.tryAppendItem(mutation.identity, mutation.body, { lifecycle: true })
if (!admission.accepted) {
return admission
}
} else {
sink.appendItem(mutation.identity, mutation.body, [], { lifecycle: true })
sink.appendItem(mutation.identity, mutation.body, { lifecycle: true })
}
} else {
if (sink.tryAppendTombstone) {
@@ -4,7 +4,6 @@ import type {
} from '../../shared/agent-session-journal-types'
import type {
StructuredAgentSessionEventSink,
StructuredAgentSessionJournalBlob,
StructuredAgentSessionSinkAdmission
} from '../native-chat/agent-session-wire/structured-agent-session-event-sink'
import type { CodexPendingJournalPrompt } from './codex-structured-journal-settlement'
@@ -20,13 +19,12 @@ function criticalAdmission(
export function appendCodexLifecycleItem(
sink: StructuredAgentSessionEventSink,
identity: AgentJournalItemIdentity,
body: AgentJournalItemBody,
blobs: readonly StructuredAgentSessionJournalBlob[] = []
body: AgentJournalItemBody
): CodexJournalTranslationAdmission {
if (sink.tryAppendItem) {
return criticalAdmission(sink.tryAppendItem(identity, body, blobs, { lifecycle: true }))
return criticalAdmission(sink.tryAppendItem(identity, body, { lifecycle: true }))
}
sink.appendItem(identity, body, blobs, { lifecycle: true })
sink.appendItem(identity, body, { lifecycle: true })
return CODEX_JOURNAL_ADMITTED
}
@@ -89,11 +89,11 @@ describe('codex journal translation', () => {
const { translator, tap } = translatorWith()
let rejectTerminal = true
const appendItem = tap.sink.appendItem
tap.sink.tryAppendItem = (identity, body, blobs, options) => {
tap.sink.tryAppendItem = (identity, body, options) => {
if (rejectTerminal && body.kind === 'tool-call' && body.state === 'failed') {
return { accepted: false as const, reason: 'backpressure' as const }
}
appendItem(identity, body, blobs, options)
appendItem(identity, body, options)
return { accepted: true as const }
}
for (let index = 0; index <= 256; index += 1) {
@@ -40,7 +40,6 @@ export function publishCodexTurnLifecycle(input: {
text: 'Codex is working…',
turnLifecycle: { turnId: input.turnId, state: input.state }
},
[],
{ lifecycle: true }
)
: (input.sink.appendItem(
@@ -50,7 +49,6 @@ export function publishCodexTurnLifecycle(input: {
text: 'Codex is working…',
turnLifecycle: { turnId: input.turnId, state: input.state }
},
[],
{ lifecycle: true }
),
ADMITTED)
+1 -4
View File
@@ -37,10 +37,7 @@ export class CodexTurnOrdinals {
const suffix = `#${digestPayload(value).slice(0, 24)}`
return `${
boundPayload(encoded, {
inlineHeadBytes: 256 - Buffer.byteLength(suffix, 'utf8'),
maxSessionBytes: Number.MAX_SAFE_INTEGER,
maxAppendsPerWindow: Number.MAX_SAFE_INTEGER,
appendWindowMs: Number.MAX_SAFE_INTEGER
inlineHeadBytes: 256 - Buffer.byteLength(suffix, 'utf8')
}).head
}${suffix}`
}

Some files were not shown because too many files have changed in this diff Show More