Merge remote-tracking branch 'origin/main' into brennanb2025/codex-windows-real-home-lane

This commit is contained in:
Merge Sim
2026-09-07 13:29:34 -07:00
433 changed files with 25090 additions and 2351 deletions
@@ -4,7 +4,7 @@ on:
workflow_dispatch:
inputs:
image-digest:
description: "Immutable relay image digest (sha256: plus 64 lowercase hex characters)"
description: 'Immutable relay image digest (sha256: plus 64 lowercase hex characters)'
required: true
type: string
regional-placement-mode:
@@ -14,6 +14,7 @@ on:
not-before: { required: true, type: string }
rate-per-minute: { required: true, type: string }
preference-max-age-ms: { required: true, type: string }
host-cooldown-ms: { required: true, type: string }
drain-grace-ms: { required: true, type: string }
confirmation: { required: true, type: string }
monitor-run-id: { required: true, type: string }
@@ -54,6 +55,7 @@ jobs:
NOT_BEFORE: ${{ inputs.not-before }}
RATE_PER_MINUTE: ${{ inputs.rate-per-minute }}
PREFERENCE_MAX_AGE_MS: ${{ inputs.preference-max-age-ms }}
HOST_COOLDOWN_MS: ${{ inputs.host-cooldown-ms }}
DRAIN_GRACE_MS: ${{ inputs.drain-grace-ms }}
CONFIRMATION: ${{ inputs.confirmation }}
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
@@ -128,6 +130,7 @@ jobs:
--expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
--not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \
--preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \
--host-cooldown-ms "${HOST_COOLDOWN_MS}" \
--drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \
| tee "${RUNNER_TEMP}/relay-rehome-control.json"
@@ -299,6 +302,7 @@ jobs:
--expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
--not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \
--preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \
--host-cooldown-ms "${HOST_COOLDOWN_MS}" \
--drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \
| tee "${RUNNER_TEMP}/relay-rehome-control.json"
@@ -52,6 +52,11 @@ on:
required: true
default: '86400000'
type: string
host-cooldown-ms:
description: Minimum gap between two rehomes of the same host
required: true
default: '604800000'
type: string
drain-grace-ms:
description: Per-host source drain grace
required: true
@@ -99,6 +104,7 @@ jobs:
not-before: ${{ inputs.not-before }}
rate-per-minute: ${{ inputs.rate-per-minute }}
preference-max-age-ms: ${{ inputs.preference-max-age-ms }}
host-cooldown-ms: ${{ inputs.host-cooldown-ms }}
drain-grace-ms: ${{ inputs.drain-grace-ms }}
confirmation: ${{ inputs.confirmation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
+1
View File
@@ -856,6 +856,7 @@ jobs:
src/main/agent-hooks/windows-hook-payload-delivery.test.ts
src/main/agent-hooks/windows-direct-cmd-hook-command.test.ts
src/main/windows/windows-pty-job.win32.test.ts
src/main/windows/windows-msys-job.win32.test.ts
src/main/windows/windows-host-job.win32.test.ts
src/main/windows/windows-process-tree-command-line-patch.test.ts
src/main/windows/windows-process-table-native-addon.win32.test.ts
+7 -1
View File
@@ -606,8 +606,9 @@ export function createRelayApp(
const source = await operations.assignments.cellDeploymentStatus(
body.data.sourceCellId
)
// Any cell that can be drained can be a rehome source, in either
// direction, so the probe is gated on the protocol and not on a region.
if (
source.region !== RELAY_DEFAULT_REGION ||
!source.runtime ||
source.runtime.cellIncarnation !== body.data.sourceCellIncarnation ||
!source.runtime.ready ||
@@ -1412,6 +1413,11 @@ const RegionalRehomeControlSchema = z.discriminatedUnion('action', [
.int()
.min(60_000)
.max(30 * 24 * 60 * 60_000),
hostCooldownMs: z
.number()
.int()
.min(60_000)
.max(30 * 24 * 60 * 60_000),
drainGraceMs: z.number().int().min(60_000).max(60 * 60_000),
confirmation: z.enum([
'ENABLE_REGIONAL_REHOMING',
@@ -1,3 +1,4 @@
import { RELAY_DEFAULT_REGION } from '@orca-cloud/relay-contract'
import type { RelayDatabase, SqlRow } from './database.js'
export type CellInventorySnapshotRow = {
@@ -92,7 +93,7 @@ export async function readAssignmentInventorySnapshot(
return {
cells: cellRows.map((row) => ({
cellId: asText(row, 'cell_id'),
region: optionalText(row, 'region') ?? 'us-central1',
region: optionalText(row, 'region') ?? RELAY_DEFAULT_REGION,
admissionState: optionalText(row, 'admission_state') ?? 'unset',
enabled: asInteger(row, 'enabled') === 1,
capacityRequests: asInteger(row, 'capacity_requests'),
+107 -28
View File
@@ -30,6 +30,9 @@ import {
ASSIGNMENT_CONNECTION_HEADROOM_QUERY
} from './assignment-connection-headroom-query.js'
import { AssignmentIdentityQueue } from './assignment-identity-queue.js'
import {
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS
} from './database.js'
import type { RelayCellConfig } from './config.js'
import type {
RelayDatabase,
@@ -121,7 +124,7 @@ export type RelayAssignmentMigration = AssignmentIdentity & {
export type RegionalRehomeAttempt = AssignmentIdentity & {
attemptId: string
preferredRegion: 'asia-east2'
preferredRegion: RelayRegion
sourceCellId: string
sourceCellUrl: string
sourceCellIncarnation: string
@@ -151,6 +154,7 @@ export type RegionalRehomeControl = {
notBefore: number
ratePerMinute: number
preferenceMaxAgeMs: number
hostCooldownMs: number
drainGraceMs: number
}
@@ -4921,6 +4925,7 @@ export class RelayAssignmentStore {
notBefore: number
ratePerMinute: number
preferenceMaxAgeMs: number
hostCooldownMs: number
drainGraceMs: number
}): Promise<RegionalRehomeControl> {
if (!Number.isSafeInteger(input.expectedGeneration) || input.expectedGeneration < 0) {
@@ -4939,6 +4944,13 @@ export class RelayAssignmentStore {
) {
throw new Error('invalid_regional_rehome_preference_age')
}
if (
!Number.isSafeInteger(input.hostCooldownMs) ||
input.hostCooldownMs < 60_000 ||
input.hostCooldownMs > 30 * 24 * 60 * 60_000
) {
throw new Error('invalid_regional_rehome_host_cooldown')
}
if (
!Number.isSafeInteger(input.drainGraceMs) ||
input.drainGraceMs < 60_000 ||
@@ -4967,14 +4979,15 @@ export class RelayAssignmentStore {
await transaction.query(
`UPDATE relay_region_rehome_control
SET generation = generation + 1, enabled = ?, not_before = ?,
rate_per_minute = ?, preference_max_age_ms = ?, drain_grace_ms = ?,
updated_at = ?
rate_per_minute = ?, preference_max_age_ms = ?, host_cooldown_ms = ?,
drain_grace_ms = ?, updated_at = ?
WHERE control_id = 'global'`,
[
input.enabled ? 1 : 0,
input.notBefore,
input.ratePerMinute,
input.preferenceMaxAgeMs,
input.hostCooldownMs,
input.drainGraceMs,
now
]
@@ -5006,10 +5019,17 @@ export class RelayAssignmentStore {
await database.query(
`INSERT INTO relay_region_rehome_control
(control_id, generation, enabled, observation_started_at, not_before,
rate_per_minute, preference_max_age_ms, drain_grace_ms, updated_at)
VALUES ('global', 0, 0, ?, 0, 10, ?, ?, ?)
rate_per_minute, preference_max_age_ms, host_cooldown_ms, drain_grace_ms,
updated_at)
VALUES ('global', 0, 0, ?, 0, 10, ?, ?, ?, ?)
ON CONFLICT (control_id) DO NOTHING`,
[now, 24 * 60 * 60_000, 60 * 60_000, now]
[
now,
24 * 60 * 60_000,
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS,
60 * 60_000,
now
]
)
}
@@ -5017,6 +5037,9 @@ export class RelayAssignmentStore {
return await this.readRegionalRehomeFleetSafety(this.database, this.now())
}
// The rehome fleet is every general cell that can be drained: those are the
// sources and, because a host must be movable back out again, the only legal
// targets. The region join stays so a cell with no region row is excluded.
private async readRegionalRehomeFleetSafety(
database: RelayDatabase,
now: number
@@ -5037,10 +5060,7 @@ export class RelayAssignmentStore {
ON safety.cell_id = runtime.cell_id
AND safety.cell_incarnation = runtime.cell_incarnation
WHERE cell.enabled = 1 AND admission.admission_state = 'general'
AND (
region.region = 'asia-east2' OR
(region.region = 'us-central1' AND capability.regional_rehome_protocol >= 1)
)`
AND capability.regional_rehome_protocol >= 1`
)
const valid = rows.filter(
(row) =>
@@ -5119,6 +5139,10 @@ export class RelayAssignmentStore {
}
const intervalMs = Math.ceil(60_000 / integer(control, 'rate_per_minute'))
const preferenceCutoff = now - integer(control, 'preference_max_age_ms')
// A host that was rehomed recently is left alone whichever way its
// preference now points: a flapping region probe must not walk one host
// back and forth across an ocean.
const cooldownCutoff = now - integer(control, 'host_cooldown_ms')
await transaction.query(
`INSERT INTO relay_region_rehome_worker_state
(worker_id, next_dispatch_at, paused_until, consecutive_failures, updated_at)
@@ -5284,9 +5308,8 @@ export class RelayAssignmentStore {
JOIN relay_cell_capabilities capability
ON capability.cell_id = runtime.cell_id
AND capability.cell_incarnation = runtime.cell_incarnation
WHERE preference.preferred_region = 'asia-east2'
WHERE preference.preferred_region <> region.region
AND preference.observed_at >= ?
AND region.region = 'us-central1'
AND admission.admission_state = 'general'
AND runtime.ready = 1 AND runtime.last_heartbeat_at > ?
AND capability.regional_rehome_protocol >= 1
@@ -5306,9 +5329,38 @@ export class RelayAssignmentStore {
AND migration.relay_host_id = assignment.relay_host_id
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL
)
AND NOT EXISTS (
SELECT 1 FROM relay_region_rehome_attempts recent
WHERE recent.user_id = preference.user_id
AND recent.relay_host_id = preference.relay_host_id
AND recent.created_at > ?
)
AND EXISTS (
SELECT 1 FROM relay_cell_regions target_region
JOIN relay_cells target_cell ON target_cell.cell_id = target_region.cell_id
JOIN relay_cell_admission target_admission
ON target_admission.cell_id = target_region.cell_id
JOIN relay_cell_runtime target_runtime
ON target_runtime.cell_id = target_region.cell_id
JOIN relay_cell_capabilities target_capability
ON target_capability.cell_id = target_runtime.cell_id
AND target_capability.cell_incarnation = target_runtime.cell_incarnation
WHERE target_region.region = preference.preferred_region
AND target_cell.enabled = 1
AND target_admission.admission_state = 'general'
AND target_runtime.ready = 1
AND target_runtime.last_heartbeat_at > ?
AND target_capability.regional_rehome_protocol >= 1
)
ORDER BY preference.observed_at, preference.user_id, preference.relay_host_id
LIMIT 10`,
[preferenceCutoff, now - this.heartbeatTtlMs, now]
[
preferenceCutoff,
now - this.heartbeatTtlMs,
now,
cooldownCutoff,
now - this.heartbeatTtlMs
]
)
candidatesTotal = candidates.length
for (const candidate of candidates) {
@@ -5320,6 +5372,7 @@ export class RelayAssignmentStore {
sourceCellId: text(candidate, 'source_cell_id'),
assignmentEpoch: integer(candidate, 'assignment_epoch'),
preferenceCutoff,
cooldownCutoff,
drainGraceMs: integer(control, 'drain_grace_ms'),
processSafety: effectiveProcessSafety,
worker,
@@ -5374,6 +5427,7 @@ export class RelayAssignmentStore {
sourceCellId: string
assignmentEpoch: number
preferenceCutoff: number
cooldownCutoff: number
drainGraceMs: number
processSafety: RegionalRehomeSafetySnapshot
worker: SqlRow
@@ -5397,14 +5451,11 @@ export class RelayAssignmentStore {
[input.identity.userId, input.identity.relayHostId]
)
)[0]
if (
!preference ||
text(preference, 'preferred_region') !== 'asia-east2' ||
integer(preference, 'observed_at') < input.preferenceCutoff
) {
if (!preference || integer(preference, 'observed_at') < input.preferenceCutoff) {
input.skips.push({ reason: 'candidate_stale' })
return null
}
const preferredRegion = relayRegion(preference, 'preferred_region')
const activeMigration = await transaction.queryLocked(
`SELECT assignment_epoch FROM relay_assignment_migrations
WHERE user_id = ? AND relay_host_id = ?
@@ -5415,6 +5466,18 @@ export class RelayAssignmentStore {
input.skips.push({ reason: 'candidate_stale' })
return null
}
// Re-read under the claim: an attempt committed between the scan and here
// would otherwise start a second move for the same host.
const recentAttempt = await transaction.query(
`SELECT 1 FROM relay_region_rehome_attempts
WHERE user_id = ? AND relay_host_id = ? AND created_at > ?
LIMIT 1`,
[input.identity.userId, input.identity.relayHostId, input.cooldownCutoff]
)
if (recentAttempt.length > 0) {
input.skips.push({ reason: 'host_cooldown' })
return null
}
const activityLeases = await this.lockAssignmentActivities(transaction, input.identity)
assertAssignmentActivityCounts(assignment, activityLeases, 0)
const cells = await this.lockCellInventory(transaction, 'nowait')
@@ -5469,11 +5532,17 @@ export class RelayAssignmentStore {
)
return null
}
// The preference read under lock can now agree with the cell the host is
// already on: nothing to move, in either direction.
if (regions.get(input.sourceCellId) === preferredRegion) {
input.skips.push({ reason: 'candidate_stale' })
return null
}
if (
!source ||
integer(source, 'enabled') !== 1 ||
admission.get(input.sourceCellId) !== 'general' ||
regions.get(input.sourceCellId) !== RELAY_DEFAULT_REGION ||
regions.get(input.sourceCellId) === undefined ||
!sourceRuntime ||
integer(sourceRuntime, 'ready') !== 1 ||
integer(sourceRuntime, 'last_heartbeat_at') <= input.now - this.heartbeatTtlMs ||
@@ -5502,17 +5571,25 @@ export class RelayAssignmentStore {
return null
}
const connectionHeadroom = await this.connectionHeadroomByCell(transaction)
// A target must be drainable too, or the host lands somewhere it can never
// be rehomed out of again -- the trap this bidirectional move exists to undo.
const eligibleTargets = cells.filter((row) => {
const cellId = text(row, 'cell_id')
const runtime = runtimes.find((candidate) => text(candidate, 'cell_id') === cellId)
const capability = capabilities.find(
(candidate) => text(candidate, 'cell_id') === cellId
)
return (
cellId !== input.sourceCellId &&
integer(row, 'enabled') === 1 &&
admission.get(cellId) === 'general' &&
regions.get(cellId) === 'asia-east2' &&
regions.get(cellId) === preferredRegion &&
runtime !== undefined &&
integer(runtime, 'ready') === 1 &&
integer(runtime, 'last_heartbeat_at') > input.now - this.heartbeatTtlMs
integer(runtime, 'last_heartbeat_at') > input.now - this.heartbeatTtlMs &&
capability !== undefined &&
text(capability, 'cell_incarnation') === text(runtime, 'cell_incarnation') &&
integer(capability, 'regional_rehome_protocol') >= 1
)
})
const targetIsClean = (row: SqlRow): boolean => {
@@ -5668,12 +5745,13 @@ export class RelayAssignmentStore {
drain_grace_ms, send_attempts, last_send_attempt_at,
drain_receipt_at, drain_outcome, completed_at, aborted_at,
created_at, updated_at)
VALUES (?, ?, ?, 'asia-east2', ?, ?, ?, ?, ?, ?, ?, 0, NULL,
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, NULL,
NULL, NULL, NULL, NULL, ?, ?)`,
[
attemptId,
input.identity.userId,
input.identity.relayHostId,
preferredRegion,
input.sourceCellId,
text(sourceRuntime, 'cell_incarnation'),
targetCellId,
@@ -5688,7 +5766,7 @@ export class RelayAssignmentStore {
return {
...input.identity,
attemptId,
preferredRegion: 'asia-east2',
preferredRegion,
sourceCellId: input.sourceCellId,
sourceCellUrl: text(source, 'cell_url'),
sourceCellIncarnation: text(sourceRuntime, 'cell_incarnation'),
@@ -8101,7 +8179,7 @@ function regionalRehomeAttempt(row: SqlRow): RegionalRehomeAttempt {
attemptId: text(row, 'attempt_id'),
userId: text(row, 'user_id'),
relayHostId: text(row, 'relay_host_id'),
preferredRegion: 'asia-east2',
preferredRegion: relayRegion(row, 'preferred_region'),
sourceCellId: text(row, 'source_cell_id'),
sourceCellUrl: text(row, 'source_cell_url'),
sourceCellIncarnation: text(row, 'source_cell_incarnation'),
@@ -8122,6 +8200,7 @@ function regionalRehomeControl(row: SqlRow): RegionalRehomeControl {
notBefore: integer(row, 'not_before'),
ratePerMinute: integer(row, 'rate_per_minute'),
preferenceMaxAgeMs: integer(row, 'preference_max_age_ms'),
hostCooldownMs: integer(row, 'host_cooldown_ms'),
drainGraceMs: integer(row, 'drain_grace_ms')
}
}
@@ -8161,10 +8240,9 @@ function regionalRehomeFleetSafetyFromInventory(input: {
return (
integer(row, 'enabled') === 1 &&
input.admission.get(cellId) === 'general' &&
(input.regions.get(cellId) === 'asia-east2' ||
(input.regions.get(cellId) === RELAY_DEFAULT_REGION &&
capability !== undefined &&
integer(capability, 'regional_rehome_protocol') >= 1))
input.regions.get(cellId) !== undefined &&
capability !== undefined &&
integer(capability, 'regional_rehome_protocol') >= 1
)
})
const valid = required.flatMap((row) => {
@@ -8231,6 +8309,7 @@ function regionalRehomeFleetSafetyFailure(
type RegionalRehomeCandidateSkip = {
reason:
| 'candidate_stale'
| 'host_cooldown'
| 'source_ineligible'
| 'source_unclean'
| 'source_control_inactive'
@@ -1,4 +1,5 @@
import { randomUUID } from 'node:crypto'
import { RELAY_DEFAULT_REGION } from '@orca-cloud/relay-contract'
import type { RelayConfig } from './config.js'
import { googleMetadataIdentityToken } from './google-metadata-identity-token.js'
import type { RegionalRehomeSafetySnapshot } from './relay-observability.js'
@@ -57,7 +58,7 @@ export function startCellHeartbeat(
v: 1,
cellId: config.cellId,
cellUrl: config.cellUrl,
region: config.region ?? 'us-central1',
region: config.region ?? RELAY_DEFAULT_REGION,
cellIncarnation,
startedAt,
ready,
@@ -34,7 +34,11 @@ vi.mock('pg', () => ({
}
}))
import { openRelayDatabase, relayPostgresStatementTimeoutMs } from './database.js'
import {
openRelayDatabase,
POSTGRES_SCHEMA_MIGRATIONS,
relayPostgresStatementTimeoutMs
} from './database.js'
import { applyPostgresSchema } from './postgres-schema-startup.js'
const SCHEMA_POOL = {
@@ -118,7 +122,9 @@ describe('PostgreSQL relay deadlines', () => {
// Statements can open with a leading `--` rationale comment.
const body = (statement: string): string =>
statement.replace(/^(?:\s*--[^\n]*\n)*\s*/, '')
expect(ddl.every((statement) => /^CREATE\b/i.test(body(statement)))).toBe(true)
expect(
ddl.every((statement) => /^(?:CREATE|ALTER TABLE)\b/i.test(body(statement)))
).toBe(true)
// The backfill is DML, so it stays on the deadline-bearing serving pool.
expect(ddl.some((statement) => statement.includes('INSERT INTO'))).toBe(false)
await database.close()
@@ -263,6 +269,54 @@ describe('PostgreSQL schema startup', () => {
expect(query).toHaveBeenCalledTimes(2)
})
it('treats an existing constraint as an applied ADD CONSTRAINT', async () => {
// Postgres has no `ADD CONSTRAINT IF NOT EXISTS`, and a retry would only
// repeat 42710, so a re-run and a concurrent startup both move on.
const error = Object.assign(new Error('already exists'), { code: '42710' })
const query = vi
.fn<(statement: string) => Promise<unknown>>()
.mockRejectedValueOnce(error)
.mockResolvedValue(undefined)
const pause = vi.fn(async () => undefined)
await applyPostgresSchema(
['ALTER TABLE test ADD CONSTRAINT test_check CHECK (id > 0)', 'CREATE TABLE test2'],
query,
{ wait: pause }
)
expect(pause).not.toHaveBeenCalled()
expect(query).toHaveBeenCalledTimes(2)
expect(query).toHaveBeenLastCalledWith('CREATE TABLE test2')
})
it('recognises every shipped ADD CONSTRAINT migration as re-runnable', async () => {
// Guards the statement text against the pattern that classifies it.
const shipped = POSTGRES_SCHEMA_MIGRATIONS.filter((statement) =>
statement.includes('ADD CONSTRAINT')
)
expect(shipped.length).toBeGreaterThan(0)
const error = Object.assign(new Error('already exists'), { code: '42710' })
const query = vi.fn<(statement: string) => Promise<unknown>>().mockRejectedValue(error)
await applyPostgresSchema(shipped, query, { wait: async () => undefined })
expect(query).toHaveBeenCalledTimes(shipped.length)
})
it('still fails an ADD CONSTRAINT that violates existing rows', async () => {
const error = Object.assign(new Error('check violation'), { code: '23514' })
const query = vi.fn<(statement: string) => Promise<unknown>>().mockRejectedValue(error)
await expect(
applyPostgresSchema(
['ALTER TABLE test ADD CONSTRAINT test_check CHECK (id > 0)'],
query,
{ wait: async () => undefined }
)
).rejects.toBe(error)
})
it.each([
['42710', 'CREATE INDEX IF NOT EXISTS test_index ON test(id)'],
['42710', 'CREATE TABLE test'],
+41 -1
View File
@@ -2,7 +2,12 @@ import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it } from 'vitest'
import { openInMemoryRelayDatabase, openRelayDatabase } from './database.js'
import {
openInMemoryRelayDatabase,
openRelayDatabase,
POSTGRES_SCHEMA_MIGRATIONS,
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS
} from './database.js'
const temporaryDirectories: string[] = []
@@ -142,6 +147,41 @@ describe('relay database', () => {
await second.close()
})
it('renders every region check from the shared region list', async () => {
// Derived, not hand-written: a third region must not leave one column
// rejecting a value the rest of the relay already accepts.
const database = await openInMemoryRelayDatabase()
const checked = await database.query(
`SELECT name, sql FROM sqlite_master
WHERE type = 'table'
AND name IN ('relay_assignment_region_preferences', 'relay_cell_regions',
'relay_region_rehome_attempts')
ORDER BY name`
)
const list = `IN ('us-central1', 'asia-east2')`
expect(checked.map((row) => row.name)).toEqual([
'relay_assignment_region_preferences',
'relay_cell_regions',
'relay_region_rehome_attempts'
])
expect(checked.every((row) => String(row.sql).includes(list))).toBe(true)
expect(
POSTGRES_SCHEMA_MIGRATIONS.some((statement) => statement.includes(list))
).toBe(true)
await database.close()
})
it('indexes rehome attempts by host recency for the per-host cooldown', async () => {
const database = await openInMemoryRelayDatabase()
const rows = await database.query(
`SELECT sql FROM sqlite_master
WHERE type = 'index' AND name = 'relay_region_rehome_attempts_host_recency'`
)
expect(rows[0]?.sql).toContain('(user_id, relay_host_id, created_at)')
expect(REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS).toBe(7 * 24 * 60 * 60_000)
await database.close()
})
it('indexes region preference expiry by observation time', async () => {
const database = await openInMemoryRelayDatabase()
const rows = await database.query(
+37 -4
View File
@@ -3,6 +3,7 @@ import { performance } from 'node:perf_hooks'
import { join } from 'node:path'
import { DatabaseSync } from 'node:sqlite'
import pg from 'pg'
import { RELAY_REGIONS } from '@orca-cloud/relay-contract'
import {
emptyPostgresPoolPressureCounts,
PostgresPoolPressure,
@@ -24,6 +25,14 @@ function setLocalLockTimeout(milliseconds: number): string {
return `SET LOCAL lock_timeout = '${milliseconds}ms'`
}
// Region CHECK lists come from the contract so a new region cannot leave a
// column rejecting values the rest of the relay already accepts.
const REGION_LIST = RELAY_REGIONS.map((region) => `'${region}'`).join(', ')
// A host that was just moved is not a candidate again for this long, so a
// desktop whose region probe flips cannot walk itself back and forth.
export const REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS = 7 * 24 * 60 * 60_000
export type SqlRow = Record<string, unknown>
export type RelayLockOptions = {
failIfUnavailable?: boolean
@@ -181,7 +190,7 @@ CREATE TABLE IF NOT EXISTS relay_assignment_region_preferences (
user_id TEXT NOT NULL,
relay_host_id TEXT NOT NULL,
preferred_region TEXT NOT NULL
CHECK (preferred_region IN ('us-central1', 'asia-east2')),
CHECK (preferred_region IN (${REGION_LIST})),
observed_at BIGINT NOT NULL,
PRIMARY KEY (user_id, relay_host_id)
);
@@ -204,6 +213,8 @@ CREATE TABLE IF NOT EXISTS relay_region_rehome_control (
not_before BIGINT NOT NULL,
rate_per_minute BIGINT NOT NULL,
preference_max_age_ms BIGINT NOT NULL,
host_cooldown_ms BIGINT NOT NULL
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS},
drain_grace_ms BIGINT NOT NULL,
updated_at BIGINT NOT NULL
);
@@ -212,7 +223,9 @@ CREATE TABLE IF NOT EXISTS relay_region_rehome_attempts (
attempt_id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
relay_host_id TEXT NOT NULL,
preferred_region TEXT NOT NULL CHECK (preferred_region = 'asia-east2'),
preferred_region TEXT NOT NULL
CONSTRAINT relay_region_rehome_attempts_preferred_region_valid
CHECK (preferred_region IN (${REGION_LIST})),
source_cell_id TEXT NOT NULL,
source_cell_incarnation TEXT NOT NULL,
target_cell_id TEXT NOT NULL,
@@ -234,6 +247,8 @@ CREATE TABLE IF NOT EXISTS relay_region_rehome_attempts (
);
CREATE INDEX IF NOT EXISTS relay_region_rehome_attempts_pending
ON relay_region_rehome_attempts(drain_receipt_at, last_send_attempt_at, completed_at, aborted_at);
CREATE INDEX IF NOT EXISTS relay_region_rehome_attempts_host_recency
ON relay_region_rehome_attempts(user_id, relay_host_id, created_at);
CREATE TABLE IF NOT EXISTS relay_cells (
cell_id TEXT PRIMARY KEY,
@@ -248,7 +263,7 @@ CREATE TABLE IF NOT EXISTS relay_cells (
CREATE TABLE IF NOT EXISTS relay_cell_regions (
cell_id TEXT PRIMARY KEY,
region TEXT NOT NULL CHECK (region IN ('us-central1', 'asia-east2'))
region TEXT NOT NULL CHECK (region IN (${REGION_LIST}))
);
CREATE TABLE IF NOT EXISTS relay_cell_admission (
@@ -580,6 +595,21 @@ CREATE TABLE IF NOT EXISTS relay_audit_events (
CREATE INDEX IF NOT EXISTS relay_audit_events_at ON relay_audit_events(at);
`
// Rehoming is bidirectional, but tables created before that carry the
// original single-region column check. The old constraint is the one Postgres
// auto-named; the replacement is named, so both statements are no-ops on a
// database the current schema created and neither can drop the other.
export const POSTGRES_SCHEMA_MIGRATIONS = [
`ALTER TABLE relay_region_rehome_attempts
DROP CONSTRAINT IF EXISTS relay_region_rehome_attempts_preferred_region_check`,
`ALTER TABLE relay_region_rehome_attempts
ADD CONSTRAINT relay_region_rehome_attempts_preferred_region_valid
CHECK (preferred_region IN (${REGION_LIST}))`,
`ALTER TABLE relay_region_rehome_control
ADD COLUMN IF NOT EXISTS host_cooldown_ms BIGINT NOT NULL
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS}`
]
function postgresSql(sql: string): string {
let index = 0
return sql.replace(/\?/g, () => `$${++index}`)
@@ -1009,7 +1039,10 @@ async function applySchemaOnUntimedPool(
const database = new PostgresDatabase(pool)
try {
await applyPostgresSchema(
SCHEMA.split(';').filter((statement) => statement.trim()),
[
...SCHEMA.split(';').filter((statement) => statement.trim()),
...POSTGRES_SCHEMA_MIGRATIONS
],
async (statement) => await database.query(statement)
)
} finally {
@@ -1,5 +1,5 @@
import { EventEmitter } from 'node:events'
import { RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract'
import { RELAY_CLOSE_CODE, RELAY_PROTOCOL_LIMITS } from '@orca-cloud/relay-contract'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type WebSocket from 'ws'
import type { RelayAssignmentStore } from './assignment-store.js'
@@ -102,7 +102,9 @@ function harness(options: { random?: () => number; now?: () => number } = {}) {
const store = {
resolveResume: vi.fn().mockResolvedValue({ userId: identity.sub }),
reserveCredential: vi.fn().mockResolvedValue(reservation),
failReservation: vi.fn().mockResolvedValue(undefined)
failReservation: vi.fn().mockResolvedValue(undefined),
recordConnectionBasis: vi.fn().mockResolvedValue(undefined),
deactivateBasis: vi.fn().mockResolvedValue(undefined)
}
const observer = {
recordAuth: vi.fn(),
@@ -110,7 +112,9 @@ function harness(options: { random?: () => number; now?: () => number } = {}) {
recordHttp: vi.fn(),
recordReconnect: vi.fn(),
recordSql: vi.fn(),
recordClientAcceptAbandoned: vi.fn()
recordClientAcceptAbandoned: vi.fn(),
recordClientAcceptCompleted: vi.fn(),
recordControlRtt: vi.fn()
} satisfies RelayRuntimeObserver
const registry = new HostSessionRegistry(
config,
@@ -325,6 +329,210 @@ describe('client accept abandoned mid-DB-phase', () => {
})
})
describe('successful client accept timing', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
it('times every serialized stage plus the attach window once relay-hello lands', async () => {
let now = 1_700_000_000_000
const h = harness({ now: () => now })
const control = await activeHost(h)
h.store.resolveResume.mockImplementationOnce(async () => {
now += 5
return { userId: identity.sub }
})
h.store.reserveCredential.mockImplementationOnce(async () => {
now += 7
return reservation
})
h.acquireActivity.mockImplementationOnce(async () => {
now += 11
})
h.store.recordConnectionBasis.mockImplementationOnce(async () => {
now += 3
})
const client = new FakeSocket()
const hostData = new FakeSocket()
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
try {
await h.registry.acceptClient(client as unknown as WebSocket, identity.relayHostId, 'cred')
const connOpen = JSON.parse(
String(control.send.mock.calls.find((call) => String(call[0]).includes('conn-open'))![0])
) as { connId: string; connTicket: string }
// The desktop's data leg is the attach window this is meant to expose.
now += 23
const accepted = await h.registry.acceptHostData(
hostData as unknown as WebSocket,
connOpen.connId,
connOpen.connTicket,
1
)
expect(accepted).toBe(true)
expect(h.observer.recordClientAcceptCompleted).toHaveBeenCalledWith({
totalMs: 49,
stageMs: { assignment: 5, credential: 7, activity: 11, attach: 23, basis: 3 }
})
const line = log.mock.calls
.map((call) => String(call[0]))
.find((entry) => entry.includes('orca_relay_client_accept_completed'))
expect(line).toBeDefined()
const event = JSON.parse(line!) as {
role: string
cellId: string
region: string
credentialKind: string
stageMs: Record<string, number>
totalMs: number
relayHostIdDigest: string
}
expect(event.credentialKind).toBe('resume')
// Joins the line back to the emitting process, like the runtime metrics event.
expect(event).toMatchObject({ role: 'cell', cellId: config.cellId, region: 'us-central1' })
expect(Object.keys(event.stageMs).sort()).toEqual([
'activity',
'assignment',
'attach',
'basis',
'credential'
])
for (const stage of Object.values(event.stageMs)) expect(stage).toBeGreaterThanOrEqual(0)
// The stages tile the accept end to end: every millisecond is attributed.
const summed = Object.values(event.stageMs).reduce((total, stage) => total + stage, 0)
expect(summed).toBe(event.totalMs)
expect(event.relayHostIdDigest).toMatch(/^[0-9a-f]{12}$/)
expect(line).not.toContain(identity.relayHostId)
} finally {
log.mockRestore()
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})
})
// Fires one heartbeat and returns the `t` of the ping it sent, which is the only
// echo the registry will time.
async function advanceToPing(control: FakeSocket, clock: { now: number }): Promise<number> {
clock.now += RELAY_PROTOCOL_LIMITS.controlPingIntervalMs
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
const ping = control.send.mock.calls
.filter((call) => String(call[0]).includes('"type":"ping"'))
.at(-1)!
return (JSON.parse(String(ping[0])) as { t: number }).t
}
describe('control round-trip sampling', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
it('logs a host once at the fourth sample and not again within the hour', async () => {
const clock = { now: 1_700_000_000_000 }
const h = harness({ now: () => clock.now })
const control = await activeHost(h)
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
const rttLines = (): string[] =>
log.mock.calls
.map((call) => String(call[0]))
.filter((entry) => entry.includes('orca_relay_host_control_rtt'))
// One heartbeat, then the desktop's echo of that ping's own `t` 40 ms later.
const roundTrip = async (): Promise<void> => {
const pingAt = await advanceToPing(control, clock)
clock.now += 40
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt }), false)
}
try {
for (let round = 0; round < 3; round++) await roundTrip()
expect(h.observer.recordControlRtt).toHaveBeenCalledTimes(3)
expect(rttLines()).toHaveLength(0)
await roundTrip()
expect(h.observer.recordControlRtt).toHaveBeenLastCalledWith(40)
expect(rttLines()).toHaveLength(1)
expect(JSON.parse(rttLines()[0]!)).toMatchObject({
event: 'orca_relay_host_control_rtt',
role: 'cell',
cellId: config.cellId,
region: 'us-central1',
rttMsMedian: 40,
sampleCount: 4
})
expect(rttLines()[0]).not.toContain(identity.relayHostId)
// Later samples keep feeding the fleet metric, but stay silent for an hour.
for (let round = 0; round < 8; round++) await roundTrip()
expect(h.observer.recordControlRtt).toHaveBeenCalledTimes(12)
expect(rttLines()).toHaveLength(1)
const elapsedStart = clock.now
while (clock.now - elapsedStart < 60 * 60 * 1000) await roundTrip()
expect(rttLines()).toHaveLength(2)
} finally {
log.mockRestore()
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})
it('ignores a pong that answers no outstanding ping', async () => {
const clock = { now: 1_700_000_000_000 }
const h = harness({ now: () => clock.now })
const control = await activeHost(h)
try {
// Nothing has been pinged yet, so even a plausible echo is not a round trip.
control.emit('message', JSON.stringify({ type: 'pong' }), false)
control.emit('message', JSON.stringify({ type: 'pong', t: 'later' }), false)
control.emit('message', JSON.stringify({ type: 'pong', t: clock.now }), false)
control.emit('message', JSON.stringify({ type: 'pong', t: clock.now - 10 }), false)
expect(h.observer.recordControlRtt).not.toHaveBeenCalled()
const pingAt = await advanceToPing(control, clock)
// A guessed timestamp is not the outstanding ping's `t`, so it is dropped.
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt - 1 }), false)
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt + 1 }), false)
expect(h.observer.recordControlRtt).not.toHaveBeenCalled()
clock.now += 10
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt }), false)
expect(h.observer.recordControlRtt).toHaveBeenCalledWith(10)
} finally {
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})
it('records one sample per ping however many pongs a host floods', async () => {
const clock = { now: 1_700_000_000_000 }
const h = harness({ now: () => clock.now })
const control = await activeHost(h)
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
try {
const pingAt = await advanceToPing(control, clock)
clock.now += 12
for (let flood = 0; flood < 5_000; flood++) {
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt }), false)
control.emit('message', JSON.stringify({ type: 'pong', t: clock.now }), false)
}
// One answered ping is one process-wide sample and one per-session sample, so
// neither the metric window nor the hourly log line can be flooded.
expect(h.observer.recordControlRtt).toHaveBeenCalledTimes(1)
expect(h.observer.recordControlRtt).toHaveBeenCalledWith(12)
expect(
log.mock.calls.filter((call) => String(call[0]).includes('orca_relay_host_control_rtt'))
).toHaveLength(0)
} finally {
log.mockRestore()
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})
})
describe('control lease jitter', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
@@ -3,6 +3,7 @@ import {
ASSIGNMENT_LIMITS,
CONTROL_CONTINUITY_LIMITS,
RELAY_CLOSE_CODE,
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS,
RELAY_PROTOCOL_LIMITS
} from '@orca-cloud/relay-contract'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
@@ -1005,3 +1006,115 @@ describe('control lease recovery after the session is gone', () => {
}
})
})
describe('host hello ack pending connections', () => {
const DETAILS = new Set([RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS])
const LEGACY_ENTRY = { connId: 'conn-1', connTicket: 'T'.repeat(43) }
const DETAILED_ENTRY = { ...LEGACY_ENTRY, kind: 'invite', relayDeviceId: 'device-1' }
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})
function newRegistry(): ReturnType<typeof createRegistry> {
return createRegistry(
vi
.fn<RelayAssignmentStore['activateControl']>()
.mockResolvedValue('control:production-gce-c3:1')
)
}
function addPendingConnection(session: HostSession): void {
session.pendingConns.set('conn-1', {
...LEGACY_ENTRY,
reservation: {
userId: identity.sub,
relayHostId: identity.relayHostId,
credentialKind: 'invite',
relayDeviceId: 'device-1'
},
client: new FakeSocket() as unknown as WebSocket,
attachTimer: setTimeout(() => {}, 60_000),
credentialActivityId: null
} as unknown as Parameters<typeof session.pendingConns.set>[1])
}
function sentAck(socket: FakeSocket): Record<string, unknown> {
const acks = socket.send.mock.calls
.map((call) => JSON.parse(String(call[0])) as Record<string, unknown>)
.filter((message) => message.type === 'host-hello-ack')
return acks.at(-1)!
}
function sessionOf(registry: HostSessionRegistry): HostSession {
return registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
}
async function ackFor(capabilities?: ReadonlySet<string>): Promise<Record<string, unknown>> {
const { registry, activate } = newRegistry()
const socket = new FakeSocket()
registry.acceptControl(
socket as unknown as WebSocket,
identity,
undefined,
capabilities ?? new Set()
)
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
const session = sessionOf(registry)
addPendingConnection(session)
socket.send.mockClear()
;(registry as unknown as { sendHelloAck(session: HostSession): void }).sendHelloAck(session)
return sentAck(socket)
}
async function ackAfterRebind(
first: ReadonlySet<string>,
successor: ReadonlySet<string>
): Promise<{ opening: Record<string, unknown>; rebound: Record<string, unknown> }> {
const { registry, activate } = newRegistry()
const opening = new FakeSocket()
registry.acceptControl(opening as unknown as WebSocket, identity, undefined, first)
await activate(opening as unknown as WebSocket, identity, null, 1, false, 1)
const session = sessionOf(registry)
addPendingConnection(session)
opening.send.mockClear()
;(registry as unknown as { sendHelloAck(session: HostSession): void }).sendHelloAck(session)
const rebound = new FakeSocket()
registry.acceptControl(rebound as unknown as WebSocket, identity, undefined, successor)
await activate(rebound as unknown as WebSocket, identity, session, 1, true, 1)
return { opening: sentAck(opening), rebound: sentAck(rebound) }
}
it('states the pending kind and device to a host that advertised it can read them', async () => {
const ack = await ackFor(DETAILS)
expect(ack.pendingConns).toEqual([DETAILED_ENTRY])
})
it('restates only the identifiers to a host that never advertised the capability', async () => {
// A shipped host parses these entries strictly, so an unannounced key fails
// the whole ack parse and kills a control that was working.
const ack = await ackFor()
expect(ack.pendingConns).toEqual([LEGACY_ENTRY])
})
it('downgrades the restated entry when the successor control drops the capability', async () => {
// The capability belongs to the socket, not the session: a rebind can land a
// control whose decoder is older than the one that opened the session.
const { opening, rebound } = await ackAfterRebind(DETAILS, new Set())
expect(opening.pendingConns).toEqual([DETAILED_ENTRY])
expect(rebound.pendingConns).toEqual([LEGACY_ENTRY])
})
it('upgrades the restated entry when the successor control adds the capability', async () => {
const { opening, rebound } = await ackAfterRebind(new Set(), DETAILS)
expect(opening.pendingConns).toEqual([LEGACY_ENTRY])
expect(rebound.pendingConns).toEqual([DETAILED_ENTRY])
})
})
+156 -5
View File
@@ -1,6 +1,7 @@
import { createHash, createHmac, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto'
import {
ASSIGNMENT_LIMITS,
RELAY_DEFAULT_REGION,
AuthRefreshSchema,
buildHostChallengePlaintext,
buildHostProofMacInput,
@@ -13,9 +14,11 @@ import {
HostChallengeAckSchema,
HostHelloSchema,
InviteCreateSchema,
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS,
RELAY_PROTOCOL_LIMITS,
RELAY_CLOSE_CODE,
type RelayHostCloseReason
type RelayHostCloseReason,
type RelayRegion
} from '@orca-cloud/relay-contract'
import nacl from 'tweetnacl'
import type WebSocket from 'ws'
@@ -29,7 +32,12 @@ import {
import { HostCloseReasonMemory } from './host-close-reason-memory.js'
import { relayHostLogDigest } from './relay-host-log-digest.js'
import type { RelayTokenClaims } from './relay-token-verifier.js'
import type { RelayClientAcceptStage, RelayRuntimeObserver } from './relay-observability.js'
import {
percentile,
type RelayClientAcceptStage,
type RelayClientAcceptTimedStage,
type RelayRuntimeObserver
} from './relay-observability.js'
import type { PendingHostDataReservation } from './relay-connection-ledger.js'
import { closeRelayWebSocket } from './relay-websocket-close.js'
import { ProcessQueuedByteBudget, wireSplice } from './splice-forwarder.js'
@@ -45,6 +53,20 @@ function printableCloseReason(reason: Buffer | string): string {
type VerifyRelayToken = (token: string) => Promise<RelayTokenClaims | null>
type HostState = 'proving' | 'active' | 'orphaned' | 'drain-only' | 'closed'
// A host's distance to its cell moves on the scale of a rehome, not a heartbeat,
// so a short window is enough to ride out one stalled ping.
const CONTROL_RTT_WINDOW = 8
const CONTROL_RTT_LOG_SAMPLE_THRESHOLD = 4
const CONTROL_RTT_LOG_INTERVAL_MS = 60 * 60 * 1000
// A pong claiming a multi-minute round trip is clock skew, not distance.
const CONTROL_RTT_MAX_PLAUSIBLE_MS = 120_000
// Wall clock can step backwards mid-accept; a negative latency would poison the
// percentiles it feeds.
function nonNegativeMs(elapsedMs: number): number {
return Math.max(0, elapsedMs)
}
const CONTROL_ACTIVITY_RENEWAL_INTERVAL_MS = RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2
// Preserve the existing 75s renewal runway after doubling the successful-call interval.
const CONTROL_ACTIVITY_LEASE_MS =
@@ -68,6 +90,10 @@ export type HostSession = {
orphanTimer: ReturnType<typeof setTimeout> | null
heartbeatTimer: ReturnType<typeof setInterval> | null
lastPongAt: number
// The `t` of the ping still waiting for its echo; null once one has answered it.
pendingPingAt: number | null
controlRttSamplesMs: number[]
controlRttLoggedAt: number | null
activityRenewalDueAt: number
activityRenewalAttempt: number
activityRenewalCompletedAttempt: number
@@ -95,6 +121,15 @@ type PendingConnection = {
attachTimer: ReturnType<typeof setTimeout>
credentialActivityId: string | null
capacityReservation?: PendingHostDataReservation
timing: ClientAcceptTiming
}
// Carries the phone-side accept clock across to the desktop's data leg, which
// lands in a separate call and is the only place the accept is known to succeed.
type ClientAcceptTiming = {
startedAt: number
connOpenAt: number
stageMs: Record<RelayClientAcceptStage, number>
}
function decodeCanonicalBase64(value: string, bytes: number): Uint8Array | null {
@@ -146,6 +181,7 @@ export class HostSessionRegistry {
// but a signed-out desktop never comes back, so the phone that asks minutes
// later would otherwise find nothing to explain its rejection with.
private readonly hostCloseReasons = new HostCloseReasonMemory(() => this.now())
private readonly hostCapabilities = new WeakMap<WebSocket, ReadonlySet<string>>()
private draining = false
constructor(
@@ -192,6 +228,17 @@ export class HostSessionRegistry {
)
return true
}
const stageMs: Record<RelayClientAcceptStage, number> = {
assignment: 0,
credential: 0,
activity: 0
}
let stageCursor = acceptStartedAt
const markStage = (stage: RelayClientAcceptStage): void => {
const at = this.now()
stageMs[stage] = at - stageCursor
stageCursor = at
}
if (this.config.role === 'cell') {
// Each lookup is its own pooled round trip; stop between them once the phone
// has left instead of running the rest of the chain for nobody.
@@ -212,6 +259,7 @@ export class HostSessionRegistry {
}
if (abandonedByClient('assignment')) return
}
markStage('assignment')
const reservation = await this.store.reserveCredential(hostId, credential)
if (!reservation) {
capacityReservation?.release()
@@ -221,6 +269,7 @@ export class HostSessionRegistry {
}
this.observer.recordAuth(true)
if (abandonedByClient('credential', () => this.failReservationBestEffort(reservation))) return
markStage('credential')
const sessionKey = this.key(reservation.userId, hostId)
const session = this.sessions.get(sessionKey)
if (
@@ -275,6 +324,7 @@ export class HostSessionRegistry {
) {
return
}
markStage('activity')
const attachTimer = setTimeout(() => {
session.pendingConns.delete(connId)
capacityReservation?.release()
@@ -289,7 +339,10 @@ export class HostSessionRegistry {
client: socket,
attachTimer,
credentialActivityId,
capacityReservation
capacityReservation,
// Attach starts where the activity stage ended, so the conn-open send is
// charged to it and no wall-clock gap goes unattributed.
timing: { startedAt: acceptStartedAt, connOpenAt: stageCursor, stageMs }
}
capacityReservation?.bind(connId)
session.pendingConns.set(connId, pending)
@@ -336,6 +389,7 @@ export class HostSessionRegistry {
return false
}
this.observer.recordAuth(true)
const attachedAt = this.now()
clearTimeout(pending.attachTimer)
session.pendingConns.delete(connId)
session.activeConnIds.add(connId)
@@ -409,6 +463,7 @@ export class HostSessionRegistry {
close()
return false
}
const helloAt = this.now()
send(pending.client, 'relay-hello', {
ok: true,
credentialKind: pending.reservation.credentialKind,
@@ -424,14 +479,92 @@ export class HostSessionRegistry {
}
: {})
})
this.recordClientAcceptCompleted(session, pending, attachedAt, helloAt)
return true
}
// The stages tile the whole accept, so their sum is the total minus only the
// clamping above: `basis` is the splice lease and connection-basis writes that
// land between the host data leg and relay-hello.
private recordClientAcceptCompleted(
session: HostSession,
pending: PendingConnection,
attachedAt: number,
helloAt: number
): void {
const stageMs: Record<RelayClientAcceptTimedStage, number> = {
assignment: nonNegativeMs(pending.timing.stageMs.assignment),
credential: nonNegativeMs(pending.timing.stageMs.credential),
activity: nonNegativeMs(pending.timing.stageMs.activity),
attach: nonNegativeMs(attachedAt - pending.timing.connOpenAt),
basis: nonNegativeMs(helloAt - attachedAt)
}
const totalMs = nonNegativeMs(helloAt - pending.timing.startedAt)
this.observer.recordClientAcceptCompleted?.({ totalMs, stageMs })
console.log(
JSON.stringify({
event: 'orca_relay_client_accept_completed',
...this.logIdentity(),
credentialKind: pending.reservation.credentialKind,
stageMs,
totalMs,
relayHostIdDigest: relayHostLogDigest(session.relayHostId)
})
)
}
// Matches the runtime metrics event so a log line and a metric point can be
// joined back to the process that emitted them.
private logIdentity(): { role: string; cellId: string; region: RelayRegion } {
return {
role: this.config.role,
cellId: this.config.cellId,
region: this.config.region ?? RELAY_DEFAULT_REGION
}
}
// Every desktop build already echoes the ping's `t`, so a pong is only timed when
// it answers the outstanding ping: at most one sample per ping this cell sent,
// however many a host floods. A pong that lost the race to the next ping is
// dropped here but still counts as proof of life for the silence watchdog.
private recordControlRtt(session: HostSession, echoedPingAt: unknown): void {
if (typeof echoedPingAt !== 'number' || echoedPingAt !== session.pendingPingAt) return
session.pendingPingAt = null
const now = this.now()
const rttMs = now - echoedPingAt
if (rttMs < 0 || rttMs > CONTROL_RTT_MAX_PLAUSIBLE_MS) return
this.observer.recordControlRtt?.(rttMs)
const samples = session.controlRttSamplesMs
samples.push(rttMs)
if (samples.length > CONTROL_RTT_WINDOW) samples.shift()
if (samples.length < CONTROL_RTT_LOG_SAMPLE_THRESHOLD) return
if (
session.controlRttLoggedAt !== null &&
now - session.controlRttLoggedAt < CONTROL_RTT_LOG_INTERVAL_MS
) {
return
}
session.controlRttLoggedAt = now
console.log(
JSON.stringify({
event: 'orca_relay_host_control_rtt',
...this.logIdentity(),
relayHostIdDigest: relayHostLogDigest(session.relayHostId),
rttMsMedian: percentile(samples, 0.5),
sampleCount: samples.length
})
)
}
acceptControl(
socket: WebSocket,
identity: RelayTokenClaims,
connectionInclusionWatermark?: number
connectionInclusionWatermark?: number,
hostCapabilities?: ReadonlySet<string>
): void {
// Keyed by socket, not session: a rebind swaps the session's socket, and the
// successor's own advertisement is the only one that describes its decoder.
if (hostCapabilities?.size) this.hostCapabilities.set(socket, hostCapabilities)
if (this.draining) {
socket.close(RELAY_CLOSE_CODE.DRAINING, 'relay draining')
return
@@ -790,6 +923,7 @@ export class HostSessionRegistry {
existing.appVersion = appVersion
existing.leaseExpiresAt = this.controlLeaseExpiresAt()
existing.lastPongAt = this.now()
existing.pendingPingAt = null
existing.activityRenewalDueAt =
this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs
this.wireActiveControl(existing)
@@ -843,6 +977,9 @@ export class HostSessionRegistry {
orphanTimer: null,
heartbeatTimer: null,
lastPongAt: this.now(),
pendingPingAt: null,
controlRttSamplesMs: [],
controlRttLoggedAt: null,
activityRenewalDueAt: this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs,
activityRenewalAttempt: 0,
activityRenewalCompletedAttempt: 0,
@@ -900,6 +1037,7 @@ export class HostSessionRegistry {
const parsed = JSON.parse(raw.toString()) as Record<string, unknown>
if (parsed.type === 'pong') {
session.lastPongAt = this.now()
this.recordControlRtt(session, parsed.t)
return
}
if (parsed.type === 'auth-refresh') {
@@ -1047,11 +1185,18 @@ export class HostSessionRegistry {
session.socket.close(RELAY_CLOSE_CODE.DRAINING, 'control lease expired')
return
}
session.pendingPingAt = now
send(session.socket, 'ping', { t: now })
}
private sendHelloAck(session: HostSession): void {
if (!session.socket) return
// Without these a host that missed the conn-open cannot dial the pending
// connection: it would have to guess the pairing kind and the device the
// relay authorized. Only sent to a host that said it can read them.
const details = this.hostCapabilities
.get(session.socket)
?.has(RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS)
send(session.socket, 'host-hello-ack', {
v: 1,
generation: session.generation,
@@ -1060,7 +1205,13 @@ export class HostSessionRegistry {
activeConnIds: [...session.activeConnIds],
pendingConns: [...session.pendingConns.values()].map((pending) => ({
connId: pending.connId,
connTicket: pending.connTicket
connTicket: pending.connTicket,
...(details
? {
kind: pending.reservation.credentialKind,
relayDeviceId: pending.reservation.relayDeviceId
}
: {})
}))
})
}
@@ -49,6 +49,20 @@ function concurrentCreateCollision(
return false
}
const ALTER_TABLE_ADD_CONSTRAINT =
/^\s*ALTER\s+TABLE\s+\S+\s+ADD\s+CONSTRAINT\b/i
// Postgres has no `ADD CONSTRAINT IF NOT EXISTS`, so a re-run and a concurrent
// startup both land on 42710 once the constraint exists. Unlike a CREATE race
// this is terminal, not transient: retrying only repeats it, so the statement
// counts as applied.
function constraintAlreadyApplied(error: unknown, statement: string): boolean {
return (
ALTER_TABLE_ADD_CONSTRAINT.test(statement) &&
(error as { code?: unknown }).code === '42710'
)
}
function retryableSchemaError(error: unknown, statement: string): boolean {
const value = error as { code?: unknown; constraint?: unknown }
return (
@@ -73,6 +87,7 @@ export async function applyPostgresSchema(
await query(statement)
break
} catch (error) {
if (constraintAlreadyApplied(error, statement)) break
const code = String((error as { code?: unknown }).code)
const remainingMs = deadlineAt - now()
const retryable = retryableSchemaError(error, statement)
@@ -315,6 +315,7 @@ describe('regional rehome director controls', () => {
notBefore: 100,
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: 7 * 24 * 60 * 60_000,
drainGraceMs: 60_000,
confirmation: 'ENABLE_REGIONAL_REHOMING'
}
@@ -343,6 +344,14 @@ describe('regional rehome director controls', () => {
'deploy-token',
{ ...apply, confirmation: 'DISABLE_REGIONAL_REHOMING' }
)).status).toBe(400)
// The per-host cooldown is part of the durable shape an operator must state.
const { hostCooldownMs: _omitted, ...withoutCooldown } = apply
expect((await postPath(
app,
'/v1/admin/regional-rehome-control',
'deploy-token',
withoutCooldown
)).status).toBe(400)
})
it('probes dedicated trust twice and returns only aggregate proof', async () => {
@@ -411,6 +420,78 @@ describe('regional rehome director controls', () => {
expect(JSON.stringify(responseBody)).not.toContain('rehome-token')
})
it('probes a source cell in any region, not only the default one', async () => {
// Rehoming moves hosts in both directions, so an asia-east2 cell is a
// source too and its trust has to be provable the same way.
const cellDeploymentStatus = vi.fn().mockResolvedValue({
cellId: 'production-gce-c27',
cellUrl: 'https://c27.relay.example.test',
region: 'asia-east2',
runtime: {
cellIncarnation,
ready: true,
heartbeatFresh: true,
regionalRehomeProtocol: 1
}
})
const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
store: {} as never,
assignments: { cellDeploymentStatus } as never,
drain: vi.fn(),
regionalRehomeIdentityToken: vi.fn(async () => 'rehome-token'),
regionalRehomeFetch: (async () =>
Response.json({
v: 1,
outcome: 'host-not-connected',
sharedRuntimeIdentityRejected: true
})) as typeof fetch,
ready: vi.fn(async () => true)
})
const response = await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ v: 1, sourceCellId: 'production-gce-c27', sourceCellIncarnation: cellIncarnation }
)
expect(response.status).toBe(200)
expect(await response.json()).toMatchObject({ proven: true })
})
it('still refuses a trust probe against a cell without the drain protocol', async () => {
const cellDeploymentStatus = vi.fn().mockResolvedValue({
cellId: 'production-gce-c27',
cellUrl: 'https://c27.relay.example.test',
region: 'asia-east2',
runtime: {
cellIncarnation,
ready: true,
heartbeatFresh: true,
regionalRehomeProtocol: 0
}
})
const sourceFetch = vi.fn<typeof fetch>()
const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
store: {} as never,
assignments: { cellDeploymentStatus } as never,
drain: vi.fn(),
regionalRehomeIdentityToken: vi.fn(async () => 'rehome-token'),
regionalRehomeFetch: sourceFetch,
ready: vi.fn(async () => true)
})
const response = await postPath(
app,
'/v1/admin/regional-rehome-trust-probe',
'deploy-token',
{ v: 1, sourceCellId: 'production-gce-c27', sourceCellIncarnation: cellIncarnation }
)
expect(response.status).toBe(409)
expect(sourceFetch).not.toHaveBeenCalled()
})
it('restricts trust probes to deploy authorization and strict input', async () => {
const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
store: {} as never,
@@ -0,0 +1,195 @@
import pg from 'pg'
import { afterAll, beforeEach, describe, expect, it } from 'vitest'
import {
openRelayDatabase,
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS,
type RelayDatabase
} from './database.js'
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
const schema = 'relay_rehome_constraint_migration_test'
// The shape shipped before rehoming became bidirectional: a single-region
// column check that Postgres auto-names.
const LEGACY_ATTEMPTS_TABLE = `
CREATE TABLE relay_region_rehome_attempts (
attempt_id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
relay_host_id TEXT NOT NULL,
preferred_region TEXT NOT NULL CHECK (preferred_region = 'asia-east2'),
source_cell_id TEXT NOT NULL,
source_cell_incarnation TEXT NOT NULL,
target_cell_id TEXT NOT NULL,
target_cell_incarnation TEXT NOT NULL,
previous_epoch BIGINT NOT NULL,
assignment_epoch BIGINT NOT NULL,
drain_grace_ms BIGINT NOT NULL,
send_attempts BIGINT NOT NULL,
last_send_attempt_at BIGINT,
drain_receipt_at BIGINT,
drain_outcome TEXT CHECK (
drain_outcome IN ('accepted', 'already-accepted', 'host-not-connected')
),
completed_at BIGINT,
aborted_at BIGINT,
created_at BIGINT NOT NULL,
updated_at BIGINT NOT NULL,
UNIQUE (user_id, relay_host_id, assignment_epoch)
)`
// The control row as it shipped before the per-host cooldown existed.
const LEGACY_CONTROL_TABLE = `
CREATE TABLE relay_region_rehome_control (
control_id TEXT PRIMARY KEY,
generation BIGINT NOT NULL,
enabled BIGINT NOT NULL,
observation_started_at BIGINT NOT NULL,
not_before BIGINT NOT NULL,
rate_per_minute BIGINT NOT NULL,
preference_max_age_ms BIGINT NOT NULL,
drain_grace_ms BIGINT NOT NULL,
updated_at BIGINT NOT NULL
)`
const attemptValues = (attemptId: string, preferredRegion: string): unknown[] => [
attemptId,
'user-1',
'abcdefghijklmnop',
preferredRegion,
'cell-source',
'11111111-1111-4111-8111-111111111111',
'cell-target',
'22222222-2222-4222-8222-222222222222',
1,
Number(attemptId.at(-1)),
0,
0,
1_000_000,
1_000_000
]
const INSERT_ATTEMPT = `INSERT INTO relay_region_rehome_attempts
(attempt_id, user_id, relay_host_id, preferred_region, source_cell_id,
source_cell_incarnation, target_cell_id, target_cell_incarnation,
previous_epoch, assignment_epoch, drain_grace_ms, send_attempts,
created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`
describePostgres('PostgreSQL regional rehome constraint migration', () => {
let scopedUrl = ''
async function withClient(
operation: (client: pg.Client) => Promise<void>
): Promise<void> {
const client = new pg.Client({ connectionString: databaseUrl })
await client.connect()
try {
await operation(client)
} finally {
await client.end()
}
}
beforeEach(async () => {
await withClient(async (client) => {
await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
await client.query(`CREATE SCHEMA ${schema}`)
await client.query(`SET search_path = ${schema}`)
await client.query(LEGACY_ATTEMPTS_TABLE)
await client.query(LEGACY_CONTROL_TABLE)
await client.query(
`INSERT INTO relay_region_rehome_control
(control_id, generation, enabled, observation_started_at, not_before,
rate_per_minute, preference_max_age_ms, drain_grace_ms, updated_at)
VALUES ('global', 3, 0, 1, 0, 10, 86400000, 60000, 1)`
)
// Production data the replacement constraint has to validate.
await client.query(INSERT_ATTEMPT, attemptValues('attempt-1', 'asia-east2'))
})
const url = new URL(databaseUrl!)
url.searchParams.set('options', `-c search_path=${schema}`)
scopedUrl = url.toString()
})
afterAll(async () => {
await withClient(async (client) => {
await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
})
})
it('upgrades a legacy single-region constraint in place', async () => {
const database = await openRelayDatabase({ databaseUrl: scopedUrl, dataDir: '' })
try {
await withClient(async (client) => {
await client.query(`SET search_path = ${schema}`)
await client.query(INSERT_ATTEMPT, attemptValues('attempt-2', 'us-central1'))
await expect(
client.query(INSERT_ATTEMPT, attemptValues('attempt-3', 'europe-west1'))
).rejects.toMatchObject({ code: '23514' })
const constraints = await client.query(
`SELECT conname FROM pg_constraint
WHERE conrelid = 'relay_region_rehome_attempts'::regclass
AND conname LIKE '%preferred_region%'
ORDER BY conname`
)
expect(constraints.rows).toEqual([
{ conname: 'relay_region_rehome_attempts_preferred_region_valid' }
])
// The existing control row keeps its tuning and gains the cooldown.
const control = await client.query(
`SELECT generation, preference_max_age_ms, host_cooldown_ms
FROM relay_region_rehome_control WHERE control_id = 'global'`
)
expect(control.rows).toEqual([
{
generation: '3',
preference_max_age_ms: '86400000',
host_cooldown_ms: String(REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS)
}
])
})
} finally {
await database.close()
}
})
it('upgrades once across concurrent startups', async () => {
const results = await Promise.allSettled(
Array.from(
{ length: 5 },
async (): Promise<RelayDatabase> =>
await openRelayDatabase({ databaseUrl: scopedUrl, dataDir: '' })
)
)
const databases = results.flatMap((result) =>
result.status === 'fulfilled' ? [result.value] : []
)
await Promise.all(databases.map(async (database) => await database.close()))
expect(
results.flatMap((result) =>
result.status === 'rejected'
? [
{
code: (result.reason as { code?: unknown }).code,
message: String(result.reason)
}
]
: []
)
).toEqual([])
await withClient(async (client) => {
await client.query(`SET search_path = ${schema}`)
await client.query(INSERT_ATTEMPT, attemptValues('attempt-4', 'us-central1'))
const constraints = await client.query(
`SELECT conname FROM pg_constraint
WHERE conrelid = 'relay_region_rehome_attempts'::regclass
AND conname LIKE '%preferred_region%'`
)
expect(constraints.rows).toEqual([
{ conname: 'relay_region_rehome_attempts_preferred_region_valid' }
])
})
}, 60_000)
})
@@ -81,6 +81,153 @@ describePostgres('PostgreSQL regional rehoming', () => {
expect(await context.store.claimRegionalRehome()).not.toBeNull()
})
it('moves a us-central1 host onto a cell in its preferred asia-east2 region', async () => {
const context = await fixture()
const attempt = await context.store.claimRegionalRehome()
expect(attempt).toMatchObject({
preferredRegion: 'asia-east2',
sourceCellId: context.source.id,
targetCellId: context.target.id
})
expect(await primary.query(
`SELECT preferred_region, source_cell_id, target_cell_id
FROM relay_region_rehome_attempts WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{
preferred_region: 'asia-east2',
source_cell_id: context.source.id,
target_cell_id: context.target.id
}])
})
it('moves an asia-east2 host back onto a cell in its preferred us-central1 region', async () => {
const context = await fixture({
sourceRegion: 'asia-east2',
targetRegion: 'us-central1'
})
const attempt = await context.store.claimRegionalRehome()
expect(attempt).toMatchObject({
preferredRegion: 'us-central1',
sourceCellId: context.source.id,
targetCellId: context.target.id
})
// The durable attempt row must accept the reverse direction too.
expect(await primary.query(
`SELECT preferred_region, source_cell_id, target_cell_id
FROM relay_region_rehome_attempts WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{
preferred_region: 'us-central1',
source_cell_id: context.source.id,
target_cell_id: context.target.id
}])
expect(await primary.query(
`SELECT cell_id FROM relay_assignments WHERE user_id = ?`,
[context.identity.userId]
)).toEqual([{ cell_id: context.target.id }])
})
it('leaves a host whose preference already matches its own region', async () => {
const context = await fixture({ preferredRegion: 'us-central1' })
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true
})
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('leaves a host whose preference is older than the configured max age', async () => {
const context = await fixture()
await primary.query(
`UPDATE relay_assignment_region_preferences SET observed_at = ?
WHERE user_id = ? AND relay_host_id = ?`,
[
context.now() - 24 * 60 * 60_000 - 1,
context.identity.userId,
context.identity.relayHostId
]
)
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true
})
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('leaves a host inside its per-host rehome cooldown, in either direction', async () => {
const context = await fixture({ hostCooldownMs: 3 * 24 * 60 * 60_000 })
// A move this host already made, whichever way it went.
await primary.query(
`INSERT INTO relay_region_rehome_attempts
(attempt_id, user_id, relay_host_id, preferred_region, source_cell_id,
source_cell_incarnation, target_cell_id, target_cell_incarnation,
previous_epoch, assignment_epoch, drain_grace_ms, send_attempts,
completed_at, created_at, updated_at)
VALUES (?, ?, ?, 'us-central1', ?, ?, ?, ?, 0, 1, 0, 0, ?, ?, ?)`,
[
`pg-rehome-cooldown-${context.identity.relayHostId}`,
context.identity.userId,
context.identity.relayHostId,
context.target.id,
'22222222-2222-4222-8222-222222222222',
context.source.id,
'11111111-1111-4111-8111-111111111111',
context.now(),
context.now() - 3 * 24 * 60 * 60_000 + 1,
context.now()
]
)
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true,
hostCooldownMs: 3 * 24 * 60 * 60_000
})
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 1,
migrations: 0
})
// One millisecond past the window the same host is a candidate again.
await primary.query(
`UPDATE relay_region_rehome_attempts SET created_at = ? WHERE user_id = ?`,
[context.now() - 3 * 24 * 60 * 60_000, context.identity.userId]
)
await expect(context.store.claimRegionalRehome()).resolves.toMatchObject({
sourceCellId: context.source.id,
targetCellId: context.target.id
})
})
it('leaves a host whose preferred region holds no drainable cell', async () => {
// A cell that cannot be drained cannot be a target: the host would land
// where no later rehome could move it out again.
const context = await fixture({ targetProtocol: 0 })
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true
})
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('skips an unclean cell without latching the control off', async () => {
const context = await fixture()
await primary.query(
@@ -281,7 +428,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
context.store,
context.target,
'22222222-2222-4222-8222-222222222222',
0,
1,
900_000,
2
)
@@ -322,7 +469,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
context.store,
context.target,
'44444444-4444-4444-8444-444444444444',
0,
1,
context.now()
)
@@ -341,7 +488,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
context.store,
context.target,
'22222222-2222-4222-8222-222222222222',
0,
1,
900_000,
2
)
@@ -414,6 +561,26 @@ describePostgres('PostgreSQL regional rehoming', () => {
})
})
async function attemptAndMigrationCounts(identity: {
userId: string
relayHostId: string
}): Promise<{ attempts: number; migrations: number }> {
const attempts = await primary.query(
`SELECT COUNT(*) AS count FROM relay_region_rehome_attempts
WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
const migrations = await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations
WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
return {
attempts: Number(attempts[0]!.count),
migrations: Number(migrations[0]!.count)
}
}
async function controlAccounting(identity: {
userId: string
relayHostId: string
@@ -436,12 +603,15 @@ describePostgres('PostgreSQL regional rehoming', () => {
}
}
async function fixture() {
async function fixture(options: FixtureOptions = {}) {
sequence++
let now = 1_000_000
const suffix = String(sequence)
const source = cell(suffix, 'source', 'us-central1')
const target = cell(suffix, 'target', 'asia-east2')
const sourceRegion = options.sourceRegion ?? 'us-central1'
const targetRegion = options.targetRegion ?? 'asia-east2'
const preferredRegion = options.preferredRegion ?? targetRegion
const source = cell(suffix, 'source', sourceRegion)
const target = cell(suffix, 'target', targetRegion)
const store = new RelayAssignmentStore(primary, () => now, storeOptions)
const competingStore = new RelayAssignmentStore(secondary, () => now, storeOptions)
await store.inspectRegionalRehomeControl()
@@ -452,6 +622,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
notBefore: now,
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: options.hostCooldownMs ?? 7 * 24 * 60 * 60_000,
drainGraceMs: 60_000
})
await store.reconcileCells([source, target])
@@ -466,21 +637,22 @@ describePostgres('PostgreSQL regional rehoming', () => {
store,
target,
'22222222-2222-4222-8222-222222222222',
0,
options.targetProtocol ?? 1,
900_000
)
const identity = {
userId: `pg-rehome-user-${suffix}`,
relayHostId: `rehomehost${suffix.padStart(6, '0')}`
}
const assignment = await store.assign(identity, undefined, 'us-central1')
const assignment = await store.assign(identity, undefined, sourceRegion)
const sourceControl = await store.activateControl(identity, {
cellId: source.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 1
})
await store.assign(identity, 'asia-east2')
await store.assign(identity, preferredRegion)
return {
preferredRegion,
store,
competingStore,
identity,
@@ -500,7 +672,16 @@ const storeOptions = {
heartbeatTtlMs: 45_000
}
function cell(suffix: string, role: string, region: 'us-central1' | 'asia-east2') {
type Region = 'us-central1' | 'asia-east2'
type FixtureOptions = {
sourceRegion?: Region
targetRegion?: Region
preferredRegion?: Region
targetProtocol?: number
hostCooldownMs?: number
}
function cell(suffix: string, role: string, region: Region) {
return {
id: `pg-rehome-cell-${suffix}-${role}`,
url: `https://pg-rehome-${suffix}-${role}.example.test`,
@@ -81,6 +81,7 @@ describe('regional rehome assignment state', () => {
notBefore: context.now(),
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: 7 * 24 * 60 * 60_000,
drainGraceMs: 60_000
})).rejects.toThrow('regional_rehome_generation_mismatch')
await expect(context.store.applyRegionalRehomeControl({
@@ -89,6 +90,7 @@ describe('regional rehome assignment state', () => {
notBefore: context.now(),
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: 7 * 24 * 60 * 60_000,
drainGraceMs: 60_000
})).resolves.toMatchObject({ generation: 3, enabled: true })
await context.database.close()
@@ -207,7 +209,7 @@ describe('regional rehome assignment state', () => {
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
})
await heartbeat(context.store, target, targetIncarnation, 0, 2, {
await heartbeat(context.store, target, targetIncarnation, 1, 2, {
observedAt: context.now(),
sqlFailures: 1,
reconnects: 3,
@@ -226,6 +228,23 @@ describe('regional rehome assignment state', () => {
await context.database.close()
})
it('counts only drainable cells as the rehome fleet, in every region', async () => {
// The fleet whose health gates a rehome is exactly the cells that can be a
// source or a target, and both roles require the drain protocol.
const context = await setup({ targetProtocol: 0 })
expect(await context.store.regionalRehomeFleetSafety()).toMatchObject({
requiredCells: 1,
missingCells: 0
})
await heartbeat(context.store, target, targetIncarnation, 1, 2)
expect(await context.store.regionalRehomeFleetSafety()).toMatchObject({
requiredCells: 2,
missingCells: 0
})
await context.database.close()
})
it('claims through the measured healthy baseline of pool micro-waits and churn', async () => {
const context = await setup()
const baseline = {
@@ -238,7 +257,7 @@ describe('regional rehome assignment state', () => {
databasePoolWaitMsMax: 1
}
await heartbeat(context.store, source, sourceIncarnation, 1, 2, baseline)
await heartbeat(context.store, target, targetIncarnation, 0, 2, baseline)
await heartbeat(context.store, target, targetIncarnation, 1, 2, baseline)
await activatePreferredSource(context, {
userId: 'user-1',
relayHostId: 'abcdefghijklmnop'
@@ -324,6 +343,204 @@ describe('regional rehome assignment state', () => {
await context.database.close()
})
it('moves a live host on an asia-east2 cell back to its preferred us-central1 cell', async () => {
const context = await setup()
const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
await activateReversePreferredSource(context, identity)
const attempt = await context.store.claimRegionalRehome()
expect(attempt).toMatchObject({
userId: identity.userId,
relayHostId: identity.relayHostId,
preferredRegion: 'us-central1',
sourceCellId: target.id,
sourceCellIncarnation: targetIncarnation,
targetCellId: source.id,
targetCellIncarnation: sourceIncarnation,
previousEpoch: 1,
assignmentEpoch: 2,
sendAttempts: 1
})
expect(
await context.database.query(
`SELECT preferred_region, source_cell_id, target_cell_id
FROM relay_region_rehome_attempts`
)
).toEqual([{
preferred_region: 'us-central1',
source_cell_id: target.id,
target_cell_id: source.id
}])
expect(await context.store.resolve(identity)).toMatchObject({ cellId: source.id })
await context.database.close()
})
it('drops a candidate at scan time when no cell in the preferred region is usable', async () => {
const context = await setup()
await activatePreferredSource(context, {
userId: 'user-1',
relayHostId: 'abcdefghijklmnop'
})
// A disabled cell is not a target, and the scan must say so: leaving it to
// the claim would burn a slot of the candidate batch on a certain skip.
await context.database.query(`UPDATE relay_cells SET enabled = 0 WHERE cell_id = ?`, [
target.id
])
const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
try {
expect(await context.store.claimRegionalRehome()).toBeNull()
} finally {
warnings.restore()
}
expect(warnings.entries).toEqual([])
expect(
await context.database.query(
`SELECT next_dispatch_at FROM relay_region_rehome_worker_state`
)
).toEqual([{ next_dispatch_at: 0 }])
expect(await context.database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
await context.database.close()
})
it('names the skip when the last target is lost between scan and claim', async () => {
const database = await openInMemoryRelayDatabase()
const context = await setup({
database,
wrap: (delegate) =>
hookAfterCandidateScan(delegate, async (transaction) => {
await transaction.query(`UPDATE relay_cells SET enabled = 0 WHERE cell_id = ?`, [
target.id
])
})
})
await activatePreferredSource(context, {
userId: 'user-1',
relayHostId: 'abcdefghijklmnop'
})
const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
try {
expect(await context.store.claimRegionalRehome()).toBeNull()
} finally {
warnings.restore()
}
expect(warnings.entries).toMatchObject([
{ skips: [{ reason: 'no_eligible_target', candidates: 1 }] }
])
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
generation: 1,
enabled: true
})
expect(await database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
await database.close()
})
it('leaves a host alone until its cooldown expires, then moves it back', async () => {
const context = await setup({ hostCooldownMs: 3 * 24 * 60 * 60_000 })
const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
const targetControl = await completeRehomeToTarget(context, identity)
// Past the dispatch interval the earlier claim charged, so the next tick
// really does scan and the cooldown is the only thing holding this host.
context.advance(10_000)
// The desktop's region probe now says us-central1 again.
await context.store.assign(identity, 'us-central1')
const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
try {
expect(await context.store.claimRegionalRehome()).toBeNull()
} finally {
warnings.restore()
}
expect(warnings.entries).toEqual([])
expect(await context.store.resolve(identity)).toMatchObject({ cellId: target.id })
context.advance(3 * 24 * 60 * 60_000)
await freshHeartbeats(context)
await context.store.renewControlActivity(identity, {
activityId: targetControl,
cellId: target.id,
expiresAt: context.now() + 90_000
})
await context.store.assign(identity, 'us-central1')
const attempt = await context.store.claimRegionalRehome()
expect(attempt).toMatchObject({
preferredRegion: 'us-central1',
sourceCellId: target.id,
targetCellId: source.id
})
await context.database.close()
})
it('rejects a host whose attempt lands between the scan and the claim', async () => {
const database = await openInMemoryRelayDatabase()
const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
const context = await setup({
database,
wrap: (delegate) =>
hookAfterCandidateScan(delegate, async (transaction) => {
await transaction.query(
`INSERT INTO relay_region_rehome_attempts
(attempt_id, user_id, relay_host_id, preferred_region, source_cell_id,
source_cell_incarnation, target_cell_id, target_cell_incarnation,
previous_epoch, assignment_epoch, drain_grace_ms, send_attempts,
created_at, updated_at)
VALUES ('raced', ?, ?, 'asia-east2', ?, ?, ?, ?, 0, 1, 0, 0, ?, ?)`,
[
identity.userId,
identity.relayHostId,
source.id,
sourceIncarnation,
target.id,
targetIncarnation,
context.now(),
context.now()
]
)
})
})
await activatePreferredSource(context, identity)
const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
try {
expect(await context.store.claimRegionalRehome()).toBeNull()
} finally {
warnings.restore()
}
expect(warnings.entries).toMatchObject([
{ skips: [{ reason: 'host_cooldown', candidates: 1 }] }
])
expect(await database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
await database.close()
})
it('does not scan a candidate whose preferred region has no drainable cell', async () => {
// A cell without the drain protocol cannot be a target: the host would land
// where no later rehome could move it out again. The candidate query drops
// it, so the tick stays idle instead of paying for an inventory scan.
const context = await setup({ targetProtocol: 0 })
await activatePreferredSource(context, {
userId: 'user-1',
relayHostId: 'abcdefghijklmnop'
})
const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
try {
expect(await context.store.claimRegionalRehome()).toBeNull()
} finally {
warnings.restore()
}
expect(warnings.entries).toEqual([])
expect(
await context.database.query(
`SELECT next_dispatch_at FROM relay_region_rehome_worker_state`
)
).toEqual([{ next_dispatch_at: 0 }])
expect(await context.database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
await context.database.close()
})
it('skips an unclean cell without latching the control off', async () => {
const context = await setup()
await activatePreferredSource(context, {
@@ -457,7 +674,7 @@ describe('regional rehome assignment state', () => {
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
})
await heartbeat(context.store, target, targetIncarnation, 0, 2, {
await heartbeat(context.store, target, targetIncarnation, 1, 2, {
observedAt: context.now(),
sqlFailures: 0,
reconnects: 0,
@@ -477,6 +694,7 @@ describe('regional rehome assignment state', () => {
notBefore: context.now(),
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: 7 * 24 * 60 * 60_000,
drainGraceMs: 60_000
})
const retry = await context.store.claimRegionalRehome()
@@ -547,7 +765,7 @@ describe('regional rehome assignment state', () => {
await activatePreferredSource(context, identity)
await context.store.claimRegionalRehome()
context.advance(6 * 60_000)
await heartbeat(context.store, target, targetIncarnation, 0, 2)
await heartbeat(context.store, target, targetIncarnation, 1, 2)
expect(await context.store.refreshRegionalRehomeLeases()).toBe(0)
expect(await context.store.abortExpiredEvacuations()).toBe(0)
@@ -1549,10 +1767,16 @@ function collectDisableWarnings() {
}
async function setup(
options: { sourceProtocol?: number; wrap?: (database: RelayDatabase) => RelayDatabase } = {}
options: {
sourceProtocol?: number
targetProtocol?: number
hostCooldownMs?: number
database?: RelayDatabase
wrap?: (database: RelayDatabase) => RelayDatabase
} = {}
) {
let clock = 1_000_000
const database = await openInMemoryRelayDatabase()
const database = options.database ?? (await openInMemoryRelayDatabase())
const store = new RelayAssignmentStore(options.wrap?.(database) ?? database, () => clock, {
requireLiveCells: true,
heartbeatTtlMs: 45_000
@@ -1565,11 +1789,12 @@ async function setup(
notBefore: clock,
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: options.hostCooldownMs ?? 7 * 24 * 60 * 60_000,
drainGraceMs: 60 * 60_000
})
await store.reconcileCells([source, target])
await heartbeat(store, source, sourceIncarnation, options.sourceProtocol ?? 1)
await heartbeat(store, target, targetIncarnation, 0)
await heartbeat(store, target, targetIncarnation, options.targetProtocol ?? 1)
return {
database,
store,
@@ -1671,7 +1896,7 @@ async function freshHeartbeats(context: Context): Promise<void> {
}
// The clock doubles as a strictly-increasing connection inclusion watermark.
await heartbeat(context.store, source, sourceIncarnation, 1, context.now(), safety)
await heartbeat(context.store, target, targetIncarnation, 0, context.now(), safety)
await heartbeat(context.store, target, targetIncarnation, 1, context.now(), safety)
}
async function activatePreferredSource(
@@ -1688,6 +1913,68 @@ async function activatePreferredSource(
return control
}
// Runs a hook inside the claim transaction, right after the candidate scan, so
// a scan-versus-claim race is deterministic instead of timing-dependent.
function hookAfterCandidateScan(
database: RelayDatabase,
hook: (transaction: RelayDatabase) => Promise<void>
): RelayDatabase {
let fired = false
const decorate = (delegate: RelayDatabase): RelayDatabase => ({
query: async (sql, params) => {
const rows = await delegate.query(sql, params)
if (!fired && sql.includes('FROM relay_assignment_region_preferences preference')) {
fired = true
await hook(delegate)
}
return rows
},
queryLocked: async (sql, params, lockOptions) =>
await delegate.queryLocked(sql, params, lockOptions),
transaction: async (operation, transactionOptions) =>
await delegate.transaction(
async (transaction) => await operation(decorate(transaction)),
transactionOptions
),
close: async () => undefined
})
return decorate(database)
}
async function completeRehomeToTarget(
context: Context,
identity: { userId: string; relayHostId: string }
): Promise<string> {
const sourceControl = await activatePreferredSource(context, identity)
const attempt = await context.store.claimRegionalRehome()
const targetControl = await context.store.activateControl(identity, {
cellId: target.id,
assignmentEpoch: attempt!.assignmentEpoch,
generation: 1
})
await context.store.markMigrationTargetRegistered(identity, {
cellId: target.id,
assignmentEpoch: attempt!.assignmentEpoch
})
await context.store.releaseActivity(identity, sourceControl)
await context.store.completeReadyRegionalRehomes()
return targetControl
}
async function activateReversePreferredSource(
context: Context,
identity: { userId: string; relayHostId: string }
): Promise<string> {
const assignment = await context.store.assign(identity, undefined, 'asia-east2')
const control = await context.store.activateControl(identity, {
cellId: target.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 1
})
await context.store.assign(identity, 'us-central1')
return control
}
async function activateSource(
context: Context,
identity: { userId: string; relayHostId: string }
@@ -36,6 +36,7 @@ async function setup() {
notBefore: clock,
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: 7 * 24 * 60 * 60_000,
drainGraceMs: 60 * 60_000
})
await store.reconcileCells([source, noHeadroom, unclean, highLoad, lowLoad])
@@ -100,22 +101,22 @@ describe('regional rehome target selection', () => {
sqlFailures: 0
})
// Lowest load but the connection hard cap is exhausted.
await context.beat(noHeadroom, 2, 0, {
await context.beat(noHeadroom, 2, 1, {
observedRequests: 0,
enforcedConnections: 999,
sqlFailures: 0
})
await context.beat(unclean, 3, 0, {
await context.beat(unclean, 3, 1, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: UNCLEAN
})
await context.beat(highLoad, 4, 0, {
await context.beat(highLoad, 4, 1, {
observedRequests: 50,
enforcedConnections: 0,
sqlFailures: 0
})
await context.beat(lowLoad, 5, 0, {
await context.beat(lowLoad, 5, 1, {
observedRequests: 10,
enforcedConnections: 0,
sqlFailures: 0
@@ -134,22 +135,22 @@ describe('regional rehome target selection', () => {
enforcedConnections: 0,
sqlFailures: 0
})
await context.beat(noHeadroom, 2, 0, {
await context.beat(noHeadroom, 2, 1, {
observedRequests: 0,
enforcedConnections: 999,
sqlFailures: 0
})
await context.beat(unclean, 3, 0, {
await context.beat(unclean, 3, 1, {
observedRequests: 0,
enforcedConnections: 0,
sqlFailures: UNCLEAN
})
await context.beat(highLoad, 4, 0, {
await context.beat(highLoad, 4, 1, {
observedRequests: 50,
enforcedConnections: 0,
sqlFailures: 0
})
await context.beat(lowLoad, 5, 0, {
await context.beat(lowLoad, 5, 1, {
observedRequests: 10,
enforcedConnections: 0,
sqlFailures: UNCLEAN
@@ -1,7 +1,9 @@
import { RELAY_REGION_METRIC_SEGMENTS, RELAY_REGIONS } from '@orca-cloud/relay-contract'
import { describe, expect, it, vi } from 'vitest'
import type { RelayDatabase } from './database.js'
import { observeRelayDatabase } from './observed-relay-database.js'
import {
CONTROL_RTT_RESERVOIR_LIMIT,
observedRelayRequests,
RelayObservability,
type RelayProcessCounts
@@ -22,6 +24,37 @@ const counts: RelayProcessCounts = {
databasePoolWaitMsMax: 1_250
}
// Two schema keys legitimately spell a policed word: the abandoned-accept bucket
// is keyed by stage name and one stage is `credential`. Rename those exact keys in
// a clone instead of rewriting the JSON, so a stray raw field or value anywhere
// else still trips the guard below.
const SCHEMA_KEY_ALIASES: Record<string, string> = {
clientAcceptCredentialMsP95: 'clientAcceptStageTwoMsP95'
}
function scrubSchemaKeys(entries: Array<Record<string, unknown>>): string {
return JSON.stringify(
entries.map((entry) =>
Object.fromEntries(
Object.entries(entry).map(([key, value]) => [
SCHEMA_KEY_ALIASES[key] ?? key,
key === 'clientAcceptsAbandonedByStageDelta' ? renameStageKeys(value) : value
])
)
)
)
}
function renameStageKeys(bucket: unknown): unknown {
if (bucket === null || typeof bucket !== 'object') return bucket
return Object.fromEntries(
Object.entries(bucket).map(([stage, count]) => [
stage === 'credential' ? 'stageTwo' : stage,
count
])
)
}
describe('relay observability', () => {
it('emits safe readiness dependency outcomes', () => {
const entries: Array<Record<string, unknown>> = []
@@ -106,14 +139,31 @@ describe('relay observability', () => {
requestedRegionsDelta: { 'asia-east2': 1, unhinted: 1 },
selectedRegionsDelta: { 'us-central1': 1 },
regionFallbacksDelta: { 'asia-east2': 1 },
unavailableRegionsDelta: { 'asia-east2': 1 }
unavailableRegionsDelta: { 'asia-east2': 1 },
// Flat per-region siblings the log-based metrics extract; `unhinted` stays map-only.
requestedRegionUsCentral1Delta: 0,
requestedRegionAsiaEast2Delta: 1,
selectedRegionUsCentral1Delta: 1,
selectedRegionAsiaEast2Delta: 0
})
expect(entries[1]).toMatchObject({
requestedRegionsDelta: {},
selectedRegionsDelta: {},
regionFallbacksDelta: {},
unavailableRegionsDelta: {}
unavailableRegionsDelta: {},
// Zeros keep publishing so an idle window cannot drop a series out of the skew join.
requestedRegionUsCentral1Delta: 0,
requestedRegionAsiaEast2Delta: 0,
selectedRegionUsCentral1Delta: 0,
selectedRegionAsiaEast2Delta: 0
})
// A region added to the contract has to reach the flat keys, or the skew alert's
// denominator silently misses it.
for (const segment of Object.values(RELAY_REGION_METRIC_SEGMENTS)) {
expect(entries[0]).toHaveProperty(`requestedRegion${segment}Delta`)
expect(entries[0]).toHaveProperty(`selectedRegion${segment}Delta`)
}
expect(Object.keys(RELAY_REGION_METRIC_SEGMENTS).sort()).toEqual([...RELAY_REGIONS].sort())
})
it('emits bounded aggregate runtime signals without identities or credentials', () => {
@@ -181,7 +231,7 @@ describe('relay observability', () => {
controlActivityRecoveryFailuresDelta: 0,
httpLatencyMsMax: 0
})
expect(JSON.stringify(entries)).not.toMatch(/token|credential|userId|relayHostId/)
expect(scrubSchemaKeys(entries)).not.toMatch(/token|credential|userId|relayHostId/i)
})
it('aggregates control and splice closes as bounded per-reason deltas', () => {
@@ -215,6 +265,117 @@ describe('relay observability', () => {
})
})
it('summarises completed client accepts and control round trips per window', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
(entry) => entries.push(entry)
)
observability.recordClientAcceptCompleted({
totalMs: 812.4567,
stageMs: { assignment: 120, credential: 90, activity: 40, attach: 500, basis: 62 }
})
observability.recordClientAcceptCompleted({
totalMs: 6_400,
stageMs: { assignment: 4_100, credential: 95, activity: 60, attach: 2_000, basis: 145 }
})
observability.recordControlRtt(28)
observability.recordControlRtt(240)
observability.recordControlRtt(31)
observability.flush(counts)
observability.flush(counts)
expect(entries[0]).toMatchObject({
clientAcceptCompletedDelta: 2,
clientAcceptTotalMsP50: 812.457,
clientAcceptTotalMsP95: 6_400,
clientAcceptTotalMsMax: 6_400,
clientAcceptAssignmentMsP95: 4_100,
clientAcceptCredentialMsP95: 95,
clientAcceptActivityMsP95: 60,
clientAcceptAttachMsP95: 2_000,
clientAcceptBasisMsP95: 145,
controlRttSamplesDelta: 3,
controlRttMsP50: 31,
controlRttMsP95: 240,
controlRttMsMax: 240
})
// Only-add: the pre-existing fields still read the same after the extension.
expect(entries[0]).toMatchObject({
event: 'orca_relay_runtime_metrics',
metricVersion: 2,
clientAcceptsAbandonedByStageDelta: {},
clientAcceptAbandonedMsMax: 0
})
// An empty window publishes counts only: a zero percentile point is
// indistinguishable from a real zero once Cloud Logging aggregates it.
expect(entries[1]).toMatchObject({ clientAcceptCompletedDelta: 0, controlRttSamplesDelta: 0 })
for (const omitted of [
'clientAcceptTotalMsP50',
'clientAcceptTotalMsP95',
'clientAcceptTotalMsMax',
'clientAcceptAssignmentMsP95',
'clientAcceptCredentialMsP95',
'clientAcceptActivityMsP95',
'clientAcceptAttachMsP95',
'clientAcceptBasisMsP95',
'controlRttMsP50',
'controlRttMsP95',
'controlRttMsMax'
]) {
expect(entries[1]).not.toHaveProperty(omitted)
expect(entries[0]).toHaveProperty(omitted)
}
expect(scrubSchemaKeys(entries)).not.toMatch(/token|credential|userId|relayHostId/i)
})
it('caps the control round-trip reservoir and reports what it dropped', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
(entry) => entries.push(entry)
)
const flooded = CONTROL_RTT_RESERVOIR_LIMIT * 20
for (let sample = 0; sample < flooded; sample++) {
observability.recordControlRtt(10 + (sample % 40))
}
observability.flush(counts)
// Dropped is observed minus retained, so this pins the retained window at the cap.
expect(entries[0]).toMatchObject({
controlRttSamplesDelta: flooded,
controlRttSamplesDroppedDelta: flooded - CONTROL_RTT_RESERVOIR_LIMIT
})
// The kept samples are real observations, not a truncated or synthesised window.
expect(entries[0]!.controlRttMsP50 as number).toBeGreaterThanOrEqual(10)
expect(entries[0]!.controlRttMsMax as number).toBeLessThanOrEqual(49)
observability.flush(counts)
expect(entries[1]).toMatchObject({
controlRttSamplesDelta: 0,
controlRttSamplesDroppedDelta: 0
})
expect(entries[1]).not.toHaveProperty('controlRttMsP50')
})
it('samples the whole flooded window rather than its first samples', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
(entry) => entries.push(entry)
)
const half = CONTROL_RTT_RESERVOIR_LIMIT * 10
for (let sample = 0; sample < half; sample++) observability.recordControlRtt(10)
for (let sample = 0; sample < half; sample++) observability.recordControlRtt(900)
observability.flush(counts)
// Keeping the first N instead would publish a window of nothing but 10s. Each
// reservoir slot ends up drawn from the late half with ~1/2 probability, so
// fewer than the 5% the p95 needs is out of reach of this suite.
expect(entries[0]!.controlRttMsP95).toBe(900)
expect(entries[0]!.controlRttMsMax).toBe(900)
})
it('observes successful and failed database calls including transactions', async () => {
const recordSql = vi.fn()
const underlying: RelayDatabase = {
+128 -14
View File
@@ -1,5 +1,5 @@
import { monitorEventLoopDelay, performance } from 'node:perf_hooks'
import type { RelayRegion } from '@orca-cloud/relay-contract'
import { RELAY_REGION_METRIC_SEGMENTS, type RelayRegion } from '@orca-cloud/relay-contract'
import type { ControlRenewalOutcome } from './assignment-store.js'
import type { CellInventoryHoldCounts } from './cell-inventory-hold-samples.js'
import type { PostgresPoolPressureCounts } from './postgres-pool-pressure.js'
@@ -65,11 +65,31 @@ export interface RelayRuntimeObserver {
recordControlClose?(code: number): void
recordSpliceClose?(trigger: string): void
recordClientAcceptAbandoned?(stage: RelayClientAcceptStage, elapsedMs: number): void
recordClientAcceptCompleted?(sample: RelayClientAcceptSample): void
recordControlRtt?(rttMs: number): void
}
// Which serialized accept step the phone had already hung up behind.
export type RelayClientAcceptStage = 'assignment' | 'credential' | 'activity'
// The attach window and the basis writes that follow it are only measurable once
// the host data leg lands, so they join the serialized pre-attach steps on
// completed accepts only.
export type RelayClientAcceptTimedStage = RelayClientAcceptStage | 'attach' | 'basis'
export const RELAY_CLIENT_ACCEPT_TIMED_STAGES = [
'assignment',
'credential',
'activity',
'attach',
'basis'
] as const satisfies readonly RelayClientAcceptTimedStage[]
export type RelayClientAcceptSample = {
totalMs: number
stageMs: Record<RelayClientAcceptTimedStage, number>
}
type RelayMetricDeltas = {
forwardedBytes: number
authSuccesses: number
@@ -93,12 +113,20 @@ type RelayMetricDeltas = {
spliceClosesByTrigger: Record<string, number>
clientAcceptsAbandonedByStage: Record<string, number>
clientAcceptAbandonedMsMax: number
clientAcceptTotalsMs: number[]
clientAcceptStageSamplesMs: Record<RelayClientAcceptTimedStage, number[]>
controlRttSamplesMs: number[]
controlRttObserved: number
controlRenewalLatenciesMs: number[]
controlRenewalsByOutcome: Record<string, number>
controlActivityRecoveries: number
controlActivityRecoveryFailures: number
}
// A host chooses how often it answers a ping, so the process-wide window is a
// reservoir: the heap cost of a flood is capped and the percentiles stay unbiased.
export const CONTROL_RTT_RESERVOIR_LIMIT = 1024
type MetricWriter = (entry: Record<string, unknown>) => void
const emptyDeltas = (): RelayMetricDeltas => ({
@@ -124,18 +152,42 @@ const emptyDeltas = (): RelayMetricDeltas => ({
spliceClosesByTrigger: {},
clientAcceptsAbandonedByStage: {},
clientAcceptAbandonedMsMax: 0,
clientAcceptTotalsMs: [],
clientAcceptStageSamplesMs: {
assignment: [],
credential: [],
activity: [],
attach: [],
basis: []
},
controlRttSamplesMs: [],
controlRttObserved: 0,
controlRenewalLatenciesMs: [],
controlRenewalsByOutcome: {},
controlActivityRecoveries: 0,
controlActivityRecoveryFailures: 0
})
function percentile(values: number[], percentileRank: number): number {
export function percentile(values: number[], percentileRank: number): number {
if (values.length === 0) return 0
const sorted = [...values].sort((left, right) => left - right)
return sorted[Math.ceil(percentileRank * sorted.length) - 1] ?? 0
}
function roundMs(value: number): number {
return Number(value.toFixed(3))
}
// Spreading a window into Math.max blows the stack once a busy cell samples
// enough of it, so the maximum is folded instead.
function latencySummary(samples: number[]): { p50: number; p95: number; max: number } {
return {
p50: roundMs(percentile(samples, 0.5)),
p95: roundMs(percentile(samples, 0.95)),
max: roundMs(samples.reduce((highest, sample) => Math.max(highest, sample), 0))
}
}
export class RelayObservability implements RelayRuntimeObserver {
private readonly eventLoop = monitorEventLoopDelay({ resolution: 20 })
private deltas = emptyDeltas()
@@ -244,6 +296,25 @@ export class RelayObservability implements RelayRuntimeObserver {
)
}
recordClientAcceptCompleted(sample: RelayClientAcceptSample): void {
this.deltas.clientAcceptTotalsMs.push(sample.totalMs)
for (const stage of RELAY_CLIENT_ACCEPT_TIMED_STAGES) {
this.deltas.clientAcceptStageSamplesMs[stage].push(sample.stageMs[stage])
}
}
recordControlRtt(rttMs: number): void {
const samples = this.deltas.controlRttSamplesMs
const observedBefore = this.deltas.controlRttObserved++
if (samples.length < CONTROL_RTT_RESERVOIR_LIMIT) {
samples.push(rttMs)
return
}
// Algorithm R: every round trip in the window keeps an equal chance of being kept.
const slot = Math.floor(Math.random() * (observedBefore + 1))
if (slot < CONTROL_RTT_RESERVOIR_LIMIT) samples[slot] = rttMs
}
start(readCounts: () => RelayProcessCounts, intervalMs = 30_000): void {
if (this.timer) return
this.eventLoop.enable()
@@ -277,6 +348,11 @@ export class RelayObservability implements RelayRuntimeObserver {
controlActivityRecoveryFailures: deltas.controlActivityRecoveryFailures
}
this.deltas = emptyDeltas()
const acceptTotals = latencySummary(deltas.clientAcceptTotalsMs)
const acceptStageP95 = (stage: RelayClientAcceptTimedStage): number =>
roundMs(percentile(deltas.clientAcceptStageSamplesMs[stage], 0.95))
const controlRtt = latencySummary(deltas.controlRttSamplesMs)
const controlRenewal = latencySummary(deltas.controlRenewalLatenciesMs)
const memory = process.memoryUsage()
const p99 = this.eventLoop.count === 0 ? 0 : this.eventLoop.percentile(99) / 1_000_000
this.eventLoop.reset()
@@ -301,15 +377,43 @@ export class RelayObservability implements RelayRuntimeObserver {
placementRejectionsByReasonDelta: deltas.placementRejectionsByReason,
requestedRegionsDelta: deltas.requestedRegions,
selectedRegionsDelta: deltas.selectedRegions,
...regionCounterFields('requestedRegion', deltas.requestedRegions),
...regionCounterFields('selectedRegion', deltas.selectedRegions),
regionFallbacksDelta: deltas.regionFallbacks,
unavailableRegionsDelta: deltas.unavailableRegions,
controlClosesByCodeDelta: deltas.controlClosesByCode,
spliceClosesByTriggerDelta: deltas.spliceClosesByTrigger,
clientAcceptsAbandonedByStageDelta: deltas.clientAcceptsAbandonedByStage,
clientAcceptAbandonedMsMax: Number(deltas.clientAcceptAbandonedMsMax.toFixed(3)),
clientAcceptAbandonedMsMax: roundMs(deltas.clientAcceptAbandonedMsMax),
clientAcceptCompletedDelta: deltas.clientAcceptTotalsMs.length,
// Accepts are sparse: publishing a zero percentile for every empty window
// would pin the p50 at 0 forever and collapse the p95 at low accept rates.
...(deltas.clientAcceptTotalsMs.length === 0
? {}
: {
clientAcceptTotalMsP50: acceptTotals.p50,
clientAcceptTotalMsP95: acceptTotals.p95,
clientAcceptTotalMsMax: acceptTotals.max,
clientAcceptAssignmentMsP95: acceptStageP95('assignment'),
clientAcceptCredentialMsP95: acceptStageP95('credential'),
clientAcceptActivityMsP95: acceptStageP95('activity'),
clientAcceptAttachMsP95: acceptStageP95('attach'),
clientAcceptBasisMsP95: acceptStageP95('basis')
}),
// Every round trip observed in the window, including the ones the reservoir
// above declined to keep; the percentiles summarise only what it kept.
controlRttSamplesDelta: deltas.controlRttObserved,
controlRttSamplesDroppedDelta: deltas.controlRttObserved - deltas.controlRttSamplesMs.length,
...(deltas.controlRttSamplesMs.length === 0
? {}
: {
controlRttMsP50: controlRtt.p50,
controlRttMsP95: controlRtt.p95,
controlRttMsMax: controlRtt.max
}),
sqlQueriesDelta: deltas.sqlQueries,
sqlFailuresDelta: deltas.sqlFailures,
sqlLatencyMsMax: Number(deltas.sqlLatencyMsMax.toFixed(3)),
sqlLatencyMsMax: roundMs(deltas.sqlLatencyMsMax),
controlRenewalsByOutcomeDelta: deltas.controlRenewalsByOutcome,
controlRenewalsDelta: deltas.controlRenewalLatenciesMs.length,
controlRenewalSuccessesDelta: deltas.controlRenewalsByOutcome.renewed ?? 0,
@@ -317,16 +421,10 @@ export class RelayObservability implements RelayRuntimeObserver {
deltas.controlRenewalsByOutcome.control_activity_not_found ?? 0,
controlActivityRecoveriesDelta: deltas.controlActivityRecoveries,
controlActivityRecoveryFailuresDelta: deltas.controlActivityRecoveryFailures,
controlRenewalLatencyMsP50: Number(
percentile(deltas.controlRenewalLatenciesMs, 0.5).toFixed(3)
),
controlRenewalLatencyMsP95: Number(
percentile(deltas.controlRenewalLatenciesMs, 0.95).toFixed(3)
),
controlRenewalLatencyMsMax: Number(
Math.max(0, ...deltas.controlRenewalLatenciesMs).toFixed(3)
),
httpLatencyMsMax: Number(deltas.httpLatencyMsMax.toFixed(3)),
controlRenewalLatencyMsP50: controlRenewal.p50,
controlRenewalLatencyMsP95: controlRenewal.p95,
controlRenewalLatencyMsMax: controlRenewal.max,
httpLatencyMsMax: roundMs(deltas.httpLatencyMsMax),
heapUsedBytes: memory.heapUsed,
heapTotalBytes: memory.heapTotal,
eventLoopDelayMsP99: Number(p99.toFixed(3))
@@ -334,6 +432,22 @@ export class RelayObservability implements RelayRuntimeObserver {
}
}
// Flat siblings of the nested region maps, always emitted for every region including zeros.
// A log-based metric cannot reach `requestedRegionsDelta."asia-east2"` without a quoted field
// path, and an absent key would drop a series out of the inner join the region-skew alert does.
// The maps stay authoritative and keep carrying anything outside the catalog, such as `unhinted`.
function regionCounterFields(
prefix: 'requestedRegion' | 'selectedRegion',
counts: Record<string, number>
): Record<string, number> {
return Object.fromEntries(
Object.entries(RELAY_REGION_METRIC_SEGMENTS).map(([region, segment]) => [
`${prefix}${segment}Delta`,
counts[region] ?? 0
])
)
}
function increment(counts: Record<string, number>, key: string): void {
counts[key] = (counts[key] ?? 0) + 1
}
+4 -1
View File
@@ -2,7 +2,9 @@ import { createAdaptorServer } from '@hono/node-server'
import {
hasAdmissionCapacity,
HostDataAuthSchema,
parseRelayHostCapabilities,
RELAY_ADMISSION_BUDGETS,
RELAY_HOST_CAPABILITIES_HEADER,
RELAY_CLOSE_CODE,
RELAY_DEFAULT_REGION,
RELAY_PROTOCOL_LIMITS,
@@ -486,7 +488,8 @@ export function createRelayServer(
sessions.acceptControl(
webSocket,
identity,
controlUpgrade?.inclusionWatermark
controlUpgrade?.inclusionWatermark,
parseRelayHostCapabilities(request.headers[RELAY_HOST_CAPABILITIES_HEADER])
)
})
} catch {
+73 -2
View File
@@ -9,7 +9,9 @@ import { fileURLToPath } from 'node:url'
import { exportJWK, generateKeyPair, jwtVerify, SignJWT } from 'jose'
import {
buildHostProofMacInput,
HOST_CHALLENGE_PLAINTEXT_DOMAIN
HOST_CHALLENGE_PLAINTEXT_DOMAIN,
RELAY_HOST_CAPABILITIES_HEADER,
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS
} from '@orca-cloud/relay-contract'
import nacl from 'tweetnacl'
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
@@ -282,11 +284,17 @@ async function openHostControl(input?: {
previousGeneration?: number
keyPair?: nacl.BoxKeyPair
assignmentEpoch?: number
capabilities?: string
}): Promise<{ socket: WebSocket; ack: Record<string, unknown>; keyPair: nacl.BoxKeyPair }> {
const keyPair = input?.keyPair ?? nacl.box.keyPair()
const hostId = createHash('sha256').update(keyPair.publicKey).digest('base64url').slice(0, 16)
const socket = new WebSocket(`${relayUrl.replace('http:', 'ws:')}/v1/host/control`, {
headers: { authorization: `Bearer ${await relayToken('orca-relay', hostId)}` },
headers: {
authorization: `Bearer ${await relayToken('orca-relay', hostId)}`,
...(input?.capabilities
? { [RELAY_HOST_CAPABILITIES_HEADER]: input.capabilities }
: {})
},
perMessageDeflate: false
})
await new Promise<void>((resolveOpen, reject) => {
@@ -653,6 +661,69 @@ describe('served relay URL', () => {
expect(result.reason).not.toContain('http')
})
it('restates a pending connection to the rebound control, detailed only when advertised', async () => {
// The one link the unit tests cannot reach: an upgrade that really carries
// x-orca-host-capabilities must reach acceptControl and change the ack. A
// typo in the header name here passes every other test in the suite.
const host = await openHostControl()
const hostId = createHash('sha256')
.update(host.keyPair.publicKey)
.digest('base64url')
.slice(0, 16)
const inviteResponse = nextMessage(host.socket)
host.socket.send(
JSON.stringify({
type: 'invite-create',
reqId: 'capability-invite',
relayDeviceId: 'capability-device'
})
)
const invite = await inviteResponse
const phone = new WebSocket(`${relayUrl.replace('http:', 'ws:')}/v1/connect/${hostId}`, {
headers: forwardedHeaders()
})
await new Promise<void>((resolveOpen, reject) => {
phone.once('open', resolveOpen)
phone.once('error', reject)
})
const connectionPromise = nextMessage(host.socket)
phone.send(
JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential: invite.inviteToken })
)
// Never attached: the connection stays pending, which is what the ack restates.
const connection = await connectionPromise
expect(connection.type).toBe('conn-open')
const capable = await openHostControl({
keyPair: host.keyPair,
controlResumeSecret: String(host.ack.controlResumeSecret),
previousGeneration: 1,
capabilities: RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS
})
expect(capable.ack.pendingConns).toEqual([
{
connId: connection.connId,
connTicket: connection.connTicket,
kind: 'invite',
relayDeviceId: 'capability-device'
}
])
const legacy = await openHostControl({
keyPair: host.keyPair,
controlResumeSecret: String(capable.ack.controlResumeSecret),
previousGeneration: 1
})
// A shipped host parses these entries strictly, so an unannounced key would
// fail the whole ack and kill a control that was working.
expect(legacy.ack.pendingConns).toEqual([
{ connId: connection.connId, connTicket: connection.connTicket }
])
phone.close()
legacy.socket.close()
})
it('keeps a pending attach usable after a bad ticket and rejects ticket replay', async () => {
const host = await openHostControl()
const hostId = createHash('sha256')
@@ -133,14 +133,17 @@
"google_logging_metric.relay_snapshot",
"google_monitoring_alert_policy.relay_assignment_5xx",
"google_monitoring_alert_policy.relay_assignment_edge_429",
"google_monitoring_alert_policy.relay_cell_control_rtt",
"google_monitoring_alert_policy.relay_cell_process_exit",
"google_monitoring_alert_policy.relay_cloud_nat_port_drops",
"google_monitoring_alert_policy.relay_cloud_sql_backends",
"google_monitoring_alert_policy.relay_cloud_sql_checkpoint_loop",
"google_monitoring_alert_policy.relay_cloud_sql_disk",
"google_monitoring_alert_policy.relay_custom",
"google_monitoring_alert_policy.relay_far_cell_accept_latency",
"google_monitoring_alert_policy.relay_gce_connection_headroom",
"google_monitoring_alert_policy.relay_postgres_retry_exhausted",
"google_monitoring_alert_policy.relay_region_hint_skew",
"google_monitoring_dashboard.relay_incident",
"google_project_iam_custom_role.github_production_relay_capacity_mutation",
"google_project_iam_custom_role.github_relay_asia_topology_mutation",
@@ -45,6 +45,7 @@ export function parseRegionalRehomeArguments(argv, environment = process.env) {
'not-before',
'rate-per-minute',
'preference-max-age-ms',
'host-cooldown-ms',
'drain-grace-ms',
'confirmation'
]
@@ -117,6 +118,11 @@ export function parseRegionalRehomeArguments(argv, environment = process.env) {
'--preference-max-age-ms',
{ minimum: 60_000, maximum: 30 * 24 * 60 * 60_000 }
),
hostCooldownMs: integer(
values['host-cooldown-ms'],
'--host-cooldown-ms',
{ minimum: 60_000, maximum: 30 * 24 * 60 * 60_000 }
),
drainGraceMs: integer(values['drain-grace-ms'], '--drain-grace-ms', {
minimum: 60_000,
maximum: 60 * 60_000
@@ -147,6 +153,11 @@ function assertControl(control, expected) {
!Number.isSafeInteger(control.notBefore) ||
!Number.isSafeInteger(control.ratePerMinute) ||
!Number.isSafeInteger(control.preferenceMaxAgeMs) ||
// A director predating the per-host cooldown does not report it. Reading
// the control and both emergency brakes must keep working against that
// image; only enable requires the field.
(control.hostCooldownMs !== undefined &&
!Number.isSafeInteger(control.hostCooldownMs)) ||
!Number.isSafeInteger(control.drainGraceMs)
) throw new Error('director returned an invalid regional rehome control')
if (expected.enabled !== undefined && control.enabled !== expected.enabled) {
@@ -155,6 +166,12 @@ function assertControl(control, expected) {
return control
}
// Echo the cooldown only when the director already reports it: a legacy
// director rejects the unknown key outright and would refuse every brake.
function cooldownField(before, value) {
return before.hostCooldownMs === undefined ? {} : { hostCooldownMs: value }
}
async function verifiedDisabledControl(post, generation) {
return assertControl((await post('/v1/admin/regional-rehome-control', {
v: 1,
@@ -171,6 +188,7 @@ async function applyDisabledControl(post, before) {
notBefore: before.notBefore,
ratePerMinute: before.ratePerMinute,
preferenceMaxAgeMs: before.preferenceMaxAgeMs,
...cooldownField(before, before.hostCooldownMs),
drainGraceMs: before.drainGraceMs,
confirmation: 'DISABLE_REGIONAL_REHOMING'
})).control, { generation: before.generation + 1, enabled: false })
@@ -270,6 +288,11 @@ export async function operateRegionalRehome(config, dependencies = {}) {
throw new Error('regional rehome is already paused')
}
const enabled = config.mode === 'enable'
if (enabled && before.hostCooldownMs === undefined) {
throw new Error(
'director does not report a per-host rehome cooldown; deploy a director that supports it before enabling'
)
}
const applied = await post('/v1/admin/regional-rehome-control', {
v: 1,
action: 'apply',
@@ -278,6 +301,7 @@ export async function operateRegionalRehome(config, dependencies = {}) {
notBefore: config.notBefore,
ratePerMinute: config.ratePerMinute,
preferenceMaxAgeMs: config.preferenceMaxAgeMs,
...cooldownField(before, config.hostCooldownMs),
drainGraceMs: config.drainGraceMs,
confirmation: enabled
? 'ENABLE_REGIONAL_REHOMING'
@@ -26,6 +26,7 @@ function argumentsFor(mode, confirmation) {
'--not-before', '2000000000000',
'--rate-per-minute', '10',
'--preference-max-age-ms', '86400000',
'--host-cooldown-ms', '604800000',
'--drain-grace-ms', '60000',
'--confirmation', confirmation
])
@@ -40,10 +41,29 @@ function control(generation, enabled) {
notBefore: 2_000_000_000_000,
ratePerMinute: 10,
preferenceMaxAgeMs: 86_400_000,
hostCooldownMs: 604_800_000,
drainGraceMs: 60_000
}
}
// The control a director predating the per-host cooldown reports.
function legacyControl(generation, enabled) {
const { hostCooldownMs: _absent, ...rest } = control(generation, enabled)
return rest
}
function legacyDirector(controls) {
const requests = []
const post = async (path, body) => {
requests.push({ path, body })
if (path === '/v1/admin/admission-selector/status') {
return { selector: { generation: 11, membership } }
}
return { v: 1, control: controls.shift() }
}
return { requests, post }
}
test('parses exact selector and typed control confirmation', () => {
const parsed = parseRegionalRehomeArguments(
argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING'),
@@ -52,6 +72,17 @@ test('parses exact selector and typed control confirmation', () => {
assert.equal(parsed.expectedSelectorGeneration, 11)
assert.equal(parsed.expectedControlGeneration, 4)
assert.equal(parsed.ratePerMinute, 10)
assert.equal(parsed.hostCooldownMs, 604_800_000)
assert.throws(
() => parseRegionalRehomeArguments(
argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING').filter(
(value, index, all) =>
value !== '--host-cooldown-ms' && all[index - 1] !== '--host-cooldown-ms'
),
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
),
/complete durable control shape/
)
assert.throws(
() => parseRegionalRehomeArguments(
argumentsFor('pause', 'DISABLE_REGIONAL_REHOMING'),
@@ -79,6 +110,7 @@ test('binds enable to exact selector and durable control generations', async ()
notBefore: 0,
ratePerMinute: 10,
preferenceMaxAgeMs: 86_400_000,
hostCooldownMs: 604_800_000,
drainGraceMs: 60_000,
...control
}))
@@ -96,6 +128,7 @@ test('binds enable to exact selector and durable control generations', async ()
}
})
assert.equal(result.control.generation, 5)
assert.equal(result.control.hostCooldownMs, 604_800_000)
assert.deepEqual(requests[2].body, {
v: 1,
action: 'apply',
@@ -104,11 +137,82 @@ test('binds enable to exact selector and durable control generations', async ()
notBefore: 2_000_000_000_000,
ratePerMinute: 10,
preferenceMaxAgeMs: 86_400_000,
hostCooldownMs: 604_800_000,
drainGraceMs: 60_000,
confirmation: 'ENABLE_REGIONAL_REHOMING'
})
})
test('inspects a director that predates the per-host cooldown', async () => {
const director = legacyDirector([legacyControl(4, true)])
const config = parseRegionalRehomeArguments(
argumentsFor('inspect'),
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
)
const result = await operateRegionalRehome(config, { post: director.post })
assert.equal(result.control.generation, 4)
assert.equal(result.control.hostCooldownMs, undefined)
})
for (const [mode, confirmation, enabledBefore] of [
['pause', 'PAUSE_REGIONAL_REHOMING', true],
['disable', 'DISABLE_REGIONAL_REHOMING', false]
]) {
test(`${mode} still brakes a director that predates the cooldown`, async () => {
const director = legacyDirector([
legacyControl(4, enabledBefore),
legacyControl(5, false),
legacyControl(5, false)
])
const config = parseRegionalRehomeArguments(
argumentsFor(mode, confirmation),
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
)
const result = await operateRegionalRehome(config, { post: director.post })
assert.equal(result.control.generation, 5)
// The unknown key would be refused by that director's strict schema.
assert.equal('hostCooldownMs' in director.requests[2].body, false)
assert.equal(director.requests[2].body.confirmation, 'DISABLE_REGIONAL_REHOMING')
})
}
test('failed-enable recovery brakes a director that predates the cooldown', async () => {
const requests = []
let current = legacyControl(7, true)
const result = await recoverRegionalRehomeEnable({
mode: 'recover-enable',
expectedControlGeneration: 4
}, async (_path, body) => {
requests.push(body)
if (body.action === 'inspect') return { control: current }
current = legacyControl(8, false)
return { control: current }
})
assert.equal(result.control.generation, 8)
assert.equal('hostCooldownMs' in requests[1], false)
})
test('refuses to enable a director that does not report the cooldown', async () => {
const director = legacyDirector([legacyControl(4, false)])
const config = parseRegionalRehomeArguments(
argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING'),
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
)
await assert.rejects(
operateRegionalRehome(config, { post: director.post }),
/per-host rehome cooldown/
)
// Read-only: selector status and the control inspect, and nothing else.
assert.equal(director.requests.length, 2)
assert.equal(director.requests.every(({ body }) => body.action !== 'apply'), true)
})
test('fails closed on selector drift before reading or mutating control', async () => {
let calls = 0
const config = parseRegionalRehomeArguments(
@@ -150,6 +254,7 @@ test('failed-enable recovery CAS-disables an advanced enabled generation', async
notBefore: 2_000_000_000_000,
ratePerMinute: 10,
preferenceMaxAgeMs: 86_400_000,
hostCooldownMs: 604_800_000,
drainGraceMs: 60_000,
confirmation: 'DISABLE_REGIONAL_REHOMING'
})
@@ -1,7 +1,9 @@
import { pathToFileURL } from 'node:url'
import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
const PRODUCTION_CELL = /^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26)$/
// Every general cell that carries the rehome identity: the sixteen US cells and the
// three asia-east2 cells that drain mis-homed hosts back the other way.
const PRODUCTION_CELL = /^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26|27|28|29)$/
const DIRECTOR_ORIGIN = 'https://relay.onorca.dev'
export function parseRehomeTrustProbeArguments(argv, environment = process.env) {
@@ -111,3 +111,23 @@ test('fails when both trust-probe attempts return a transient 503', async () =>
)
assert.equal(calls, 2)
})
test('approves the asia-east2 rehome sources and still rejects unlisted cells', () => {
for (const cellId of ['production-gce-c27', 'production-gce-c28', 'production-gce-c29']) {
const parsed = parseRehomeTrustProbeArguments(
argv.map((value) => (value === 'production-gce-c7' ? cellId : value)),
environment
)
assert.equal(parsed.cellId, cellId)
}
for (const cellId of ['production-gce-c1', 'production-gce-c17', 'production-gce-c30']) {
assert.throws(
() =>
parseRehomeTrustProbeArguments(
argv.map((value) => (value === 'production-gce-c7' ? cellId : value)),
environment
),
/--cell-id is not approved/
)
}
})
@@ -0,0 +1,84 @@
import assert from 'node:assert/strict'
import { readFileSync } from 'node:fs'
import test from 'node:test'
import { fileURLToPath } from 'node:url'
// Why: the region-skew alert compares asia-east2's share of assignment hints against its share of
// actual placements. Both shares are sums over one log-based metric per region, and the region
// list is written out by hand in Terraform. A region added to the contract without matching
// metrics would silently drop out of both denominators and move the ratio the alert fires on.
const read = (relative) => readFileSync(fileURLToPath(new URL(relative, import.meta.url)), 'utf8')
const collapse = (text) => text.replaceAll(/\s+/g, ' ')
const contractRegions = (() => {
const source = read('../../packages/relay-contract/src/relay-regions.ts')
const literal = /export const RELAY_REGIONS = \[([^\]]*)\]/.exec(source)
assert.ok(literal, 'RELAY_REGIONS literal not found in relay-regions.ts')
return [...literal[1].matchAll(/'([^']+)'/g)].map((match) => match[1])
})()
const terraform = read('../../infra/terraform/relay-observability.tf')
const terraformRegions = (() => {
const literal = /relay_region_keys = \[([^\]]*)\]/.exec(terraform)
assert.ok(literal, 'relay_region_keys not found in relay-observability.tf')
return [...literal[1].matchAll(/"([^"]+)"/g)].map((match) => match[1])
})()
// Both sides now spell the field-name segments out, so the test compares the two declared maps
// rather than two source expressions. Reformatting either file cannot break this, and a literal
// expected value below still catches an identical wrong edit made to both.
const declaredSegments = (source, open, close) => {
const body = source.slice(source.indexOf(open) + open.length, source.indexOf(close, source.indexOf(open)))
return Object.fromEntries(
[...body.matchAll(/'?"?([a-z0-9-]+)'?"?\s*[:=]\s*'?"?([A-Za-z0-9]+)'?"?/g)].map((match) => [
match[1],
match[2]
])
)
}
const terraformSegments = declaredSegments(terraform, 'relay_region_field_segments = {', '}')
const contractSegments = declaredSegments(
read('../../packages/relay-contract/src/relay-regions.ts'),
'RELAY_REGION_METRIC_SEGMENTS = {',
'}'
)
test('terraform covers exactly the regions the contract can hint or select', () => {
assert.deepEqual([...terraformRegions].sort(), [...contractRegions].sort())
})
test('terraform and the contract declare the same flat field segments', () => {
assert.deepEqual(terraformSegments, contractSegments)
// Pinned literally so the same wrong edit applied to both sides still fails.
assert.deepEqual(terraformSegments, { 'us-central1': 'UsCentral1', 'asia-east2': 'AsiaEast2' })
assert.deepEqual(Object.keys(terraformSegments).sort(), [...contractRegions].sort())
})
test('the skew query compares a catalogued region against itself', () => {
const columns = terraformRegions.map((region) => region.replaceAll('-', '_'))
const hint = /hint_share: req_([a-z0-9_]+) \//.exec(terraform)
const placement = /placement_share: sel_([a-z0-9_]+) \//.exec(terraform)
assert.ok(hint && placement, 'skew query share columns not found')
assert.equal(hint[1], placement[1], 'the two shares must be about the same region')
assert.ok(columns.includes(hint[1]), `${hint[1]} is not one of ${columns.join(', ')}`)
})
test('the skew condition never divides by the placement share', () => {
// A zero-placement hour is the worst skew there is; MQL drops the row on x/0, so the ratio form
// silences exactly the case the alert exists for.
assert.ok(
!/hint_share \/ placement_share/.test(terraform),
'cross-multiply instead: hint_share > 2 * placement_share'
)
assert.match(collapse(terraform), /condition hint_share > 2 \* placement_share/)
})
test('the unhinted bucket stays out of the skew denominators', () => {
assert.ok(
!terraformRegions.includes('unhinted'),
'unhinted requests are a client-side choice, not a region; including them moves the share'
)
})
@@ -179,9 +179,10 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
it('validates a correct plan for every wave cell at that cell\'s rehome protocol', () => {
for (const cellId of SAME_CAP_CELLS) {
const [region, cap] = resolveCellShape(cellId).stdout.trim().split(' ')
const [, cap] = resolveCellShape(cellId).stdout.trim().split(' ')
const protocol = REHOME_SOURCE_CELLS.has(cellId) ? 1 : 0
assert.equal(protocol, region === 'us-central1' ? 1 : 0, cellId)
// Every reviewed serving cell carries rehome trust now, in either region.
assert.equal(protocol, 1, cellId)
const config = {
mode: 'same-cap-cell',
cellId,
@@ -211,6 +212,29 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
}
})
it('validates a protocol-0 plan for a cell outside the rehome source list', () => {
const cellId = 'production-gce-c17'
assert.equal(REHOME_SOURCE_CELLS.has(cellId), false)
const config = {
mode: 'same-cap-cell',
cellId,
hardCap: 1000,
unobservedBound: 60,
image: TARGET_IMAGE,
rollbackImage: ROLLBACK_IMAGE,
rehomeDirectorServiceAccount: DIRECTOR_IDENTITY,
rehomeAudience: AUDIENCE,
regionalRehomeProtocol: '0'
}
const plan = rollPlan({ cellId, cap: 1000, protocol: 0 })
assert.deepEqual(validateCapacityPlan(plan, config), { mode: 'same-cap-cell', changes: 2 })
// Protocol 1 must reject a plan with no rehome lines, or the absent-line rule decides nothing.
assert.throws(
() => validateCapacityPlan(plan, { ...config, regionalRehomeProtocol: '1' }),
/reviewed image and capacity/
)
})
it('leaves the US-only capacity job on the default allowlist', () => {
assert.doesNotMatch(capacityWorkflow, /--approved-cells/)
})
+12
View File
@@ -464,6 +464,18 @@ Once a target control is registered, do not force the pre-registration rollback.
After a deployment traffic shift, preserve the old revision/tag until metrics and live reconnect checks pass. If the new revision is unhealthy, shift traffic back only while old controls are still valid, then issue a strictly newer director migration rather than reusing a prior epoch.
## Regional rehoming
Rehoming moves a host to a general cell in the region its desktop last reported, in either
direction. Both roles need the drain protocol: a cell without it can be neither a source nor a
target, and it is not part of the fleet whose telemetry gates the worker. Until the asia-east2
cells run `regionalRehomeProtocol` 1 they are none of the three, so no host is moved into or out
of Asia and an Asia cell in distress does not pause the worker.
`host-cooldown-ms` is the minimum gap between two rehomes of one host. It bounds the damage from
a desktop whose region probe flips: without it the host would be dragged back across the ocean on
every flip, since the preference age never expires while the host keeps reconnecting.
## Game-day matrix
Run and record each scenario in staging before launch:
+73
View File
@@ -121,6 +121,79 @@ durably marked consumed before mutation and cannot authorize another run.
Expected enabled cells must also have a powered runtime, healthy and ready endpoints, fresh
heartbeats, and matching live admission.
## Region placement alert policies
Cloud Monitoring alert policies, not monitor freeze bars: these page from
`cloud/infra/terraform/relay-observability.tf` on the shared relay channel in
`relay_alert_notification_channels`, and they do not gate any workflow. All
three exist because US desktops sat on asia-east2 cells for weeks in 2026-08
with every existing bar green.
| Alert policy | Condition |
| --- | ---: |
| Orca Relay: far-cell phone accept latency | per cell, median 30-second `clientAcceptTotalMsP95` over 15 minutes above 2,000 ms with at least 20 completed accepts |
| Orca Relay: cell control round trip | per cell, median `controlRttMsP50` over one hour above 150 ms with at least 500 samples |
| Orca Relay: region hint skew | fleet-wide, asia-east2 share of hinted requests over one hour more than 2x and more than 15 points above its share of actual placements, with at least 500 hinted requests |
Threshold basis:
- Accept latency. An in-region phone accept completes in 0.3-0.6 s and a
cross-Pacific one in 5-10 s, so 2,000 ms sits outside in-region noise and
well under the far-cell floor. The 20-accept minimum keeps one slow accept
on a quiet cell off the pager. The p95 is the published value, so the
window aggregate is its median, not its max.
- Control round trip. In-region is tens of milliseconds; a US desktop on an
asia-east2 cell is 200 ms or more. Only the p50 is used. The desktop echoes
the pong on its main thread, so the published p95 and max track renderer
stalls rather than distance. 500 samples per hour is about two
continuously connected hosts at the 15-second control ping. Tuning risk: EU
desktops on us-central1 sit at 100-130 ms, so a cell whose population is
mostly European can approach the bar while correctly homed. Check where the
hosts are before reading a first breach as mis-homing.
- Region hint skew. This compares two shares of the same hour rather than
testing one absolute share, because an absolute bar is wrong at both ends.
Measured over twelve hours on 2026-09-07, while the desktop region probe
was still mis-picking: asia-east2 was 33.8% of the 33,800 hinted requests
and only 7.9% of the 45,364 assignments, a divergence of 4.27x and a gap of
25.9 points. A fixed 40% bar would have stayed silent through that, and
once the probe is fixed the genuine APAC share climbs past any such bar and
pages forever on the correct end state. The 2x and 15-point bars sit inside
the broken state and outside a healthy one. `unhinted` requests are
excluded from the denominator: they were 27% of all requests, so a client
change that always sends a hint would move the number with no behaviour
change at all. The two bars are cross-multiplied rather than divided. An
hour that placed nobody in the region is the most extreme skew there is,
and it happens whenever the region is drained, fenced, or at capacity, but
dividing by that zero placement share makes MQL drop the row and lose the
series before any other clause runs.
Expect the skew alert to stay lit after a client fix until the mis-homed
backlog is rehomed. Sticky assignment never re-consults the hint, so a
desktop already on an asia cell keeps being placed there whatever it now
asks for; the ratio clears only once the rehome sweep has drained.
All three conditions are written in MQL rather than the metric filters the
other relay policies use. Every runtime metric is a DELTA DISTRIBUTION, and
the only scalar aligners a filter condition can apply to one are percentiles;
each of these alerts needs the sum of the extracted values as a volume floor,
which is `sum(value.<metric>)` in MQL and unreachable otherwise. None of the
metrics they read exists in the project yet, so what was checked against
production is the query shape: the same MQL run over existing metrics of the
same kind confirmed the distribution sum, the join arity, the unit literals,
and the condition clause.
The skew shares are built from one log-based metric per region for hints and
one per region for placements. They read flat `requestedRegion<Region>Delta`
and `selectedRegion<Region>Delta` fields that the relay publishes as zeros in
every interval, not the nested region maps: a log-based metric would need a
quoted field path to reach a hyphenated map key, and an absent key would drop
a series out of the inner join. The region list lives in Terraform as
`relay_region_keys` and is pinned to relay-contract's `RELAY_REGIONS` by
`dev/scripts/relay-region-hint-metrics.test.mjs`. Both sides spell the field
name segments out as literal maps rather than deriving them, so the same test
compares the two declarations directly. Adding a region to the contract
without its segment is a compile error in relay-contract, not a silent gap.
## Implementation log
- Recalibrated the relay pool freezes from 30 waiters / 1,000 ms to
@@ -402,7 +402,11 @@ relay_region_rehome_source_cell_ids = [
"production-gce-c23",
"production-gce-c24",
"production-gce-c25",
"production-gce-c26"
"production-gce-c26",
# Asia cells carry the same trust so mis-homed hosts can be drained back off them.
"production-gce-c27",
"production-gce-c28",
"production-gce-c29"
]
# Slack #orca-relay-alerts, created out of band on 2026-08-05. Declared here because an apply
+3 -2
View File
@@ -82,14 +82,15 @@ check "relay_gce_fixed_one_topology" {
assert {
condition = alltrue([
# Region is not asserted here: the director's own rehome source and target predicates
# own eligibility, so this pins only cell shape.
for cell_id in var.relay_region_rehome_source_cell_ids : try(
var.relay_gce_cells[cell_id].region == var.region &&
var.relay_gce_cells[cell_id].connection_hard_cap != null &&
!contains(var.relay_gce_fenced_cells, cell_id),
false
)
])
error_message = "Regional rehome sources must be configured, unfenced primary-region GCE cells with explicit connection limits."
error_message = "Regional rehome sources must be configured, unfenced GCE cells with explicit connection limits."
}
assert {
+215 -3
View File
@@ -65,6 +65,20 @@ locals {
control_renewal_lease_misses = { field = "controlRenewalLeaseMissesDelta", description = "Control renewals that found their activity lease missing." }
control_activity_recoveries = { field = "controlActivityRecoveriesDelta", description = "Control activity leases recovered after a renewal miss." }
control_activity_recovery_failures = { field = "controlActivityRecoveryFailuresDelta", description = "Control activity lease recovery attempts that failed." }
control_rtt_ms_p50 = { field = "controlRttMsP50", description = "Control-socket ping round trip p50 in the interval. The desktop echoes the pong on its main thread, so only the median reads as distance; the p95 and max below are dominated by desktop stalls." }
control_rtt_ms_p95 = { field = "controlRttMsP95", description = "Control-socket ping round trip p95 in the interval; a desktop-stall signal, not a distance one." }
control_rtt_ms_max = { field = "controlRttMsMax", description = "Maximum control-socket ping round trip in the interval; a desktop-stall signal, not a distance one." }
control_rtt_samples = { field = "controlRttSamplesDelta", description = "Control-socket round trips observed in the interval, one per ping answered; the percentiles above are omitted when this is zero." }
control_rtt_samples_dropped = { field = "controlRttSamplesDroppedDelta", description = "Observed round trips the bounded percentile reservoir did not keep; non-zero means the percentiles above summarise a uniform sample of the interval." }
client_accepts_completed = { field = "clientAcceptCompletedDelta", description = "Phone accepts that reached relay-hello in the interval; the percentiles below are omitted when this is zero." }
client_accept_total_ms_p50 = { field = "clientAcceptTotalMsP50", description = "Successful phone-accept duration p50, dial to relay-hello." }
client_accept_total_ms_p95 = { field = "clientAcceptTotalMsP95", description = "Successful phone-accept duration p95, dial to relay-hello." }
client_accept_total_ms_max = { field = "clientAcceptTotalMsMax", description = "Maximum successful phone-accept duration in the interval." }
client_accept_assignment_ms_p95 = { field = "clientAcceptAssignmentMsP95", description = "Accept stage p95: resume/invite lookup plus assignment resolve." }
client_accept_credential_ms_p95 = { field = "clientAcceptCredentialMsP95", description = "Accept stage p95: outer credential reservation." }
client_accept_activity_ms_p95 = { field = "clientAcceptActivityMsP95", description = "Accept stage p95: credential activity lease acquisition." }
client_accept_attach_ms_p95 = { field = "clientAcceptAttachMsP95", description = "Accept stage p95: conn-open sent until the desktop's data leg authenticated." }
client_accept_basis_ms_p95 = { field = "clientAcceptBasisMsP95", description = "Accept stage p95: splice lease and connection-basis writes between the data leg and relay-hello." }
heap_used_bytes = { field = "heapUsedBytes", description = "Node.js heap bytes used by the relay process." }
event_loop_ms_p99 = { field = "eventLoopDelayMsP99", description = "Node.js event-loop delay p99 in milliseconds." }
forwarded_bytes = { field = "forwardedBytesDelta", description = "Ciphertext bytes admitted for forwarding." }
@@ -80,6 +94,80 @@ locals {
db_oldest_wait_ms = { field = "databasePoolOldestWaitMs", description = "Current oldest PostgreSQL pool waiter age." }
db_wait_ms_max = { field = "databasePoolWaitMsMax", description = "Maximum PostgreSQL pool wait during the interval." }
}
# Regions the director can hint or select. Pinned to relay-contract's RELAY_REGIONS by
# dev/scripts/relay-region-hint-metrics.test.mjs, which also checks the flat field names below
# against the emitter. A region missing here drops out of both shares the skew alert compares.
relay_region_keys = ["us-central1", "asia-east2"]
# Flat emitter fields, not the nested `requestedRegionsDelta` map: a log-based metric would need
# a quoted field path to reach a hyphenated map key, and the relay publishes these as zeros in
# every interval so no series can drop out of the alert's inner join. Spelled out rather than
# derived, so this literal and relay-contract's RELAY_REGION_METRIC_SEGMENTS can be compared
# directly; reformatting either side cannot break the check and neither can drift alone.
relay_region_field_segments = {
"us-central1" = "UsCentral1"
"asia-east2" = "AsiaEast2"
}
relay_region_columns = { for key in local.relay_region_keys : key => replace(key, "-", "_") }
relay_region_share_metrics = merge(
{
for key in local.relay_region_keys :
"requested_regions_${local.relay_region_columns[key]}" => {
field = "requestedRegion${local.relay_region_field_segments[key]}Delta"
description = "Assignment requests that hinted ${key}."
}
},
{
for key in local.relay_region_keys :
"selected_regions_${local.relay_region_columns[key]}" => {
field = "selectedRegion${local.relay_region_field_segments[key]}Delta"
description = "Assignments that placed a host in ${key}."
}
}
)
relay_region_hinted_total = join(" + ", [for key in local.relay_region_keys : "req_${local.relay_region_columns[key]}"])
relay_region_selected_total = join(" + ", [for key in local.relay_region_keys : "sel_${local.relay_region_columns[key]}"])
# MQL, not a filter condition: every runtime metric is a DELTA DISTRIBUTION, and the only scalar
# aligners a `condition_threshold` can apply to one are percentiles. Both shares need the sum of
# the extracted values, which is `sum(value.<metric>)` in MQL and unreachable otherwise.
relay_region_hint_skew_query = join("\n", concat(
["{"],
flatten([
for index, entry in [
for key in local.relay_region_keys : { metric = "requested_regions_${local.relay_region_columns[key]}", column = "req_${local.relay_region_columns[key]}" }
] : [
index == 0 ? "" : ";",
" fetch cloud_run_revision::logging.googleapis.com/user/orca_relay_${entry.metric}",
" | align delta(1h) | every 1h",
" | group_by [], [${entry.column}: sum(value.orca_relay_${entry.metric})]"
]
]),
flatten([
for key in local.relay_region_keys : [
";",
" fetch cloud_run_revision::logging.googleapis.com/user/orca_relay_selected_regions_${local.relay_region_columns[key]}",
" | align delta(1h) | every 1h",
" | group_by [], [sel_${local.relay_region_columns[key]}: sum(value.orca_relay_selected_regions_${local.relay_region_columns[key]})]"
]
]),
[
"}",
"| join",
"| value [",
" hint_share: req_asia_east2 / (${local.relay_region_hinted_total}),",
" placement_share: sel_asia_east2 / (${local.relay_region_selected_total}),",
" hinted_requests: ${local.relay_region_hinted_total}",
" ]",
# Cross-multiplied, never a plain ratio of the two shares: an hour that placed nobody in the
# region makes that ratio 0/0 or x/0, and MQL drops the row instead of yielding a number, so
# the whole series vanishes before the other clauses run. That hour is the worst skew there
# is - every desktop asking for a region the director is putting nobody in - and it happens
# whenever the region is drained, fenced, or at capacity. Both forms were run read-only
# against production surrogates with a zero denominator: the ratio returned no rows, this
# returned the series with the condition true.
"| condition hint_share > 2 * placement_share && hint_share - placement_share > 0.15 '1' && hinted_requests > 500 '1'"
]
))
relay_custom_alerts = {
connection_headroom = {
pages_oncall = true
@@ -201,7 +289,9 @@ locals {
}
resource "google_logging_metric" "relay_snapshot" {
for_each = local.relay_runtime_metrics
# Region-request metrics ride the same event and shape; merging adds map entries only, so the
# existing metric instances are untouched (a label change, not a new key, is what recreates them).
for_each = merge(local.relay_runtime_metrics, local.relay_region_share_metrics)
project = var.project_id
name = "orca_relay_${each.key}"
@@ -211,14 +301,14 @@ resource "google_logging_metric" "relay_snapshot" {
label_extractors = {
role = "EXTRACT(jsonPayload.role)"
cell_id = "EXTRACT(jsonPayload.cellId)"
# No region label: adding one replaces all 21 live metrics (label change = delete+create),
# No region label: adding one replaces all 42 live metrics (label change = delete+create),
# which resets history and blanks the relay alert policies during the swap.
}
metric_descriptor {
metric_kind = "DELTA"
value_type = "DISTRIBUTION"
unit = contains(["sql_latency_ms", "control_renewal_latency_ms_p50", "control_renewal_latency_ms_p95", "control_renewal_latency_ms_max", "http_latency_ms", "event_loop_ms_p99", "db_oldest_wait_ms", "db_wait_ms_max"], each.key) ? "ms" : each.key == "queued_bytes" || each.key == "heap_used_bytes" || each.key == "forwarded_bytes" ? "By" : "1"
unit = contains(["sql_latency_ms", "control_rtt_ms_p50", "control_rtt_ms_p95", "control_rtt_ms_max", "client_accept_total_ms_p50", "client_accept_total_ms_p95", "client_accept_total_ms_max", "client_accept_assignment_ms_p95", "client_accept_credential_ms_p95", "client_accept_activity_ms_p95", "client_accept_attach_ms_p95", "client_accept_basis_ms_p95", "control_renewal_latency_ms_p50", "control_renewal_latency_ms_p95", "control_renewal_latency_ms_max", "http_latency_ms", "event_loop_ms_p99", "db_oldest_wait_ms", "db_wait_ms_max"], each.key) ? "ms" : each.key == "queued_bytes" || each.key == "heap_used_bytes" || each.key == "forwarded_bytes" ? "By" : "1"
labels {
key = "role"
@@ -672,6 +762,128 @@ resource "google_monitoring_alert_policy" "relay_cell_process_exit" {
depends_on = [google_logging_metric.relay_incident]
}
# Why: nothing fired while US desktops sat on asia-east2 cells for weeks in 2026-08. The two
# per-cell policies below read that as distance, and the fleet-wide one reads it as a bad region
# hint. All three are MQL because each needs the sum of a DELTA DISTRIBUTION as a volume floor,
# and the only scalar aligners a `condition_threshold` can apply to a distribution are percentiles.
# `join` is an inner join and the relay omits its percentile fields on an empty interval, so an
# idle cell drops out rather than alerting on nothing. The per-cell arms fetch `gce_instance`
# only: production runs no Cloud Run cells (`relay_cells` is empty), and a future one would need
# its own arm here. None of the metrics these query exist in the project yet, so what was checked
# against production is the query shape: the same MQL run over existing metrics of the same kind
# confirmed the distribution sum, the join arity, the unit literals, and the condition clause.
resource "google_monitoring_alert_policy" "relay_far_cell_accept_latency" {
project = var.project_id
display_name = "Orca Relay: far-cell phone accept latency"
combiner = "OR"
enabled = true
notification_channels = var.relay_alert_notification_channels
conditions {
display_name = "Phone accept p95 above 2 s for 15 minutes"
condition_monitoring_query_language {
# percentile(..., 50) over the window, not max: the published value is already a p95, so the
# median of the interval p95s reads as sustained slowness instead of one bad 30-second flush.
query = <<-EOT
{
fetch gce_instance::logging.googleapis.com/user/orca_relay_client_accept_total_ms_p95
| align delta(15m) | every 15m
| group_by [metric.cell_id], [accept_p95_ms: percentile(value.orca_relay_client_accept_total_ms_p95, 50)]
;
fetch gce_instance::logging.googleapis.com/user/orca_relay_client_accepts_completed
| align delta(15m) | every 15m
| group_by [metric.cell_id], [accepts: sum(value.orca_relay_client_accepts_completed)]
}
| join
| condition accept_p95_ms > 2000 'ms' && accepts >= 20 '1'
EOT
duration = "0s"
trigger {
count = 1
}
}
}
documentation {
content = "Phones on this cell are taking over two seconds to reach relay-hello. Measured separation: an in-region accept completes in 0.3-0.6 s and a cross-Pacific one in 5-10 s, so 2 s sits well outside in-region noise and well below the far-cell floor. The 20-accept floor over 15 minutes keeps a single slow accept on a quiet cell from paging. Check which regions the cell's hosts are actually in before touching capacity: the 2026-08 cause was desktops requesting the wrong region, not a slow cell. Read the per-stage `orca_relay_client_accept_*_ms_p95` metrics to separate distance from assignment, credential, or attach work."
mime_type = "text/markdown"
}
depends_on = [google_logging_metric.relay_snapshot]
}
resource "google_monitoring_alert_policy" "relay_cell_control_rtt" {
project = var.project_id
display_name = "Orca Relay: cell control round trip"
combiner = "OR"
enabled = true
notification_channels = var.relay_alert_notification_channels
conditions {
display_name = "Control ping p50 above 150 ms for an hour"
condition_monitoring_query_language {
# p50 only. The desktop echoes the pong on its main thread, so the published p95 and max
# track renderer stalls, not distance; the median is the only column that reads as distance.
query = <<-EOT
{
fetch gce_instance::logging.googleapis.com/user/orca_relay_control_rtt_ms_p50
| align delta(1h) | every 1h
| group_by [metric.cell_id], [control_rtt_p50_ms: percentile(value.orca_relay_control_rtt_ms_p50, 50)]
;
fetch gce_instance::logging.googleapis.com/user/orca_relay_control_rtt_samples
| align delta(1h) | every 1h
| group_by [metric.cell_id], [samples: sum(value.orca_relay_control_rtt_samples)]
}
| join
| condition control_rtt_p50_ms > 150 'ms' && samples >= 500 '1'
EOT
duration = "0s"
trigger {
count = 1
}
}
}
documentation {
content = "The median desktop on this cell is more than 150 ms away from it, which is a mis-homed population rather than a cell fault: an in-region control ping is tens of milliseconds and a US desktop on an asia-east2 cell is 200 ms or more. This is the signal that was missing while roughly 226 of 332 hosts on the asia cells were non-APAC for weeks in 2026-08. Confirm with the assignment table which regions those hosts requested, then rehome; do not restart or drain the cell on this alert alone. The 500-sample floor is about two continuously connected hosts at the 15-second control ping, so a nearly idle cell cannot alert on one desktop. Tuning risk: EU desktops on us-central1 sit at 100-130 ms, so a cell whose population is mostly European can approach 150 ms while correctly homed. Check where the hosts are before treating a first breach as mis-homing, and raise the bar only with that evidence."
mime_type = "text/markdown"
}
depends_on = [google_logging_metric.relay_snapshot]
}
resource "google_monitoring_alert_policy" "relay_region_hint_skew" {
project = var.project_id
display_name = "Orca Relay: region hint skew"
combiner = "OR"
enabled = true
notification_channels = var.relay_alert_notification_channels
conditions {
display_name = "asia-east2 hint share above 2x its placement share for an hour"
condition_monitoring_query_language {
query = local.relay_region_hint_skew_query
duration = "0s"
trigger {
count = 1
}
}
}
documentation {
content = "Desktops are asking the director for asia-east2 far more often than the director actually places them there, which is what silently homed US desktops on asia cells through 2026-08. The alert compares two shares of the same hour and never an absolute share, because an absolute bar is wrong at both ends: measured over twelve hours on 2026-09-07, while the desktop region probe was still mis-picking, asia-east2 was 33.8% of the 33,800 hinted requests but only 7.9% of the 45,364 assignments, and once the probe is fixed the genuine APAC share will climb past any fixed bar that would have caught this. Divergence was 4.27x with a 25.9-point gap, so the 2x and 15-point bars sit well inside the broken state and well outside a healthy one. `unhinted` requests are excluded from the denominator: they were 27% of all requests, and a client change that always sends a hint would move this number without any behaviour changing. Expect this to stay lit until the mis-homed backlog is rehomed, because sticky assignment never re-consults the hint, so a desktop already on an asia cell keeps being placed there no matter what it now asks for. Investigate the desktop region probe first, not relay placement."
mime_type = "text/markdown"
}
depends_on = [google_logging_metric.relay_snapshot]
}
# Why: the four signals that had to be assembled by hand during the 2026-09-04 incident.
resource "google_monitoring_dashboard" "relay_incident" {
project = var.project_id
+1 -1
View File
@@ -245,7 +245,7 @@ variable "relay_regional_placement_enabled" {
variable "relay_region_rehome_source_cell_ids" {
type = set(string)
description = "Reviewed US Relay cells allowed to advertise and accept the regional rehome source protocol."
description = "Reviewed Relay cells, in any configured region, allowed to advertise and accept the regional rehome source protocol."
default = []
}
+1 -1
View File
@@ -20,7 +20,7 @@
"load:relay:model": "node dev/scripts/run-relay-load-model.mjs",
"load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs",
"ops:relay": "pnpm --filter @orca-cloud/relay-ops dev",
"pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-admission-workflow.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs",
"pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-admission-workflow.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs",
"test": "pnpm -r test && node --test dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-regional-rehome-workflow.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs",
"typecheck": "pnpm -r typecheck"
},
@@ -6,7 +6,10 @@ import {
HostChallengeSchema,
HostDataAuthSchema,
HostHelloAckSchema,
HostHelloSchema
HostHelloSchema,
parseRelayHostCapabilities,
RELAY_HOST_CAPABILITIES_HEADER,
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS
} from './control-messages.js'
import {
DeviceCredentialInstallSchema,
@@ -345,3 +348,47 @@ describe('relay protocol contract', () => {
).toBe(false)
})
})
describe('pending connection details capability', () => {
it('reads a pending entry with or without the stated kind and device', () => {
const ack = {
v: 1 as const,
generation: 3,
controlResumeSecret: 'R'.repeat(43),
leaseExpiresAt: 1_800_000_000_000,
activeConnIds: []
}
const identifiers = { connId: 'conn-1', connTicket: 'T'.repeat(43) }
expect(HostHelloAckSchema.safeParse({ ...ack, pendingConns: [identifiers] }).success).toBe(true)
expect(
HostHelloAckSchema.safeParse({
...ack,
pendingConns: [{ ...identifiers, kind: 'resume', relayDeviceId: 'device-1' }]
}).success
).toBe(true)
// Still strict otherwise: an unannounced key must not slip through as data.
expect(
HostHelloAckSchema.safeParse({
...ack,
pendingConns: [{ ...identifiers, reservationId: 'injected' }]
}).success
).toBe(false)
})
it('pins the header and token the desktop mirrors by hand', () => {
// The desktop cannot import this package; drift silently disables the
// feature, so both literals are asserted on each side.
expect(RELAY_HOST_CAPABILITIES_HEADER).toBe('x-orca-host-capabilities')
expect(RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS).toBe('pending-conn-details')
})
it('reads the advertised capabilities from a control upgrade header', () => {
expect(
parseRelayHostCapabilities(` ${RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS} , future-thing`)
).toEqual(new Set([RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS, 'future-thing']))
// A host that predates the header sends nothing; absence is never capable.
expect(parseRelayHostCapabilities(undefined).size).toBe(0)
expect(parseRelayHostCapabilities('').size).toBe(0)
expect(parseRelayHostCapabilities('x'.repeat(65)).size).toBe(0)
})
})
@@ -44,8 +44,35 @@ export const HostChallengeAckSchema = z
.object({ challengeId: OpaqueIdSchema, proofB64: Base6432ByteSchema })
.strict()
// Advertised on the control upgrade rather than in host-hello: HostHelloSchema
// is strict, so a new hello key is refused by every already-deployed cell.
export const RELAY_HOST_CAPABILITIES_HEADER = 'x-orca-host-capabilities'
// The host accepts kind/relayDeviceId on a pendingConns entry. A host that does
// not advertise this parses those entries strictly and would drop the whole ack.
export const RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS = 'pending-conn-details'
export function parseRelayHostCapabilities(
header: string | string[] | undefined
): ReadonlySet<string> {
const raw = Array.isArray(header) ? header.join(',') : (header ?? '')
return new Set(
raw
.split(',')
.map((token) => token.trim())
.filter((token) => token.length > 0 && token.length <= 64)
.slice(0, 16)
)
}
// kind/relayDeviceId are optional so an entry stays readable by a host that
// predates them; the cell only emits them to a host that advertised support.
const PendingConnectionSchema = z
.object({ connId: OpaqueIdSchema, connTicket: Base64Url32ByteSchema })
.object({
connId: OpaqueIdSchema,
connTicket: Base64Url32ByteSchema,
kind: ConnectionKindSchema.optional(),
relayDeviceId: OpaqueIdSchema.optional()
})
.strict()
export const HostHelloAckSchema = z
@@ -8,6 +8,15 @@ export type RelayRegion = z.infer<typeof RelayRegionSchema>
export const RELAY_DEFAULT_REGION: RelayRegion = 'us-central1'
// Field-name segment for the flat per-region runtime counters, spelled out rather than derived so
// the Terraform side can hold the same literal and a test can compare the two. `satisfies` makes a
// new region a compile error here, which is the point: a region with no segment would silently
// drop out of the region-skew alert's denominators.
export const RELAY_REGION_METRIC_SEGMENTS = {
'us-central1': 'UsCentral1',
'asia-east2': 'AsiaEast2'
} as const satisfies Record<RelayRegion, string>
const RelayProbeOriginSchema = z.string().url().max(2_048).refine(isCanonicalHttpsOrigin)
export const RelayRegionCatalogResponseSchema = z
+69 -38
View File
@@ -603,7 +603,7 @@ index 7b4b9e1f990fbf95b51528bb56dc9717f5b87532..2ae787c5bd4f3eba470584dc658a01a5
}
#endif
diff --git a/src/win/conpty.cc b/src/win/conpty.cc
index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c97209248e 100644
index 7b286d3d644c26141df516929703aa6e129df4b2..4b06d18576c807c3d1181a7bd714140c6678cf86 100644
--- a/src/win/conpty.cc
+++ b/src/win/conpty.cc
@@ -18,6 +18,7 @@
@@ -614,7 +614,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
#include <vector>
#include <Windows.h>
#include <strsafe.h>
@@ -44,12 +45,39 @@ struct pty_baton {
@@ -44,12 +45,40 @@ struct pty_baton {
HANDLE hOut;
HPCON hpc;
@@ -630,6 +630,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
+ // refused to create or assign one (an outer job without breakaway rights),
+ // in which case callers fall back to their pre-job behaviour.
+ HANDLE hJob = nullptr;
+ bool allowJobBreakaway = true;
+
+ // Orca: teardown needs BOTH the shell's death and an explicit kill() before
+ // the baton can be freed, so each side records that it has run. Whichever
@@ -655,7 +656,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
static volatile LONG ptyCounter;
static pty_baton* get_pty_baton(int id) {
@@ -102,8 +130,31 @@ void SetupExitCallback(Napi::Env env, Napi::Function cb, pty_baton* baton) {
@@ -102,8 +131,31 @@ void SetupExitCallback(Napi::Env env, Napi::Function cb, pty_baton* baton) {
// Get process exit code.
GetExitCodeProcess(baton->hShell, (LPDWORD)(&exit_event->exit_code));
// Clean up handles
@@ -689,7 +690,36 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
auto status = tsfn.BlockingCall(exit_event, callback); // In main thread
switch (status) {
@@ -409,6 +460,15 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
@@ -242,6 +294,20 @@
return HRESULT_FROM_WIN32(GetLastError());
}
+// Cygwin and MSYS request breakaway for every child whenever the job allows it,
+// so their shells need one that does not. The runtime DLL on the exe's search
+// path is the signal; Git for Windows ships bash.exe in bin\ beside usr\bin\.
+static bool usesCygwinRuntime(const std::wstring& shellpath) {
+ const size_t separator = shellpath.find_last_of(L"\\/");
+ if (separator == std::wstring::npos) return false;
+ const std::wstring directory = shellpath.substr(0, separator + 1);
+ for (const wchar_t* dll : {L"msys-2.0.dll", L"cygwin1.dll"}) {
+ if (path_util::file_exists(directory + dll) ||
+ path_util::file_exists(directory + L"..\\usr\\bin\\" + dll)) return true;
+ }
+ return false;
+}
+
static Napi::Value PtyStartProcess(const Napi::CallbackInfo& info) {
Napi::Env env(info.Env());
Napi::HandleScope scope(env);
@@ -303,6 +369,7 @@
marshal.Set("pty", Napi::Number::New(env, ptyId));
ptyHandles.emplace_back(
std::make_unique<pty_baton>(ptyId, hIn, hOut, hpc));
+ ptyHandles.back()->allowJobBreakaway = !usesCygwinRuntime(shellpath);
} else {
throw Napi::Error::New(env, "Cannot launch conpty");
}
@@ -409,6 +476,15 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
throw errorWithCode(info, "UpdateProcThreadAttribute failed");
}
@@ -705,7 +735,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
PROCESS_INFORMATION piClient{};
fSuccess = !!CreateProcessW(
nullptr,
@@ -416,7 +476,10 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
@@ -416,7 +492,10 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
nullptr, // lpProcessAttributes
nullptr, // lpThreadAttributes
false, // bInheritHandles VERY IMPORTANT that this is false
@@ -717,7 +747,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
envArg, // lpEnvironment
mutableCwd.get(), // lpCurrentDirectory
&siEx.StartupInfo, // lpStartupInfo
@@ -426,8 +489,47 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
@@ -426,8 +505,48 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
throw errorWithCode(info, "Cannot create process");
}
@@ -735,13 +765,14 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
+ // EXPLICIT teardown exact, not to redefine what a clean exit means.
+ HANDLE hJob = CreateJobObjectW(nullptr, nullptr);
+ if (hJob != nullptr) {
+ // Why BREAKAWAY_OK and not a bare job: with no limits set, a child asking
+ // for CREATE_BREAKAWAY_FROM_JOB is refused with ERROR_ACCESS_DENIED.
+ // Installers, msiexec and some updater and service-control paths spawn that
+ // way deliberately, so a bare job breaks them ONLY inside an Orca terminal.
+ // With this flag a child has to ask, so ordinary descendants stay owned.
+ // Native shells retain explicit breakaway for installers and updaters.
+ // Cygwin/MSYS shells take it automatically for ordinary children whenever
+ // this flag is present, so they get strict per-PTY membership instead.
+ // Explicit breakaway requests inside such a pane are consequently denied;
+ // ordinary backgrounding and clean shell exit remain supported.
+ JOBOBJECT_EXTENDED_LIMIT_INFORMATION jobLimits{};
+ jobLimits.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_BREAKAWAY_OK;
+ jobLimits.BasicLimitInformation.LimitFlags =
+ handle->allowJobBreakaway ? JOB_OBJECT_LIMIT_BREAKAWAY_OK : 0;
+ if (!SetInformationJobObject(hJob, JobObjectExtendedLimitInformation, &jobLimits, sizeof(jobLimits)) ||
+ !AssignProcessToJobObject(hJob, piClient.hProcess)) {
+ // Why tolerate failure: an outer job without JOB_OBJECT_LIMIT_BREAKAWAY_OK
@@ -767,7 +798,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
if (useConptyDll && fLoadedDll)
{
PFNRELEASEPSEUDOCONSOLE const pfnReleasePseudoConsole = (PFNRELEASEPSEUDOCONSOLE)GetProcAddress(
@@ -440,6 +542,8 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
@@ -440,6 +559,8 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
// Update handle
handle->hShell = piClient.hProcess;
@@ -776,11 +807,16 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
// Close the thread handle to avoid resource leak
CloseHandle(piClient.hThread);
@@ -544,29 +648,215 @@ static Napi::Value PtyKill(const Napi::CallbackInfo& info) {
@@ -544,27 +665,213 @@ static Napi::Value PtyKill(const Napi::CallbackInfo& info) {
int id = info[0].As<Napi::Number>().Int32Value();
const bool useConptyDll = info[1].As<Napi::Boolean>().Value();
- const pty_baton* handle = get_pty_baton(id);
-
- if (handle != nullptr) {
- HANDLE hLibrary = LoadConptyDll(info, useConptyDll);
- bool fLoadedDll = hLibrary != nullptr;
- if (fLoadedDll)
+ // Orca: resolve the DLL BEFORE touching any baton state, for the same reason
+ // PtyConnect does it before creating anything. LoadConptyDll throws when
+ // conpty.dll is missing, and a throw after consoleClosed was set would strand
@@ -794,18 +830,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
+ (HMODULE)hLibrary,
+ useConptyDll ? "ConptyClosePseudoConsole" : "ClosePseudoConsole");
+ }
- if (handle != nullptr) {
- HANDLE hLibrary = LoadConptyDll(info, useConptyDll);
- bool fLoadedDll = hLibrary != nullptr;
- if (fLoadedDll)
- {
- PFNCLOSEPSEUDOCONSOLE const pfnClosePseudoConsole = (PFNCLOSEPSEUDOCONSOLE)GetProcAddress(
- (HMODULE)hLibrary,
- useConptyDll ? "ConptyClosePseudoConsole" : "ClosePseudoConsole");
- if (pfnClosePseudoConsole)
- {
- pfnClosePseudoConsole(handle->hpc);
+
+ // Orca: the baton now outlives the shell, so this runs on a self-exited pty
+ // too -- that is the whole point. Take what we need under the lock: the
+ // watcher thread nulls hShell the moment the shell dies, and TerminateProcess
@@ -841,18 +866,26 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
+ const bool removed = remove_pty_baton(id);
+ assert(removed);
+ (void)removed;
}
+ }
+ // Else the shell is still running and the watcher frees the baton.
}
- if (useConptyDll) {
- TerminateProcess(handle->hShell, 1);
+ }
+ }
+
+ // Why outside the lock: ClosePseudoConsole blocks until the conout side has
+ // drained, and the watcher must be able to take the lock while it does.
+ if (owed) {
+ if (pfnClosePseudoConsole)
+ {
{
- PFNCLOSEPSEUDOCONSOLE const pfnClosePseudoConsole = (PFNCLOSEPSEUDOCONSOLE)GetProcAddress(
- (HMODULE)hLibrary,
- useConptyDll ? "ConptyClosePseudoConsole" : "ClosePseudoConsole");
- if (pfnClosePseudoConsole)
- {
- pfnClosePseudoConsole(handle->hpc);
- }
- }
- if (useConptyDll) {
- TerminateProcess(handle->hShell, 1);
+ pfnClosePseudoConsole(hpc);
+ }
+ if (hShellDup != nullptr) {
@@ -862,8 +895,8 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
}
return env.Undefined();
}
+}
+
+/**
+ * Orca: confirm a baton really is the pty the caller means.
+ *
@@ -1001,12 +1034,10 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
+ }
+ hHostJob = job;
+ return Napi::Boolean::New(env, true);
+}
+
}
/**
* Init
*/
@@ -577,6 +867,9 @@ Napi::Object init(Napi::Env env, Napi::Object exports) {
@@ -577,6 +884,9 @@ Napi::Object init(Napi::Env env, Napi::Object exports) {
exports.Set("resize", Napi::Function::New(env, PtyResize));
exports.Set("clear", Napi::Function::New(env, PtyClear));
exports.Set("kill", Napi::Function::New(env, PtyKill));
+16 -51
View File
@@ -13325,9 +13325,7 @@
},
{
"file": "src/main/runtime/orchestration/mailbox-pointer-stage.test.ts",
"assertions": [
"a refused pointer write drains a delivery parked behind its watermark"
]
"assertions": ["a refused pointer write drains a delivery parked behind its watermark"]
},
{
"file": "src/main/providers/settled-pty-writer-census.test.ts",
@@ -18549,27 +18547,13 @@
"protection": "partial",
"owner": "browser-runtime",
"layer": "electron-packaged",
"surfaces": [
"paired browser placement"
],
"platforms": [
"linux",
"macos",
"windows"
],
"providers": [
"paired-runtime"
],
"coveredPlatforms": [
"linux"
],
"coveredProviders": [
"paired-runtime"
],
"surfaces": ["paired browser placement"],
"platforms": ["linux", "macos", "windows"],
"providers": ["paired-runtime"],
"coveredPlatforms": ["linux"],
"coveredProviders": ["paired-runtime"],
"coverageNotes": "Published Linux 1.4.188 desktop against current source in both directions; scheduled weekly and manually runnable. No required PR check.",
"motivatingLinks": [
"https://github.com/stablyai/orca/actions/runs/34069063016"
],
"motivatingLinks": ["https://github.com/stablyai/orca/actions/runs/34069063016"],
"invariant": "A paired client and host without client-hosted browser capabilities retain server-hosted browser placement across supported version skew.",
"oracle": "Require both existing named browser placement scenarios to pass three times with one attempt, zero skips, zero failures, and no report errors.",
"commands": [
@@ -18593,9 +18577,7 @@
},
{
"file": "config/scripts/verify-packaged-browser-participation.test.mjs",
"assertions": [
"reject missing, substituted, skipped and retried scenarios"
]
"assertions": ["reject missing, substituted, skipped and retried scenarios"]
},
{
"file": "config/scripts/packaged-browser-lane-contract.test.mjs",
@@ -18650,26 +18632,13 @@
"protection": "partial",
"owner": "terminal-input",
"layer": "electron-native-ime-e2e",
"surfaces": [
"native Hangul composition",
"Wayland terminal input"
],
"platforms": [
"linux"
],
"providers": [
"local"
],
"coveredPlatforms": [
"linux"
],
"coveredProviders": [
"local"
],
"surfaces": ["native Hangul composition", "Wayland terminal input"],
"platforms": ["linux"],
"providers": ["local"],
"coveredPlatforms": ["linux"],
"coveredProviders": ["local"],
"coverageNotes": "Ubuntu 22.04 nested GNOME and IBus Hangul drive three complete native executions in GitHub Actions. GNOME owns IBus; daemon and CLI share its default config discovery path.",
"motivatingLinks": [
"https://github.com/stablyai/orca/pull/19174"
],
"motivatingLinks": ["https://github.com/stablyai/orca/pull/19174"],
"invariant": "Typing d k 1 Return through native IBus Hangul delivers exactly 아1 followed by newline without missing, duplicate, or reordered characters.",
"oracle": "Three executions each assert three exact UTF-8 PTY lines. Verify the exact Playwright title, zero skips/retries, each individual native composition receipt, and the nested launch Wayland flag.",
"commands": [
@@ -18685,15 +18654,11 @@
"assertionRefs": [
{
"file": "tests/e2e/terminal-hangul-terminating-digit-native.spec.ts",
"assertions": [
"a digit typed right after a Hangul syllable reaches the pty"
]
"assertions": ["a digit typed right after a Hangul syllable reaches the pty"]
},
{
"file": "config/scripts/terminal-ime-e2e-workflow.test.mjs",
"assertions": [
"runs native Wayland independently with CJK fonts and retained evidence"
]
"assertions": ["runs native Wayland independently with CJK fonts and retained evidence"]
}
],
"evidenceRuns": [
@@ -38,6 +38,16 @@ const SUPPRESSED_REACT_DOCTOR_DIAGNOSTICS = new Map([
new Set([
'src/renderer/src/components/editor/combined-diff/review-controls/use-combined-diff-view-preferences.ts'
])
],
[
// The rule wants one named handle cleared by name. Both startup effects arm a variable number
// of refresh timers, every one of them through addTimer into `timers`, which their cleanups
// clear -- a shape the rule reports whether the handles live in an array, a Set, or a nested
// helper. The finding predates this list; it surfaced when the effect body changed. This map
// keys on file, not line, so the entry covers both effects in it; nothing else in the file
// arms a timer, so widening it further is the only alternative, not a narrower option.
'react-doctor(effect-needs-cleanup)',
new Set(['mobile/src/session/use-mobile-session-startup.ts'])
]
])
+1
View File
@@ -224,6 +224,7 @@ const WINDOWS_PACKAGE_TESTS = [
'src/main/agent-hooks/windows-hook-payload-delivery.test.ts',
'src/main/agent-hooks/windows-direct-cmd-hook-command.test.ts',
'src/main/windows/windows-pty-job.win32.test.ts',
'src/main/windows/windows-msys-job.win32.test.ts',
'src/main/windows/windows-host-job.win32.test.ts',
'src/main/windows/windows-process-tree-command-line-patch.test.ts',
'src/main/windows/windows-process-table-native-addon.win32.test.ts',
@@ -173,6 +173,28 @@ describe('rebuild-native-deps patched node-pty rebuild', () => {
}
})
it('refuses a Windows rebuild when the process creation-time patch is missing', () => {
const projectDir = mkTempProject()
try {
writeFakeUsableElectronPackage(projectDir, { platform: 'win32' })
writeFakeElectronRebuild(projectDir)
writeFakeNodePtyConptyPayload(projectDir, 'x64')
writeFakeWindowsProcessTreeWithNodeAddonApi(projectDir, { creationTimePatchApplied: false })
const result = runRebuildScript(
projectDir,
{ npm_config_platform: 'win32', npm_config_arch: 'x64' },
['--platform=win32', '--arch=x64', '--force']
)
expect(result.status).not.toBe(0)
expect(result.stderr).toContain('process creation-time patch')
} finally {
removeTreeSync(projectDir)
}
})
it('restores the ConPTY runtime payload after a Windows Electron rebuild', () => {
const projectDir = mkTempProject()
@@ -374,13 +374,18 @@ export function writeFakeWindowsProcessTree(projectDir) {
export function writeFakeWindowsProcessTreeWithNodeAddonApi(
projectDir,
{ commandLinePatchApplied = true } = {}
{ commandLinePatchApplied = true, creationTimePatchApplied = true } = {}
) {
const processTreeDir = join(projectDir, 'node_modules', '@vscode', 'windows-process-tree')
const nodeAddonApiDir = join(processTreeDir, 'node_modules', 'node-addon-api')
mkdirSync(nodeAddonApiDir, { recursive: true })
writeFileSync(join(processTreeDir, 'package.json'), '{"dependencies":{"node-addon-api":"*"}}\n')
writeFileSync(join(processTreeDir, 'index.js'), 'module.exports = {}\n')
writeFileSync(
join(processTreeDir, 'index.js'),
creationTimePatchApplied
? 'exports.ProcessDataFlag = { None: 0, Memory: 1, CommandLine: 2, CreationTime: 4 }\n'
: 'exports.ProcessDataFlag = { None: 0, Memory: 1, CommandLine: 2 }\n'
)
mkdirSync(join(processTreeDir, 'src'), { recursive: true })
writeFileSync(
join(processTreeDir, 'src', 'process_commandline.cc'),
@@ -388,6 +393,36 @@ export function writeFakeWindowsProcessTreeWithNodeAddonApi(
? '// kProcessCommandLineInformation = 60\n'
: unpatchedWindowsProcessTreeCommandLineSource()
)
writeFileSync(
join(processTreeDir, 'src', 'process.h'),
creationTimePatchApplied
? 'enum ProcessDataFlags { NONE = 0, MEMORY = 1, COMMANDLINE = 2, CREATIONTIME = 4 };\nULONGLONG creationTimeMs;\n'
: 'enum ProcessDataFlags { NONE = 0, MEMORY = 1, COMMANDLINE = 2 };\n'
)
writeFileSync(
join(processTreeDir, 'src', 'process.cc'),
creationTimePatchApplied
? 'GetProcessCreationTime(pinfo);\nGetProcessTimes(hProcess, &creationTime, &exitTime, &kernelTime, &userTime);\n'
: 'GetProcessMemoryUsage(pinfo);\n'
)
writeFileSync(
join(processTreeDir, 'src', 'process_worker.cc'),
creationTimePatchApplied ? 'object.Set("creationTimeMs", process.creationTimeMs);\n' : '\n'
)
mkdirSync(join(processTreeDir, 'lib'), { recursive: true })
writeFileSync(
join(processTreeDir, 'lib', 'index.js'),
creationTimePatchApplied ? 'exports.ProcessDataFlag["CreationTime"] = 4;\n' : '\n'
)
writeFileSync(
join(processTreeDir, 'lib', 'index.ts'),
creationTimePatchApplied ? 'export enum ProcessDataFlag { CreationTime = 4 }\n' : '\n'
)
mkdirSync(join(processTreeDir, 'typings'), { recursive: true })
writeFileSync(
join(processTreeDir, 'typings', 'windows-process-tree.d.ts'),
creationTimePatchApplied ? 'creationTimeMs?: number\n' : '\n'
)
writeFileSync(join(nodeAddonApiDir, 'package.json'), '{"name":"node-addon-api"}\n')
writeFileSync(join(nodeAddonApiDir, 'napi.h'), '// napi.h\n')
writeFileSync(join(nodeAddonApiDir, 'napi-inl.h'), '// napi-inl.h\n')
@@ -33,6 +33,17 @@ export const WINDOWS_PROCESS_TREE_PATCH_PATH = join(
/** Only the patched reader defines this; the upstream one walks the PEB. */
const COMMAND_LINE_PATCH_MARKER = 'kProcessCommandLineInformation'
const CREATION_TIME_PATCH_MARKERS = [
['src/process.h', 'CREATIONTIME = 4'],
['src/process.h', 'ULONGLONG creationTimeMs'],
['src/process.cc', 'GetProcessCreationTime(pinfo)'],
['src/process.cc', 'GetProcessTimes(hProcess, &creationTime'],
['src/process_worker.cc', 'object.Set("creationTimeMs"'],
['lib/index.js', '["CreationTime"] = 4'],
['lib/index.ts', 'CreationTime = 4'],
['typings/windows-process-tree.d.ts', 'creationTimeMs?: number']
]
export const WINDOWS_PROCESS_TREE_NODE_ADDON_API_HEADERS = [
'napi.h',
'napi-inl.h',
@@ -83,6 +94,36 @@ export function inspectWindowsProcessTreeAddon(addonPath) {
return readFileSync(addonPath).includes(FLAGGED_IMPORT) ? 'unpatched' : 'clean'
}
export function assertWindowsProcessTreeCreationTimePatch(
packageDir = WINDOWS_PROCESS_TREE_PACKAGE_DIR
) {
for (const [relativePath, expected] of CREATION_TIME_PATCH_MARKERS) {
const filePath = join(packageDir, relativePath)
if (!existsSync(filePath)) {
throw new Error(
`${filePath} is missing, so the process creation-time patch cannot be verified. ` +
'Run pnpm install.'
)
}
if (!readFileSync(filePath, 'utf8').includes(expected)) {
throw new Error(
`${relativePath} does not contain the process creation-time patch (${expected}). ` +
'Run pnpm install.'
)
}
}
}
export function assertWindowsProcessTreeRuntimeCreationTime(windowsProcessTree) {
if (windowsProcessTree?.ProcessDataFlag?.CreationTime !== 4) {
throw new Error(
'@vscode/windows-process-tree does not expose ProcessDataFlag.CreationTime, so native ' +
'Windows structured agent-session process ownership cannot be PID-reuse safe. Rebuild it ' +
'(pnpm run rebuild:electron) rather than using the published prebuild.'
)
}
}
/**
* Refuse to compile or load the upstream command-line reader.
*
@@ -159,6 +200,7 @@ export function ensureWindowsProcessTreeCommandLinePatch(
rmSync(windowsProcessTreeAddonPath(packageDir), { force: true })
repaired = true
}
assertWindowsProcessTreeCreationTimePatch(packageDir)
return repaired
}
@@ -12,12 +12,15 @@ import { tmpdir } from 'node:os'
import { join, resolve } from 'node:path'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import {
assertWindowsProcessTreeCreationTimePatch,
assertWindowsProcessTreeRuntimeCreationTime,
inspectWindowsProcessTreeAddon,
nodeGypRebuildInvocation,
stageWindowsProcessTreeNodeAddonApiHeaders,
WINDOWS_PROCESS_TREE_NODE_ADDON_API_HEADERS,
WINDOWS_PROCESS_TREE_PACKAGE_DIR
} from './windows-process-tree-gyp-rebuild.mjs'
import { writeFakeWindowsProcessTreeWithNodeAddonApi } from './rebuild-native-deps-test-fixtures.mjs'
describe('windows-process-tree node-gyp rebuild', () => {
it("resolves node-addon-api's gyp target from the rebuild cwd", () => {
@@ -97,3 +100,47 @@ describe('inspecting a compiled windows-process-tree addon', () => {
expect(inspectWindowsProcessTreeAddon(staged)).toBe('unpatched')
})
})
describe('windows-process-tree CreationTime patch assertion', () => {
let dir
beforeEach(() => {
dir = mkdtempSync(join(tmpdir(), 'orca-windows-process-tree-creation-time-'))
})
afterEach(() => {
rmSync(dir, { recursive: true, force: true })
})
it('accepts a package whose source and JS surfaces expose process creation time', () => {
writeFakeWindowsProcessTreeWithNodeAddonApi(dir)
expect(() =>
assertWindowsProcessTreeCreationTimePatch(
join(dir, 'node_modules', '@vscode', 'windows-process-tree')
)
).not.toThrow()
})
it('rejects a package missing the process creation-time patch', () => {
writeFakeWindowsProcessTreeWithNodeAddonApi(dir, { creationTimePatchApplied: false })
expect(() =>
assertWindowsProcessTreeCreationTimePatch(
join(dir, 'node_modules', '@vscode', 'windows-process-tree')
)
).toThrow('process creation-time patch')
})
it('requires the runtime ProcessDataFlag.CreationTime enum', () => {
expect(() =>
assertWindowsProcessTreeRuntimeCreationTime({
ProcessDataFlag: { None: 0, Memory: 1, CommandLine: 2, CreationTime: 4 }
})
).not.toThrow()
expect(() =>
assertWindowsProcessTreeRuntimeCreationTime({
ProcessDataFlag: { None: 0, Memory: 1, CommandLine: 2 }
})
).toThrow('ProcessDataFlag.CreationTime')
})
})
+1
View File
@@ -32,6 +32,7 @@
"../src/main/codex/codex-app-server-capability-cache.ts",
"../src/main/codex/codex-app-server-capability-signal.ts",
"../src/main/codex/codex-app-server-client.ts",
"../src/main/codex/codex-app-server-process-tree-kill.ts",
"../src/main/codex/codex-app-server-record-reader.ts",
"../src/main/codex/codex-app-server-session.ts",
"../src/main/codex/codex-config-mirror.ts",
+21 -2
View File
@@ -2,8 +2,27 @@
Orca selects a Relay region in the Electron main process before requesting a new assignment. The
director publishes an allowlisted region catalog containing only HTTPS cell subdomains of that
director; Orca takes three bounded `/health` latency samples per region and caches the stable choice
for 24 hours. A cached region changes only when the alternative is materially faster.
director. Orca discards one warm-up `/health` request per probe origin — a cold request pays TCP and
TLS setup that can exceed the round trip it measures — then takes three bounded samples and compares
regions by their minimum. A wide spread still rejects a region, but only a genuinely flapping one.
The stable choice is cached for 24 hours, and a cached region changes only when the alternative is
materially faster.
A region wins only against a measured competitor. If any region in the catalog is rejected or cannot
be measured, Orca sends no hint rather than selecting the sole survivor. Sending no hint is not
neutral placement: the director assigns `preferredRegion ?? RELAY_DEFAULT_REGION`, and the default
is `us-central1`. So an `asia-east2` user whose `us-central1` probe fails or flaps once is placed in
`us-central1` for that refresh. That trade is accepted because the relay database is
`us-central1`-only, and it is bounded: the withheld hint is cached for one hour, not the 24 hours a
chosen region gets, so the next hour re-measures. An origin that fails its warm-up probe is dropped
before the sampling rounds, so an unreachable region costs one probe timeout rather than four.
After a control socket registers, Orca probes the cell it actually landed on, once per cell URL per
process. The cache is deleted only when it names a region other than the best measured one and the
assigned cell is more than three times slower than that region — a far cell under a cache that still
names the best region means the director declined the hint, and re-measuring would return the same
answer. Self-heal skips an absent, expired, or no-hint cache, and never runs under
`ORCA_RELAY_REGION_OVERRIDE`.
The assignment request sends only `preferredRegion`. It does not send latency, IP address, country,
pairing data, or credentials. Catalog, probe, and cache failures fall back to an assignment without
+31 -31
View File
@@ -31,12 +31,12 @@ administrators can do about it.
Four independent evidence clusters, from six incidents:
| Cluster | Incidents | Evidence |
| ----------------- | --------- | -------------------------------------------------------------------------------------------------------------------- |
| **Update** | A, B, C | `orca-windows-setup.exe``old-uninstaller.exe`, `Uninstall Orca.exe` (electron-builder generates these; they are in no repo file) |
| Cluster | Incidents | Evidence |
| ----------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------- |
| **Update** | A, B, C | `orca-windows-setup.exe``old-uninstaller.exe`, `Uninstall Orca.exe` (electron-builder generates these; they are in no repo file) |
| **Spawn** | all six | `Orca.exe``orca-terminal-daemon.exe``powershell.exe` / `pwsh.exe` / `cmd.exe` / `reg.exe``claude.exe`, `gh.exe`, `codex.cmd` |
| **Process table** | D | "suspicious memory activity" — `OpenProcess` plus a PEB read against every process on a repeating cadence |
| **Computer use** | E, F | `runtime.ps1`, `computer-sidecar.js`, many `operation.json`, a burst of ~10 short-lived `powershell.exe` |
| **Process table** | D | "suspicious memory activity" — `OpenProcess` plus a PEB read against every process on a repeating cadence |
| **Computer use** | E, F | `runtime.ps1`, `computer-sidecar.js`, many `operation.json`, a burst of ~10 short-lived `powershell.exe` |
Incident E is the one to look at hardest: 5 alerts, 37 evidence items, ATT&CK
**Execution + Collection**, and a description reading _"Screenshots were taken
@@ -143,11 +143,11 @@ PEB fallback to reinstate it — a hooked `ntdll` answering
`STATUS_INVALID_INFO_CLASS` for one target would have flipped a process-wide,
one-way switch back to `PROCESS_VM_READ` on exactly the machines this exists for.
Because the property is the *absence* of an import, it is checkable on the
Because the property is the _absence_ of an import, it is checkable on the
artifact rather than the source: `inspectWindowsProcessTreeAddon()` answers
`clean` / `unpatched` / `missing`, and the rebuild, `ensure-native-runtime.mjs`,
the relay build and `loadWindowsProcessTree()` all key on it. That check is load-
bearing because the published tarball ships a *loadable* prebuilt built from
bearing because the published tarball ships a _loadable_ prebuilt built from
unpatched source, so "it required cleanly" is not evidence.
What to declare to administrators is now one
@@ -180,7 +180,7 @@ Three sites are named in the incident analysis:
`src/shared/setup-agent-sequencing.ts`,
`src/shared/windows-cmd-runner-delayed-launch.ts` and
`src/shared/windows-interactive-login-spawn.ts` each dropped
`-ExecutionPolicy Bypass` as a measured no-op: the policy gates script *files*,
`-ExecutionPolicy Bypass` as a measured no-op: the policy gates script _files_,
never `-EncodedCommand`. Where the bypass was load-bearing it moved in-payload as
a process-scope `Set-ExecutionPolicy` (`setup-agent-sequencing.ts`), which is the
pattern to copy rather than restoring the switch — the switch loses to a GPO
@@ -192,7 +192,7 @@ What remains is `-EncodedCommand` without the bypass: the PTY bootstraps
(`src/main/agent-hooks/windows-powershell-hook-launcher.ts` and its callers
`src/main/agent-hooks/runtime-home-hook-command.ts`,
`src/main/agent-hooks/installer-utils.ts`, and `src/main/claude/hook-settings.ts`
— that last one only as a *fallback* since #18875, see below),
— that last one only as a _fallback_ since #18875, see below),
`src/main/runtime/windows-default-route-interfaces.ts`,
`src/main/runtime/orchestration/setup-completion-signal.ts`,
`src/shared/hermes-startup-query.ts`, and the four ex-bypass sites above.
@@ -224,7 +224,7 @@ denies the analyser the payload it would otherwise clear.
The hook launcher is prior art worth knowing about. #16003 measured, on a
reporting Kaspersky host, that `-WindowStyle Hidden` paired with
`-EncodedCommand` was denied at `CreateProcess` with exit 126 regardless of
payload — `exit 0` was denied too. The fix was to stop *spelling* the flags:
payload — `exit 0` was denied too. The fix was to stop _spelling_ the flags:
`WINDOWS_POWERSHELL_HOOK_SWITCHES` is now just `-NoProfile`, and separately, in
#16576, the execution policy bypass moved in-payload as a process-scope
`Set-ExecutionPolicy` — a real command-line signal reduction, though #16003's
@@ -247,7 +247,7 @@ a quoted token, each `%` is broken with `"^%"`.
The escaping is not decorative. Measured on Windows 11 against a real `.cmd`
shim, `["a b", 'c"d', "e%F%g", "h&i", "j^k"]` came back as `["a b", 'c"d',
"e^%F^%g", "h"]` — the `&` truncated the argument *and* ran the remainder as a
"e^%F^%g", "h"]` — the `&` truncated the argument _and_ ran the remainder as a
command.
**How an EDR reads it:** caret escaping is the canonical obfuscation marker in
@@ -259,7 +259,7 @@ obfuscated-command-line detector is tuned on.
`Orca.exe` → the relocated daemon host (`orca-terminal-daemon.exe` in the builds
these incidents cover, `Orca.exe` since) → a shell → an agent CLI is what a
terminal multiplexer for coding agents *is*. `reg.exe` appears from
terminal multiplexer for coding agents _is_. `reg.exe` appears from
`src/main/win32-utils.ts`,
`src/main/agent-hooks/managed-hook-owner-identity.ts` and
`src/relay/pty-shell-utils.ts` (reading the OpenSSH `DefaultShell`).
@@ -267,7 +267,7 @@ terminal multiplexer for coding agents *is*. `reg.exe` appears from
Nothing here is avoidable in principle. What is controllable is depth and
breadth: every interpreter hop between Orca and the thing the user asked for adds
a scored edge, which is why the shipped doctrine of #15520 and #15595 is to
*shorten the interpreter chain* rather than to hide a window.
_shorten the interpreter chain_ rather than to hide a window.
#18875 is a worked example of that doctrine. The Claude Code lifecycle hook was
registered as `powershell.exe -NoProfile -EncodedCommand <...>` whose entire
@@ -295,7 +295,7 @@ That last clause is the standing assumption of this change, and it is worth
stating plainly because it is **not** measured. `||` parses in Git Bash, cmd.exe
and pwsh, but not in Windows PowerShell 5.1, so the direct shape is correct for
any host that is one of the first three. Claude Code itself is a Git Bash host on
native Windows. What no one here has verified is which host a *compat consumer*
native Windows. What no one here has verified is which host a _compat consumer_
uses: cursor-agent and Devin import `~/.claude/settings.json` and run `command`
through their own launcher (the managed `.cmd` carries a `DEVIN_PROJECT_DIR` skip
for exactly that). If one of them spawns hook strings through Windows PowerShell
@@ -318,12 +318,12 @@ then captures the screen through `Graphics.CopyFromScreen`.
That is four separate high-signal behaviours stacked in one process:
| Behaviour | How it is scored |
| ----------------------------------------------- | ---------------------------------------------------- |
| `Graphics.CopyFromScreen` | **MITRE T1113**, screen capture — Collection tactic |
| `SendInput` synthetic keyboard/mouse | input synthesis against other applications |
| `Add-Type -TypeDefinition` on every operation | MSIL compiled at runtime; incident F's "suspicious MSIL code" |
| One `powershell.exe` per operation | a burst of short-lived interpreters under one parent |
| Behaviour | How it is scored |
| --------------------------------------------- | ------------------------------------------------------------- |
| `Graphics.CopyFromScreen` | **MITRE T1113**, screen capture — Collection tactic |
| `SendInput` synthetic keyboard/mouse | input synthesis against other applications |
| `Add-Type -TypeDefinition` on every operation | MSIL compiled at runtime; incident F's "suspicious MSIL code" |
| One `powershell.exe` per operation | a burst of short-lived interpreters under one parent |
The bottom two rows are the two the incident text named directly, and they are
also the two a persistent runtime host would remove: a long-lived helper compiles
@@ -381,16 +381,16 @@ changed. Check the code before relying on it.
The checklist. On Windows, do not reach for:
| Don't | Instead |
| ----------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- |
| `-ExecutionPolicy Bypass` on the command line | Set the policy in-payload at process scope, as `windows-powershell-hook-launcher.ts` does, or do not run a `.ps1` at all |
| `-EncodedCommand` | A temp `.ps1` with an argument, or no PowerShell hop: prefer a native API or an existing Node path |
| `cmd.exe /c` carrying escaped free text | Spawn the real target directly. `cmd.exe` is only unavoidable for `.cmd`/`.bat`; keep free text out of the line where you can |
| Forking `powershell.exe` to read system state | The native reader — [`windows-process-enumeration.md`](./windows-process-enumeration.md) is the standing rule for the process table |
| A process per operation in a loop | One long-lived helper with a request channel. A burst of short-lived interpreters under one parent is itself the signal |
| `Add-Type -TypeDefinition` at runtime | A precompiled, signed assembly, or a native helper |
| Copying our own image under a different name | Copy it verbatim — [`windows-daemon-host-relocation.md`](./windows-daemon-host-relocation.md) (done for the daemon host) |
| Deriving a script runner from a UI preference | [`windows-setup-shell.md`](./windows-setup-shell.md) — the script declares its own interpreter |
| Don't | Instead |
| --------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- |
| `-ExecutionPolicy Bypass` on the command line | Set the policy in-payload at process scope, as `windows-powershell-hook-launcher.ts` does, or do not run a `.ps1` at all |
| `-EncodedCommand` | A temp `.ps1` with an argument, or no PowerShell hop: prefer a native API or an existing Node path |
| `cmd.exe /c` carrying escaped free text | Spawn the real target directly. `cmd.exe` is only unavoidable for `.cmd`/`.bat`; keep free text out of the line where you can |
| Forking `powershell.exe` to read system state | The native reader — [`windows-process-enumeration.md`](./windows-process-enumeration.md) is the standing rule for the process table |
| A process per operation in a loop | One long-lived helper with a request channel. A burst of short-lived interpreters under one parent is itself the signal |
| `Add-Type -TypeDefinition` at runtime | A precompiled, signed assembly, or a native helper |
| Copying our own image under a different name | Copy it verbatim — [`windows-daemon-host-relocation.md`](./windows-daemon-host-relocation.md) (done for the daemon host) |
| Deriving a script runner from a UI preference | [`windows-setup-shell.md`](./windows-setup-shell.md) — the script declares its own interpreter |
Two framing rules that outlast the table:
@@ -407,7 +407,7 @@ Two framing rules that outlast the table:
This is the single most important operational point, and it is the one most
commonly got wrong. The six incidents are **MDE EDR behavioural alerts**.
Defender Antivirus path exclusions suppress *scan* detections; they do not
Defender Antivirus path exclusions suppress _scan_ detections; they do not
suppress EDR behavioural alerts the same way. Adding
`%LOCALAPPDATA%\Programs\orca\` to the AV exclusion list and expecting the
incidents to stop will not work.
+31 -17
View File
@@ -64,10 +64,10 @@ identity scan opens nothing.
So the module exposes two snapshots, and the row types differ so a cheap caller
cannot read what its flag set did not pay for:
| reader | row type | flags | per-process handles |
| ------------------------------------------ | ---------------------------- | --------------------------- | ------------------- |
| `readWindowsProcessIdentityTable[Fresh]()` | `WindowsProcessIdentityRow` | `None \| CreationTime` | none |
| `readWindowsProcessTable[Fresh]()` | `WindowsProcessRow` | `+ CommandLine` | one `OpenProcess` |
| reader | row type | flags | per-process handles |
| ------------------------------------------ | --------------------------- | ---------------------- | ------------------- |
| `readWindowsProcessIdentityTable[Fresh]()` | `WindowsProcessIdentityRow` | `None \| CreationTime` | none |
| `readWindowsProcessTable[Fresh]()` | `WindowsProcessRow` | `+ CommandLine` | one `OpenProcess` |
`Memory` is requested by neither. Nothing reads a working set off this table —
`windows-process-resource-collector.ts` runs its own sweep because it needs
@@ -103,7 +103,7 @@ only under concurrency.
Nothing else in this module prevents that. Each snapshot cache single-flights
only within itself (`inFlight` is a closure per reader), and the wedge set
latches only *after* a read misses its 3 s deadline, so through the healthy
latches only _after_ a read misses its 3 s deadline, so through the healthy
~12 ms of a scan neither excludes the other. Overlap is the normal state rather
than an edge case: other panes keep polling detailed at 750 ms while a teardown
takes identity snapshots, and `codex-structured-turn-processes.ts` issues fresh
@@ -166,15 +166,15 @@ through `toIdentityRow`, so an identity row carries no command line on any host.
### Which callers need which
| caller | reads | flag set |
| --------------------------------------------- | ------------------ | -------- |
| `windows-agent-foreground-process.ts` | `command` (agent recognition) | detailed |
| `local-workspace-platform-port-scanner.ts` | `command` (port attribution) | detailed |
| `codex-structured-turn-processes.ts` | `command` (turn-process identity) | detailed |
| `structured-tui-process-identity.ts` | `command` (child match) | detailed |
| `windows-pty-root-identity.ts` | `pid` / `ppid` only | identity |
| `agent-session-process-identity-probe.ts` | `creationTimeMs` only | identity |
| `relay/windows-port-scan.ts` | `name` (port owner label) | detailed |
| caller | reads | flag set |
| ------------------------------------------ | --------------------------------- | -------- |
| `windows-agent-foreground-process.ts` | `command` (agent recognition) | detailed |
| `local-workspace-platform-port-scanner.ts` | `command` (port attribution) | detailed |
| `codex-structured-turn-processes.ts` | `command` (turn-process identity) | detailed |
| `structured-tui-process-identity.ts` | `command` (child match) | detailed |
| `windows-pty-root-identity.ts` | `pid` / `ppid` only | identity |
| `agent-session-process-identity-probe.ts` | `creationTimeMs` only | identity |
| `relay/windows-port-scan.ts` | `name` (port owner label) | detailed |
`windows-port-scan.ts` is the one mismatch in the table: it reads only `pid` and
`name`, which the identity set answers, but it calls the detailed reader. On a
@@ -344,7 +344,7 @@ on any other OS keeps using the scan.
## Why the package is patched
`config/patches/@vscode__windows-process-tree@0.8.0.patch` carries five changes.
`config/patches/@vscode__windows-process-tree@0.8.0.patch` carries six changes.
1. **Spectre mitigation.** The upstream `binding.gyp` requires Spectre-mitigated
libraries, which Orca's Windows build agents do not install. `node-pty` is
@@ -368,10 +368,10 @@ on any other OS keeps using the scan.
to Unix ms; a process that denies the handle is emitted with the field
absent, never zero, because callers must be able to tell "cannot identify"
from a timestamp.
5. **`supportedProcessDataFlags`.** `addon.cc` exports the flag bits the
6. **`supportedProcessDataFlags`.** `addon.cc` exports the flag bits the
compiled binary understands, and `lib/index.js` re-exports it.
Why a fifth hunk and not just the enum: unlike `node-pty`, this package
Why a separate hunk and not just the enum: unlike `node-pty`, this package
publishes a prebuilt `.node` at the same `build/Release/` path node-gyp
writes to. pnpm patches the source tree and leaves that prebuilt alone, so a
host can hold a patched `lib/index.js``ProcessDataFlag.CreationTime` and
@@ -575,6 +575,20 @@ running, so typing `exit` in a pane reaped a `start /b` server that used to
survive. The job exists to make an _explicit_ teardown exact, not to redefine
what a clean exit means.
Git Bash needs one additional restriction. The Cygwin runtime — and the MSYS2
fork of it that Git for Windows ships — reads `JOB_OBJECT_LIMIT_BREAKAWAY_OK`
off its own job and then adds `CREATE_BREAKAWAY_FROM_JOB` to **every** child it
spawns when that flag is set (`spawn.cc`, there since 2011), so offering
breakaway hands the whole tree its escape. The per-PTY job therefore omits
`BREAKAWAY_OK` whenever `msys-2.0.dll` or `cygwin1.dll` sits on the shell's DLL
search path — beside the executable, or under `usr/bin` for Git's `bin`
launcher. Native shells keep explicit breakaway. Denying it costs Cygwin
nothing, because it *pre-checks* the limit rather than retrying, so no spawn
fails; but a *native* program that passes `CREATE_BREAKAWAY_FROM_JOB` itself
inside such a pane now gets `ERROR_ACCESS_DENIED`. `nohup` and `disown` are
unaffected — they are Cygwin signal/session concepts, unrelated to job
membership. The daemon's host job is unchanged.
Reaping a dead daemon's shells (#9195, #10415) is therefore a **second, nested
job**, not this one. The terminal daemon assigns itself to a kill-on-close job
at startup (`assignHostProcessToKillOnCloseJob`); children inherit membership,
+2 -3
View File
@@ -30,8 +30,7 @@ import { Callout } from '@/components/docs/prose'
[installer](https://github.com/stablyai/orca/releases/latest/download/orca-windows-setup.exe)
</li>
<li>
**Linux:**
AppImage
**Linux:** AppImage
[x64](https://github.com/stablyai/orca/releases/latest/download/orca-linux.AppImage) ·
[arm64](https://github.com/stablyai/orca/releases/latest/download/orca-linux-arm64.AppImage) ·
[.deb](https://github.com/stablyai/orca/releases) ·
@@ -131,7 +130,7 @@ On Linux the [Orca CLI](/docs/cli/reference) installs as **`orca-ide`**, not `or
- The `.deb` and `.rpm` put `orca-ide` on your `PATH` at install time, as `/usr/bin/orca-ide`.
- With the AppImage, register the CLI from [Settings → General → Orca CLI](/docs/settings). That installs `~/.local/bin/orca-ide`.
- Inside Orca's own terminals, bare `orca` works. Orca puts a shim on the `PATH` of the terminals it manages, so agents and scripts running there use the same command as on macOS and Windows.
- On a headless host, a packaged `orca serve` writes a bare `orca` into `~/.local/bin` as it starts, unless a file it does not own already holds that name. It writes that *during* startup, so it is never what starts the server — the first launch is always [`orca-ide serve`](/docs/remote-servers).
- On a headless host, a packaged `orca serve` writes a bare `orca` into `~/.local/bin` as it starts, unless a file it does not own already holds that name. It writes that _during_ startup, so it is never what starts the server — the first launch is always [`orca-ide serve`](/docs/remote-servers).
Do not verify with `command -v orca`: on a GNOME desktop that succeeds and resolves to the screen reader. Use `orca-ide` in your own shell and `orca` inside Orca. If you want the short name everywhere and you do not use the screen reader, link it yourself:
+11 -3
View File
@@ -129,19 +129,23 @@ Install Orca and its bundled CLI on the server, then run:
<Callout title="On Linux, start it with orca-ide serve">
The Linux CLI is named `orca-ide`, because GNOME Orca's screen reader already owns
`/usr/bin/orca`. A packaged `orca serve` does write a bare `orca` into `~/.local/bin`, but only
while it is starting, so that shim can never be the command that starts the server. Read
`orca serve` as `orca-ide serve` throughout this page when the host is Linux. See
[Install → Linux](/docs/install#linux).
while it is starting, so that shim can never be the command that starts the server. Read `orca
serve` as `orca-ide serve` throughout this page when the host is Linux. See [Install →
Linux](/docs/install#linux).
</Callout>
```bash
orca serve --pairing-address <server-tailscale-ip-or-hostname>
# Linux
orca-ide serve --pairing-address <server-tailscale-ip-or-hostname>
```
For example:
```bash
orca serve --pairing-address 100.64.1.20
# Linux
orca-ide serve --pairing-address 100.64.1.20
```
The command:
@@ -157,6 +161,8 @@ Add `--port 6768` when a firewall, tunnel, or service definition requires a fixe
```bash
orca serve --port 6768 --pairing-address 100.64.1.20
# Linux
orca-ide serve --port 6768 --pairing-address 100.64.1.20
```
Use only one host mode at a time. If the Orca desktop app is already sharing that computer, do not start a second `orca serve` process for the same setup.
@@ -167,6 +173,8 @@ For the Orca mobile app, request a mobile-scoped QR code and link:
```bash
orca serve --pairing-address 100.64.1.20 --mobile-pairing
# Linux
orca-ide serve --pairing-address 100.64.1.20 --mobile-pairing
```
Keep the phone on the same tailnet, open Orca Mobile, choose **Pair**, and scan the terminal QR code or paste the printed link.
+406
View File
@@ -0,0 +1,406 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
const asyncStorage = vi.hoisted(() => ({
getItem: vi.fn(),
setItem: vi.fn(),
removeItem: vi.fn()
}))
vi.mock('@react-native-async-storage/async-storage', () => ({ default: asyncStorage }))
import {
deleteCachedSessionTabStripForHost,
getSessionTabStripCacheKey,
loadCachedSessionTabStrip,
readCachedSessionTabStrip,
resetSessionTabStripCacheForTests,
saveCachedSessionTabStrip
} from './session-tab-strip-cache'
import type { MobileSessionTabStripPreview } from '../session/mobile-session-tab-strip-entries'
const STORAGE_KEY = 'orca:session-tab-strip:v1'
function preview(...ids: string[]): MobileSessionTabStripPreview {
return {
tabs: ids.map((id) => ({ id, type: 'terminal' as const, title: id, agentId: null })),
activeTabId: ids[0] ?? null
}
}
function lastWrittenFile(): { workspaces: { key: string }[] } {
const call = asyncStorage.setItem.mock.calls.at(-1)
return JSON.parse(String(call?.[1]))
}
beforeEach(() => {
vi.useFakeTimers()
asyncStorage.getItem.mockReset().mockResolvedValue(null)
asyncStorage.setItem.mockReset().mockResolvedValue(undefined)
resetSessionTabStripCacheForTests()
})
afterEach(() => {
vi.useRealTimers()
})
describe('getSessionTabStripCacheKey', () => {
it('digests the workspace id so no filesystem path reaches the key', () => {
const path = '/Users/someone/private-client/worktrees/acquisition'
const key = getSessionTabStripCacheKey('host-1', `repo::${path}`)
expect(key).not.toContain(path)
expect(key).not.toContain('someone')
expect(key).toMatch(/^\["host-1","[0-9a-f]{32}"\]$/)
})
it('joins the two ids unambiguously, whatever a worktree path contains', () => {
expect(getSessionTabStripCacheKey('host', 'a\nb')).not.toBe(
getSessionTabStripCacheKey('host\na', 'b')
)
expect(getSessionTabStripCacheKey('host-1', 'wt-1')).not.toBe(
getSessionTabStripCacheKey('host-1', 'wt-2')
)
})
it('needs both a host and a workspace', () => {
expect(getSessionTabStripCacheKey(undefined, 'wt-1')).toBeNull()
expect(getSessionTabStripCacheKey('host-1', undefined)).toBeNull()
})
})
describe('session tab strip cache', () => {
it('serves a save back synchronously and persists it once the write settles', async () => {
const key = getSessionTabStripCacheKey('host-1', 'wt-1')
saveCachedSessionTabStrip(key, preview('tab-1', 'tab-2'))
expect(readCachedSessionTabStrip(key)?.tabs.map((tab) => tab.id)).toEqual(['tab-1', 'tab-2'])
expect(asyncStorage.setItem).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(300)
expect(asyncStorage.setItem.mock.calls[0]?.[0]).toBe(STORAGE_KEY)
expect(lastWrittenFile().workspaces.map((w) => w.key)).toEqual([key])
})
it('reads nothing synchronously before the stored file is loaded', async () => {
const key = getSessionTabStripCacheKey('host-1', 'wt-1')
asyncStorage.getItem.mockResolvedValue(
JSON.stringify({ workspaces: [{ key, preview: preview('tab-1') }] })
)
expect(readCachedSessionTabStrip(key)).toBeNull()
expect((await loadCachedSessionTabStrip(key))?.tabs.map((tab) => tab.id)).toEqual(['tab-1'])
expect(readCachedSessionTabStrip(key)?.tabs).toHaveLength(1)
})
it('returns null for a workspace with no stored strip', async () => {
expect(await loadCachedSessionTabStrip(getSessionTabStripCacheKey('host-1', 'wt-9'))).toBeNull()
expect(await loadCachedSessionTabStrip(null)).toBeNull()
})
it('survives unreadable storage', async () => {
asyncStorage.getItem.mockResolvedValue('{not json')
expect(await loadCachedSessionTabStrip(getSessionTabStripCacheKey('host-1', 'wt-1'))).toBeNull()
})
it('evicts the least recently written workspace past the cap', async () => {
for (let i = 0; i < 14; i++) {
saveCachedSessionTabStrip(getSessionTabStripCacheKey('host-1', `wt-${i}`), preview('tab-1'))
}
await vi.advanceTimersByTimeAsync(300)
const keys = lastWrittenFile().workspaces.map((w) => w.key)
expect(keys).toHaveLength(12)
expect(keys).not.toContain(getSessionTabStripCacheKey('host-1', 'wt-0'))
expect(keys.at(-1)).toBe(getSessionTabStripCacheKey('host-1', 'wt-13'))
})
it('re-writing a workspace makes it the newest, not the oldest', async () => {
for (let i = 0; i < 12; i++) {
saveCachedSessionTabStrip(getSessionTabStripCacheKey('host-1', `wt-${i}`), preview('tab-1'))
}
saveCachedSessionTabStrip(getSessionTabStripCacheKey('host-1', 'wt-0'), preview('tab-2'))
saveCachedSessionTabStrip(getSessionTabStripCacheKey('host-1', 'wt-99'), preview('tab-1'))
await vi.advanceTimersByTimeAsync(300)
const keys = lastWrittenFile().workspaces.map((w) => w.key)
expect(keys).toContain(getSessionTabStripCacheKey('host-1', 'wt-0'))
expect(keys).not.toContain(getSessionTabStripCacheKey('host-1', 'wt-1'))
})
it('records a workspace the host has emptied, so a stale strip cannot outlive it', async () => {
const key = getSessionTabStripCacheKey('host-1', 'wt-1')
saveCachedSessionTabStrip(key, preview('tab-1'))
saveCachedSessionTabStrip(key, { tabs: [], activeTabId: null })
expect(readCachedSessionTabStrip(key)).toEqual({ tabs: [], activeTabId: null })
})
it('caps tabs per workspace and title length, and drops an unmatched active id', async () => {
const key = getSessionTabStripCacheKey('host-1', 'wt-1')
saveCachedSessionTabStrip(key, {
// A file tab, because the titles that survive redaction at all are the ones the cap has
// to bound.
tabs: Array.from({ length: 30 }, (_, i) => ({
id: `tab-${i}`,
type: 'file' as const,
title: 'x'.repeat(200),
agentId: null
})),
activeTabId: 'tab-29'
})
const stored = readCachedSessionTabStrip(key)
expect(stored?.tabs).toHaveLength(24)
expect(stored?.tabs[0]?.title).toHaveLength(64)
expect(stored?.activeTabId).toBeNull()
})
it('drops fields a future tab type might smuggle into storage', async () => {
const key = getSessionTabStripCacheKey('host-1', 'wt-1')
saveCachedSessionTabStrip(key, {
tabs: [
{
id: 'tab-1',
type: 'file',
title: 'notes.md',
agentId: null,
filePath: '/Users/someone/secret/notes.md'
} as never
],
activeTabId: 'tab-1'
})
await vi.advanceTimersByTimeAsync(300)
expect(String(asyncStorage.setItem.mock.calls.at(-1)?.[1])).not.toContain('/Users/someone')
})
it('drops a stored entry naming a tab type this build cannot draw', async () => {
const key = getSessionTabStripCacheKey('host-1', 'wt-1')
saveCachedSessionTabStrip(key, {
tabs: [
{ id: 'tab-1', type: 'from-a-newer-build', title: 'raw title', agentId: null } as never,
{ id: 'tab-2', type: 'file', title: 'notes.md', agentId: null }
],
activeTabId: 'tab-2'
})
expect(readCachedSessionTabStrip(key)?.tabs.map((tab) => tab.id)).toEqual(['tab-2'])
})
it('never writes a shell-controlled terminal title, however it arrives', async () => {
const secret = 'psql postgres://admin:hunter2@db.internal/prod'
const key = getSessionTabStripCacheKey('host-1', 'wt-1')
saveCachedSessionTabStrip(key, {
tabs: [
{ id: 'tab-1', type: 'terminal', title: secret, agentId: null },
{ id: 'tab-2', type: 'terminal', title: secret, agentId: 'claude' },
{ id: 'tab-3', type: 'terminal', title: secret, agentId: 'not-a-known-agent' },
{ id: 'tab-4', type: 'browser', title: 'Acme Corp — Q3 layoffs memo', agentId: null }
],
activeTabId: 'tab-1'
})
await vi.advanceTimersByTimeAsync(300)
expect(readCachedSessionTabStrip(key)?.tabs.map((tab) => tab.title)).toEqual([
'Terminal',
'Claude',
'Terminal',
'Browser'
])
const written = String(asyncStorage.setItem.mock.calls.at(-1)?.[1])
expect(written).not.toContain('hunter2')
expect(written).not.toContain('postgres://')
expect(written).not.toContain('layoffs')
})
it('scrubs a stored title written by an older build on the way back out', async () => {
const key = getSessionTabStripCacheKey('host-1', 'wt-1')
asyncStorage.getItem.mockResolvedValue(
JSON.stringify({
workspaces: [
{
key,
preview: {
tabs: [{ id: 'tab-1', type: 'terminal', title: 'curl -H token', agentId: null }],
activeTabId: 'tab-1'
}
}
]
})
)
expect((await loadCachedSessionTabStrip(key))?.tabs[0]?.title).toBe('Terminal')
})
it('forgets an unpaired host and cannot resurrect it from a later save', async () => {
const hostA = getSessionTabStripCacheKey('host-a', 'wt-1')
const hostB = getSessionTabStripCacheKey('host-b', 'wt-1')
saveCachedSessionTabStrip(hostA, preview('tab-a'))
saveCachedSessionTabStrip(hostB, preview('tab-b'))
await vi.advanceTimersByTimeAsync(300)
await deleteCachedSessionTabStripForHost('host-a')
expect(readCachedSessionTabStrip(hostA)).toBeNull()
expect(readCachedSessionTabStrip(hostB)?.tabs).toHaveLength(1)
expect(lastWrittenFile().workspaces.map((w) => w.key)).toEqual([hostB])
saveCachedSessionTabStrip(hostB, preview('tab-b2'))
await vi.advanceTimersByTimeAsync(300)
expect(lastWrittenFile().workspaces.map((w) => w.key)).toEqual([hostB])
})
it('forgets a host whose rows are only on disk, never read this session', async () => {
const hostA = getSessionTabStripCacheKey('host-a', 'wt-1')
const hostB = getSessionTabStripCacheKey('host-b', 'wt-1')
asyncStorage.getItem.mockResolvedValue(
JSON.stringify({
workspaces: [
{ key: hostA, preview: preview('tab-a') },
{ key: hostB, preview: preview('tab-b') }
]
})
)
await deleteCachedSessionTabStripForHost('host-a')
expect(lastWrittenFile().workspaces.map((w) => w.key)).toEqual([hostB])
})
it('drops a pending debounced write so it cannot restore the forgotten host', async () => {
const hostA = getSessionTabStripCacheKey('host-a', 'wt-1')
saveCachedSessionTabStrip(hostA, preview('tab-a'))
await deleteCachedSessionTabStripForHost('host-a')
await vi.advanceTimersByTimeAsync(300)
expect(lastWrittenFile().workspaces).toEqual([])
})
it('rejects a deletion whose write never landed, rather than reporting it as done', async () => {
// A resolved delete over a failed write leaves the forgotten host's tab titles in
// plaintext on disk while every caller believes they are gone.
const hostA = getSessionTabStripCacheKey('host-a', 'wt-1')
saveCachedSessionTabStrip(hostA, preview('tab-a'))
await vi.advanceTimersByTimeAsync(300)
asyncStorage.setItem.mockRejectedValue(new Error('storage full'))
await expect(deleteCachedSessionTabStripForHost('host-a')).rejects.toThrow('storage full')
})
it('keeps a debounced save best effort, so one failed write cannot reject unowned', async () => {
asyncStorage.setItem.mockRejectedValue(new Error('storage full'))
saveCachedSessionTabStrip(getSessionTabStripCacheKey('host-a', 'wt-1'), preview('tab-a'))
// No throw and no unhandled rejection: the write is fire-and-forget by design.
await vi.advanceTimersByTimeAsync(300)
expect(asyncStorage.setItem).toHaveBeenCalledOnce()
})
it('refuses a save for the host it is in the middle of forgetting', async () => {
const hostA = getSessionTabStripCacheKey('host-a', 'wt-1')
saveCachedSessionTabStrip(hostA, preview('tab-a'))
await vi.advanceTimersByTimeAsync(300)
let releaseWrite!: () => void
asyncStorage.setItem.mockImplementationOnce(
async () =>
new Promise<void>((resolve) => {
releaseWrite = () => resolve()
})
)
const deletion = deleteCachedSessionTabStripForHost('host-a')
// The purge has run and its write is on the wire; a snapshot queued for the
// workspace the user just unpaired now lands in that window.
await vi.advanceTimersByTimeAsync(0)
saveCachedSessionTabStrip(hostA, preview('tab-a2'))
releaseWrite()
await deletion
await vi.advanceTimersByTimeAsync(300)
expect(readCachedSessionTabStrip(hostA)).toBeNull()
expect(lastWrittenFile().workspaces).toEqual([])
})
it('cannot be talked back into a host whose deletion write failed', async () => {
const hostA = getSessionTabStripCacheKey('host-a', 'wt-1')
saveCachedSessionTabStrip(hostA, preview('tab-a'))
await vi.advanceTimersByTimeAsync(300)
asyncStorage.setItem.mockRejectedValueOnce(new Error('storage full'))
await expect(deleteCachedSessionTabStripForHost('host-a')).rejects.toThrow('storage full')
const writesSoFar = asyncStorage.setItem.mock.calls.length
saveCachedSessionTabStrip(hostA, preview('tab-a3'))
await vi.advanceTimersByTimeAsync(300)
expect(readCachedSessionTabStrip(hostA)).toBeNull()
expect(asyncStorage.setItem).toHaveBeenCalledTimes(writesSoFar)
})
it('lets a debounced write that already snapshotted the removed host land first', async () => {
// The tombstone stops new saves, but a debounced write that fired a moment earlier
// built its blob from the map as it was and is still on the wire. Writing over it
// concurrently leaves which blob lands last up to storage.
const hostA = getSessionTabStripCacheKey('host-a', 'wt-1')
const hostB = getSessionTabStripCacheKey('host-b', 'wt-1')
saveCachedSessionTabStrip(hostA, preview('tab-a'))
saveCachedSessionTabStrip(hostB, preview('tab-b'))
let releaseDebounced!: () => void
asyncStorage.setItem.mockImplementationOnce(
async () =>
new Promise<void>((resolve) => {
releaseDebounced = () => resolve()
})
)
await vi.advanceTimersByTimeAsync(300)
const deletion = deleteCachedSessionTabStripForHost('host-a')
await vi.advanceTimersByTimeAsync(0)
expect(asyncStorage.setItem).toHaveBeenCalledOnce()
releaseDebounced()
await deletion
expect(asyncStorage.setItem).toHaveBeenCalledTimes(2)
expect(lastWrittenFile().workspaces.map((w) => w.key)).toEqual([hostB])
})
it('cannot let an older overlapping write commit after the purge', async () => {
// Why: two debounced writes can sit on the bridge at once, and the second used to replace
// the in-flight handle. The purge then awaited only the newer one, so the older blob --
// snapshotted while the forgotten host was still in the map -- could commit last.
const hostA = getSessionTabStripCacheKey('host-a', 'wt-1')
const hostB = getSessionTabStripCacheKey('host-b', 'wt-1')
let stored = ''
const gates: Array<() => void> = []
asyncStorage.setItem.mockImplementation(
(_key: string, value: string) =>
new Promise<void>((resolve) => {
gates.push(() => {
stored = value
resolve()
})
})
)
saveCachedSessionTabStrip(hostA, preview('tab-a'))
await vi.advanceTimersByTimeAsync(300)
saveCachedSessionTabStrip(hostB, preview('tab-b'))
await vi.advanceTimersByTimeAsync(300)
const deletion = deleteCachedSessionTabStripForHost('host-a')
// Newest released first: only writes that queue behind one another survive this.
for (let step = 0; step < 6 && gates.length > 0; step += 1) {
gates.pop()?.()
await vi.advanceTimersByTimeAsync(0)
}
await deletion
const keys = (JSON.parse(stored) as { workspaces: { key: string }[] }).workspaces.map(
(workspace) => workspace.key
)
expect(keys).toEqual([hostB])
})
})
+260
View File
@@ -0,0 +1,260 @@
// Why: reconnecting to a workspace the phone opened a minute ago tears the session screen back
// to an empty strip and a spinner, even though the tab list it is about to be handed is the one
// it just displayed. Persist the shape of the strip per workspace so a reconnect paints the
// known tabs immediately and swaps in live rows under the same keys.
//
// This file is the authority on what reaches plaintext storage, not its callers: every entry is
// rebuilt field by field on the way in, and shell-controlled titles are replaced with fixed
// labels here rather than trusted to have been scrubbed upstream.
import AsyncStorage from '@react-native-async-storage/async-storage'
import { sha256 } from '@noble/hashes/sha256'
import {
getPersistableTabStripTitle,
isDrawableTabStripType,
type MobileSessionTabStripEntry,
type MobileSessionTabStripPreview
} from '../session/mobile-session-tab-strip-entries'
const STORAGE_KEY = 'orca:session-tab-strip:v1'
// A phone realistically revisits a handful of workspaces; the caps bound both the stored blob
// and the cost of a single write.
const MAX_WORKSPACES = 12
const MAX_TABS_PER_WORKSPACE = 24
const MAX_TITLE_LENGTH = 64
const WRITE_DEBOUNCE_MS = 250
// 128 bits of a digest: far past collision range for a dozen workspaces, and short enough that
// the stored blob stays small.
const WORKSPACE_DIGEST_LENGTH = 32
type StoredWorkspace = { key: string; preview: MobileSessionTabStripPreview }
type StoredFile = { workspaces: StoredWorkspace[] }
// Insertion-ordered, so the first key is the least recently written one to evict.
let memoryCache: Map<string, MobileSessionTabStripPreview> | null = null
let loadPromise: Promise<Map<string, MobileSessionTabStripPreview>> | null = null
let writeTimer: ReturnType<typeof setTimeout> | null = null
// Tail of the write chain. Every write queues behind it, so an older setItem can never
// settle after a newer one and make its stale blob the last word on disk.
let writeInFlight: Promise<void> | null = null
// Hosts forgotten this session. A save racing the deletion would re-insert the host and
// the next debounced write would put its tab titles back on disk, so refuse those saves
// outright. Re-pairing the same host caches again from the next app launch — the cheap
// direction for a deletion the user asked for.
const forgottenHosts = new Set<string>()
/**
* A workspace id ends in a filesystem path, so it is digested rather than stored. The host id
* stays readable because forgetting a host has to be able to find that host's rows, and because
* host ids already key several other entries in this store.
*/
export function getSessionTabStripCacheKey(
hostId: string | undefined,
worktreeId: string | undefined
): string | null {
if (!hostId || !worktreeId) {
return null
}
return JSON.stringify([hostId, digestWorkspaceId(worktreeId)])
}
/** Whatever this process already knows, with no await — so a revisit paints on the first frame. */
export function readCachedSessionTabStrip(key: string | null): MobileSessionTabStripPreview | null {
if (!key || !memoryCache) {
return null
}
return memoryCache.get(key) ?? null
}
export async function loadCachedSessionTabStrip(
key: string | null
): Promise<MobileSessionTabStripPreview | null> {
if (!key) {
return null
}
const cache = await loadFile()
return cache.get(key) ?? null
}
export function saveCachedSessionTabStrip(
key: string | null,
preview: MobileSessionTabStripPreview
): void {
if (!key) {
return
}
const hostId = readHostIdFromKey(key)
if (hostId !== null && forgottenHosts.has(hostId)) {
return
}
const redacted = redactPreview(preview)
const cache = memoryCache ?? new Map()
memoryCache = cache
// Map.set on an existing key keeps its original iteration position, so delete first to make
// the re-inserted key the newest and give the cap true LRU eviction.
cache.delete(key)
cache.set(key, redacted)
while (cache.size > MAX_WORKSPACES) {
const oldest = cache.keys().next().value
if (oldest === undefined) {
break
}
cache.delete(oldest)
}
scheduleWrite(cache)
}
/**
* Drop every workspace belonging to a host the user has unpaired. Both the in-memory rows and
* the stored blob have to go: leaving either behind means the next save for any other host
* serializes the forgotten host's tabs straight back to disk.
*/
export async function deleteCachedSessionTabStripForHost(hostId: string): Promise<void> {
// Before the first await: a save landing during the load or the write must not
// re-insert the host the caller is in the middle of forgetting.
forgottenHosts.add(hostId)
// Load first so the rewrite below preserves other hosts. If storage is unreadable we still
// rewrite, which can cost another host its rows — the wrong direction for a cache, the right
// one for a deletion the user asked for.
const cache = await loadFile()
// Deleting the entry the iterator is standing on is well-defined for a Map.
for (const key of cache.keys()) {
if (readHostIdFromKey(key) === hostId) {
cache.delete(key)
}
}
if (writeTimer) {
clearTimeout(writeTimer)
writeTimer = null
}
// Queued, not raced: the purge is the last write, and its failure is the caller's.
await enqueueWrite(cache)
}
export function resetSessionTabStripCacheForTests(): void {
if (writeTimer) {
clearTimeout(writeTimer)
writeTimer = null
}
memoryCache = null
loadPromise = null
writeInFlight = null
forgottenHosts.clear()
}
function digestWorkspaceId(worktreeId: string): string {
const digest = sha256(new TextEncoder().encode(worktreeId))
let hex = ''
for (const byte of digest) {
hex += byte.toString(16).padStart(2, '0')
}
return hex.slice(0, WORKSPACE_DIGEST_LENGTH)
}
function readHostIdFromKey(key: string): string | null {
try {
const parsed = JSON.parse(key) as unknown
return Array.isArray(parsed) && typeof parsed[0] === 'string' ? parsed[0] : null
} catch {
return null
}
}
async function loadFile(): Promise<Map<string, MobileSessionTabStripPreview>> {
if (memoryCache) {
return memoryCache
}
loadPromise ??= (async () => {
const parsed = await readStoredFile()
// A save that landed while the read was in flight owns the newer truth.
const cache = memoryCache ?? new Map<string, MobileSessionTabStripPreview>()
for (const workspace of parsed) {
if (!cache.has(workspace.key)) {
cache.set(workspace.key, workspace.preview)
}
}
memoryCache = cache
return cache
})()
return loadPromise
}
async function readStoredFile(): Promise<StoredWorkspace[]> {
try {
const raw = await AsyncStorage.getItem(STORAGE_KEY)
if (!raw) {
return []
}
const parsed = JSON.parse(raw) as StoredFile
if (typeof parsed !== 'object' || parsed === null || !Array.isArray(parsed.workspaces)) {
return []
}
return parsed.workspaces.flatMap((workspace) => {
if (typeof workspace?.key !== 'string' || !Array.isArray(workspace.preview?.tabs)) {
return []
}
return [{ key: workspace.key, preview: redactPreview(workspace.preview) }]
})
} catch {
return []
}
}
// Why: a flurry of snapshots (one per desktop republication) must not hammer AsyncStorage.
function scheduleWrite(cache: Map<string, MobileSessionTabStripPreview>): void {
if (writeTimer) {
clearTimeout(writeTimer)
}
writeTimer = setTimeout(() => {
writeTimer = null
// Best effort by design: a dropped cache refresh costs one repaint, and the next
// save rewrites the whole map. Only the deletion path needs the failure.
void enqueueWrite(cache).catch(() => {})
}, WRITE_DEBOUNCE_MS)
}
// Why the chain rather than one handle: two debounced writes can overlap on the bridge, and
// the second overwrote the handle. A deletion then awaited only the newer one, so the older
// write -- serialized before the purge, host rows and all -- could land last and restore them.
function enqueueWrite(cache: Map<string, MobileSessionTabStripPreview>): Promise<void> {
const queued = (writeInFlight ?? Promise.resolve()).then(() => writeFile(cache))
// A rejected link must not break the chain for the writes queued behind it.
writeInFlight = queued.catch(() => {})
return queued
}
async function writeFile(cache: Map<string, MobileSessionTabStripPreview>): Promise<void> {
const workspaces: StoredWorkspace[] = [...cache].map(([key, preview]) => ({ key, preview }))
// Throws on purpose: a deletion that only removed the in-memory rows must not be
// reported as a deletion, or the forgotten host's titles stay in plaintext on disk.
await AsyncStorage.setItem(STORAGE_KEY, JSON.stringify({ workspaces }))
}
// Rebuilt field by field so a field later added to the live tab type cannot ride into storage
// without someone deciding it belongs there.
function redactPreview(preview: MobileSessionTabStripPreview): MobileSessionTabStripPreview {
const tabs: MobileSessionTabStripEntry[] = []
for (const tab of preview.tabs ?? []) {
if (typeof tab?.id !== 'string' || !isDrawableTabStripType(tab.type)) {
continue
}
const agentId = typeof tab.agentId === 'string' ? tab.agentId : null
const title = typeof tab.title === 'string' ? tab.title : ''
tabs.push({
id: tab.id,
type: tab.type,
title: getPersistableTabStripTitle({ type: tab.type, title, agentId }).slice(
0,
MAX_TITLE_LENGTH
),
agentId
})
if (tabs.length === MAX_TABS_PER_WORKSPACE) {
break
}
}
const activeTabId =
typeof preview.activeTabId === 'string' && tabs.some((tab) => tab.id === preview.activeTabId)
? preview.activeTabId
: null
return { tabs, activeTabId }
}
+172 -4
View File
@@ -44,6 +44,12 @@ function GateConsumer() {
return createElement('GateStatus', null, hostCapabilities.join(','))
}
// Separate from GateStatus so the capability assertions keep their exact rendered shape.
function VerifiedConsumer() {
const { compatVerified } = useHostProtocolGates()
return createElement('GateVerified', null, compatVerified ? 'verified' : 'unverified')
}
// Counts mounts so a test can prove the routes were never torn down, which presence alone can't.
const probeMounts = { count: 0 }
function MountProbe() {
@@ -57,7 +63,13 @@ function gateElement() {
return createElement(
HostProtocolGate,
{ hostId: 'host-1' },
createElement('HostContent', null, createElement(GateConsumer), createElement(MountProbe))
createElement(
'HostContent',
null,
createElement(GateConsumer),
createElement(VerifiedConsumer),
createElement(MountProbe)
)
)
}
@@ -154,13 +166,90 @@ describe('HostProtocolGate', () => {
expect(client.sendRequest).toHaveBeenCalledOnce()
})
it('serves every descendant capability read from the one status.get it issues', async () => {
const client = clientWithStatus({
protocolVersion: 5,
minCompatibleMobileVersion: 0,
capabilities: ['browser.screencast.v1', 'terminal.queryReplyInput.v1']
})
hostClient.current = { client, state: 'connected' }
renderer = await act(async () => {
const created = create(
createElement(
HostProtocolGate,
{ hostId: 'host-1' },
createElement(GateConsumer),
createElement(GateConsumer)
)
)
await Promise.resolve()
return created
})
// Why: the session route used to run its own retrying status.get on top of this one, so a
// cold open cost two round trips for the same answer. Consumers now read the gate's copy.
expect(client.sendRequest).toHaveBeenCalledOnce()
expect(client.sendRequest).toHaveBeenCalledWith('status.get')
const statuses = renderer.root.findAllByType('GateStatus')
expect(statuses).toHaveLength(2)
for (const status of statuses) {
expect(status.props.children).toBe('browser.screencast.v1,terminal.queryReplyInput.v1')
}
})
it('releases the cover on a failed status.get and upgrades when a retry lands', async () => {
vi.useFakeTimers({ shouldAdvanceTime: true })
const sendRequest = vi
.fn()
.mockRejectedValueOnce(new Error('status.get timed out'))
.mockResolvedValue({
ok: true,
result: { protocolVersion: 5, minCompatibleMobileVersion: 0, capabilities: ['late.v1'] }
})
hostClient.current = { client: { sendRequest } as unknown as RpcClient, state: 'connected' }
renderer = await renderGate()
// Why: a wedged status.get must never trap the routes behind the cover, so the first miss
// settles conservative gates immediately — no capabilities, but a usable UI.
let output = renderedText(renderer)
expect(output).toContain('HostContent')
expect(output).not.toContain('Checking host compatibility')
expect(output).toContain('"type":"GateStatus","props":{},"children":null')
await act(async () => {
await vi.advanceTimersByTimeAsync(1_100)
})
// The probe kept retrying underneath, so the answer arrives without a remount.
expect(sendRequest).toHaveBeenCalledTimes(2)
expect(renderedText(renderer)).toContain('late.v1')
expect(probeMounts.count).toBe(1)
vi.useRealTimers()
})
it('blocks a desktop that omits protocolVersion, so a pending verdict is not a formality', async () => {
vi.spyOn(console, 'warn').mockImplementation(() => {})
// Why this case and not just an explicit old version: evaluateCompat reads a missing
// protocolVersion as 0, so the everyday shape of an old desktop is a blocking one.
hostClient.current = {
client: clientWithStatus({ capabilities: [] }),
state: 'connected'
}
renderer = await renderGate()
const output = renderedText(renderer)
expect(output).toContain('Update Orca on your computer')
expect(output).not.toContain('HostContent')
})
it('renders the host UI while the host connection is still pending', async () => {
hostClient.current = { client: null, state: 'connecting' }
renderer = await renderGate()
expect(renderedText(renderer)).toContain('HostContent')
})
it('does not mount host routes before a connected host passes the compatibility probe', async () => {
// Was: the routes were held back until status.get resolved, which serialised every route's
// own startup RPC behind this one round trip. They now mount immediately and are covered.
it('mounts host routes under the pending cover while status.get is still in flight', async () => {
const client = {
sendRequest: vi.fn().mockReturnValue(new Promise(() => {}))
} as unknown as RpcClient
@@ -168,9 +257,40 @@ describe('HostProtocolGate', () => {
renderer = await renderGate()
const output = renderedText(renderer)
expect(output).toContain('Checking host compatibility')
expect(output).not.toContain('HostContent')
expect(probeMounts.count).toBe(0)
expect(output).toContain('HostContent')
expect(probeMounts.count).toBe(1)
expect(client.sendRequest).toHaveBeenCalledOnce()
// Why: mounting early must not leak an unproven host's capabilities to the routes below;
// an empty join renders no children, so the consumer saw none.
expect(output).toContain('"type":"GateStatus","props":{},"children":null')
const overlay = renderer.root
.findAllByType('View')
.find((node) => node.props.accessibilityViewIsModal === true)
expect(overlay?.props.pointerEvents).toBe('auto')
})
it('unmounts the routes it mounted early when the verdict comes back blocked', async () => {
vi.spyOn(console, 'warn').mockImplementation(() => {})
let settle: ((response: unknown) => void) | null = null
const client = {
sendRequest: vi.fn().mockReturnValue(
new Promise((resolve) => {
settle = resolve
})
)
} as unknown as RpcClient
hostClient.current = { client, state: 'connected' }
renderer = await renderGate()
expect(renderedText(renderer)).toContain('HostContent')
await act(async () => {
settle?.({ ok: true, result: { protocolVersion: 5, minCompatibleMobileVersion: 999 } })
await Promise.resolve()
})
const output = renderedText(renderer)
expect(output).toContain('Update Orca Mobile')
expect(output).not.toContain('HostContent')
})
it('overlays the pending spinner instead of unmounting routes mounted while connecting', async () => {
@@ -259,4 +379,52 @@ describe('HostProtocolGate', () => {
renderer = await renderGate()
expect(renderedText(renderer)).toContain('HostContent')
})
it('reports a rejected status.get as unverified, so failing open is not a passing verdict', async () => {
const sendRequest = vi
.fn()
.mockResolvedValue({ ok: false, error: { message: 'no such method' } })
hostClient.current = { client: { sendRequest } as unknown as RpcClient, state: 'connected' }
renderer = await renderGate()
// Navigation still works: the host said no, and that must not lock the user out of the route.
const output = renderedText(renderer)
expect(output).toContain('HostContent')
expect(output).not.toContain('Checking host compatibility')
// Why: `compatVerdict` is `ok` here purely as a fallback. Nothing about this host was proven,
// so callers that write to it read this flag instead of the verdict.
expect(output).toContain('["unverified"]')
})
it('reports a passing status reply as verified', async () => {
hostClient.current = {
client: clientWithStatus({ protocolVersion: 5, minCompatibleMobileVersion: 0 }),
state: 'connected'
}
renderer = await renderGate()
expect(renderedText(renderer)).toContain('["verified"]')
})
it('stays unverified through a failed status.get and flips once a retry answers', async () => {
vi.useFakeTimers({ shouldAdvanceTime: true })
const sendRequest = vi
.fn()
.mockRejectedValueOnce(new Error('status.get timed out'))
.mockResolvedValue({
ok: true,
result: { protocolVersion: 5, minCompatibleMobileVersion: 0 }
})
hostClient.current = { client: { sendRequest } as unknown as RpcClient, state: 'connected' }
renderer = await renderGate()
expect(renderedText(renderer)).toContain('["unverified"]')
await act(async () => {
await vi.advanceTimersByTimeAsync(1_100)
})
// The retry landed, so the fallback is replaced by a real answer and writes are released.
expect(renderedText(renderer)).toContain('["verified"]')
vi.useRealTimers()
})
})
+9 -33
View File
@@ -22,45 +22,26 @@ export function useHostProtocolGates(): HostStatusGates {
// Why: single choke point above every /h/[hostId] route so a blocked verdict replaces the
// whole host UI (sidebar + detail stack) while the host list and other hosts stay usable.
// The routes mount as soon as the connection does, so their startup RPCs (session.tabs.list,
// terminal.list) fly alongside this status.get instead of queueing behind it; a blocked verdict
// then unmounts them and their answers are discarded.
export function HostProtocolGate({ hostId, children }: Props) {
const { client, state } = useHostClient(hostId)
const gates = useHostStatusGates({ hostId, client, connState: state })
const { compatVerdict, statusPending } = gates
const resolvedHostIdRef = useRef<string | null>(null)
const mountedHostIdRef = useRef<string | null>(null)
const hostKey = hostId ?? null
const resolvedNow = state === 'connected' && client !== null && !statusPending
const blocked = compatVerdict.kind === 'blocked'
const pending = statusPending && resolvedHostIdRef.current !== hostKey
const holdBack = pending && mountedHostIdRef.current !== hostKey
// Why: React can replay or discard a render, so the latches record committed
// outcomes only — a discarded children render must not count as mounted.
// Why: React can replay or discard a render, so the latch records committed outcomes only.
useEffect(() => {
if (resolvedNow) {
resolvedHostIdRef.current = hostKey
}
if (blocked) {
// Why: the block screen unmounts the routes, so a later pending window
// must not assume a live tree it can overlay.
mountedHostIdRef.current = null
} else if (!holdBack) {
mountedHostIdRef.current = hostKey
}
})
if (holdBack) {
// Why: nothing is mounted yet for this host, so hold the routes back entirely
// rather than letting them mount (and fire their connect RPCs) pre-verdict.
return (
<View style={styles.pending}>
<ActivityIndicator
color={colors.textSecondary}
accessibilityLabel="Checking host compatibility"
/>
</View>
)
}
if (blocked) {
return <ProtocolBlockScreen verdict={compatVerdict} />
}
@@ -77,10 +58,11 @@ export function HostProtocolGate({ hostId, children }: Props) {
{children}
</View>
{pending ? (
// Why: once the stack is mounted, unmounting it for a pending status.get destroys
// in-flight nested navigation, so cover it instead. Mount effects underneath still
// run — they wait for connState 'connected' and every capability-dependent call
// re-probes status.get itself, so nothing newer than the baseline fires here.
// Why: cover the stack rather than unmounting it — unmounting for a pending status.get
// destroys in-flight nested navigation, and holding it back would serialise every route's
// startup RPC behind this one. Mount effects underneath run pre-verdict by design; they
// read capabilities from this gate, which reports none until the verdict lands, so every
// capability-dependent surface stays closed rather than guessing.
<View
style={styles.pendingOverlay}
// Why: the fill owns the hit test for in-tree views only — native-Modal-hosted
@@ -100,12 +82,6 @@ export function HostProtocolGate({ hostId, children }: Props) {
}
const styles = StyleSheet.create({
pending: {
flex: 1,
alignItems: 'center',
justifyContent: 'center',
backgroundColor: colors.bgBase
},
// Stays mounted across the overlay toggling so the routes below keep their identity.
host: {
flex: 1
@@ -7,7 +7,7 @@ const probe = vi.hoisted(() => ({
start: vi.fn()
}))
vi.mock('../transport/runtime-capability-probe', () => ({
vi.mock('../transport/runtime-status-probe', () => ({
startRuntimeCapabilityProbe: probe.start
}))
@@ -1,7 +1,7 @@
import { useEffect, useState } from 'react'
import { CODEX_RESET_CREDIT_RUNTIME_CAPABILITY } from '../../../src/shared/protocol-version'
import type { RpcClient } from '../transport/rpc-client'
import { startRuntimeCapabilityProbe } from '../transport/runtime-capability-probe'
import { startRuntimeCapabilityProbe } from '../transport/runtime-status-probe'
// Why: source the capability string from the shared contract so a host bump can never
// silently drift from the mobile probe.
@@ -1,8 +1,36 @@
import { describe, expect, it } from 'vitest'
import { buildConnectionDiagnosticsReport } from './connection-diagnostics-report'
import type { ConnectionLogEntry } from '../transport/types'
const NOW = Date.UTC(2026, 6, 9, 22, 0, 0)
function stageEntry(
id: string,
ts: number,
name: string,
ms: number,
complete: boolean
): ConnectionLogEntry {
return {
id,
ts,
level: complete ? 'info' : 'warn',
path: 'relay',
message: `Relay dial stage ${name} ${complete ? 'finished' : 'did not finish'}`,
timing: { kind: 'relay-dial-stage', name, ms, complete }
}
}
function stateEntry(id: string, ts: number, name: string, ms: number): ConnectionLogEntry {
return {
id,
ts,
level: 'info',
message: `Connection state ${name} → connected`,
timing: { kind: 'connection-state', name, ms, complete: true }
}
}
describe('buildConnectionDiagnosticsReport', () => {
it('summarizes a failing Tailscale host with its log', () => {
const report = buildConnectionDiagnosticsReport({
@@ -68,13 +96,28 @@ describe('buildConnectionDiagnosticsReport', () => {
activePath: 'tailscale',
pendingPath: 'relay',
entries: [
{
id: 'relay-stage-opening',
ts: NOW - 6_000,
level: 'info',
path: 'relay',
message: 'Relay dial stage opening finished',
detail: '118ms — resumeToken=secret-resume-token',
timing: { kind: 'relay-dial-stage', name: 'opening', ms: 118, complete: true }
},
{
id: 'relay-failure',
ts: NOW - 5_000,
level: 'error',
message: 'Relay: relay dial failed',
detail:
'RelayDirectorHttpError: relay director resolve failed (503); retry after 30000ms; resumeToken=secret-resume-token'
'RelayDirectorHttpError: relay director resolve failed (503); retry after 30000ms; resumeToken=secret-resume-token',
timing: {
kind: 'relay-dial-stage',
name: 'awaiting-hello',
ms: 9_100,
complete: false
}
}
],
nowMs: NOW
@@ -87,6 +130,9 @@ describe('buildConnectionDiagnosticsReport', () => {
expect(report).toContain('Next step: Keep Orca open; recovery should retry automatically.')
expect(report).toContain('resumeToken=[redacted]')
expect(report).not.toContain('secret-resume-token')
expect(report).toContain(
'Relay dial stages: opening 118ms · awaiting-hello 9.1s (did not finish) — total 9.2s'
)
})
it('redacts quoted JSON credentials and never echoes an invalid endpoint', () => {
@@ -116,6 +162,52 @@ describe('buildConnectionDiagnosticsReport', () => {
expect(report).not.toContain('bearer-secret')
})
it('breaks a slow connect down by dial stage and connection state', () => {
const report = buildConnectionDiagnosticsReport({
hostName: 'Host 6',
endpoint: 'ws://192.168.1.50:6768',
state: 'connected',
reconnectAttempts: 2,
lastConnectedAt: NOW,
platform: 'ios 26.5.1',
appVersion: '0.0.47',
entries: [
stageEntry('a1', NOW - 30_000, 'opening', 90, false),
stateEntry('s1', NOW - 29_000, 'connecting', 12_000),
stageEntry('b1', NOW - 20_000, 'opening', 120, true),
stageEntry('b2', NOW - 19_000, 'awaiting-hello', 6_400, true),
stageEntry('b3', NOW - 13_000, 'handshaking', 240, true),
stageEntry('b4', NOW - 12_000, 'confirming', 1_180, true),
stateEntry('s2', NOW - 11_000, 'connecting', 8_000)
],
nowMs: NOW
})
// Only the latest dial is broken out, so a reconnect loop cannot average away
// the attempt the reporter is complaining about.
expect(report).toContain(
'Relay dial stages (latest of 2): opening 120ms · awaiting-hello 6.4s · handshaking 240ms · confirming 1.2s — total 7.9s'
)
expect(report).toContain('Connection state dwell: connecting 20.0s ×2')
})
it('omits the timing lines when nothing recorded a phase duration', () => {
const report = buildConnectionDiagnosticsReport({
hostName: 'Host 7',
endpoint: 'ws://192.168.1.50:6768',
state: 'connected',
reconnectAttempts: 0,
lastConnectedAt: NOW,
platform: 'ios 26.5.1',
appVersion: '0.0.47',
entries: [{ id: 'plain', ts: NOW, level: 'info', message: 'Authenticated' }],
nowMs: NOW
})
expect(report).not.toContain('Relay dial stages')
expect(report).not.toContain('Connection state dwell')
})
it('bounds a single event line before submission while preserving its identity', () => {
const report = buildConnectionDiagnosticsReport({
hostName: 'Host 5',
@@ -8,6 +8,7 @@ import { normalizeHostAppVersion } from '../transport/host-app-version-store'
import { formatEndpoint } from './host-reachability'
import { diagnoseConnection } from './connection-diagnostics-analysis'
import { redactConnectionLogEntry, redactConnectionLogText } from './connection-log-redaction'
import { summarizeConnectionLogTimings } from './connection-log-timing-summary'
const MAX_EVENT_LINE_BYTES = 2 * 1024
const EVENT_TRUNCATION_MARKER = ' … [truncated]'
@@ -59,6 +60,7 @@ export function buildConnectionDiagnosticsReport(args: {
? 'Last connected: never this session'
: `Last connected: ${new Date(args.lastConnectedAt).toISOString()} (${formatAgo(now - args.lastConnectedAt)} ago)`
)
lines.push(...summarizeConnectionLogTimings(entries))
lines.push('')
lines.push(`Likely cause: ${diagnosis.likelyCause}`)
lines.push(`Next step: ${diagnosis.nextStep}`)
@@ -0,0 +1,66 @@
import type { ConnectionLogEntry, ConnectionLogTiming } from '../transport/types'
// Why: a report that only says "connecting for 10s" cannot be triaged. These lines
// turn the per-phase timings the transport now records into the two questions
// support actually asks: which relay dial stage ate the time, and how long the
// client sat in each connection state.
export function summarizeConnectionLogTimings(entries: readonly ConnectionLogEntry[]): string[] {
const timings = entries.flatMap((entry) => (entry.timing ? [entry.timing] : []))
const lines: string[] = []
const dials = groupRelayDials(timings.filter((timing) => timing.kind === 'relay-dial-stage'))
const latestDial = dials.at(-1)
if (latestDial) {
const label =
dials.length > 1 ? `Relay dial stages (latest of ${dials.length})` : 'Relay dial stages'
const total = latestDial.reduce((sum, timing) => sum + timing.ms, 0)
lines.push(
`${label}: ${latestDial.map(formatStageTiming).join(' · ')} — total ${formatDurationMs(total)}`
)
}
const states = totalPerName(timings.filter((timing) => timing.kind === 'connection-state'))
if (states.length > 0) {
lines.push(
`Connection state dwell: ${states
.map(
({ name, ms, count }) => `${name} ${formatDurationMs(ms)}${count > 1 ? ` ×${count}` : ''}`
)
.join(' · ')}`
)
}
return lines
}
// Relay dial stages are strictly ordered and every dial starts in 'opening', so an
// 'opening' timing opens a new group. Reporting only the latest keeps a reconnect
// loop from averaging away the attempt the reporter is complaining about.
function groupRelayDials(timings: readonly ConnectionLogTiming[]): ConnectionLogTiming[][] {
const dials: ConnectionLogTiming[][] = []
for (const timing of timings) {
if (timing.name === 'opening' || dials.length === 0) {
dials.push([])
}
dials.at(-1)!.push(timing)
}
return dials
}
function totalPerName(
timings: readonly ConnectionLogTiming[]
): { name: string; ms: number; count: number }[] {
const totals = new Map<string, { name: string; ms: number; count: number }>()
for (const timing of timings) {
const total = totals.get(timing.name) ?? { name: timing.name, ms: 0, count: 0 }
total.ms += timing.ms
total.count += 1
totals.set(timing.name, total)
}
return [...totals.values()]
}
function formatStageTiming(timing: ConnectionLogTiming): string {
return `${timing.name} ${formatDurationMs(timing.ms)}${timing.complete ? '' : ' (did not finish)'}`
}
function formatDurationMs(ms: number): string {
return ms < 1000 ? `${Math.round(ms)}ms` : `${(ms / 1000).toFixed(1)}s`
}
@@ -2,6 +2,8 @@ import { Animated, View, Text, Pressable, ActivityIndicator } from 'react-native
import { saveTerminalTextScale } from '../storage/preferences'
import { MobileBrowserPane } from '../browser/MobileBrowserPane'
import { TerminalPaneView } from './TerminalPaneView'
import { TerminalEnginePrewarm } from './TerminalEnginePrewarm'
import { MOBILE_SESSION_TAB_BAR_HEIGHT } from './mobile-session-frame-styles'
import { MobileNativeChatOverlay } from './MobileNativeChatOverlay'
import { colors } from '../theme/mobile-theme'
import { styles } from './mobile-session-styles'
@@ -74,16 +76,38 @@ export function MobileSessionActiveContent({
activePendingTerminalTab,
isPendingTerminalRecoveryParked,
retryPendingTerminalRecovery,
reconnectViewState,
tabStripRows,
showLoadingState,
measurePrewarmViewport,
showEmptyState,
keyboardLift,
activeTerminalKeyboardLift,
toastAnimatedStyle,
createTabBusy
} = controller
return showLoadingState ? (
<View style={styles.emptyState}>
<ActivityIndicator size="small" color={colors.textSecondary} />
// Why the same list the header gates on: an unmounted tab bar gives the content row its band
// back, so the pre-warm would measure a taller box than the pane ever gets. Reading the header's
// own rows (live or cached preview) keeps the two from drifting.
const prewarmReservedTabBarHeight = tabStripRows.length > 0 ? 0 : MOBILE_SESSION_TAB_BAR_HEIGHT
// Why: the cached strip in the header is the content during a reconnect; the terminal body
// cannot be, because replaying stored scrollback into the WebView would double-render once the
// live stream replays the same rows. See mobile-session-reconnect-view-state. The engine still
// boots inside the real terminal frame while the startup RPCs are in flight, so the first pane
// inherits a warm WebView and a measured viewport (see prewarm).
return reconnectViewState.kind === 'reconnecting-with-cache' || showLoadingState ? (
<View style={styles.terminalFrame}>
<View style={styles.emptyState}>
<ActivityIndicator size="small" color={colors.textSecondary} />
{reconnectViewState.kind === 'reconnecting-with-cache' ? (
<Text style={styles.emptyText}>{reconnectViewState.label}</Text>
) : null}
</View>
<TerminalEnginePrewarm
reservedTabBarHeight={prewarmReservedTabBarHeight}
textScale={terminalTextScale}
onEngineMeasured={measurePrewarmViewport}
/>
</View>
) : showEmptyState ? (
<View style={styles.emptyState}>
@@ -171,25 +195,35 @@ export function MobileSessionActiveContent({
)}
</View>
) : activePendingTerminalTab ? (
<View style={styles.emptyState}>
{!isPendingTerminalRecoveryParked && (
<ActivityIndicator size="small" color={colors.textSecondary} />
)}
<Text style={styles.emptyText}>
{isPendingTerminalRecoveryParked
? 'Terminal is taking longer than expected'
: activePendingTerminalTab.title || 'Loading terminal'}
</Text>
{isPendingTerminalRecoveryParked && (
<Pressable
accessibilityRole="button"
accessibilityLabel="Retry loading terminal"
style={({ pressed }) => [styles.createButton, pressed && styles.newTerminalButtonPressed]}
onPress={() => void retryPendingTerminalRecovery()}
>
<Text style={styles.createButtonText}>Retry</Text>
</Pressable>
)}
<View style={styles.terminalFrame}>
<View style={styles.emptyState}>
{!isPendingTerminalRecoveryParked && (
<ActivityIndicator size="small" color={colors.textSecondary} />
)}
<Text style={styles.emptyText}>
{isPendingTerminalRecoveryParked
? 'Terminal is taking longer than expected'
: activePendingTerminalTab.title || 'Loading terminal'}
</Text>
{isPendingTerminalRecoveryParked && (
<Pressable
accessibilityRole="button"
accessibilityLabel="Retry loading terminal"
style={({ pressed }) => [
styles.createButton,
pressed && styles.newTerminalButtonPressed
]}
onPress={() => void retryPendingTerminalRecovery()}
>
<Text style={styles.createButtonText}>Retry</Text>
</Pressable>
)}
</View>
<TerminalEnginePrewarm
reservedTabBarHeight={prewarmReservedTabBarHeight}
textScale={terminalTextScale}
onEngineMeasured={measurePrewarmViewport}
/>
</View>
) : (
<View
+30 -29
View File
@@ -14,10 +14,6 @@ import { MobileSessionHeaderIconButton } from './MobileSessionHeaderIconButton'
import { triggerMediumImpact } from '../platform/haptics'
import { StatusDot } from '../components/StatusDot'
import { MobileAgentIcon } from '../components/MobileAgentIcon'
import {
getMobileSessionTabTitle,
resolveMobileTerminalTabAgentId
} from './mobile-terminal-tab-agent'
import { colors } from '../theme/mobile-theme'
import { QuickCommandsTabButton } from './QuickCommandsTabButton'
import { styles } from './mobile-session-styles'
@@ -32,7 +28,6 @@ export function MobileSessionHeader({ controller }: { controller: MobileSessionC
forceReconnectHost,
worktreeName,
activePanel,
activeSessionTabId,
activeSessionTabIdRef,
tabStripRef,
tabStripOffsetRef,
@@ -52,7 +47,7 @@ export function MobileSessionHeader({ controller }: { controller: MobileSessionC
scrollActiveTabIntoView,
switchSessionTab,
openSessionTabActionSheetAfterKeyboardDismiss,
visibleTabs,
tabStripRows,
showConnectionRetry,
terminalSummary,
handlePanelTap,
@@ -117,7 +112,7 @@ export function MobileSessionHeader({ controller }: { controller: MobileSessionC
) : null}
</View>
{visibleTabs.length > 0 && (
{tabStripRows.length > 0 && (
<View style={styles.tabBar}>
{/* Why: tab taps must register on first press with the keyboard open instead of being eaten by dismissal (#5106). */}
<ScrollView
@@ -140,45 +135,51 @@ export function MobileSessionHeader({ controller }: { controller: MobileSessionC
scrollActiveTabIntoView(activeSessionTabIdRef.current, false)
}}
>
{visibleTabs.map((t) => (
{tabStripRows.map(({ entry, isActive, tab }) => (
<Pressable
key={t.id}
style={[styles.tab, t.id === activeSessionTabId && styles.tabActive]}
key={entry.id}
style={[
styles.tab,
isActive && styles.tabActive,
tab === null && styles.tabPreview
]}
onLayout={(e) => {
const { x, width } = e.nativeEvent.layout
tabLayoutsRef.current.set(t.id, { x, width })
if (t.id === activeSessionTabIdRef.current) {
scrollActiveTabIntoView(t.id, false)
tabLayoutsRef.current.set(entry.id, { x, width })
if (entry.id === activeSessionTabIdRef.current) {
scrollActiveTabIntoView(entry.id, false)
}
}}
onPress={() => switchSessionTab(t)}
onLongPress={() => {
triggerMediumImpact()
openSessionTabActionSheetAfterKeyboardDismiss(t)
}}
// A cached preview row has no live tab behind it, so both gestures need the
// reconnect to land first.
disabled={tab === null}
onPress={tab === null ? undefined : () => switchSessionTab(tab)}
onLongPress={
tab === null
? undefined
: () => {
triggerMediumImpact()
openSessionTabActionSheetAfterKeyboardDismiss(tab)
}
}
delayLongPress={400}
>
<View style={styles.tabLabelRow}>
{t.type === 'browser' && (
{entry.type === 'browser' && (
<Globe size={13} color={colors.textSecondary} strokeWidth={2.1} />
)}
{t.type === 'markdown' && (
{entry.type === 'markdown' && (
<FileText size={13} color={colors.textSecondary} strokeWidth={2.1} />
)}
{t.type === 'file' && (
{entry.type === 'file' && (
<File size={13} color={colors.textSecondary} strokeWidth={2.1} />
)}
{t.type === 'agent-session' && <MobileAgentIcon agentId={t.agent} size={13} />}
{t.type === 'terminal' &&
(() => {
const agentId = resolveMobileTerminalTabAgentId(t)
return agentId ? <MobileAgentIcon agentId={agentId} size={13} /> : null
})()}
{entry.agentId !== null && <MobileAgentIcon agentId={entry.agentId} size={13} />}
<Text
style={[styles.tabText, t.id === activeSessionTabId && styles.tabTextActive]}
style={[styles.tabText, isActive && styles.tabTextActive]}
numberOfLines={1}
>
{getMobileSessionTabTitle(t)}
{entry.title}
</Text>
</View>
</Pressable>
@@ -0,0 +1,238 @@
import { createElement } from 'react'
import { act, create, type ReactTestRenderer } from 'react-test-renderer'
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { TerminalWebViewHandle } from '../terminal/terminal-webview-contract'
const engine = vi.hoisted(() => ({
init: vi.fn((_cols: number, _rows: number) => {}),
awaitReady: vi.fn(async () => {}),
measureFitDimensions: vi.fn(async (_containerHeight?: number) => ({ cols: 120, rows: 40 })),
onWebReady: null as (() => void) | null,
textScale: undefined as number | undefined
}))
vi.mock('react-native', () => ({
StyleSheet: {
create: <T>(styles: T) => styles,
absoluteFillObject: { position: 'absolute', top: 0, left: 0, right: 0, bottom: 0 }
},
View: 'View'
}))
// Stands in for the real engine: records the ref the pre-warm pane holds and the ready callback
// it arms, so a test can drive web-ready and layout in either order.
vi.mock('../terminal/TerminalWebView', async () => {
const { forwardRef, useImperativeHandle } = await import('react')
return {
TerminalWebView: forwardRef<
TerminalWebViewHandle,
{ onWebReady?: () => void; textScale?: number }
>(function MockTerminalWebView(props, ref) {
engine.onWebReady = props.onWebReady ?? null
engine.textScale = props.textScale
useImperativeHandle(ref, () => engine as unknown as TerminalWebViewHandle, [])
return createElement('MockTerminalWebView')
})
}
})
import { TerminalEnginePrewarm } from './TerminalEnginePrewarm'
const FRAME = { x: 0, y: 0, width: 390, height: 700 }
const TEXT_SCALE = 1.25
function renderPrewarm(onEngineMeasured: (ref: TerminalWebViewHandle, height: number) => void): {
renderer: ReactTestRenderer
layout: (frame: { x: number; y: number; width: number; height: number }) => void
webReady: () => void
} {
let renderer: ReactTestRenderer | null = null
act(() => {
renderer = create(
createElement(TerminalEnginePrewarm, {
reservedTabBarHeight: 0,
textScale: TEXT_SCALE,
onEngineMeasured
})
)
})
const created = renderer as unknown as ReactTestRenderer
return {
renderer: created,
layout: (frame) =>
act(() => {
created.root.findAllByType('View')[0]?.props.onLayout({ nativeEvent: { layout: frame } })
}),
webReady: () =>
act(() => {
engine.onWebReady?.()
})
}
}
// The handoff now waits on the engine's ready promise, so tests have to let microtasks run.
async function flushReady(): Promise<void> {
await act(async () => {})
}
afterEach(() => {
engine.measureFitDimensions.mockClear()
engine.init.mockClear()
engine.awaitReady.mockReset()
engine.awaitReady.mockResolvedValue(undefined)
engine.onWebReady = null
engine.textScale = undefined
})
describe('TerminalEnginePrewarm', () => {
it('boots the engine without waiting for a terminal to attach', () => {
const measured = vi.fn()
const { renderer } = renderPrewarm(measured)
// The engine mounts on the first render, so its bundle loads while the startup RPCs fly.
expect(renderer.root.findAllByType('MockTerminalWebView')).toHaveLength(1)
expect(measured).not.toHaveBeenCalled()
})
it('withholds the measurement until the pane has a real layout', async () => {
const measured = vi.fn()
const { webReady, layout } = renderPrewarm(measured)
webReady()
// Why: this is the 80x24 trap — an unsized engine answers with xterm's default, and that
// number would ride the first subscribe to the host as the PTY size.
expect(measured).not.toHaveBeenCalled()
layout({ ...FRAME, width: 0, height: 0 })
expect(measured).not.toHaveBeenCalled()
layout(FRAME)
await flushReady()
expect(measured).toHaveBeenCalledOnce()
expect(measured.mock.calls[0]?.[1]).toBe(FRAME.height)
})
it('withholds the measurement until the engine reports ready', async () => {
const measured = vi.fn()
const { layout, webReady } = renderPrewarm(measured)
layout(FRAME)
expect(measured).not.toHaveBeenCalled()
webReady()
await flushReady()
expect(measured).toHaveBeenCalledOnce()
})
it('measures once however many times layout and web-ready repeat', async () => {
const measured = vi.fn()
const { layout, webReady } = renderPrewarm(measured)
layout(FRAME)
webReady()
webReady()
layout({ ...FRAME, height: 640 })
layout(FRAME)
await flushReady()
expect(measured).toHaveBeenCalledOnce()
})
it('opens the engine before handing it over, because web-ready alone builds no terminal', async () => {
const measured = vi.fn()
let releaseReady: (() => void) | null = null
engine.awaitReady.mockImplementation(
() =>
new Promise<void>((resolve) => {
releaseReady = resolve
})
)
const { layout, webReady } = renderPrewarm(measured)
layout(FRAME)
webReady()
// Why: the WebView answers `measure` with null while it has no terminal, and the pane latches
// once, so handing the engine over before init would spend the one measurement on nothing.
expect(engine.init).toHaveBeenCalledOnce()
expect(measured).not.toHaveBeenCalled()
releaseReady?.()
await flushReady()
expect(measured).toHaveBeenCalledOnce()
expect(measured.mock.calls[0]?.[0]).toBe(engine)
})
it('pre-warms at the text size the first pane will open with', () => {
renderPrewarm(vi.fn())
// Cell size is what the frame gets divided by, so a default-sized engine would measure a
// different phone than the one the user is looking at.
expect(engine.textScale).toBe(TEXT_SCALE)
})
it('reports the frame the pane ended up with when a resize lands during engine start-up', async () => {
const measured = vi.fn()
let releaseReady: (() => void) | null = null
engine.awaitReady.mockImplementation(
() =>
new Promise<void>((resolve) => {
releaseReady = resolve
})
)
const { layout, webReady } = renderPrewarm(measured)
layout(FRAME)
webReady()
expect(measured).not.toHaveBeenCalled()
// A rotation or split-screen resize while the engine is still coming up. The latch has already
// fired, so this is the last chance to correct the height the one measurement is taken against.
const resized = { ...FRAME, width: 700, height: 360 }
layout(resized)
releaseReady?.()
await flushReady()
expect(measured).toHaveBeenCalledOnce()
expect(measured.mock.calls[0]?.[1]).toBe(resized.height)
})
it('drops the handoff when the pane unmounts before the engine is ready', async () => {
const measured = vi.fn()
let releaseReady: (() => void) | null = null
engine.awaitReady.mockImplementation(
() =>
new Promise<void>((resolve) => {
releaseReady = resolve
})
)
const { renderer, layout, webReady } = renderPrewarm(measured)
layout(FRAME)
webReady()
act(() => {
renderer.unmount()
})
releaseReady?.()
await flushReady()
// The frame this measurement was taken against is gone, so it describes nothing.
expect(measured).not.toHaveBeenCalled()
})
it('is inert: no touches, no accessibility, and nothing sent to a terminal', async () => {
const measured = vi.fn()
const { renderer, layout, webReady } = renderPrewarm(measured)
layout(FRAME)
webReady()
await flushReady()
const pane = renderer.root.findAllByType('View')[0]
expect(pane?.props.pointerEvents).toBe('none')
expect(pane?.props.accessibilityElementsHidden).toBe(true)
expect(pane?.props.importantForAccessibility).toBe('no-hide-descendants')
// The pane owns no handle, so it has no way to subscribe, send input, or resize a PTY.
// Opening the engine is WebView-local; the measurement itself is the caller's to take.
expect(engine.measureFitDimensions).not.toHaveBeenCalled()
expect(measured.mock.calls[0]?.[0]).toBe(engine)
})
})
@@ -0,0 +1,111 @@
import { useCallback, useRef } from 'react'
import { StyleSheet, View, type LayoutChangeEvent } from 'react-native'
import { TerminalWebView } from '../terminal/TerminalWebView'
import type { TerminalWebViewHandle } from '../terminal/terminal-webview-contract'
// Diagnostics label for the measurement this pane contributes; it is not a PTY handle.
export const TERMINAL_ENGINE_PREWARM_HANDLE = '(engine-prewarm)'
// Why: the WebView builds no xterm until it is told to, and `measure` answers null while `term`
// is null, so the engine has to be opened before it can be asked anything. These are placeholder
// dimensions for an empty buffer nobody reads; the measurement derives its own cols and rows from
// the frame and the font's cell size, so nothing downstream inherits them.
const PREWARM_INIT_COLS = 80
const PREWARM_INIT_ROWS = 24
type Props = {
// Height the tab bar will claim from the top of this frame once the session has a tab. The
// loading state has no visible tab, so the bar is not mounted yet and the box the pane will
// finally occupy is this much shorter. Reserving it keeps the measurement honest; measuring
// the taller box would latch too many rows and send them to the host as the PTY size.
reservedTabBarHeight: number
// Why: the first pane opens at the user's saved text size, and cell size is what the
// measurement divides the frame by. Pre-warming at a different size measures a different phone.
textScale: number
onEngineMeasured: (ref: TerminalWebViewHandle, frameHeight: number) => void
}
// Why: a session still resolving its tabs already knows it is heading for a terminal, so load
// the xterm engine alongside the startup RPCs instead of after terminal.list returns. This pane
// owns no handle: it never subscribes, never sends input, and can never resize a PTY. Its only
// output is the viewport measurement the first real pane would otherwise pay a round trip for.
export function TerminalEnginePrewarm({
reservedTabBarHeight,
textScale,
onEngineMeasured
}: Props) {
const engineRef = useRef<TerminalWebViewHandle | null>(null)
const frameHeightRef = useRef(0)
const webReadyRef = useRef(false)
const measuredRef = useRef(false)
// Idempotent by construction: both triggers funnel here and the latch fires once per mount.
const measureWhenSized = useCallback(() => {
const engine = engineRef.current
// Why: an unsized or unmounted WebView measures xterm's 80x24 default, and that number
// rides the first subscribe to the host. Only a laid-out engine is allowed to answer.
if (measuredRef.current || !webReadyRef.current || !engine || frameHeightRef.current <= 0) {
return
}
measuredRef.current = true
// `web-ready` only says the xterm bundle loaded. Opening the engine is what creates `term`,
// and `awaitReady` is what lets its cell dimensions exist before anything reads them.
engine.init(PREWARM_INIT_COLS, PREWARM_INIT_ROWS)
void engine.awaitReady().then(() => {
// React nulls the ref on unmount, so this proves the pane the frame belongs to is still up.
if (engineRef.current !== engine) {
return
}
// Why read the height here and not before the wait: a rotation or split-screen resize during
// engine start-up re-lays out this pane, and the latch above already refused the second
// handoff, so a height captured earlier would be the only one this pane ever reports.
onEngineMeasured(engine, frameHeightRef.current)
})
}, [onEngineMeasured])
const handleLayout = useCallback(
(event: LayoutChangeEvent) => {
const { height, width } = event.nativeEvent.layout
if (width <= 0 || height <= 0) {
return
}
frameHeightRef.current = height
measureWhenSized()
},
[measureWhenSized]
)
const handleWebReady = useCallback(() => {
webReadyRef.current = true
measureWhenSized()
}, [measureWhenSized])
return (
<View
// Why: sized like the real pane so the measurement matches, but invisible and inert so it
// cannot paint over the loading state or steal a touch from the retry affordance above it.
accessibilityElementsHidden
importantForAccessibility="no-hide-descendants"
pointerEvents="none"
style={[styles.prewarmPane, { top: reservedTabBarHeight }]}
onLayout={handleLayout}
>
<TerminalWebView
ref={engineRef}
style={styles.prewarmWebView}
textScale={textScale}
onWebReady={handleWebReady}
/>
</View>
)
}
const styles = StyleSheet.create({
prewarmPane: {
...StyleSheet.absoluteFillObject,
opacity: 0
},
prewarmWebView: {
flex: 1
}
})
@@ -2,6 +2,20 @@ import { StyleSheet } from 'react-native'
import { colors, spacing, radii, typography } from '../theme/mobile-theme'
// Why one constant for the whole strip: the terminal frame is whatever the tab bar leaves behind,
// and the engine pre-warm has to reserve exactly that much before the bar exists. Every row child
// is pinned to this height so nothing can grow the bar without moving the reservation with it.
//
// The row deliberately has NO explicit height. React Native lays out border-box, so `height: 36`
// with a 1 px top border would render a 36 px row over a 35 px content area and squeeze children
// that are themselves 36 -- and it would leave this constant one pixel long, which is a whole row
// of drift once a frame sits near a row boundary. Left to size itself the row takes its tallest
// child and adds the border outside it, which is exactly the sum below.
export const MOBILE_SESSION_TAB_BAR_CONTENT_HEIGHT = 36
export const MOBILE_SESSION_TAB_BAR_BORDER_WIDTH = 1
export const MOBILE_SESSION_TAB_BAR_HEIGHT =
MOBILE_SESSION_TAB_BAR_CONTENT_HEIGHT + MOBILE_SESSION_TAB_BAR_BORDER_WIDTH
export const mobileSessionFrameStyles = StyleSheet.create({
container: {
flex: 1,
@@ -80,12 +94,12 @@ export const mobileSessionFrameStyles = StyleSheet.create({
tabBar: {
flexDirection: 'row',
alignItems: 'center',
borderTopWidth: 1,
borderTopWidth: MOBILE_SESSION_TAB_BAR_BORDER_WIDTH,
borderTopColor: colors.borderSubtle
},
tabScroll: {
flex: 1,
maxHeight: 36
maxHeight: MOBILE_SESSION_TAB_BAR_CONTENT_HEIGHT
},
tabContent: {
paddingLeft: spacing.sm,
@@ -94,7 +108,7 @@ export const mobileSessionFrameStyles = StyleSheet.create({
tab: {
width: 128,
maxWidth: 128,
minHeight: 36,
minHeight: MOBILE_SESSION_TAB_BAR_CONTENT_HEIGHT,
alignItems: 'center',
justifyContent: 'center',
paddingHorizontal: spacing.sm,
@@ -102,6 +116,11 @@ export const mobileSessionFrameStyles = StyleSheet.create({
borderBottomWidth: 2,
borderBottomColor: 'transparent'
},
// Why: a cached row is inert until the reconnect lands, so it carries the same de-emphasis as
// the disabled tab-bar buttons beside it rather than passing for a live tab.
tabPreview: {
opacity: 0.45
},
tabActive: {
// Neutral grey underline, matching the desktop terminal tab's active
// indicator (a muted foreground/card mix), not a blue accent.
@@ -123,7 +142,7 @@ export const mobileSessionFrameStyles = StyleSheet.create({
},
newTerminalButton: {
width: 40,
height: 36,
height: MOBILE_SESSION_TAB_BAR_CONTENT_HEIGHT,
alignItems: 'center',
justifyContent: 'center',
borderBottomWidth: 2,
@@ -0,0 +1,155 @@
import { describe, expect, it } from 'vitest'
import { selectMobileSessionReconnectViewState } from './mobile-session-reconnect-view-state'
import {
getMobileSessionTabStripRows,
toMobileSessionTabStripPreview,
type MobileSessionTabStripPreview
} from './mobile-session-tab-strip-entries'
import type { MobileSessionTab } from './mobile-session-route-types'
function terminalTab(id: string, title: string, isActive = false): MobileSessionTab {
return { type: 'terminal', id, title, terminal: `h-${id}`, isActive }
}
const cachedPreview: MobileSessionTabStripPreview = {
tabs: [
{ id: 'tab-1', type: 'terminal', title: 'claude', agentId: 'claude' },
{ id: 'tab-2', type: 'terminal', title: 'shell', agentId: null }
],
activeTabId: 'tab-1'
}
const base = {
connState: 'reconnecting',
verdictKind: 'normal',
terminalsLoaded: false,
liveTabCount: 0,
activeHandle: null,
cachedPreview: null
} as const
describe('selectMobileSessionReconnectViewState', () => {
it('renders the cached strip with a progress label while reconnecting', () => {
const state = selectMobileSessionReconnectViewState({ ...base, cachedPreview })
expect(state).toEqual({
kind: 'reconnecting-with-cache',
preview: cachedPreview,
label: 'Reconnecting…'
})
})
it('labels the post-connect hydration gap as loading, not reconnecting', () => {
const state = selectMobileSessionReconnectViewState({
...base,
connState: 'connected',
cachedPreview
})
expect(state.kind === 'reconnecting-with-cache' && state.label).toBe('Loading tabs…')
})
it('blocks when nothing is cached for this workspace', () => {
expect(selectMobileSessionReconnectViewState(base)).toEqual({ kind: 'blocking' })
expect(
selectMobileSessionReconnectViewState({
...base,
cachedPreview: { tabs: [], activeTabId: null }
})
).toEqual({ kind: 'blocking' })
})
it('keeps mounted live content instead of swapping in its own cached snapshot', () => {
expect(
selectMobileSessionReconnectViewState({ ...base, liveTabCount: 2, cachedPreview })
).toEqual({ kind: 'live' })
expect(
selectMobileSessionReconnectViewState({ ...base, activeHandle: 'h-1', cachedPreview })
).toEqual({ kind: 'live' })
})
it('treats a host-confirmed empty workspace as live', () => {
expect(
selectMobileSessionReconnectViewState({
...base,
connState: 'connected',
terminalsLoaded: true,
cachedPreview
})
).toEqual({ kind: 'live' })
})
it('falls back to the offline state once the retry loop or the pairing has failed', () => {
expect(
selectMobileSessionReconnectViewState({ ...base, verdictKind: 'unreachable', cachedPreview })
).toEqual({ kind: 'offline' })
expect(
selectMobileSessionReconnectViewState({ ...base, verdictKind: 'auth-failed', cachedPreview })
).toEqual({ kind: 'offline' })
})
it('keeps showing the cache through a transient warning verdict', () => {
expect(
selectMobileSessionReconnectViewState({ ...base, verdictKind: 'warning', cachedPreview }).kind
).toBe('reconnecting-with-cache')
})
})
describe('getMobileSessionTabStripRows', () => {
it('draws disabled preview rows while reconnecting, then the live tabs under the same keys', () => {
const preview = selectMobileSessionReconnectViewState({ ...base, cachedPreview })
const previewRows = getMobileSessionTabStripRows({
liveTabs: [],
activeSessionTabId: null,
preview: preview.kind === 'reconnecting-with-cache' ? preview.preview : null
})
expect(previewRows.map((row) => row.entry.id)).toEqual(['tab-1', 'tab-2'])
expect(previewRows.map((row) => row.tab)).toEqual([null, null])
expect(previewRows.map((row) => row.isActive)).toEqual([true, false])
const liveTabs = [terminalTab('tab-1', 'claude', true), terminalTab('tab-2', 'shell')]
const liveRows = getMobileSessionTabStripRows({
liveTabs,
activeSessionTabId: 'tab-1',
preview: null
})
expect(liveRows.map((row) => row.entry.id)).toEqual(previewRows.map((row) => row.entry.id))
expect(liveRows.map((row) => row.isActive)).toEqual(previewRows.map((row) => row.isActive))
expect(liveRows.every((row) => row.tab !== null)).toBe(true)
})
it('prefers live tabs over a preview that is still present', () => {
const rows = getMobileSessionTabStripRows({
liveTabs: [terminalTab('tab-9', 'fresh', true)],
activeSessionTabId: 'tab-9',
preview: cachedPreview
})
expect(rows.map((row) => row.entry.id)).toEqual(['tab-9'])
})
it('keeps only the drawn fields when projecting a preview to persist', () => {
const preview = toMobileSessionTabStripPreview(
[
{
type: 'terminal',
id: 'tab-1',
title: 'claude',
terminal: 'h-1',
launchAgent: 'claude',
launchDraft: 'unsent secret prompt',
isActive: true
}
],
'tab-1'
)
expect(preview).toEqual({
tabs: [{ id: 'tab-1', type: 'terminal', title: 'claude', agentId: 'claude' }],
activeTabId: 'tab-1'
})
expect(JSON.stringify(preview)).not.toContain('unsent secret prompt')
})
})
@@ -0,0 +1,61 @@
import type { ConnectionVerdict } from '../transport/connection-health'
import type { ConnectionState } from '../transport/types'
import type { MobileSessionTabStripPreview } from './mobile-session-tab-strip-entries'
/**
* What the session screen should draw while the phone is not yet serving live tabs.
*
* - `live`: real tabs are mounted (or the host has confirmed there are none). The existing
* loading/empty/content branches own the screen.
* - `reconnecting-with-cache`: nothing live yet, but this workspace's last strip is on the
* device. Draw it, disabled, with a compact progress line instead of a bare spinner.
* - `offline`: the retry loop has given up or the pairing is rejected. A stale strip would
* imply a session we cannot reach, so fall back to the existing offline affordance.
* - `blocking`: nothing live and nothing cached. Unchanged from before this state existed.
*/
export type MobileSessionReconnectViewState =
| { kind: 'live' }
| { kind: 'reconnecting-with-cache'; preview: MobileSessionTabStripPreview; label: string }
| { kind: 'offline' }
| { kind: 'blocking' }
export function selectMobileSessionReconnectViewState(args: {
connState: ConnectionState
verdictKind: ConnectionVerdict['kind']
terminalsLoaded: boolean
liveTabCount: number
activeHandle: string | null
cachedPreview: MobileSessionTabStripPreview | null
}): MobileSessionReconnectViewState {
const { connState, verdictKind, terminalsLoaded, liveTabCount, activeHandle, cachedPreview } =
args
// A mounted terminal or tab is the real thing; a mid-session drop must never trade it for a
// snapshot of itself, however the connection is faring.
if (liveTabCount > 0 || activeHandle !== null) {
return { kind: 'live' }
}
// The host has answered and said this workspace is empty — that is live truth, not a gap.
if (connState === 'connected' && terminalsLoaded) {
return { kind: 'live' }
}
if (verdictKind === 'unreachable' || verdictKind === 'auth-failed') {
return { kind: 'offline' }
}
if (cachedPreview && cachedPreview.tabs.length > 0) {
return {
kind: 'reconnecting-with-cache',
preview: cachedPreview,
label: reconnectProgressLabel(connState)
}
}
return { kind: 'blocking' }
}
function reconnectProgressLabel(connState: ConnectionState): string {
if (connState === 'connected') {
return 'Loading tabs…'
}
return connState === 'reconnecting' || connState === 'disconnected'
? 'Reconnecting…'
: 'Connecting…'
}
@@ -37,6 +37,7 @@ const LOGIC_EXPANSION_NAMES = new Set([
'useMobileSessionContentCreateActions',
'useMobileSessionCloseActions',
'useMobileSessionBulkClose',
'useMobileSessionTabStripCache',
'useMobileSessionPresentation',
'useMobileSessionPanelRouteActions'
])
@@ -62,32 +63,32 @@ const HOST_COMPONENT_NAMES = new Set([
'View'
])
const HEAD_MAIN_HOOK_SHA256 = '10071240ef9edafc2b9c8bed73be83dceaf7828e3b29f17dab55da020a7697a6'
const HEAD_HOOK_BINDING_SHA256 = '1dadb8c3dc0573ea20659ce7251629669e618dd0effaeac3a4536b29c2e865a1'
const HEAD_MAIN_HOOK_SHA256 = 'e22e7d3a1147ef19c747f0e216b778e794a73e027e96cf84fac2dde03b37b640'
const HEAD_HOOK_BINDING_SHA256 = '531fe06cf2c261b1346bbc949c9ceba5aea8b8ace2dcb8a1898e9759745e013c'
const HEAD_CALLBACK_IDENTITY_SHA256 =
'2a9e4825df007f6ef53b81aa5004991d6318eee7507b44d625c07e630be432eb'
const HEAD_CALLBACK_BODY_SHA256 = '22103ba85a86e3a3fcb80a7509c7a455d79863010cde3af02db6565b55e3ebe9'
const HEAD_EFFECT_SHA256 = 'd9ebfaabc1e79773cdada7ab370b20459ed972f1f8edce1652199f4d0391cd13'
'e5df1043256bcb0b3813bf89161d91f5e65c00749fbb6d98176bca82e878d061'
const HEAD_CALLBACK_BODY_SHA256 = '6d9ed614ed139aef5cc911c33ea4220cc1fc5f888a1a564ef85e6910cc118bc3'
const HEAD_EFFECT_SHA256 = 'a6d4d5cb573926f40faa7701cef7885a0f2c7e7c5cfaa91f4e480c29aba44d79'
const HEAD_CONTENT_HOOK_SHA256 = '9c3b612fef3f370d66873aefdbe1d701f20cb64ded31fef5cc45fde6f8189581'
const HEAD_NESTED_FUNCTION_SHA256 =
'536c72b233c813bb0cea164b090bdce5406ceb965bbc5b83c1f89b89b46f3821'
'0e553eb5ec7aeda8f8336b8da85ff87eb3657a21fa32d3c75c9cc32e36860244'
const HEAD_NATIVE_REGISTRATION_SHA256 =
'cab85e4e4a3f43289ba93ddea9ccce57aea83e0bf14fd1620a965aad0c1cb49e'
const HEAD_NATIVE_REMOVAL_SHA256 =
'4c994574675a2a0f9c607b3ea89ab7a2ed5a83f7c72fa42342ddcb5f00fc3f4f'
const HEAD_TIMER_CREATION_SHA256 =
'1a31b625e2174c3db77272249843196d2b6b06ab1e654a96d8f7858e3082e66b'
const HEAD_TIMER_CLEANUP_SHA256 = 'c73f1d1c2cc89642f3d727d6f3b6b81860a9d6f34234541a2065ec3d1a8cd116'
'36c3ccef371698e25cd2eb239df7a8dea6dcc674d9da43cc38cabfa3a8f64929'
const HEAD_TIMER_CLEANUP_SHA256 = '2f41ddc30d0e9c1b6d1d6b5e09d96d1b3facd3133acae1ff7436bb40e4ef39dc'
const HEAD_RUNTIME_STRING_SHA256 =
'31951b0b83be01ebfa659c4b94df9ad7eaff6404df5338fbade89eb7473a3cb4'
const HEAD_HOST_JSX_SHA256 = '390405926b1695fa3a33686f0bc192b432f5468d8576499d7cafbb4922defbb5'
const HEAD_LEAF_JSX_SHA256 = '21dba981875e173f692590bf910d60964660c5f4cbb79f3a377c7e54f6a1f016'
'694a22ed924ebc2a7d380089ff2cfd3e27f5d72d3c4d4b7b06aa3006db93c053'
const HEAD_HOST_JSX_SHA256 = '0aca9fe4b6738228020fe20334fe2716471a2fdf57e4feea6a2a92cac1c04c58'
const HEAD_LEAF_JSX_SHA256 = '2c38e19ffbcaae14f9df2fdb44751546d2b936f9a4b2c5e90727a5f74f3c2665'
const HEAD_STYLE_REFERENCE_SHA256 =
'295a3501c2c6d7bea7c8bbf38b3f3534f01344cd7e1b91bb8e07c040821d596a'
'da81d6065c5c1ebafbbd721321023cddd0bfc1afa0325749f736bb97898f9556'
const HEAD_IDENTITY_FIELD_SHA256 =
'91146853930a34dd1f3d80e5c97fbacd7cf19fb93dd26fe8fc6f29169622f9d6'
'a7444b7d0953edb34abc77180ba11d458b02081547b8499249571efd30ac0609'
const HEAD_NAVIGATION_SHA256 = '9d96f5dad7de555d6553eac39c0fab00efad507470fd562cb9beaa32db16f512'
const HEAD_CAPABILITY_SHA256 = 'ca219f7909a091717110b823d5b94a20770ad3ae51894e0fa765e8628309392d'
const HEAD_CAPABILITY_SHA256 = '54c74cdb468d015c31517004e005187f6cff2ddb07e25fdb4a7060a2fac6b786'
type Definition = { declaration: ts.FunctionDeclaration; sourceFile: ts.SourceFile }
type HookFacts = {
@@ -456,8 +457,10 @@ function readCompatibilityFacts(definitions: ReadonlyMap<string, Definition>): {
: ''
const callText = canonical(node, sourceFile)
if (
['startRuntimeCapabilityProbe', 'supportsMobileQuickCommands'].includes(callName) ||
(callName === 'includes' && callText.includes('capabilities.includes'))
// hostCapabilities.* is included: the session route now reads the gate's shared status.get
// answer instead of running its own probe, and those reads still have to stay ratcheted.
['useHostProtocolGates', 'supportsMobileQuickCommands'].includes(callName) ||
(callName === 'includes' && /[cC]apabilities\.includes/.test(callText))
) {
capabilities.push(callText)
}
@@ -472,18 +475,18 @@ describe('mobile session route extraction parity', () => {
const contentBindings = CONTENT_COMPONENT_NAMES.flatMap(
(name) => readHookFacts(name, definitions).bindings
)
expect(main.hooks).toHaveLength(266)
expect(main.hooks).toHaveLength(272)
expect(hash(main.hooks)).toBe(HEAD_MAIN_HOOK_SHA256)
expect(hash(main.bindings)).toBe(HEAD_HOOK_BINDING_SHA256)
expect(main.callbacks).toHaveLength(77)
expect(main.callbacks).toHaveLength(78)
expect(hash(main.callbacks)).toBe(HEAD_CALLBACK_IDENTITY_SHA256)
expect(hash(main.callbackBodies)).toBe(HEAD_CALLBACK_BODY_SHA256)
expect(main.effects).toHaveLength(24)
expect(main.effects).toHaveLength(27)
expect(hash(main.effects)).toBe(HEAD_EFFECT_SHA256)
expect(contentBindings).toHaveLength(14)
expect(hash(contentBindings)).toBe(HEAD_CONTENT_HOOK_SHA256)
const nestedFunctions = readNestedFunctions(definitions)
expect(nestedFunctions).toHaveLength(12)
expect(nestedFunctions).toHaveLength(13)
expect(hash(nestedFunctions)).toBe(HEAD_NESTED_FUNCTION_SHA256)
})
@@ -494,20 +497,23 @@ describe('mobile session route extraction parity', () => {
expect(hash(native.registrations)).toBe(HEAD_NATIVE_REGISTRATION_SHA256)
expect(native.removals).toHaveLength(9)
expect(hash(native.removals)).toBe(HEAD_NATIVE_REMOVAL_SHA256)
expect(native.creations.filter((fact) => fact.startsWith('setTimeout'))).toHaveLength(7)
expect(native.creations.filter((fact) => fact.startsWith('setTimeout'))).toHaveLength(8)
expect(native.creations.filter((fact) => fact.startsWith('setInterval'))).toHaveLength(1)
expect(
native.creations.filter((fact) => fact.startsWith('requestAnimationFrame'))
).toHaveLength(1)
expect(hash(native.creations)).toBe(HEAD_TIMER_CREATION_SHA256)
expect(native.cleanups.filter((fact) => fact.startsWith('clearTimeout'))).toHaveLength(11)
expect(native.cleanups.filter((fact) => fact.startsWith('clearTimeout'))).toHaveLength(12)
expect(native.cleanups.filter((fact) => fact.startsWith('clearInterval'))).toHaveLength(1)
expect(native.cleanups.filter((fact) => fact.startsWith('cancelAnimationFrame'))).toHaveLength(
1
)
expect(hash(native.cleanups)).toBe(HEAD_TIMER_CLEANUP_SHA256)
const compatibility = readCompatibilityFacts(definitions)
expect(compatibility.identityFields).toHaveLength(14)
// 13, not 14: both worktree.activate call sites now share one payload builder, so the
// literal `notifyClients: false` they used to repeat appears once. The guarantee itself is
// pinned in mobile-session-startup-source.test.ts, which requires exactly one call site.
expect(compatibility.identityFields).toHaveLength(13)
expect(hash(compatibility.identityFields)).toBe(HEAD_IDENTITY_FIELD_SHA256)
expect(compatibility.navigation).toHaveLength(6)
expect(hash(compatibility.navigation)).toBe(HEAD_NAVIGATION_SHA256)
@@ -517,14 +523,14 @@ describe('mobile session route extraction parity', () => {
it('preserves runtime strings, styles, and the expanded JSX tree', () => {
const strings = readRuntimeStrings()
expect(strings).toHaveLength(546)
expect(strings).toHaveLength(545)
expect(hash(strings)).toBe(HEAD_RUNTIME_STRING_SHA256)
const jsx = readJsxFacts(readDefinitions())
expect(jsx.host).toHaveLength(124)
expect(jsx.host).toHaveLength(127)
expect(hash(jsx.host)).toBe(HEAD_HOST_JSX_SHA256)
expect(jsx.leaf).toHaveLength(61)
expect(jsx.leaf).toHaveLength(62)
expect(hash(jsx.leaf)).toBe(HEAD_LEAF_JSX_SHA256)
expect(jsx.styleReferences).toHaveLength(172)
expect(jsx.styleReferences).toHaveLength(176)
expect(hash(jsx.styleReferences)).toBe(HEAD_STYLE_REFERENCE_SHA256)
})
})
@@ -33,6 +33,7 @@ export const MOBILE_SESSION_ROUTE_SOURCE_FILES = [
'./use-mobile-session-content-create-actions.ts',
'./use-mobile-session-close-actions.ts',
'./use-mobile-session-bulk-close.ts',
'./use-mobile-session-tab-strip-cache.ts',
'./use-mobile-session-presentation.ts',
'./use-mobile-session-panel-route-actions.tsx',
'./MobileSessionMarkdownReader.tsx',
@@ -0,0 +1,279 @@
import { createElement, type ReactElement } from 'react'
import { act, create, type ReactTestRenderer } from 'react-test-renderer'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { useMobileSessionStartup } from './use-mobile-session-startup'
import type { MobileSessionKeyboardStateModel } from './use-mobile-session-keyboard-state'
type Deferred<T> = { promise: Promise<T>; resolve: (value: T) => void; reject: (e: Error) => void }
function defer<T>(): Deferred<T> {
let resolve!: (value: T) => void
let reject!: (error: Error) => void
const promise = new Promise<T>((res, rej) => {
resolve = res
reject = rej
})
return { promise, resolve, reject }
}
type StartupCall = { rpc: 'session.tabs.list' | 'terminal.list'; worktreeId: string }
// One session's worth of scope: only the fields useMobileSessionStartup actually reads, plus
// the two reads under test wired to deferreds so a test controls exactly when they settle.
function makeScope(worktreeId: string, calls: StartupCall[], protocolVerified = true) {
const tabs = defer<void>()
const terminals = defer<boolean>()
const sendRequest = vi.fn().mockResolvedValue({ ok: true, result: {} })
const scope = {
hostId: 'host-1',
worktreeId,
created: '0',
isFloatingWorkspaceRoute: false,
connState: 'connected',
client: { sendRequest },
protocolVerified,
setTerminals: vi.fn(),
terminalsRef: { current: [] },
setSessionTabs: vi.fn(),
appliedSnapshotMarkerRef: { current: { epoch: null, version: -1 } },
closedTabTombstonesRef: { current: new Map() },
setTerminalsLoaded: vi.fn(),
setActiveHandle: vi.fn(),
setActiveSessionTabId: vi.fn(),
setMarkdownDocs: vi.fn(),
setFileDocs: vi.fn(),
terminalGestureInputQueuesRef: { current: new Map() },
terminalGestureInputInFlightRef: { current: new Set() },
sessionTabActionSheetKeyboardHideSubRef: { current: null },
sessionTabActionSheetRequestSeqRef: { current: 0 },
initializedHandlesRef: { current: new Set<string>() },
terminalDiagnosticsRef: { current: { resetRoute: vi.fn() } },
activeHandleRef: { current: null },
activeSessionTabTypeRef: { current: null },
pendingActiveSessionTabIdRef: { current: null },
selectedSessionTabIdRef: { current: null },
pendingActiveTerminalHandleRef: { current: null },
pendingBrowserFocusPageIdRef: { current: null },
pendingTerminalActivationAttemptRef: { current: null },
initialSessionAutoCreateRef: { current: null },
bufferedTerminalDraftState: { resetDrafts: vi.fn(), clearPendingRestorations: vi.fn() },
clearPendingLiveInputCommit: vi.fn(),
clearDelayedActionTimers: vi.fn(),
showToast: vi.fn(),
clearTerminalCache: vi.fn(),
fetchTerminals: vi.fn(() => {
calls.push({ rpc: 'terminal.list', worktreeId })
return terminals.promise
}),
ensureSessionTabs: vi.fn(() => {
calls.push({ rpc: 'session.tabs.list', worktreeId })
return tabs.promise
})
}
return {
scope: scope as unknown as MobileSessionKeyboardStateModel,
tabs,
terminals,
sendRequest,
activateCalls: () =>
sendRequest.mock.calls.filter(([method]) => method === 'worktree.activate').length
}
}
function StartupHarness({
scope
}: {
scope: MobileSessionKeyboardStateModel
}): ReactElement | null {
useMobileSessionStartup(scope)
return null
}
async function flush(): Promise<void> {
await act(async () => {
await Promise.resolve()
await Promise.resolve()
await Promise.resolve()
})
}
describe('mobile session startup parallelism', () => {
let renderer: ReactTestRenderer | null = null
beforeEach(() => {
vi.useFakeTimers({ shouldAdvanceTime: true })
})
afterEach(() => {
act(() => renderer?.unmount())
renderer = null
vi.useRealTimers()
})
it('puts session.tabs.list and terminal.list on the wire together', async () => {
const calls: StartupCall[] = []
const { scope } = makeScope('wt-1', calls)
await act(async () => {
renderer = create(createElement(StartupHarness, { scope }))
await Promise.resolve()
})
await flush()
// Neither deferred has settled, so both requests are in flight at the same moment. Under the
// old chain the second call could not have been made until the first resolved.
expect(calls).toEqual([
{ rpc: 'session.tabs.list', worktreeId: 'wt-1' },
{ rpc: 'terminal.list', worktreeId: 'wt-1' }
])
})
it('isolates each read so one rejection cannot strand the follow-up refreshes', async () => {
const calls: StartupCall[] = []
const { scope, tabs, terminals } = makeScope('wt-1', calls)
await act(async () => {
renderer = create(createElement(StartupHarness, { scope }))
await Promise.resolve()
})
await flush()
await act(async () => {
tabs.reject(new Error('tabs rejected'))
terminals.reject(new Error('terminals rejected'))
await Promise.resolve()
})
await flush()
await act(async () => {
vi.advanceTimersByTime(1600)
await Promise.resolve()
})
// The 750 ms and 1500 ms follow-up refreshes still armed despite both rejections; an
// unguarded await would have thrown out of the startup block and armed neither.
expect(calls.filter((call) => call.rpc === 'terminal.list')).toHaveLength(3)
})
it('drops results that land after the route moved to another session', async () => {
const calls: StartupCall[] = []
const first = makeScope('wt-1', calls)
const second = makeScope('wt-2', calls)
await act(async () => {
renderer = create(createElement(StartupHarness, { scope: first.scope }))
await Promise.resolve()
})
await flush()
await act(async () => {
renderer?.update(createElement(StartupHarness, { scope: second.scope }))
await Promise.resolve()
})
await flush()
// The first session's reads land only now, after its effect was torn down.
await act(async () => {
first.tabs.resolve(undefined)
first.terminals.resolve(true)
await Promise.resolve()
})
await flush()
await act(async () => {
vi.advanceTimersByTime(1600)
await Promise.resolve()
})
// Why: a stale settlement must not schedule refreshes for a worktree the route has left.
expect(calls.filter((call) => call.worktreeId === 'wt-1')).toHaveLength(2)
})
it('withholds worktree.activate until the compatibility verdict lands', async () => {
const calls: StartupCall[] = []
const pending = makeScope('wt-1', calls, false)
await act(async () => {
renderer = create(createElement(StartupHarness, { scope: pending.scope }))
await Promise.resolve()
})
await flush()
// Why: a desktop that omits protocolVersion evaluates as version 0 and IS blocked, so the
// routes that now mount pre-verdict must not mutate a host the gate is about to refuse.
expect(pending.activateCalls()).toBe(0)
// The reads are not held back with it; that is the whole point of mounting early.
expect(calls).toHaveLength(2)
})
it('activates once the verdict lands without re-issuing the reads', async () => {
const calls: StartupCall[] = []
const pending = makeScope('wt-1', calls, false)
await act(async () => {
renderer = create(createElement(StartupHarness, { scope: pending.scope }))
await Promise.resolve()
})
await flush()
expect(pending.activateCalls()).toBe(0)
// Same session, verdict now proven: only the activation effect may re-run.
const verified = {
...(pending.scope as unknown as Record<string, unknown>),
protocolVerified: true
} as unknown as MobileSessionKeyboardStateModel
await act(async () => {
renderer?.update(createElement(StartupHarness, { scope: verified }))
await Promise.resolve()
})
await flush()
expect(pending.activateCalls()).toBe(1)
expect(pending.sendRequest).toHaveBeenCalledWith('worktree.activate', {
worktree: 'id:wt-1',
notifyClients: false,
navigation: 'caller'
})
expect(calls).toHaveLength(2)
})
it('discards both parallel results when the session changes mid-flight', async () => {
const calls: StartupCall[] = []
const first = makeScope('wt-1', calls)
const second = makeScope('wt-2', calls)
await act(async () => {
renderer = create(createElement(StartupHarness, { scope: first.scope }))
await Promise.resolve()
})
await flush()
expect(calls.filter((call) => call.worktreeId === 'wt-1')).toHaveLength(2)
await act(async () => {
renderer?.update(createElement(StartupHarness, { scope: second.scope }))
await Promise.resolve()
})
await flush()
// Tabs land late first, then terminals, so each is separately proven inert.
await act(async () => {
first.tabs.resolve(undefined)
await Promise.resolve()
})
await flush()
await act(async () => {
vi.advanceTimersByTime(1600)
await Promise.resolve()
})
expect(calls.filter((call) => call.worktreeId === 'wt-1')).toHaveLength(2)
await act(async () => {
first.terminals.resolve(true)
await Promise.resolve()
})
await flush()
await act(async () => {
vi.advanceTimersByTime(1600)
await Promise.resolve()
})
expect(calls.filter((call) => call.worktreeId === 'wt-1')).toHaveLength(2)
})
})
@@ -30,6 +30,10 @@ const autoCreateHookSource = readMobileSessionRouteSource(
'./use-initial-session-terminal-autocreate.ts'
)
const foundationSource = readMobileSessionRouteSource('./use-mobile-session-foundation.ts')
const activeContentSource = readMobileSessionRouteSource('./MobileSessionActiveContent.tsx')
const subscriptionFoundationSource = readMobileSessionRouteSource(
'./use-mobile-session-terminal-subscription-foundation.ts'
)
const terminalRuntimeSource = readMobileSessionRouteSource(
'./use-mobile-session-terminal-runtime.ts'
)
@@ -147,35 +151,72 @@ describe('mobile session startup', () => {
)
})
it('loads session tabs without waiting for desktop activation', () => {
const startupEffect = sliceBetween(
// Was: one effect that awaited tabs, then terminals, and fired worktree.activate alongside them.
// The reads are now concurrent and unblocked, while the activation moved to its own effect that
// waits for the compatibility verdict, because it writes host state.
it('loads session tabs and terminals concurrently, ahead of any desktop activation', () => {
const readEffect = sliceBetween(
'void (async () => {',
'return () => {\n disposed = true',
startupSource
)
expect(startupEffect).toContain("void client\n .sendRequest('worktree.activate'")
expect(startupEffect).toContain("if (client && created !== '1' && !isFloatingWorkspaceRoute)")
expect(startupEffect).toContain("if (client && created === '1' && !isFloatingWorkspaceRoute)")
expect(startupEffect).toContain('notifyClients: false')
expect(startupEffect).toContain("navigation: 'caller'")
expect(startupEffect).not.toContain("await client\n .sendRequest('worktree.activate'")
expect(startupEffect.indexOf("sendRequest('worktree.activate'")).toBeLessThan(
startupEffect.indexOf('await ensureSessionTabs()')
expect(readEffect).toContain(
'await Promise.all([\n ensureSessionTabs().catch(() => null),\n fetchTerminals({ allowEmptyLoaded: false }).catch(() => false)\n ])'
)
expect(startupEffect).toContain('headlessActivationNeedsHostRenderer(response.result)')
expect(startupEffect).toContain("showToast('Open Orca on the host to wake sleeping agents.'")
// The reads must not wait on the verdict; that is the point of mounting under the gate.
expect(readEffect).not.toContain('protocolVerified')
expect(readEffect).not.toContain('worktree.activate')
expect(startupSource).toContain('}, [connState, fetchTerminals, ensureSessionTabs])')
})
it('fails runtime capability gates closed before probing a replacement client', () => {
it('holds worktree.activate until the compatibility verdict lands', () => {
const activationEffect = sliceBetween(
"if (connState !== 'connected' || !client || !protocolVerified || isFloatingWorkspaceRoute) {",
'return () => {\n disposed = true',
startupSource.slice(startupSource.indexOf('worktree.activate') - 2000)
)
// Why: a desktop that omits protocolVersion reads as version 0 and IS blocked, so mounting
// this route pre-verdict must not let it mutate a host the gate is about to refuse.
expect(activationEffect).toContain("sendRequest('worktree.activate'")
expect(activationEffect).toContain('notifyClients: false')
expect(activationEffect).toContain("navigation: 'caller'")
expect(activationEffect).toContain("if (created !== '1') {")
expect(activationEffect).toContain('headlessActivationNeedsHostRenderer(response.result)')
expect(activationEffect).toContain("showToast('Open Orca on the host to wake sleeping agents.'")
// The only worktree.activate calls in the route are the two this gated effect owns.
expect(startupSource.split("sendRequest('worktree.activate'")).toHaveLength(2)
expect(startupSource).toContain(' protocolVerified,\n showToast,\n worktreeId\n ])')
})
// Was: this route ran its own retrying status.get. The gate above every /h/ route already
// holds that answer, so the second request is gone and the gates read it instead.
it('fails runtime capability gates closed until the shared status.get is proven', () => {
const capabilityEffect = sliceBetween(
'const hostQueryReplyInputSupportedRef = useRef(false)',
'return {\n consumeAcceptedSessionTabs',
tabReconciliationSource
)
const probeStart = capabilityEffect.indexOf('startRuntimeCapabilityProbe(client,')
expect(probeStart).toBeGreaterThanOrEqual(0)
expect(tabReconciliationSource).not.toContain('startRuntimeCapabilityProbe')
expect(tabReconciliationSource).not.toContain('useHostProtocolGates')
// One read of the gate for the whole route, taken in the foundation and passed down.
expect(foundationSource).toContain(
'const { compatVerdict, compatVerified, hostCapabilities, statusPending } = useHostProtocolGates()'
)
// Settled is not passing, and passing-by-fallback is not answered. The write gate reads all
// three, so a host that never answered status.get cannot be mistaken for a verified one.
expect(foundationSource).toContain(
"const protocolVerified = !statusPending && compatVerified && compatVerdict.kind === 'ok'"
)
expect(capabilityEffect).toContain(
"if (!client || connState !== 'connected' || !protocolVerified) {"
)
const readStart = capabilityEffect.indexOf(
"setBrowserScreencastSupported(hostCapabilities.includes('browser.screencast.v1'))"
)
expect(readStart).toBeGreaterThanOrEqual(0)
for (const reset of [
'setBrowserScreencastSupported(null)',
'setAgentSessionHistorySupported(null)',
@@ -185,7 +226,7 @@ describe('mobile session startup', () => {
]) {
const resetIndex = capabilityEffect.lastIndexOf(reset)
expect(resetIndex).toBeGreaterThanOrEqual(0)
expect(resetIndex).toBeLessThan(probeStart)
expect(resetIndex).toBeLessThan(readStart)
}
})
@@ -290,4 +331,43 @@ describe('mobile session startup', () => {
expect(source).toContain('onPendingTerminalRecoveryParked: setParkedPendingTerminalContext')
expect(source).toContain('retryPendingTerminalRecovery()')
})
it('boots the terminal engine while the startup reads are still in flight', () => {
// Why: the loading and pending-terminal states are exactly the window in which the startup
// RPCs are outstanding, so the engine loads there rather than after terminal.list answers.
const loadingBranch = sliceBetween(
"return reconnectViewState.kind === 'reconnecting-with-cache' || showLoadingState ? (",
') : showEmptyState ? (',
activeContentSource
)
const prewarmElement =
'<TerminalEnginePrewarm\n reservedTabBarHeight={prewarmReservedTabBarHeight}\n textScale={terminalTextScale}\n onEngineMeasured={measurePrewarmViewport}\n />'
expect(loadingBranch).toContain(prewarmElement)
expect(loadingBranch).toContain('<View style={styles.terminalFrame}>')
const pendingBranch = sliceBetween(
') : activePendingTerminalTab ? (',
') : (\n <View\n style={styles.terminalFrame}',
activeContentSource
)
expect(pendingBranch).toContain(prewarmElement)
// The pre-warm never reaches a terminal: the pane list is still the only attachment point.
expect(activeContentSource).toContain('{terminals.map((terminal) => (')
expect(activeContentSource.indexOf('<TerminalEnginePrewarm')).toBeLessThan(
activeContentSource.indexOf('{terminals.map((terminal) => (')
)
})
it('refuses a pre-warm viewport measured before the frame had a height', () => {
const measure = sliceBetween(
'const measurePrewarmViewport = useCallback(',
' return {\n getTerminalRef',
subscriptionFoundationSource
)
expect(measure).toContain('if (viewportMeasuredRef.current || frameHeight <= 0) {')
expect(measure).toContain('await engine.measureFitDimensions(frameHeight)')
// Why: the latch is re-checked after the await so a real pane that measured first wins.
expect(measure).toContain('if (dims && !viewportMeasuredRef.current) {')
})
})
@@ -0,0 +1,116 @@
import { TUI_AGENT_DISPLAY_NAMES } from '../../../src/shared/tui-agent-display-names'
import type { MobileSessionTab, MobileSessionTabType } from './mobile-session-route-types'
import {
getMobileSessionTabTitle,
resolveMobileTerminalTabAgentId
} from './mobile-terminal-tab-agent'
/**
* The only session-tab fields the tab strip draws. Everything else the live tab carries (unsent
* launch drafts, absolute file paths, browser URLs, agent session ids) stays on the wire.
*/
export type MobileSessionTabStripEntry = {
id: string
type: MobileSessionTabType
title: string
agentId: string | null
}
export type MobileSessionTabStripPreview = {
tabs: readonly MobileSessionTabStripEntry[]
activeTabId: string | null
}
export type MobileSessionTabStripRow = {
entry: MobileSessionTabStripEntry
isActive: boolean
/** null on a preview row: switching to that tab needs a live connection. */
tab: MobileSessionTab | null
}
export function toMobileSessionTabStripEntry(tab: MobileSessionTab): MobileSessionTabStripEntry {
return {
id: tab.id,
type: tab.type,
title: getMobileSessionTabTitle(tab),
agentId:
tab.type === 'agent-session'
? tab.agent
: tab.type === 'terminal'
? resolveMobileTerminalTabAgentId(tab)
: null
}
}
/**
* Every tab type the strip knows how to draw. A stored entry naming anything else is dropped
* rather than trusted, so a type added later fails closed: its rows go missing from the preview
* instead of carrying an unreviewed title into storage.
*/
const drawableTabTypes = new Set<string>([
'terminal',
'markdown',
'file',
'browser',
'agent-session'
] satisfies readonly MobileSessionTabType[])
export function isDrawableTabStripType(type: string): type is MobileSessionTabType {
return drawableTabTypes.has(type)
}
const agentDisplayNames: Readonly<Record<string, string>> = TUI_AGENT_DISPLAY_NAMES
/**
* The title a strip entry may be written to disk under.
*
* A terminal's title is whatever the shell last set, which is routinely the command line
* `psql postgres://user:password@host/db`, `curl -H "Authorization: Bearer ..."`. None of that
* belongs in plaintext storage, and a browser tab's page title is no better. Both collapse to a
* fixed label, so what survives is the shape of the strip, not its contents. A resolved agent
* still names itself, because that lookup is a closed enum: an unrecognised id yields the
* generic label rather than passing text through.
*/
export function getPersistableTabStripTitle(
entry: Pick<MobileSessionTabStripEntry, 'type' | 'title' | 'agentId'>
): string {
if (entry.type === 'terminal') {
const agentLabel = entry.agentId === null ? undefined : agentDisplayNames[entry.agentId]
return agentLabel ?? 'Terminal'
}
if (entry.type === 'browser') {
return 'Browser'
}
return entry.title
}
export function toMobileSessionTabStripPreview(
tabs: readonly MobileSessionTab[],
activeTabId: string | null
): MobileSessionTabStripPreview {
return { tabs: tabs.map(toMobileSessionTabStripEntry), activeTabId }
}
/**
* Rows for the header strip. Live tabs always win; the preview only fills a strip that has no
* live rows yet, and its ids are the live ids, so the swap reuses the same React keys.
*/
export function getMobileSessionTabStripRows(args: {
liveTabs: readonly MobileSessionTab[]
activeSessionTabId: string | null
preview: MobileSessionTabStripPreview | null
}): MobileSessionTabStripRow[] {
const { liveTabs, activeSessionTabId, preview } = args
if (liveTabs.length > 0 || !preview) {
return liveTabs.map((tab) => ({
entry: toMobileSessionTabStripEntry(tab),
isActive: tab.id === activeSessionTabId,
tab
}))
}
return preview.tabs.map((entry) => ({
entry,
isActive: entry.id === preview.activeTabId,
tab: null
}))
}
@@ -0,0 +1,181 @@
import { createElement } from 'react'
import { act, create, type ReactTestRenderer } from 'react-test-renderer'
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { TerminalWebViewHandle } from '../terminal/terminal-webview-contract'
import { readMobileSessionRouteSource } from './mobile-session-route-source-family.test-support'
type StyleLayer = { top?: number }
// The applied top offset, read off the rendered pane rather than assumed.
function appliedTopOffset(style: unknown): number {
const layers = (Array.isArray(style) ? style : [style]) as (StyleLayer | null | undefined)[]
return layers.reduce<number>(
(top, layer) => (typeof layer?.top === 'number' ? layer.top : top),
0
)
}
const engine = vi.hoisted(() => ({
init: vi.fn((_cols: number, _rows: number) => {}),
awaitReady: vi.fn(async () => {}),
measureFitDimensions: vi.fn(async (_containerHeight?: number) => ({ cols: 100, rows: 40 })),
onWebReady: null as (() => void) | null
}))
vi.mock('react-native', () => ({
StyleSheet: {
create: <T>(styles: T) => styles,
absoluteFillObject: { position: 'absolute', top: 0, left: 0, right: 0, bottom: 0 }
},
View: 'View'
}))
vi.mock('../terminal/TerminalWebView', async () => {
const { forwardRef, useImperativeHandle } = await import('react')
return {
TerminalWebView: forwardRef<TerminalWebViewHandle, { onWebReady?: () => void }>(
function MockTerminalWebView(props, ref) {
engine.onWebReady = props.onWebReady ?? null
useImperativeHandle(ref, () => engine as unknown as TerminalWebViewHandle, [])
return createElement('MockTerminalWebView')
}
)
}
})
import { TerminalEnginePrewarm } from './TerminalEnginePrewarm'
import {
MOBILE_SESSION_TAB_BAR_BORDER_WIDTH,
MOBILE_SESSION_TAB_BAR_CONTENT_HEIGHT,
MOBILE_SESSION_TAB_BAR_HEIGHT,
mobileSessionFrameStyles
} from './mobile-session-frame-styles'
// The box the session content row occupies. Both states below live in it, so it is the one
// number the two frame heights are derived from.
const CONTENT_ROW_HEIGHT = 700
// The bar's rendered height, derived from the styles the header actually mounts rather than from
// the constant the pre-warm consumes — otherwise the comparison below would just restate itself.
// React Native sizes a row with no explicit height to its tallest child and puts the border
// outside that, so this is max(children) + border.
function renderedTabBarHeight(): number {
const row = mobileSessionFrameStyles.tabBar as { height?: number; borderTopWidth: number }
// An explicit height here would be border-box and would shrink the row below its children.
expect(row.height).toBeUndefined()
const tallestChild = Math.max(
mobileSessionFrameStyles.tabScroll.maxHeight,
mobileSessionFrameStyles.tab.minHeight,
mobileSessionFrameStyles.newTerminalButton.height,
mobileSessionFrameStyles.tabActionDivider.height
)
return tallestChild + row.borderTopWidth
}
// What the first real pane gets once its tab exists and the bar mounts above it.
function firstPaneFrameHeight(): number {
return CONTENT_ROW_HEIGHT - renderedTabBarHeight()
}
const headerSource = readMobileSessionRouteSource('./MobileSessionHeader.tsx')
const activeContentSource = readMobileSessionRouteSource('./MobileSessionActiveContent.tsx')
// Reproduces React Native's absolute-fill layout: a box pinned to every edge of its parent with
// a top offset gets exactly that much less height. The offset is read off the component, never
// assumed, so a pre-warm that stopped reserving the bar would report the taller box here.
function measuredPrewarmHeight(reservedTabBarHeight: number): number {
let renderer: ReactTestRenderer | null = null
act(() => {
renderer = create(
createElement(TerminalEnginePrewarm, { reservedTabBarHeight, onEngineMeasured: () => {} })
)
})
const created = renderer as unknown as ReactTestRenderer
const applied = appliedTopOffset(created.root.findAllByType('View')[0]?.props.style)
act(() => created.unmount())
return CONTENT_ROW_HEIGHT - applied
}
afterEach(() => {
engine.measureFitDimensions.mockClear()
engine.onWebReady = null
})
describe('terminal pre-warm frame geometry', () => {
it('states the height the bar actually renders at', () => {
// The constant is what the pre-warm reserves, so it has to equal what the header mounts.
// Deriving the latter from the styles catches the border-box trap: pinning an explicit
// height on the row would render it a pixel short of this sum and drift a whole row.
expect(renderedTabBarHeight()).toBe(MOBILE_SESSION_TAB_BAR_HEIGHT)
expect(MOBILE_SESSION_TAB_BAR_HEIGHT).toBe(
MOBILE_SESSION_TAB_BAR_CONTENT_HEIGHT + MOBILE_SESSION_TAB_BAR_BORDER_WIDTH
)
expect(mobileSessionFrameStyles.tabBar.borderTopWidth).toBe(MOBILE_SESSION_TAB_BAR_BORDER_WIDTH)
// Every child is pinned to the content height, so nothing can grow the row unnoticed.
expect(mobileSessionFrameStyles.tabScroll.maxHeight).toBe(MOBILE_SESSION_TAB_BAR_CONTENT_HEIGHT)
expect(mobileSessionFrameStyles.tab.minHeight).toBe(MOBILE_SESSION_TAB_BAR_CONTENT_HEIGHT)
expect(mobileSessionFrameStyles.newTerminalButton.height).toBe(
MOBILE_SESSION_TAB_BAR_CONTENT_HEIGHT
)
})
it('mounts the tab bar only once a tab is visible, which is what shortens the pane', () => {
expect(headerSource).toContain(
'{tabStripRows.length > 0 && (\n <View style={styles.tabBar}>'
)
// So the reservation has to be the exact complement of that condition, read off the same rows
// the header gates on (live tabs or the cached reconnect preview) rather than a proxy for it.
expect(activeContentSource).toContain(
'const prewarmReservedTabBarHeight = tabStripRows.length > 0 ? 0 : MOBILE_SESSION_TAB_BAR_HEIGHT'
)
})
it('measures the same frame height the first real pane will get', () => {
// Loading: no visible tab, so no tab bar, so the content row is all the pre-warm's to fill,
// minus whatever it reserves. Loaded: the first terminal produces a tab, the bar mounts, and
// the pane gets what is left. The right side is derived from the header's own styles.
expect(measuredPrewarmHeight(MOBILE_SESSION_TAB_BAR_HEIGHT)).toBe(firstPaneFrameHeight())
})
it('would latch a taller frame than the pane if the bar were not reserved', () => {
// Guards the fix rather than the code: without the reservation the pre-warm measures the
// pre-tab-bar box, and every row of that difference is a row the host never had.
const unreserved = measuredPrewarmHeight(0)
expect(unreserved).toBe(CONTENT_ROW_HEIGHT)
expect(unreserved - firstPaneFrameHeight()).toBe(renderedTabBarHeight())
})
it('hands the engine the reserved height, so no refit is owed after the first subscribe', async () => {
let measuredWith: number | null = null
let renderer: ReactTestRenderer | null = null
act(() => {
renderer = create(
createElement(TerminalEnginePrewarm, {
reservedTabBarHeight: MOBILE_SESSION_TAB_BAR_HEIGHT,
textScale: 1,
onEngineMeasured: (_ref: unknown, frameHeight: number) => {
measuredWith = frameHeight
}
})
)
})
const created = renderer as unknown as ReactTestRenderer
const pane = created.root.findAllByType('View')[0]
const applied = appliedTopOffset(pane?.props.style)
act(() => {
pane?.props.onLayout({
nativeEvent: { layout: { x: 0, y: 0, width: 390, height: CONTENT_ROW_HEIGHT - applied } }
})
})
act(() => {
engine.onWebReady?.()
})
// The handoff waits on the engine's ready promise, so let those microtasks land.
await act(async () => {})
// The height the latched viewport is computed from equals the real pane's frame height, so
// the frame-height refit re-measures the same cols/rows and returns before it would send
// terminal.updateViewport (see the prev-dims guard in terminal-viewport-refit.ts).
expect(measuredWith).toBe(firstPaneFrameHeight())
act(() => created.unmount())
})
})
@@ -0,0 +1,147 @@
import { createElement, useRef, type ReactElement } from 'react'
import { act, create, type ReactTestRenderer } from 'react-test-renderer'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type { RpcClient } from '../transport/rpc-client'
import type { TerminalWebViewHandle } from '../terminal/terminal-webview-contract'
vi.mock('react-native', () => ({
AppState: { currentState: 'active', addEventListener: () => ({ remove: () => {} }) },
Platform: { OS: 'android' },
StyleSheet: {
create: <T>(styles: T) => styles,
absoluteFillObject: { position: 'absolute', top: 0, left: 0, right: 0, bottom: 0 },
hairlineWidth: 1
},
useWindowDimensions: () => ({ width: 390, height: 844 }),
View: 'View'
}))
import { useTerminalViewportRefit } from '../terminal/terminal-viewport-refit'
import { MOBILE_SESSION_TAB_BAR_HEIGHT } from './mobile-session-frame-styles'
const CONTENT_ROW_HEIGHT = 700
const CELL_HEIGHT = 17
const HANDLE = 'term-1'
const REFIT_DEBOUNCE_MS = 150
// Stands in for the WebView's fit: the taller the box it is handed, the more rows it reports.
// This is what turns a frame that is one tab bar too tall into a row count the host never had.
function fitDimensions(containerHeight: number): { cols: number; rows: number } {
return { cols: 100, rows: Math.floor(containerHeight / CELL_HEIGHT) }
}
type ColdOpenResult = {
updateViewportCalls: number
resubscribes: number
latchedRows: number
}
// Replays a single-terminal cold open: the pre-warm measured `prewarmFrameHeight` and latched it,
// the first pane subscribed with those dims, and only then does the real frame report its layout.
async function runSingleTerminalColdOpen(prewarmFrameHeight: number): Promise<ColdOpenResult> {
const firstPaneFrameHeight = CONTENT_ROW_HEIGHT - MOBILE_SESSION_TAB_BAR_HEIGHT
const sendRequest = vi.fn(async () => ({ ok: true, result: { updated: true, applied: true } }))
const client = {
sendRequest,
updateTerminalSubscriptionViewport: vi.fn()
} as unknown as RpcClient
const engine = {
measureFitDimensions: vi.fn(async (containerHeight?: number) =>
fitDimensions(containerHeight ?? 0)
),
reflow: vi.fn()
} as unknown as TerminalWebViewHandle
const subscribeToTerminal = vi.fn()
const unsubscribeTerminal = vi.fn()
const viewport = { current: fitDimensions(prewarmFrameHeight) as { cols: number; rows: number } }
const viewportMeasured = { current: true }
// The real pane's frame, reported by its onLayout once the tab bar has mounted.
const frameHeight = { current: firstPaneFrameHeight }
let notify: ((height: number) => void) | null = null
function RefitHarness(): ReactElement | null {
const terminalRefs = useRef(new Map([[HANDLE, engine]]))
const { notifyTerminalFrameHeight } = useTerminalViewportRefit({
activeHandleRef: useRef<string | null>(HANDLE),
terminalRefs,
terminalFrameHeightRef: frameHeight,
viewportRef: viewport,
viewportMeasuredRef: viewportMeasured,
nativeChatCoveredRef: useRef(false),
clientRef: useRef<RpcClient | null>(client),
deviceTokenRef: useRef<string | null>('device-1'),
initializedHandlesRef: useRef(new Set([HANDLE])),
connState: 'connected',
// One terminal, so the tab-strip corrector is not armed — this is the case that used to
// fall through to the frame-height reducer and pay for the mis-measurement.
tabStripVisible: false,
textScale: 1,
terminalFrameWidth: 390,
unsubscribeTerminal,
subscribeToTerminal
})
notify = notifyTerminalFrameHeight
return null
}
let renderer: ReactTestRenderer | null = null
await act(async () => {
renderer = create(createElement(RefitHarness))
})
await act(async () => {
notify?.(firstPaneFrameHeight)
})
// Why drain microtasks between ticks and before unmount: the refit measures and sends inside an
// async block that bails once disposedRef flips, so tearing down early would fake a clean run.
await act(async () => {
vi.advanceTimersByTime(REFIT_DEBOUNCE_MS + 1)
for (let i = 0; i < 10; i += 1) {
await Promise.resolve()
}
})
await act(async () => {
vi.advanceTimersByTime(REFIT_DEBOUNCE_MS + 1)
for (let i = 0; i < 10; i += 1) {
await Promise.resolve()
}
})
act(() => (renderer as unknown as ReactTestRenderer).unmount())
return {
updateViewportCalls: sendRequest.mock.calls.filter(
([method]) => method === 'terminal.updateViewport'
).length,
resubscribes: subscribeToTerminal.mock.calls.length,
latchedRows: viewport.current.rows
}
}
describe('terminal pre-warm refit debt', () => {
beforeEach(() => {
vi.useFakeTimers({ shouldAdvanceTime: true })
})
afterEach(() => {
vi.useRealTimers()
})
it('owes the host nothing after the first subscribe when the pre-warm reserved the tab bar', async () => {
const reserved = CONTENT_ROW_HEIGHT - MOBILE_SESSION_TAB_BAR_HEIGHT
const result = await runSingleTerminalColdOpen(reserved)
expect(result.updateViewportCalls).toBe(0)
expect(result.resubscribes).toBe(0)
expect(result.latchedRows).toBe(fitDimensions(reserved).rows)
})
it('pays a terminal.updateViewport round trip if the pre-warm measured the pre-tab-bar box', async () => {
// Guards the fix, not the code: this is the frame the pre-warm saw before it reserved the bar.
const result = await runSingleTerminalColdOpen(CONTENT_ROW_HEIGHT)
expect(result.updateViewportCalls).toBe(1)
// And the rows it had to correct are rows the host was told about and never had.
expect(fitDimensions(CONTENT_ROW_HEIGHT).rows).toBeGreaterThan(
fitDimensions(CONTENT_ROW_HEIGHT - MOBILE_SESSION_TAB_BAR_HEIGHT).rows
)
})
})
@@ -27,6 +27,7 @@ import { useMobileSessionTerminalCreateActions } from './use-mobile-session-term
import { useMobileSessionContentCreateActions } from './use-mobile-session-content-create-actions'
import { useMobileSessionCloseActions } from './use-mobile-session-close-actions'
import { useMobileSessionBulkClose } from './use-mobile-session-bulk-close'
import { useMobileSessionTabStripCache } from './use-mobile-session-tab-strip-cache'
import { useMobileSessionPresentation } from './use-mobile-session-presentation'
import { useMobileSessionPanelRouteActions } from './use-mobile-session-panel-route-actions'
@@ -113,7 +114,8 @@ export function useMobileSessionController() {
useMobileSessionCloseActions(contentCreateActions)
)
const bulkClose = Object.assign(closeActions, useMobileSessionBulkClose(closeActions))
const presentation = Object.assign(bulkClose, useMobileSessionPresentation(bulkClose))
const tabStripCache = Object.assign(bulkClose, useMobileSessionTabStripCache(bulkClose))
const presentation = Object.assign(tabStripCache, useMobileSessionPresentation(tabStripCache))
const panelRouteActions = Object.assign(
presentation,
useMobileSessionPanelRouteActions(presentation)
@@ -14,6 +14,7 @@ import { isFloatingWorkspaceWorktreeId } from './floating-workspace'
import { useLiveWorktreeName } from './use-live-worktree-name'
import { useMissingWorktreeBounce } from './use-missing-worktree-bounce'
import { hostRouteWithNotice } from '../host-route-notice'
import { useHostProtocolGates } from '../components/HostProtocolGate'
export function useMobileSessionFoundation() {
const {
@@ -36,6 +37,14 @@ export function useMobileSessionFoundation() {
const insets = useSafeAreaInsets()
// Why: shared client per host owned by RpcClientProvider (docs/mobile-shared-client-per-host.md).
const { client, clientId, state: connState } = useHostClient(hostId)
// Why: HostProtocolGate holds this connection's single status.get. Reading it here gives the
// whole route one source for host capabilities and for whether the compatibility verdict has
// landed — the routes now mount while it is still in flight, so "not yet known" is a real state.
const { compatVerdict, compatVerified, hostCapabilities, statusPending } = useHostProtocolGates()
// Why all three: a settled verdict is not necessarily a passing one, and a settled *passing*
// verdict is not necessarily an answered one — a host that cannot answer status.get fails open
// to `ok` so navigation still works. Writes read this flag, so they wait for a real reply.
const protocolVerified = !statusPending && compatVerified && compatVerdict.kind === 'ok'
const reconnectAttempts = useReconnectAttempt(hostId)
const lastConnectedAt = useLastConnectedAt(hostId)
const forceReconnectHost = useForceReconnect()
@@ -98,6 +107,8 @@ export function useMobileSessionFoundation() {
client,
clientId,
connState,
hostCapabilities,
protocolVerified,
reconnectAttempts,
lastConnectedAt,
forceReconnectHost,
@@ -3,9 +3,11 @@ import { classifyConnection, verdictDisplayLabel } from '../transport/connection
import { computeActiveTerminalKeyboardLift } from '../terminal/terminal-keyboard-avoidance-lift'
import { useInitialSessionTerminalAutoCreate } from './use-initial-session-terminal-autocreate'
import { MOBILE_SESSION_STATUS_LABELS } from './mobile-session-route-helpers'
import type { MobileSessionBulkCloseModel } from './use-mobile-session-bulk-close'
import { selectMobileSessionReconnectViewState } from './mobile-session-reconnect-view-state'
import { getMobileSessionTabStripRows } from './mobile-session-tab-strip-entries'
import type { MobileSessionTabStripCacheModel } from './use-mobile-session-tab-strip-cache'
export function useMobileSessionPresentation(scope: MobileSessionBulkCloseModel) {
export function useMobileSessionPresentation(scope: MobileSessionTabStripCacheModel) {
const {
created,
worktreeId,
@@ -24,6 +26,8 @@ export function useMobileSessionPresentation(scope: MobileSessionBulkCloseModel)
terminalKeyboardMetrics,
toastOpacityRef,
hostEndpoint,
activeSessionTabId,
cachedTabStrip,
initialSessionAutoCreateRef,
terminalFrameHeightRef,
handleCreateTerminal,
@@ -58,6 +62,23 @@ export function useMobileSessionPresentation(scope: MobileSessionBulkCloseModel)
const showConnectionRetry =
connectionVerdict.kind === 'warning' || connectionVerdict.kind === 'unreachable'
// Why: a reconnect to a workspace this phone has already drawn should re-draw it, not blank
// the screen while the RPCs land. See mobile-session-reconnect-view-state.
const reconnectViewState = selectMobileSessionReconnectViewState({
connState,
verdictKind: connectionVerdict.kind,
terminalsLoaded,
liveTabCount: visibleTabs.length,
activeHandle,
cachedPreview: cachedTabStrip
})
const tabStripRows = getMobileSessionTabStripRows({
liveTabs: visibleTabs,
activeSessionTabId,
preview:
reconnectViewState.kind === 'reconnecting-with-cache' ? reconnectViewState.preview : null
})
const terminalSummary =
connState === 'connected'
? showLoadingState
@@ -88,6 +109,8 @@ export function useMobileSessionPresentation(scope: MobileSessionBulkCloseModel)
return {
showLoadingState,
showEmptyState,
reconnectViewState,
tabStripRows,
connectionVerdict,
showConnectionRetry,
terminalSummary,
@@ -97,5 +120,5 @@ export function useMobileSessionPresentation(scope: MobileSessionBulkCloseModel)
}
}
export type MobileSessionPresentationModel = MobileSessionBulkCloseModel &
export type MobileSessionPresentationModel = MobileSessionTabStripCacheModel &
ReturnType<typeof useMobileSessionPresentation>
@@ -12,6 +12,7 @@ export function useMobileSessionStartup(scope: MobileSessionKeyboardStateModel)
isFloatingWorkspaceRoute,
connState,
client,
protocolVerified,
setTerminals,
terminalsRef,
setSessionTabs,
@@ -95,6 +96,8 @@ export function useMobileSessionStartup(scope: MobileSessionKeyboardStateModel)
worktreeId
])
// Reads only. They carry no side effect on the host, so they do not wait on the compatibility
// verdict — that is the whole point of mounting this route while status.get is still in flight.
// Every setTimeout goes through addTimer into `timers`, which the returned cleanup clears.
// react-doctor-disable-next-line react-doctor/effect-needs-cleanup
useEffect(() => {
@@ -116,58 +119,81 @@ export function useMobileSessionStartup(scope: MobileSessionKeyboardStateModel)
timers.push(setTimeout(fn, ms))
}
void (async () => {
const reportActivationOutcome = (response: RpcSuccess | null): void => {
if (!disposed && response && headlessActivationNeedsHostRenderer(response.result)) {
showToast('Open Orca on the host to wake sleeping agents.', 3000)
}
}
if (client && created !== '1' && !isFloatingWorkspaceRoute) {
// Why: hydrate host-owned tabs without pulling other paired clients (esp. desktop) into this worktree.
void client
.sendRequest('worktree.activate', {
worktree: `id:${worktreeId}`,
notifyClients: false,
navigation: 'caller'
})
.then((response) => reportActivationOutcome(response.ok ? response : null))
.catch(() => null)
}
if (disposed) {
return
}
await ensureSessionTabs().catch(() => null)
if (disposed) {
return
}
await fetchTerminals({ allowEmptyLoaded: false })
// Why: session.tabs.list and terminal.list are independent reads, so issue both now and
// wait for the pair. Serialising them cost a full extra round trip before the first
// terminal could paint, which on a far relay cell is seconds, not milliseconds. Each
// call keeps its own catch so one rejection cannot strand the other's follow-up refreshes.
await Promise.all([
ensureSessionTabs().catch(() => null),
fetchTerminals({ allowEmptyLoaded: false }).catch(() => false)
])
if (disposed) {
return
}
addTimer(() => void fetchTerminals({ allowEmptyLoaded: false }), 750)
addTimer(() => void fetchTerminals({ allowEmptyLoaded: true }), 1500)
if (client && created === '1' && !isFloatingWorkspaceRoute) {
addTimer(() => {
if (activeHandleRef.current) {
})()
return () => {
disposed = true
for (const t of timers) {
clearTimeout(t)
}
}
// Why no client/worktreeId here: both reads are useCallbacks that already list them, so a
// host or worktree change replaces their identity and re-runs this effect with them.
}, [connState, fetchTerminals, ensureSessionTabs])
// worktree.activate writes host state, so unlike the reads above it waits for the compatibility
// verdict. A missing protocolVersion reads as 0 and is blocked, so "pending" is not a formality:
// mounting early must not let this route mutate a host the gate is about to refuse.
// Every setTimeout goes through addTimer into `timers`, which the returned cleanup clears.
// react-doctor-disable-next-line react-doctor/effect-needs-cleanup
useEffect(() => {
if (connState !== 'connected' || !client || !protocolVerified || isFloatingWorkspaceRoute) {
return
}
let disposed = false
const timers: ReturnType<typeof setTimeout>[] = []
function addTimer(fn: () => void, ms: number) {
if (disposed) {
return
}
timers.push(setTimeout(fn, ms))
}
const activateWorktree = () =>
client
.sendRequest('worktree.activate', {
worktree: `id:${worktreeId}`,
notifyClients: false,
navigation: 'caller'
})
.catch(() => null)
const reportActivationOutcome = (response: RpcSuccess | null): void => {
if (!disposed && response && headlessActivationNeedsHostRenderer(response.result)) {
showToast('Open Orca on the host to wake sleeping agents.', 3000)
}
}
if (created !== '1') {
// Why: hydrate host-owned tabs without pulling other paired clients (esp. desktop) into this worktree.
void activateWorktree().then((response) =>
reportActivationOutcome(response?.ok ? response : null)
)
} else {
addTimer(() => {
if (activeHandleRef.current) {
return
}
void (async () => {
const activationResponse = await activateWorktree()
reportActivationOutcome(activationResponse?.ok ? activationResponse : null)
if (disposed) {
return
}
void (async () => {
const activationResponse = await client
.sendRequest('worktree.activate', {
worktree: `id:${worktreeId}`,
notifyClients: false,
navigation: 'caller'
})
.catch(() => null)
reportActivationOutcome(activationResponse?.ok ? activationResponse : null)
if (disposed) {
return
}
await fetchTerminals({ allowEmptyLoaded: true })
addTimer(() => void fetchTerminals({ allowEmptyLoaded: true }), 750)
})()
}, 1800)
}
})()
await fetchTerminals({ allowEmptyLoaded: true })
addTimer(() => void fetchTerminals({ allowEmptyLoaded: true }), 750)
})()
}, 1800)
}
return () => {
disposed = true
for (const t of timers) {
@@ -179,8 +205,8 @@ export function useMobileSessionStartup(scope: MobileSessionKeyboardStateModel)
connState,
created,
fetchTerminals,
ensureSessionTabs,
isFloatingWorkspaceRoute,
protocolVerified,
showToast,
worktreeId
])
@@ -1,5 +1,4 @@
import { useEffect, useRef, useCallback, useMemo, useState } from 'react'
import { startRuntimeCapabilityProbe } from '../transport/runtime-capability-probe'
import { supportsMobileQuickCommands } from '../terminal/quick-commands'
import { MOBILE_AI_VAULT_CAPABILITY } from '../agent-history/agent-history-capability'
import { TERMINAL_QUERY_REPLY_INPUT_RUNTIME_CAPABILITY } from '../../../src/shared/protocol-version'
@@ -17,6 +16,8 @@ export function useMobileSessionTabReconciliation(scope: MobileSessionMarkdownAc
worktreeId,
client,
connState,
hostCapabilities,
protocolVerified,
sessionTabsRef,
activeSessionTabIdRef,
terminalsRef,
@@ -144,8 +145,14 @@ export function useMobileSessionTabReconciliation(scope: MobileSessionMarkdownAc
const hostQueryReplyInputSupportedRef = useRef(false)
// Why: the gate above every /h/ route already holds this connection's status.get answer (and
// retries it until one lands), so the route reads it through the foundation instead of issuing
// a second one. It reports no capabilities until the verdict is proven, which keeps the
// fail-closed reset below identical to the old pre-probe clear.
useEffect(() => {
if (!client || connState !== 'connected') {
// Why: a client swap can keep the route connected while moving to an older
// host; clear the prior capability before exposing host-specific actions.
if (!client || connState !== 'connected' || !protocolVerified) {
setBrowserScreencastSupported(null)
setAgentSessionHistorySupported(null)
setQuickCommandsSupported(null)
@@ -153,26 +160,15 @@ export function useMobileSessionTabReconciliation(scope: MobileSessionMarkdownAc
hostQueryReplyInputSupportedRef.current = false
return
}
// Why: a client swap can keep the route connected while moving to an older
// host; clear the prior capability before exposing host-specific actions.
setBrowserScreencastSupported(null)
setAgentSessionHistorySupported(null)
setQuickCommandsSupported(null)
setShowQuickCommands(false)
hostQueryReplyInputSupportedRef.current = false
// Why: the probe retries — a relay→direct cutover or request timeout rejects
// status.get without changing connState, which used to latch these hidden.
return startRuntimeCapabilityProbe(client, (capabilities) => {
setBrowserScreencastSupported(capabilities.includes('browser.screencast.v1'))
setAgentSessionHistorySupported(capabilities.includes(MOBILE_AI_VAULT_CAPABILITY))
setQuickCommandsSupported(supportsMobileQuickCommands(capabilities))
// Why: hosts without this capability strip inputKind from terminal.send,
// so a forwarded xterm reply would become floor-stealing shell input.
hostQueryReplyInputSupportedRef.current = capabilities.includes(
TERMINAL_QUERY_REPLY_INPUT_RUNTIME_CAPABILITY
)
})
}, [client, connState])
setBrowserScreencastSupported(hostCapabilities.includes('browser.screencast.v1'))
setAgentSessionHistorySupported(hostCapabilities.includes(MOBILE_AI_VAULT_CAPABILITY))
setQuickCommandsSupported(supportsMobileQuickCommands(hostCapabilities))
// Why: hosts without this capability strip inputKind from terminal.send,
// so a forwarded xterm reply would become floor-stealing shell input.
hostQueryReplyInputSupportedRef.current = hostCapabilities.includes(
TERMINAL_QUERY_REPLY_INPUT_RUNTIME_CAPABILITY
)
}, [client, connState, hostCapabilities, protocolVerified])
return {
consumeAcceptedSessionTabs,
hasSessionTabsRecoveryNeed,
@@ -0,0 +1,66 @@
import { useEffect, useState } from 'react'
import {
getSessionTabStripCacheKey,
loadCachedSessionTabStrip,
readCachedSessionTabStrip,
saveCachedSessionTabStrip
} from '../cache/session-tab-strip-cache'
import {
toMobileSessionTabStripPreview,
type MobileSessionTabStripPreview
} from './mobile-session-tab-strip-entries'
import type { MobileSessionBulkCloseModel } from './use-mobile-session-bulk-close'
/**
* Keeps the last drawn tab strip for this workspace on the device, so a reconnect has something
* to render before the first snapshot lands. See mobile-session-reconnect-view-state.
*/
export function useMobileSessionTabStripCache(scope: MobileSessionBulkCloseModel) {
const { hostId, worktreeId, connState, terminalsLoaded } = scope
const { visibleTabs, activeSessionTabId, activeHandle } = scope
const cacheKey = getSessionTabStripCacheKey(hostId, worktreeId)
// Why: state settles a commit behind the key it was read for, so carry the key with it —
// otherwise the first render after a workspace switch draws the previous workspace's strip.
const [loaded, setLoaded] = useState<{
key: string | null
preview: MobileSessionTabStripPreview | null
}>(() => ({ key: cacheKey, preview: readCachedSessionTabStrip(cacheKey) }))
useEffect(() => {
// Synchronous first, so an in-session revisit never blinks through the uncached branch.
setLoaded({ key: cacheKey, preview: readCachedSessionTabStrip(cacheKey) })
let disposed = false
void loadCachedSessionTabStrip(cacheKey).then((preview) => {
if (!disposed) {
setLoaded({ key: cacheKey, preview })
}
})
return () => {
disposed = true
}
}, [cacheKey])
const cachedTabStrip = loaded.key === cacheKey ? loaded.preview : null
// Only a host-confirmed strip is worth persisting, and an emptied workspace has to be written
// too — skipping it would leave yesterday's tabs to be drawn over a session that no longer has
// them. The one reading we do not trust is a live terminal with no tab record behind it, which
// is the same case the empty state refuses to claim (use-mobile-session-presentation).
// react-doctor-disable-next-line react-doctor/effect-needs-cleanup
useEffect(() => {
if (connState !== 'connected' || !terminalsLoaded) {
return
}
if (visibleTabs.length === 0 && activeHandle !== null) {
return
}
saveCachedSessionTabStrip(
cacheKey,
toMobileSessionTabStripPreview(visibleTabs, activeSessionTabId)
)
}, [activeHandle, activeSessionTabId, cacheKey, connState, terminalsLoaded, visibleTabs])
return { cachedTabStrip }
}
export type MobileSessionTabStripCacheModel = MobileSessionBulkCloseModel &
ReturnType<typeof useMobileSessionTabStripCache>
@@ -1,4 +1,6 @@
import { useRef, useCallback } from 'react'
import type { TerminalWebViewHandle } from '../terminal/terminal-webview-contract'
import { TERMINAL_ENGINE_PREWARM_HANDLE } from './TerminalEnginePrewarm'
import type { MobileSessionNativeChatDictationModel } from './use-mobile-session-native-chat-dictation'
export function useMobileSessionTerminalSubscriptionFoundation(
@@ -101,12 +103,34 @@ export function useMobileSessionTerminalSubscriptionFoundation(
},
[getTerminalRef]
)
// Why: the pre-warm engine occupies the frame the first pane will occupy, so let it satisfy
// the one-shot measurement. It has no handle, so it is passed its own ref instead of looking
// one up, and it must never latch a measurement taken before the frame has a real height.
const measurePrewarmViewport = useCallback(
async (engine: TerminalWebViewHandle, frameHeight: number) => {
if (viewportMeasuredRef.current || frameHeight <= 0) {
return
}
const dims = await engine.measureFitDimensions(frameHeight)
terminalDiagnosticsRef.current.viewportMeasured(
TERMINAL_ENGINE_PREWARM_HANDLE,
dims,
frameHeight
)
if (dims && !viewportMeasuredRef.current) {
viewportRef.current = dims
viewportMeasuredRef.current = true
}
},
[]
)
return {
getTerminalRef,
unsubscribeTerminal,
unsubscribeTerminalRef,
clearTerminalCache,
measureViewportOnce
measureViewportOnce,
measurePrewarmViewport
}
}
@@ -16,6 +16,29 @@ describe('connection log buffer', () => {
expect(store.get('host-b').map((e) => e.id)).toEqual(['log-2'])
})
it('retains phase timings through redaction and evicts them with the cap', () => {
const store = createConnectionLogStore(2)
store.append('host-a', {
...entry(1),
timing: { kind: 'connection-state', name: 'reconnecting', ms: 800, complete: true }
})
store.append('host-a', {
...entry(2),
detail: '4280ms in connecting; resumeToken=secret-resume-token',
timing: { kind: 'connection-state', name: 'connecting', ms: 4_280, complete: true }
})
store.append('host-a', {
...entry(3),
timing: { kind: 'relay-dial-stage', name: 'awaiting-hello', ms: 9_100, complete: false }
})
expect(store.get('host-a').map((e) => e.timing)).toEqual([
{ kind: 'connection-state', name: 'connecting', ms: 4_280, complete: true },
{ kind: 'relay-dial-stage', name: 'awaiting-hello', ms: 9_100, complete: false }
])
expect(store.get('host-a')[0]!.detail).toBe('4280ms in connecting; resumeToken=[redacted]')
})
it('drops the oldest entries past the cap', () => {
const store = createConnectionLogStore(3)
for (let i = 1; i <= 5; i++) {
@@ -0,0 +1,77 @@
import { describe, expect, it } from 'vitest'
import { DirectConnectionLog } from './direct-connection-log'
import { RpcClientConnectionState } from './rpc-client-connection-state'
import type { ConnectionLogEntry, ConnectionState } from './types'
function openStateWithLog(sink?: (entry: ConnectionLogEntry) => void) {
const entries: ConnectionLogEntry[] = []
const log = new DirectConnectionLog(
'ws://192.168.1.50:6768',
sink ?? ((entry) => entries.push(entry))
)
let now = 0
const state = new RpcClientConnectionState({
endpoint: 'ws://192.168.1.50:6768',
getReconnectAttempt: () => 0,
isClosed: () => false,
onStateDwell: log.stateDwell,
now: () => now
})
const publishAfter = (elapsedMs: number, next: ConnectionState): void => {
now += elapsedMs
state.publish(next)
}
return { entries, state, publishAfter }
}
describe('connection state dwell logging', () => {
it('records the time spent in each state as a structured log entry', () => {
const { entries, publishAfter } = openStateWithLog()
publishAfter(300, 'connecting')
publishAfter(4_200, 'handshaking')
publishAfter(250, 'connected')
expect(entries.map((entry) => entry.timing)).toEqual([
{ kind: 'connection-state', name: 'disconnected', ms: 300, complete: true },
{ kind: 'connection-state', name: 'connecting', ms: 4_200, complete: true },
{ kind: 'connection-state', name: 'handshaking', ms: 250, complete: true }
])
expect(entries[1]!.message).toBe('Connection state connecting → handshaking')
expect(entries[1]!.detail).toBe('4200ms in connecting')
})
it('skips transitions too short to explain a slow connect', () => {
const { entries, publishAfter } = openStateWithLog()
publishAfter(99, 'connecting')
publishAfter(100, 'handshaking')
expect(entries.map((entry) => entry.timing?.name)).toEqual(['connecting'])
})
it('does not log a dwell when the state does not change', () => {
const { entries, publishAfter } = openStateWithLog()
publishAfter(500, 'connecting')
publishAfter(500, 'connecting')
expect(entries).toHaveLength(1)
})
it('still publishes the state when the log sink throws', () => {
const seen: ConnectionState[] = []
const { state, publishAfter } = openStateWithLog(() => {
throw new Error('sink exploded')
})
state.addListener((next) => seen.push(next))
const connected = state.waitForConnected()
publishAfter(500, 'connecting')
publishAfter(500, 'connected')
expect(seen).toEqual(['connecting', 'connected'])
expect(state.get()).toBe('connected')
return expect(connected).resolves.toBeUndefined()
})
})
+27 -1
View File
@@ -4,9 +4,15 @@ import type {
ConnectionLogEntry,
ConnectionLogLevel,
ConnectionLogSink,
ConnectionState,
MobileConnectionDiagnosticPath
} from './types'
// Why: every reconnect cycle walks four states, and the per-host buffer is capped.
// Logging sub-100ms transitions would halve the history a report can show while
// telling support nothing — those states are never where a slow connect spent time.
const MIN_LOGGED_DWELL_MS = 100
export class DirectConnectionLog {
private sequence = 0
private readonly path: MobileConnectionDiagnosticPath
@@ -22,7 +28,7 @@ export class DirectConnectionLog {
level: ConnectionLogLevel,
message: string,
detail?: string,
evidence?: Pick<ConnectionLogEntry, 'code' | 'path'>
evidence?: Pick<ConnectionLogEntry, 'code' | 'path' | 'timing'>
): void => {
this.sink?.({
id: `log-${++this.sequence}-${Date.now()}`,
@@ -44,6 +50,26 @@ export class DirectConnectionLog {
)
}
// Why: how long the client sat in each ConnectionState used to go only to
// console, so a shared diagnostics report could not show where a slow connect
// spent its seconds.
stateDwell = (previous: ConnectionState, next: ConnectionState, dweltMs: number): void => {
if (dweltMs < MIN_LOGGED_DWELL_MS) {
return
}
this.emit('info', `Connection state ${previous}${next}`, `${dweltMs}ms in ${previous}`, {
timing: { kind: 'connection-state', name: previous, ms: dweltMs, complete: true }
})
}
retryScheduled = (message: string, detail?: string): void => {
this.emit('info', message, detail, { code: 'retry-scheduled' })
}
authenticationRejected = (message: string, detail?: string): void => {
this.emit('warn', message, detail, { code: 'authentication-rejected' })
}
connected = (): void => {
this.emit('success', 'Authenticated', 'Channel ready for RPC', { code: 'direct-connected' })
}
+4 -5
View File
@@ -48,14 +48,14 @@ export class DirectRpcClient implements RpcClient {
this.reconnect = new RpcClientReconnectSchedule({
openConnection: () => this.openConnection(),
rejectConnectWaiters: (reason) => this.connectionState.rejectWaiters(reason),
emitLog: (message, detail) =>
this.connectionLog.emit('info', message, detail, { code: 'retry-scheduled' })
emitLog: this.connectionLog.retryScheduled
})
this.connectionState = new RpcClientConnectionState({
endpoint,
initialListener: options.onStateChange,
getReconnectAttempt: () => this.reconnect.getAttempt(),
isClosed: () => this.intentionallyClosed
isClosed: () => this.intentionallyClosed,
onStateDwell: this.connectionLog.stateDwell
})
this.streams = new RpcClientStreamRegistry({
nextId: () => this.nextId(),
@@ -102,8 +102,7 @@ export class DirectRpcClient implements RpcClient {
this.authenticationRetry = new RpcClientAuthenticationRetry({
endpoint,
stopLiveness: () => this.stopLiveness(),
emitWarning: (message, detail) =>
this.connectionLog.emit('warn', message, detail, { code: 'authentication-rejected' }),
emitWarning: this.connectionLog.authenticationRejected,
retry: (reason) => this.retryAuthentication(reason),
latchFailure: (reason) => this.latchAuthenticationFailure(reason)
})
@@ -17,6 +17,12 @@ vi.mock('./host-store', () => ({
}))
import { removeHostAndCloseClient } from './host-removal-lifecycle'
import {
getSessionTabStripCacheKey,
readCachedSessionTabStrip,
resetSessionTabStripCacheForTests,
saveCachedSessionTabStrip
} from '../cache/session-tab-strip-cache'
import {
getHostNotificationSession,
resetHostNotificationSessionsForTests
@@ -26,7 +32,9 @@ describe('host removal lifecycle', () => {
beforeEach(() => {
removeHostMock.mockReset()
asyncStorage.removeItem.mockClear()
asyncStorage.setItem.mockReset().mockResolvedValue(undefined)
resetHostNotificationSessionsForTests()
resetSessionTabStripCacheForTests()
})
it('closes the client only after metadata removal commits', async () => {
@@ -88,4 +96,40 @@ describe('host removal lifecycle', () => {
expect(asyncStorage.removeItem).toHaveBeenCalledWith('orca:mobileNotificationsWatermark:host-1')
})
it('drops the removed host cached tab strip and keeps every other host', async () => {
// Why: the strip is plaintext and nothing else in the app ever expires an entry, so a
// forgotten host would keep its tab titles on disk and get them rewritten by the next
// save for any surviving host.
removeHostMock.mockResolvedValue(undefined)
const removed = getSessionTabStripCacheKey('host-1', 'wt-1')
const kept = getSessionTabStripCacheKey('host-2', 'wt-1')
const strip = {
tabs: [{ id: 'tab-1', type: 'terminal' as const, title: 'Terminal', agentId: null }],
activeTabId: 'tab-1'
}
saveCachedSessionTabStrip(removed, strip)
saveCachedSessionTabStrip(kept, strip)
await removeHostAndCloseClient('host-1', vi.fn())
// Fire-and-forget, like clearWatermark above; let its microtasks land.
await vi.waitFor(() => expect(readCachedSessionTabStrip(removed)).toBeNull())
expect(readCachedSessionTabStrip(kept)?.tabs).toHaveLength(1)
})
it('finishes the removal even when the cached tab strip write fails', async () => {
// The metadata removal has already committed and the client is closed by this
// point, so a cache write that fails must be reported, not thrown: surfacing it
// as a failed removal would leave the user staring at a host that is really gone.
removeHostMock.mockResolvedValue(undefined)
asyncStorage.setItem.mockRejectedValue(new Error('storage full'))
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const closeHostClient = vi.fn()
await expect(removeHostAndCloseClient('host-1', closeHostClient)).resolves.toBeUndefined()
expect(closeHostClient).toHaveBeenCalledWith('host-1')
expect(warn).toHaveBeenCalled()
warn.mockRestore()
})
})
@@ -1,3 +1,4 @@
import { deleteCachedSessionTabStripForHost } from '../cache/session-tab-strip-cache'
import {
clearWatermark,
forgetHostNotificationSession
@@ -17,4 +18,12 @@ export async function removeHostAndCloseClient(
// re-pair of the same host would inherit a watermark for a counter it never saw.
forgetHostNotificationSession(hostId)
void clearWatermark(hostId)
// Why: the cached tab strip is plaintext and host-scoped, so forgetting the host has to drop
// it here too — nothing else in the app ever expires an entry. Awaited so a storage failure
// is observed rather than swallowed, but never fatal: the metadata removal has already
// committed and the client is closed, so failing here would report a finished removal as
// failed. The cache refuses further saves for this host either way.
await deleteCachedSessionTabStripForHost(hostId).catch((error: unknown) => {
console.warn('[host-removal] cached tab strip delete failed', error)
})
}
+39 -38
View File
@@ -1,6 +1,7 @@
import { useEffect, useState } from 'react'
import type { RpcClient } from './rpc-client'
import type { ConnectionState, RpcSuccess } from './types'
import type { ConnectionState } from './types'
import { readRuntimeCapabilities, startRuntimeStatusProbe } from './runtime-status-probe'
import { evaluateCompat, type CompatVerdict } from './protocol-compat'
import type { DesktopStatus } from '../worktree/host-worktree-rpc-types'
import { normalizeHostAppVersion, recordHostAppVersion } from './host-app-version-store'
@@ -10,6 +11,10 @@ export type HostStatusGates = {
floatingWorkspaceEnabled: boolean
desktopAppVersion: string | null
compatVerdict: CompatVerdict
// Why: `compatVerdict.kind === 'ok'` is not proof. A host that never answers status.get settles
// the same `ok` so navigation is not trapped, and that fallback must not read as a passing
// verdict. Only an evaluated status reply sets this, so writes to the host can gate on it.
compatVerified: boolean
statusPending: boolean
}
@@ -21,8 +26,11 @@ type LoadedHostStatusGates = Omit<HostStatusGates, 'statusPending'> & {
const EMPTY_HOST_CAPABILITIES: string[] = []
// Reads status.get on connect for capabilities, protocol-compat verdict, and the
// floating-workspace flag. Compat constants are wide-open today so this never blocks yet.
// The route tree's single status.get: it reads capabilities, the protocol-compat verdict, and
// the floating-workspace flag once per connection and publishes them through HostProtocolGate,
// so no descendant issues its own. The verdict really can block — evaluateCompat reads a missing
// protocolVersion as 0, below MIN_COMPATIBLE_DESKTOP_VERSION — so a pending verdict is a real
// state, not a formality, and anything that writes to the host must wait for it.
export function useHostStatusGates(args: {
hostId: string | undefined
client: RpcClient | null
@@ -39,30 +47,33 @@ export function useHostStatusGates(args: {
setUnverified(true)
return
}
let cancelled = false
const requestClient = client
const settle = (gates: Omit<HostStatusGates, 'statusPending'>) => {
setLoaded({ hostId, client: requestClient, ...gates })
setUnverified(false)
}
void (async () => {
try {
const response = await requestClient.sendRequest('status.get')
if (cancelled) {
return
}
if (!response.ok) {
settle({
hostCapabilities: [],
floatingWorkspaceEnabled: false,
desktopAppVersion: null,
compatVerdict: { kind: 'ok' }
})
return
}
const status = (response as RpcSuccess).result as DesktopStatus & {
capabilities?: string[]
}
// Why: a transient status failure must not trap navigation, so the first miss settles
// conservative gates and releases the pending overlay; the probe keeps retrying underneath
// so a cutover or timeout no longer latches capability-gated UI hidden until a remount.
// compatVerified stays false: this releases the UI, it proves nothing about the host.
let failedOpen = false
const failOpen = () => {
if (failedOpen) {
return
}
failedOpen = true
settle({
hostCapabilities: [],
floatingWorkspaceEnabled: false,
desktopAppVersion: null,
compatVerdict: { kind: 'ok' },
compatVerified: false
})
}
return startRuntimeStatusProbe(requestClient, {
onUnavailable: failOpen,
onStatus: (result) => {
const status = result as DesktopStatus & { capabilities?: string[] }
const verdict = evaluateCompat({
desktopProtocolVersion: status.protocolVersion,
desktopMinCompatibleMobileVersion: status.minCompatibleMobileVersion
@@ -72,10 +83,11 @@ export function useHostStatusGates(args: {
void recordHostAppVersion(hostId, desktopAppVersion)
}
settle({
hostCapabilities: status.capabilities ?? [],
hostCapabilities: [...readRuntimeCapabilities(result)],
floatingWorkspaceEnabled: status.floatingWorkspaceEnabled === true,
desktopAppVersion,
compatVerdict: verdict
compatVerdict: verdict,
compatVerified: true
})
if (verdict.kind === 'blocked') {
// Why: support breadcrumb to confirm a block fired vs a render bug; no PII, just version ints.
@@ -86,21 +98,8 @@ export function useHostStatusGates(args: {
requiredDesktopVersion: verdict.requiredDesktopVersion
})
}
} catch {
// Why: a transient status failure must not trap navigation; conservative feature gates remain disabled.
if (!cancelled) {
settle({
hostCapabilities: [],
floatingWorkspaceEnabled: false,
desktopAppVersion: null,
compatVerdict: { kind: 'ok' }
})
}
}
})()
return () => {
cancelled = true
}
})
}, [client, connState, hostId])
// Why: effects run after render, so key loaded gates by host and client to fail closed during route reuse.
@@ -111,6 +110,7 @@ export function useHostStatusGates(args: {
floatingWorkspaceEnabled: false,
desktopAppVersion: null,
compatVerdict: { kind: 'ok' },
compatVerified: false,
statusPending: connState === 'connected' && client !== null
}
}
@@ -119,6 +119,7 @@ export function useHostStatusGates(args: {
floatingWorkspaceEnabled: proven.floatingWorkspaceEnabled,
desktopAppVersion: proven.desktopAppVersion,
compatVerdict: proven.compatVerdict,
compatVerified: proven.compatVerified,
// Why (F10): unchanged pending timing — the reconnect refetch is still "unknown", it just no
// longer blanks the capabilities this same host already proved.
statusPending: connState === 'connected' && unverified
@@ -0,0 +1,96 @@
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
import { DirectReturnProbe } from './mobile-direct-return-probe'
import { MobileEndpointHysteresis } from './mobile-endpoint-hysteresis'
import { FakeSession, host } from './mobile-endpoint-supervisor-test-fakes'
vi.mock('react-native', () => ({ Platform: { OS: 'ios' } }))
// A LAN that never answers: every dial sits open until the probe's own 12s budget.
function fixture() {
const opened: FakeSession[] = []
const probe = new DirectReturnProbe(
{
now: Date.now,
setTimer: setTimeout,
clearTimer: clearTimeout,
openDirect: () => {
const candidate = new FakeSession('connecting')
opened.push(candidate)
return candidate
}
},
{
hysteresis: new MobileEndpointHysteresis(Date.now(), {
directSuccessesRequired: 1,
directObservationMs: 60_000,
failureCooldownMs: 0,
minimumDwellMs: 0
}),
host: () => host,
canSchedule: () => true,
canDial: () => true,
canAttempt: () => true,
// These cases model a live relay session, so hysteresis still arbitrates.
adoptsOutright: () => false,
beginOperation: () => {},
migrate: async () => {},
onDirectMigrated: async () => {},
afterProbe: () => {}
}
)
return { opened, probe }
}
beforeEach(() => vi.useFakeTimers())
afterEach(() => vi.useRealTimers())
it('never opens a second dial while one is still in flight', async () => {
const { opened, probe } = fixture()
probe.schedule(0)
await vi.advanceTimersByTimeAsync(0)
expect(opened).toHaveLength(1)
// A relay drop and a foreground return both ask for an immediate probe while the
// first dial is still awaiting authentication.
probe.schedule(0)
probe.schedule(0)
await vi.advanceTimersByTimeAsync(0)
expect(opened).toHaveLength(1)
// Why this is the assertion that matters: a second probe would have overwritten
// activeProbe, so stop() would abort only the newest dial and leave this socket
// open for the rest of its 12s budget.
probe.stop()
await vi.advanceTimersByTimeAsync(0)
expect(opened[0]!.close).toHaveBeenCalledOnce()
expect(vi.getTimerCount()).toBe(0)
})
it('honors an urgent reprobe asked for mid-dial instead of dropping it on the 15s floor', async () => {
const { opened, probe } = fixture()
probe.schedule(0)
await vi.advanceTimersByTimeAsync(0)
probe.schedule(0)
await vi.advanceTimersByTimeAsync(0)
expect(opened).toHaveLength(1)
// The deferred ask survives the dial and runs at once when it settles, so holding
// the slot does not cost the caller the 15s it was trying to skip.
await vi.advanceTimersByTimeAsync(12_000)
await vi.advanceTimersByTimeAsync(1)
expect(opened).toHaveLength(2)
probe.stop()
})
it('falls back to the ordinary interval when nothing asked for a sooner probe', async () => {
const { opened, probe } = fixture()
probe.schedule(0)
await vi.advanceTimersByTimeAsync(12_000)
expect(opened).toHaveLength(1)
await vi.advanceTimersByTimeAsync(14_999)
expect(opened).toHaveLength(1)
await vi.advanceTimersByTimeAsync(1)
expect(opened).toHaveLength(2)
probe.stop()
})
@@ -6,13 +6,18 @@ import type { MobileConnectionPath } from './stable-logical-rpc-client'
const DIRECT_PROBE_INTERVAL_MS = 15_000
// While the runtime channel rides the relay, periodically probe the direct
// endpoint and migrate back once hysteresis proves it stable.
// Re-acquires the direct endpoint while the runtime channel rides the relay.
// Two adoption policies, because what is at stake differs:
// - against a live relay, hysteresis must prove direct stable before the swap;
// - during a reconnect nothing is live, so this dial races the relay dial from
// t=0 and the first authenticated socket is adopted outright.
export class DirectReturnProbe {
private timer: ReturnType<typeof setTimeout> | null = null
private stopped = false
private activeProbe: AbortController | null = null
// Soonest delay a caller asked for while a dial was in flight.
private deferredDelayMs: number | null = null
constructor(
private readonly deps: {
@@ -25,7 +30,14 @@ export class DirectReturnProbe {
hysteresis: MobileEndpointHysteresis
host: () => HostProfile
canSchedule: () => boolean
// A dial is a pure observation on its own socket, so it only needs a live
// supervisor; the cutover is the part that needs the operation mutex.
canDial: () => boolean
canAttempt: () => boolean
// True while no session is live: the reconnect is a race, so an
// authenticated direct socket wins without consulting hysteresis.
adoptsOutright: () => boolean
// Takes the supervisor's operation mutex, now held for the cutover only.
beginOperation: () => void
migrate: (
client: RpcClient,
@@ -38,7 +50,18 @@ export class DirectReturnProbe {
) {}
schedule(delayMs = DIRECT_PROBE_INTERVAL_MS): void {
if (this.stopped || !this.hooks.canSchedule() || this.timer) {
if (this.stopped || !this.hooks.canSchedule()) {
return
}
// Why: the dial no longer holds the supervisor's mutex, so nothing else stops a
// second probe from overwriting activeProbe — stop() would then reach only the
// newest socket and leave the earlier one dialing for its full 12s budget. The
// in-flight probe owns the next slot and re-arms it on the soonest ask.
if (this.activeProbe) {
this.deferredDelayMs = Math.min(this.deferredDelayMs ?? delayMs, delayMs)
return
}
if (this.timer) {
return
}
this.timer = this.deps.setTimer(() => {
@@ -47,7 +70,18 @@ export class DirectReturnProbe {
}, delayMs)
}
// Why: a reconnect races both paths from t=0, and schedule(0) yields to a
// pending 15s tick — that would hand the relay dial a head start by another name.
probeNow(): void {
if (this.stopped || this.activeProbe) {
return
}
this.clear()
this.schedule(0)
}
clear(): void {
this.deferredDelayMs = null
if (this.timer) {
this.deps.clearTimer(this.timer)
this.timer = null
@@ -64,15 +98,23 @@ export class DirectReturnProbe {
if (this.stopped) {
return
}
if (!this.hooks.canAttempt() || !this.hooks.hysteresis.canProbe(this.deps.now())) {
// Why: the failure cooldown exists to stop a healthy relay flapping onto a
// marginal LAN. With nothing connected there is no session to protect, and
// honouring it would leave the phone waiting on relay alone.
const racing = this.hooks.adoptsOutright()
if (!this.hooks.canDial() || (!racing && !this.hooks.hysteresis.canProbe(this.deps.now()))) {
this.schedule()
return
}
const controller = new AbortController()
this.activeProbe = controller
this.hooks.beginOperation()
let owned = false
let successful: Awaited<ReturnType<typeof openAuthenticatedDirectEndpoint>> = null
try {
// Why: the dial is a pure observation on its own socket — holding the
// supervisor's mutex across its 12s budget stalled every relay recovery
// that landed during a foreground return, and makes the reconnect race
// unwinnable while a relay dial holds it.
successful = await openAuthenticatedDirectEndpoint(
this.hooks.host(),
this.deps.openDirect,
@@ -86,17 +128,42 @@ export class DirectReturnProbe {
this.hooks.hysteresis.recordDirectFailure(this.deps.now())
return
}
if (!this.hooks.hysteresis.recordDirectSuccess(this.deps.now())) {
successful.client.close()
// Both early returns leave the candidate to the finally, which owns it until
// migration takes over — closing here too would double-close it.
const outright = this.hooks.adoptsOutright()
// Why: a socket that entered the race and lost books nothing and leaves the
// promotion streak untouched — the winner is this reconnect's whole verdict.
if (!outright && (racing || !this.hooks.hysteresis.recordDirectSuccess(this.deps.now()))) {
return
}
const mutexFree = this.hooks.canAttempt()
if (!mutexFree && !outright) {
// A relay dial owns the mutex; the streak survives, so the next probe
// promotes direct instead of this one.
return
}
if (mutexFree) {
this.hooks.beginOperation()
owned = true
}
// Why: when a relay dial holds the mutex the race still cuts over — that
// dial withdraws itself in migrateTo and books no failure against relay.
const candidate = successful
// Migration owns the candidate, including closing it if cutover is canceled.
successful = null
// Why: the relay dial can authenticate between this socket's authentication
// and the swap. migrateTo re-checks after auth, so the loser withdraws.
const abortCutover = outright
? (): boolean => this.stopped || !this.hooks.adoptsOutright()
: (): boolean => this.stopped
try {
await this.hooks.migrate(candidate.client, candidate.path, () => this.stopped)
await this.hooks.migrate(candidate.client, candidate.path, abortCutover)
} catch (error) {
if (this.stopped) {
// Why: a withdrawn cutover is the ordinary end of a lost race, and
// migrateTo has already closed the candidate. Only the timer calls this
// method, and it discards the promise, so rethrowing here would surface
// a routine loss as an unhandled rejection.
if (this.stopped || abortCutover()) {
return
}
throw error
@@ -109,10 +176,14 @@ export class DirectReturnProbe {
} finally {
this.activeProbe = null
successful?.client.close()
// Why: a relay drop or backoff timer can arrive while the probe owns the
// Why: a relay drop or backoff timer can arrive while the cutover owns the
// operation mutex; afterProbe releases it and replays deferred recovery.
this.hooks.afterProbe()
this.schedule()
if (owned) {
this.hooks.afterProbe()
}
const deferred = this.deferredDelayMs
this.deferredDelayMs = null
this.schedule(deferred ?? undefined)
}
}
}
@@ -86,7 +86,7 @@ function createSupervisor(
): MobileEndpointSupervisor {
return new MobileEndpointSupervisor(logical, host, {
openDirect: (endpoint) => connect(endpoint, host.deviceToken, host.publicKeyB64, { onLog }),
openRelay: (relay, credential, confirmReqId, onHostCloseReason) =>
openRelay: (relay, credential, confirmReqId, onHostCloseReason, isForeground) =>
connectMobileRelayRpcSession({
relay,
resumeToken: credential.token,
@@ -94,6 +94,7 @@ function createSupervisor(
resumeConfirmReqId: confirmReqId,
deviceToken: host.deviceToken,
desktopPublicKeyB64: host.publicKeyB64,
isForeground,
onHostCloseReason,
onLog
}),
@@ -0,0 +1,362 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { MobileEndpointHysteresis } from './mobile-endpoint-hysteresis'
import { MobileEndpointSupervisor } from './mobile-endpoint-supervisor'
import { RelayOuterError } from './mobile-relay-e2ee-link'
import {
dependencies,
FakeLogicalClient,
FakeRelaySession,
FakeSession,
host,
unreachableDirect
} from './mobile-endpoint-supervisor-test-fakes'
vi.mock('react-native', () => ({ Platform: { OS: 'ios' } }))
vi.mock('expo-secure-store', () => ({ WHEN_UNLOCKED_THIS_DEVICE_ONLY: 'when-unlocked' }))
vi.mock('expo-crypto', () => ({ getRandomBytes: (length: number) => new Uint8Array(length) }))
// Holds the relay cutover open so the direct path can authenticate mid-dial. The
// fake's migrateTo otherwise settles inside the dial, which no real cell does.
function holdRelayCutover(logical: FakeLogicalClient): () => void {
const settle = logical.migrateTo.getMockImplementation()!
let release!: () => void
const held = new Promise<void>((resolve) => {
release = resolve
})
logical.migrateTo.mockImplementationOnce(async (session, path, timeoutMs, shouldAbort) => {
await held
// Why: replays the real post-authentication checks, so a superseded dial
// still withdraws instead of stealing the client from the winner.
return await settle(session, path, timeoutMs, shouldAbort)
})
return release
}
// One full lost race: the relay dial starts, direct returns mid-cutover and wins.
async function loseOneRace(
logical: FakeLogicalClient,
openRelay: ReturnType<typeof vi.fn>
): Promise<void> {
const before = openRelay.mock.calls.length
const release = holdRelayCutover(logical)
logical.publishState('reconnecting')
await vi.advanceTimersByTimeAsync(0)
expect(openRelay.mock.calls.length).toBe(before + 1)
logical.publishState('connected')
release()
await vi.advanceTimersByTimeAsync(0)
}
function relaySessionsFrom(openRelay: ReturnType<typeof vi.fn>): FakeRelaySession[] {
return openRelay.mock.results.map((result) => result.value as FakeRelaySession)
}
describe('mobile endpoint reconnect race', () => {
beforeEach(() => {
vi.useFakeTimers()
vi.setSystemTime(new Date('2026-07-13T12:00:00Z'))
})
afterEach(() => {
vi.useRealTimers()
})
it('dials relay at t=0 while the direct dial is still connecting', async () => {
const logical = new FakeLogicalClient('connecting', 'lan')
const deps = dependencies({ openDirect: unreachableDirect() })
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
// No timer advance at all: an unfinished direct dial buys no head start.
await supervisor.start()
expect(deps.openRelay).toHaveBeenCalledOnce()
expect(logical.getActivePath()).toBe('relay')
expect(logical.migrateTo).toHaveBeenCalledWith(
expect.any(FakeRelaySession),
'relay',
undefined,
expect.any(Function)
)
supervisor.stop()
})
it('adopts the direct dial and withdraws the slower relay dial without booking it', async () => {
const logical = new FakeLogicalClient('connecting', 'lan')
const openRelay = vi.fn(() => new FakeRelaySession('connected'))
const deps = dependencies({ openRelay, openDirect: unreachableDirect() })
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
const release = holdRelayCutover(logical)
const starting = supervisor.start()
await vi.advanceTimersByTimeAsync(0)
expect(openRelay).toHaveBeenCalledOnce()
// The direct dial authenticates while the cell is still cutting over.
logical.publishState('connected')
release()
await starting
expect(logical.getActivePath()).toBe('lan')
expect(relaySessionsFrom(openRelay)[0]!.close).toHaveBeenCalled()
// A withdrawn dial is not a failure: no cooldown is armed, so no redial lands.
await vi.advanceTimersByTimeAsync(60_000)
expect(openRelay).toHaveBeenCalledOnce()
expect(logical.setRecoveryPath).toHaveBeenLastCalledWith(null)
supervisor.stop()
})
it('adopts a direct socket that wins a reconnect the relay path started', async () => {
const recordMigration = vi.spyOn(MobileEndpointHysteresis.prototype, 'recordMigration')
const logical = new FakeLogicalClient('connected', 'relay')
const openRelay = vi.fn(() => new FakeRelaySession('connected'))
const deps = dependencies({ openRelay })
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
const release = holdRelayCutover(logical)
logical.publishState('disconnected')
// The direct dial runs while the relay dial is still in flight and wins it.
await vi.advanceTimersByTimeAsync(0)
expect(deps.openDirect).toHaveBeenCalledOnce()
expect(logical.getActivePath()).toBe('lan')
release()
await vi.advanceTimersByTimeAsync(0)
expect(relaySessionsFrom(openRelay)[0]!.close).toHaveBeenCalled()
// Hysteresis stamps the dwell, and the losing relay dial books no backoff.
expect(recordMigration).toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(60_000)
expect(openRelay).toHaveBeenCalledOnce()
supervisor.stop()
})
it('books one backoff, not two, when both paths lose the reconnect', async () => {
const recordDirectFailure = vi.spyOn(MobileEndpointHysteresis.prototype, 'recordDirectFailure')
const logical = new FakeLogicalClient('connected', 'relay')
const openRelay = vi.fn(() => new FakeRelaySession('disconnected', new RelayOuterError(4408)))
const deps = dependencies({
openRelay,
openDirect: unreachableDirect(),
randomBytes: () => new Uint8Array([128, 0])
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
logical.publishState('disconnected')
await vi.advanceTimersByTimeAsync(0)
expect(openRelay).toHaveBeenCalledOnce()
expect(deps.openDirect).toHaveBeenCalledOnce()
expect(recordDirectFailure).toHaveBeenCalledOnce()
// One failure, so one 250ms step. A double-booked loss would redial at 500ms.
await vi.advanceTimersByTimeAsync(249)
expect(openRelay).toHaveBeenCalledOnce()
await vi.advanceTimersByTimeAsync(1)
expect(openRelay).toHaveBeenCalledTimes(2)
supervisor.stop()
})
it('leaves the promotion streak alone when the direct socket loses the race', async () => {
const recordDirectSuccess = vi.spyOn(MobileEndpointHysteresis.prototype, 'recordDirectSuccess')
const logical = new FakeLogicalClient('connected', 'relay')
const direct = new FakeSession('connecting')
const openRelay = vi.fn(() => new FakeRelaySession('connected'))
const deps = dependencies({ openRelay, openDirect: vi.fn(() => direct) })
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
const release = holdRelayCutover(logical)
logical.publishState('disconnected')
await vi.advanceTimersByTimeAsync(0)
expect(deps.openDirect).toHaveBeenCalledOnce()
// Relay authenticates first, then the direct socket finally answers.
release()
await vi.advanceTimersByTimeAsync(0)
expect(logical.getActivePath()).toBe('relay')
direct.publishState('connected')
await vi.advanceTimersByTimeAsync(0)
expect(direct.close).toHaveBeenCalled()
expect(recordDirectSuccess).not.toHaveBeenCalled()
expect(logical.getActivePath()).toBe('relay')
supervisor.stop()
})
it('ignores a loser that closes after the winner has been adopted', async () => {
const logical = new FakeLogicalClient('connecting', 'lan')
const openRelay = vi.fn(() => new FakeRelaySession('connected'))
const deps = dependencies({ openRelay, openDirect: unreachableDirect() })
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
const release = holdRelayCutover(logical)
const starting = supervisor.start()
await vi.advanceTimersByTimeAsync(0)
logical.publishState('connected')
release()
await starting
expect(logical.getActivePath()).toBe('lan')
// The withdrawn cell socket reports its close afterwards.
relaySessionsFrom(openRelay)[0]!.publishState('disconnected')
await vi.advanceTimersByTimeAsync(60_000)
expect(logical.getState()).toBe('connected')
expect(logical.getActivePath()).toBe('lan')
expect(openRelay).toHaveBeenCalledOnce()
supervisor.stop()
})
it('withdraws the relay socket before it authenticates once direct wins', async () => {
const logical = new FakeLogicalClient('connecting', 'lan')
const relaySession = new FakeRelaySession('connecting')
const openRelay = vi.fn(() => relaySession)
const deps = dependencies({ openRelay, openDirect: unreachableDirect() })
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
const release = holdRelayCutover(logical)
const starting = supervisor.start()
await vi.advanceTimersByTimeAsync(0)
expect(openRelay).toHaveBeenCalledOnce()
expect(relaySession.close).not.toHaveBeenCalled()
// The direct dial authenticates while the cell socket is still pre-handshake.
// migrateTo would not withdraw until after E2EE auth, so the cell would have
// reserved a splice and the desktop would have finished a handshake for it.
logical.publishState('connected')
expect(relaySession.close).toHaveBeenCalled()
expect(relaySession.getState()).not.toBe('connected')
release()
await starting
await vi.advanceTimersByTimeAsync(60_000)
expect(logical.getActivePath()).toBe('lan')
expect(openRelay).toHaveBeenCalledOnce()
supervisor.stop()
})
it('damps the race after a loss so a flapping LAN opens one cell socket', async () => {
const logical = new FakeLogicalClient('connecting', 'lan')
const openRelay = vi.fn(() => new FakeRelaySession('connecting'))
const deps = dependencies({ openRelay, openDirect: unreachableDirect() })
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
// The first blip races, and the returning direct dial wins it.
const release = holdRelayCutover(logical)
const starting = supervisor.start()
await vi.advanceTimersByTimeAsync(0)
logical.publishState('connected')
release()
await starting
expect(openRelay).toHaveBeenCalledOnce()
// Two more blips inside the damper window open no further cell socket.
for (const _blip of [1, 2]) {
logical.publishState('reconnecting')
await vi.advanceTimersByTimeAsync(100)
logical.publishState('connected')
await vi.advanceTimersByTimeAsync(400)
}
expect(openRelay).toHaveBeenCalledOnce()
// The window lapses against a live direct path, so it still opens nothing.
await vi.advanceTimersByTimeAsync(10_000)
expect(openRelay).toHaveBeenCalledOnce()
supervisor.stop()
})
it('races at once when the LAN dies inside a damper window grown to the cap', async () => {
const logical = new FakeLogicalClient('connecting', 'lan')
const openRelay = vi.fn(() => new FakeRelaySession('connecting'))
const deps = dependencies({ openRelay, openDirect: unreachableDirect() })
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
const release = holdRelayCutover(logical)
const starting = supervisor.start()
await vi.advanceTimersByTimeAsync(0)
logical.publishState('connected')
release()
await starting
// Four more losses, each once its window has run: 2s, 4s, 8s, 16s, then the
// fifth earns the 30s cap.
for (const window of [2_000, 4_000, 8_000, 16_000]) {
await vi.advanceTimersByTimeAsync(window)
await loseOneRace(logical, openRelay)
}
expect(openRelay).toHaveBeenCalledTimes(5)
// This time direct does not come back. Waiting out the window a blip earned
// would strand the phone offline for 30s with nothing else scheduled.
logical.publishState('reconnecting')
await vi.advanceTimersByTimeAsync(249)
expect(openRelay).toHaveBeenCalledTimes(5)
await vi.advanceTimersByTimeAsync(1)
expect(openRelay.mock.calls.length).toBeGreaterThan(5)
supervisor.stop()
})
it('lets a foreground resume race immediately inside a damper window', async () => {
const logical = new FakeLogicalClient('connecting', 'lan')
const openRelay = vi.fn(() => new FakeRelaySession('connecting'))
const deps = dependencies({ openRelay, openDirect: unreachableDirect() })
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
const release = holdRelayCutover(logical)
const starting = supervisor.start()
await vi.advanceTimersByTimeAsync(0)
logical.publishState('connected')
release()
await starting
logical.publishState('reconnecting')
await vi.advanceTimersByTimeAsync(100)
expect(openRelay).toHaveBeenCalledOnce()
// A resume is the user waiting on the screen; it never serves out the window.
supervisor.setForeground(false)
supervisor.setForeground(true)
await vi.advanceTimersByTimeAsync(0)
expect(openRelay.mock.calls.length).toBeGreaterThan(1)
supervisor.stop()
})
it('starts no dial in the background and races both paths on resume', async () => {
const logical = new FakeLogicalClient('connecting', 'lan')
const deps = dependencies({ openDirect: unreachableDirect() })
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
supervisor.setForeground(false)
await supervisor.start()
await vi.advanceTimersByTimeAsync(60_000)
expect(deps.openRelay).not.toHaveBeenCalled()
supervisor.setForeground(true)
await vi.advanceTimersByTimeAsync(0)
expect(deps.openRelay).toHaveBeenCalledOnce()
expect(logical.getActivePath()).toBe('relay')
supervisor.stop()
})
it('runs the resume probe against a relay that survived the background grace', async () => {
const logical = new FakeLogicalClient('connected', 'relay')
const deps = dependencies()
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
supervisor.setForeground(false)
await vi.advanceTimersByTimeAsync(1_000)
expect(deps.openDirect).not.toHaveBeenCalled()
// A live relay is not a reconnect: the resume probe dials direct, but the
// promotion still has to earn its hysteresis streak.
supervisor.setForeground(true)
await vi.advanceTimersByTimeAsync(0)
expect(deps.openDirect).toHaveBeenCalledOnce()
expect(logical.getActivePath()).toBe('relay')
expect(deps.openRelay).not.toHaveBeenCalled()
supervisor.stop()
})
})
@@ -12,7 +12,9 @@ export type MobileEndpointSupervisorDependencies = {
relay: MobileRelayEndpoint,
credential: { token: string; version: number },
confirmReqId: string,
onHostCloseReason?: (reason: RelayHostCloseReason) => void
onHostCloseReason?: (reason: RelayHostCloseReason) => void,
// Gates the session's idle liveness sweep; a backgrounded app spends no probes.
isForeground?: () => boolean
) => MobileRelayRpcSession
resolveRelay: typeof resolveMobileRelayEndpoint
readBundle: (hostId: string) => Promise<MobileRelayCredentialBundle | null>
@@ -1,13 +1,26 @@
import { beforeEach, afterEach, describe, expect, it, vi } from 'vitest'
import { MobileEndpointSupervisor } from './mobile-endpoint-supervisor'
import { MobileEndpointHysteresis } from './mobile-endpoint-hysteresis'
import {
dependencies,
FakeLogicalClient,
FakeRelaySession,
FakeSession,
host
host,
unreachableDirect
} from './mobile-endpoint-supervisor-test-fakes'
// A cell that authenticates and then answers the confirm for a different relay host
// — what a rehomed desktop produces. The session fails after the logical cutover.
function confirmRejectingRelaySession(logical: FakeLogicalClient): FakeRelaySession {
const session = new FakeRelaySession('connected', new Error('relay resume confirmation missing'))
session.whenResumeConfirmed = async () => {
session.publishState('disconnected')
logical.publishState('disconnected')
}
return session
}
vi.mock('react-native', () => ({ Platform: { OS: 'ios' } }))
vi.mock('expo-secure-store', () => ({ WHEN_UNLOCKED_THIS_DEVICE_ONLY: 'when-unlocked' }))
vi.mock('expo-crypto', () => ({ getRandomBytes: (length: number) => new Uint8Array(length) }))
@@ -48,4 +61,103 @@ describe('mobile endpoint supervisor direct probe', () => {
expect(logical.getActivePath()).toBe('relay')
supervisor.stop()
})
it('recovers the relay at once while the probe is still dialing direct', async () => {
const logical = new FakeLogicalClient('connected', 'relay')
// A black-holed LAN endpoint: the dial sits unanswered for its whole 12s budget.
const direct = new FakeSession('connecting')
const openRelay = vi.fn(() => new FakeRelaySession('connected'))
const deps = dependencies({ openDirect: vi.fn(() => direct), openRelay })
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
await vi.advanceTimersByTimeAsync(15_000)
expect(deps.openDirect).toHaveBeenCalledOnce()
logical.publishState('disconnected')
await vi.advanceTimersByTimeAsync(0)
// Why: the dial is a pure observation, so it no longer owns the operation
// mutex — recovery does not wait out the probe's budget.
expect(openRelay).toHaveBeenCalledOnce()
expect(logical.getState()).toBe('connected')
expect(logical.getActivePath()).toBe('relay')
supervisor.stop()
})
it('backs off a dial whose resume confirm fails after the cutover', async () => {
const recordMigration = vi.spyOn(MobileEndpointHysteresis.prototype, 'recordMigration')
const logical = new FakeLogicalClient('disconnected', 'lan')
const openRelay = vi.fn(() => confirmRejectingRelaySession(logical))
// No LAN to race: this is about the relay cadence after a confirm failure.
const deps = dependencies({
openRelay,
openDirect: unreachableDirect(),
randomBytes: () => new Uint8Array([128, 0])
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
// Two sockets per pass: a confirm mismatch reads as a stale cell assignment, so
// the existing director fallback re-resolves and dials the authoritative target.
expect(openRelay).toHaveBeenCalledTimes(2)
expect(logical.migrateTo).toHaveBeenCalledTimes(2)
// Why: `connected` is published at authentication, so the cutover happens before
// the confirm answers. A confirm that then fails must still book the shared
// cooldown — reporting it as an established dial redials in a tight loop.
await vi.advanceTimersByTimeAsync(0)
expect(openRelay).toHaveBeenCalledTimes(2)
// 250ms, then 500ms, then 1000ms: the streak grows instead of resetting, which
// it could not do if setActiveSession had run for this dying session.
await vi.advanceTimersByTimeAsync(249)
expect(openRelay).toHaveBeenCalledTimes(2)
await vi.advanceTimersByTimeAsync(1)
expect(openRelay).toHaveBeenCalledTimes(4)
await vi.advanceTimersByTimeAsync(250)
expect(openRelay).toHaveBeenCalledTimes(4)
await vi.advanceTimersByTimeAsync(250)
expect(openRelay).toHaveBeenCalledTimes(6)
await vi.advanceTimersByTimeAsync(999)
expect(openRelay).toHaveBeenCalledTimes(6)
await vi.advanceTimersByTimeAsync(1)
expect(openRelay).toHaveBeenCalledTimes(8)
// No session whose confirm failed is ever booked as a migration.
expect(recordMigration).not.toHaveBeenCalled()
supervisor.stop()
})
it('replays a relay recovery that landed while the direct cutover owned the mutex', async () => {
const logical = new FakeLogicalClient('connected', 'relay')
const openRelay = vi.fn(() => new FakeRelaySession('connected'))
const deps = dependencies({ openDirect: vi.fn(() => new FakeSession('connected')), openRelay })
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
let release!: () => void
const cutover = new Promise<void>((resolve) => {
release = resolve
})
// The candidate loses the cutover, so the logical client stays on the relay path.
logical.migrateTo.mockImplementationOnce(async (candidate) => {
await cutover
candidate.close()
})
// Three authenticated probes plus the observation and dwell windows.
await vi.advanceTimersByTimeAsync(60_000)
expect(logical.migrateTo).toHaveBeenCalledOnce()
logical.publishState('disconnected')
await vi.advanceTimersByTimeAsync(0)
expect(openRelay).not.toHaveBeenCalled()
release()
await vi.advanceTimersByTimeAsync(0)
// The queued request is replayed by afterProbe, never dropped.
expect(openRelay).toHaveBeenCalledOnce()
expect(logical.getState()).toBe('connected')
supervisor.stop()
})
})
@@ -65,6 +65,7 @@ export class FakeRelaySession extends FakeSession implements MobileRelayRpcSessi
renewed: this.renewed,
resumeExpiresAt: this.resumeExpiry
})
whenResumeConfirmed = () => Promise.resolve()
getFailure = () => this.failure
}
@@ -203,6 +204,15 @@ export const bundle: MobileRelayCredentialBundle = {
}
}
// Why: LAN unreachable. A throwing open beats a never-answering socket — the
// direct dial resolves synchronously, so a relay-only test leaves no probe timer
// behind and the reconnect race has exactly one runner.
export function unreachableDirect(): MobileEndpointSupervisorDependencies['openDirect'] {
return vi.fn(() => {
throw new Error('direct endpoint unreachable')
})
}
export function dependencies(
overrides: Partial<MobileEndpointSupervisorDependencies> = {}
): MobileEndpointSupervisorDependencies {
@@ -10,7 +10,8 @@ import {
FakeSession,
host,
mockCredentialRotation,
relay
relay,
unreachableDirect
} from './mobile-endpoint-supervisor-test-fakes'
import { MobileEndpointSupervisor } from './mobile-endpoint-supervisor'
@@ -48,19 +49,17 @@ describe('mobile endpoint supervisor', () => {
supervisor.stop()
})
it('fails over when the direct retry loop publishes reconnecting', async () => {
it('fails over while the direct retry loop is still dialing', async () => {
const logical = new FakeLogicalClient('connecting', 'lan')
const deps = dependencies()
const deps = dependencies({ openDirect: unreachableDirect() })
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
// An unfinished direct dial no longer holds relay back, so the failover has
// already happened by the time the direct client gives up.
logical.publishState('handshaking')
await vi.advanceTimersByTimeAsync(0)
expect(deps.openRelay).not.toHaveBeenCalled()
supervisor.setForeground(true)
await vi.advanceTimersByTimeAsync(0)
expect(deps.openRelay).not.toHaveBeenCalled()
expect(deps.openRelay).toHaveBeenCalledOnce()
logical.publishState('reconnecting')
await vi.waitFor(() => expect(logical.getActivePath()).toBe('relay'))
@@ -148,11 +147,12 @@ describe('mobile endpoint supervisor', () => {
expect(logical.getPendingPath()).toBeNull()
})
it('does not spend a queued relay retry while direct authentication is progressing', async () => {
it('keeps retrying relay on its own cadence while a direct handshake drags on', async () => {
const logical = new FakeLogicalClient('disconnected', 'lan')
const openRelay = vi.fn(() => new FakeRelaySession('disconnected', new RelayOuterError(4408)))
const deps = dependencies({
openRelay,
openDirect: unreachableDirect(),
randomBytes: () => new Uint8Array([128, 0])
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
@@ -160,12 +160,13 @@ describe('mobile endpoint supervisor', () => {
await supervisor.start()
expect(openRelay).toHaveBeenCalledOnce()
// A direct dial that reaches 'handshaking' and stays there used to park relay
// recovery until it gave up; the retry now runs on the failure cadence alone.
logical.publishState('handshaking')
await vi.advanceTimersByTimeAsync(250)
await vi.advanceTimersByTimeAsync(249)
expect(openRelay).toHaveBeenCalledOnce()
logical.publishState('disconnected')
await vi.waitFor(() => expect(openRelay).toHaveBeenCalledTimes(2))
await vi.advanceTimersByTimeAsync(1)
expect(openRelay).toHaveBeenCalledTimes(2)
supervisor.stop()
})
@@ -189,6 +190,7 @@ describe('mobile endpoint supervisor', () => {
resolved,
expect.any(Object),
expect.any(String),
expect.any(Function),
expect.any(Function)
)
expect(deps.saveHost).toHaveBeenCalledWith(
@@ -243,6 +245,7 @@ describe('mobile endpoint supervisor', () => {
const deps = dependencies({
openRelay,
onLog,
openDirect: unreachableDirect(),
randomBytes: () => new Uint8Array([128, 0])
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
@@ -287,6 +290,7 @@ describe('mobile endpoint supervisor', () => {
const openRelay = vi.fn(() => new FakeRelaySession('connected', new RelayOuterError(4408)))
const deps = dependencies({
openRelay,
openDirect: unreachableDirect(),
randomBytes: () => new Uint8Array([128, 0])
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
@@ -469,6 +473,7 @@ describe('mobile endpoint supervisor', () => {
.mockImplementation(() => new FakeRelaySession('connected'))
const deps = dependencies({
openRelay,
openDirect: unreachableDirect(),
writeBundle: vi.fn(() => writePending),
randomBytes: () => new Uint8Array([128, 0])
})
@@ -562,6 +567,7 @@ describe('mobile endpoint supervisor', () => {
relay,
expect.objectContaining({ version: 3 }),
expect.any(String),
expect.any(Function),
expect.any(Function)
)
supervisor.stop()
@@ -610,6 +616,7 @@ describe('mobile endpoint supervisor', () => {
relay,
expect.objectContaining({ version: 3 }),
expect.any(String),
expect.any(Function),
expect.any(Function)
)
supervisor.stop()
@@ -805,6 +812,7 @@ describe('mobile endpoint supervisor', () => {
.mockImplementation(() => new FakeRelaySession('connected'))
const deps = dependencies({
openRelay,
openDirect: unreachableDirect(),
randomBytes: () => new Uint8Array([128, 0])
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
@@ -841,43 +849,6 @@ describe('mobile endpoint supervisor', () => {
supervisor.stop()
})
it('races a relay dial when the direct dial stalls unauthenticated', async () => {
const logical = new FakeLogicalClient('connecting', 'lan')
const deps = dependencies()
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
await vi.advanceTimersByTimeAsync(2_499)
expect(deps.openRelay).not.toHaveBeenCalled()
expect(logical.getState()).toBe('connecting')
// The direct dial never authenticates; the relay wins the race through migrateTo.
await vi.advanceTimersByTimeAsync(1)
await vi.waitFor(() => expect(logical.getActivePath()).toBe('relay'))
expect(logical.migrateTo).toHaveBeenCalledWith(
expect.any(FakeRelaySession),
'relay',
undefined,
expect.any(Function)
)
supervisor.stop()
})
it('cancels the grace race when the direct dial authenticates first', async () => {
const logical = new FakeLogicalClient('connecting', 'lan')
const deps = dependencies()
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
logical.publishState('connected')
expect(vi.getTimerCount()).toBe(0)
await vi.advanceTimersByTimeAsync(5_000)
expect(deps.openRelay).not.toHaveBeenCalled()
expect(logical.getActivePath()).toBe('lan')
supervisor.stop()
})
it('never races a relay dial against a desktop with no relay endpoint', async () => {
const logical = new FakeLogicalClient('connecting', 'lan')
const deps = dependencies()
@@ -890,39 +861,4 @@ describe('mobile endpoint supervisor', () => {
expect(vi.getTimerCount()).toBe(0)
supervisor.stop()
})
it('drops the pending grace race when the phone backgrounds', async () => {
const logical = new FakeLogicalClient('connecting', 'lan')
const deps = dependencies()
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
supervisor.setForeground(false)
await vi.advanceTimersByTimeAsync(5_000)
expect(deps.openRelay).not.toHaveBeenCalled()
expect(vi.getTimerCount()).toBe(0)
supervisor.stop()
})
it('books the shared cooldown when the grace race loses its dial', async () => {
const logical = new FakeLogicalClient('connecting', 'lan')
const openRelay = vi.fn(() => new FakeRelaySession('disconnected', new RelayOuterError(4408)))
const deps = dependencies({ openRelay, randomBytes: () => new Uint8Array([128, 0]) })
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
await vi.advanceTimersByTimeAsync(2_500)
expect(openRelay).toHaveBeenCalledOnce()
// The armed retry runs unforced, so it yields to the still-progressing direct
// dial: the race gets one attempt, never a socket-per-cooldown loop.
await vi.advanceTimersByTimeAsync(60_000)
expect(openRelay).toHaveBeenCalledOnce()
// Direct finally gives up: ordinary recovery still owns the failure.
logical.publishState('reconnecting')
await vi.waitFor(() => expect(openRelay).toHaveBeenCalledTimes(2))
supervisor.stop()
})
})

Some files were not shown because too many files have changed in this diff Show More