mirror of
https://github.com/stablyai/orca.git
synced 2026-09-23 16:02:24 +00:00
Merge remote-tracking branch 'origin/main' into brennanb2025/codex-windows-real-home-lane
This commit is contained in:
@@ -4,7 +4,7 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
image-digest:
|
||||
description: "Immutable relay image digest (sha256: plus 64 lowercase hex characters)"
|
||||
description: 'Immutable relay image digest (sha256: plus 64 lowercase hex characters)'
|
||||
required: true
|
||||
type: string
|
||||
regional-placement-mode:
|
||||
|
||||
@@ -14,6 +14,7 @@ on:
|
||||
not-before: { required: true, type: string }
|
||||
rate-per-minute: { required: true, type: string }
|
||||
preference-max-age-ms: { required: true, type: string }
|
||||
host-cooldown-ms: { required: true, type: string }
|
||||
drain-grace-ms: { required: true, type: string }
|
||||
confirmation: { required: true, type: string }
|
||||
monitor-run-id: { required: true, type: string }
|
||||
@@ -54,6 +55,7 @@ jobs:
|
||||
NOT_BEFORE: ${{ inputs.not-before }}
|
||||
RATE_PER_MINUTE: ${{ inputs.rate-per-minute }}
|
||||
PREFERENCE_MAX_AGE_MS: ${{ inputs.preference-max-age-ms }}
|
||||
HOST_COOLDOWN_MS: ${{ inputs.host-cooldown-ms }}
|
||||
DRAIN_GRACE_MS: ${{ inputs.drain-grace-ms }}
|
||||
CONFIRMATION: ${{ inputs.confirmation }}
|
||||
MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
|
||||
@@ -128,6 +130,7 @@ jobs:
|
||||
--expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
|
||||
--not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \
|
||||
--preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \
|
||||
--host-cooldown-ms "${HOST_COOLDOWN_MS}" \
|
||||
--drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \
|
||||
| tee "${RUNNER_TEMP}/relay-rehome-control.json"
|
||||
|
||||
@@ -299,6 +302,7 @@ jobs:
|
||||
--expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
|
||||
--not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \
|
||||
--preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \
|
||||
--host-cooldown-ms "${HOST_COOLDOWN_MS}" \
|
||||
--drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \
|
||||
| tee "${RUNNER_TEMP}/relay-rehome-control.json"
|
||||
|
||||
|
||||
@@ -52,6 +52,11 @@ on:
|
||||
required: true
|
||||
default: '86400000'
|
||||
type: string
|
||||
host-cooldown-ms:
|
||||
description: Minimum gap between two rehomes of the same host
|
||||
required: true
|
||||
default: '604800000'
|
||||
type: string
|
||||
drain-grace-ms:
|
||||
description: Per-host source drain grace
|
||||
required: true
|
||||
@@ -99,6 +104,7 @@ jobs:
|
||||
not-before: ${{ inputs.not-before }}
|
||||
rate-per-minute: ${{ inputs.rate-per-minute }}
|
||||
preference-max-age-ms: ${{ inputs.preference-max-age-ms }}
|
||||
host-cooldown-ms: ${{ inputs.host-cooldown-ms }}
|
||||
drain-grace-ms: ${{ inputs.drain-grace-ms }}
|
||||
confirmation: ${{ inputs.confirmation }}
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
|
||||
@@ -856,6 +856,7 @@ jobs:
|
||||
src/main/agent-hooks/windows-hook-payload-delivery.test.ts
|
||||
src/main/agent-hooks/windows-direct-cmd-hook-command.test.ts
|
||||
src/main/windows/windows-pty-job.win32.test.ts
|
||||
src/main/windows/windows-msys-job.win32.test.ts
|
||||
src/main/windows/windows-host-job.win32.test.ts
|
||||
src/main/windows/windows-process-tree-command-line-patch.test.ts
|
||||
src/main/windows/windows-process-table-native-addon.win32.test.ts
|
||||
|
||||
@@ -606,8 +606,9 @@ export function createRelayApp(
|
||||
const source = await operations.assignments.cellDeploymentStatus(
|
||||
body.data.sourceCellId
|
||||
)
|
||||
// Any cell that can be drained can be a rehome source, in either
|
||||
// direction, so the probe is gated on the protocol and not on a region.
|
||||
if (
|
||||
source.region !== RELAY_DEFAULT_REGION ||
|
||||
!source.runtime ||
|
||||
source.runtime.cellIncarnation !== body.data.sourceCellIncarnation ||
|
||||
!source.runtime.ready ||
|
||||
@@ -1412,6 +1413,11 @@ const RegionalRehomeControlSchema = z.discriminatedUnion('action', [
|
||||
.int()
|
||||
.min(60_000)
|
||||
.max(30 * 24 * 60 * 60_000),
|
||||
hostCooldownMs: z
|
||||
.number()
|
||||
.int()
|
||||
.min(60_000)
|
||||
.max(30 * 24 * 60 * 60_000),
|
||||
drainGraceMs: z.number().int().min(60_000).max(60 * 60_000),
|
||||
confirmation: z.enum([
|
||||
'ENABLE_REGIONAL_REHOMING',
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { RELAY_DEFAULT_REGION } from '@orca-cloud/relay-contract'
|
||||
import type { RelayDatabase, SqlRow } from './database.js'
|
||||
|
||||
export type CellInventorySnapshotRow = {
|
||||
@@ -92,7 +93,7 @@ export async function readAssignmentInventorySnapshot(
|
||||
return {
|
||||
cells: cellRows.map((row) => ({
|
||||
cellId: asText(row, 'cell_id'),
|
||||
region: optionalText(row, 'region') ?? 'us-central1',
|
||||
region: optionalText(row, 'region') ?? RELAY_DEFAULT_REGION,
|
||||
admissionState: optionalText(row, 'admission_state') ?? 'unset',
|
||||
enabled: asInteger(row, 'enabled') === 1,
|
||||
capacityRequests: asInteger(row, 'capacity_requests'),
|
||||
|
||||
@@ -30,6 +30,9 @@ import {
|
||||
ASSIGNMENT_CONNECTION_HEADROOM_QUERY
|
||||
} from './assignment-connection-headroom-query.js'
|
||||
import { AssignmentIdentityQueue } from './assignment-identity-queue.js'
|
||||
import {
|
||||
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS
|
||||
} from './database.js'
|
||||
import type { RelayCellConfig } from './config.js'
|
||||
import type {
|
||||
RelayDatabase,
|
||||
@@ -121,7 +124,7 @@ export type RelayAssignmentMigration = AssignmentIdentity & {
|
||||
|
||||
export type RegionalRehomeAttempt = AssignmentIdentity & {
|
||||
attemptId: string
|
||||
preferredRegion: 'asia-east2'
|
||||
preferredRegion: RelayRegion
|
||||
sourceCellId: string
|
||||
sourceCellUrl: string
|
||||
sourceCellIncarnation: string
|
||||
@@ -151,6 +154,7 @@ export type RegionalRehomeControl = {
|
||||
notBefore: number
|
||||
ratePerMinute: number
|
||||
preferenceMaxAgeMs: number
|
||||
hostCooldownMs: number
|
||||
drainGraceMs: number
|
||||
}
|
||||
|
||||
@@ -4921,6 +4925,7 @@ export class RelayAssignmentStore {
|
||||
notBefore: number
|
||||
ratePerMinute: number
|
||||
preferenceMaxAgeMs: number
|
||||
hostCooldownMs: number
|
||||
drainGraceMs: number
|
||||
}): Promise<RegionalRehomeControl> {
|
||||
if (!Number.isSafeInteger(input.expectedGeneration) || input.expectedGeneration < 0) {
|
||||
@@ -4939,6 +4944,13 @@ export class RelayAssignmentStore {
|
||||
) {
|
||||
throw new Error('invalid_regional_rehome_preference_age')
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(input.hostCooldownMs) ||
|
||||
input.hostCooldownMs < 60_000 ||
|
||||
input.hostCooldownMs > 30 * 24 * 60 * 60_000
|
||||
) {
|
||||
throw new Error('invalid_regional_rehome_host_cooldown')
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(input.drainGraceMs) ||
|
||||
input.drainGraceMs < 60_000 ||
|
||||
@@ -4967,14 +4979,15 @@ export class RelayAssignmentStore {
|
||||
await transaction.query(
|
||||
`UPDATE relay_region_rehome_control
|
||||
SET generation = generation + 1, enabled = ?, not_before = ?,
|
||||
rate_per_minute = ?, preference_max_age_ms = ?, drain_grace_ms = ?,
|
||||
updated_at = ?
|
||||
rate_per_minute = ?, preference_max_age_ms = ?, host_cooldown_ms = ?,
|
||||
drain_grace_ms = ?, updated_at = ?
|
||||
WHERE control_id = 'global'`,
|
||||
[
|
||||
input.enabled ? 1 : 0,
|
||||
input.notBefore,
|
||||
input.ratePerMinute,
|
||||
input.preferenceMaxAgeMs,
|
||||
input.hostCooldownMs,
|
||||
input.drainGraceMs,
|
||||
now
|
||||
]
|
||||
@@ -5006,10 +5019,17 @@ export class RelayAssignmentStore {
|
||||
await database.query(
|
||||
`INSERT INTO relay_region_rehome_control
|
||||
(control_id, generation, enabled, observation_started_at, not_before,
|
||||
rate_per_minute, preference_max_age_ms, drain_grace_ms, updated_at)
|
||||
VALUES ('global', 0, 0, ?, 0, 10, ?, ?, ?)
|
||||
rate_per_minute, preference_max_age_ms, host_cooldown_ms, drain_grace_ms,
|
||||
updated_at)
|
||||
VALUES ('global', 0, 0, ?, 0, 10, ?, ?, ?, ?)
|
||||
ON CONFLICT (control_id) DO NOTHING`,
|
||||
[now, 24 * 60 * 60_000, 60 * 60_000, now]
|
||||
[
|
||||
now,
|
||||
24 * 60 * 60_000,
|
||||
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS,
|
||||
60 * 60_000,
|
||||
now
|
||||
]
|
||||
)
|
||||
}
|
||||
|
||||
@@ -5017,6 +5037,9 @@ export class RelayAssignmentStore {
|
||||
return await this.readRegionalRehomeFleetSafety(this.database, this.now())
|
||||
}
|
||||
|
||||
// The rehome fleet is every general cell that can be drained: those are the
|
||||
// sources and, because a host must be movable back out again, the only legal
|
||||
// targets. The region join stays so a cell with no region row is excluded.
|
||||
private async readRegionalRehomeFleetSafety(
|
||||
database: RelayDatabase,
|
||||
now: number
|
||||
@@ -5037,10 +5060,7 @@ export class RelayAssignmentStore {
|
||||
ON safety.cell_id = runtime.cell_id
|
||||
AND safety.cell_incarnation = runtime.cell_incarnation
|
||||
WHERE cell.enabled = 1 AND admission.admission_state = 'general'
|
||||
AND (
|
||||
region.region = 'asia-east2' OR
|
||||
(region.region = 'us-central1' AND capability.regional_rehome_protocol >= 1)
|
||||
)`
|
||||
AND capability.regional_rehome_protocol >= 1`
|
||||
)
|
||||
const valid = rows.filter(
|
||||
(row) =>
|
||||
@@ -5119,6 +5139,10 @@ export class RelayAssignmentStore {
|
||||
}
|
||||
const intervalMs = Math.ceil(60_000 / integer(control, 'rate_per_minute'))
|
||||
const preferenceCutoff = now - integer(control, 'preference_max_age_ms')
|
||||
// A host that was rehomed recently is left alone whichever way its
|
||||
// preference now points: a flapping region probe must not walk one host
|
||||
// back and forth across an ocean.
|
||||
const cooldownCutoff = now - integer(control, 'host_cooldown_ms')
|
||||
await transaction.query(
|
||||
`INSERT INTO relay_region_rehome_worker_state
|
||||
(worker_id, next_dispatch_at, paused_until, consecutive_failures, updated_at)
|
||||
@@ -5284,9 +5308,8 @@ export class RelayAssignmentStore {
|
||||
JOIN relay_cell_capabilities capability
|
||||
ON capability.cell_id = runtime.cell_id
|
||||
AND capability.cell_incarnation = runtime.cell_incarnation
|
||||
WHERE preference.preferred_region = 'asia-east2'
|
||||
WHERE preference.preferred_region <> region.region
|
||||
AND preference.observed_at >= ?
|
||||
AND region.region = 'us-central1'
|
||||
AND admission.admission_state = 'general'
|
||||
AND runtime.ready = 1 AND runtime.last_heartbeat_at > ?
|
||||
AND capability.regional_rehome_protocol >= 1
|
||||
@@ -5306,9 +5329,38 @@ export class RelayAssignmentStore {
|
||||
AND migration.relay_host_id = assignment.relay_host_id
|
||||
AND migration.completed_at IS NULL AND migration.aborted_at IS NULL
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM relay_region_rehome_attempts recent
|
||||
WHERE recent.user_id = preference.user_id
|
||||
AND recent.relay_host_id = preference.relay_host_id
|
||||
AND recent.created_at > ?
|
||||
)
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM relay_cell_regions target_region
|
||||
JOIN relay_cells target_cell ON target_cell.cell_id = target_region.cell_id
|
||||
JOIN relay_cell_admission target_admission
|
||||
ON target_admission.cell_id = target_region.cell_id
|
||||
JOIN relay_cell_runtime target_runtime
|
||||
ON target_runtime.cell_id = target_region.cell_id
|
||||
JOIN relay_cell_capabilities target_capability
|
||||
ON target_capability.cell_id = target_runtime.cell_id
|
||||
AND target_capability.cell_incarnation = target_runtime.cell_incarnation
|
||||
WHERE target_region.region = preference.preferred_region
|
||||
AND target_cell.enabled = 1
|
||||
AND target_admission.admission_state = 'general'
|
||||
AND target_runtime.ready = 1
|
||||
AND target_runtime.last_heartbeat_at > ?
|
||||
AND target_capability.regional_rehome_protocol >= 1
|
||||
)
|
||||
ORDER BY preference.observed_at, preference.user_id, preference.relay_host_id
|
||||
LIMIT 10`,
|
||||
[preferenceCutoff, now - this.heartbeatTtlMs, now]
|
||||
[
|
||||
preferenceCutoff,
|
||||
now - this.heartbeatTtlMs,
|
||||
now,
|
||||
cooldownCutoff,
|
||||
now - this.heartbeatTtlMs
|
||||
]
|
||||
)
|
||||
candidatesTotal = candidates.length
|
||||
for (const candidate of candidates) {
|
||||
@@ -5320,6 +5372,7 @@ export class RelayAssignmentStore {
|
||||
sourceCellId: text(candidate, 'source_cell_id'),
|
||||
assignmentEpoch: integer(candidate, 'assignment_epoch'),
|
||||
preferenceCutoff,
|
||||
cooldownCutoff,
|
||||
drainGraceMs: integer(control, 'drain_grace_ms'),
|
||||
processSafety: effectiveProcessSafety,
|
||||
worker,
|
||||
@@ -5374,6 +5427,7 @@ export class RelayAssignmentStore {
|
||||
sourceCellId: string
|
||||
assignmentEpoch: number
|
||||
preferenceCutoff: number
|
||||
cooldownCutoff: number
|
||||
drainGraceMs: number
|
||||
processSafety: RegionalRehomeSafetySnapshot
|
||||
worker: SqlRow
|
||||
@@ -5397,14 +5451,11 @@ export class RelayAssignmentStore {
|
||||
[input.identity.userId, input.identity.relayHostId]
|
||||
)
|
||||
)[0]
|
||||
if (
|
||||
!preference ||
|
||||
text(preference, 'preferred_region') !== 'asia-east2' ||
|
||||
integer(preference, 'observed_at') < input.preferenceCutoff
|
||||
) {
|
||||
if (!preference || integer(preference, 'observed_at') < input.preferenceCutoff) {
|
||||
input.skips.push({ reason: 'candidate_stale' })
|
||||
return null
|
||||
}
|
||||
const preferredRegion = relayRegion(preference, 'preferred_region')
|
||||
const activeMigration = await transaction.queryLocked(
|
||||
`SELECT assignment_epoch FROM relay_assignment_migrations
|
||||
WHERE user_id = ? AND relay_host_id = ?
|
||||
@@ -5415,6 +5466,18 @@ export class RelayAssignmentStore {
|
||||
input.skips.push({ reason: 'candidate_stale' })
|
||||
return null
|
||||
}
|
||||
// Re-read under the claim: an attempt committed between the scan and here
|
||||
// would otherwise start a second move for the same host.
|
||||
const recentAttempt = await transaction.query(
|
||||
`SELECT 1 FROM relay_region_rehome_attempts
|
||||
WHERE user_id = ? AND relay_host_id = ? AND created_at > ?
|
||||
LIMIT 1`,
|
||||
[input.identity.userId, input.identity.relayHostId, input.cooldownCutoff]
|
||||
)
|
||||
if (recentAttempt.length > 0) {
|
||||
input.skips.push({ reason: 'host_cooldown' })
|
||||
return null
|
||||
}
|
||||
const activityLeases = await this.lockAssignmentActivities(transaction, input.identity)
|
||||
assertAssignmentActivityCounts(assignment, activityLeases, 0)
|
||||
const cells = await this.lockCellInventory(transaction, 'nowait')
|
||||
@@ -5469,11 +5532,17 @@ export class RelayAssignmentStore {
|
||||
)
|
||||
return null
|
||||
}
|
||||
// The preference read under lock can now agree with the cell the host is
|
||||
// already on: nothing to move, in either direction.
|
||||
if (regions.get(input.sourceCellId) === preferredRegion) {
|
||||
input.skips.push({ reason: 'candidate_stale' })
|
||||
return null
|
||||
}
|
||||
if (
|
||||
!source ||
|
||||
integer(source, 'enabled') !== 1 ||
|
||||
admission.get(input.sourceCellId) !== 'general' ||
|
||||
regions.get(input.sourceCellId) !== RELAY_DEFAULT_REGION ||
|
||||
regions.get(input.sourceCellId) === undefined ||
|
||||
!sourceRuntime ||
|
||||
integer(sourceRuntime, 'ready') !== 1 ||
|
||||
integer(sourceRuntime, 'last_heartbeat_at') <= input.now - this.heartbeatTtlMs ||
|
||||
@@ -5502,17 +5571,25 @@ export class RelayAssignmentStore {
|
||||
return null
|
||||
}
|
||||
const connectionHeadroom = await this.connectionHeadroomByCell(transaction)
|
||||
// A target must be drainable too, or the host lands somewhere it can never
|
||||
// be rehomed out of again -- the trap this bidirectional move exists to undo.
|
||||
const eligibleTargets = cells.filter((row) => {
|
||||
const cellId = text(row, 'cell_id')
|
||||
const runtime = runtimes.find((candidate) => text(candidate, 'cell_id') === cellId)
|
||||
const capability = capabilities.find(
|
||||
(candidate) => text(candidate, 'cell_id') === cellId
|
||||
)
|
||||
return (
|
||||
cellId !== input.sourceCellId &&
|
||||
integer(row, 'enabled') === 1 &&
|
||||
admission.get(cellId) === 'general' &&
|
||||
regions.get(cellId) === 'asia-east2' &&
|
||||
regions.get(cellId) === preferredRegion &&
|
||||
runtime !== undefined &&
|
||||
integer(runtime, 'ready') === 1 &&
|
||||
integer(runtime, 'last_heartbeat_at') > input.now - this.heartbeatTtlMs
|
||||
integer(runtime, 'last_heartbeat_at') > input.now - this.heartbeatTtlMs &&
|
||||
capability !== undefined &&
|
||||
text(capability, 'cell_incarnation') === text(runtime, 'cell_incarnation') &&
|
||||
integer(capability, 'regional_rehome_protocol') >= 1
|
||||
)
|
||||
})
|
||||
const targetIsClean = (row: SqlRow): boolean => {
|
||||
@@ -5668,12 +5745,13 @@ export class RelayAssignmentStore {
|
||||
drain_grace_ms, send_attempts, last_send_attempt_at,
|
||||
drain_receipt_at, drain_outcome, completed_at, aborted_at,
|
||||
created_at, updated_at)
|
||||
VALUES (?, ?, ?, 'asia-east2', ?, ?, ?, ?, ?, ?, ?, 0, NULL,
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, NULL,
|
||||
NULL, NULL, NULL, NULL, ?, ?)`,
|
||||
[
|
||||
attemptId,
|
||||
input.identity.userId,
|
||||
input.identity.relayHostId,
|
||||
preferredRegion,
|
||||
input.sourceCellId,
|
||||
text(sourceRuntime, 'cell_incarnation'),
|
||||
targetCellId,
|
||||
@@ -5688,7 +5766,7 @@ export class RelayAssignmentStore {
|
||||
return {
|
||||
...input.identity,
|
||||
attemptId,
|
||||
preferredRegion: 'asia-east2',
|
||||
preferredRegion,
|
||||
sourceCellId: input.sourceCellId,
|
||||
sourceCellUrl: text(source, 'cell_url'),
|
||||
sourceCellIncarnation: text(sourceRuntime, 'cell_incarnation'),
|
||||
@@ -8101,7 +8179,7 @@ function regionalRehomeAttempt(row: SqlRow): RegionalRehomeAttempt {
|
||||
attemptId: text(row, 'attempt_id'),
|
||||
userId: text(row, 'user_id'),
|
||||
relayHostId: text(row, 'relay_host_id'),
|
||||
preferredRegion: 'asia-east2',
|
||||
preferredRegion: relayRegion(row, 'preferred_region'),
|
||||
sourceCellId: text(row, 'source_cell_id'),
|
||||
sourceCellUrl: text(row, 'source_cell_url'),
|
||||
sourceCellIncarnation: text(row, 'source_cell_incarnation'),
|
||||
@@ -8122,6 +8200,7 @@ function regionalRehomeControl(row: SqlRow): RegionalRehomeControl {
|
||||
notBefore: integer(row, 'not_before'),
|
||||
ratePerMinute: integer(row, 'rate_per_minute'),
|
||||
preferenceMaxAgeMs: integer(row, 'preference_max_age_ms'),
|
||||
hostCooldownMs: integer(row, 'host_cooldown_ms'),
|
||||
drainGraceMs: integer(row, 'drain_grace_ms')
|
||||
}
|
||||
}
|
||||
@@ -8161,10 +8240,9 @@ function regionalRehomeFleetSafetyFromInventory(input: {
|
||||
return (
|
||||
integer(row, 'enabled') === 1 &&
|
||||
input.admission.get(cellId) === 'general' &&
|
||||
(input.regions.get(cellId) === 'asia-east2' ||
|
||||
(input.regions.get(cellId) === RELAY_DEFAULT_REGION &&
|
||||
capability !== undefined &&
|
||||
integer(capability, 'regional_rehome_protocol') >= 1))
|
||||
input.regions.get(cellId) !== undefined &&
|
||||
capability !== undefined &&
|
||||
integer(capability, 'regional_rehome_protocol') >= 1
|
||||
)
|
||||
})
|
||||
const valid = required.flatMap((row) => {
|
||||
@@ -8231,6 +8309,7 @@ function regionalRehomeFleetSafetyFailure(
|
||||
type RegionalRehomeCandidateSkip = {
|
||||
reason:
|
||||
| 'candidate_stale'
|
||||
| 'host_cooldown'
|
||||
| 'source_ineligible'
|
||||
| 'source_unclean'
|
||||
| 'source_control_inactive'
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { RELAY_DEFAULT_REGION } from '@orca-cloud/relay-contract'
|
||||
import type { RelayConfig } from './config.js'
|
||||
import { googleMetadataIdentityToken } from './google-metadata-identity-token.js'
|
||||
import type { RegionalRehomeSafetySnapshot } from './relay-observability.js'
|
||||
@@ -57,7 +58,7 @@ export function startCellHeartbeat(
|
||||
v: 1,
|
||||
cellId: config.cellId,
|
||||
cellUrl: config.cellUrl,
|
||||
region: config.region ?? 'us-central1',
|
||||
region: config.region ?? RELAY_DEFAULT_REGION,
|
||||
cellIncarnation,
|
||||
startedAt,
|
||||
ready,
|
||||
|
||||
@@ -34,7 +34,11 @@ vi.mock('pg', () => ({
|
||||
}
|
||||
}))
|
||||
|
||||
import { openRelayDatabase, relayPostgresStatementTimeoutMs } from './database.js'
|
||||
import {
|
||||
openRelayDatabase,
|
||||
POSTGRES_SCHEMA_MIGRATIONS,
|
||||
relayPostgresStatementTimeoutMs
|
||||
} from './database.js'
|
||||
import { applyPostgresSchema } from './postgres-schema-startup.js'
|
||||
|
||||
const SCHEMA_POOL = {
|
||||
@@ -118,7 +122,9 @@ describe('PostgreSQL relay deadlines', () => {
|
||||
// Statements can open with a leading `--` rationale comment.
|
||||
const body = (statement: string): string =>
|
||||
statement.replace(/^(?:\s*--[^\n]*\n)*\s*/, '')
|
||||
expect(ddl.every((statement) => /^CREATE\b/i.test(body(statement)))).toBe(true)
|
||||
expect(
|
||||
ddl.every((statement) => /^(?:CREATE|ALTER TABLE)\b/i.test(body(statement)))
|
||||
).toBe(true)
|
||||
// The backfill is DML, so it stays on the deadline-bearing serving pool.
|
||||
expect(ddl.some((statement) => statement.includes('INSERT INTO'))).toBe(false)
|
||||
await database.close()
|
||||
@@ -263,6 +269,54 @@ describe('PostgreSQL schema startup', () => {
|
||||
expect(query).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('treats an existing constraint as an applied ADD CONSTRAINT', async () => {
|
||||
// Postgres has no `ADD CONSTRAINT IF NOT EXISTS`, and a retry would only
|
||||
// repeat 42710, so a re-run and a concurrent startup both move on.
|
||||
const error = Object.assign(new Error('already exists'), { code: '42710' })
|
||||
const query = vi
|
||||
.fn<(statement: string) => Promise<unknown>>()
|
||||
.mockRejectedValueOnce(error)
|
||||
.mockResolvedValue(undefined)
|
||||
const pause = vi.fn(async () => undefined)
|
||||
|
||||
await applyPostgresSchema(
|
||||
['ALTER TABLE test ADD CONSTRAINT test_check CHECK (id > 0)', 'CREATE TABLE test2'],
|
||||
query,
|
||||
{ wait: pause }
|
||||
)
|
||||
|
||||
expect(pause).not.toHaveBeenCalled()
|
||||
expect(query).toHaveBeenCalledTimes(2)
|
||||
expect(query).toHaveBeenLastCalledWith('CREATE TABLE test2')
|
||||
})
|
||||
|
||||
it('recognises every shipped ADD CONSTRAINT migration as re-runnable', async () => {
|
||||
// Guards the statement text against the pattern that classifies it.
|
||||
const shipped = POSTGRES_SCHEMA_MIGRATIONS.filter((statement) =>
|
||||
statement.includes('ADD CONSTRAINT')
|
||||
)
|
||||
expect(shipped.length).toBeGreaterThan(0)
|
||||
const error = Object.assign(new Error('already exists'), { code: '42710' })
|
||||
const query = vi.fn<(statement: string) => Promise<unknown>>().mockRejectedValue(error)
|
||||
|
||||
await applyPostgresSchema(shipped, query, { wait: async () => undefined })
|
||||
|
||||
expect(query).toHaveBeenCalledTimes(shipped.length)
|
||||
})
|
||||
|
||||
it('still fails an ADD CONSTRAINT that violates existing rows', async () => {
|
||||
const error = Object.assign(new Error('check violation'), { code: '23514' })
|
||||
const query = vi.fn<(statement: string) => Promise<unknown>>().mockRejectedValue(error)
|
||||
|
||||
await expect(
|
||||
applyPostgresSchema(
|
||||
['ALTER TABLE test ADD CONSTRAINT test_check CHECK (id > 0)'],
|
||||
query,
|
||||
{ wait: async () => undefined }
|
||||
)
|
||||
).rejects.toBe(error)
|
||||
})
|
||||
|
||||
it.each([
|
||||
['42710', 'CREATE INDEX IF NOT EXISTS test_index ON test(id)'],
|
||||
['42710', 'CREATE TABLE test'],
|
||||
|
||||
@@ -2,7 +2,12 @@ import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { openInMemoryRelayDatabase, openRelayDatabase } from './database.js'
|
||||
import {
|
||||
openInMemoryRelayDatabase,
|
||||
openRelayDatabase,
|
||||
POSTGRES_SCHEMA_MIGRATIONS,
|
||||
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS
|
||||
} from './database.js'
|
||||
|
||||
const temporaryDirectories: string[] = []
|
||||
|
||||
@@ -142,6 +147,41 @@ describe('relay database', () => {
|
||||
await second.close()
|
||||
})
|
||||
|
||||
it('renders every region check from the shared region list', async () => {
|
||||
// Derived, not hand-written: a third region must not leave one column
|
||||
// rejecting a value the rest of the relay already accepts.
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
const checked = await database.query(
|
||||
`SELECT name, sql FROM sqlite_master
|
||||
WHERE type = 'table'
|
||||
AND name IN ('relay_assignment_region_preferences', 'relay_cell_regions',
|
||||
'relay_region_rehome_attempts')
|
||||
ORDER BY name`
|
||||
)
|
||||
const list = `IN ('us-central1', 'asia-east2')`
|
||||
expect(checked.map((row) => row.name)).toEqual([
|
||||
'relay_assignment_region_preferences',
|
||||
'relay_cell_regions',
|
||||
'relay_region_rehome_attempts'
|
||||
])
|
||||
expect(checked.every((row) => String(row.sql).includes(list))).toBe(true)
|
||||
expect(
|
||||
POSTGRES_SCHEMA_MIGRATIONS.some((statement) => statement.includes(list))
|
||||
).toBe(true)
|
||||
await database.close()
|
||||
})
|
||||
|
||||
it('indexes rehome attempts by host recency for the per-host cooldown', async () => {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
const rows = await database.query(
|
||||
`SELECT sql FROM sqlite_master
|
||||
WHERE type = 'index' AND name = 'relay_region_rehome_attempts_host_recency'`
|
||||
)
|
||||
expect(rows[0]?.sql).toContain('(user_id, relay_host_id, created_at)')
|
||||
expect(REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS).toBe(7 * 24 * 60 * 60_000)
|
||||
await database.close()
|
||||
})
|
||||
|
||||
it('indexes region preference expiry by observation time', async () => {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
const rows = await database.query(
|
||||
|
||||
@@ -3,6 +3,7 @@ import { performance } from 'node:perf_hooks'
|
||||
import { join } from 'node:path'
|
||||
import { DatabaseSync } from 'node:sqlite'
|
||||
import pg from 'pg'
|
||||
import { RELAY_REGIONS } from '@orca-cloud/relay-contract'
|
||||
import {
|
||||
emptyPostgresPoolPressureCounts,
|
||||
PostgresPoolPressure,
|
||||
@@ -24,6 +25,14 @@ function setLocalLockTimeout(milliseconds: number): string {
|
||||
return `SET LOCAL lock_timeout = '${milliseconds}ms'`
|
||||
}
|
||||
|
||||
// Region CHECK lists come from the contract so a new region cannot leave a
|
||||
// column rejecting values the rest of the relay already accepts.
|
||||
const REGION_LIST = RELAY_REGIONS.map((region) => `'${region}'`).join(', ')
|
||||
|
||||
// A host that was just moved is not a candidate again for this long, so a
|
||||
// desktop whose region probe flips cannot walk itself back and forth.
|
||||
export const REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS = 7 * 24 * 60 * 60_000
|
||||
|
||||
export type SqlRow = Record<string, unknown>
|
||||
export type RelayLockOptions = {
|
||||
failIfUnavailable?: boolean
|
||||
@@ -181,7 +190,7 @@ CREATE TABLE IF NOT EXISTS relay_assignment_region_preferences (
|
||||
user_id TEXT NOT NULL,
|
||||
relay_host_id TEXT NOT NULL,
|
||||
preferred_region TEXT NOT NULL
|
||||
CHECK (preferred_region IN ('us-central1', 'asia-east2')),
|
||||
CHECK (preferred_region IN (${REGION_LIST})),
|
||||
observed_at BIGINT NOT NULL,
|
||||
PRIMARY KEY (user_id, relay_host_id)
|
||||
);
|
||||
@@ -204,6 +213,8 @@ CREATE TABLE IF NOT EXISTS relay_region_rehome_control (
|
||||
not_before BIGINT NOT NULL,
|
||||
rate_per_minute BIGINT NOT NULL,
|
||||
preference_max_age_ms BIGINT NOT NULL,
|
||||
host_cooldown_ms BIGINT NOT NULL
|
||||
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS},
|
||||
drain_grace_ms BIGINT NOT NULL,
|
||||
updated_at BIGINT NOT NULL
|
||||
);
|
||||
@@ -212,7 +223,9 @@ CREATE TABLE IF NOT EXISTS relay_region_rehome_attempts (
|
||||
attempt_id TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL,
|
||||
relay_host_id TEXT NOT NULL,
|
||||
preferred_region TEXT NOT NULL CHECK (preferred_region = 'asia-east2'),
|
||||
preferred_region TEXT NOT NULL
|
||||
CONSTRAINT relay_region_rehome_attempts_preferred_region_valid
|
||||
CHECK (preferred_region IN (${REGION_LIST})),
|
||||
source_cell_id TEXT NOT NULL,
|
||||
source_cell_incarnation TEXT NOT NULL,
|
||||
target_cell_id TEXT NOT NULL,
|
||||
@@ -234,6 +247,8 @@ CREATE TABLE IF NOT EXISTS relay_region_rehome_attempts (
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS relay_region_rehome_attempts_pending
|
||||
ON relay_region_rehome_attempts(drain_receipt_at, last_send_attempt_at, completed_at, aborted_at);
|
||||
CREATE INDEX IF NOT EXISTS relay_region_rehome_attempts_host_recency
|
||||
ON relay_region_rehome_attempts(user_id, relay_host_id, created_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS relay_cells (
|
||||
cell_id TEXT PRIMARY KEY,
|
||||
@@ -248,7 +263,7 @@ CREATE TABLE IF NOT EXISTS relay_cells (
|
||||
|
||||
CREATE TABLE IF NOT EXISTS relay_cell_regions (
|
||||
cell_id TEXT PRIMARY KEY,
|
||||
region TEXT NOT NULL CHECK (region IN ('us-central1', 'asia-east2'))
|
||||
region TEXT NOT NULL CHECK (region IN (${REGION_LIST}))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS relay_cell_admission (
|
||||
@@ -580,6 +595,21 @@ CREATE TABLE IF NOT EXISTS relay_audit_events (
|
||||
CREATE INDEX IF NOT EXISTS relay_audit_events_at ON relay_audit_events(at);
|
||||
`
|
||||
|
||||
// Rehoming is bidirectional, but tables created before that carry the
|
||||
// original single-region column check. The old constraint is the one Postgres
|
||||
// auto-named; the replacement is named, so both statements are no-ops on a
|
||||
// database the current schema created and neither can drop the other.
|
||||
export const POSTGRES_SCHEMA_MIGRATIONS = [
|
||||
`ALTER TABLE relay_region_rehome_attempts
|
||||
DROP CONSTRAINT IF EXISTS relay_region_rehome_attempts_preferred_region_check`,
|
||||
`ALTER TABLE relay_region_rehome_attempts
|
||||
ADD CONSTRAINT relay_region_rehome_attempts_preferred_region_valid
|
||||
CHECK (preferred_region IN (${REGION_LIST}))`,
|
||||
`ALTER TABLE relay_region_rehome_control
|
||||
ADD COLUMN IF NOT EXISTS host_cooldown_ms BIGINT NOT NULL
|
||||
DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS}`
|
||||
]
|
||||
|
||||
function postgresSql(sql: string): string {
|
||||
let index = 0
|
||||
return sql.replace(/\?/g, () => `$${++index}`)
|
||||
@@ -1009,7 +1039,10 @@ async function applySchemaOnUntimedPool(
|
||||
const database = new PostgresDatabase(pool)
|
||||
try {
|
||||
await applyPostgresSchema(
|
||||
SCHEMA.split(';').filter((statement) => statement.trim()),
|
||||
[
|
||||
...SCHEMA.split(';').filter((statement) => statement.trim()),
|
||||
...POSTGRES_SCHEMA_MIGRATIONS
|
||||
],
|
||||
async (statement) => await database.query(statement)
|
||||
)
|
||||
} finally {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract'
|
||||
import { RELAY_CLOSE_CODE, RELAY_PROTOCOL_LIMITS } from '@orca-cloud/relay-contract'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type WebSocket from 'ws'
|
||||
import type { RelayAssignmentStore } from './assignment-store.js'
|
||||
@@ -102,7 +102,9 @@ function harness(options: { random?: () => number; now?: () => number } = {}) {
|
||||
const store = {
|
||||
resolveResume: vi.fn().mockResolvedValue({ userId: identity.sub }),
|
||||
reserveCredential: vi.fn().mockResolvedValue(reservation),
|
||||
failReservation: vi.fn().mockResolvedValue(undefined)
|
||||
failReservation: vi.fn().mockResolvedValue(undefined),
|
||||
recordConnectionBasis: vi.fn().mockResolvedValue(undefined),
|
||||
deactivateBasis: vi.fn().mockResolvedValue(undefined)
|
||||
}
|
||||
const observer = {
|
||||
recordAuth: vi.fn(),
|
||||
@@ -110,7 +112,9 @@ function harness(options: { random?: () => number; now?: () => number } = {}) {
|
||||
recordHttp: vi.fn(),
|
||||
recordReconnect: vi.fn(),
|
||||
recordSql: vi.fn(),
|
||||
recordClientAcceptAbandoned: vi.fn()
|
||||
recordClientAcceptAbandoned: vi.fn(),
|
||||
recordClientAcceptCompleted: vi.fn(),
|
||||
recordControlRtt: vi.fn()
|
||||
} satisfies RelayRuntimeObserver
|
||||
const registry = new HostSessionRegistry(
|
||||
config,
|
||||
@@ -325,6 +329,210 @@ describe('client accept abandoned mid-DB-phase', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('successful client accept timing', () => {
|
||||
beforeEach(() => vi.useFakeTimers())
|
||||
afterEach(() => {
|
||||
vi.clearAllTimers()
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('times every serialized stage plus the attach window once relay-hello lands', async () => {
|
||||
let now = 1_700_000_000_000
|
||||
const h = harness({ now: () => now })
|
||||
const control = await activeHost(h)
|
||||
h.store.resolveResume.mockImplementationOnce(async () => {
|
||||
now += 5
|
||||
return { userId: identity.sub }
|
||||
})
|
||||
h.store.reserveCredential.mockImplementationOnce(async () => {
|
||||
now += 7
|
||||
return reservation
|
||||
})
|
||||
h.acquireActivity.mockImplementationOnce(async () => {
|
||||
now += 11
|
||||
})
|
||||
h.store.recordConnectionBasis.mockImplementationOnce(async () => {
|
||||
now += 3
|
||||
})
|
||||
const client = new FakeSocket()
|
||||
const hostData = new FakeSocket()
|
||||
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
|
||||
try {
|
||||
await h.registry.acceptClient(client as unknown as WebSocket, identity.relayHostId, 'cred')
|
||||
const connOpen = JSON.parse(
|
||||
String(control.send.mock.calls.find((call) => String(call[0]).includes('conn-open'))![0])
|
||||
) as { connId: string; connTicket: string }
|
||||
// The desktop's data leg is the attach window this is meant to expose.
|
||||
now += 23
|
||||
const accepted = await h.registry.acceptHostData(
|
||||
hostData as unknown as WebSocket,
|
||||
connOpen.connId,
|
||||
connOpen.connTicket,
|
||||
1
|
||||
)
|
||||
|
||||
expect(accepted).toBe(true)
|
||||
expect(h.observer.recordClientAcceptCompleted).toHaveBeenCalledWith({
|
||||
totalMs: 49,
|
||||
stageMs: { assignment: 5, credential: 7, activity: 11, attach: 23, basis: 3 }
|
||||
})
|
||||
const line = log.mock.calls
|
||||
.map((call) => String(call[0]))
|
||||
.find((entry) => entry.includes('orca_relay_client_accept_completed'))
|
||||
expect(line).toBeDefined()
|
||||
const event = JSON.parse(line!) as {
|
||||
role: string
|
||||
cellId: string
|
||||
region: string
|
||||
credentialKind: string
|
||||
stageMs: Record<string, number>
|
||||
totalMs: number
|
||||
relayHostIdDigest: string
|
||||
}
|
||||
expect(event.credentialKind).toBe('resume')
|
||||
// Joins the line back to the emitting process, like the runtime metrics event.
|
||||
expect(event).toMatchObject({ role: 'cell', cellId: config.cellId, region: 'us-central1' })
|
||||
expect(Object.keys(event.stageMs).sort()).toEqual([
|
||||
'activity',
|
||||
'assignment',
|
||||
'attach',
|
||||
'basis',
|
||||
'credential'
|
||||
])
|
||||
for (const stage of Object.values(event.stageMs)) expect(stage).toBeGreaterThanOrEqual(0)
|
||||
// The stages tile the accept end to end: every millisecond is attributed.
|
||||
const summed = Object.values(event.stageMs).reduce((total, stage) => total + stage, 0)
|
||||
expect(summed).toBe(event.totalMs)
|
||||
expect(event.relayHostIdDigest).toMatch(/^[0-9a-f]{12}$/)
|
||||
expect(line).not.toContain(identity.relayHostId)
|
||||
} finally {
|
||||
log.mockRestore()
|
||||
h.registry.drain(0)
|
||||
vi.advanceTimersByTime(0)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
// Fires one heartbeat and returns the `t` of the ping it sent, which is the only
|
||||
// echo the registry will time.
|
||||
async function advanceToPing(control: FakeSocket, clock: { now: number }): Promise<number> {
|
||||
clock.now += RELAY_PROTOCOL_LIMITS.controlPingIntervalMs
|
||||
await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
|
||||
const ping = control.send.mock.calls
|
||||
.filter((call) => String(call[0]).includes('"type":"ping"'))
|
||||
.at(-1)!
|
||||
return (JSON.parse(String(ping[0])) as { t: number }).t
|
||||
}
|
||||
|
||||
describe('control round-trip sampling', () => {
|
||||
beforeEach(() => vi.useFakeTimers())
|
||||
afterEach(() => {
|
||||
vi.clearAllTimers()
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('logs a host once at the fourth sample and not again within the hour', async () => {
|
||||
const clock = { now: 1_700_000_000_000 }
|
||||
const h = harness({ now: () => clock.now })
|
||||
const control = await activeHost(h)
|
||||
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
|
||||
const rttLines = (): string[] =>
|
||||
log.mock.calls
|
||||
.map((call) => String(call[0]))
|
||||
.filter((entry) => entry.includes('orca_relay_host_control_rtt'))
|
||||
// One heartbeat, then the desktop's echo of that ping's own `t` 40 ms later.
|
||||
const roundTrip = async (): Promise<void> => {
|
||||
const pingAt = await advanceToPing(control, clock)
|
||||
clock.now += 40
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt }), false)
|
||||
}
|
||||
try {
|
||||
for (let round = 0; round < 3; round++) await roundTrip()
|
||||
expect(h.observer.recordControlRtt).toHaveBeenCalledTimes(3)
|
||||
expect(rttLines()).toHaveLength(0)
|
||||
|
||||
await roundTrip()
|
||||
expect(h.observer.recordControlRtt).toHaveBeenLastCalledWith(40)
|
||||
expect(rttLines()).toHaveLength(1)
|
||||
expect(JSON.parse(rttLines()[0]!)).toMatchObject({
|
||||
event: 'orca_relay_host_control_rtt',
|
||||
role: 'cell',
|
||||
cellId: config.cellId,
|
||||
region: 'us-central1',
|
||||
rttMsMedian: 40,
|
||||
sampleCount: 4
|
||||
})
|
||||
expect(rttLines()[0]).not.toContain(identity.relayHostId)
|
||||
|
||||
// Later samples keep feeding the fleet metric, but stay silent for an hour.
|
||||
for (let round = 0; round < 8; round++) await roundTrip()
|
||||
expect(h.observer.recordControlRtt).toHaveBeenCalledTimes(12)
|
||||
expect(rttLines()).toHaveLength(1)
|
||||
|
||||
const elapsedStart = clock.now
|
||||
while (clock.now - elapsedStart < 60 * 60 * 1000) await roundTrip()
|
||||
expect(rttLines()).toHaveLength(2)
|
||||
} finally {
|
||||
log.mockRestore()
|
||||
h.registry.drain(0)
|
||||
vi.advanceTimersByTime(0)
|
||||
}
|
||||
})
|
||||
|
||||
it('ignores a pong that answers no outstanding ping', async () => {
|
||||
const clock = { now: 1_700_000_000_000 }
|
||||
const h = harness({ now: () => clock.now })
|
||||
const control = await activeHost(h)
|
||||
try {
|
||||
// Nothing has been pinged yet, so even a plausible echo is not a round trip.
|
||||
control.emit('message', JSON.stringify({ type: 'pong' }), false)
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: 'later' }), false)
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: clock.now }), false)
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: clock.now - 10 }), false)
|
||||
expect(h.observer.recordControlRtt).not.toHaveBeenCalled()
|
||||
|
||||
const pingAt = await advanceToPing(control, clock)
|
||||
// A guessed timestamp is not the outstanding ping's `t`, so it is dropped.
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt - 1 }), false)
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt + 1 }), false)
|
||||
expect(h.observer.recordControlRtt).not.toHaveBeenCalled()
|
||||
|
||||
clock.now += 10
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt }), false)
|
||||
expect(h.observer.recordControlRtt).toHaveBeenCalledWith(10)
|
||||
} finally {
|
||||
h.registry.drain(0)
|
||||
vi.advanceTimersByTime(0)
|
||||
}
|
||||
})
|
||||
|
||||
it('records one sample per ping however many pongs a host floods', async () => {
|
||||
const clock = { now: 1_700_000_000_000 }
|
||||
const h = harness({ now: () => clock.now })
|
||||
const control = await activeHost(h)
|
||||
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
|
||||
try {
|
||||
const pingAt = await advanceToPing(control, clock)
|
||||
clock.now += 12
|
||||
for (let flood = 0; flood < 5_000; flood++) {
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: pingAt }), false)
|
||||
control.emit('message', JSON.stringify({ type: 'pong', t: clock.now }), false)
|
||||
}
|
||||
// One answered ping is one process-wide sample and one per-session sample, so
|
||||
// neither the metric window nor the hourly log line can be flooded.
|
||||
expect(h.observer.recordControlRtt).toHaveBeenCalledTimes(1)
|
||||
expect(h.observer.recordControlRtt).toHaveBeenCalledWith(12)
|
||||
expect(
|
||||
log.mock.calls.filter((call) => String(call[0]).includes('orca_relay_host_control_rtt'))
|
||||
).toHaveLength(0)
|
||||
} finally {
|
||||
log.mockRestore()
|
||||
h.registry.drain(0)
|
||||
vi.advanceTimersByTime(0)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('control lease jitter', () => {
|
||||
beforeEach(() => vi.useFakeTimers())
|
||||
afterEach(() => {
|
||||
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
ASSIGNMENT_LIMITS,
|
||||
CONTROL_CONTINUITY_LIMITS,
|
||||
RELAY_CLOSE_CODE,
|
||||
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS,
|
||||
RELAY_PROTOCOL_LIMITS
|
||||
} from '@orca-cloud/relay-contract'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
@@ -1005,3 +1006,115 @@ describe('control lease recovery after the session is gone', () => {
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('host hello ack pending connections', () => {
|
||||
const DETAILS = new Set([RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS])
|
||||
const LEGACY_ENTRY = { connId: 'conn-1', connTicket: 'T'.repeat(43) }
|
||||
const DETAILED_ENTRY = { ...LEGACY_ENTRY, kind: 'invite', relayDeviceId: 'device-1' }
|
||||
|
||||
beforeEach(() => vi.useFakeTimers())
|
||||
afterEach(() => {
|
||||
vi.clearAllTimers()
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
function newRegistry(): ReturnType<typeof createRegistry> {
|
||||
return createRegistry(
|
||||
vi
|
||||
.fn<RelayAssignmentStore['activateControl']>()
|
||||
.mockResolvedValue('control:production-gce-c3:1')
|
||||
)
|
||||
}
|
||||
|
||||
function addPendingConnection(session: HostSession): void {
|
||||
session.pendingConns.set('conn-1', {
|
||||
...LEGACY_ENTRY,
|
||||
reservation: {
|
||||
userId: identity.sub,
|
||||
relayHostId: identity.relayHostId,
|
||||
credentialKind: 'invite',
|
||||
relayDeviceId: 'device-1'
|
||||
},
|
||||
client: new FakeSocket() as unknown as WebSocket,
|
||||
attachTimer: setTimeout(() => {}, 60_000),
|
||||
credentialActivityId: null
|
||||
} as unknown as Parameters<typeof session.pendingConns.set>[1])
|
||||
}
|
||||
|
||||
function sentAck(socket: FakeSocket): Record<string, unknown> {
|
||||
const acks = socket.send.mock.calls
|
||||
.map((call) => JSON.parse(String(call[0])) as Record<string, unknown>)
|
||||
.filter((message) => message.type === 'host-hello-ack')
|
||||
return acks.at(-1)!
|
||||
}
|
||||
|
||||
function sessionOf(registry: HostSessionRegistry): HostSession {
|
||||
return registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
|
||||
}
|
||||
|
||||
async function ackFor(capabilities?: ReadonlySet<string>): Promise<Record<string, unknown>> {
|
||||
const { registry, activate } = newRegistry()
|
||||
const socket = new FakeSocket()
|
||||
registry.acceptControl(
|
||||
socket as unknown as WebSocket,
|
||||
identity,
|
||||
undefined,
|
||||
capabilities ?? new Set()
|
||||
)
|
||||
await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
|
||||
const session = sessionOf(registry)
|
||||
addPendingConnection(session)
|
||||
socket.send.mockClear()
|
||||
;(registry as unknown as { sendHelloAck(session: HostSession): void }).sendHelloAck(session)
|
||||
return sentAck(socket)
|
||||
}
|
||||
|
||||
async function ackAfterRebind(
|
||||
first: ReadonlySet<string>,
|
||||
successor: ReadonlySet<string>
|
||||
): Promise<{ opening: Record<string, unknown>; rebound: Record<string, unknown> }> {
|
||||
const { registry, activate } = newRegistry()
|
||||
const opening = new FakeSocket()
|
||||
registry.acceptControl(opening as unknown as WebSocket, identity, undefined, first)
|
||||
await activate(opening as unknown as WebSocket, identity, null, 1, false, 1)
|
||||
const session = sessionOf(registry)
|
||||
addPendingConnection(session)
|
||||
opening.send.mockClear()
|
||||
;(registry as unknown as { sendHelloAck(session: HostSession): void }).sendHelloAck(session)
|
||||
|
||||
const rebound = new FakeSocket()
|
||||
registry.acceptControl(rebound as unknown as WebSocket, identity, undefined, successor)
|
||||
await activate(rebound as unknown as WebSocket, identity, session, 1, true, 1)
|
||||
return { opening: sentAck(opening), rebound: sentAck(rebound) }
|
||||
}
|
||||
|
||||
it('states the pending kind and device to a host that advertised it can read them', async () => {
|
||||
const ack = await ackFor(DETAILS)
|
||||
|
||||
expect(ack.pendingConns).toEqual([DETAILED_ENTRY])
|
||||
})
|
||||
|
||||
it('restates only the identifiers to a host that never advertised the capability', async () => {
|
||||
// A shipped host parses these entries strictly, so an unannounced key fails
|
||||
// the whole ack parse and kills a control that was working.
|
||||
const ack = await ackFor()
|
||||
|
||||
expect(ack.pendingConns).toEqual([LEGACY_ENTRY])
|
||||
})
|
||||
|
||||
it('downgrades the restated entry when the successor control drops the capability', async () => {
|
||||
// The capability belongs to the socket, not the session: a rebind can land a
|
||||
// control whose decoder is older than the one that opened the session.
|
||||
const { opening, rebound } = await ackAfterRebind(DETAILS, new Set())
|
||||
|
||||
expect(opening.pendingConns).toEqual([DETAILED_ENTRY])
|
||||
expect(rebound.pendingConns).toEqual([LEGACY_ENTRY])
|
||||
})
|
||||
|
||||
it('upgrades the restated entry when the successor control adds the capability', async () => {
|
||||
const { opening, rebound } = await ackAfterRebind(new Set(), DETAILS)
|
||||
|
||||
expect(opening.pendingConns).toEqual([LEGACY_ENTRY])
|
||||
expect(rebound.pendingConns).toEqual([DETAILED_ENTRY])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { createHash, createHmac, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto'
|
||||
import {
|
||||
ASSIGNMENT_LIMITS,
|
||||
RELAY_DEFAULT_REGION,
|
||||
AuthRefreshSchema,
|
||||
buildHostChallengePlaintext,
|
||||
buildHostProofMacInput,
|
||||
@@ -13,9 +14,11 @@ import {
|
||||
HostChallengeAckSchema,
|
||||
HostHelloSchema,
|
||||
InviteCreateSchema,
|
||||
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS,
|
||||
RELAY_PROTOCOL_LIMITS,
|
||||
RELAY_CLOSE_CODE,
|
||||
type RelayHostCloseReason
|
||||
type RelayHostCloseReason,
|
||||
type RelayRegion
|
||||
} from '@orca-cloud/relay-contract'
|
||||
import nacl from 'tweetnacl'
|
||||
import type WebSocket from 'ws'
|
||||
@@ -29,7 +32,12 @@ import {
|
||||
import { HostCloseReasonMemory } from './host-close-reason-memory.js'
|
||||
import { relayHostLogDigest } from './relay-host-log-digest.js'
|
||||
import type { RelayTokenClaims } from './relay-token-verifier.js'
|
||||
import type { RelayClientAcceptStage, RelayRuntimeObserver } from './relay-observability.js'
|
||||
import {
|
||||
percentile,
|
||||
type RelayClientAcceptStage,
|
||||
type RelayClientAcceptTimedStage,
|
||||
type RelayRuntimeObserver
|
||||
} from './relay-observability.js'
|
||||
import type { PendingHostDataReservation } from './relay-connection-ledger.js'
|
||||
import { closeRelayWebSocket } from './relay-websocket-close.js'
|
||||
import { ProcessQueuedByteBudget, wireSplice } from './splice-forwarder.js'
|
||||
@@ -45,6 +53,20 @@ function printableCloseReason(reason: Buffer | string): string {
|
||||
type VerifyRelayToken = (token: string) => Promise<RelayTokenClaims | null>
|
||||
type HostState = 'proving' | 'active' | 'orphaned' | 'drain-only' | 'closed'
|
||||
|
||||
// A host's distance to its cell moves on the scale of a rehome, not a heartbeat,
|
||||
// so a short window is enough to ride out one stalled ping.
|
||||
const CONTROL_RTT_WINDOW = 8
|
||||
const CONTROL_RTT_LOG_SAMPLE_THRESHOLD = 4
|
||||
const CONTROL_RTT_LOG_INTERVAL_MS = 60 * 60 * 1000
|
||||
// A pong claiming a multi-minute round trip is clock skew, not distance.
|
||||
const CONTROL_RTT_MAX_PLAUSIBLE_MS = 120_000
|
||||
|
||||
// Wall clock can step backwards mid-accept; a negative latency would poison the
|
||||
// percentiles it feeds.
|
||||
function nonNegativeMs(elapsedMs: number): number {
|
||||
return Math.max(0, elapsedMs)
|
||||
}
|
||||
|
||||
const CONTROL_ACTIVITY_RENEWAL_INTERVAL_MS = RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2
|
||||
// Preserve the existing 75s renewal runway after doubling the successful-call interval.
|
||||
const CONTROL_ACTIVITY_LEASE_MS =
|
||||
@@ -68,6 +90,10 @@ export type HostSession = {
|
||||
orphanTimer: ReturnType<typeof setTimeout> | null
|
||||
heartbeatTimer: ReturnType<typeof setInterval> | null
|
||||
lastPongAt: number
|
||||
// The `t` of the ping still waiting for its echo; null once one has answered it.
|
||||
pendingPingAt: number | null
|
||||
controlRttSamplesMs: number[]
|
||||
controlRttLoggedAt: number | null
|
||||
activityRenewalDueAt: number
|
||||
activityRenewalAttempt: number
|
||||
activityRenewalCompletedAttempt: number
|
||||
@@ -95,6 +121,15 @@ type PendingConnection = {
|
||||
attachTimer: ReturnType<typeof setTimeout>
|
||||
credentialActivityId: string | null
|
||||
capacityReservation?: PendingHostDataReservation
|
||||
timing: ClientAcceptTiming
|
||||
}
|
||||
|
||||
// Carries the phone-side accept clock across to the desktop's data leg, which
|
||||
// lands in a separate call and is the only place the accept is known to succeed.
|
||||
type ClientAcceptTiming = {
|
||||
startedAt: number
|
||||
connOpenAt: number
|
||||
stageMs: Record<RelayClientAcceptStage, number>
|
||||
}
|
||||
|
||||
function decodeCanonicalBase64(value: string, bytes: number): Uint8Array | null {
|
||||
@@ -146,6 +181,7 @@ export class HostSessionRegistry {
|
||||
// but a signed-out desktop never comes back, so the phone that asks minutes
|
||||
// later would otherwise find nothing to explain its rejection with.
|
||||
private readonly hostCloseReasons = new HostCloseReasonMemory(() => this.now())
|
||||
private readonly hostCapabilities = new WeakMap<WebSocket, ReadonlySet<string>>()
|
||||
private draining = false
|
||||
|
||||
constructor(
|
||||
@@ -192,6 +228,17 @@ export class HostSessionRegistry {
|
||||
)
|
||||
return true
|
||||
}
|
||||
const stageMs: Record<RelayClientAcceptStage, number> = {
|
||||
assignment: 0,
|
||||
credential: 0,
|
||||
activity: 0
|
||||
}
|
||||
let stageCursor = acceptStartedAt
|
||||
const markStage = (stage: RelayClientAcceptStage): void => {
|
||||
const at = this.now()
|
||||
stageMs[stage] = at - stageCursor
|
||||
stageCursor = at
|
||||
}
|
||||
if (this.config.role === 'cell') {
|
||||
// Each lookup is its own pooled round trip; stop between them once the phone
|
||||
// has left instead of running the rest of the chain for nobody.
|
||||
@@ -212,6 +259,7 @@ export class HostSessionRegistry {
|
||||
}
|
||||
if (abandonedByClient('assignment')) return
|
||||
}
|
||||
markStage('assignment')
|
||||
const reservation = await this.store.reserveCredential(hostId, credential)
|
||||
if (!reservation) {
|
||||
capacityReservation?.release()
|
||||
@@ -221,6 +269,7 @@ export class HostSessionRegistry {
|
||||
}
|
||||
this.observer.recordAuth(true)
|
||||
if (abandonedByClient('credential', () => this.failReservationBestEffort(reservation))) return
|
||||
markStage('credential')
|
||||
const sessionKey = this.key(reservation.userId, hostId)
|
||||
const session = this.sessions.get(sessionKey)
|
||||
if (
|
||||
@@ -275,6 +324,7 @@ export class HostSessionRegistry {
|
||||
) {
|
||||
return
|
||||
}
|
||||
markStage('activity')
|
||||
const attachTimer = setTimeout(() => {
|
||||
session.pendingConns.delete(connId)
|
||||
capacityReservation?.release()
|
||||
@@ -289,7 +339,10 @@ export class HostSessionRegistry {
|
||||
client: socket,
|
||||
attachTimer,
|
||||
credentialActivityId,
|
||||
capacityReservation
|
||||
capacityReservation,
|
||||
// Attach starts where the activity stage ended, so the conn-open send is
|
||||
// charged to it and no wall-clock gap goes unattributed.
|
||||
timing: { startedAt: acceptStartedAt, connOpenAt: stageCursor, stageMs }
|
||||
}
|
||||
capacityReservation?.bind(connId)
|
||||
session.pendingConns.set(connId, pending)
|
||||
@@ -336,6 +389,7 @@ export class HostSessionRegistry {
|
||||
return false
|
||||
}
|
||||
this.observer.recordAuth(true)
|
||||
const attachedAt = this.now()
|
||||
clearTimeout(pending.attachTimer)
|
||||
session.pendingConns.delete(connId)
|
||||
session.activeConnIds.add(connId)
|
||||
@@ -409,6 +463,7 @@ export class HostSessionRegistry {
|
||||
close()
|
||||
return false
|
||||
}
|
||||
const helloAt = this.now()
|
||||
send(pending.client, 'relay-hello', {
|
||||
ok: true,
|
||||
credentialKind: pending.reservation.credentialKind,
|
||||
@@ -424,14 +479,92 @@ export class HostSessionRegistry {
|
||||
}
|
||||
: {})
|
||||
})
|
||||
this.recordClientAcceptCompleted(session, pending, attachedAt, helloAt)
|
||||
return true
|
||||
}
|
||||
|
||||
// The stages tile the whole accept, so their sum is the total minus only the
|
||||
// clamping above: `basis` is the splice lease and connection-basis writes that
|
||||
// land between the host data leg and relay-hello.
|
||||
private recordClientAcceptCompleted(
|
||||
session: HostSession,
|
||||
pending: PendingConnection,
|
||||
attachedAt: number,
|
||||
helloAt: number
|
||||
): void {
|
||||
const stageMs: Record<RelayClientAcceptTimedStage, number> = {
|
||||
assignment: nonNegativeMs(pending.timing.stageMs.assignment),
|
||||
credential: nonNegativeMs(pending.timing.stageMs.credential),
|
||||
activity: nonNegativeMs(pending.timing.stageMs.activity),
|
||||
attach: nonNegativeMs(attachedAt - pending.timing.connOpenAt),
|
||||
basis: nonNegativeMs(helloAt - attachedAt)
|
||||
}
|
||||
const totalMs = nonNegativeMs(helloAt - pending.timing.startedAt)
|
||||
this.observer.recordClientAcceptCompleted?.({ totalMs, stageMs })
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_client_accept_completed',
|
||||
...this.logIdentity(),
|
||||
credentialKind: pending.reservation.credentialKind,
|
||||
stageMs,
|
||||
totalMs,
|
||||
relayHostIdDigest: relayHostLogDigest(session.relayHostId)
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// Matches the runtime metrics event so a log line and a metric point can be
|
||||
// joined back to the process that emitted them.
|
||||
private logIdentity(): { role: string; cellId: string; region: RelayRegion } {
|
||||
return {
|
||||
role: this.config.role,
|
||||
cellId: this.config.cellId,
|
||||
region: this.config.region ?? RELAY_DEFAULT_REGION
|
||||
}
|
||||
}
|
||||
|
||||
// Every desktop build already echoes the ping's `t`, so a pong is only timed when
|
||||
// it answers the outstanding ping: at most one sample per ping this cell sent,
|
||||
// however many a host floods. A pong that lost the race to the next ping is
|
||||
// dropped here but still counts as proof of life for the silence watchdog.
|
||||
private recordControlRtt(session: HostSession, echoedPingAt: unknown): void {
|
||||
if (typeof echoedPingAt !== 'number' || echoedPingAt !== session.pendingPingAt) return
|
||||
session.pendingPingAt = null
|
||||
const now = this.now()
|
||||
const rttMs = now - echoedPingAt
|
||||
if (rttMs < 0 || rttMs > CONTROL_RTT_MAX_PLAUSIBLE_MS) return
|
||||
this.observer.recordControlRtt?.(rttMs)
|
||||
const samples = session.controlRttSamplesMs
|
||||
samples.push(rttMs)
|
||||
if (samples.length > CONTROL_RTT_WINDOW) samples.shift()
|
||||
if (samples.length < CONTROL_RTT_LOG_SAMPLE_THRESHOLD) return
|
||||
if (
|
||||
session.controlRttLoggedAt !== null &&
|
||||
now - session.controlRttLoggedAt < CONTROL_RTT_LOG_INTERVAL_MS
|
||||
) {
|
||||
return
|
||||
}
|
||||
session.controlRttLoggedAt = now
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
event: 'orca_relay_host_control_rtt',
|
||||
...this.logIdentity(),
|
||||
relayHostIdDigest: relayHostLogDigest(session.relayHostId),
|
||||
rttMsMedian: percentile(samples, 0.5),
|
||||
sampleCount: samples.length
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
acceptControl(
|
||||
socket: WebSocket,
|
||||
identity: RelayTokenClaims,
|
||||
connectionInclusionWatermark?: number
|
||||
connectionInclusionWatermark?: number,
|
||||
hostCapabilities?: ReadonlySet<string>
|
||||
): void {
|
||||
// Keyed by socket, not session: a rebind swaps the session's socket, and the
|
||||
// successor's own advertisement is the only one that describes its decoder.
|
||||
if (hostCapabilities?.size) this.hostCapabilities.set(socket, hostCapabilities)
|
||||
if (this.draining) {
|
||||
socket.close(RELAY_CLOSE_CODE.DRAINING, 'relay draining')
|
||||
return
|
||||
@@ -790,6 +923,7 @@ export class HostSessionRegistry {
|
||||
existing.appVersion = appVersion
|
||||
existing.leaseExpiresAt = this.controlLeaseExpiresAt()
|
||||
existing.lastPongAt = this.now()
|
||||
existing.pendingPingAt = null
|
||||
existing.activityRenewalDueAt =
|
||||
this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs
|
||||
this.wireActiveControl(existing)
|
||||
@@ -843,6 +977,9 @@ export class HostSessionRegistry {
|
||||
orphanTimer: null,
|
||||
heartbeatTimer: null,
|
||||
lastPongAt: this.now(),
|
||||
pendingPingAt: null,
|
||||
controlRttSamplesMs: [],
|
||||
controlRttLoggedAt: null,
|
||||
activityRenewalDueAt: this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs,
|
||||
activityRenewalAttempt: 0,
|
||||
activityRenewalCompletedAttempt: 0,
|
||||
@@ -900,6 +1037,7 @@ export class HostSessionRegistry {
|
||||
const parsed = JSON.parse(raw.toString()) as Record<string, unknown>
|
||||
if (parsed.type === 'pong') {
|
||||
session.lastPongAt = this.now()
|
||||
this.recordControlRtt(session, parsed.t)
|
||||
return
|
||||
}
|
||||
if (parsed.type === 'auth-refresh') {
|
||||
@@ -1047,11 +1185,18 @@ export class HostSessionRegistry {
|
||||
session.socket.close(RELAY_CLOSE_CODE.DRAINING, 'control lease expired')
|
||||
return
|
||||
}
|
||||
session.pendingPingAt = now
|
||||
send(session.socket, 'ping', { t: now })
|
||||
}
|
||||
|
||||
private sendHelloAck(session: HostSession): void {
|
||||
if (!session.socket) return
|
||||
// Without these a host that missed the conn-open cannot dial the pending
|
||||
// connection: it would have to guess the pairing kind and the device the
|
||||
// relay authorized. Only sent to a host that said it can read them.
|
||||
const details = this.hostCapabilities
|
||||
.get(session.socket)
|
||||
?.has(RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS)
|
||||
send(session.socket, 'host-hello-ack', {
|
||||
v: 1,
|
||||
generation: session.generation,
|
||||
@@ -1060,7 +1205,13 @@ export class HostSessionRegistry {
|
||||
activeConnIds: [...session.activeConnIds],
|
||||
pendingConns: [...session.pendingConns.values()].map((pending) => ({
|
||||
connId: pending.connId,
|
||||
connTicket: pending.connTicket
|
||||
connTicket: pending.connTicket,
|
||||
...(details
|
||||
? {
|
||||
kind: pending.reservation.credentialKind,
|
||||
relayDeviceId: pending.reservation.relayDeviceId
|
||||
}
|
||||
: {})
|
||||
}))
|
||||
})
|
||||
}
|
||||
|
||||
@@ -49,6 +49,20 @@ function concurrentCreateCollision(
|
||||
return false
|
||||
}
|
||||
|
||||
const ALTER_TABLE_ADD_CONSTRAINT =
|
||||
/^\s*ALTER\s+TABLE\s+\S+\s+ADD\s+CONSTRAINT\b/i
|
||||
|
||||
// Postgres has no `ADD CONSTRAINT IF NOT EXISTS`, so a re-run and a concurrent
|
||||
// startup both land on 42710 once the constraint exists. Unlike a CREATE race
|
||||
// this is terminal, not transient: retrying only repeats it, so the statement
|
||||
// counts as applied.
|
||||
function constraintAlreadyApplied(error: unknown, statement: string): boolean {
|
||||
return (
|
||||
ALTER_TABLE_ADD_CONSTRAINT.test(statement) &&
|
||||
(error as { code?: unknown }).code === '42710'
|
||||
)
|
||||
}
|
||||
|
||||
function retryableSchemaError(error: unknown, statement: string): boolean {
|
||||
const value = error as { code?: unknown; constraint?: unknown }
|
||||
return (
|
||||
@@ -73,6 +87,7 @@ export async function applyPostgresSchema(
|
||||
await query(statement)
|
||||
break
|
||||
} catch (error) {
|
||||
if (constraintAlreadyApplied(error, statement)) break
|
||||
const code = String((error as { code?: unknown }).code)
|
||||
const remainingMs = deadlineAt - now()
|
||||
const retryable = retryableSchemaError(error, statement)
|
||||
|
||||
@@ -315,6 +315,7 @@ describe('regional rehome director controls', () => {
|
||||
notBefore: 100,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 24 * 60 * 60_000,
|
||||
hostCooldownMs: 7 * 24 * 60 * 60_000,
|
||||
drainGraceMs: 60_000,
|
||||
confirmation: 'ENABLE_REGIONAL_REHOMING'
|
||||
}
|
||||
@@ -343,6 +344,14 @@ describe('regional rehome director controls', () => {
|
||||
'deploy-token',
|
||||
{ ...apply, confirmation: 'DISABLE_REGIONAL_REHOMING' }
|
||||
)).status).toBe(400)
|
||||
// The per-host cooldown is part of the durable shape an operator must state.
|
||||
const { hostCooldownMs: _omitted, ...withoutCooldown } = apply
|
||||
expect((await postPath(
|
||||
app,
|
||||
'/v1/admin/regional-rehome-control',
|
||||
'deploy-token',
|
||||
withoutCooldown
|
||||
)).status).toBe(400)
|
||||
})
|
||||
|
||||
it('probes dedicated trust twice and returns only aggregate proof', async () => {
|
||||
@@ -411,6 +420,78 @@ describe('regional rehome director controls', () => {
|
||||
expect(JSON.stringify(responseBody)).not.toContain('rehome-token')
|
||||
})
|
||||
|
||||
it('probes a source cell in any region, not only the default one', async () => {
|
||||
// Rehoming moves hosts in both directions, so an asia-east2 cell is a
|
||||
// source too and its trust has to be provable the same way.
|
||||
const cellDeploymentStatus = vi.fn().mockResolvedValue({
|
||||
cellId: 'production-gce-c27',
|
||||
cellUrl: 'https://c27.relay.example.test',
|
||||
region: 'asia-east2',
|
||||
runtime: {
|
||||
cellIncarnation,
|
||||
ready: true,
|
||||
heartbeatFresh: true,
|
||||
regionalRehomeProtocol: 1
|
||||
}
|
||||
})
|
||||
const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
|
||||
store: {} as never,
|
||||
assignments: { cellDeploymentStatus } as never,
|
||||
drain: vi.fn(),
|
||||
regionalRehomeIdentityToken: vi.fn(async () => 'rehome-token'),
|
||||
regionalRehomeFetch: (async () =>
|
||||
Response.json({
|
||||
v: 1,
|
||||
outcome: 'host-not-connected',
|
||||
sharedRuntimeIdentityRejected: true
|
||||
})) as typeof fetch,
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
|
||||
const response = await postPath(
|
||||
app,
|
||||
'/v1/admin/regional-rehome-trust-probe',
|
||||
'deploy-token',
|
||||
{ v: 1, sourceCellId: 'production-gce-c27', sourceCellIncarnation: cellIncarnation }
|
||||
)
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(await response.json()).toMatchObject({ proven: true })
|
||||
})
|
||||
|
||||
it('still refuses a trust probe against a cell without the drain protocol', async () => {
|
||||
const cellDeploymentStatus = vi.fn().mockResolvedValue({
|
||||
cellId: 'production-gce-c27',
|
||||
cellUrl: 'https://c27.relay.example.test',
|
||||
region: 'asia-east2',
|
||||
runtime: {
|
||||
cellIncarnation,
|
||||
ready: true,
|
||||
heartbeatFresh: true,
|
||||
regionalRehomeProtocol: 0
|
||||
}
|
||||
})
|
||||
const sourceFetch = vi.fn<typeof fetch>()
|
||||
const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
|
||||
store: {} as never,
|
||||
assignments: { cellDeploymentStatus } as never,
|
||||
drain: vi.fn(),
|
||||
regionalRehomeIdentityToken: vi.fn(async () => 'rehome-token'),
|
||||
regionalRehomeFetch: sourceFetch,
|
||||
ready: vi.fn(async () => true)
|
||||
})
|
||||
|
||||
const response = await postPath(
|
||||
app,
|
||||
'/v1/admin/regional-rehome-trust-probe',
|
||||
'deploy-token',
|
||||
{ v: 1, sourceCellId: 'production-gce-c27', sourceCellIncarnation: cellIncarnation }
|
||||
)
|
||||
|
||||
expect(response.status).toBe(409)
|
||||
expect(sourceFetch).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('restricts trust probes to deploy authorization and strict input', async () => {
|
||||
const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
|
||||
store: {} as never,
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
import pg from 'pg'
|
||||
import { afterAll, beforeEach, describe, expect, it } from 'vitest'
|
||||
import {
|
||||
openRelayDatabase,
|
||||
REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS,
|
||||
type RelayDatabase
|
||||
} from './database.js'
|
||||
|
||||
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
|
||||
const describePostgres = databaseUrl ? describe : describe.skip
|
||||
const schema = 'relay_rehome_constraint_migration_test'
|
||||
|
||||
// The shape shipped before rehoming became bidirectional: a single-region
|
||||
// column check that Postgres auto-names.
|
||||
const LEGACY_ATTEMPTS_TABLE = `
|
||||
CREATE TABLE relay_region_rehome_attempts (
|
||||
attempt_id TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL,
|
||||
relay_host_id TEXT NOT NULL,
|
||||
preferred_region TEXT NOT NULL CHECK (preferred_region = 'asia-east2'),
|
||||
source_cell_id TEXT NOT NULL,
|
||||
source_cell_incarnation TEXT NOT NULL,
|
||||
target_cell_id TEXT NOT NULL,
|
||||
target_cell_incarnation TEXT NOT NULL,
|
||||
previous_epoch BIGINT NOT NULL,
|
||||
assignment_epoch BIGINT NOT NULL,
|
||||
drain_grace_ms BIGINT NOT NULL,
|
||||
send_attempts BIGINT NOT NULL,
|
||||
last_send_attempt_at BIGINT,
|
||||
drain_receipt_at BIGINT,
|
||||
drain_outcome TEXT CHECK (
|
||||
drain_outcome IN ('accepted', 'already-accepted', 'host-not-connected')
|
||||
),
|
||||
completed_at BIGINT,
|
||||
aborted_at BIGINT,
|
||||
created_at BIGINT NOT NULL,
|
||||
updated_at BIGINT NOT NULL,
|
||||
UNIQUE (user_id, relay_host_id, assignment_epoch)
|
||||
)`
|
||||
|
||||
// The control row as it shipped before the per-host cooldown existed.
|
||||
const LEGACY_CONTROL_TABLE = `
|
||||
CREATE TABLE relay_region_rehome_control (
|
||||
control_id TEXT PRIMARY KEY,
|
||||
generation BIGINT NOT NULL,
|
||||
enabled BIGINT NOT NULL,
|
||||
observation_started_at BIGINT NOT NULL,
|
||||
not_before BIGINT NOT NULL,
|
||||
rate_per_minute BIGINT NOT NULL,
|
||||
preference_max_age_ms BIGINT NOT NULL,
|
||||
drain_grace_ms BIGINT NOT NULL,
|
||||
updated_at BIGINT NOT NULL
|
||||
)`
|
||||
|
||||
const attemptValues = (attemptId: string, preferredRegion: string): unknown[] => [
|
||||
attemptId,
|
||||
'user-1',
|
||||
'abcdefghijklmnop',
|
||||
preferredRegion,
|
||||
'cell-source',
|
||||
'11111111-1111-4111-8111-111111111111',
|
||||
'cell-target',
|
||||
'22222222-2222-4222-8222-222222222222',
|
||||
1,
|
||||
Number(attemptId.at(-1)),
|
||||
0,
|
||||
0,
|
||||
1_000_000,
|
||||
1_000_000
|
||||
]
|
||||
|
||||
const INSERT_ATTEMPT = `INSERT INTO relay_region_rehome_attempts
|
||||
(attempt_id, user_id, relay_host_id, preferred_region, source_cell_id,
|
||||
source_cell_incarnation, target_cell_id, target_cell_incarnation,
|
||||
previous_epoch, assignment_epoch, drain_grace_ms, send_attempts,
|
||||
created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`
|
||||
|
||||
describePostgres('PostgreSQL regional rehome constraint migration', () => {
|
||||
let scopedUrl = ''
|
||||
|
||||
async function withClient(
|
||||
operation: (client: pg.Client) => Promise<void>
|
||||
): Promise<void> {
|
||||
const client = new pg.Client({ connectionString: databaseUrl })
|
||||
await client.connect()
|
||||
try {
|
||||
await operation(client)
|
||||
} finally {
|
||||
await client.end()
|
||||
}
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
await withClient(async (client) => {
|
||||
await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
|
||||
await client.query(`CREATE SCHEMA ${schema}`)
|
||||
await client.query(`SET search_path = ${schema}`)
|
||||
await client.query(LEGACY_ATTEMPTS_TABLE)
|
||||
await client.query(LEGACY_CONTROL_TABLE)
|
||||
await client.query(
|
||||
`INSERT INTO relay_region_rehome_control
|
||||
(control_id, generation, enabled, observation_started_at, not_before,
|
||||
rate_per_minute, preference_max_age_ms, drain_grace_ms, updated_at)
|
||||
VALUES ('global', 3, 0, 1, 0, 10, 86400000, 60000, 1)`
|
||||
)
|
||||
// Production data the replacement constraint has to validate.
|
||||
await client.query(INSERT_ATTEMPT, attemptValues('attempt-1', 'asia-east2'))
|
||||
})
|
||||
const url = new URL(databaseUrl!)
|
||||
url.searchParams.set('options', `-c search_path=${schema}`)
|
||||
scopedUrl = url.toString()
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
await withClient(async (client) => {
|
||||
await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
|
||||
})
|
||||
})
|
||||
|
||||
it('upgrades a legacy single-region constraint in place', async () => {
|
||||
const database = await openRelayDatabase({ databaseUrl: scopedUrl, dataDir: '' })
|
||||
try {
|
||||
await withClient(async (client) => {
|
||||
await client.query(`SET search_path = ${schema}`)
|
||||
await client.query(INSERT_ATTEMPT, attemptValues('attempt-2', 'us-central1'))
|
||||
await expect(
|
||||
client.query(INSERT_ATTEMPT, attemptValues('attempt-3', 'europe-west1'))
|
||||
).rejects.toMatchObject({ code: '23514' })
|
||||
const constraints = await client.query(
|
||||
`SELECT conname FROM pg_constraint
|
||||
WHERE conrelid = 'relay_region_rehome_attempts'::regclass
|
||||
AND conname LIKE '%preferred_region%'
|
||||
ORDER BY conname`
|
||||
)
|
||||
expect(constraints.rows).toEqual([
|
||||
{ conname: 'relay_region_rehome_attempts_preferred_region_valid' }
|
||||
])
|
||||
// The existing control row keeps its tuning and gains the cooldown.
|
||||
const control = await client.query(
|
||||
`SELECT generation, preference_max_age_ms, host_cooldown_ms
|
||||
FROM relay_region_rehome_control WHERE control_id = 'global'`
|
||||
)
|
||||
expect(control.rows).toEqual([
|
||||
{
|
||||
generation: '3',
|
||||
preference_max_age_ms: '86400000',
|
||||
host_cooldown_ms: String(REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS)
|
||||
}
|
||||
])
|
||||
})
|
||||
} finally {
|
||||
await database.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('upgrades once across concurrent startups', async () => {
|
||||
const results = await Promise.allSettled(
|
||||
Array.from(
|
||||
{ length: 5 },
|
||||
async (): Promise<RelayDatabase> =>
|
||||
await openRelayDatabase({ databaseUrl: scopedUrl, dataDir: '' })
|
||||
)
|
||||
)
|
||||
const databases = results.flatMap((result) =>
|
||||
result.status === 'fulfilled' ? [result.value] : []
|
||||
)
|
||||
await Promise.all(databases.map(async (database) => await database.close()))
|
||||
|
||||
expect(
|
||||
results.flatMap((result) =>
|
||||
result.status === 'rejected'
|
||||
? [
|
||||
{
|
||||
code: (result.reason as { code?: unknown }).code,
|
||||
message: String(result.reason)
|
||||
}
|
||||
]
|
||||
: []
|
||||
)
|
||||
).toEqual([])
|
||||
await withClient(async (client) => {
|
||||
await client.query(`SET search_path = ${schema}`)
|
||||
await client.query(INSERT_ATTEMPT, attemptValues('attempt-4', 'us-central1'))
|
||||
const constraints = await client.query(
|
||||
`SELECT conname FROM pg_constraint
|
||||
WHERE conrelid = 'relay_region_rehome_attempts'::regclass
|
||||
AND conname LIKE '%preferred_region%'`
|
||||
)
|
||||
expect(constraints.rows).toEqual([
|
||||
{ conname: 'relay_region_rehome_attempts_preferred_region_valid' }
|
||||
])
|
||||
})
|
||||
}, 60_000)
|
||||
})
|
||||
@@ -81,6 +81,153 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
expect(await context.store.claimRegionalRehome()).not.toBeNull()
|
||||
})
|
||||
|
||||
it('moves a us-central1 host onto a cell in its preferred asia-east2 region', async () => {
|
||||
const context = await fixture()
|
||||
|
||||
const attempt = await context.store.claimRegionalRehome()
|
||||
expect(attempt).toMatchObject({
|
||||
preferredRegion: 'asia-east2',
|
||||
sourceCellId: context.source.id,
|
||||
targetCellId: context.target.id
|
||||
})
|
||||
expect(await primary.query(
|
||||
`SELECT preferred_region, source_cell_id, target_cell_id
|
||||
FROM relay_region_rehome_attempts WHERE user_id = ?`,
|
||||
[context.identity.userId]
|
||||
)).toEqual([{
|
||||
preferred_region: 'asia-east2',
|
||||
source_cell_id: context.source.id,
|
||||
target_cell_id: context.target.id
|
||||
}])
|
||||
})
|
||||
|
||||
it('moves an asia-east2 host back onto a cell in its preferred us-central1 region', async () => {
|
||||
const context = await fixture({
|
||||
sourceRegion: 'asia-east2',
|
||||
targetRegion: 'us-central1'
|
||||
})
|
||||
|
||||
const attempt = await context.store.claimRegionalRehome()
|
||||
expect(attempt).toMatchObject({
|
||||
preferredRegion: 'us-central1',
|
||||
sourceCellId: context.source.id,
|
||||
targetCellId: context.target.id
|
||||
})
|
||||
// The durable attempt row must accept the reverse direction too.
|
||||
expect(await primary.query(
|
||||
`SELECT preferred_region, source_cell_id, target_cell_id
|
||||
FROM relay_region_rehome_attempts WHERE user_id = ?`,
|
||||
[context.identity.userId]
|
||||
)).toEqual([{
|
||||
preferred_region: 'us-central1',
|
||||
source_cell_id: context.source.id,
|
||||
target_cell_id: context.target.id
|
||||
}])
|
||||
expect(await primary.query(
|
||||
`SELECT cell_id FROM relay_assignments WHERE user_id = ?`,
|
||||
[context.identity.userId]
|
||||
)).toEqual([{ cell_id: context.target.id }])
|
||||
})
|
||||
|
||||
it('leaves a host whose preference already matches its own region', async () => {
|
||||
const context = await fixture({ preferredRegion: 'us-central1' })
|
||||
|
||||
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
|
||||
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
|
||||
generation: 1,
|
||||
enabled: true
|
||||
})
|
||||
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
|
||||
attempts: 0,
|
||||
migrations: 0
|
||||
})
|
||||
})
|
||||
|
||||
it('leaves a host whose preference is older than the configured max age', async () => {
|
||||
const context = await fixture()
|
||||
await primary.query(
|
||||
`UPDATE relay_assignment_region_preferences SET observed_at = ?
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[
|
||||
context.now() - 24 * 60 * 60_000 - 1,
|
||||
context.identity.userId,
|
||||
context.identity.relayHostId
|
||||
]
|
||||
)
|
||||
|
||||
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
|
||||
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
|
||||
generation: 1,
|
||||
enabled: true
|
||||
})
|
||||
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
|
||||
attempts: 0,
|
||||
migrations: 0
|
||||
})
|
||||
})
|
||||
|
||||
it('leaves a host inside its per-host rehome cooldown, in either direction', async () => {
|
||||
const context = await fixture({ hostCooldownMs: 3 * 24 * 60 * 60_000 })
|
||||
// A move this host already made, whichever way it went.
|
||||
await primary.query(
|
||||
`INSERT INTO relay_region_rehome_attempts
|
||||
(attempt_id, user_id, relay_host_id, preferred_region, source_cell_id,
|
||||
source_cell_incarnation, target_cell_id, target_cell_incarnation,
|
||||
previous_epoch, assignment_epoch, drain_grace_ms, send_attempts,
|
||||
completed_at, created_at, updated_at)
|
||||
VALUES (?, ?, ?, 'us-central1', ?, ?, ?, ?, 0, 1, 0, 0, ?, ?, ?)`,
|
||||
[
|
||||
`pg-rehome-cooldown-${context.identity.relayHostId}`,
|
||||
context.identity.userId,
|
||||
context.identity.relayHostId,
|
||||
context.target.id,
|
||||
'22222222-2222-4222-8222-222222222222',
|
||||
context.source.id,
|
||||
'11111111-1111-4111-8111-111111111111',
|
||||
context.now(),
|
||||
context.now() - 3 * 24 * 60 * 60_000 + 1,
|
||||
context.now()
|
||||
]
|
||||
)
|
||||
|
||||
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
|
||||
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
|
||||
generation: 1,
|
||||
enabled: true,
|
||||
hostCooldownMs: 3 * 24 * 60 * 60_000
|
||||
})
|
||||
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
|
||||
attempts: 1,
|
||||
migrations: 0
|
||||
})
|
||||
|
||||
// One millisecond past the window the same host is a candidate again.
|
||||
await primary.query(
|
||||
`UPDATE relay_region_rehome_attempts SET created_at = ? WHERE user_id = ?`,
|
||||
[context.now() - 3 * 24 * 60 * 60_000, context.identity.userId]
|
||||
)
|
||||
await expect(context.store.claimRegionalRehome()).resolves.toMatchObject({
|
||||
sourceCellId: context.source.id,
|
||||
targetCellId: context.target.id
|
||||
})
|
||||
})
|
||||
|
||||
it('leaves a host whose preferred region holds no drainable cell', async () => {
|
||||
// A cell that cannot be drained cannot be a target: the host would land
|
||||
// where no later rehome could move it out again.
|
||||
const context = await fixture({ targetProtocol: 0 })
|
||||
|
||||
await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
|
||||
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
|
||||
generation: 1,
|
||||
enabled: true
|
||||
})
|
||||
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
|
||||
attempts: 0,
|
||||
migrations: 0
|
||||
})
|
||||
})
|
||||
|
||||
it('skips an unclean cell without latching the control off', async () => {
|
||||
const context = await fixture()
|
||||
await primary.query(
|
||||
@@ -281,7 +428,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
context.store,
|
||||
context.target,
|
||||
'22222222-2222-4222-8222-222222222222',
|
||||
0,
|
||||
1,
|
||||
900_000,
|
||||
2
|
||||
)
|
||||
@@ -322,7 +469,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
context.store,
|
||||
context.target,
|
||||
'44444444-4444-4444-8444-444444444444',
|
||||
0,
|
||||
1,
|
||||
context.now()
|
||||
)
|
||||
|
||||
@@ -341,7 +488,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
context.store,
|
||||
context.target,
|
||||
'22222222-2222-4222-8222-222222222222',
|
||||
0,
|
||||
1,
|
||||
900_000,
|
||||
2
|
||||
)
|
||||
@@ -414,6 +561,26 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
})
|
||||
})
|
||||
|
||||
async function attemptAndMigrationCounts(identity: {
|
||||
userId: string
|
||||
relayHostId: string
|
||||
}): Promise<{ attempts: number; migrations: number }> {
|
||||
const attempts = await primary.query(
|
||||
`SELECT COUNT(*) AS count FROM relay_region_rehome_attempts
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[identity.userId, identity.relayHostId]
|
||||
)
|
||||
const migrations = await primary.query(
|
||||
`SELECT COUNT(*) AS count FROM relay_assignment_migrations
|
||||
WHERE user_id = ? AND relay_host_id = ?`,
|
||||
[identity.userId, identity.relayHostId]
|
||||
)
|
||||
return {
|
||||
attempts: Number(attempts[0]!.count),
|
||||
migrations: Number(migrations[0]!.count)
|
||||
}
|
||||
}
|
||||
|
||||
async function controlAccounting(identity: {
|
||||
userId: string
|
||||
relayHostId: string
|
||||
@@ -436,12 +603,15 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
}
|
||||
}
|
||||
|
||||
async function fixture() {
|
||||
async function fixture(options: FixtureOptions = {}) {
|
||||
sequence++
|
||||
let now = 1_000_000
|
||||
const suffix = String(sequence)
|
||||
const source = cell(suffix, 'source', 'us-central1')
|
||||
const target = cell(suffix, 'target', 'asia-east2')
|
||||
const sourceRegion = options.sourceRegion ?? 'us-central1'
|
||||
const targetRegion = options.targetRegion ?? 'asia-east2'
|
||||
const preferredRegion = options.preferredRegion ?? targetRegion
|
||||
const source = cell(suffix, 'source', sourceRegion)
|
||||
const target = cell(suffix, 'target', targetRegion)
|
||||
const store = new RelayAssignmentStore(primary, () => now, storeOptions)
|
||||
const competingStore = new RelayAssignmentStore(secondary, () => now, storeOptions)
|
||||
await store.inspectRegionalRehomeControl()
|
||||
@@ -452,6 +622,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
notBefore: now,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 24 * 60 * 60_000,
|
||||
hostCooldownMs: options.hostCooldownMs ?? 7 * 24 * 60 * 60_000,
|
||||
drainGraceMs: 60_000
|
||||
})
|
||||
await store.reconcileCells([source, target])
|
||||
@@ -466,21 +637,22 @@ describePostgres('PostgreSQL regional rehoming', () => {
|
||||
store,
|
||||
target,
|
||||
'22222222-2222-4222-8222-222222222222',
|
||||
0,
|
||||
options.targetProtocol ?? 1,
|
||||
900_000
|
||||
)
|
||||
const identity = {
|
||||
userId: `pg-rehome-user-${suffix}`,
|
||||
relayHostId: `rehomehost${suffix.padStart(6, '0')}`
|
||||
}
|
||||
const assignment = await store.assign(identity, undefined, 'us-central1')
|
||||
const assignment = await store.assign(identity, undefined, sourceRegion)
|
||||
const sourceControl = await store.activateControl(identity, {
|
||||
cellId: source.id,
|
||||
assignmentEpoch: assignment.assignmentEpoch,
|
||||
generation: 1
|
||||
})
|
||||
await store.assign(identity, 'asia-east2')
|
||||
await store.assign(identity, preferredRegion)
|
||||
return {
|
||||
preferredRegion,
|
||||
store,
|
||||
competingStore,
|
||||
identity,
|
||||
@@ -500,7 +672,16 @@ const storeOptions = {
|
||||
heartbeatTtlMs: 45_000
|
||||
}
|
||||
|
||||
function cell(suffix: string, role: string, region: 'us-central1' | 'asia-east2') {
|
||||
type Region = 'us-central1' | 'asia-east2'
|
||||
type FixtureOptions = {
|
||||
sourceRegion?: Region
|
||||
targetRegion?: Region
|
||||
preferredRegion?: Region
|
||||
targetProtocol?: number
|
||||
hostCooldownMs?: number
|
||||
}
|
||||
|
||||
function cell(suffix: string, role: string, region: Region) {
|
||||
return {
|
||||
id: `pg-rehome-cell-${suffix}-${role}`,
|
||||
url: `https://pg-rehome-${suffix}-${role}.example.test`,
|
||||
|
||||
@@ -81,6 +81,7 @@ describe('regional rehome assignment state', () => {
|
||||
notBefore: context.now(),
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 24 * 60 * 60_000,
|
||||
hostCooldownMs: 7 * 24 * 60 * 60_000,
|
||||
drainGraceMs: 60_000
|
||||
})).rejects.toThrow('regional_rehome_generation_mismatch')
|
||||
await expect(context.store.applyRegionalRehomeControl({
|
||||
@@ -89,6 +90,7 @@ describe('regional rehome assignment state', () => {
|
||||
notBefore: context.now(),
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 24 * 60 * 60_000,
|
||||
hostCooldownMs: 7 * 24 * 60 * 60_000,
|
||||
drainGraceMs: 60_000
|
||||
})).resolves.toMatchObject({ generation: 3, enabled: true })
|
||||
await context.database.close()
|
||||
@@ -207,7 +209,7 @@ describe('regional rehome assignment state', () => {
|
||||
databasePoolWaitersMax: 0,
|
||||
databasePoolWaitMsMax: 0
|
||||
})
|
||||
await heartbeat(context.store, target, targetIncarnation, 0, 2, {
|
||||
await heartbeat(context.store, target, targetIncarnation, 1, 2, {
|
||||
observedAt: context.now(),
|
||||
sqlFailures: 1,
|
||||
reconnects: 3,
|
||||
@@ -226,6 +228,23 @@ describe('regional rehome assignment state', () => {
|
||||
await context.database.close()
|
||||
})
|
||||
|
||||
it('counts only drainable cells as the rehome fleet, in every region', async () => {
|
||||
// The fleet whose health gates a rehome is exactly the cells that can be a
|
||||
// source or a target, and both roles require the drain protocol.
|
||||
const context = await setup({ targetProtocol: 0 })
|
||||
|
||||
expect(await context.store.regionalRehomeFleetSafety()).toMatchObject({
|
||||
requiredCells: 1,
|
||||
missingCells: 0
|
||||
})
|
||||
await heartbeat(context.store, target, targetIncarnation, 1, 2)
|
||||
expect(await context.store.regionalRehomeFleetSafety()).toMatchObject({
|
||||
requiredCells: 2,
|
||||
missingCells: 0
|
||||
})
|
||||
await context.database.close()
|
||||
})
|
||||
|
||||
it('claims through the measured healthy baseline of pool micro-waits and churn', async () => {
|
||||
const context = await setup()
|
||||
const baseline = {
|
||||
@@ -238,7 +257,7 @@ describe('regional rehome assignment state', () => {
|
||||
databasePoolWaitMsMax: 1
|
||||
}
|
||||
await heartbeat(context.store, source, sourceIncarnation, 1, 2, baseline)
|
||||
await heartbeat(context.store, target, targetIncarnation, 0, 2, baseline)
|
||||
await heartbeat(context.store, target, targetIncarnation, 1, 2, baseline)
|
||||
await activatePreferredSource(context, {
|
||||
userId: 'user-1',
|
||||
relayHostId: 'abcdefghijklmnop'
|
||||
@@ -324,6 +343,204 @@ describe('regional rehome assignment state', () => {
|
||||
await context.database.close()
|
||||
})
|
||||
|
||||
it('moves a live host on an asia-east2 cell back to its preferred us-central1 cell', async () => {
|
||||
const context = await setup()
|
||||
const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
|
||||
await activateReversePreferredSource(context, identity)
|
||||
|
||||
const attempt = await context.store.claimRegionalRehome()
|
||||
expect(attempt).toMatchObject({
|
||||
userId: identity.userId,
|
||||
relayHostId: identity.relayHostId,
|
||||
preferredRegion: 'us-central1',
|
||||
sourceCellId: target.id,
|
||||
sourceCellIncarnation: targetIncarnation,
|
||||
targetCellId: source.id,
|
||||
targetCellIncarnation: sourceIncarnation,
|
||||
previousEpoch: 1,
|
||||
assignmentEpoch: 2,
|
||||
sendAttempts: 1
|
||||
})
|
||||
expect(
|
||||
await context.database.query(
|
||||
`SELECT preferred_region, source_cell_id, target_cell_id
|
||||
FROM relay_region_rehome_attempts`
|
||||
)
|
||||
).toEqual([{
|
||||
preferred_region: 'us-central1',
|
||||
source_cell_id: target.id,
|
||||
target_cell_id: source.id
|
||||
}])
|
||||
expect(await context.store.resolve(identity)).toMatchObject({ cellId: source.id })
|
||||
await context.database.close()
|
||||
})
|
||||
|
||||
it('drops a candidate at scan time when no cell in the preferred region is usable', async () => {
|
||||
const context = await setup()
|
||||
await activatePreferredSource(context, {
|
||||
userId: 'user-1',
|
||||
relayHostId: 'abcdefghijklmnop'
|
||||
})
|
||||
// A disabled cell is not a target, and the scan must say so: leaving it to
|
||||
// the claim would burn a slot of the candidate batch on a certain skip.
|
||||
await context.database.query(`UPDATE relay_cells SET enabled = 0 WHERE cell_id = ?`, [
|
||||
target.id
|
||||
])
|
||||
|
||||
const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
|
||||
try {
|
||||
expect(await context.store.claimRegionalRehome()).toBeNull()
|
||||
} finally {
|
||||
warnings.restore()
|
||||
}
|
||||
expect(warnings.entries).toEqual([])
|
||||
expect(
|
||||
await context.database.query(
|
||||
`SELECT next_dispatch_at FROM relay_region_rehome_worker_state`
|
||||
)
|
||||
).toEqual([{ next_dispatch_at: 0 }])
|
||||
expect(await context.database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
|
||||
await context.database.close()
|
||||
})
|
||||
|
||||
it('names the skip when the last target is lost between scan and claim', async () => {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
const context = await setup({
|
||||
database,
|
||||
wrap: (delegate) =>
|
||||
hookAfterCandidateScan(delegate, async (transaction) => {
|
||||
await transaction.query(`UPDATE relay_cells SET enabled = 0 WHERE cell_id = ?`, [
|
||||
target.id
|
||||
])
|
||||
})
|
||||
})
|
||||
await activatePreferredSource(context, {
|
||||
userId: 'user-1',
|
||||
relayHostId: 'abcdefghijklmnop'
|
||||
})
|
||||
|
||||
const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
|
||||
try {
|
||||
expect(await context.store.claimRegionalRehome()).toBeNull()
|
||||
} finally {
|
||||
warnings.restore()
|
||||
}
|
||||
expect(warnings.entries).toMatchObject([
|
||||
{ skips: [{ reason: 'no_eligible_target', candidates: 1 }] }
|
||||
])
|
||||
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
|
||||
generation: 1,
|
||||
enabled: true
|
||||
})
|
||||
expect(await database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
|
||||
await database.close()
|
||||
})
|
||||
|
||||
it('leaves a host alone until its cooldown expires, then moves it back', async () => {
|
||||
const context = await setup({ hostCooldownMs: 3 * 24 * 60 * 60_000 })
|
||||
const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
|
||||
const targetControl = await completeRehomeToTarget(context, identity)
|
||||
// Past the dispatch interval the earlier claim charged, so the next tick
|
||||
// really does scan and the cooldown is the only thing holding this host.
|
||||
context.advance(10_000)
|
||||
// The desktop's region probe now says us-central1 again.
|
||||
await context.store.assign(identity, 'us-central1')
|
||||
|
||||
const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
|
||||
try {
|
||||
expect(await context.store.claimRegionalRehome()).toBeNull()
|
||||
} finally {
|
||||
warnings.restore()
|
||||
}
|
||||
expect(warnings.entries).toEqual([])
|
||||
expect(await context.store.resolve(identity)).toMatchObject({ cellId: target.id })
|
||||
|
||||
context.advance(3 * 24 * 60 * 60_000)
|
||||
await freshHeartbeats(context)
|
||||
await context.store.renewControlActivity(identity, {
|
||||
activityId: targetControl,
|
||||
cellId: target.id,
|
||||
expiresAt: context.now() + 90_000
|
||||
})
|
||||
await context.store.assign(identity, 'us-central1')
|
||||
|
||||
const attempt = await context.store.claimRegionalRehome()
|
||||
expect(attempt).toMatchObject({
|
||||
preferredRegion: 'us-central1',
|
||||
sourceCellId: target.id,
|
||||
targetCellId: source.id
|
||||
})
|
||||
await context.database.close()
|
||||
})
|
||||
|
||||
it('rejects a host whose attempt lands between the scan and the claim', async () => {
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
|
||||
const context = await setup({
|
||||
database,
|
||||
wrap: (delegate) =>
|
||||
hookAfterCandidateScan(delegate, async (transaction) => {
|
||||
await transaction.query(
|
||||
`INSERT INTO relay_region_rehome_attempts
|
||||
(attempt_id, user_id, relay_host_id, preferred_region, source_cell_id,
|
||||
source_cell_incarnation, target_cell_id, target_cell_incarnation,
|
||||
previous_epoch, assignment_epoch, drain_grace_ms, send_attempts,
|
||||
created_at, updated_at)
|
||||
VALUES ('raced', ?, ?, 'asia-east2', ?, ?, ?, ?, 0, 1, 0, 0, ?, ?)`,
|
||||
[
|
||||
identity.userId,
|
||||
identity.relayHostId,
|
||||
source.id,
|
||||
sourceIncarnation,
|
||||
target.id,
|
||||
targetIncarnation,
|
||||
context.now(),
|
||||
context.now()
|
||||
]
|
||||
)
|
||||
})
|
||||
})
|
||||
await activatePreferredSource(context, identity)
|
||||
|
||||
const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
|
||||
try {
|
||||
expect(await context.store.claimRegionalRehome()).toBeNull()
|
||||
} finally {
|
||||
warnings.restore()
|
||||
}
|
||||
expect(warnings.entries).toMatchObject([
|
||||
{ skips: [{ reason: 'host_cooldown', candidates: 1 }] }
|
||||
])
|
||||
expect(await database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
|
||||
await database.close()
|
||||
})
|
||||
|
||||
it('does not scan a candidate whose preferred region has no drainable cell', async () => {
|
||||
// A cell without the drain protocol cannot be a target: the host would land
|
||||
// where no later rehome could move it out again. The candidate query drops
|
||||
// it, so the tick stays idle instead of paying for an inventory scan.
|
||||
const context = await setup({ targetProtocol: 0 })
|
||||
await activatePreferredSource(context, {
|
||||
userId: 'user-1',
|
||||
relayHostId: 'abcdefghijklmnop'
|
||||
})
|
||||
|
||||
const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
|
||||
try {
|
||||
expect(await context.store.claimRegionalRehome()).toBeNull()
|
||||
} finally {
|
||||
warnings.restore()
|
||||
}
|
||||
expect(warnings.entries).toEqual([])
|
||||
expect(
|
||||
await context.database.query(
|
||||
`SELECT next_dispatch_at FROM relay_region_rehome_worker_state`
|
||||
)
|
||||
).toEqual([{ next_dispatch_at: 0 }])
|
||||
expect(await context.database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
|
||||
await context.database.close()
|
||||
})
|
||||
|
||||
it('skips an unclean cell without latching the control off', async () => {
|
||||
const context = await setup()
|
||||
await activatePreferredSource(context, {
|
||||
@@ -457,7 +674,7 @@ describe('regional rehome assignment state', () => {
|
||||
databasePoolWaitersMax: 0,
|
||||
databasePoolWaitMsMax: 0
|
||||
})
|
||||
await heartbeat(context.store, target, targetIncarnation, 0, 2, {
|
||||
await heartbeat(context.store, target, targetIncarnation, 1, 2, {
|
||||
observedAt: context.now(),
|
||||
sqlFailures: 0,
|
||||
reconnects: 0,
|
||||
@@ -477,6 +694,7 @@ describe('regional rehome assignment state', () => {
|
||||
notBefore: context.now(),
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 24 * 60 * 60_000,
|
||||
hostCooldownMs: 7 * 24 * 60 * 60_000,
|
||||
drainGraceMs: 60_000
|
||||
})
|
||||
const retry = await context.store.claimRegionalRehome()
|
||||
@@ -547,7 +765,7 @@ describe('regional rehome assignment state', () => {
|
||||
await activatePreferredSource(context, identity)
|
||||
await context.store.claimRegionalRehome()
|
||||
context.advance(6 * 60_000)
|
||||
await heartbeat(context.store, target, targetIncarnation, 0, 2)
|
||||
await heartbeat(context.store, target, targetIncarnation, 1, 2)
|
||||
|
||||
expect(await context.store.refreshRegionalRehomeLeases()).toBe(0)
|
||||
expect(await context.store.abortExpiredEvacuations()).toBe(0)
|
||||
@@ -1549,10 +1767,16 @@ function collectDisableWarnings() {
|
||||
}
|
||||
|
||||
async function setup(
|
||||
options: { sourceProtocol?: number; wrap?: (database: RelayDatabase) => RelayDatabase } = {}
|
||||
options: {
|
||||
sourceProtocol?: number
|
||||
targetProtocol?: number
|
||||
hostCooldownMs?: number
|
||||
database?: RelayDatabase
|
||||
wrap?: (database: RelayDatabase) => RelayDatabase
|
||||
} = {}
|
||||
) {
|
||||
let clock = 1_000_000
|
||||
const database = await openInMemoryRelayDatabase()
|
||||
const database = options.database ?? (await openInMemoryRelayDatabase())
|
||||
const store = new RelayAssignmentStore(options.wrap?.(database) ?? database, () => clock, {
|
||||
requireLiveCells: true,
|
||||
heartbeatTtlMs: 45_000
|
||||
@@ -1565,11 +1789,12 @@ async function setup(
|
||||
notBefore: clock,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 24 * 60 * 60_000,
|
||||
hostCooldownMs: options.hostCooldownMs ?? 7 * 24 * 60 * 60_000,
|
||||
drainGraceMs: 60 * 60_000
|
||||
})
|
||||
await store.reconcileCells([source, target])
|
||||
await heartbeat(store, source, sourceIncarnation, options.sourceProtocol ?? 1)
|
||||
await heartbeat(store, target, targetIncarnation, 0)
|
||||
await heartbeat(store, target, targetIncarnation, options.targetProtocol ?? 1)
|
||||
return {
|
||||
database,
|
||||
store,
|
||||
@@ -1671,7 +1896,7 @@ async function freshHeartbeats(context: Context): Promise<void> {
|
||||
}
|
||||
// The clock doubles as a strictly-increasing connection inclusion watermark.
|
||||
await heartbeat(context.store, source, sourceIncarnation, 1, context.now(), safety)
|
||||
await heartbeat(context.store, target, targetIncarnation, 0, context.now(), safety)
|
||||
await heartbeat(context.store, target, targetIncarnation, 1, context.now(), safety)
|
||||
}
|
||||
|
||||
async function activatePreferredSource(
|
||||
@@ -1688,6 +1913,68 @@ async function activatePreferredSource(
|
||||
return control
|
||||
}
|
||||
|
||||
// Runs a hook inside the claim transaction, right after the candidate scan, so
|
||||
// a scan-versus-claim race is deterministic instead of timing-dependent.
|
||||
function hookAfterCandidateScan(
|
||||
database: RelayDatabase,
|
||||
hook: (transaction: RelayDatabase) => Promise<void>
|
||||
): RelayDatabase {
|
||||
let fired = false
|
||||
const decorate = (delegate: RelayDatabase): RelayDatabase => ({
|
||||
query: async (sql, params) => {
|
||||
const rows = await delegate.query(sql, params)
|
||||
if (!fired && sql.includes('FROM relay_assignment_region_preferences preference')) {
|
||||
fired = true
|
||||
await hook(delegate)
|
||||
}
|
||||
return rows
|
||||
},
|
||||
queryLocked: async (sql, params, lockOptions) =>
|
||||
await delegate.queryLocked(sql, params, lockOptions),
|
||||
transaction: async (operation, transactionOptions) =>
|
||||
await delegate.transaction(
|
||||
async (transaction) => await operation(decorate(transaction)),
|
||||
transactionOptions
|
||||
),
|
||||
close: async () => undefined
|
||||
})
|
||||
return decorate(database)
|
||||
}
|
||||
|
||||
async function completeRehomeToTarget(
|
||||
context: Context,
|
||||
identity: { userId: string; relayHostId: string }
|
||||
): Promise<string> {
|
||||
const sourceControl = await activatePreferredSource(context, identity)
|
||||
const attempt = await context.store.claimRegionalRehome()
|
||||
const targetControl = await context.store.activateControl(identity, {
|
||||
cellId: target.id,
|
||||
assignmentEpoch: attempt!.assignmentEpoch,
|
||||
generation: 1
|
||||
})
|
||||
await context.store.markMigrationTargetRegistered(identity, {
|
||||
cellId: target.id,
|
||||
assignmentEpoch: attempt!.assignmentEpoch
|
||||
})
|
||||
await context.store.releaseActivity(identity, sourceControl)
|
||||
await context.store.completeReadyRegionalRehomes()
|
||||
return targetControl
|
||||
}
|
||||
|
||||
async function activateReversePreferredSource(
|
||||
context: Context,
|
||||
identity: { userId: string; relayHostId: string }
|
||||
): Promise<string> {
|
||||
const assignment = await context.store.assign(identity, undefined, 'asia-east2')
|
||||
const control = await context.store.activateControl(identity, {
|
||||
cellId: target.id,
|
||||
assignmentEpoch: assignment.assignmentEpoch,
|
||||
generation: 1
|
||||
})
|
||||
await context.store.assign(identity, 'us-central1')
|
||||
return control
|
||||
}
|
||||
|
||||
async function activateSource(
|
||||
context: Context,
|
||||
identity: { userId: string; relayHostId: string }
|
||||
|
||||
@@ -36,6 +36,7 @@ async function setup() {
|
||||
notBefore: clock,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 24 * 60 * 60_000,
|
||||
hostCooldownMs: 7 * 24 * 60 * 60_000,
|
||||
drainGraceMs: 60 * 60_000
|
||||
})
|
||||
await store.reconcileCells([source, noHeadroom, unclean, highLoad, lowLoad])
|
||||
@@ -100,22 +101,22 @@ describe('regional rehome target selection', () => {
|
||||
sqlFailures: 0
|
||||
})
|
||||
// Lowest load but the connection hard cap is exhausted.
|
||||
await context.beat(noHeadroom, 2, 0, {
|
||||
await context.beat(noHeadroom, 2, 1, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 999,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.beat(unclean, 3, 0, {
|
||||
await context.beat(unclean, 3, 1, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: UNCLEAN
|
||||
})
|
||||
await context.beat(highLoad, 4, 0, {
|
||||
await context.beat(highLoad, 4, 1, {
|
||||
observedRequests: 50,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.beat(lowLoad, 5, 0, {
|
||||
await context.beat(lowLoad, 5, 1, {
|
||||
observedRequests: 10,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: 0
|
||||
@@ -134,22 +135,22 @@ describe('regional rehome target selection', () => {
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.beat(noHeadroom, 2, 0, {
|
||||
await context.beat(noHeadroom, 2, 1, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 999,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.beat(unclean, 3, 0, {
|
||||
await context.beat(unclean, 3, 1, {
|
||||
observedRequests: 0,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: UNCLEAN
|
||||
})
|
||||
await context.beat(highLoad, 4, 0, {
|
||||
await context.beat(highLoad, 4, 1, {
|
||||
observedRequests: 50,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: 0
|
||||
})
|
||||
await context.beat(lowLoad, 5, 0, {
|
||||
await context.beat(lowLoad, 5, 1, {
|
||||
observedRequests: 10,
|
||||
enforcedConnections: 0,
|
||||
sqlFailures: UNCLEAN
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import { RELAY_REGION_METRIC_SEGMENTS, RELAY_REGIONS } from '@orca-cloud/relay-contract'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { RelayDatabase } from './database.js'
|
||||
import { observeRelayDatabase } from './observed-relay-database.js'
|
||||
import {
|
||||
CONTROL_RTT_RESERVOIR_LIMIT,
|
||||
observedRelayRequests,
|
||||
RelayObservability,
|
||||
type RelayProcessCounts
|
||||
@@ -22,6 +24,37 @@ const counts: RelayProcessCounts = {
|
||||
databasePoolWaitMsMax: 1_250
|
||||
}
|
||||
|
||||
// Two schema keys legitimately spell a policed word: the abandoned-accept bucket
|
||||
// is keyed by stage name and one stage is `credential`. Rename those exact keys in
|
||||
// a clone instead of rewriting the JSON, so a stray raw field or value anywhere
|
||||
// else still trips the guard below.
|
||||
const SCHEMA_KEY_ALIASES: Record<string, string> = {
|
||||
clientAcceptCredentialMsP95: 'clientAcceptStageTwoMsP95'
|
||||
}
|
||||
|
||||
function scrubSchemaKeys(entries: Array<Record<string, unknown>>): string {
|
||||
return JSON.stringify(
|
||||
entries.map((entry) =>
|
||||
Object.fromEntries(
|
||||
Object.entries(entry).map(([key, value]) => [
|
||||
SCHEMA_KEY_ALIASES[key] ?? key,
|
||||
key === 'clientAcceptsAbandonedByStageDelta' ? renameStageKeys(value) : value
|
||||
])
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
function renameStageKeys(bucket: unknown): unknown {
|
||||
if (bucket === null || typeof bucket !== 'object') return bucket
|
||||
return Object.fromEntries(
|
||||
Object.entries(bucket).map(([stage, count]) => [
|
||||
stage === 'credential' ? 'stageTwo' : stage,
|
||||
count
|
||||
])
|
||||
)
|
||||
}
|
||||
|
||||
describe('relay observability', () => {
|
||||
it('emits safe readiness dependency outcomes', () => {
|
||||
const entries: Array<Record<string, unknown>> = []
|
||||
@@ -106,14 +139,31 @@ describe('relay observability', () => {
|
||||
requestedRegionsDelta: { 'asia-east2': 1, unhinted: 1 },
|
||||
selectedRegionsDelta: { 'us-central1': 1 },
|
||||
regionFallbacksDelta: { 'asia-east2': 1 },
|
||||
unavailableRegionsDelta: { 'asia-east2': 1 }
|
||||
unavailableRegionsDelta: { 'asia-east2': 1 },
|
||||
// Flat per-region siblings the log-based metrics extract; `unhinted` stays map-only.
|
||||
requestedRegionUsCentral1Delta: 0,
|
||||
requestedRegionAsiaEast2Delta: 1,
|
||||
selectedRegionUsCentral1Delta: 1,
|
||||
selectedRegionAsiaEast2Delta: 0
|
||||
})
|
||||
expect(entries[1]).toMatchObject({
|
||||
requestedRegionsDelta: {},
|
||||
selectedRegionsDelta: {},
|
||||
regionFallbacksDelta: {},
|
||||
unavailableRegionsDelta: {}
|
||||
unavailableRegionsDelta: {},
|
||||
// Zeros keep publishing so an idle window cannot drop a series out of the skew join.
|
||||
requestedRegionUsCentral1Delta: 0,
|
||||
requestedRegionAsiaEast2Delta: 0,
|
||||
selectedRegionUsCentral1Delta: 0,
|
||||
selectedRegionAsiaEast2Delta: 0
|
||||
})
|
||||
// A region added to the contract has to reach the flat keys, or the skew alert's
|
||||
// denominator silently misses it.
|
||||
for (const segment of Object.values(RELAY_REGION_METRIC_SEGMENTS)) {
|
||||
expect(entries[0]).toHaveProperty(`requestedRegion${segment}Delta`)
|
||||
expect(entries[0]).toHaveProperty(`selectedRegion${segment}Delta`)
|
||||
}
|
||||
expect(Object.keys(RELAY_REGION_METRIC_SEGMENTS).sort()).toEqual([...RELAY_REGIONS].sort())
|
||||
})
|
||||
|
||||
it('emits bounded aggregate runtime signals without identities or credentials', () => {
|
||||
@@ -181,7 +231,7 @@ describe('relay observability', () => {
|
||||
controlActivityRecoveryFailuresDelta: 0,
|
||||
httpLatencyMsMax: 0
|
||||
})
|
||||
expect(JSON.stringify(entries)).not.toMatch(/token|credential|userId|relayHostId/)
|
||||
expect(scrubSchemaKeys(entries)).not.toMatch(/token|credential|userId|relayHostId/i)
|
||||
})
|
||||
|
||||
it('aggregates control and splice closes as bounded per-reason deltas', () => {
|
||||
@@ -215,6 +265,117 @@ describe('relay observability', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('summarises completed client accepts and control round trips per window', () => {
|
||||
const entries: Array<Record<string, unknown>> = []
|
||||
const observability = new RelayObservability(
|
||||
{ role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
|
||||
(entry) => entries.push(entry)
|
||||
)
|
||||
observability.recordClientAcceptCompleted({
|
||||
totalMs: 812.4567,
|
||||
stageMs: { assignment: 120, credential: 90, activity: 40, attach: 500, basis: 62 }
|
||||
})
|
||||
observability.recordClientAcceptCompleted({
|
||||
totalMs: 6_400,
|
||||
stageMs: { assignment: 4_100, credential: 95, activity: 60, attach: 2_000, basis: 145 }
|
||||
})
|
||||
observability.recordControlRtt(28)
|
||||
observability.recordControlRtt(240)
|
||||
observability.recordControlRtt(31)
|
||||
observability.flush(counts)
|
||||
observability.flush(counts)
|
||||
|
||||
expect(entries[0]).toMatchObject({
|
||||
clientAcceptCompletedDelta: 2,
|
||||
clientAcceptTotalMsP50: 812.457,
|
||||
clientAcceptTotalMsP95: 6_400,
|
||||
clientAcceptTotalMsMax: 6_400,
|
||||
clientAcceptAssignmentMsP95: 4_100,
|
||||
clientAcceptCredentialMsP95: 95,
|
||||
clientAcceptActivityMsP95: 60,
|
||||
clientAcceptAttachMsP95: 2_000,
|
||||
clientAcceptBasisMsP95: 145,
|
||||
controlRttSamplesDelta: 3,
|
||||
controlRttMsP50: 31,
|
||||
controlRttMsP95: 240,
|
||||
controlRttMsMax: 240
|
||||
})
|
||||
// Only-add: the pre-existing fields still read the same after the extension.
|
||||
expect(entries[0]).toMatchObject({
|
||||
event: 'orca_relay_runtime_metrics',
|
||||
metricVersion: 2,
|
||||
clientAcceptsAbandonedByStageDelta: {},
|
||||
clientAcceptAbandonedMsMax: 0
|
||||
})
|
||||
// An empty window publishes counts only: a zero percentile point is
|
||||
// indistinguishable from a real zero once Cloud Logging aggregates it.
|
||||
expect(entries[1]).toMatchObject({ clientAcceptCompletedDelta: 0, controlRttSamplesDelta: 0 })
|
||||
for (const omitted of [
|
||||
'clientAcceptTotalMsP50',
|
||||
'clientAcceptTotalMsP95',
|
||||
'clientAcceptTotalMsMax',
|
||||
'clientAcceptAssignmentMsP95',
|
||||
'clientAcceptCredentialMsP95',
|
||||
'clientAcceptActivityMsP95',
|
||||
'clientAcceptAttachMsP95',
|
||||
'clientAcceptBasisMsP95',
|
||||
'controlRttMsP50',
|
||||
'controlRttMsP95',
|
||||
'controlRttMsMax'
|
||||
]) {
|
||||
expect(entries[1]).not.toHaveProperty(omitted)
|
||||
expect(entries[0]).toHaveProperty(omitted)
|
||||
}
|
||||
expect(scrubSchemaKeys(entries)).not.toMatch(/token|credential|userId|relayHostId/i)
|
||||
})
|
||||
|
||||
it('caps the control round-trip reservoir and reports what it dropped', () => {
|
||||
const entries: Array<Record<string, unknown>> = []
|
||||
const observability = new RelayObservability(
|
||||
{ role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
|
||||
(entry) => entries.push(entry)
|
||||
)
|
||||
const flooded = CONTROL_RTT_RESERVOIR_LIMIT * 20
|
||||
for (let sample = 0; sample < flooded; sample++) {
|
||||
observability.recordControlRtt(10 + (sample % 40))
|
||||
}
|
||||
observability.flush(counts)
|
||||
|
||||
// Dropped is observed minus retained, so this pins the retained window at the cap.
|
||||
expect(entries[0]).toMatchObject({
|
||||
controlRttSamplesDelta: flooded,
|
||||
controlRttSamplesDroppedDelta: flooded - CONTROL_RTT_RESERVOIR_LIMIT
|
||||
})
|
||||
// The kept samples are real observations, not a truncated or synthesised window.
|
||||
expect(entries[0]!.controlRttMsP50 as number).toBeGreaterThanOrEqual(10)
|
||||
expect(entries[0]!.controlRttMsMax as number).toBeLessThanOrEqual(49)
|
||||
|
||||
observability.flush(counts)
|
||||
expect(entries[1]).toMatchObject({
|
||||
controlRttSamplesDelta: 0,
|
||||
controlRttSamplesDroppedDelta: 0
|
||||
})
|
||||
expect(entries[1]).not.toHaveProperty('controlRttMsP50')
|
||||
})
|
||||
|
||||
it('samples the whole flooded window rather than its first samples', () => {
|
||||
const entries: Array<Record<string, unknown>> = []
|
||||
const observability = new RelayObservability(
|
||||
{ role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
|
||||
(entry) => entries.push(entry)
|
||||
)
|
||||
const half = CONTROL_RTT_RESERVOIR_LIMIT * 10
|
||||
for (let sample = 0; sample < half; sample++) observability.recordControlRtt(10)
|
||||
for (let sample = 0; sample < half; sample++) observability.recordControlRtt(900)
|
||||
observability.flush(counts)
|
||||
|
||||
// Keeping the first N instead would publish a window of nothing but 10s. Each
|
||||
// reservoir slot ends up drawn from the late half with ~1/2 probability, so
|
||||
// fewer than the 5% the p95 needs is out of reach of this suite.
|
||||
expect(entries[0]!.controlRttMsP95).toBe(900)
|
||||
expect(entries[0]!.controlRttMsMax).toBe(900)
|
||||
})
|
||||
|
||||
it('observes successful and failed database calls including transactions', async () => {
|
||||
const recordSql = vi.fn()
|
||||
const underlying: RelayDatabase = {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { monitorEventLoopDelay, performance } from 'node:perf_hooks'
|
||||
import type { RelayRegion } from '@orca-cloud/relay-contract'
|
||||
import { RELAY_REGION_METRIC_SEGMENTS, type RelayRegion } from '@orca-cloud/relay-contract'
|
||||
import type { ControlRenewalOutcome } from './assignment-store.js'
|
||||
import type { CellInventoryHoldCounts } from './cell-inventory-hold-samples.js'
|
||||
import type { PostgresPoolPressureCounts } from './postgres-pool-pressure.js'
|
||||
@@ -65,11 +65,31 @@ export interface RelayRuntimeObserver {
|
||||
recordControlClose?(code: number): void
|
||||
recordSpliceClose?(trigger: string): void
|
||||
recordClientAcceptAbandoned?(stage: RelayClientAcceptStage, elapsedMs: number): void
|
||||
recordClientAcceptCompleted?(sample: RelayClientAcceptSample): void
|
||||
recordControlRtt?(rttMs: number): void
|
||||
}
|
||||
|
||||
// Which serialized accept step the phone had already hung up behind.
|
||||
export type RelayClientAcceptStage = 'assignment' | 'credential' | 'activity'
|
||||
|
||||
// The attach window and the basis writes that follow it are only measurable once
|
||||
// the host data leg lands, so they join the serialized pre-attach steps on
|
||||
// completed accepts only.
|
||||
export type RelayClientAcceptTimedStage = RelayClientAcceptStage | 'attach' | 'basis'
|
||||
|
||||
export const RELAY_CLIENT_ACCEPT_TIMED_STAGES = [
|
||||
'assignment',
|
||||
'credential',
|
||||
'activity',
|
||||
'attach',
|
||||
'basis'
|
||||
] as const satisfies readonly RelayClientAcceptTimedStage[]
|
||||
|
||||
export type RelayClientAcceptSample = {
|
||||
totalMs: number
|
||||
stageMs: Record<RelayClientAcceptTimedStage, number>
|
||||
}
|
||||
|
||||
type RelayMetricDeltas = {
|
||||
forwardedBytes: number
|
||||
authSuccesses: number
|
||||
@@ -93,12 +113,20 @@ type RelayMetricDeltas = {
|
||||
spliceClosesByTrigger: Record<string, number>
|
||||
clientAcceptsAbandonedByStage: Record<string, number>
|
||||
clientAcceptAbandonedMsMax: number
|
||||
clientAcceptTotalsMs: number[]
|
||||
clientAcceptStageSamplesMs: Record<RelayClientAcceptTimedStage, number[]>
|
||||
controlRttSamplesMs: number[]
|
||||
controlRttObserved: number
|
||||
controlRenewalLatenciesMs: number[]
|
||||
controlRenewalsByOutcome: Record<string, number>
|
||||
controlActivityRecoveries: number
|
||||
controlActivityRecoveryFailures: number
|
||||
}
|
||||
|
||||
// A host chooses how often it answers a ping, so the process-wide window is a
|
||||
// reservoir: the heap cost of a flood is capped and the percentiles stay unbiased.
|
||||
export const CONTROL_RTT_RESERVOIR_LIMIT = 1024
|
||||
|
||||
type MetricWriter = (entry: Record<string, unknown>) => void
|
||||
|
||||
const emptyDeltas = (): RelayMetricDeltas => ({
|
||||
@@ -124,18 +152,42 @@ const emptyDeltas = (): RelayMetricDeltas => ({
|
||||
spliceClosesByTrigger: {},
|
||||
clientAcceptsAbandonedByStage: {},
|
||||
clientAcceptAbandonedMsMax: 0,
|
||||
clientAcceptTotalsMs: [],
|
||||
clientAcceptStageSamplesMs: {
|
||||
assignment: [],
|
||||
credential: [],
|
||||
activity: [],
|
||||
attach: [],
|
||||
basis: []
|
||||
},
|
||||
controlRttSamplesMs: [],
|
||||
controlRttObserved: 0,
|
||||
controlRenewalLatenciesMs: [],
|
||||
controlRenewalsByOutcome: {},
|
||||
controlActivityRecoveries: 0,
|
||||
controlActivityRecoveryFailures: 0
|
||||
})
|
||||
|
||||
function percentile(values: number[], percentileRank: number): number {
|
||||
export function percentile(values: number[], percentileRank: number): number {
|
||||
if (values.length === 0) return 0
|
||||
const sorted = [...values].sort((left, right) => left - right)
|
||||
return sorted[Math.ceil(percentileRank * sorted.length) - 1] ?? 0
|
||||
}
|
||||
|
||||
function roundMs(value: number): number {
|
||||
return Number(value.toFixed(3))
|
||||
}
|
||||
|
||||
// Spreading a window into Math.max blows the stack once a busy cell samples
|
||||
// enough of it, so the maximum is folded instead.
|
||||
function latencySummary(samples: number[]): { p50: number; p95: number; max: number } {
|
||||
return {
|
||||
p50: roundMs(percentile(samples, 0.5)),
|
||||
p95: roundMs(percentile(samples, 0.95)),
|
||||
max: roundMs(samples.reduce((highest, sample) => Math.max(highest, sample), 0))
|
||||
}
|
||||
}
|
||||
|
||||
export class RelayObservability implements RelayRuntimeObserver {
|
||||
private readonly eventLoop = monitorEventLoopDelay({ resolution: 20 })
|
||||
private deltas = emptyDeltas()
|
||||
@@ -244,6 +296,25 @@ export class RelayObservability implements RelayRuntimeObserver {
|
||||
)
|
||||
}
|
||||
|
||||
recordClientAcceptCompleted(sample: RelayClientAcceptSample): void {
|
||||
this.deltas.clientAcceptTotalsMs.push(sample.totalMs)
|
||||
for (const stage of RELAY_CLIENT_ACCEPT_TIMED_STAGES) {
|
||||
this.deltas.clientAcceptStageSamplesMs[stage].push(sample.stageMs[stage])
|
||||
}
|
||||
}
|
||||
|
||||
recordControlRtt(rttMs: number): void {
|
||||
const samples = this.deltas.controlRttSamplesMs
|
||||
const observedBefore = this.deltas.controlRttObserved++
|
||||
if (samples.length < CONTROL_RTT_RESERVOIR_LIMIT) {
|
||||
samples.push(rttMs)
|
||||
return
|
||||
}
|
||||
// Algorithm R: every round trip in the window keeps an equal chance of being kept.
|
||||
const slot = Math.floor(Math.random() * (observedBefore + 1))
|
||||
if (slot < CONTROL_RTT_RESERVOIR_LIMIT) samples[slot] = rttMs
|
||||
}
|
||||
|
||||
start(readCounts: () => RelayProcessCounts, intervalMs = 30_000): void {
|
||||
if (this.timer) return
|
||||
this.eventLoop.enable()
|
||||
@@ -277,6 +348,11 @@ export class RelayObservability implements RelayRuntimeObserver {
|
||||
controlActivityRecoveryFailures: deltas.controlActivityRecoveryFailures
|
||||
}
|
||||
this.deltas = emptyDeltas()
|
||||
const acceptTotals = latencySummary(deltas.clientAcceptTotalsMs)
|
||||
const acceptStageP95 = (stage: RelayClientAcceptTimedStage): number =>
|
||||
roundMs(percentile(deltas.clientAcceptStageSamplesMs[stage], 0.95))
|
||||
const controlRtt = latencySummary(deltas.controlRttSamplesMs)
|
||||
const controlRenewal = latencySummary(deltas.controlRenewalLatenciesMs)
|
||||
const memory = process.memoryUsage()
|
||||
const p99 = this.eventLoop.count === 0 ? 0 : this.eventLoop.percentile(99) / 1_000_000
|
||||
this.eventLoop.reset()
|
||||
@@ -301,15 +377,43 @@ export class RelayObservability implements RelayRuntimeObserver {
|
||||
placementRejectionsByReasonDelta: deltas.placementRejectionsByReason,
|
||||
requestedRegionsDelta: deltas.requestedRegions,
|
||||
selectedRegionsDelta: deltas.selectedRegions,
|
||||
...regionCounterFields('requestedRegion', deltas.requestedRegions),
|
||||
...regionCounterFields('selectedRegion', deltas.selectedRegions),
|
||||
regionFallbacksDelta: deltas.regionFallbacks,
|
||||
unavailableRegionsDelta: deltas.unavailableRegions,
|
||||
controlClosesByCodeDelta: deltas.controlClosesByCode,
|
||||
spliceClosesByTriggerDelta: deltas.spliceClosesByTrigger,
|
||||
clientAcceptsAbandonedByStageDelta: deltas.clientAcceptsAbandonedByStage,
|
||||
clientAcceptAbandonedMsMax: Number(deltas.clientAcceptAbandonedMsMax.toFixed(3)),
|
||||
clientAcceptAbandonedMsMax: roundMs(deltas.clientAcceptAbandonedMsMax),
|
||||
clientAcceptCompletedDelta: deltas.clientAcceptTotalsMs.length,
|
||||
// Accepts are sparse: publishing a zero percentile for every empty window
|
||||
// would pin the p50 at 0 forever and collapse the p95 at low accept rates.
|
||||
...(deltas.clientAcceptTotalsMs.length === 0
|
||||
? {}
|
||||
: {
|
||||
clientAcceptTotalMsP50: acceptTotals.p50,
|
||||
clientAcceptTotalMsP95: acceptTotals.p95,
|
||||
clientAcceptTotalMsMax: acceptTotals.max,
|
||||
clientAcceptAssignmentMsP95: acceptStageP95('assignment'),
|
||||
clientAcceptCredentialMsP95: acceptStageP95('credential'),
|
||||
clientAcceptActivityMsP95: acceptStageP95('activity'),
|
||||
clientAcceptAttachMsP95: acceptStageP95('attach'),
|
||||
clientAcceptBasisMsP95: acceptStageP95('basis')
|
||||
}),
|
||||
// Every round trip observed in the window, including the ones the reservoir
|
||||
// above declined to keep; the percentiles summarise only what it kept.
|
||||
controlRttSamplesDelta: deltas.controlRttObserved,
|
||||
controlRttSamplesDroppedDelta: deltas.controlRttObserved - deltas.controlRttSamplesMs.length,
|
||||
...(deltas.controlRttSamplesMs.length === 0
|
||||
? {}
|
||||
: {
|
||||
controlRttMsP50: controlRtt.p50,
|
||||
controlRttMsP95: controlRtt.p95,
|
||||
controlRttMsMax: controlRtt.max
|
||||
}),
|
||||
sqlQueriesDelta: deltas.sqlQueries,
|
||||
sqlFailuresDelta: deltas.sqlFailures,
|
||||
sqlLatencyMsMax: Number(deltas.sqlLatencyMsMax.toFixed(3)),
|
||||
sqlLatencyMsMax: roundMs(deltas.sqlLatencyMsMax),
|
||||
controlRenewalsByOutcomeDelta: deltas.controlRenewalsByOutcome,
|
||||
controlRenewalsDelta: deltas.controlRenewalLatenciesMs.length,
|
||||
controlRenewalSuccessesDelta: deltas.controlRenewalsByOutcome.renewed ?? 0,
|
||||
@@ -317,16 +421,10 @@ export class RelayObservability implements RelayRuntimeObserver {
|
||||
deltas.controlRenewalsByOutcome.control_activity_not_found ?? 0,
|
||||
controlActivityRecoveriesDelta: deltas.controlActivityRecoveries,
|
||||
controlActivityRecoveryFailuresDelta: deltas.controlActivityRecoveryFailures,
|
||||
controlRenewalLatencyMsP50: Number(
|
||||
percentile(deltas.controlRenewalLatenciesMs, 0.5).toFixed(3)
|
||||
),
|
||||
controlRenewalLatencyMsP95: Number(
|
||||
percentile(deltas.controlRenewalLatenciesMs, 0.95).toFixed(3)
|
||||
),
|
||||
controlRenewalLatencyMsMax: Number(
|
||||
Math.max(0, ...deltas.controlRenewalLatenciesMs).toFixed(3)
|
||||
),
|
||||
httpLatencyMsMax: Number(deltas.httpLatencyMsMax.toFixed(3)),
|
||||
controlRenewalLatencyMsP50: controlRenewal.p50,
|
||||
controlRenewalLatencyMsP95: controlRenewal.p95,
|
||||
controlRenewalLatencyMsMax: controlRenewal.max,
|
||||
httpLatencyMsMax: roundMs(deltas.httpLatencyMsMax),
|
||||
heapUsedBytes: memory.heapUsed,
|
||||
heapTotalBytes: memory.heapTotal,
|
||||
eventLoopDelayMsP99: Number(p99.toFixed(3))
|
||||
@@ -334,6 +432,22 @@ export class RelayObservability implements RelayRuntimeObserver {
|
||||
}
|
||||
}
|
||||
|
||||
// Flat siblings of the nested region maps, always emitted for every region including zeros.
|
||||
// A log-based metric cannot reach `requestedRegionsDelta."asia-east2"` without a quoted field
|
||||
// path, and an absent key would drop a series out of the inner join the region-skew alert does.
|
||||
// The maps stay authoritative and keep carrying anything outside the catalog, such as `unhinted`.
|
||||
function regionCounterFields(
|
||||
prefix: 'requestedRegion' | 'selectedRegion',
|
||||
counts: Record<string, number>
|
||||
): Record<string, number> {
|
||||
return Object.fromEntries(
|
||||
Object.entries(RELAY_REGION_METRIC_SEGMENTS).map(([region, segment]) => [
|
||||
`${prefix}${segment}Delta`,
|
||||
counts[region] ?? 0
|
||||
])
|
||||
)
|
||||
}
|
||||
|
||||
function increment(counts: Record<string, number>, key: string): void {
|
||||
counts[key] = (counts[key] ?? 0) + 1
|
||||
}
|
||||
|
||||
@@ -2,7 +2,9 @@ import { createAdaptorServer } from '@hono/node-server'
|
||||
import {
|
||||
hasAdmissionCapacity,
|
||||
HostDataAuthSchema,
|
||||
parseRelayHostCapabilities,
|
||||
RELAY_ADMISSION_BUDGETS,
|
||||
RELAY_HOST_CAPABILITIES_HEADER,
|
||||
RELAY_CLOSE_CODE,
|
||||
RELAY_DEFAULT_REGION,
|
||||
RELAY_PROTOCOL_LIMITS,
|
||||
@@ -486,7 +488,8 @@ export function createRelayServer(
|
||||
sessions.acceptControl(
|
||||
webSocket,
|
||||
identity,
|
||||
controlUpgrade?.inclusionWatermark
|
||||
controlUpgrade?.inclusionWatermark,
|
||||
parseRelayHostCapabilities(request.headers[RELAY_HOST_CAPABILITIES_HEADER])
|
||||
)
|
||||
})
|
||||
} catch {
|
||||
|
||||
@@ -9,7 +9,9 @@ import { fileURLToPath } from 'node:url'
|
||||
import { exportJWK, generateKeyPair, jwtVerify, SignJWT } from 'jose'
|
||||
import {
|
||||
buildHostProofMacInput,
|
||||
HOST_CHALLENGE_PLAINTEXT_DOMAIN
|
||||
HOST_CHALLENGE_PLAINTEXT_DOMAIN,
|
||||
RELAY_HOST_CAPABILITIES_HEADER,
|
||||
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS
|
||||
} from '@orca-cloud/relay-contract'
|
||||
import nacl from 'tweetnacl'
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
|
||||
@@ -282,11 +284,17 @@ async function openHostControl(input?: {
|
||||
previousGeneration?: number
|
||||
keyPair?: nacl.BoxKeyPair
|
||||
assignmentEpoch?: number
|
||||
capabilities?: string
|
||||
}): Promise<{ socket: WebSocket; ack: Record<string, unknown>; keyPair: nacl.BoxKeyPair }> {
|
||||
const keyPair = input?.keyPair ?? nacl.box.keyPair()
|
||||
const hostId = createHash('sha256').update(keyPair.publicKey).digest('base64url').slice(0, 16)
|
||||
const socket = new WebSocket(`${relayUrl.replace('http:', 'ws:')}/v1/host/control`, {
|
||||
headers: { authorization: `Bearer ${await relayToken('orca-relay', hostId)}` },
|
||||
headers: {
|
||||
authorization: `Bearer ${await relayToken('orca-relay', hostId)}`,
|
||||
...(input?.capabilities
|
||||
? { [RELAY_HOST_CAPABILITIES_HEADER]: input.capabilities }
|
||||
: {})
|
||||
},
|
||||
perMessageDeflate: false
|
||||
})
|
||||
await new Promise<void>((resolveOpen, reject) => {
|
||||
@@ -653,6 +661,69 @@ describe('served relay URL', () => {
|
||||
expect(result.reason).not.toContain('http')
|
||||
})
|
||||
|
||||
it('restates a pending connection to the rebound control, detailed only when advertised', async () => {
|
||||
// The one link the unit tests cannot reach: an upgrade that really carries
|
||||
// x-orca-host-capabilities must reach acceptControl and change the ack. A
|
||||
// typo in the header name here passes every other test in the suite.
|
||||
const host = await openHostControl()
|
||||
const hostId = createHash('sha256')
|
||||
.update(host.keyPair.publicKey)
|
||||
.digest('base64url')
|
||||
.slice(0, 16)
|
||||
const inviteResponse = nextMessage(host.socket)
|
||||
host.socket.send(
|
||||
JSON.stringify({
|
||||
type: 'invite-create',
|
||||
reqId: 'capability-invite',
|
||||
relayDeviceId: 'capability-device'
|
||||
})
|
||||
)
|
||||
const invite = await inviteResponse
|
||||
const phone = new WebSocket(`${relayUrl.replace('http:', 'ws:')}/v1/connect/${hostId}`, {
|
||||
headers: forwardedHeaders()
|
||||
})
|
||||
await new Promise<void>((resolveOpen, reject) => {
|
||||
phone.once('open', resolveOpen)
|
||||
phone.once('error', reject)
|
||||
})
|
||||
const connectionPromise = nextMessage(host.socket)
|
||||
phone.send(
|
||||
JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential: invite.inviteToken })
|
||||
)
|
||||
// Never attached: the connection stays pending, which is what the ack restates.
|
||||
const connection = await connectionPromise
|
||||
expect(connection.type).toBe('conn-open')
|
||||
|
||||
const capable = await openHostControl({
|
||||
keyPair: host.keyPair,
|
||||
controlResumeSecret: String(host.ack.controlResumeSecret),
|
||||
previousGeneration: 1,
|
||||
capabilities: RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS
|
||||
})
|
||||
expect(capable.ack.pendingConns).toEqual([
|
||||
{
|
||||
connId: connection.connId,
|
||||
connTicket: connection.connTicket,
|
||||
kind: 'invite',
|
||||
relayDeviceId: 'capability-device'
|
||||
}
|
||||
])
|
||||
|
||||
const legacy = await openHostControl({
|
||||
keyPair: host.keyPair,
|
||||
controlResumeSecret: String(capable.ack.controlResumeSecret),
|
||||
previousGeneration: 1
|
||||
})
|
||||
// A shipped host parses these entries strictly, so an unannounced key would
|
||||
// fail the whole ack and kill a control that was working.
|
||||
expect(legacy.ack.pendingConns).toEqual([
|
||||
{ connId: connection.connId, connTicket: connection.connTicket }
|
||||
])
|
||||
|
||||
phone.close()
|
||||
legacy.socket.close()
|
||||
})
|
||||
|
||||
it('keeps a pending attach usable after a bad ticket and rejects ticket replay', async () => {
|
||||
const host = await openHostControl()
|
||||
const hostId = createHash('sha256')
|
||||
|
||||
@@ -133,14 +133,17 @@
|
||||
"google_logging_metric.relay_snapshot",
|
||||
"google_monitoring_alert_policy.relay_assignment_5xx",
|
||||
"google_monitoring_alert_policy.relay_assignment_edge_429",
|
||||
"google_monitoring_alert_policy.relay_cell_control_rtt",
|
||||
"google_monitoring_alert_policy.relay_cell_process_exit",
|
||||
"google_monitoring_alert_policy.relay_cloud_nat_port_drops",
|
||||
"google_monitoring_alert_policy.relay_cloud_sql_backends",
|
||||
"google_monitoring_alert_policy.relay_cloud_sql_checkpoint_loop",
|
||||
"google_monitoring_alert_policy.relay_cloud_sql_disk",
|
||||
"google_monitoring_alert_policy.relay_custom",
|
||||
"google_monitoring_alert_policy.relay_far_cell_accept_latency",
|
||||
"google_monitoring_alert_policy.relay_gce_connection_headroom",
|
||||
"google_monitoring_alert_policy.relay_postgres_retry_exhausted",
|
||||
"google_monitoring_alert_policy.relay_region_hint_skew",
|
||||
"google_monitoring_dashboard.relay_incident",
|
||||
"google_project_iam_custom_role.github_production_relay_capacity_mutation",
|
||||
"google_project_iam_custom_role.github_relay_asia_topology_mutation",
|
||||
|
||||
@@ -45,6 +45,7 @@ export function parseRegionalRehomeArguments(argv, environment = process.env) {
|
||||
'not-before',
|
||||
'rate-per-minute',
|
||||
'preference-max-age-ms',
|
||||
'host-cooldown-ms',
|
||||
'drain-grace-ms',
|
||||
'confirmation'
|
||||
]
|
||||
@@ -117,6 +118,11 @@ export function parseRegionalRehomeArguments(argv, environment = process.env) {
|
||||
'--preference-max-age-ms',
|
||||
{ minimum: 60_000, maximum: 30 * 24 * 60 * 60_000 }
|
||||
),
|
||||
hostCooldownMs: integer(
|
||||
values['host-cooldown-ms'],
|
||||
'--host-cooldown-ms',
|
||||
{ minimum: 60_000, maximum: 30 * 24 * 60 * 60_000 }
|
||||
),
|
||||
drainGraceMs: integer(values['drain-grace-ms'], '--drain-grace-ms', {
|
||||
minimum: 60_000,
|
||||
maximum: 60 * 60_000
|
||||
@@ -147,6 +153,11 @@ function assertControl(control, expected) {
|
||||
!Number.isSafeInteger(control.notBefore) ||
|
||||
!Number.isSafeInteger(control.ratePerMinute) ||
|
||||
!Number.isSafeInteger(control.preferenceMaxAgeMs) ||
|
||||
// A director predating the per-host cooldown does not report it. Reading
|
||||
// the control and both emergency brakes must keep working against that
|
||||
// image; only enable requires the field.
|
||||
(control.hostCooldownMs !== undefined &&
|
||||
!Number.isSafeInteger(control.hostCooldownMs)) ||
|
||||
!Number.isSafeInteger(control.drainGraceMs)
|
||||
) throw new Error('director returned an invalid regional rehome control')
|
||||
if (expected.enabled !== undefined && control.enabled !== expected.enabled) {
|
||||
@@ -155,6 +166,12 @@ function assertControl(control, expected) {
|
||||
return control
|
||||
}
|
||||
|
||||
// Echo the cooldown only when the director already reports it: a legacy
|
||||
// director rejects the unknown key outright and would refuse every brake.
|
||||
function cooldownField(before, value) {
|
||||
return before.hostCooldownMs === undefined ? {} : { hostCooldownMs: value }
|
||||
}
|
||||
|
||||
async function verifiedDisabledControl(post, generation) {
|
||||
return assertControl((await post('/v1/admin/regional-rehome-control', {
|
||||
v: 1,
|
||||
@@ -171,6 +188,7 @@ async function applyDisabledControl(post, before) {
|
||||
notBefore: before.notBefore,
|
||||
ratePerMinute: before.ratePerMinute,
|
||||
preferenceMaxAgeMs: before.preferenceMaxAgeMs,
|
||||
...cooldownField(before, before.hostCooldownMs),
|
||||
drainGraceMs: before.drainGraceMs,
|
||||
confirmation: 'DISABLE_REGIONAL_REHOMING'
|
||||
})).control, { generation: before.generation + 1, enabled: false })
|
||||
@@ -270,6 +288,11 @@ export async function operateRegionalRehome(config, dependencies = {}) {
|
||||
throw new Error('regional rehome is already paused')
|
||||
}
|
||||
const enabled = config.mode === 'enable'
|
||||
if (enabled && before.hostCooldownMs === undefined) {
|
||||
throw new Error(
|
||||
'director does not report a per-host rehome cooldown; deploy a director that supports it before enabling'
|
||||
)
|
||||
}
|
||||
const applied = await post('/v1/admin/regional-rehome-control', {
|
||||
v: 1,
|
||||
action: 'apply',
|
||||
@@ -278,6 +301,7 @@ export async function operateRegionalRehome(config, dependencies = {}) {
|
||||
notBefore: config.notBefore,
|
||||
ratePerMinute: config.ratePerMinute,
|
||||
preferenceMaxAgeMs: config.preferenceMaxAgeMs,
|
||||
...cooldownField(before, config.hostCooldownMs),
|
||||
drainGraceMs: config.drainGraceMs,
|
||||
confirmation: enabled
|
||||
? 'ENABLE_REGIONAL_REHOMING'
|
||||
|
||||
@@ -26,6 +26,7 @@ function argumentsFor(mode, confirmation) {
|
||||
'--not-before', '2000000000000',
|
||||
'--rate-per-minute', '10',
|
||||
'--preference-max-age-ms', '86400000',
|
||||
'--host-cooldown-ms', '604800000',
|
||||
'--drain-grace-ms', '60000',
|
||||
'--confirmation', confirmation
|
||||
])
|
||||
@@ -40,10 +41,29 @@ function control(generation, enabled) {
|
||||
notBefore: 2_000_000_000_000,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 86_400_000,
|
||||
hostCooldownMs: 604_800_000,
|
||||
drainGraceMs: 60_000
|
||||
}
|
||||
}
|
||||
|
||||
// The control a director predating the per-host cooldown reports.
|
||||
function legacyControl(generation, enabled) {
|
||||
const { hostCooldownMs: _absent, ...rest } = control(generation, enabled)
|
||||
return rest
|
||||
}
|
||||
|
||||
function legacyDirector(controls) {
|
||||
const requests = []
|
||||
const post = async (path, body) => {
|
||||
requests.push({ path, body })
|
||||
if (path === '/v1/admin/admission-selector/status') {
|
||||
return { selector: { generation: 11, membership } }
|
||||
}
|
||||
return { v: 1, control: controls.shift() }
|
||||
}
|
||||
return { requests, post }
|
||||
}
|
||||
|
||||
test('parses exact selector and typed control confirmation', () => {
|
||||
const parsed = parseRegionalRehomeArguments(
|
||||
argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING'),
|
||||
@@ -52,6 +72,17 @@ test('parses exact selector and typed control confirmation', () => {
|
||||
assert.equal(parsed.expectedSelectorGeneration, 11)
|
||||
assert.equal(parsed.expectedControlGeneration, 4)
|
||||
assert.equal(parsed.ratePerMinute, 10)
|
||||
assert.equal(parsed.hostCooldownMs, 604_800_000)
|
||||
assert.throws(
|
||||
() => parseRegionalRehomeArguments(
|
||||
argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING').filter(
|
||||
(value, index, all) =>
|
||||
value !== '--host-cooldown-ms' && all[index - 1] !== '--host-cooldown-ms'
|
||||
),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
),
|
||||
/complete durable control shape/
|
||||
)
|
||||
assert.throws(
|
||||
() => parseRegionalRehomeArguments(
|
||||
argumentsFor('pause', 'DISABLE_REGIONAL_REHOMING'),
|
||||
@@ -79,6 +110,7 @@ test('binds enable to exact selector and durable control generations', async ()
|
||||
notBefore: 0,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 86_400_000,
|
||||
hostCooldownMs: 604_800_000,
|
||||
drainGraceMs: 60_000,
|
||||
...control
|
||||
}))
|
||||
@@ -96,6 +128,7 @@ test('binds enable to exact selector and durable control generations', async ()
|
||||
}
|
||||
})
|
||||
assert.equal(result.control.generation, 5)
|
||||
assert.equal(result.control.hostCooldownMs, 604_800_000)
|
||||
assert.deepEqual(requests[2].body, {
|
||||
v: 1,
|
||||
action: 'apply',
|
||||
@@ -104,11 +137,82 @@ test('binds enable to exact selector and durable control generations', async ()
|
||||
notBefore: 2_000_000_000_000,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 86_400_000,
|
||||
hostCooldownMs: 604_800_000,
|
||||
drainGraceMs: 60_000,
|
||||
confirmation: 'ENABLE_REGIONAL_REHOMING'
|
||||
})
|
||||
})
|
||||
|
||||
test('inspects a director that predates the per-host cooldown', async () => {
|
||||
const director = legacyDirector([legacyControl(4, true)])
|
||||
const config = parseRegionalRehomeArguments(
|
||||
argumentsFor('inspect'),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
)
|
||||
|
||||
const result = await operateRegionalRehome(config, { post: director.post })
|
||||
|
||||
assert.equal(result.control.generation, 4)
|
||||
assert.equal(result.control.hostCooldownMs, undefined)
|
||||
})
|
||||
|
||||
for (const [mode, confirmation, enabledBefore] of [
|
||||
['pause', 'PAUSE_REGIONAL_REHOMING', true],
|
||||
['disable', 'DISABLE_REGIONAL_REHOMING', false]
|
||||
]) {
|
||||
test(`${mode} still brakes a director that predates the cooldown`, async () => {
|
||||
const director = legacyDirector([
|
||||
legacyControl(4, enabledBefore),
|
||||
legacyControl(5, false),
|
||||
legacyControl(5, false)
|
||||
])
|
||||
const config = parseRegionalRehomeArguments(
|
||||
argumentsFor(mode, confirmation),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
)
|
||||
|
||||
const result = await operateRegionalRehome(config, { post: director.post })
|
||||
|
||||
assert.equal(result.control.generation, 5)
|
||||
// The unknown key would be refused by that director's strict schema.
|
||||
assert.equal('hostCooldownMs' in director.requests[2].body, false)
|
||||
assert.equal(director.requests[2].body.confirmation, 'DISABLE_REGIONAL_REHOMING')
|
||||
})
|
||||
}
|
||||
|
||||
test('failed-enable recovery brakes a director that predates the cooldown', async () => {
|
||||
const requests = []
|
||||
let current = legacyControl(7, true)
|
||||
const result = await recoverRegionalRehomeEnable({
|
||||
mode: 'recover-enable',
|
||||
expectedControlGeneration: 4
|
||||
}, async (_path, body) => {
|
||||
requests.push(body)
|
||||
if (body.action === 'inspect') return { control: current }
|
||||
current = legacyControl(8, false)
|
||||
return { control: current }
|
||||
})
|
||||
|
||||
assert.equal(result.control.generation, 8)
|
||||
assert.equal('hostCooldownMs' in requests[1], false)
|
||||
})
|
||||
|
||||
test('refuses to enable a director that does not report the cooldown', async () => {
|
||||
const director = legacyDirector([legacyControl(4, false)])
|
||||
const config = parseRegionalRehomeArguments(
|
||||
argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING'),
|
||||
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
|
||||
)
|
||||
|
||||
await assert.rejects(
|
||||
operateRegionalRehome(config, { post: director.post }),
|
||||
/per-host rehome cooldown/
|
||||
)
|
||||
// Read-only: selector status and the control inspect, and nothing else.
|
||||
assert.equal(director.requests.length, 2)
|
||||
assert.equal(director.requests.every(({ body }) => body.action !== 'apply'), true)
|
||||
})
|
||||
|
||||
test('fails closed on selector drift before reading or mutating control', async () => {
|
||||
let calls = 0
|
||||
const config = parseRegionalRehomeArguments(
|
||||
@@ -150,6 +254,7 @@ test('failed-enable recovery CAS-disables an advanced enabled generation', async
|
||||
notBefore: 2_000_000_000_000,
|
||||
ratePerMinute: 10,
|
||||
preferenceMaxAgeMs: 86_400_000,
|
||||
hostCooldownMs: 604_800_000,
|
||||
drainGraceMs: 60_000,
|
||||
confirmation: 'DISABLE_REGIONAL_REHOMING'
|
||||
})
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
|
||||
|
||||
const PRODUCTION_CELL = /^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26)$/
|
||||
// Every general cell that carries the rehome identity: the sixteen US cells and the
|
||||
// three asia-east2 cells that drain mis-homed hosts back the other way.
|
||||
const PRODUCTION_CELL = /^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26|27|28|29)$/
|
||||
const DIRECTOR_ORIGIN = 'https://relay.onorca.dev'
|
||||
|
||||
export function parseRehomeTrustProbeArguments(argv, environment = process.env) {
|
||||
|
||||
@@ -111,3 +111,23 @@ test('fails when both trust-probe attempts return a transient 503', async () =>
|
||||
)
|
||||
assert.equal(calls, 2)
|
||||
})
|
||||
|
||||
test('approves the asia-east2 rehome sources and still rejects unlisted cells', () => {
|
||||
for (const cellId of ['production-gce-c27', 'production-gce-c28', 'production-gce-c29']) {
|
||||
const parsed = parseRehomeTrustProbeArguments(
|
||||
argv.map((value) => (value === 'production-gce-c7' ? cellId : value)),
|
||||
environment
|
||||
)
|
||||
assert.equal(parsed.cellId, cellId)
|
||||
}
|
||||
for (const cellId of ['production-gce-c1', 'production-gce-c17', 'production-gce-c30']) {
|
||||
assert.throws(
|
||||
() =>
|
||||
parseRehomeTrustProbeArguments(
|
||||
argv.map((value) => (value === 'production-gce-c7' ? cellId : value)),
|
||||
environment
|
||||
),
|
||||
/--cell-id is not approved/
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import test from 'node:test'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
// Why: the region-skew alert compares asia-east2's share of assignment hints against its share of
|
||||
// actual placements. Both shares are sums over one log-based metric per region, and the region
|
||||
// list is written out by hand in Terraform. A region added to the contract without matching
|
||||
// metrics would silently drop out of both denominators and move the ratio the alert fires on.
|
||||
|
||||
const read = (relative) => readFileSync(fileURLToPath(new URL(relative, import.meta.url)), 'utf8')
|
||||
const collapse = (text) => text.replaceAll(/\s+/g, ' ')
|
||||
|
||||
const contractRegions = (() => {
|
||||
const source = read('../../packages/relay-contract/src/relay-regions.ts')
|
||||
const literal = /export const RELAY_REGIONS = \[([^\]]*)\]/.exec(source)
|
||||
assert.ok(literal, 'RELAY_REGIONS literal not found in relay-regions.ts')
|
||||
return [...literal[1].matchAll(/'([^']+)'/g)].map((match) => match[1])
|
||||
})()
|
||||
|
||||
const terraform = read('../../infra/terraform/relay-observability.tf')
|
||||
|
||||
const terraformRegions = (() => {
|
||||
const literal = /relay_region_keys = \[([^\]]*)\]/.exec(terraform)
|
||||
assert.ok(literal, 'relay_region_keys not found in relay-observability.tf')
|
||||
return [...literal[1].matchAll(/"([^"]+)"/g)].map((match) => match[1])
|
||||
})()
|
||||
|
||||
// Both sides now spell the field-name segments out, so the test compares the two declared maps
|
||||
// rather than two source expressions. Reformatting either file cannot break this, and a literal
|
||||
// expected value below still catches an identical wrong edit made to both.
|
||||
const declaredSegments = (source, open, close) => {
|
||||
const body = source.slice(source.indexOf(open) + open.length, source.indexOf(close, source.indexOf(open)))
|
||||
return Object.fromEntries(
|
||||
[...body.matchAll(/'?"?([a-z0-9-]+)'?"?\s*[:=]\s*'?"?([A-Za-z0-9]+)'?"?/g)].map((match) => [
|
||||
match[1],
|
||||
match[2]
|
||||
])
|
||||
)
|
||||
}
|
||||
|
||||
const terraformSegments = declaredSegments(terraform, 'relay_region_field_segments = {', '}')
|
||||
const contractSegments = declaredSegments(
|
||||
read('../../packages/relay-contract/src/relay-regions.ts'),
|
||||
'RELAY_REGION_METRIC_SEGMENTS = {',
|
||||
'}'
|
||||
)
|
||||
|
||||
test('terraform covers exactly the regions the contract can hint or select', () => {
|
||||
assert.deepEqual([...terraformRegions].sort(), [...contractRegions].sort())
|
||||
})
|
||||
|
||||
test('terraform and the contract declare the same flat field segments', () => {
|
||||
assert.deepEqual(terraformSegments, contractSegments)
|
||||
// Pinned literally so the same wrong edit applied to both sides still fails.
|
||||
assert.deepEqual(terraformSegments, { 'us-central1': 'UsCentral1', 'asia-east2': 'AsiaEast2' })
|
||||
assert.deepEqual(Object.keys(terraformSegments).sort(), [...contractRegions].sort())
|
||||
})
|
||||
|
||||
test('the skew query compares a catalogued region against itself', () => {
|
||||
const columns = terraformRegions.map((region) => region.replaceAll('-', '_'))
|
||||
const hint = /hint_share: req_([a-z0-9_]+) \//.exec(terraform)
|
||||
const placement = /placement_share: sel_([a-z0-9_]+) \//.exec(terraform)
|
||||
assert.ok(hint && placement, 'skew query share columns not found')
|
||||
assert.equal(hint[1], placement[1], 'the two shares must be about the same region')
|
||||
assert.ok(columns.includes(hint[1]), `${hint[1]} is not one of ${columns.join(', ')}`)
|
||||
})
|
||||
|
||||
test('the skew condition never divides by the placement share', () => {
|
||||
// A zero-placement hour is the worst skew there is; MQL drops the row on x/0, so the ratio form
|
||||
// silences exactly the case the alert exists for.
|
||||
assert.ok(
|
||||
!/hint_share \/ placement_share/.test(terraform),
|
||||
'cross-multiply instead: hint_share > 2 * placement_share'
|
||||
)
|
||||
assert.match(collapse(terraform), /condition hint_share > 2 \* placement_share/)
|
||||
})
|
||||
|
||||
test('the unhinted bucket stays out of the skew denominators', () => {
|
||||
assert.ok(
|
||||
!terraformRegions.includes('unhinted'),
|
||||
'unhinted requests are a client-side choice, not a region; including them moves the share'
|
||||
)
|
||||
})
|
||||
@@ -179,9 +179,10 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
|
||||
|
||||
it('validates a correct plan for every wave cell at that cell\'s rehome protocol', () => {
|
||||
for (const cellId of SAME_CAP_CELLS) {
|
||||
const [region, cap] = resolveCellShape(cellId).stdout.trim().split(' ')
|
||||
const [, cap] = resolveCellShape(cellId).stdout.trim().split(' ')
|
||||
const protocol = REHOME_SOURCE_CELLS.has(cellId) ? 1 : 0
|
||||
assert.equal(protocol, region === 'us-central1' ? 1 : 0, cellId)
|
||||
// Every reviewed serving cell carries rehome trust now, in either region.
|
||||
assert.equal(protocol, 1, cellId)
|
||||
const config = {
|
||||
mode: 'same-cap-cell',
|
||||
cellId,
|
||||
@@ -211,6 +212,29 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('validates a protocol-0 plan for a cell outside the rehome source list', () => {
|
||||
const cellId = 'production-gce-c17'
|
||||
assert.equal(REHOME_SOURCE_CELLS.has(cellId), false)
|
||||
const config = {
|
||||
mode: 'same-cap-cell',
|
||||
cellId,
|
||||
hardCap: 1000,
|
||||
unobservedBound: 60,
|
||||
image: TARGET_IMAGE,
|
||||
rollbackImage: ROLLBACK_IMAGE,
|
||||
rehomeDirectorServiceAccount: DIRECTOR_IDENTITY,
|
||||
rehomeAudience: AUDIENCE,
|
||||
regionalRehomeProtocol: '0'
|
||||
}
|
||||
const plan = rollPlan({ cellId, cap: 1000, protocol: 0 })
|
||||
assert.deepEqual(validateCapacityPlan(plan, config), { mode: 'same-cap-cell', changes: 2 })
|
||||
// Protocol 1 must reject a plan with no rehome lines, or the absent-line rule decides nothing.
|
||||
assert.throws(
|
||||
() => validateCapacityPlan(plan, { ...config, regionalRehomeProtocol: '1' }),
|
||||
/reviewed image and capacity/
|
||||
)
|
||||
})
|
||||
|
||||
it('leaves the US-only capacity job on the default allowlist', () => {
|
||||
assert.doesNotMatch(capacityWorkflow, /--approved-cells/)
|
||||
})
|
||||
|
||||
@@ -464,6 +464,18 @@ Once a target control is registered, do not force the pre-registration rollback.
|
||||
|
||||
After a deployment traffic shift, preserve the old revision/tag until metrics and live reconnect checks pass. If the new revision is unhealthy, shift traffic back only while old controls are still valid, then issue a strictly newer director migration rather than reusing a prior epoch.
|
||||
|
||||
## Regional rehoming
|
||||
|
||||
Rehoming moves a host to a general cell in the region its desktop last reported, in either
|
||||
direction. Both roles need the drain protocol: a cell without it can be neither a source nor a
|
||||
target, and it is not part of the fleet whose telemetry gates the worker. Until the asia-east2
|
||||
cells run `regionalRehomeProtocol` 1 they are none of the three, so no host is moved into or out
|
||||
of Asia and an Asia cell in distress does not pause the worker.
|
||||
|
||||
`host-cooldown-ms` is the minimum gap between two rehomes of one host. It bounds the damage from
|
||||
a desktop whose region probe flips: without it the host would be dragged back across the ocean on
|
||||
every flip, since the preference age never expires while the host keeps reconnecting.
|
||||
|
||||
## Game-day matrix
|
||||
|
||||
Run and record each scenario in staging before launch:
|
||||
|
||||
@@ -121,6 +121,79 @@ durably marked consumed before mutation and cannot authorize another run.
|
||||
Expected enabled cells must also have a powered runtime, healthy and ready endpoints, fresh
|
||||
heartbeats, and matching live admission.
|
||||
|
||||
## Region placement alert policies
|
||||
|
||||
Cloud Monitoring alert policies, not monitor freeze bars: these page from
|
||||
`cloud/infra/terraform/relay-observability.tf` on the shared relay channel in
|
||||
`relay_alert_notification_channels`, and they do not gate any workflow. All
|
||||
three exist because US desktops sat on asia-east2 cells for weeks in 2026-08
|
||||
with every existing bar green.
|
||||
|
||||
| Alert policy | Condition |
|
||||
| --- | ---: |
|
||||
| Orca Relay: far-cell phone accept latency | per cell, median 30-second `clientAcceptTotalMsP95` over 15 minutes above 2,000 ms with at least 20 completed accepts |
|
||||
| Orca Relay: cell control round trip | per cell, median `controlRttMsP50` over one hour above 150 ms with at least 500 samples |
|
||||
| Orca Relay: region hint skew | fleet-wide, asia-east2 share of hinted requests over one hour more than 2x and more than 15 points above its share of actual placements, with at least 500 hinted requests |
|
||||
|
||||
Threshold basis:
|
||||
|
||||
- Accept latency. An in-region phone accept completes in 0.3-0.6 s and a
|
||||
cross-Pacific one in 5-10 s, so 2,000 ms sits outside in-region noise and
|
||||
well under the far-cell floor. The 20-accept minimum keeps one slow accept
|
||||
on a quiet cell off the pager. The p95 is the published value, so the
|
||||
window aggregate is its median, not its max.
|
||||
- Control round trip. In-region is tens of milliseconds; a US desktop on an
|
||||
asia-east2 cell is 200 ms or more. Only the p50 is used. The desktop echoes
|
||||
the pong on its main thread, so the published p95 and max track renderer
|
||||
stalls rather than distance. 500 samples per hour is about two
|
||||
continuously connected hosts at the 15-second control ping. Tuning risk: EU
|
||||
desktops on us-central1 sit at 100-130 ms, so a cell whose population is
|
||||
mostly European can approach the bar while correctly homed. Check where the
|
||||
hosts are before reading a first breach as mis-homing.
|
||||
- Region hint skew. This compares two shares of the same hour rather than
|
||||
testing one absolute share, because an absolute bar is wrong at both ends.
|
||||
Measured over twelve hours on 2026-09-07, while the desktop region probe
|
||||
was still mis-picking: asia-east2 was 33.8% of the 33,800 hinted requests
|
||||
and only 7.9% of the 45,364 assignments, a divergence of 4.27x and a gap of
|
||||
25.9 points. A fixed 40% bar would have stayed silent through that, and
|
||||
once the probe is fixed the genuine APAC share climbs past any such bar and
|
||||
pages forever on the correct end state. The 2x and 15-point bars sit inside
|
||||
the broken state and outside a healthy one. `unhinted` requests are
|
||||
excluded from the denominator: they were 27% of all requests, so a client
|
||||
change that always sends a hint would move the number with no behaviour
|
||||
change at all. The two bars are cross-multiplied rather than divided. An
|
||||
hour that placed nobody in the region is the most extreme skew there is,
|
||||
and it happens whenever the region is drained, fenced, or at capacity, but
|
||||
dividing by that zero placement share makes MQL drop the row and lose the
|
||||
series before any other clause runs.
|
||||
|
||||
Expect the skew alert to stay lit after a client fix until the mis-homed
|
||||
backlog is rehomed. Sticky assignment never re-consults the hint, so a
|
||||
desktop already on an asia cell keeps being placed there whatever it now
|
||||
asks for; the ratio clears only once the rehome sweep has drained.
|
||||
|
||||
All three conditions are written in MQL rather than the metric filters the
|
||||
other relay policies use. Every runtime metric is a DELTA DISTRIBUTION, and
|
||||
the only scalar aligners a filter condition can apply to one are percentiles;
|
||||
each of these alerts needs the sum of the extracted values as a volume floor,
|
||||
which is `sum(value.<metric>)` in MQL and unreachable otherwise. None of the
|
||||
metrics they read exists in the project yet, so what was checked against
|
||||
production is the query shape: the same MQL run over existing metrics of the
|
||||
same kind confirmed the distribution sum, the join arity, the unit literals,
|
||||
and the condition clause.
|
||||
|
||||
The skew shares are built from one log-based metric per region for hints and
|
||||
one per region for placements. They read flat `requestedRegion<Region>Delta`
|
||||
and `selectedRegion<Region>Delta` fields that the relay publishes as zeros in
|
||||
every interval, not the nested region maps: a log-based metric would need a
|
||||
quoted field path to reach a hyphenated map key, and an absent key would drop
|
||||
a series out of the inner join. The region list lives in Terraform as
|
||||
`relay_region_keys` and is pinned to relay-contract's `RELAY_REGIONS` by
|
||||
`dev/scripts/relay-region-hint-metrics.test.mjs`. Both sides spell the field
|
||||
name segments out as literal maps rather than deriving them, so the same test
|
||||
compares the two declarations directly. Adding a region to the contract
|
||||
without its segment is a compile error in relay-contract, not a silent gap.
|
||||
|
||||
## Implementation log
|
||||
|
||||
- Recalibrated the relay pool freezes from 30 waiters / 1,000 ms to
|
||||
|
||||
@@ -402,7 +402,11 @@ relay_region_rehome_source_cell_ids = [
|
||||
"production-gce-c23",
|
||||
"production-gce-c24",
|
||||
"production-gce-c25",
|
||||
"production-gce-c26"
|
||||
"production-gce-c26",
|
||||
# Asia cells carry the same trust so mis-homed hosts can be drained back off them.
|
||||
"production-gce-c27",
|
||||
"production-gce-c28",
|
||||
"production-gce-c29"
|
||||
]
|
||||
|
||||
# Slack #orca-relay-alerts, created out of band on 2026-08-05. Declared here because an apply
|
||||
|
||||
@@ -82,14 +82,15 @@ check "relay_gce_fixed_one_topology" {
|
||||
|
||||
assert {
|
||||
condition = alltrue([
|
||||
# Region is not asserted here: the director's own rehome source and target predicates
|
||||
# own eligibility, so this pins only cell shape.
|
||||
for cell_id in var.relay_region_rehome_source_cell_ids : try(
|
||||
var.relay_gce_cells[cell_id].region == var.region &&
|
||||
var.relay_gce_cells[cell_id].connection_hard_cap != null &&
|
||||
!contains(var.relay_gce_fenced_cells, cell_id),
|
||||
false
|
||||
)
|
||||
])
|
||||
error_message = "Regional rehome sources must be configured, unfenced primary-region GCE cells with explicit connection limits."
|
||||
error_message = "Regional rehome sources must be configured, unfenced GCE cells with explicit connection limits."
|
||||
}
|
||||
|
||||
assert {
|
||||
|
||||
@@ -65,6 +65,20 @@ locals {
|
||||
control_renewal_lease_misses = { field = "controlRenewalLeaseMissesDelta", description = "Control renewals that found their activity lease missing." }
|
||||
control_activity_recoveries = { field = "controlActivityRecoveriesDelta", description = "Control activity leases recovered after a renewal miss." }
|
||||
control_activity_recovery_failures = { field = "controlActivityRecoveryFailuresDelta", description = "Control activity lease recovery attempts that failed." }
|
||||
control_rtt_ms_p50 = { field = "controlRttMsP50", description = "Control-socket ping round trip p50 in the interval. The desktop echoes the pong on its main thread, so only the median reads as distance; the p95 and max below are dominated by desktop stalls." }
|
||||
control_rtt_ms_p95 = { field = "controlRttMsP95", description = "Control-socket ping round trip p95 in the interval; a desktop-stall signal, not a distance one." }
|
||||
control_rtt_ms_max = { field = "controlRttMsMax", description = "Maximum control-socket ping round trip in the interval; a desktop-stall signal, not a distance one." }
|
||||
control_rtt_samples = { field = "controlRttSamplesDelta", description = "Control-socket round trips observed in the interval, one per ping answered; the percentiles above are omitted when this is zero." }
|
||||
control_rtt_samples_dropped = { field = "controlRttSamplesDroppedDelta", description = "Observed round trips the bounded percentile reservoir did not keep; non-zero means the percentiles above summarise a uniform sample of the interval." }
|
||||
client_accepts_completed = { field = "clientAcceptCompletedDelta", description = "Phone accepts that reached relay-hello in the interval; the percentiles below are omitted when this is zero." }
|
||||
client_accept_total_ms_p50 = { field = "clientAcceptTotalMsP50", description = "Successful phone-accept duration p50, dial to relay-hello." }
|
||||
client_accept_total_ms_p95 = { field = "clientAcceptTotalMsP95", description = "Successful phone-accept duration p95, dial to relay-hello." }
|
||||
client_accept_total_ms_max = { field = "clientAcceptTotalMsMax", description = "Maximum successful phone-accept duration in the interval." }
|
||||
client_accept_assignment_ms_p95 = { field = "clientAcceptAssignmentMsP95", description = "Accept stage p95: resume/invite lookup plus assignment resolve." }
|
||||
client_accept_credential_ms_p95 = { field = "clientAcceptCredentialMsP95", description = "Accept stage p95: outer credential reservation." }
|
||||
client_accept_activity_ms_p95 = { field = "clientAcceptActivityMsP95", description = "Accept stage p95: credential activity lease acquisition." }
|
||||
client_accept_attach_ms_p95 = { field = "clientAcceptAttachMsP95", description = "Accept stage p95: conn-open sent until the desktop's data leg authenticated." }
|
||||
client_accept_basis_ms_p95 = { field = "clientAcceptBasisMsP95", description = "Accept stage p95: splice lease and connection-basis writes between the data leg and relay-hello." }
|
||||
heap_used_bytes = { field = "heapUsedBytes", description = "Node.js heap bytes used by the relay process." }
|
||||
event_loop_ms_p99 = { field = "eventLoopDelayMsP99", description = "Node.js event-loop delay p99 in milliseconds." }
|
||||
forwarded_bytes = { field = "forwardedBytesDelta", description = "Ciphertext bytes admitted for forwarding." }
|
||||
@@ -80,6 +94,80 @@ locals {
|
||||
db_oldest_wait_ms = { field = "databasePoolOldestWaitMs", description = "Current oldest PostgreSQL pool waiter age." }
|
||||
db_wait_ms_max = { field = "databasePoolWaitMsMax", description = "Maximum PostgreSQL pool wait during the interval." }
|
||||
}
|
||||
|
||||
# Regions the director can hint or select. Pinned to relay-contract's RELAY_REGIONS by
|
||||
# dev/scripts/relay-region-hint-metrics.test.mjs, which also checks the flat field names below
|
||||
# against the emitter. A region missing here drops out of both shares the skew alert compares.
|
||||
relay_region_keys = ["us-central1", "asia-east2"]
|
||||
# Flat emitter fields, not the nested `requestedRegionsDelta` map: a log-based metric would need
|
||||
# a quoted field path to reach a hyphenated map key, and the relay publishes these as zeros in
|
||||
# every interval so no series can drop out of the alert's inner join. Spelled out rather than
|
||||
# derived, so this literal and relay-contract's RELAY_REGION_METRIC_SEGMENTS can be compared
|
||||
# directly; reformatting either side cannot break the check and neither can drift alone.
|
||||
relay_region_field_segments = {
|
||||
"us-central1" = "UsCentral1"
|
||||
"asia-east2" = "AsiaEast2"
|
||||
}
|
||||
relay_region_columns = { for key in local.relay_region_keys : key => replace(key, "-", "_") }
|
||||
relay_region_share_metrics = merge(
|
||||
{
|
||||
for key in local.relay_region_keys :
|
||||
"requested_regions_${local.relay_region_columns[key]}" => {
|
||||
field = "requestedRegion${local.relay_region_field_segments[key]}Delta"
|
||||
description = "Assignment requests that hinted ${key}."
|
||||
}
|
||||
},
|
||||
{
|
||||
for key in local.relay_region_keys :
|
||||
"selected_regions_${local.relay_region_columns[key]}" => {
|
||||
field = "selectedRegion${local.relay_region_field_segments[key]}Delta"
|
||||
description = "Assignments that placed a host in ${key}."
|
||||
}
|
||||
}
|
||||
)
|
||||
relay_region_hinted_total = join(" + ", [for key in local.relay_region_keys : "req_${local.relay_region_columns[key]}"])
|
||||
relay_region_selected_total = join(" + ", [for key in local.relay_region_keys : "sel_${local.relay_region_columns[key]}"])
|
||||
# MQL, not a filter condition: every runtime metric is a DELTA DISTRIBUTION, and the only scalar
|
||||
# aligners a `condition_threshold` can apply to one are percentiles. Both shares need the sum of
|
||||
# the extracted values, which is `sum(value.<metric>)` in MQL and unreachable otherwise.
|
||||
relay_region_hint_skew_query = join("\n", concat(
|
||||
["{"],
|
||||
flatten([
|
||||
for index, entry in [
|
||||
for key in local.relay_region_keys : { metric = "requested_regions_${local.relay_region_columns[key]}", column = "req_${local.relay_region_columns[key]}" }
|
||||
] : [
|
||||
index == 0 ? "" : ";",
|
||||
" fetch cloud_run_revision::logging.googleapis.com/user/orca_relay_${entry.metric}",
|
||||
" | align delta(1h) | every 1h",
|
||||
" | group_by [], [${entry.column}: sum(value.orca_relay_${entry.metric})]"
|
||||
]
|
||||
]),
|
||||
flatten([
|
||||
for key in local.relay_region_keys : [
|
||||
";",
|
||||
" fetch cloud_run_revision::logging.googleapis.com/user/orca_relay_selected_regions_${local.relay_region_columns[key]}",
|
||||
" | align delta(1h) | every 1h",
|
||||
" | group_by [], [sel_${local.relay_region_columns[key]}: sum(value.orca_relay_selected_regions_${local.relay_region_columns[key]})]"
|
||||
]
|
||||
]),
|
||||
[
|
||||
"}",
|
||||
"| join",
|
||||
"| value [",
|
||||
" hint_share: req_asia_east2 / (${local.relay_region_hinted_total}),",
|
||||
" placement_share: sel_asia_east2 / (${local.relay_region_selected_total}),",
|
||||
" hinted_requests: ${local.relay_region_hinted_total}",
|
||||
" ]",
|
||||
# Cross-multiplied, never a plain ratio of the two shares: an hour that placed nobody in the
|
||||
# region makes that ratio 0/0 or x/0, and MQL drops the row instead of yielding a number, so
|
||||
# the whole series vanishes before the other clauses run. That hour is the worst skew there
|
||||
# is - every desktop asking for a region the director is putting nobody in - and it happens
|
||||
# whenever the region is drained, fenced, or at capacity. Both forms were run read-only
|
||||
# against production surrogates with a zero denominator: the ratio returned no rows, this
|
||||
# returned the series with the condition true.
|
||||
"| condition hint_share > 2 * placement_share && hint_share - placement_share > 0.15 '1' && hinted_requests > 500 '1'"
|
||||
]
|
||||
))
|
||||
relay_custom_alerts = {
|
||||
connection_headroom = {
|
||||
pages_oncall = true
|
||||
@@ -201,7 +289,9 @@ locals {
|
||||
}
|
||||
|
||||
resource "google_logging_metric" "relay_snapshot" {
|
||||
for_each = local.relay_runtime_metrics
|
||||
# Region-request metrics ride the same event and shape; merging adds map entries only, so the
|
||||
# existing metric instances are untouched (a label change, not a new key, is what recreates them).
|
||||
for_each = merge(local.relay_runtime_metrics, local.relay_region_share_metrics)
|
||||
|
||||
project = var.project_id
|
||||
name = "orca_relay_${each.key}"
|
||||
@@ -211,14 +301,14 @@ resource "google_logging_metric" "relay_snapshot" {
|
||||
label_extractors = {
|
||||
role = "EXTRACT(jsonPayload.role)"
|
||||
cell_id = "EXTRACT(jsonPayload.cellId)"
|
||||
# No region label: adding one replaces all 21 live metrics (label change = delete+create),
|
||||
# No region label: adding one replaces all 42 live metrics (label change = delete+create),
|
||||
# which resets history and blanks the relay alert policies during the swap.
|
||||
}
|
||||
|
||||
metric_descriptor {
|
||||
metric_kind = "DELTA"
|
||||
value_type = "DISTRIBUTION"
|
||||
unit = contains(["sql_latency_ms", "control_renewal_latency_ms_p50", "control_renewal_latency_ms_p95", "control_renewal_latency_ms_max", "http_latency_ms", "event_loop_ms_p99", "db_oldest_wait_ms", "db_wait_ms_max"], each.key) ? "ms" : each.key == "queued_bytes" || each.key == "heap_used_bytes" || each.key == "forwarded_bytes" ? "By" : "1"
|
||||
unit = contains(["sql_latency_ms", "control_rtt_ms_p50", "control_rtt_ms_p95", "control_rtt_ms_max", "client_accept_total_ms_p50", "client_accept_total_ms_p95", "client_accept_total_ms_max", "client_accept_assignment_ms_p95", "client_accept_credential_ms_p95", "client_accept_activity_ms_p95", "client_accept_attach_ms_p95", "client_accept_basis_ms_p95", "control_renewal_latency_ms_p50", "control_renewal_latency_ms_p95", "control_renewal_latency_ms_max", "http_latency_ms", "event_loop_ms_p99", "db_oldest_wait_ms", "db_wait_ms_max"], each.key) ? "ms" : each.key == "queued_bytes" || each.key == "heap_used_bytes" || each.key == "forwarded_bytes" ? "By" : "1"
|
||||
|
||||
labels {
|
||||
key = "role"
|
||||
@@ -672,6 +762,128 @@ resource "google_monitoring_alert_policy" "relay_cell_process_exit" {
|
||||
depends_on = [google_logging_metric.relay_incident]
|
||||
}
|
||||
|
||||
# Why: nothing fired while US desktops sat on asia-east2 cells for weeks in 2026-08. The two
|
||||
# per-cell policies below read that as distance, and the fleet-wide one reads it as a bad region
|
||||
# hint. All three are MQL because each needs the sum of a DELTA DISTRIBUTION as a volume floor,
|
||||
# and the only scalar aligners a `condition_threshold` can apply to a distribution are percentiles.
|
||||
# `join` is an inner join and the relay omits its percentile fields on an empty interval, so an
|
||||
# idle cell drops out rather than alerting on nothing. The per-cell arms fetch `gce_instance`
|
||||
# only: production runs no Cloud Run cells (`relay_cells` is empty), and a future one would need
|
||||
# its own arm here. None of the metrics these query exist in the project yet, so what was checked
|
||||
# against production is the query shape: the same MQL run over existing metrics of the same kind
|
||||
# confirmed the distribution sum, the join arity, the unit literals, and the condition clause.
|
||||
resource "google_monitoring_alert_policy" "relay_far_cell_accept_latency" {
|
||||
project = var.project_id
|
||||
display_name = "Orca Relay: far-cell phone accept latency"
|
||||
combiner = "OR"
|
||||
enabled = true
|
||||
notification_channels = var.relay_alert_notification_channels
|
||||
|
||||
conditions {
|
||||
display_name = "Phone accept p95 above 2 s for 15 minutes"
|
||||
|
||||
condition_monitoring_query_language {
|
||||
# percentile(..., 50) over the window, not max: the published value is already a p95, so the
|
||||
# median of the interval p95s reads as sustained slowness instead of one bad 30-second flush.
|
||||
query = <<-EOT
|
||||
{
|
||||
fetch gce_instance::logging.googleapis.com/user/orca_relay_client_accept_total_ms_p95
|
||||
| align delta(15m) | every 15m
|
||||
| group_by [metric.cell_id], [accept_p95_ms: percentile(value.orca_relay_client_accept_total_ms_p95, 50)]
|
||||
;
|
||||
fetch gce_instance::logging.googleapis.com/user/orca_relay_client_accepts_completed
|
||||
| align delta(15m) | every 15m
|
||||
| group_by [metric.cell_id], [accepts: sum(value.orca_relay_client_accepts_completed)]
|
||||
}
|
||||
| join
|
||||
| condition accept_p95_ms > 2000 'ms' && accepts >= 20 '1'
|
||||
EOT
|
||||
duration = "0s"
|
||||
|
||||
trigger {
|
||||
count = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
documentation {
|
||||
content = "Phones on this cell are taking over two seconds to reach relay-hello. Measured separation: an in-region accept completes in 0.3-0.6 s and a cross-Pacific one in 5-10 s, so 2 s sits well outside in-region noise and well below the far-cell floor. The 20-accept floor over 15 minutes keeps a single slow accept on a quiet cell from paging. Check which regions the cell's hosts are actually in before touching capacity: the 2026-08 cause was desktops requesting the wrong region, not a slow cell. Read the per-stage `orca_relay_client_accept_*_ms_p95` metrics to separate distance from assignment, credential, or attach work."
|
||||
mime_type = "text/markdown"
|
||||
}
|
||||
|
||||
depends_on = [google_logging_metric.relay_snapshot]
|
||||
}
|
||||
|
||||
resource "google_monitoring_alert_policy" "relay_cell_control_rtt" {
|
||||
project = var.project_id
|
||||
display_name = "Orca Relay: cell control round trip"
|
||||
combiner = "OR"
|
||||
enabled = true
|
||||
notification_channels = var.relay_alert_notification_channels
|
||||
|
||||
conditions {
|
||||
display_name = "Control ping p50 above 150 ms for an hour"
|
||||
|
||||
condition_monitoring_query_language {
|
||||
# p50 only. The desktop echoes the pong on its main thread, so the published p95 and max
|
||||
# track renderer stalls, not distance; the median is the only column that reads as distance.
|
||||
query = <<-EOT
|
||||
{
|
||||
fetch gce_instance::logging.googleapis.com/user/orca_relay_control_rtt_ms_p50
|
||||
| align delta(1h) | every 1h
|
||||
| group_by [metric.cell_id], [control_rtt_p50_ms: percentile(value.orca_relay_control_rtt_ms_p50, 50)]
|
||||
;
|
||||
fetch gce_instance::logging.googleapis.com/user/orca_relay_control_rtt_samples
|
||||
| align delta(1h) | every 1h
|
||||
| group_by [metric.cell_id], [samples: sum(value.orca_relay_control_rtt_samples)]
|
||||
}
|
||||
| join
|
||||
| condition control_rtt_p50_ms > 150 'ms' && samples >= 500 '1'
|
||||
EOT
|
||||
duration = "0s"
|
||||
|
||||
trigger {
|
||||
count = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
documentation {
|
||||
content = "The median desktop on this cell is more than 150 ms away from it, which is a mis-homed population rather than a cell fault: an in-region control ping is tens of milliseconds and a US desktop on an asia-east2 cell is 200 ms or more. This is the signal that was missing while roughly 226 of 332 hosts on the asia cells were non-APAC for weeks in 2026-08. Confirm with the assignment table which regions those hosts requested, then rehome; do not restart or drain the cell on this alert alone. The 500-sample floor is about two continuously connected hosts at the 15-second control ping, so a nearly idle cell cannot alert on one desktop. Tuning risk: EU desktops on us-central1 sit at 100-130 ms, so a cell whose population is mostly European can approach 150 ms while correctly homed. Check where the hosts are before treating a first breach as mis-homing, and raise the bar only with that evidence."
|
||||
mime_type = "text/markdown"
|
||||
}
|
||||
|
||||
depends_on = [google_logging_metric.relay_snapshot]
|
||||
}
|
||||
|
||||
resource "google_monitoring_alert_policy" "relay_region_hint_skew" {
|
||||
project = var.project_id
|
||||
display_name = "Orca Relay: region hint skew"
|
||||
combiner = "OR"
|
||||
enabled = true
|
||||
notification_channels = var.relay_alert_notification_channels
|
||||
|
||||
conditions {
|
||||
display_name = "asia-east2 hint share above 2x its placement share for an hour"
|
||||
|
||||
condition_monitoring_query_language {
|
||||
query = local.relay_region_hint_skew_query
|
||||
duration = "0s"
|
||||
|
||||
trigger {
|
||||
count = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
documentation {
|
||||
content = "Desktops are asking the director for asia-east2 far more often than the director actually places them there, which is what silently homed US desktops on asia cells through 2026-08. The alert compares two shares of the same hour and never an absolute share, because an absolute bar is wrong at both ends: measured over twelve hours on 2026-09-07, while the desktop region probe was still mis-picking, asia-east2 was 33.8% of the 33,800 hinted requests but only 7.9% of the 45,364 assignments, and once the probe is fixed the genuine APAC share will climb past any fixed bar that would have caught this. Divergence was 4.27x with a 25.9-point gap, so the 2x and 15-point bars sit well inside the broken state and well outside a healthy one. `unhinted` requests are excluded from the denominator: they were 27% of all requests, and a client change that always sends a hint would move this number without any behaviour changing. Expect this to stay lit until the mis-homed backlog is rehomed, because sticky assignment never re-consults the hint, so a desktop already on an asia cell keeps being placed there no matter what it now asks for. Investigate the desktop region probe first, not relay placement."
|
||||
mime_type = "text/markdown"
|
||||
}
|
||||
|
||||
depends_on = [google_logging_metric.relay_snapshot]
|
||||
}
|
||||
|
||||
# Why: the four signals that had to be assembled by hand during the 2026-09-04 incident.
|
||||
resource "google_monitoring_dashboard" "relay_incident" {
|
||||
project = var.project_id
|
||||
|
||||
@@ -245,7 +245,7 @@ variable "relay_regional_placement_enabled" {
|
||||
|
||||
variable "relay_region_rehome_source_cell_ids" {
|
||||
type = set(string)
|
||||
description = "Reviewed US Relay cells allowed to advertise and accept the regional rehome source protocol."
|
||||
description = "Reviewed Relay cells, in any configured region, allowed to advertise and accept the regional rehome source protocol."
|
||||
default = []
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@
|
||||
"load:relay:model": "node dev/scripts/run-relay-load-model.mjs",
|
||||
"load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs",
|
||||
"ops:relay": "pnpm --filter @orca-cloud/relay-ops dev",
|
||||
"pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-admission-workflow.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs",
|
||||
"pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-admission-workflow.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs",
|
||||
"test": "pnpm -r test && node --test dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-regional-rehome-workflow.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs",
|
||||
"typecheck": "pnpm -r typecheck"
|
||||
},
|
||||
|
||||
@@ -6,7 +6,10 @@ import {
|
||||
HostChallengeSchema,
|
||||
HostDataAuthSchema,
|
||||
HostHelloAckSchema,
|
||||
HostHelloSchema
|
||||
HostHelloSchema,
|
||||
parseRelayHostCapabilities,
|
||||
RELAY_HOST_CAPABILITIES_HEADER,
|
||||
RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS
|
||||
} from './control-messages.js'
|
||||
import {
|
||||
DeviceCredentialInstallSchema,
|
||||
@@ -345,3 +348,47 @@ describe('relay protocol contract', () => {
|
||||
).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('pending connection details capability', () => {
|
||||
it('reads a pending entry with or without the stated kind and device', () => {
|
||||
const ack = {
|
||||
v: 1 as const,
|
||||
generation: 3,
|
||||
controlResumeSecret: 'R'.repeat(43),
|
||||
leaseExpiresAt: 1_800_000_000_000,
|
||||
activeConnIds: []
|
||||
}
|
||||
const identifiers = { connId: 'conn-1', connTicket: 'T'.repeat(43) }
|
||||
expect(HostHelloAckSchema.safeParse({ ...ack, pendingConns: [identifiers] }).success).toBe(true)
|
||||
expect(
|
||||
HostHelloAckSchema.safeParse({
|
||||
...ack,
|
||||
pendingConns: [{ ...identifiers, kind: 'resume', relayDeviceId: 'device-1' }]
|
||||
}).success
|
||||
).toBe(true)
|
||||
// Still strict otherwise: an unannounced key must not slip through as data.
|
||||
expect(
|
||||
HostHelloAckSchema.safeParse({
|
||||
...ack,
|
||||
pendingConns: [{ ...identifiers, reservationId: 'injected' }]
|
||||
}).success
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('pins the header and token the desktop mirrors by hand', () => {
|
||||
// The desktop cannot import this package; drift silently disables the
|
||||
// feature, so both literals are asserted on each side.
|
||||
expect(RELAY_HOST_CAPABILITIES_HEADER).toBe('x-orca-host-capabilities')
|
||||
expect(RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS).toBe('pending-conn-details')
|
||||
})
|
||||
|
||||
it('reads the advertised capabilities from a control upgrade header', () => {
|
||||
expect(
|
||||
parseRelayHostCapabilities(` ${RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS} , future-thing`)
|
||||
).toEqual(new Set([RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS, 'future-thing']))
|
||||
// A host that predates the header sends nothing; absence is never capable.
|
||||
expect(parseRelayHostCapabilities(undefined).size).toBe(0)
|
||||
expect(parseRelayHostCapabilities('').size).toBe(0)
|
||||
expect(parseRelayHostCapabilities('x'.repeat(65)).size).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -44,8 +44,35 @@ export const HostChallengeAckSchema = z
|
||||
.object({ challengeId: OpaqueIdSchema, proofB64: Base6432ByteSchema })
|
||||
.strict()
|
||||
|
||||
// Advertised on the control upgrade rather than in host-hello: HostHelloSchema
|
||||
// is strict, so a new hello key is refused by every already-deployed cell.
|
||||
export const RELAY_HOST_CAPABILITIES_HEADER = 'x-orca-host-capabilities'
|
||||
// The host accepts kind/relayDeviceId on a pendingConns entry. A host that does
|
||||
// not advertise this parses those entries strictly and would drop the whole ack.
|
||||
export const RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS = 'pending-conn-details'
|
||||
|
||||
export function parseRelayHostCapabilities(
|
||||
header: string | string[] | undefined
|
||||
): ReadonlySet<string> {
|
||||
const raw = Array.isArray(header) ? header.join(',') : (header ?? '')
|
||||
return new Set(
|
||||
raw
|
||||
.split(',')
|
||||
.map((token) => token.trim())
|
||||
.filter((token) => token.length > 0 && token.length <= 64)
|
||||
.slice(0, 16)
|
||||
)
|
||||
}
|
||||
|
||||
// kind/relayDeviceId are optional so an entry stays readable by a host that
|
||||
// predates them; the cell only emits them to a host that advertised support.
|
||||
const PendingConnectionSchema = z
|
||||
.object({ connId: OpaqueIdSchema, connTicket: Base64Url32ByteSchema })
|
||||
.object({
|
||||
connId: OpaqueIdSchema,
|
||||
connTicket: Base64Url32ByteSchema,
|
||||
kind: ConnectionKindSchema.optional(),
|
||||
relayDeviceId: OpaqueIdSchema.optional()
|
||||
})
|
||||
.strict()
|
||||
|
||||
export const HostHelloAckSchema = z
|
||||
|
||||
@@ -8,6 +8,15 @@ export type RelayRegion = z.infer<typeof RelayRegionSchema>
|
||||
|
||||
export const RELAY_DEFAULT_REGION: RelayRegion = 'us-central1'
|
||||
|
||||
// Field-name segment for the flat per-region runtime counters, spelled out rather than derived so
|
||||
// the Terraform side can hold the same literal and a test can compare the two. `satisfies` makes a
|
||||
// new region a compile error here, which is the point: a region with no segment would silently
|
||||
// drop out of the region-skew alert's denominators.
|
||||
export const RELAY_REGION_METRIC_SEGMENTS = {
|
||||
'us-central1': 'UsCentral1',
|
||||
'asia-east2': 'AsiaEast2'
|
||||
} as const satisfies Record<RelayRegion, string>
|
||||
|
||||
const RelayProbeOriginSchema = z.string().url().max(2_048).refine(isCanonicalHttpsOrigin)
|
||||
|
||||
export const RelayRegionCatalogResponseSchema = z
|
||||
|
||||
@@ -603,7 +603,7 @@ index 7b4b9e1f990fbf95b51528bb56dc9717f5b87532..2ae787c5bd4f3eba470584dc658a01a5
|
||||
}
|
||||
#endif
|
||||
diff --git a/src/win/conpty.cc b/src/win/conpty.cc
|
||||
index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c97209248e 100644
|
||||
index 7b286d3d644c26141df516929703aa6e129df4b2..4b06d18576c807c3d1181a7bd714140c6678cf86 100644
|
||||
--- a/src/win/conpty.cc
|
||||
+++ b/src/win/conpty.cc
|
||||
@@ -18,6 +18,7 @@
|
||||
@@ -614,7 +614,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
|
||||
#include <vector>
|
||||
#include <Windows.h>
|
||||
#include <strsafe.h>
|
||||
@@ -44,12 +45,39 @@ struct pty_baton {
|
||||
@@ -44,12 +45,40 @@ struct pty_baton {
|
||||
HANDLE hOut;
|
||||
HPCON hpc;
|
||||
|
||||
@@ -630,6 +630,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
|
||||
+ // refused to create or assign one (an outer job without breakaway rights),
|
||||
+ // in which case callers fall back to their pre-job behaviour.
|
||||
+ HANDLE hJob = nullptr;
|
||||
+ bool allowJobBreakaway = true;
|
||||
+
|
||||
+ // Orca: teardown needs BOTH the shell's death and an explicit kill() before
|
||||
+ // the baton can be freed, so each side records that it has run. Whichever
|
||||
@@ -655,7 +656,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
|
||||
static volatile LONG ptyCounter;
|
||||
|
||||
static pty_baton* get_pty_baton(int id) {
|
||||
@@ -102,8 +130,31 @@ void SetupExitCallback(Napi::Env env, Napi::Function cb, pty_baton* baton) {
|
||||
@@ -102,8 +131,31 @@ void SetupExitCallback(Napi::Env env, Napi::Function cb, pty_baton* baton) {
|
||||
// Get process exit code.
|
||||
GetExitCodeProcess(baton->hShell, (LPDWORD)(&exit_event->exit_code));
|
||||
// Clean up handles
|
||||
@@ -689,7 +690,36 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
|
||||
|
||||
auto status = tsfn.BlockingCall(exit_event, callback); // In main thread
|
||||
switch (status) {
|
||||
@@ -409,6 +460,15 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
|
||||
@@ -242,6 +294,20 @@
|
||||
return HRESULT_FROM_WIN32(GetLastError());
|
||||
}
|
||||
|
||||
+// Cygwin and MSYS request breakaway for every child whenever the job allows it,
|
||||
+// so their shells need one that does not. The runtime DLL on the exe's search
|
||||
+// path is the signal; Git for Windows ships bash.exe in bin\ beside usr\bin\.
|
||||
+static bool usesCygwinRuntime(const std::wstring& shellpath) {
|
||||
+ const size_t separator = shellpath.find_last_of(L"\\/");
|
||||
+ if (separator == std::wstring::npos) return false;
|
||||
+ const std::wstring directory = shellpath.substr(0, separator + 1);
|
||||
+ for (const wchar_t* dll : {L"msys-2.0.dll", L"cygwin1.dll"}) {
|
||||
+ if (path_util::file_exists(directory + dll) ||
|
||||
+ path_util::file_exists(directory + L"..\\usr\\bin\\" + dll)) return true;
|
||||
+ }
|
||||
+ return false;
|
||||
+}
|
||||
+
|
||||
static Napi::Value PtyStartProcess(const Napi::CallbackInfo& info) {
|
||||
Napi::Env env(info.Env());
|
||||
Napi::HandleScope scope(env);
|
||||
@@ -303,6 +369,7 @@
|
||||
marshal.Set("pty", Napi::Number::New(env, ptyId));
|
||||
ptyHandles.emplace_back(
|
||||
std::make_unique<pty_baton>(ptyId, hIn, hOut, hpc));
|
||||
+ ptyHandles.back()->allowJobBreakaway = !usesCygwinRuntime(shellpath);
|
||||
} else {
|
||||
throw Napi::Error::New(env, "Cannot launch conpty");
|
||||
}
|
||||
@@ -409,6 +476,15 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
|
||||
throw errorWithCode(info, "UpdateProcThreadAttribute failed");
|
||||
}
|
||||
|
||||
@@ -705,7 +735,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
|
||||
PROCESS_INFORMATION piClient{};
|
||||
fSuccess = !!CreateProcessW(
|
||||
nullptr,
|
||||
@@ -416,7 +476,10 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
|
||||
@@ -416,7 +492,10 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
|
||||
nullptr, // lpProcessAttributes
|
||||
nullptr, // lpThreadAttributes
|
||||
false, // bInheritHandles VERY IMPORTANT that this is false
|
||||
@@ -717,7 +747,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
|
||||
envArg, // lpEnvironment
|
||||
mutableCwd.get(), // lpCurrentDirectory
|
||||
&siEx.StartupInfo, // lpStartupInfo
|
||||
@@ -426,8 +489,47 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
|
||||
@@ -426,8 +505,48 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
|
||||
throw errorWithCode(info, "Cannot create process");
|
||||
}
|
||||
|
||||
@@ -735,13 +765,14 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
|
||||
+ // EXPLICIT teardown exact, not to redefine what a clean exit means.
|
||||
+ HANDLE hJob = CreateJobObjectW(nullptr, nullptr);
|
||||
+ if (hJob != nullptr) {
|
||||
+ // Why BREAKAWAY_OK and not a bare job: with no limits set, a child asking
|
||||
+ // for CREATE_BREAKAWAY_FROM_JOB is refused with ERROR_ACCESS_DENIED.
|
||||
+ // Installers, msiexec and some updater and service-control paths spawn that
|
||||
+ // way deliberately, so a bare job breaks them ONLY inside an Orca terminal.
|
||||
+ // With this flag a child has to ask, so ordinary descendants stay owned.
|
||||
+ // Native shells retain explicit breakaway for installers and updaters.
|
||||
+ // Cygwin/MSYS shells take it automatically for ordinary children whenever
|
||||
+ // this flag is present, so they get strict per-PTY membership instead.
|
||||
+ // Explicit breakaway requests inside such a pane are consequently denied;
|
||||
+ // ordinary backgrounding and clean shell exit remain supported.
|
||||
+ JOBOBJECT_EXTENDED_LIMIT_INFORMATION jobLimits{};
|
||||
+ jobLimits.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_BREAKAWAY_OK;
|
||||
+ jobLimits.BasicLimitInformation.LimitFlags =
|
||||
+ handle->allowJobBreakaway ? JOB_OBJECT_LIMIT_BREAKAWAY_OK : 0;
|
||||
+ if (!SetInformationJobObject(hJob, JobObjectExtendedLimitInformation, &jobLimits, sizeof(jobLimits)) ||
|
||||
+ !AssignProcessToJobObject(hJob, piClient.hProcess)) {
|
||||
+ // Why tolerate failure: an outer job without JOB_OBJECT_LIMIT_BREAKAWAY_OK
|
||||
@@ -767,7 +798,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
|
||||
if (useConptyDll && fLoadedDll)
|
||||
{
|
||||
PFNRELEASEPSEUDOCONSOLE const pfnReleasePseudoConsole = (PFNRELEASEPSEUDOCONSOLE)GetProcAddress(
|
||||
@@ -440,6 +542,8 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
|
||||
@@ -440,6 +559,8 @@ static Napi::Value PtyConnect(const Napi::CallbackInfo& info) {
|
||||
|
||||
// Update handle
|
||||
handle->hShell = piClient.hProcess;
|
||||
@@ -776,11 +807,16 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
|
||||
|
||||
// Close the thread handle to avoid resource leak
|
||||
CloseHandle(piClient.hThread);
|
||||
@@ -544,29 +648,215 @@ static Napi::Value PtyKill(const Napi::CallbackInfo& info) {
|
||||
@@ -544,27 +665,213 @@ static Napi::Value PtyKill(const Napi::CallbackInfo& info) {
|
||||
int id = info[0].As<Napi::Number>().Int32Value();
|
||||
const bool useConptyDll = info[1].As<Napi::Boolean>().Value();
|
||||
|
||||
- const pty_baton* handle = get_pty_baton(id);
|
||||
-
|
||||
- if (handle != nullptr) {
|
||||
- HANDLE hLibrary = LoadConptyDll(info, useConptyDll);
|
||||
- bool fLoadedDll = hLibrary != nullptr;
|
||||
- if (fLoadedDll)
|
||||
+ // Orca: resolve the DLL BEFORE touching any baton state, for the same reason
|
||||
+ // PtyConnect does it before creating anything. LoadConptyDll throws when
|
||||
+ // conpty.dll is missing, and a throw after consoleClosed was set would strand
|
||||
@@ -794,18 +830,7 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
|
||||
+ (HMODULE)hLibrary,
|
||||
+ useConptyDll ? "ConptyClosePseudoConsole" : "ClosePseudoConsole");
|
||||
+ }
|
||||
|
||||
- if (handle != nullptr) {
|
||||
- HANDLE hLibrary = LoadConptyDll(info, useConptyDll);
|
||||
- bool fLoadedDll = hLibrary != nullptr;
|
||||
- if (fLoadedDll)
|
||||
- {
|
||||
- PFNCLOSEPSEUDOCONSOLE const pfnClosePseudoConsole = (PFNCLOSEPSEUDOCONSOLE)GetProcAddress(
|
||||
- (HMODULE)hLibrary,
|
||||
- useConptyDll ? "ConptyClosePseudoConsole" : "ClosePseudoConsole");
|
||||
- if (pfnClosePseudoConsole)
|
||||
- {
|
||||
- pfnClosePseudoConsole(handle->hpc);
|
||||
+
|
||||
+ // Orca: the baton now outlives the shell, so this runs on a self-exited pty
|
||||
+ // too -- that is the whole point. Take what we need under the lock: the
|
||||
+ // watcher thread nulls hShell the moment the shell dies, and TerminateProcess
|
||||
@@ -841,18 +866,26 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
|
||||
+ const bool removed = remove_pty_baton(id);
|
||||
+ assert(removed);
|
||||
+ (void)removed;
|
||||
}
|
||||
+ }
|
||||
+ // Else the shell is still running and the watcher frees the baton.
|
||||
}
|
||||
- if (useConptyDll) {
|
||||
- TerminateProcess(handle->hShell, 1);
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ // Why outside the lock: ClosePseudoConsole blocks until the conout side has
|
||||
+ // drained, and the watcher must be able to take the lock while it does.
|
||||
+ if (owed) {
|
||||
+ if (pfnClosePseudoConsole)
|
||||
+ {
|
||||
{
|
||||
- PFNCLOSEPSEUDOCONSOLE const pfnClosePseudoConsole = (PFNCLOSEPSEUDOCONSOLE)GetProcAddress(
|
||||
- (HMODULE)hLibrary,
|
||||
- useConptyDll ? "ConptyClosePseudoConsole" : "ClosePseudoConsole");
|
||||
- if (pfnClosePseudoConsole)
|
||||
- {
|
||||
- pfnClosePseudoConsole(handle->hpc);
|
||||
- }
|
||||
- }
|
||||
- if (useConptyDll) {
|
||||
- TerminateProcess(handle->hShell, 1);
|
||||
+ pfnClosePseudoConsole(hpc);
|
||||
+ }
|
||||
+ if (hShellDup != nullptr) {
|
||||
@@ -862,8 +895,8 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
|
||||
}
|
||||
|
||||
return env.Undefined();
|
||||
}
|
||||
|
||||
+}
|
||||
+
|
||||
+/**
|
||||
+ * Orca: confirm a baton really is the pty the caller means.
|
||||
+ *
|
||||
@@ -1001,12 +1034,10 @@ index 7b286d3d644c26141df516929703aa6e129df4b2..4aed260dd68e6a171dcfd349e9a7c5c9
|
||||
+ }
|
||||
+ hHostJob = job;
|
||||
+ return Napi::Boolean::New(env, true);
|
||||
+}
|
||||
+
|
||||
}
|
||||
|
||||
/**
|
||||
* Init
|
||||
*/
|
||||
@@ -577,6 +867,9 @@ Napi::Object init(Napi::Env env, Napi::Object exports) {
|
||||
@@ -577,6 +884,9 @@ Napi::Object init(Napi::Env env, Napi::Object exports) {
|
||||
exports.Set("resize", Napi::Function::New(env, PtyResize));
|
||||
exports.Set("clear", Napi::Function::New(env, PtyClear));
|
||||
exports.Set("kill", Napi::Function::New(env, PtyKill));
|
||||
|
||||
@@ -13325,9 +13325,7 @@
|
||||
},
|
||||
{
|
||||
"file": "src/main/runtime/orchestration/mailbox-pointer-stage.test.ts",
|
||||
"assertions": [
|
||||
"a refused pointer write drains a delivery parked behind its watermark"
|
||||
]
|
||||
"assertions": ["a refused pointer write drains a delivery parked behind its watermark"]
|
||||
},
|
||||
{
|
||||
"file": "src/main/providers/settled-pty-writer-census.test.ts",
|
||||
@@ -18549,27 +18547,13 @@
|
||||
"protection": "partial",
|
||||
"owner": "browser-runtime",
|
||||
"layer": "electron-packaged",
|
||||
"surfaces": [
|
||||
"paired browser placement"
|
||||
],
|
||||
"platforms": [
|
||||
"linux",
|
||||
"macos",
|
||||
"windows"
|
||||
],
|
||||
"providers": [
|
||||
"paired-runtime"
|
||||
],
|
||||
"coveredPlatforms": [
|
||||
"linux"
|
||||
],
|
||||
"coveredProviders": [
|
||||
"paired-runtime"
|
||||
],
|
||||
"surfaces": ["paired browser placement"],
|
||||
"platforms": ["linux", "macos", "windows"],
|
||||
"providers": ["paired-runtime"],
|
||||
"coveredPlatforms": ["linux"],
|
||||
"coveredProviders": ["paired-runtime"],
|
||||
"coverageNotes": "Published Linux 1.4.188 desktop against current source in both directions; scheduled weekly and manually runnable. No required PR check.",
|
||||
"motivatingLinks": [
|
||||
"https://github.com/stablyai/orca/actions/runs/34069063016"
|
||||
],
|
||||
"motivatingLinks": ["https://github.com/stablyai/orca/actions/runs/34069063016"],
|
||||
"invariant": "A paired client and host without client-hosted browser capabilities retain server-hosted browser placement across supported version skew.",
|
||||
"oracle": "Require both existing named browser placement scenarios to pass three times with one attempt, zero skips, zero failures, and no report errors.",
|
||||
"commands": [
|
||||
@@ -18593,9 +18577,7 @@
|
||||
},
|
||||
{
|
||||
"file": "config/scripts/verify-packaged-browser-participation.test.mjs",
|
||||
"assertions": [
|
||||
"reject missing, substituted, skipped and retried scenarios"
|
||||
]
|
||||
"assertions": ["reject missing, substituted, skipped and retried scenarios"]
|
||||
},
|
||||
{
|
||||
"file": "config/scripts/packaged-browser-lane-contract.test.mjs",
|
||||
@@ -18650,26 +18632,13 @@
|
||||
"protection": "partial",
|
||||
"owner": "terminal-input",
|
||||
"layer": "electron-native-ime-e2e",
|
||||
"surfaces": [
|
||||
"native Hangul composition",
|
||||
"Wayland terminal input"
|
||||
],
|
||||
"platforms": [
|
||||
"linux"
|
||||
],
|
||||
"providers": [
|
||||
"local"
|
||||
],
|
||||
"coveredPlatforms": [
|
||||
"linux"
|
||||
],
|
||||
"coveredProviders": [
|
||||
"local"
|
||||
],
|
||||
"surfaces": ["native Hangul composition", "Wayland terminal input"],
|
||||
"platforms": ["linux"],
|
||||
"providers": ["local"],
|
||||
"coveredPlatforms": ["linux"],
|
||||
"coveredProviders": ["local"],
|
||||
"coverageNotes": "Ubuntu 22.04 nested GNOME and IBus Hangul drive three complete native executions in GitHub Actions. GNOME owns IBus; daemon and CLI share its default config discovery path.",
|
||||
"motivatingLinks": [
|
||||
"https://github.com/stablyai/orca/pull/19174"
|
||||
],
|
||||
"motivatingLinks": ["https://github.com/stablyai/orca/pull/19174"],
|
||||
"invariant": "Typing d k 1 Return through native IBus Hangul delivers exactly 아1 followed by newline without missing, duplicate, or reordered characters.",
|
||||
"oracle": "Three executions each assert three exact UTF-8 PTY lines. Verify the exact Playwright title, zero skips/retries, each individual native composition receipt, and the nested launch Wayland flag.",
|
||||
"commands": [
|
||||
@@ -18685,15 +18654,11 @@
|
||||
"assertionRefs": [
|
||||
{
|
||||
"file": "tests/e2e/terminal-hangul-terminating-digit-native.spec.ts",
|
||||
"assertions": [
|
||||
"a digit typed right after a Hangul syllable reaches the pty"
|
||||
]
|
||||
"assertions": ["a digit typed right after a Hangul syllable reaches the pty"]
|
||||
},
|
||||
{
|
||||
"file": "config/scripts/terminal-ime-e2e-workflow.test.mjs",
|
||||
"assertions": [
|
||||
"runs native Wayland independently with CJK fonts and retained evidence"
|
||||
]
|
||||
"assertions": ["runs native Wayland independently with CJK fonts and retained evidence"]
|
||||
}
|
||||
],
|
||||
"evidenceRuns": [
|
||||
|
||||
@@ -38,6 +38,16 @@ const SUPPRESSED_REACT_DOCTOR_DIAGNOSTICS = new Map([
|
||||
new Set([
|
||||
'src/renderer/src/components/editor/combined-diff/review-controls/use-combined-diff-view-preferences.ts'
|
||||
])
|
||||
],
|
||||
[
|
||||
// The rule wants one named handle cleared by name. Both startup effects arm a variable number
|
||||
// of refresh timers, every one of them through addTimer into `timers`, which their cleanups
|
||||
// clear -- a shape the rule reports whether the handles live in an array, a Set, or a nested
|
||||
// helper. The finding predates this list; it surfaced when the effect body changed. This map
|
||||
// keys on file, not line, so the entry covers both effects in it; nothing else in the file
|
||||
// arms a timer, so widening it further is the only alternative, not a narrower option.
|
||||
'react-doctor(effect-needs-cleanup)',
|
||||
new Set(['mobile/src/session/use-mobile-session-startup.ts'])
|
||||
]
|
||||
])
|
||||
|
||||
|
||||
@@ -224,6 +224,7 @@ const WINDOWS_PACKAGE_TESTS = [
|
||||
'src/main/agent-hooks/windows-hook-payload-delivery.test.ts',
|
||||
'src/main/agent-hooks/windows-direct-cmd-hook-command.test.ts',
|
||||
'src/main/windows/windows-pty-job.win32.test.ts',
|
||||
'src/main/windows/windows-msys-job.win32.test.ts',
|
||||
'src/main/windows/windows-host-job.win32.test.ts',
|
||||
'src/main/windows/windows-process-tree-command-line-patch.test.ts',
|
||||
'src/main/windows/windows-process-table-native-addon.win32.test.ts',
|
||||
|
||||
@@ -173,6 +173,28 @@ describe('rebuild-native-deps patched node-pty rebuild', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('refuses a Windows rebuild when the process creation-time patch is missing', () => {
|
||||
const projectDir = mkTempProject()
|
||||
|
||||
try {
|
||||
writeFakeUsableElectronPackage(projectDir, { platform: 'win32' })
|
||||
writeFakeElectronRebuild(projectDir)
|
||||
writeFakeNodePtyConptyPayload(projectDir, 'x64')
|
||||
writeFakeWindowsProcessTreeWithNodeAddonApi(projectDir, { creationTimePatchApplied: false })
|
||||
|
||||
const result = runRebuildScript(
|
||||
projectDir,
|
||||
{ npm_config_platform: 'win32', npm_config_arch: 'x64' },
|
||||
['--platform=win32', '--arch=x64', '--force']
|
||||
)
|
||||
|
||||
expect(result.status).not.toBe(0)
|
||||
expect(result.stderr).toContain('process creation-time patch')
|
||||
} finally {
|
||||
removeTreeSync(projectDir)
|
||||
}
|
||||
})
|
||||
|
||||
it('restores the ConPTY runtime payload after a Windows Electron rebuild', () => {
|
||||
const projectDir = mkTempProject()
|
||||
|
||||
|
||||
@@ -374,13 +374,18 @@ export function writeFakeWindowsProcessTree(projectDir) {
|
||||
|
||||
export function writeFakeWindowsProcessTreeWithNodeAddonApi(
|
||||
projectDir,
|
||||
{ commandLinePatchApplied = true } = {}
|
||||
{ commandLinePatchApplied = true, creationTimePatchApplied = true } = {}
|
||||
) {
|
||||
const processTreeDir = join(projectDir, 'node_modules', '@vscode', 'windows-process-tree')
|
||||
const nodeAddonApiDir = join(processTreeDir, 'node_modules', 'node-addon-api')
|
||||
mkdirSync(nodeAddonApiDir, { recursive: true })
|
||||
writeFileSync(join(processTreeDir, 'package.json'), '{"dependencies":{"node-addon-api":"*"}}\n')
|
||||
writeFileSync(join(processTreeDir, 'index.js'), 'module.exports = {}\n')
|
||||
writeFileSync(
|
||||
join(processTreeDir, 'index.js'),
|
||||
creationTimePatchApplied
|
||||
? 'exports.ProcessDataFlag = { None: 0, Memory: 1, CommandLine: 2, CreationTime: 4 }\n'
|
||||
: 'exports.ProcessDataFlag = { None: 0, Memory: 1, CommandLine: 2 }\n'
|
||||
)
|
||||
mkdirSync(join(processTreeDir, 'src'), { recursive: true })
|
||||
writeFileSync(
|
||||
join(processTreeDir, 'src', 'process_commandline.cc'),
|
||||
@@ -388,6 +393,36 @@ export function writeFakeWindowsProcessTreeWithNodeAddonApi(
|
||||
? '// kProcessCommandLineInformation = 60\n'
|
||||
: unpatchedWindowsProcessTreeCommandLineSource()
|
||||
)
|
||||
writeFileSync(
|
||||
join(processTreeDir, 'src', 'process.h'),
|
||||
creationTimePatchApplied
|
||||
? 'enum ProcessDataFlags { NONE = 0, MEMORY = 1, COMMANDLINE = 2, CREATIONTIME = 4 };\nULONGLONG creationTimeMs;\n'
|
||||
: 'enum ProcessDataFlags { NONE = 0, MEMORY = 1, COMMANDLINE = 2 };\n'
|
||||
)
|
||||
writeFileSync(
|
||||
join(processTreeDir, 'src', 'process.cc'),
|
||||
creationTimePatchApplied
|
||||
? 'GetProcessCreationTime(pinfo);\nGetProcessTimes(hProcess, &creationTime, &exitTime, &kernelTime, &userTime);\n'
|
||||
: 'GetProcessMemoryUsage(pinfo);\n'
|
||||
)
|
||||
writeFileSync(
|
||||
join(processTreeDir, 'src', 'process_worker.cc'),
|
||||
creationTimePatchApplied ? 'object.Set("creationTimeMs", process.creationTimeMs);\n' : '\n'
|
||||
)
|
||||
mkdirSync(join(processTreeDir, 'lib'), { recursive: true })
|
||||
writeFileSync(
|
||||
join(processTreeDir, 'lib', 'index.js'),
|
||||
creationTimePatchApplied ? 'exports.ProcessDataFlag["CreationTime"] = 4;\n' : '\n'
|
||||
)
|
||||
writeFileSync(
|
||||
join(processTreeDir, 'lib', 'index.ts'),
|
||||
creationTimePatchApplied ? 'export enum ProcessDataFlag { CreationTime = 4 }\n' : '\n'
|
||||
)
|
||||
mkdirSync(join(processTreeDir, 'typings'), { recursive: true })
|
||||
writeFileSync(
|
||||
join(processTreeDir, 'typings', 'windows-process-tree.d.ts'),
|
||||
creationTimePatchApplied ? 'creationTimeMs?: number\n' : '\n'
|
||||
)
|
||||
writeFileSync(join(nodeAddonApiDir, 'package.json'), '{"name":"node-addon-api"}\n')
|
||||
writeFileSync(join(nodeAddonApiDir, 'napi.h'), '// napi.h\n')
|
||||
writeFileSync(join(nodeAddonApiDir, 'napi-inl.h'), '// napi-inl.h\n')
|
||||
|
||||
@@ -33,6 +33,17 @@ export const WINDOWS_PROCESS_TREE_PATCH_PATH = join(
|
||||
/** Only the patched reader defines this; the upstream one walks the PEB. */
|
||||
const COMMAND_LINE_PATCH_MARKER = 'kProcessCommandLineInformation'
|
||||
|
||||
const CREATION_TIME_PATCH_MARKERS = [
|
||||
['src/process.h', 'CREATIONTIME = 4'],
|
||||
['src/process.h', 'ULONGLONG creationTimeMs'],
|
||||
['src/process.cc', 'GetProcessCreationTime(pinfo)'],
|
||||
['src/process.cc', 'GetProcessTimes(hProcess, &creationTime'],
|
||||
['src/process_worker.cc', 'object.Set("creationTimeMs"'],
|
||||
['lib/index.js', '["CreationTime"] = 4'],
|
||||
['lib/index.ts', 'CreationTime = 4'],
|
||||
['typings/windows-process-tree.d.ts', 'creationTimeMs?: number']
|
||||
]
|
||||
|
||||
export const WINDOWS_PROCESS_TREE_NODE_ADDON_API_HEADERS = [
|
||||
'napi.h',
|
||||
'napi-inl.h',
|
||||
@@ -83,6 +94,36 @@ export function inspectWindowsProcessTreeAddon(addonPath) {
|
||||
return readFileSync(addonPath).includes(FLAGGED_IMPORT) ? 'unpatched' : 'clean'
|
||||
}
|
||||
|
||||
export function assertWindowsProcessTreeCreationTimePatch(
|
||||
packageDir = WINDOWS_PROCESS_TREE_PACKAGE_DIR
|
||||
) {
|
||||
for (const [relativePath, expected] of CREATION_TIME_PATCH_MARKERS) {
|
||||
const filePath = join(packageDir, relativePath)
|
||||
if (!existsSync(filePath)) {
|
||||
throw new Error(
|
||||
`${filePath} is missing, so the process creation-time patch cannot be verified. ` +
|
||||
'Run pnpm install.'
|
||||
)
|
||||
}
|
||||
if (!readFileSync(filePath, 'utf8').includes(expected)) {
|
||||
throw new Error(
|
||||
`${relativePath} does not contain the process creation-time patch (${expected}). ` +
|
||||
'Run pnpm install.'
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function assertWindowsProcessTreeRuntimeCreationTime(windowsProcessTree) {
|
||||
if (windowsProcessTree?.ProcessDataFlag?.CreationTime !== 4) {
|
||||
throw new Error(
|
||||
'@vscode/windows-process-tree does not expose ProcessDataFlag.CreationTime, so native ' +
|
||||
'Windows structured agent-session process ownership cannot be PID-reuse safe. Rebuild it ' +
|
||||
'(pnpm run rebuild:electron) rather than using the published prebuild.'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuse to compile or load the upstream command-line reader.
|
||||
*
|
||||
@@ -159,6 +200,7 @@ export function ensureWindowsProcessTreeCommandLinePatch(
|
||||
rmSync(windowsProcessTreeAddonPath(packageDir), { force: true })
|
||||
repaired = true
|
||||
}
|
||||
assertWindowsProcessTreeCreationTimePatch(packageDir)
|
||||
|
||||
return repaired
|
||||
}
|
||||
|
||||
@@ -12,12 +12,15 @@ import { tmpdir } from 'node:os'
|
||||
import { join, resolve } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import {
|
||||
assertWindowsProcessTreeCreationTimePatch,
|
||||
assertWindowsProcessTreeRuntimeCreationTime,
|
||||
inspectWindowsProcessTreeAddon,
|
||||
nodeGypRebuildInvocation,
|
||||
stageWindowsProcessTreeNodeAddonApiHeaders,
|
||||
WINDOWS_PROCESS_TREE_NODE_ADDON_API_HEADERS,
|
||||
WINDOWS_PROCESS_TREE_PACKAGE_DIR
|
||||
} from './windows-process-tree-gyp-rebuild.mjs'
|
||||
import { writeFakeWindowsProcessTreeWithNodeAddonApi } from './rebuild-native-deps-test-fixtures.mjs'
|
||||
|
||||
describe('windows-process-tree node-gyp rebuild', () => {
|
||||
it("resolves node-addon-api's gyp target from the rebuild cwd", () => {
|
||||
@@ -97,3 +100,47 @@ describe('inspecting a compiled windows-process-tree addon', () => {
|
||||
expect(inspectWindowsProcessTreeAddon(staged)).toBe('unpatched')
|
||||
})
|
||||
})
|
||||
|
||||
describe('windows-process-tree CreationTime patch assertion', () => {
|
||||
let dir
|
||||
|
||||
beforeEach(() => {
|
||||
dir = mkdtempSync(join(tmpdir(), 'orca-windows-process-tree-creation-time-'))
|
||||
})
|
||||
afterEach(() => {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('accepts a package whose source and JS surfaces expose process creation time', () => {
|
||||
writeFakeWindowsProcessTreeWithNodeAddonApi(dir)
|
||||
|
||||
expect(() =>
|
||||
assertWindowsProcessTreeCreationTimePatch(
|
||||
join(dir, 'node_modules', '@vscode', 'windows-process-tree')
|
||||
)
|
||||
).not.toThrow()
|
||||
})
|
||||
|
||||
it('rejects a package missing the process creation-time patch', () => {
|
||||
writeFakeWindowsProcessTreeWithNodeAddonApi(dir, { creationTimePatchApplied: false })
|
||||
|
||||
expect(() =>
|
||||
assertWindowsProcessTreeCreationTimePatch(
|
||||
join(dir, 'node_modules', '@vscode', 'windows-process-tree')
|
||||
)
|
||||
).toThrow('process creation-time patch')
|
||||
})
|
||||
|
||||
it('requires the runtime ProcessDataFlag.CreationTime enum', () => {
|
||||
expect(() =>
|
||||
assertWindowsProcessTreeRuntimeCreationTime({
|
||||
ProcessDataFlag: { None: 0, Memory: 1, CommandLine: 2, CreationTime: 4 }
|
||||
})
|
||||
).not.toThrow()
|
||||
expect(() =>
|
||||
assertWindowsProcessTreeRuntimeCreationTime({
|
||||
ProcessDataFlag: { None: 0, Memory: 1, CommandLine: 2 }
|
||||
})
|
||||
).toThrow('ProcessDataFlag.CreationTime')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -32,6 +32,7 @@
|
||||
"../src/main/codex/codex-app-server-capability-cache.ts",
|
||||
"../src/main/codex/codex-app-server-capability-signal.ts",
|
||||
"../src/main/codex/codex-app-server-client.ts",
|
||||
"../src/main/codex/codex-app-server-process-tree-kill.ts",
|
||||
"../src/main/codex/codex-app-server-record-reader.ts",
|
||||
"../src/main/codex/codex-app-server-session.ts",
|
||||
"../src/main/codex/codex-config-mirror.ts",
|
||||
|
||||
@@ -2,8 +2,27 @@
|
||||
|
||||
Orca selects a Relay region in the Electron main process before requesting a new assignment. The
|
||||
director publishes an allowlisted region catalog containing only HTTPS cell subdomains of that
|
||||
director; Orca takes three bounded `/health` latency samples per region and caches the stable choice
|
||||
for 24 hours. A cached region changes only when the alternative is materially faster.
|
||||
director. Orca discards one warm-up `/health` request per probe origin — a cold request pays TCP and
|
||||
TLS setup that can exceed the round trip it measures — then takes three bounded samples and compares
|
||||
regions by their minimum. A wide spread still rejects a region, but only a genuinely flapping one.
|
||||
The stable choice is cached for 24 hours, and a cached region changes only when the alternative is
|
||||
materially faster.
|
||||
|
||||
A region wins only against a measured competitor. If any region in the catalog is rejected or cannot
|
||||
be measured, Orca sends no hint rather than selecting the sole survivor. Sending no hint is not
|
||||
neutral placement: the director assigns `preferredRegion ?? RELAY_DEFAULT_REGION`, and the default
|
||||
is `us-central1`. So an `asia-east2` user whose `us-central1` probe fails or flaps once is placed in
|
||||
`us-central1` for that refresh. That trade is accepted because the relay database is
|
||||
`us-central1`-only, and it is bounded: the withheld hint is cached for one hour, not the 24 hours a
|
||||
chosen region gets, so the next hour re-measures. An origin that fails its warm-up probe is dropped
|
||||
before the sampling rounds, so an unreachable region costs one probe timeout rather than four.
|
||||
|
||||
After a control socket registers, Orca probes the cell it actually landed on, once per cell URL per
|
||||
process. The cache is deleted only when it names a region other than the best measured one and the
|
||||
assigned cell is more than three times slower than that region — a far cell under a cache that still
|
||||
names the best region means the director declined the hint, and re-measuring would return the same
|
||||
answer. Self-heal skips an absent, expired, or no-hint cache, and never runs under
|
||||
`ORCA_RELAY_REGION_OVERRIDE`.
|
||||
|
||||
The assignment request sends only `preferredRegion`. It does not send latency, IP address, country,
|
||||
pairing data, or credentials. Catalog, probe, and cache failures fall back to an assignment without
|
||||
|
||||
@@ -31,12 +31,12 @@ administrators can do about it.
|
||||
|
||||
Four independent evidence clusters, from six incidents:
|
||||
|
||||
| Cluster | Incidents | Evidence |
|
||||
| ----------------- | --------- | -------------------------------------------------------------------------------------------------------------------- |
|
||||
| **Update** | A, B, C | `orca-windows-setup.exe` → `old-uninstaller.exe`, `Uninstall Orca.exe` (electron-builder generates these; they are in no repo file) |
|
||||
| Cluster | Incidents | Evidence |
|
||||
| ----------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| **Update** | A, B, C | `orca-windows-setup.exe` → `old-uninstaller.exe`, `Uninstall Orca.exe` (electron-builder generates these; they are in no repo file) |
|
||||
| **Spawn** | all six | `Orca.exe` → `orca-terminal-daemon.exe` → `powershell.exe` / `pwsh.exe` / `cmd.exe` / `reg.exe` → `claude.exe`, `gh.exe`, `codex.cmd` |
|
||||
| **Process table** | D | "suspicious memory activity" — `OpenProcess` plus a PEB read against every process on a repeating cadence |
|
||||
| **Computer use** | E, F | `runtime.ps1`, `computer-sidecar.js`, many `operation.json`, a burst of ~10 short-lived `powershell.exe` |
|
||||
| **Process table** | D | "suspicious memory activity" — `OpenProcess` plus a PEB read against every process on a repeating cadence |
|
||||
| **Computer use** | E, F | `runtime.ps1`, `computer-sidecar.js`, many `operation.json`, a burst of ~10 short-lived `powershell.exe` |
|
||||
|
||||
Incident E is the one to look at hardest: 5 alerts, 37 evidence items, ATT&CK
|
||||
**Execution + Collection**, and a description reading _"Screenshots were taken
|
||||
@@ -143,11 +143,11 @@ PEB fallback to reinstate it — a hooked `ntdll` answering
|
||||
`STATUS_INVALID_INFO_CLASS` for one target would have flipped a process-wide,
|
||||
one-way switch back to `PROCESS_VM_READ` on exactly the machines this exists for.
|
||||
|
||||
Because the property is the *absence* of an import, it is checkable on the
|
||||
Because the property is the _absence_ of an import, it is checkable on the
|
||||
artifact rather than the source: `inspectWindowsProcessTreeAddon()` answers
|
||||
`clean` / `unpatched` / `missing`, and the rebuild, `ensure-native-runtime.mjs`,
|
||||
the relay build and `loadWindowsProcessTree()` all key on it. That check is load-
|
||||
bearing because the published tarball ships a *loadable* prebuilt built from
|
||||
bearing because the published tarball ships a _loadable_ prebuilt built from
|
||||
unpatched source, so "it required cleanly" is not evidence.
|
||||
|
||||
What to declare to administrators is now one
|
||||
@@ -180,7 +180,7 @@ Three sites are named in the incident analysis:
|
||||
`src/shared/setup-agent-sequencing.ts`,
|
||||
`src/shared/windows-cmd-runner-delayed-launch.ts` and
|
||||
`src/shared/windows-interactive-login-spawn.ts` each dropped
|
||||
`-ExecutionPolicy Bypass` as a measured no-op: the policy gates script *files*,
|
||||
`-ExecutionPolicy Bypass` as a measured no-op: the policy gates script _files_,
|
||||
never `-EncodedCommand`. Where the bypass was load-bearing it moved in-payload as
|
||||
a process-scope `Set-ExecutionPolicy` (`setup-agent-sequencing.ts`), which is the
|
||||
pattern to copy rather than restoring the switch — the switch loses to a GPO
|
||||
@@ -192,7 +192,7 @@ What remains is `-EncodedCommand` without the bypass: the PTY bootstraps
|
||||
(`src/main/agent-hooks/windows-powershell-hook-launcher.ts` and its callers
|
||||
`src/main/agent-hooks/runtime-home-hook-command.ts`,
|
||||
`src/main/agent-hooks/installer-utils.ts`, and `src/main/claude/hook-settings.ts`
|
||||
— that last one only as a *fallback* since #18875, see below),
|
||||
— that last one only as a _fallback_ since #18875, see below),
|
||||
`src/main/runtime/windows-default-route-interfaces.ts`,
|
||||
`src/main/runtime/orchestration/setup-completion-signal.ts`,
|
||||
`src/shared/hermes-startup-query.ts`, and the four ex-bypass sites above.
|
||||
@@ -224,7 +224,7 @@ denies the analyser the payload it would otherwise clear.
|
||||
The hook launcher is prior art worth knowing about. #16003 measured, on a
|
||||
reporting Kaspersky host, that `-WindowStyle Hidden` paired with
|
||||
`-EncodedCommand` was denied at `CreateProcess` with exit 126 regardless of
|
||||
payload — `exit 0` was denied too. The fix was to stop *spelling* the flags:
|
||||
payload — `exit 0` was denied too. The fix was to stop _spelling_ the flags:
|
||||
`WINDOWS_POWERSHELL_HOOK_SWITCHES` is now just `-NoProfile`, and separately, in
|
||||
#16576, the execution policy bypass moved in-payload as a process-scope
|
||||
`Set-ExecutionPolicy` — a real command-line signal reduction, though #16003's
|
||||
@@ -247,7 +247,7 @@ a quoted token, each `%` is broken with `"^%"`.
|
||||
|
||||
The escaping is not decorative. Measured on Windows 11 against a real `.cmd`
|
||||
shim, `["a b", 'c"d', "e%F%g", "h&i", "j^k"]` came back as `["a b", 'c"d',
|
||||
"e^%F^%g", "h"]` — the `&` truncated the argument *and* ran the remainder as a
|
||||
"e^%F^%g", "h"]` — the `&` truncated the argument _and_ ran the remainder as a
|
||||
command.
|
||||
|
||||
**How an EDR reads it:** caret escaping is the canonical obfuscation marker in
|
||||
@@ -259,7 +259,7 @@ obfuscated-command-line detector is tuned on.
|
||||
|
||||
`Orca.exe` → the relocated daemon host (`orca-terminal-daemon.exe` in the builds
|
||||
these incidents cover, `Orca.exe` since) → a shell → an agent CLI is what a
|
||||
terminal multiplexer for coding agents *is*. `reg.exe` appears from
|
||||
terminal multiplexer for coding agents _is_. `reg.exe` appears from
|
||||
`src/main/win32-utils.ts`,
|
||||
`src/main/agent-hooks/managed-hook-owner-identity.ts` and
|
||||
`src/relay/pty-shell-utils.ts` (reading the OpenSSH `DefaultShell`).
|
||||
@@ -267,7 +267,7 @@ terminal multiplexer for coding agents *is*. `reg.exe` appears from
|
||||
Nothing here is avoidable in principle. What is controllable is depth and
|
||||
breadth: every interpreter hop between Orca and the thing the user asked for adds
|
||||
a scored edge, which is why the shipped doctrine of #15520 and #15595 is to
|
||||
*shorten the interpreter chain* rather than to hide a window.
|
||||
_shorten the interpreter chain_ rather than to hide a window.
|
||||
|
||||
#18875 is a worked example of that doctrine. The Claude Code lifecycle hook was
|
||||
registered as `powershell.exe -NoProfile -EncodedCommand <...>` whose entire
|
||||
@@ -295,7 +295,7 @@ That last clause is the standing assumption of this change, and it is worth
|
||||
stating plainly because it is **not** measured. `||` parses in Git Bash, cmd.exe
|
||||
and pwsh, but not in Windows PowerShell 5.1, so the direct shape is correct for
|
||||
any host that is one of the first three. Claude Code itself is a Git Bash host on
|
||||
native Windows. What no one here has verified is which host a *compat consumer*
|
||||
native Windows. What no one here has verified is which host a _compat consumer_
|
||||
uses: cursor-agent and Devin import `~/.claude/settings.json` and run `command`
|
||||
through their own launcher (the managed `.cmd` carries a `DEVIN_PROJECT_DIR` skip
|
||||
for exactly that). If one of them spawns hook strings through Windows PowerShell
|
||||
@@ -318,12 +318,12 @@ then captures the screen through `Graphics.CopyFromScreen`.
|
||||
|
||||
That is four separate high-signal behaviours stacked in one process:
|
||||
|
||||
| Behaviour | How it is scored |
|
||||
| ----------------------------------------------- | ---------------------------------------------------- |
|
||||
| `Graphics.CopyFromScreen` | **MITRE T1113**, screen capture — Collection tactic |
|
||||
| `SendInput` synthetic keyboard/mouse | input synthesis against other applications |
|
||||
| `Add-Type -TypeDefinition` on every operation | MSIL compiled at runtime; incident F's "suspicious MSIL code" |
|
||||
| One `powershell.exe` per operation | a burst of short-lived interpreters under one parent |
|
||||
| Behaviour | How it is scored |
|
||||
| --------------------------------------------- | ------------------------------------------------------------- |
|
||||
| `Graphics.CopyFromScreen` | **MITRE T1113**, screen capture — Collection tactic |
|
||||
| `SendInput` synthetic keyboard/mouse | input synthesis against other applications |
|
||||
| `Add-Type -TypeDefinition` on every operation | MSIL compiled at runtime; incident F's "suspicious MSIL code" |
|
||||
| One `powershell.exe` per operation | a burst of short-lived interpreters under one parent |
|
||||
|
||||
The bottom two rows are the two the incident text named directly, and they are
|
||||
also the two a persistent runtime host would remove: a long-lived helper compiles
|
||||
@@ -381,16 +381,16 @@ changed. Check the code before relying on it.
|
||||
|
||||
The checklist. On Windows, do not reach for:
|
||||
|
||||
| Don't | Instead |
|
||||
| ----------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `-ExecutionPolicy Bypass` on the command line | Set the policy in-payload at process scope, as `windows-powershell-hook-launcher.ts` does, or do not run a `.ps1` at all |
|
||||
| `-EncodedCommand` | A temp `.ps1` with an argument, or no PowerShell hop: prefer a native API or an existing Node path |
|
||||
| `cmd.exe /c` carrying escaped free text | Spawn the real target directly. `cmd.exe` is only unavoidable for `.cmd`/`.bat`; keep free text out of the line where you can |
|
||||
| Forking `powershell.exe` to read system state | The native reader — [`windows-process-enumeration.md`](./windows-process-enumeration.md) is the standing rule for the process table |
|
||||
| A process per operation in a loop | One long-lived helper with a request channel. A burst of short-lived interpreters under one parent is itself the signal |
|
||||
| `Add-Type -TypeDefinition` at runtime | A precompiled, signed assembly, or a native helper |
|
||||
| Copying our own image under a different name | Copy it verbatim — [`windows-daemon-host-relocation.md`](./windows-daemon-host-relocation.md) (done for the daemon host) |
|
||||
| Deriving a script runner from a UI preference | [`windows-setup-shell.md`](./windows-setup-shell.md) — the script declares its own interpreter |
|
||||
| Don't | Instead |
|
||||
| --------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `-ExecutionPolicy Bypass` on the command line | Set the policy in-payload at process scope, as `windows-powershell-hook-launcher.ts` does, or do not run a `.ps1` at all |
|
||||
| `-EncodedCommand` | A temp `.ps1` with an argument, or no PowerShell hop: prefer a native API or an existing Node path |
|
||||
| `cmd.exe /c` carrying escaped free text | Spawn the real target directly. `cmd.exe` is only unavoidable for `.cmd`/`.bat`; keep free text out of the line where you can |
|
||||
| Forking `powershell.exe` to read system state | The native reader — [`windows-process-enumeration.md`](./windows-process-enumeration.md) is the standing rule for the process table |
|
||||
| A process per operation in a loop | One long-lived helper with a request channel. A burst of short-lived interpreters under one parent is itself the signal |
|
||||
| `Add-Type -TypeDefinition` at runtime | A precompiled, signed assembly, or a native helper |
|
||||
| Copying our own image under a different name | Copy it verbatim — [`windows-daemon-host-relocation.md`](./windows-daemon-host-relocation.md) (done for the daemon host) |
|
||||
| Deriving a script runner from a UI preference | [`windows-setup-shell.md`](./windows-setup-shell.md) — the script declares its own interpreter |
|
||||
|
||||
Two framing rules that outlast the table:
|
||||
|
||||
@@ -407,7 +407,7 @@ Two framing rules that outlast the table:
|
||||
|
||||
This is the single most important operational point, and it is the one most
|
||||
commonly got wrong. The six incidents are **MDE EDR behavioural alerts**.
|
||||
Defender Antivirus path exclusions suppress *scan* detections; they do not
|
||||
Defender Antivirus path exclusions suppress _scan_ detections; they do not
|
||||
suppress EDR behavioural alerts the same way. Adding
|
||||
`%LOCALAPPDATA%\Programs\orca\` to the AV exclusion list and expecting the
|
||||
incidents to stop will not work.
|
||||
|
||||
@@ -64,10 +64,10 @@ identity scan opens nothing.
|
||||
So the module exposes two snapshots, and the row types differ so a cheap caller
|
||||
cannot read what its flag set did not pay for:
|
||||
|
||||
| reader | row type | flags | per-process handles |
|
||||
| ------------------------------------------ | ---------------------------- | --------------------------- | ------------------- |
|
||||
| `readWindowsProcessIdentityTable[Fresh]()` | `WindowsProcessIdentityRow` | `None \| CreationTime` | none |
|
||||
| `readWindowsProcessTable[Fresh]()` | `WindowsProcessRow` | `+ CommandLine` | one `OpenProcess` |
|
||||
| reader | row type | flags | per-process handles |
|
||||
| ------------------------------------------ | --------------------------- | ---------------------- | ------------------- |
|
||||
| `readWindowsProcessIdentityTable[Fresh]()` | `WindowsProcessIdentityRow` | `None \| CreationTime` | none |
|
||||
| `readWindowsProcessTable[Fresh]()` | `WindowsProcessRow` | `+ CommandLine` | one `OpenProcess` |
|
||||
|
||||
`Memory` is requested by neither. Nothing reads a working set off this table —
|
||||
`windows-process-resource-collector.ts` runs its own sweep because it needs
|
||||
@@ -103,7 +103,7 @@ only under concurrency.
|
||||
|
||||
Nothing else in this module prevents that. Each snapshot cache single-flights
|
||||
only within itself (`inFlight` is a closure per reader), and the wedge set
|
||||
latches only *after* a read misses its 3 s deadline, so through the healthy
|
||||
latches only _after_ a read misses its 3 s deadline, so through the healthy
|
||||
~12 ms of a scan neither excludes the other. Overlap is the normal state rather
|
||||
than an edge case: other panes keep polling detailed at 750 ms while a teardown
|
||||
takes identity snapshots, and `codex-structured-turn-processes.ts` issues fresh
|
||||
@@ -166,15 +166,15 @@ through `toIdentityRow`, so an identity row carries no command line on any host.
|
||||
|
||||
### Which callers need which
|
||||
|
||||
| caller | reads | flag set |
|
||||
| --------------------------------------------- | ------------------ | -------- |
|
||||
| `windows-agent-foreground-process.ts` | `command` (agent recognition) | detailed |
|
||||
| `local-workspace-platform-port-scanner.ts` | `command` (port attribution) | detailed |
|
||||
| `codex-structured-turn-processes.ts` | `command` (turn-process identity) | detailed |
|
||||
| `structured-tui-process-identity.ts` | `command` (child match) | detailed |
|
||||
| `windows-pty-root-identity.ts` | `pid` / `ppid` only | identity |
|
||||
| `agent-session-process-identity-probe.ts` | `creationTimeMs` only | identity |
|
||||
| `relay/windows-port-scan.ts` | `name` (port owner label) | detailed |
|
||||
| caller | reads | flag set |
|
||||
| ------------------------------------------ | --------------------------------- | -------- |
|
||||
| `windows-agent-foreground-process.ts` | `command` (agent recognition) | detailed |
|
||||
| `local-workspace-platform-port-scanner.ts` | `command` (port attribution) | detailed |
|
||||
| `codex-structured-turn-processes.ts` | `command` (turn-process identity) | detailed |
|
||||
| `structured-tui-process-identity.ts` | `command` (child match) | detailed |
|
||||
| `windows-pty-root-identity.ts` | `pid` / `ppid` only | identity |
|
||||
| `agent-session-process-identity-probe.ts` | `creationTimeMs` only | identity |
|
||||
| `relay/windows-port-scan.ts` | `name` (port owner label) | detailed |
|
||||
|
||||
`windows-port-scan.ts` is the one mismatch in the table: it reads only `pid` and
|
||||
`name`, which the identity set answers, but it calls the detailed reader. On a
|
||||
@@ -344,7 +344,7 @@ on any other OS keeps using the scan.
|
||||
|
||||
## Why the package is patched
|
||||
|
||||
`config/patches/@vscode__windows-process-tree@0.8.0.patch` carries five changes.
|
||||
`config/patches/@vscode__windows-process-tree@0.8.0.patch` carries six changes.
|
||||
|
||||
1. **Spectre mitigation.** The upstream `binding.gyp` requires Spectre-mitigated
|
||||
libraries, which Orca's Windows build agents do not install. `node-pty` is
|
||||
@@ -368,10 +368,10 @@ on any other OS keeps using the scan.
|
||||
to Unix ms; a process that denies the handle is emitted with the field
|
||||
absent, never zero, because callers must be able to tell "cannot identify"
|
||||
from a timestamp.
|
||||
5. **`supportedProcessDataFlags`.** `addon.cc` exports the flag bits the
|
||||
6. **`supportedProcessDataFlags`.** `addon.cc` exports the flag bits the
|
||||
compiled binary understands, and `lib/index.js` re-exports it.
|
||||
|
||||
Why a fifth hunk and not just the enum: unlike `node-pty`, this package
|
||||
Why a separate hunk and not just the enum: unlike `node-pty`, this package
|
||||
publishes a prebuilt `.node` at the same `build/Release/` path node-gyp
|
||||
writes to. pnpm patches the source tree and leaves that prebuilt alone, so a
|
||||
host can hold a patched `lib/index.js` — `ProcessDataFlag.CreationTime` and
|
||||
@@ -575,6 +575,20 @@ running, so typing `exit` in a pane reaped a `start /b` server that used to
|
||||
survive. The job exists to make an _explicit_ teardown exact, not to redefine
|
||||
what a clean exit means.
|
||||
|
||||
Git Bash needs one additional restriction. The Cygwin runtime — and the MSYS2
|
||||
fork of it that Git for Windows ships — reads `JOB_OBJECT_LIMIT_BREAKAWAY_OK`
|
||||
off its own job and then adds `CREATE_BREAKAWAY_FROM_JOB` to **every** child it
|
||||
spawns when that flag is set (`spawn.cc`, there since 2011), so offering
|
||||
breakaway hands the whole tree its escape. The per-PTY job therefore omits
|
||||
`BREAKAWAY_OK` whenever `msys-2.0.dll` or `cygwin1.dll` sits on the shell's DLL
|
||||
search path — beside the executable, or under `usr/bin` for Git's `bin`
|
||||
launcher. Native shells keep explicit breakaway. Denying it costs Cygwin
|
||||
nothing, because it *pre-checks* the limit rather than retrying, so no spawn
|
||||
fails; but a *native* program that passes `CREATE_BREAKAWAY_FROM_JOB` itself
|
||||
inside such a pane now gets `ERROR_ACCESS_DENIED`. `nohup` and `disown` are
|
||||
unaffected — they are Cygwin signal/session concepts, unrelated to job
|
||||
membership. The daemon's host job is unchanged.
|
||||
|
||||
Reaping a dead daemon's shells (#9195, #10415) is therefore a **second, nested
|
||||
job**, not this one. The terminal daemon assigns itself to a kill-on-close job
|
||||
at startup (`assignHostProcessToKillOnCloseJob`); children inherit membership,
|
||||
|
||||
@@ -30,8 +30,7 @@ import { Callout } from '@/components/docs/prose'
|
||||
[installer](https://github.com/stablyai/orca/releases/latest/download/orca-windows-setup.exe)
|
||||
</li>
|
||||
<li>
|
||||
**Linux:**
|
||||
AppImage
|
||||
**Linux:** AppImage
|
||||
[x64](https://github.com/stablyai/orca/releases/latest/download/orca-linux.AppImage) ·
|
||||
[arm64](https://github.com/stablyai/orca/releases/latest/download/orca-linux-arm64.AppImage) ·
|
||||
[.deb](https://github.com/stablyai/orca/releases) ·
|
||||
@@ -131,7 +130,7 @@ On Linux the [Orca CLI](/docs/cli/reference) installs as **`orca-ide`**, not `or
|
||||
- The `.deb` and `.rpm` put `orca-ide` on your `PATH` at install time, as `/usr/bin/orca-ide`.
|
||||
- With the AppImage, register the CLI from [Settings → General → Orca CLI](/docs/settings). That installs `~/.local/bin/orca-ide`.
|
||||
- Inside Orca's own terminals, bare `orca` works. Orca puts a shim on the `PATH` of the terminals it manages, so agents and scripts running there use the same command as on macOS and Windows.
|
||||
- On a headless host, a packaged `orca serve` writes a bare `orca` into `~/.local/bin` as it starts, unless a file it does not own already holds that name. It writes that *during* startup, so it is never what starts the server — the first launch is always [`orca-ide serve`](/docs/remote-servers).
|
||||
- On a headless host, a packaged `orca serve` writes a bare `orca` into `~/.local/bin` as it starts, unless a file it does not own already holds that name. It writes that _during_ startup, so it is never what starts the server — the first launch is always [`orca-ide serve`](/docs/remote-servers).
|
||||
|
||||
Do not verify with `command -v orca`: on a GNOME desktop that succeeds and resolves to the screen reader. Use `orca-ide` in your own shell and `orca` inside Orca. If you want the short name everywhere and you do not use the screen reader, link it yourself:
|
||||
|
||||
|
||||
@@ -129,19 +129,23 @@ Install Orca and its bundled CLI on the server, then run:
|
||||
<Callout title="On Linux, start it with orca-ide serve">
|
||||
The Linux CLI is named `orca-ide`, because GNOME Orca's screen reader already owns
|
||||
`/usr/bin/orca`. A packaged `orca serve` does write a bare `orca` into `~/.local/bin`, but only
|
||||
while it is starting, so that shim can never be the command that starts the server. Read
|
||||
`orca serve` as `orca-ide serve` throughout this page when the host is Linux. See
|
||||
[Install → Linux](/docs/install#linux).
|
||||
while it is starting, so that shim can never be the command that starts the server. Read `orca
|
||||
serve` as `orca-ide serve` throughout this page when the host is Linux. See [Install →
|
||||
Linux](/docs/install#linux).
|
||||
</Callout>
|
||||
|
||||
```bash
|
||||
orca serve --pairing-address <server-tailscale-ip-or-hostname>
|
||||
# Linux
|
||||
orca-ide serve --pairing-address <server-tailscale-ip-or-hostname>
|
||||
```
|
||||
|
||||
For example:
|
||||
|
||||
```bash
|
||||
orca serve --pairing-address 100.64.1.20
|
||||
# Linux
|
||||
orca-ide serve --pairing-address 100.64.1.20
|
||||
```
|
||||
|
||||
The command:
|
||||
@@ -157,6 +161,8 @@ Add `--port 6768` when a firewall, tunnel, or service definition requires a fixe
|
||||
|
||||
```bash
|
||||
orca serve --port 6768 --pairing-address 100.64.1.20
|
||||
# Linux
|
||||
orca-ide serve --port 6768 --pairing-address 100.64.1.20
|
||||
```
|
||||
|
||||
Use only one host mode at a time. If the Orca desktop app is already sharing that computer, do not start a second `orca serve` process for the same setup.
|
||||
@@ -167,6 +173,8 @@ For the Orca mobile app, request a mobile-scoped QR code and link:
|
||||
|
||||
```bash
|
||||
orca serve --pairing-address 100.64.1.20 --mobile-pairing
|
||||
# Linux
|
||||
orca-ide serve --pairing-address 100.64.1.20 --mobile-pairing
|
||||
```
|
||||
|
||||
Keep the phone on the same tailnet, open Orca Mobile, choose **Pair**, and scan the terminal QR code or paste the printed link.
|
||||
|
||||
+406
@@ -0,0 +1,406 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const asyncStorage = vi.hoisted(() => ({
|
||||
getItem: vi.fn(),
|
||||
setItem: vi.fn(),
|
||||
removeItem: vi.fn()
|
||||
}))
|
||||
|
||||
vi.mock('@react-native-async-storage/async-storage', () => ({ default: asyncStorage }))
|
||||
|
||||
import {
|
||||
deleteCachedSessionTabStripForHost,
|
||||
getSessionTabStripCacheKey,
|
||||
loadCachedSessionTabStrip,
|
||||
readCachedSessionTabStrip,
|
||||
resetSessionTabStripCacheForTests,
|
||||
saveCachedSessionTabStrip
|
||||
} from './session-tab-strip-cache'
|
||||
import type { MobileSessionTabStripPreview } from '../session/mobile-session-tab-strip-entries'
|
||||
|
||||
const STORAGE_KEY = 'orca:session-tab-strip:v1'
|
||||
|
||||
function preview(...ids: string[]): MobileSessionTabStripPreview {
|
||||
return {
|
||||
tabs: ids.map((id) => ({ id, type: 'terminal' as const, title: id, agentId: null })),
|
||||
activeTabId: ids[0] ?? null
|
||||
}
|
||||
}
|
||||
|
||||
function lastWrittenFile(): { workspaces: { key: string }[] } {
|
||||
const call = asyncStorage.setItem.mock.calls.at(-1)
|
||||
return JSON.parse(String(call?.[1]))
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
asyncStorage.getItem.mockReset().mockResolvedValue(null)
|
||||
asyncStorage.setItem.mockReset().mockResolvedValue(undefined)
|
||||
resetSessionTabStripCacheForTests()
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
describe('getSessionTabStripCacheKey', () => {
|
||||
it('digests the workspace id so no filesystem path reaches the key', () => {
|
||||
const path = '/Users/someone/private-client/worktrees/acquisition'
|
||||
const key = getSessionTabStripCacheKey('host-1', `repo::${path}`)
|
||||
|
||||
expect(key).not.toContain(path)
|
||||
expect(key).not.toContain('someone')
|
||||
expect(key).toMatch(/^\["host-1","[0-9a-f]{32}"\]$/)
|
||||
})
|
||||
|
||||
it('joins the two ids unambiguously, whatever a worktree path contains', () => {
|
||||
expect(getSessionTabStripCacheKey('host', 'a\nb')).not.toBe(
|
||||
getSessionTabStripCacheKey('host\na', 'b')
|
||||
)
|
||||
expect(getSessionTabStripCacheKey('host-1', 'wt-1')).not.toBe(
|
||||
getSessionTabStripCacheKey('host-1', 'wt-2')
|
||||
)
|
||||
})
|
||||
|
||||
it('needs both a host and a workspace', () => {
|
||||
expect(getSessionTabStripCacheKey(undefined, 'wt-1')).toBeNull()
|
||||
expect(getSessionTabStripCacheKey('host-1', undefined)).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('session tab strip cache', () => {
|
||||
it('serves a save back synchronously and persists it once the write settles', async () => {
|
||||
const key = getSessionTabStripCacheKey('host-1', 'wt-1')
|
||||
saveCachedSessionTabStrip(key, preview('tab-1', 'tab-2'))
|
||||
|
||||
expect(readCachedSessionTabStrip(key)?.tabs.map((tab) => tab.id)).toEqual(['tab-1', 'tab-2'])
|
||||
expect(asyncStorage.setItem).not.toHaveBeenCalled()
|
||||
|
||||
await vi.advanceTimersByTimeAsync(300)
|
||||
|
||||
expect(asyncStorage.setItem.mock.calls[0]?.[0]).toBe(STORAGE_KEY)
|
||||
expect(lastWrittenFile().workspaces.map((w) => w.key)).toEqual([key])
|
||||
})
|
||||
|
||||
it('reads nothing synchronously before the stored file is loaded', async () => {
|
||||
const key = getSessionTabStripCacheKey('host-1', 'wt-1')
|
||||
asyncStorage.getItem.mockResolvedValue(
|
||||
JSON.stringify({ workspaces: [{ key, preview: preview('tab-1') }] })
|
||||
)
|
||||
|
||||
expect(readCachedSessionTabStrip(key)).toBeNull()
|
||||
expect((await loadCachedSessionTabStrip(key))?.tabs.map((tab) => tab.id)).toEqual(['tab-1'])
|
||||
expect(readCachedSessionTabStrip(key)?.tabs).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('returns null for a workspace with no stored strip', async () => {
|
||||
expect(await loadCachedSessionTabStrip(getSessionTabStripCacheKey('host-1', 'wt-9'))).toBeNull()
|
||||
expect(await loadCachedSessionTabStrip(null)).toBeNull()
|
||||
})
|
||||
|
||||
it('survives unreadable storage', async () => {
|
||||
asyncStorage.getItem.mockResolvedValue('{not json')
|
||||
|
||||
expect(await loadCachedSessionTabStrip(getSessionTabStripCacheKey('host-1', 'wt-1'))).toBeNull()
|
||||
})
|
||||
|
||||
it('evicts the least recently written workspace past the cap', async () => {
|
||||
for (let i = 0; i < 14; i++) {
|
||||
saveCachedSessionTabStrip(getSessionTabStripCacheKey('host-1', `wt-${i}`), preview('tab-1'))
|
||||
}
|
||||
await vi.advanceTimersByTimeAsync(300)
|
||||
|
||||
const keys = lastWrittenFile().workspaces.map((w) => w.key)
|
||||
expect(keys).toHaveLength(12)
|
||||
expect(keys).not.toContain(getSessionTabStripCacheKey('host-1', 'wt-0'))
|
||||
expect(keys.at(-1)).toBe(getSessionTabStripCacheKey('host-1', 'wt-13'))
|
||||
})
|
||||
|
||||
it('re-writing a workspace makes it the newest, not the oldest', async () => {
|
||||
for (let i = 0; i < 12; i++) {
|
||||
saveCachedSessionTabStrip(getSessionTabStripCacheKey('host-1', `wt-${i}`), preview('tab-1'))
|
||||
}
|
||||
saveCachedSessionTabStrip(getSessionTabStripCacheKey('host-1', 'wt-0'), preview('tab-2'))
|
||||
saveCachedSessionTabStrip(getSessionTabStripCacheKey('host-1', 'wt-99'), preview('tab-1'))
|
||||
await vi.advanceTimersByTimeAsync(300)
|
||||
|
||||
const keys = lastWrittenFile().workspaces.map((w) => w.key)
|
||||
expect(keys).toContain(getSessionTabStripCacheKey('host-1', 'wt-0'))
|
||||
expect(keys).not.toContain(getSessionTabStripCacheKey('host-1', 'wt-1'))
|
||||
})
|
||||
|
||||
it('records a workspace the host has emptied, so a stale strip cannot outlive it', async () => {
|
||||
const key = getSessionTabStripCacheKey('host-1', 'wt-1')
|
||||
saveCachedSessionTabStrip(key, preview('tab-1'))
|
||||
saveCachedSessionTabStrip(key, { tabs: [], activeTabId: null })
|
||||
|
||||
expect(readCachedSessionTabStrip(key)).toEqual({ tabs: [], activeTabId: null })
|
||||
})
|
||||
|
||||
it('caps tabs per workspace and title length, and drops an unmatched active id', async () => {
|
||||
const key = getSessionTabStripCacheKey('host-1', 'wt-1')
|
||||
saveCachedSessionTabStrip(key, {
|
||||
// A file tab, because the titles that survive redaction at all are the ones the cap has
|
||||
// to bound.
|
||||
tabs: Array.from({ length: 30 }, (_, i) => ({
|
||||
id: `tab-${i}`,
|
||||
type: 'file' as const,
|
||||
title: 'x'.repeat(200),
|
||||
agentId: null
|
||||
})),
|
||||
activeTabId: 'tab-29'
|
||||
})
|
||||
|
||||
const stored = readCachedSessionTabStrip(key)
|
||||
expect(stored?.tabs).toHaveLength(24)
|
||||
expect(stored?.tabs[0]?.title).toHaveLength(64)
|
||||
expect(stored?.activeTabId).toBeNull()
|
||||
})
|
||||
|
||||
it('drops fields a future tab type might smuggle into storage', async () => {
|
||||
const key = getSessionTabStripCacheKey('host-1', 'wt-1')
|
||||
saveCachedSessionTabStrip(key, {
|
||||
tabs: [
|
||||
{
|
||||
id: 'tab-1',
|
||||
type: 'file',
|
||||
title: 'notes.md',
|
||||
agentId: null,
|
||||
filePath: '/Users/someone/secret/notes.md'
|
||||
} as never
|
||||
],
|
||||
activeTabId: 'tab-1'
|
||||
})
|
||||
await vi.advanceTimersByTimeAsync(300)
|
||||
|
||||
expect(String(asyncStorage.setItem.mock.calls.at(-1)?.[1])).not.toContain('/Users/someone')
|
||||
})
|
||||
|
||||
it('drops a stored entry naming a tab type this build cannot draw', async () => {
|
||||
const key = getSessionTabStripCacheKey('host-1', 'wt-1')
|
||||
saveCachedSessionTabStrip(key, {
|
||||
tabs: [
|
||||
{ id: 'tab-1', type: 'from-a-newer-build', title: 'raw title', agentId: null } as never,
|
||||
{ id: 'tab-2', type: 'file', title: 'notes.md', agentId: null }
|
||||
],
|
||||
activeTabId: 'tab-2'
|
||||
})
|
||||
|
||||
expect(readCachedSessionTabStrip(key)?.tabs.map((tab) => tab.id)).toEqual(['tab-2'])
|
||||
})
|
||||
|
||||
it('never writes a shell-controlled terminal title, however it arrives', async () => {
|
||||
const secret = 'psql postgres://admin:hunter2@db.internal/prod'
|
||||
const key = getSessionTabStripCacheKey('host-1', 'wt-1')
|
||||
saveCachedSessionTabStrip(key, {
|
||||
tabs: [
|
||||
{ id: 'tab-1', type: 'terminal', title: secret, agentId: null },
|
||||
{ id: 'tab-2', type: 'terminal', title: secret, agentId: 'claude' },
|
||||
{ id: 'tab-3', type: 'terminal', title: secret, agentId: 'not-a-known-agent' },
|
||||
{ id: 'tab-4', type: 'browser', title: 'Acme Corp — Q3 layoffs memo', agentId: null }
|
||||
],
|
||||
activeTabId: 'tab-1'
|
||||
})
|
||||
await vi.advanceTimersByTimeAsync(300)
|
||||
|
||||
expect(readCachedSessionTabStrip(key)?.tabs.map((tab) => tab.title)).toEqual([
|
||||
'Terminal',
|
||||
'Claude',
|
||||
'Terminal',
|
||||
'Browser'
|
||||
])
|
||||
const written = String(asyncStorage.setItem.mock.calls.at(-1)?.[1])
|
||||
expect(written).not.toContain('hunter2')
|
||||
expect(written).not.toContain('postgres://')
|
||||
expect(written).not.toContain('layoffs')
|
||||
})
|
||||
|
||||
it('scrubs a stored title written by an older build on the way back out', async () => {
|
||||
const key = getSessionTabStripCacheKey('host-1', 'wt-1')
|
||||
asyncStorage.getItem.mockResolvedValue(
|
||||
JSON.stringify({
|
||||
workspaces: [
|
||||
{
|
||||
key,
|
||||
preview: {
|
||||
tabs: [{ id: 'tab-1', type: 'terminal', title: 'curl -H token', agentId: null }],
|
||||
activeTabId: 'tab-1'
|
||||
}
|
||||
}
|
||||
]
|
||||
})
|
||||
)
|
||||
|
||||
expect((await loadCachedSessionTabStrip(key))?.tabs[0]?.title).toBe('Terminal')
|
||||
})
|
||||
|
||||
it('forgets an unpaired host and cannot resurrect it from a later save', async () => {
|
||||
const hostA = getSessionTabStripCacheKey('host-a', 'wt-1')
|
||||
const hostB = getSessionTabStripCacheKey('host-b', 'wt-1')
|
||||
saveCachedSessionTabStrip(hostA, preview('tab-a'))
|
||||
saveCachedSessionTabStrip(hostB, preview('tab-b'))
|
||||
await vi.advanceTimersByTimeAsync(300)
|
||||
|
||||
await deleteCachedSessionTabStripForHost('host-a')
|
||||
|
||||
expect(readCachedSessionTabStrip(hostA)).toBeNull()
|
||||
expect(readCachedSessionTabStrip(hostB)?.tabs).toHaveLength(1)
|
||||
expect(lastWrittenFile().workspaces.map((w) => w.key)).toEqual([hostB])
|
||||
|
||||
saveCachedSessionTabStrip(hostB, preview('tab-b2'))
|
||||
await vi.advanceTimersByTimeAsync(300)
|
||||
|
||||
expect(lastWrittenFile().workspaces.map((w) => w.key)).toEqual([hostB])
|
||||
})
|
||||
|
||||
it('forgets a host whose rows are only on disk, never read this session', async () => {
|
||||
const hostA = getSessionTabStripCacheKey('host-a', 'wt-1')
|
||||
const hostB = getSessionTabStripCacheKey('host-b', 'wt-1')
|
||||
asyncStorage.getItem.mockResolvedValue(
|
||||
JSON.stringify({
|
||||
workspaces: [
|
||||
{ key: hostA, preview: preview('tab-a') },
|
||||
{ key: hostB, preview: preview('tab-b') }
|
||||
]
|
||||
})
|
||||
)
|
||||
|
||||
await deleteCachedSessionTabStripForHost('host-a')
|
||||
|
||||
expect(lastWrittenFile().workspaces.map((w) => w.key)).toEqual([hostB])
|
||||
})
|
||||
|
||||
it('drops a pending debounced write so it cannot restore the forgotten host', async () => {
|
||||
const hostA = getSessionTabStripCacheKey('host-a', 'wt-1')
|
||||
saveCachedSessionTabStrip(hostA, preview('tab-a'))
|
||||
|
||||
await deleteCachedSessionTabStripForHost('host-a')
|
||||
await vi.advanceTimersByTimeAsync(300)
|
||||
|
||||
expect(lastWrittenFile().workspaces).toEqual([])
|
||||
})
|
||||
it('rejects a deletion whose write never landed, rather than reporting it as done', async () => {
|
||||
// A resolved delete over a failed write leaves the forgotten host's tab titles in
|
||||
// plaintext on disk while every caller believes they are gone.
|
||||
const hostA = getSessionTabStripCacheKey('host-a', 'wt-1')
|
||||
saveCachedSessionTabStrip(hostA, preview('tab-a'))
|
||||
await vi.advanceTimersByTimeAsync(300)
|
||||
asyncStorage.setItem.mockRejectedValue(new Error('storage full'))
|
||||
|
||||
await expect(deleteCachedSessionTabStripForHost('host-a')).rejects.toThrow('storage full')
|
||||
})
|
||||
|
||||
it('keeps a debounced save best effort, so one failed write cannot reject unowned', async () => {
|
||||
asyncStorage.setItem.mockRejectedValue(new Error('storage full'))
|
||||
saveCachedSessionTabStrip(getSessionTabStripCacheKey('host-a', 'wt-1'), preview('tab-a'))
|
||||
|
||||
// No throw and no unhandled rejection: the write is fire-and-forget by design.
|
||||
await vi.advanceTimersByTimeAsync(300)
|
||||
expect(asyncStorage.setItem).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('refuses a save for the host it is in the middle of forgetting', async () => {
|
||||
const hostA = getSessionTabStripCacheKey('host-a', 'wt-1')
|
||||
saveCachedSessionTabStrip(hostA, preview('tab-a'))
|
||||
await vi.advanceTimersByTimeAsync(300)
|
||||
|
||||
let releaseWrite!: () => void
|
||||
asyncStorage.setItem.mockImplementationOnce(
|
||||
async () =>
|
||||
new Promise<void>((resolve) => {
|
||||
releaseWrite = () => resolve()
|
||||
})
|
||||
)
|
||||
const deletion = deleteCachedSessionTabStripForHost('host-a')
|
||||
// The purge has run and its write is on the wire; a snapshot queued for the
|
||||
// workspace the user just unpaired now lands in that window.
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
saveCachedSessionTabStrip(hostA, preview('tab-a2'))
|
||||
releaseWrite()
|
||||
await deletion
|
||||
await vi.advanceTimersByTimeAsync(300)
|
||||
|
||||
expect(readCachedSessionTabStrip(hostA)).toBeNull()
|
||||
expect(lastWrittenFile().workspaces).toEqual([])
|
||||
})
|
||||
|
||||
it('cannot be talked back into a host whose deletion write failed', async () => {
|
||||
const hostA = getSessionTabStripCacheKey('host-a', 'wt-1')
|
||||
saveCachedSessionTabStrip(hostA, preview('tab-a'))
|
||||
await vi.advanceTimersByTimeAsync(300)
|
||||
asyncStorage.setItem.mockRejectedValueOnce(new Error('storage full'))
|
||||
|
||||
await expect(deleteCachedSessionTabStripForHost('host-a')).rejects.toThrow('storage full')
|
||||
const writesSoFar = asyncStorage.setItem.mock.calls.length
|
||||
|
||||
saveCachedSessionTabStrip(hostA, preview('tab-a3'))
|
||||
await vi.advanceTimersByTimeAsync(300)
|
||||
|
||||
expect(readCachedSessionTabStrip(hostA)).toBeNull()
|
||||
expect(asyncStorage.setItem).toHaveBeenCalledTimes(writesSoFar)
|
||||
})
|
||||
it('lets a debounced write that already snapshotted the removed host land first', async () => {
|
||||
// The tombstone stops new saves, but a debounced write that fired a moment earlier
|
||||
// built its blob from the map as it was and is still on the wire. Writing over it
|
||||
// concurrently leaves which blob lands last up to storage.
|
||||
const hostA = getSessionTabStripCacheKey('host-a', 'wt-1')
|
||||
const hostB = getSessionTabStripCacheKey('host-b', 'wt-1')
|
||||
saveCachedSessionTabStrip(hostA, preview('tab-a'))
|
||||
saveCachedSessionTabStrip(hostB, preview('tab-b'))
|
||||
|
||||
let releaseDebounced!: () => void
|
||||
asyncStorage.setItem.mockImplementationOnce(
|
||||
async () =>
|
||||
new Promise<void>((resolve) => {
|
||||
releaseDebounced = () => resolve()
|
||||
})
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(300)
|
||||
|
||||
const deletion = deleteCachedSessionTabStripForHost('host-a')
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(asyncStorage.setItem).toHaveBeenCalledOnce()
|
||||
|
||||
releaseDebounced()
|
||||
await deletion
|
||||
|
||||
expect(asyncStorage.setItem).toHaveBeenCalledTimes(2)
|
||||
expect(lastWrittenFile().workspaces.map((w) => w.key)).toEqual([hostB])
|
||||
})
|
||||
it('cannot let an older overlapping write commit after the purge', async () => {
|
||||
// Why: two debounced writes can sit on the bridge at once, and the second used to replace
|
||||
// the in-flight handle. The purge then awaited only the newer one, so the older blob --
|
||||
// snapshotted while the forgotten host was still in the map -- could commit last.
|
||||
const hostA = getSessionTabStripCacheKey('host-a', 'wt-1')
|
||||
const hostB = getSessionTabStripCacheKey('host-b', 'wt-1')
|
||||
let stored = ''
|
||||
const gates: Array<() => void> = []
|
||||
asyncStorage.setItem.mockImplementation(
|
||||
(_key: string, value: string) =>
|
||||
new Promise<void>((resolve) => {
|
||||
gates.push(() => {
|
||||
stored = value
|
||||
resolve()
|
||||
})
|
||||
})
|
||||
)
|
||||
|
||||
saveCachedSessionTabStrip(hostA, preview('tab-a'))
|
||||
await vi.advanceTimersByTimeAsync(300)
|
||||
saveCachedSessionTabStrip(hostB, preview('tab-b'))
|
||||
await vi.advanceTimersByTimeAsync(300)
|
||||
|
||||
const deletion = deleteCachedSessionTabStripForHost('host-a')
|
||||
// Newest released first: only writes that queue behind one another survive this.
|
||||
for (let step = 0; step < 6 && gates.length > 0; step += 1) {
|
||||
gates.pop()?.()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
}
|
||||
await deletion
|
||||
|
||||
const keys = (JSON.parse(stored) as { workspaces: { key: string }[] }).workspaces.map(
|
||||
(workspace) => workspace.key
|
||||
)
|
||||
expect(keys).toEqual([hostB])
|
||||
})
|
||||
})
|
||||
+260
@@ -0,0 +1,260 @@
|
||||
// Why: reconnecting to a workspace the phone opened a minute ago tears the session screen back
|
||||
// to an empty strip and a spinner, even though the tab list it is about to be handed is the one
|
||||
// it just displayed. Persist the shape of the strip per workspace so a reconnect paints the
|
||||
// known tabs immediately and swaps in live rows under the same keys.
|
||||
//
|
||||
// This file is the authority on what reaches plaintext storage, not its callers: every entry is
|
||||
// rebuilt field by field on the way in, and shell-controlled titles are replaced with fixed
|
||||
// labels here rather than trusted to have been scrubbed upstream.
|
||||
import AsyncStorage from '@react-native-async-storage/async-storage'
|
||||
import { sha256 } from '@noble/hashes/sha256'
|
||||
import {
|
||||
getPersistableTabStripTitle,
|
||||
isDrawableTabStripType,
|
||||
type MobileSessionTabStripEntry,
|
||||
type MobileSessionTabStripPreview
|
||||
} from '../session/mobile-session-tab-strip-entries'
|
||||
|
||||
const STORAGE_KEY = 'orca:session-tab-strip:v1'
|
||||
// A phone realistically revisits a handful of workspaces; the caps bound both the stored blob
|
||||
// and the cost of a single write.
|
||||
const MAX_WORKSPACES = 12
|
||||
const MAX_TABS_PER_WORKSPACE = 24
|
||||
const MAX_TITLE_LENGTH = 64
|
||||
const WRITE_DEBOUNCE_MS = 250
|
||||
// 128 bits of a digest: far past collision range for a dozen workspaces, and short enough that
|
||||
// the stored blob stays small.
|
||||
const WORKSPACE_DIGEST_LENGTH = 32
|
||||
|
||||
type StoredWorkspace = { key: string; preview: MobileSessionTabStripPreview }
|
||||
type StoredFile = { workspaces: StoredWorkspace[] }
|
||||
|
||||
// Insertion-ordered, so the first key is the least recently written one to evict.
|
||||
let memoryCache: Map<string, MobileSessionTabStripPreview> | null = null
|
||||
let loadPromise: Promise<Map<string, MobileSessionTabStripPreview>> | null = null
|
||||
let writeTimer: ReturnType<typeof setTimeout> | null = null
|
||||
// Tail of the write chain. Every write queues behind it, so an older setItem can never
|
||||
// settle after a newer one and make its stale blob the last word on disk.
|
||||
let writeInFlight: Promise<void> | null = null
|
||||
// Hosts forgotten this session. A save racing the deletion would re-insert the host and
|
||||
// the next debounced write would put its tab titles back on disk, so refuse those saves
|
||||
// outright. Re-pairing the same host caches again from the next app launch — the cheap
|
||||
// direction for a deletion the user asked for.
|
||||
const forgottenHosts = new Set<string>()
|
||||
|
||||
/**
|
||||
* A workspace id ends in a filesystem path, so it is digested rather than stored. The host id
|
||||
* stays readable because forgetting a host has to be able to find that host's rows, and because
|
||||
* host ids already key several other entries in this store.
|
||||
*/
|
||||
export function getSessionTabStripCacheKey(
|
||||
hostId: string | undefined,
|
||||
worktreeId: string | undefined
|
||||
): string | null {
|
||||
if (!hostId || !worktreeId) {
|
||||
return null
|
||||
}
|
||||
return JSON.stringify([hostId, digestWorkspaceId(worktreeId)])
|
||||
}
|
||||
|
||||
/** Whatever this process already knows, with no await — so a revisit paints on the first frame. */
|
||||
export function readCachedSessionTabStrip(key: string | null): MobileSessionTabStripPreview | null {
|
||||
if (!key || !memoryCache) {
|
||||
return null
|
||||
}
|
||||
return memoryCache.get(key) ?? null
|
||||
}
|
||||
|
||||
export async function loadCachedSessionTabStrip(
|
||||
key: string | null
|
||||
): Promise<MobileSessionTabStripPreview | null> {
|
||||
if (!key) {
|
||||
return null
|
||||
}
|
||||
const cache = await loadFile()
|
||||
return cache.get(key) ?? null
|
||||
}
|
||||
|
||||
export function saveCachedSessionTabStrip(
|
||||
key: string | null,
|
||||
preview: MobileSessionTabStripPreview
|
||||
): void {
|
||||
if (!key) {
|
||||
return
|
||||
}
|
||||
const hostId = readHostIdFromKey(key)
|
||||
if (hostId !== null && forgottenHosts.has(hostId)) {
|
||||
return
|
||||
}
|
||||
const redacted = redactPreview(preview)
|
||||
const cache = memoryCache ?? new Map()
|
||||
memoryCache = cache
|
||||
// Map.set on an existing key keeps its original iteration position, so delete first to make
|
||||
// the re-inserted key the newest and give the cap true LRU eviction.
|
||||
cache.delete(key)
|
||||
cache.set(key, redacted)
|
||||
while (cache.size > MAX_WORKSPACES) {
|
||||
const oldest = cache.keys().next().value
|
||||
if (oldest === undefined) {
|
||||
break
|
||||
}
|
||||
cache.delete(oldest)
|
||||
}
|
||||
scheduleWrite(cache)
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop every workspace belonging to a host the user has unpaired. Both the in-memory rows and
|
||||
* the stored blob have to go: leaving either behind means the next save for any other host
|
||||
* serializes the forgotten host's tabs straight back to disk.
|
||||
*/
|
||||
export async function deleteCachedSessionTabStripForHost(hostId: string): Promise<void> {
|
||||
// Before the first await: a save landing during the load or the write must not
|
||||
// re-insert the host the caller is in the middle of forgetting.
|
||||
forgottenHosts.add(hostId)
|
||||
// Load first so the rewrite below preserves other hosts. If storage is unreadable we still
|
||||
// rewrite, which can cost another host its rows — the wrong direction for a cache, the right
|
||||
// one for a deletion the user asked for.
|
||||
const cache = await loadFile()
|
||||
// Deleting the entry the iterator is standing on is well-defined for a Map.
|
||||
for (const key of cache.keys()) {
|
||||
if (readHostIdFromKey(key) === hostId) {
|
||||
cache.delete(key)
|
||||
}
|
||||
}
|
||||
if (writeTimer) {
|
||||
clearTimeout(writeTimer)
|
||||
writeTimer = null
|
||||
}
|
||||
// Queued, not raced: the purge is the last write, and its failure is the caller's.
|
||||
await enqueueWrite(cache)
|
||||
}
|
||||
|
||||
export function resetSessionTabStripCacheForTests(): void {
|
||||
if (writeTimer) {
|
||||
clearTimeout(writeTimer)
|
||||
writeTimer = null
|
||||
}
|
||||
memoryCache = null
|
||||
loadPromise = null
|
||||
writeInFlight = null
|
||||
forgottenHosts.clear()
|
||||
}
|
||||
|
||||
function digestWorkspaceId(worktreeId: string): string {
|
||||
const digest = sha256(new TextEncoder().encode(worktreeId))
|
||||
let hex = ''
|
||||
for (const byte of digest) {
|
||||
hex += byte.toString(16).padStart(2, '0')
|
||||
}
|
||||
return hex.slice(0, WORKSPACE_DIGEST_LENGTH)
|
||||
}
|
||||
|
||||
function readHostIdFromKey(key: string): string | null {
|
||||
try {
|
||||
const parsed = JSON.parse(key) as unknown
|
||||
return Array.isArray(parsed) && typeof parsed[0] === 'string' ? parsed[0] : null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
async function loadFile(): Promise<Map<string, MobileSessionTabStripPreview>> {
|
||||
if (memoryCache) {
|
||||
return memoryCache
|
||||
}
|
||||
loadPromise ??= (async () => {
|
||||
const parsed = await readStoredFile()
|
||||
// A save that landed while the read was in flight owns the newer truth.
|
||||
const cache = memoryCache ?? new Map<string, MobileSessionTabStripPreview>()
|
||||
for (const workspace of parsed) {
|
||||
if (!cache.has(workspace.key)) {
|
||||
cache.set(workspace.key, workspace.preview)
|
||||
}
|
||||
}
|
||||
memoryCache = cache
|
||||
return cache
|
||||
})()
|
||||
return loadPromise
|
||||
}
|
||||
|
||||
async function readStoredFile(): Promise<StoredWorkspace[]> {
|
||||
try {
|
||||
const raw = await AsyncStorage.getItem(STORAGE_KEY)
|
||||
if (!raw) {
|
||||
return []
|
||||
}
|
||||
const parsed = JSON.parse(raw) as StoredFile
|
||||
if (typeof parsed !== 'object' || parsed === null || !Array.isArray(parsed.workspaces)) {
|
||||
return []
|
||||
}
|
||||
return parsed.workspaces.flatMap((workspace) => {
|
||||
if (typeof workspace?.key !== 'string' || !Array.isArray(workspace.preview?.tabs)) {
|
||||
return []
|
||||
}
|
||||
return [{ key: workspace.key, preview: redactPreview(workspace.preview) }]
|
||||
})
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
// Why: a flurry of snapshots (one per desktop republication) must not hammer AsyncStorage.
|
||||
function scheduleWrite(cache: Map<string, MobileSessionTabStripPreview>): void {
|
||||
if (writeTimer) {
|
||||
clearTimeout(writeTimer)
|
||||
}
|
||||
writeTimer = setTimeout(() => {
|
||||
writeTimer = null
|
||||
// Best effort by design: a dropped cache refresh costs one repaint, and the next
|
||||
// save rewrites the whole map. Only the deletion path needs the failure.
|
||||
void enqueueWrite(cache).catch(() => {})
|
||||
}, WRITE_DEBOUNCE_MS)
|
||||
}
|
||||
|
||||
// Why the chain rather than one handle: two debounced writes can overlap on the bridge, and
|
||||
// the second overwrote the handle. A deletion then awaited only the newer one, so the older
|
||||
// write -- serialized before the purge, host rows and all -- could land last and restore them.
|
||||
function enqueueWrite(cache: Map<string, MobileSessionTabStripPreview>): Promise<void> {
|
||||
const queued = (writeInFlight ?? Promise.resolve()).then(() => writeFile(cache))
|
||||
// A rejected link must not break the chain for the writes queued behind it.
|
||||
writeInFlight = queued.catch(() => {})
|
||||
return queued
|
||||
}
|
||||
|
||||
async function writeFile(cache: Map<string, MobileSessionTabStripPreview>): Promise<void> {
|
||||
const workspaces: StoredWorkspace[] = [...cache].map(([key, preview]) => ({ key, preview }))
|
||||
// Throws on purpose: a deletion that only removed the in-memory rows must not be
|
||||
// reported as a deletion, or the forgotten host's titles stay in plaintext on disk.
|
||||
await AsyncStorage.setItem(STORAGE_KEY, JSON.stringify({ workspaces }))
|
||||
}
|
||||
|
||||
// Rebuilt field by field so a field later added to the live tab type cannot ride into storage
|
||||
// without someone deciding it belongs there.
|
||||
function redactPreview(preview: MobileSessionTabStripPreview): MobileSessionTabStripPreview {
|
||||
const tabs: MobileSessionTabStripEntry[] = []
|
||||
for (const tab of preview.tabs ?? []) {
|
||||
if (typeof tab?.id !== 'string' || !isDrawableTabStripType(tab.type)) {
|
||||
continue
|
||||
}
|
||||
const agentId = typeof tab.agentId === 'string' ? tab.agentId : null
|
||||
const title = typeof tab.title === 'string' ? tab.title : ''
|
||||
tabs.push({
|
||||
id: tab.id,
|
||||
type: tab.type,
|
||||
title: getPersistableTabStripTitle({ type: tab.type, title, agentId }).slice(
|
||||
0,
|
||||
MAX_TITLE_LENGTH
|
||||
),
|
||||
agentId
|
||||
})
|
||||
if (tabs.length === MAX_TABS_PER_WORKSPACE) {
|
||||
break
|
||||
}
|
||||
}
|
||||
const activeTabId =
|
||||
typeof preview.activeTabId === 'string' && tabs.some((tab) => tab.id === preview.activeTabId)
|
||||
? preview.activeTabId
|
||||
: null
|
||||
return { tabs, activeTabId }
|
||||
}
|
||||
@@ -44,6 +44,12 @@ function GateConsumer() {
|
||||
return createElement('GateStatus', null, hostCapabilities.join(','))
|
||||
}
|
||||
|
||||
// Separate from GateStatus so the capability assertions keep their exact rendered shape.
|
||||
function VerifiedConsumer() {
|
||||
const { compatVerified } = useHostProtocolGates()
|
||||
return createElement('GateVerified', null, compatVerified ? 'verified' : 'unverified')
|
||||
}
|
||||
|
||||
// Counts mounts so a test can prove the routes were never torn down, which presence alone can't.
|
||||
const probeMounts = { count: 0 }
|
||||
function MountProbe() {
|
||||
@@ -57,7 +63,13 @@ function gateElement() {
|
||||
return createElement(
|
||||
HostProtocolGate,
|
||||
{ hostId: 'host-1' },
|
||||
createElement('HostContent', null, createElement(GateConsumer), createElement(MountProbe))
|
||||
createElement(
|
||||
'HostContent',
|
||||
null,
|
||||
createElement(GateConsumer),
|
||||
createElement(VerifiedConsumer),
|
||||
createElement(MountProbe)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
@@ -154,13 +166,90 @@ describe('HostProtocolGate', () => {
|
||||
expect(client.sendRequest).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('serves every descendant capability read from the one status.get it issues', async () => {
|
||||
const client = clientWithStatus({
|
||||
protocolVersion: 5,
|
||||
minCompatibleMobileVersion: 0,
|
||||
capabilities: ['browser.screencast.v1', 'terminal.queryReplyInput.v1']
|
||||
})
|
||||
hostClient.current = { client, state: 'connected' }
|
||||
renderer = await act(async () => {
|
||||
const created = create(
|
||||
createElement(
|
||||
HostProtocolGate,
|
||||
{ hostId: 'host-1' },
|
||||
createElement(GateConsumer),
|
||||
createElement(GateConsumer)
|
||||
)
|
||||
)
|
||||
await Promise.resolve()
|
||||
return created
|
||||
})
|
||||
|
||||
// Why: the session route used to run its own retrying status.get on top of this one, so a
|
||||
// cold open cost two round trips for the same answer. Consumers now read the gate's copy.
|
||||
expect(client.sendRequest).toHaveBeenCalledOnce()
|
||||
expect(client.sendRequest).toHaveBeenCalledWith('status.get')
|
||||
const statuses = renderer.root.findAllByType('GateStatus')
|
||||
expect(statuses).toHaveLength(2)
|
||||
for (const status of statuses) {
|
||||
expect(status.props.children).toBe('browser.screencast.v1,terminal.queryReplyInput.v1')
|
||||
}
|
||||
})
|
||||
|
||||
it('releases the cover on a failed status.get and upgrades when a retry lands', async () => {
|
||||
vi.useFakeTimers({ shouldAdvanceTime: true })
|
||||
const sendRequest = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error('status.get timed out'))
|
||||
.mockResolvedValue({
|
||||
ok: true,
|
||||
result: { protocolVersion: 5, minCompatibleMobileVersion: 0, capabilities: ['late.v1'] }
|
||||
})
|
||||
hostClient.current = { client: { sendRequest } as unknown as RpcClient, state: 'connected' }
|
||||
renderer = await renderGate()
|
||||
|
||||
// Why: a wedged status.get must never trap the routes behind the cover, so the first miss
|
||||
// settles conservative gates immediately — no capabilities, but a usable UI.
|
||||
let output = renderedText(renderer)
|
||||
expect(output).toContain('HostContent')
|
||||
expect(output).not.toContain('Checking host compatibility')
|
||||
expect(output).toContain('"type":"GateStatus","props":{},"children":null')
|
||||
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(1_100)
|
||||
})
|
||||
|
||||
// The probe kept retrying underneath, so the answer arrives without a remount.
|
||||
expect(sendRequest).toHaveBeenCalledTimes(2)
|
||||
expect(renderedText(renderer)).toContain('late.v1')
|
||||
expect(probeMounts.count).toBe(1)
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('blocks a desktop that omits protocolVersion, so a pending verdict is not a formality', async () => {
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
// Why this case and not just an explicit old version: evaluateCompat reads a missing
|
||||
// protocolVersion as 0, so the everyday shape of an old desktop is a blocking one.
|
||||
hostClient.current = {
|
||||
client: clientWithStatus({ capabilities: [] }),
|
||||
state: 'connected'
|
||||
}
|
||||
renderer = await renderGate()
|
||||
const output = renderedText(renderer)
|
||||
expect(output).toContain('Update Orca on your computer')
|
||||
expect(output).not.toContain('HostContent')
|
||||
})
|
||||
|
||||
it('renders the host UI while the host connection is still pending', async () => {
|
||||
hostClient.current = { client: null, state: 'connecting' }
|
||||
renderer = await renderGate()
|
||||
expect(renderedText(renderer)).toContain('HostContent')
|
||||
})
|
||||
|
||||
it('does not mount host routes before a connected host passes the compatibility probe', async () => {
|
||||
// Was: the routes were held back until status.get resolved, which serialised every route's
|
||||
// own startup RPC behind this one round trip. They now mount immediately and are covered.
|
||||
it('mounts host routes under the pending cover while status.get is still in flight', async () => {
|
||||
const client = {
|
||||
sendRequest: vi.fn().mockReturnValue(new Promise(() => {}))
|
||||
} as unknown as RpcClient
|
||||
@@ -168,9 +257,40 @@ describe('HostProtocolGate', () => {
|
||||
renderer = await renderGate()
|
||||
const output = renderedText(renderer)
|
||||
expect(output).toContain('Checking host compatibility')
|
||||
expect(output).not.toContain('HostContent')
|
||||
expect(probeMounts.count).toBe(0)
|
||||
expect(output).toContain('HostContent')
|
||||
expect(probeMounts.count).toBe(1)
|
||||
expect(client.sendRequest).toHaveBeenCalledOnce()
|
||||
// Why: mounting early must not leak an unproven host's capabilities to the routes below;
|
||||
// an empty join renders no children, so the consumer saw none.
|
||||
expect(output).toContain('"type":"GateStatus","props":{},"children":null')
|
||||
const overlay = renderer.root
|
||||
.findAllByType('View')
|
||||
.find((node) => node.props.accessibilityViewIsModal === true)
|
||||
expect(overlay?.props.pointerEvents).toBe('auto')
|
||||
})
|
||||
|
||||
it('unmounts the routes it mounted early when the verdict comes back blocked', async () => {
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
let settle: ((response: unknown) => void) | null = null
|
||||
const client = {
|
||||
sendRequest: vi.fn().mockReturnValue(
|
||||
new Promise((resolve) => {
|
||||
settle = resolve
|
||||
})
|
||||
)
|
||||
} as unknown as RpcClient
|
||||
hostClient.current = { client, state: 'connected' }
|
||||
renderer = await renderGate()
|
||||
expect(renderedText(renderer)).toContain('HostContent')
|
||||
|
||||
await act(async () => {
|
||||
settle?.({ ok: true, result: { protocolVersion: 5, minCompatibleMobileVersion: 999 } })
|
||||
await Promise.resolve()
|
||||
})
|
||||
|
||||
const output = renderedText(renderer)
|
||||
expect(output).toContain('Update Orca Mobile')
|
||||
expect(output).not.toContain('HostContent')
|
||||
})
|
||||
|
||||
it('overlays the pending spinner instead of unmounting routes mounted while connecting', async () => {
|
||||
@@ -259,4 +379,52 @@ describe('HostProtocolGate', () => {
|
||||
renderer = await renderGate()
|
||||
expect(renderedText(renderer)).toContain('HostContent')
|
||||
})
|
||||
|
||||
it('reports a rejected status.get as unverified, so failing open is not a passing verdict', async () => {
|
||||
const sendRequest = vi
|
||||
.fn()
|
||||
.mockResolvedValue({ ok: false, error: { message: 'no such method' } })
|
||||
hostClient.current = { client: { sendRequest } as unknown as RpcClient, state: 'connected' }
|
||||
renderer = await renderGate()
|
||||
|
||||
// Navigation still works: the host said no, and that must not lock the user out of the route.
|
||||
const output = renderedText(renderer)
|
||||
expect(output).toContain('HostContent')
|
||||
expect(output).not.toContain('Checking host compatibility')
|
||||
// Why: `compatVerdict` is `ok` here purely as a fallback. Nothing about this host was proven,
|
||||
// so callers that write to it read this flag instead of the verdict.
|
||||
expect(output).toContain('["unverified"]')
|
||||
})
|
||||
|
||||
it('reports a passing status reply as verified', async () => {
|
||||
hostClient.current = {
|
||||
client: clientWithStatus({ protocolVersion: 5, minCompatibleMobileVersion: 0 }),
|
||||
state: 'connected'
|
||||
}
|
||||
renderer = await renderGate()
|
||||
expect(renderedText(renderer)).toContain('["verified"]')
|
||||
})
|
||||
|
||||
it('stays unverified through a failed status.get and flips once a retry answers', async () => {
|
||||
vi.useFakeTimers({ shouldAdvanceTime: true })
|
||||
const sendRequest = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error('status.get timed out'))
|
||||
.mockResolvedValue({
|
||||
ok: true,
|
||||
result: { protocolVersion: 5, minCompatibleMobileVersion: 0 }
|
||||
})
|
||||
hostClient.current = { client: { sendRequest } as unknown as RpcClient, state: 'connected' }
|
||||
renderer = await renderGate()
|
||||
|
||||
expect(renderedText(renderer)).toContain('["unverified"]')
|
||||
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(1_100)
|
||||
})
|
||||
|
||||
// The retry landed, so the fallback is replaced by a real answer and writes are released.
|
||||
expect(renderedText(renderer)).toContain('["verified"]')
|
||||
vi.useRealTimers()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -22,45 +22,26 @@ export function useHostProtocolGates(): HostStatusGates {
|
||||
|
||||
// Why: single choke point above every /h/[hostId] route so a blocked verdict replaces the
|
||||
// whole host UI (sidebar + detail stack) while the host list and other hosts stay usable.
|
||||
// The routes mount as soon as the connection does, so their startup RPCs (session.tabs.list,
|
||||
// terminal.list) fly alongside this status.get instead of queueing behind it; a blocked verdict
|
||||
// then unmounts them and their answers are discarded.
|
||||
export function HostProtocolGate({ hostId, children }: Props) {
|
||||
const { client, state } = useHostClient(hostId)
|
||||
const gates = useHostStatusGates({ hostId, client, connState: state })
|
||||
const { compatVerdict, statusPending } = gates
|
||||
const resolvedHostIdRef = useRef<string | null>(null)
|
||||
const mountedHostIdRef = useRef<string | null>(null)
|
||||
const hostKey = hostId ?? null
|
||||
const resolvedNow = state === 'connected' && client !== null && !statusPending
|
||||
const blocked = compatVerdict.kind === 'blocked'
|
||||
const pending = statusPending && resolvedHostIdRef.current !== hostKey
|
||||
const holdBack = pending && mountedHostIdRef.current !== hostKey
|
||||
|
||||
// Why: React can replay or discard a render, so the latches record committed
|
||||
// outcomes only — a discarded children render must not count as mounted.
|
||||
// Why: React can replay or discard a render, so the latch records committed outcomes only.
|
||||
useEffect(() => {
|
||||
if (resolvedNow) {
|
||||
resolvedHostIdRef.current = hostKey
|
||||
}
|
||||
if (blocked) {
|
||||
// Why: the block screen unmounts the routes, so a later pending window
|
||||
// must not assume a live tree it can overlay.
|
||||
mountedHostIdRef.current = null
|
||||
} else if (!holdBack) {
|
||||
mountedHostIdRef.current = hostKey
|
||||
}
|
||||
})
|
||||
|
||||
if (holdBack) {
|
||||
// Why: nothing is mounted yet for this host, so hold the routes back entirely
|
||||
// rather than letting them mount (and fire their connect RPCs) pre-verdict.
|
||||
return (
|
||||
<View style={styles.pending}>
|
||||
<ActivityIndicator
|
||||
color={colors.textSecondary}
|
||||
accessibilityLabel="Checking host compatibility"
|
||||
/>
|
||||
</View>
|
||||
)
|
||||
}
|
||||
if (blocked) {
|
||||
return <ProtocolBlockScreen verdict={compatVerdict} />
|
||||
}
|
||||
@@ -77,10 +58,11 @@ export function HostProtocolGate({ hostId, children }: Props) {
|
||||
{children}
|
||||
</View>
|
||||
{pending ? (
|
||||
// Why: once the stack is mounted, unmounting it for a pending status.get destroys
|
||||
// in-flight nested navigation, so cover it instead. Mount effects underneath still
|
||||
// run — they wait for connState 'connected' and every capability-dependent call
|
||||
// re-probes status.get itself, so nothing newer than the baseline fires here.
|
||||
// Why: cover the stack rather than unmounting it — unmounting for a pending status.get
|
||||
// destroys in-flight nested navigation, and holding it back would serialise every route's
|
||||
// startup RPC behind this one. Mount effects underneath run pre-verdict by design; they
|
||||
// read capabilities from this gate, which reports none until the verdict lands, so every
|
||||
// capability-dependent surface stays closed rather than guessing.
|
||||
<View
|
||||
style={styles.pendingOverlay}
|
||||
// Why: the fill owns the hit test for in-tree views only — native-Modal-hosted
|
||||
@@ -100,12 +82,6 @@ export function HostProtocolGate({ hostId, children }: Props) {
|
||||
}
|
||||
|
||||
const styles = StyleSheet.create({
|
||||
pending: {
|
||||
flex: 1,
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
backgroundColor: colors.bgBase
|
||||
},
|
||||
// Stays mounted across the overlay toggling so the routes below keep their identity.
|
||||
host: {
|
||||
flex: 1
|
||||
|
||||
@@ -7,7 +7,7 @@ const probe = vi.hoisted(() => ({
|
||||
start: vi.fn()
|
||||
}))
|
||||
|
||||
vi.mock('../transport/runtime-capability-probe', () => ({
|
||||
vi.mock('../transport/runtime-status-probe', () => ({
|
||||
startRuntimeCapabilityProbe: probe.start
|
||||
}))
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
import { CODEX_RESET_CREDIT_RUNTIME_CAPABILITY } from '../../../src/shared/protocol-version'
|
||||
import type { RpcClient } from '../transport/rpc-client'
|
||||
import { startRuntimeCapabilityProbe } from '../transport/runtime-capability-probe'
|
||||
import { startRuntimeCapabilityProbe } from '../transport/runtime-status-probe'
|
||||
|
||||
// Why: source the capability string from the shared contract so a host bump can never
|
||||
// silently drift from the mobile probe.
|
||||
|
||||
@@ -1,8 +1,36 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { buildConnectionDiagnosticsReport } from './connection-diagnostics-report'
|
||||
import type { ConnectionLogEntry } from '../transport/types'
|
||||
|
||||
const NOW = Date.UTC(2026, 6, 9, 22, 0, 0)
|
||||
|
||||
function stageEntry(
|
||||
id: string,
|
||||
ts: number,
|
||||
name: string,
|
||||
ms: number,
|
||||
complete: boolean
|
||||
): ConnectionLogEntry {
|
||||
return {
|
||||
id,
|
||||
ts,
|
||||
level: complete ? 'info' : 'warn',
|
||||
path: 'relay',
|
||||
message: `Relay dial stage ${name} ${complete ? 'finished' : 'did not finish'}`,
|
||||
timing: { kind: 'relay-dial-stage', name, ms, complete }
|
||||
}
|
||||
}
|
||||
|
||||
function stateEntry(id: string, ts: number, name: string, ms: number): ConnectionLogEntry {
|
||||
return {
|
||||
id,
|
||||
ts,
|
||||
level: 'info',
|
||||
message: `Connection state ${name} → connected`,
|
||||
timing: { kind: 'connection-state', name, ms, complete: true }
|
||||
}
|
||||
}
|
||||
|
||||
describe('buildConnectionDiagnosticsReport', () => {
|
||||
it('summarizes a failing Tailscale host with its log', () => {
|
||||
const report = buildConnectionDiagnosticsReport({
|
||||
@@ -68,13 +96,28 @@ describe('buildConnectionDiagnosticsReport', () => {
|
||||
activePath: 'tailscale',
|
||||
pendingPath: 'relay',
|
||||
entries: [
|
||||
{
|
||||
id: 'relay-stage-opening',
|
||||
ts: NOW - 6_000,
|
||||
level: 'info',
|
||||
path: 'relay',
|
||||
message: 'Relay dial stage opening finished',
|
||||
detail: '118ms — resumeToken=secret-resume-token',
|
||||
timing: { kind: 'relay-dial-stage', name: 'opening', ms: 118, complete: true }
|
||||
},
|
||||
{
|
||||
id: 'relay-failure',
|
||||
ts: NOW - 5_000,
|
||||
level: 'error',
|
||||
message: 'Relay: relay dial failed',
|
||||
detail:
|
||||
'RelayDirectorHttpError: relay director resolve failed (503); retry after 30000ms; resumeToken=secret-resume-token'
|
||||
'RelayDirectorHttpError: relay director resolve failed (503); retry after 30000ms; resumeToken=secret-resume-token',
|
||||
timing: {
|
||||
kind: 'relay-dial-stage',
|
||||
name: 'awaiting-hello',
|
||||
ms: 9_100,
|
||||
complete: false
|
||||
}
|
||||
}
|
||||
],
|
||||
nowMs: NOW
|
||||
@@ -87,6 +130,9 @@ describe('buildConnectionDiagnosticsReport', () => {
|
||||
expect(report).toContain('Next step: Keep Orca open; recovery should retry automatically.')
|
||||
expect(report).toContain('resumeToken=[redacted]')
|
||||
expect(report).not.toContain('secret-resume-token')
|
||||
expect(report).toContain(
|
||||
'Relay dial stages: opening 118ms · awaiting-hello 9.1s (did not finish) — total 9.2s'
|
||||
)
|
||||
})
|
||||
|
||||
it('redacts quoted JSON credentials and never echoes an invalid endpoint', () => {
|
||||
@@ -116,6 +162,52 @@ describe('buildConnectionDiagnosticsReport', () => {
|
||||
expect(report).not.toContain('bearer-secret')
|
||||
})
|
||||
|
||||
it('breaks a slow connect down by dial stage and connection state', () => {
|
||||
const report = buildConnectionDiagnosticsReport({
|
||||
hostName: 'Host 6',
|
||||
endpoint: 'ws://192.168.1.50:6768',
|
||||
state: 'connected',
|
||||
reconnectAttempts: 2,
|
||||
lastConnectedAt: NOW,
|
||||
platform: 'ios 26.5.1',
|
||||
appVersion: '0.0.47',
|
||||
entries: [
|
||||
stageEntry('a1', NOW - 30_000, 'opening', 90, false),
|
||||
stateEntry('s1', NOW - 29_000, 'connecting', 12_000),
|
||||
stageEntry('b1', NOW - 20_000, 'opening', 120, true),
|
||||
stageEntry('b2', NOW - 19_000, 'awaiting-hello', 6_400, true),
|
||||
stageEntry('b3', NOW - 13_000, 'handshaking', 240, true),
|
||||
stageEntry('b4', NOW - 12_000, 'confirming', 1_180, true),
|
||||
stateEntry('s2', NOW - 11_000, 'connecting', 8_000)
|
||||
],
|
||||
nowMs: NOW
|
||||
})
|
||||
|
||||
// Only the latest dial is broken out, so a reconnect loop cannot average away
|
||||
// the attempt the reporter is complaining about.
|
||||
expect(report).toContain(
|
||||
'Relay dial stages (latest of 2): opening 120ms · awaiting-hello 6.4s · handshaking 240ms · confirming 1.2s — total 7.9s'
|
||||
)
|
||||
expect(report).toContain('Connection state dwell: connecting 20.0s ×2')
|
||||
})
|
||||
|
||||
it('omits the timing lines when nothing recorded a phase duration', () => {
|
||||
const report = buildConnectionDiagnosticsReport({
|
||||
hostName: 'Host 7',
|
||||
endpoint: 'ws://192.168.1.50:6768',
|
||||
state: 'connected',
|
||||
reconnectAttempts: 0,
|
||||
lastConnectedAt: NOW,
|
||||
platform: 'ios 26.5.1',
|
||||
appVersion: '0.0.47',
|
||||
entries: [{ id: 'plain', ts: NOW, level: 'info', message: 'Authenticated' }],
|
||||
nowMs: NOW
|
||||
})
|
||||
|
||||
expect(report).not.toContain('Relay dial stages')
|
||||
expect(report).not.toContain('Connection state dwell')
|
||||
})
|
||||
|
||||
it('bounds a single event line before submission while preserving its identity', () => {
|
||||
const report = buildConnectionDiagnosticsReport({
|
||||
hostName: 'Host 5',
|
||||
|
||||
@@ -8,6 +8,7 @@ import { normalizeHostAppVersion } from '../transport/host-app-version-store'
|
||||
import { formatEndpoint } from './host-reachability'
|
||||
import { diagnoseConnection } from './connection-diagnostics-analysis'
|
||||
import { redactConnectionLogEntry, redactConnectionLogText } from './connection-log-redaction'
|
||||
import { summarizeConnectionLogTimings } from './connection-log-timing-summary'
|
||||
|
||||
const MAX_EVENT_LINE_BYTES = 2 * 1024
|
||||
const EVENT_TRUNCATION_MARKER = ' … [truncated]'
|
||||
@@ -59,6 +60,7 @@ export function buildConnectionDiagnosticsReport(args: {
|
||||
? 'Last connected: never this session'
|
||||
: `Last connected: ${new Date(args.lastConnectedAt).toISOString()} (${formatAgo(now - args.lastConnectedAt)} ago)`
|
||||
)
|
||||
lines.push(...summarizeConnectionLogTimings(entries))
|
||||
lines.push('')
|
||||
lines.push(`Likely cause: ${diagnosis.likelyCause}`)
|
||||
lines.push(`Next step: ${diagnosis.nextStep}`)
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import type { ConnectionLogEntry, ConnectionLogTiming } from '../transport/types'
|
||||
|
||||
// Why: a report that only says "connecting for 10s" cannot be triaged. These lines
|
||||
// turn the per-phase timings the transport now records into the two questions
|
||||
// support actually asks: which relay dial stage ate the time, and how long the
|
||||
// client sat in each connection state.
|
||||
export function summarizeConnectionLogTimings(entries: readonly ConnectionLogEntry[]): string[] {
|
||||
const timings = entries.flatMap((entry) => (entry.timing ? [entry.timing] : []))
|
||||
const lines: string[] = []
|
||||
const dials = groupRelayDials(timings.filter((timing) => timing.kind === 'relay-dial-stage'))
|
||||
const latestDial = dials.at(-1)
|
||||
if (latestDial) {
|
||||
const label =
|
||||
dials.length > 1 ? `Relay dial stages (latest of ${dials.length})` : 'Relay dial stages'
|
||||
const total = latestDial.reduce((sum, timing) => sum + timing.ms, 0)
|
||||
lines.push(
|
||||
`${label}: ${latestDial.map(formatStageTiming).join(' · ')} — total ${formatDurationMs(total)}`
|
||||
)
|
||||
}
|
||||
const states = totalPerName(timings.filter((timing) => timing.kind === 'connection-state'))
|
||||
if (states.length > 0) {
|
||||
lines.push(
|
||||
`Connection state dwell: ${states
|
||||
.map(
|
||||
({ name, ms, count }) => `${name} ${formatDurationMs(ms)}${count > 1 ? ` ×${count}` : ''}`
|
||||
)
|
||||
.join(' · ')}`
|
||||
)
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
// Relay dial stages are strictly ordered and every dial starts in 'opening', so an
|
||||
// 'opening' timing opens a new group. Reporting only the latest keeps a reconnect
|
||||
// loop from averaging away the attempt the reporter is complaining about.
|
||||
function groupRelayDials(timings: readonly ConnectionLogTiming[]): ConnectionLogTiming[][] {
|
||||
const dials: ConnectionLogTiming[][] = []
|
||||
for (const timing of timings) {
|
||||
if (timing.name === 'opening' || dials.length === 0) {
|
||||
dials.push([])
|
||||
}
|
||||
dials.at(-1)!.push(timing)
|
||||
}
|
||||
return dials
|
||||
}
|
||||
|
||||
function totalPerName(
|
||||
timings: readonly ConnectionLogTiming[]
|
||||
): { name: string; ms: number; count: number }[] {
|
||||
const totals = new Map<string, { name: string; ms: number; count: number }>()
|
||||
for (const timing of timings) {
|
||||
const total = totals.get(timing.name) ?? { name: timing.name, ms: 0, count: 0 }
|
||||
total.ms += timing.ms
|
||||
total.count += 1
|
||||
totals.set(timing.name, total)
|
||||
}
|
||||
return [...totals.values()]
|
||||
}
|
||||
|
||||
function formatStageTiming(timing: ConnectionLogTiming): string {
|
||||
return `${timing.name} ${formatDurationMs(timing.ms)}${timing.complete ? '' : ' (did not finish)'}`
|
||||
}
|
||||
|
||||
function formatDurationMs(ms: number): string {
|
||||
return ms < 1000 ? `${Math.round(ms)}ms` : `${(ms / 1000).toFixed(1)}s`
|
||||
}
|
||||
@@ -2,6 +2,8 @@ import { Animated, View, Text, Pressable, ActivityIndicator } from 'react-native
|
||||
import { saveTerminalTextScale } from '../storage/preferences'
|
||||
import { MobileBrowserPane } from '../browser/MobileBrowserPane'
|
||||
import { TerminalPaneView } from './TerminalPaneView'
|
||||
import { TerminalEnginePrewarm } from './TerminalEnginePrewarm'
|
||||
import { MOBILE_SESSION_TAB_BAR_HEIGHT } from './mobile-session-frame-styles'
|
||||
import { MobileNativeChatOverlay } from './MobileNativeChatOverlay'
|
||||
import { colors } from '../theme/mobile-theme'
|
||||
import { styles } from './mobile-session-styles'
|
||||
@@ -74,16 +76,38 @@ export function MobileSessionActiveContent({
|
||||
activePendingTerminalTab,
|
||||
isPendingTerminalRecoveryParked,
|
||||
retryPendingTerminalRecovery,
|
||||
reconnectViewState,
|
||||
tabStripRows,
|
||||
showLoadingState,
|
||||
measurePrewarmViewport,
|
||||
showEmptyState,
|
||||
keyboardLift,
|
||||
activeTerminalKeyboardLift,
|
||||
toastAnimatedStyle,
|
||||
createTabBusy
|
||||
} = controller
|
||||
return showLoadingState ? (
|
||||
<View style={styles.emptyState}>
|
||||
<ActivityIndicator size="small" color={colors.textSecondary} />
|
||||
// Why the same list the header gates on: an unmounted tab bar gives the content row its band
|
||||
// back, so the pre-warm would measure a taller box than the pane ever gets. Reading the header's
|
||||
// own rows (live or cached preview) keeps the two from drifting.
|
||||
const prewarmReservedTabBarHeight = tabStripRows.length > 0 ? 0 : MOBILE_SESSION_TAB_BAR_HEIGHT
|
||||
// Why: the cached strip in the header is the content during a reconnect; the terminal body
|
||||
// cannot be, because replaying stored scrollback into the WebView would double-render once the
|
||||
// live stream replays the same rows. See mobile-session-reconnect-view-state. The engine still
|
||||
// boots inside the real terminal frame while the startup RPCs are in flight, so the first pane
|
||||
// inherits a warm WebView and a measured viewport (see prewarm).
|
||||
return reconnectViewState.kind === 'reconnecting-with-cache' || showLoadingState ? (
|
||||
<View style={styles.terminalFrame}>
|
||||
<View style={styles.emptyState}>
|
||||
<ActivityIndicator size="small" color={colors.textSecondary} />
|
||||
{reconnectViewState.kind === 'reconnecting-with-cache' ? (
|
||||
<Text style={styles.emptyText}>{reconnectViewState.label}</Text>
|
||||
) : null}
|
||||
</View>
|
||||
<TerminalEnginePrewarm
|
||||
reservedTabBarHeight={prewarmReservedTabBarHeight}
|
||||
textScale={terminalTextScale}
|
||||
onEngineMeasured={measurePrewarmViewport}
|
||||
/>
|
||||
</View>
|
||||
) : showEmptyState ? (
|
||||
<View style={styles.emptyState}>
|
||||
@@ -171,25 +195,35 @@ export function MobileSessionActiveContent({
|
||||
)}
|
||||
</View>
|
||||
) : activePendingTerminalTab ? (
|
||||
<View style={styles.emptyState}>
|
||||
{!isPendingTerminalRecoveryParked && (
|
||||
<ActivityIndicator size="small" color={colors.textSecondary} />
|
||||
)}
|
||||
<Text style={styles.emptyText}>
|
||||
{isPendingTerminalRecoveryParked
|
||||
? 'Terminal is taking longer than expected'
|
||||
: activePendingTerminalTab.title || 'Loading terminal'}
|
||||
</Text>
|
||||
{isPendingTerminalRecoveryParked && (
|
||||
<Pressable
|
||||
accessibilityRole="button"
|
||||
accessibilityLabel="Retry loading terminal"
|
||||
style={({ pressed }) => [styles.createButton, pressed && styles.newTerminalButtonPressed]}
|
||||
onPress={() => void retryPendingTerminalRecovery()}
|
||||
>
|
||||
<Text style={styles.createButtonText}>Retry</Text>
|
||||
</Pressable>
|
||||
)}
|
||||
<View style={styles.terminalFrame}>
|
||||
<View style={styles.emptyState}>
|
||||
{!isPendingTerminalRecoveryParked && (
|
||||
<ActivityIndicator size="small" color={colors.textSecondary} />
|
||||
)}
|
||||
<Text style={styles.emptyText}>
|
||||
{isPendingTerminalRecoveryParked
|
||||
? 'Terminal is taking longer than expected'
|
||||
: activePendingTerminalTab.title || 'Loading terminal'}
|
||||
</Text>
|
||||
{isPendingTerminalRecoveryParked && (
|
||||
<Pressable
|
||||
accessibilityRole="button"
|
||||
accessibilityLabel="Retry loading terminal"
|
||||
style={({ pressed }) => [
|
||||
styles.createButton,
|
||||
pressed && styles.newTerminalButtonPressed
|
||||
]}
|
||||
onPress={() => void retryPendingTerminalRecovery()}
|
||||
>
|
||||
<Text style={styles.createButtonText}>Retry</Text>
|
||||
</Pressable>
|
||||
)}
|
||||
</View>
|
||||
<TerminalEnginePrewarm
|
||||
reservedTabBarHeight={prewarmReservedTabBarHeight}
|
||||
textScale={terminalTextScale}
|
||||
onEngineMeasured={measurePrewarmViewport}
|
||||
/>
|
||||
</View>
|
||||
) : (
|
||||
<View
|
||||
|
||||
@@ -14,10 +14,6 @@ import { MobileSessionHeaderIconButton } from './MobileSessionHeaderIconButton'
|
||||
import { triggerMediumImpact } from '../platform/haptics'
|
||||
import { StatusDot } from '../components/StatusDot'
|
||||
import { MobileAgentIcon } from '../components/MobileAgentIcon'
|
||||
import {
|
||||
getMobileSessionTabTitle,
|
||||
resolveMobileTerminalTabAgentId
|
||||
} from './mobile-terminal-tab-agent'
|
||||
import { colors } from '../theme/mobile-theme'
|
||||
import { QuickCommandsTabButton } from './QuickCommandsTabButton'
|
||||
import { styles } from './mobile-session-styles'
|
||||
@@ -32,7 +28,6 @@ export function MobileSessionHeader({ controller }: { controller: MobileSessionC
|
||||
forceReconnectHost,
|
||||
worktreeName,
|
||||
activePanel,
|
||||
activeSessionTabId,
|
||||
activeSessionTabIdRef,
|
||||
tabStripRef,
|
||||
tabStripOffsetRef,
|
||||
@@ -52,7 +47,7 @@ export function MobileSessionHeader({ controller }: { controller: MobileSessionC
|
||||
scrollActiveTabIntoView,
|
||||
switchSessionTab,
|
||||
openSessionTabActionSheetAfterKeyboardDismiss,
|
||||
visibleTabs,
|
||||
tabStripRows,
|
||||
showConnectionRetry,
|
||||
terminalSummary,
|
||||
handlePanelTap,
|
||||
@@ -117,7 +112,7 @@ export function MobileSessionHeader({ controller }: { controller: MobileSessionC
|
||||
) : null}
|
||||
</View>
|
||||
|
||||
{visibleTabs.length > 0 && (
|
||||
{tabStripRows.length > 0 && (
|
||||
<View style={styles.tabBar}>
|
||||
{/* Why: tab taps must register on first press with the keyboard open instead of being eaten by dismissal (#5106). */}
|
||||
<ScrollView
|
||||
@@ -140,45 +135,51 @@ export function MobileSessionHeader({ controller }: { controller: MobileSessionC
|
||||
scrollActiveTabIntoView(activeSessionTabIdRef.current, false)
|
||||
}}
|
||||
>
|
||||
{visibleTabs.map((t) => (
|
||||
{tabStripRows.map(({ entry, isActive, tab }) => (
|
||||
<Pressable
|
||||
key={t.id}
|
||||
style={[styles.tab, t.id === activeSessionTabId && styles.tabActive]}
|
||||
key={entry.id}
|
||||
style={[
|
||||
styles.tab,
|
||||
isActive && styles.tabActive,
|
||||
tab === null && styles.tabPreview
|
||||
]}
|
||||
onLayout={(e) => {
|
||||
const { x, width } = e.nativeEvent.layout
|
||||
tabLayoutsRef.current.set(t.id, { x, width })
|
||||
if (t.id === activeSessionTabIdRef.current) {
|
||||
scrollActiveTabIntoView(t.id, false)
|
||||
tabLayoutsRef.current.set(entry.id, { x, width })
|
||||
if (entry.id === activeSessionTabIdRef.current) {
|
||||
scrollActiveTabIntoView(entry.id, false)
|
||||
}
|
||||
}}
|
||||
onPress={() => switchSessionTab(t)}
|
||||
onLongPress={() => {
|
||||
triggerMediumImpact()
|
||||
openSessionTabActionSheetAfterKeyboardDismiss(t)
|
||||
}}
|
||||
// A cached preview row has no live tab behind it, so both gestures need the
|
||||
// reconnect to land first.
|
||||
disabled={tab === null}
|
||||
onPress={tab === null ? undefined : () => switchSessionTab(tab)}
|
||||
onLongPress={
|
||||
tab === null
|
||||
? undefined
|
||||
: () => {
|
||||
triggerMediumImpact()
|
||||
openSessionTabActionSheetAfterKeyboardDismiss(tab)
|
||||
}
|
||||
}
|
||||
delayLongPress={400}
|
||||
>
|
||||
<View style={styles.tabLabelRow}>
|
||||
{t.type === 'browser' && (
|
||||
{entry.type === 'browser' && (
|
||||
<Globe size={13} color={colors.textSecondary} strokeWidth={2.1} />
|
||||
)}
|
||||
{t.type === 'markdown' && (
|
||||
{entry.type === 'markdown' && (
|
||||
<FileText size={13} color={colors.textSecondary} strokeWidth={2.1} />
|
||||
)}
|
||||
{t.type === 'file' && (
|
||||
{entry.type === 'file' && (
|
||||
<File size={13} color={colors.textSecondary} strokeWidth={2.1} />
|
||||
)}
|
||||
{t.type === 'agent-session' && <MobileAgentIcon agentId={t.agent} size={13} />}
|
||||
{t.type === 'terminal' &&
|
||||
(() => {
|
||||
const agentId = resolveMobileTerminalTabAgentId(t)
|
||||
return agentId ? <MobileAgentIcon agentId={agentId} size={13} /> : null
|
||||
})()}
|
||||
{entry.agentId !== null && <MobileAgentIcon agentId={entry.agentId} size={13} />}
|
||||
<Text
|
||||
style={[styles.tabText, t.id === activeSessionTabId && styles.tabTextActive]}
|
||||
style={[styles.tabText, isActive && styles.tabTextActive]}
|
||||
numberOfLines={1}
|
||||
>
|
||||
{getMobileSessionTabTitle(t)}
|
||||
{entry.title}
|
||||
</Text>
|
||||
</View>
|
||||
</Pressable>
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
import { createElement } from 'react'
|
||||
import { act, create, type ReactTestRenderer } from 'react-test-renderer'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { TerminalWebViewHandle } from '../terminal/terminal-webview-contract'
|
||||
|
||||
const engine = vi.hoisted(() => ({
|
||||
init: vi.fn((_cols: number, _rows: number) => {}),
|
||||
awaitReady: vi.fn(async () => {}),
|
||||
measureFitDimensions: vi.fn(async (_containerHeight?: number) => ({ cols: 120, rows: 40 })),
|
||||
onWebReady: null as (() => void) | null,
|
||||
textScale: undefined as number | undefined
|
||||
}))
|
||||
|
||||
vi.mock('react-native', () => ({
|
||||
StyleSheet: {
|
||||
create: <T>(styles: T) => styles,
|
||||
absoluteFillObject: { position: 'absolute', top: 0, left: 0, right: 0, bottom: 0 }
|
||||
},
|
||||
View: 'View'
|
||||
}))
|
||||
|
||||
// Stands in for the real engine: records the ref the pre-warm pane holds and the ready callback
|
||||
// it arms, so a test can drive web-ready and layout in either order.
|
||||
vi.mock('../terminal/TerminalWebView', async () => {
|
||||
const { forwardRef, useImperativeHandle } = await import('react')
|
||||
return {
|
||||
TerminalWebView: forwardRef<
|
||||
TerminalWebViewHandle,
|
||||
{ onWebReady?: () => void; textScale?: number }
|
||||
>(function MockTerminalWebView(props, ref) {
|
||||
engine.onWebReady = props.onWebReady ?? null
|
||||
engine.textScale = props.textScale
|
||||
useImperativeHandle(ref, () => engine as unknown as TerminalWebViewHandle, [])
|
||||
return createElement('MockTerminalWebView')
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
import { TerminalEnginePrewarm } from './TerminalEnginePrewarm'
|
||||
|
||||
const FRAME = { x: 0, y: 0, width: 390, height: 700 }
|
||||
|
||||
const TEXT_SCALE = 1.25
|
||||
|
||||
function renderPrewarm(onEngineMeasured: (ref: TerminalWebViewHandle, height: number) => void): {
|
||||
renderer: ReactTestRenderer
|
||||
layout: (frame: { x: number; y: number; width: number; height: number }) => void
|
||||
webReady: () => void
|
||||
} {
|
||||
let renderer: ReactTestRenderer | null = null
|
||||
act(() => {
|
||||
renderer = create(
|
||||
createElement(TerminalEnginePrewarm, {
|
||||
reservedTabBarHeight: 0,
|
||||
textScale: TEXT_SCALE,
|
||||
onEngineMeasured
|
||||
})
|
||||
)
|
||||
})
|
||||
const created = renderer as unknown as ReactTestRenderer
|
||||
return {
|
||||
renderer: created,
|
||||
layout: (frame) =>
|
||||
act(() => {
|
||||
created.root.findAllByType('View')[0]?.props.onLayout({ nativeEvent: { layout: frame } })
|
||||
}),
|
||||
webReady: () =>
|
||||
act(() => {
|
||||
engine.onWebReady?.()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The handoff now waits on the engine's ready promise, so tests have to let microtasks run.
|
||||
async function flushReady(): Promise<void> {
|
||||
await act(async () => {})
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
engine.measureFitDimensions.mockClear()
|
||||
engine.init.mockClear()
|
||||
engine.awaitReady.mockReset()
|
||||
engine.awaitReady.mockResolvedValue(undefined)
|
||||
engine.onWebReady = null
|
||||
engine.textScale = undefined
|
||||
})
|
||||
|
||||
describe('TerminalEnginePrewarm', () => {
|
||||
it('boots the engine without waiting for a terminal to attach', () => {
|
||||
const measured = vi.fn()
|
||||
const { renderer } = renderPrewarm(measured)
|
||||
// The engine mounts on the first render, so its bundle loads while the startup RPCs fly.
|
||||
expect(renderer.root.findAllByType('MockTerminalWebView')).toHaveLength(1)
|
||||
expect(measured).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('withholds the measurement until the pane has a real layout', async () => {
|
||||
const measured = vi.fn()
|
||||
const { webReady, layout } = renderPrewarm(measured)
|
||||
|
||||
webReady()
|
||||
// Why: this is the 80x24 trap — an unsized engine answers with xterm's default, and that
|
||||
// number would ride the first subscribe to the host as the PTY size.
|
||||
expect(measured).not.toHaveBeenCalled()
|
||||
|
||||
layout({ ...FRAME, width: 0, height: 0 })
|
||||
expect(measured).not.toHaveBeenCalled()
|
||||
|
||||
layout(FRAME)
|
||||
await flushReady()
|
||||
expect(measured).toHaveBeenCalledOnce()
|
||||
expect(measured.mock.calls[0]?.[1]).toBe(FRAME.height)
|
||||
})
|
||||
|
||||
it('withholds the measurement until the engine reports ready', async () => {
|
||||
const measured = vi.fn()
|
||||
const { layout, webReady } = renderPrewarm(measured)
|
||||
|
||||
layout(FRAME)
|
||||
expect(measured).not.toHaveBeenCalled()
|
||||
|
||||
webReady()
|
||||
await flushReady()
|
||||
expect(measured).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('measures once however many times layout and web-ready repeat', async () => {
|
||||
const measured = vi.fn()
|
||||
const { layout, webReady } = renderPrewarm(measured)
|
||||
|
||||
layout(FRAME)
|
||||
webReady()
|
||||
webReady()
|
||||
layout({ ...FRAME, height: 640 })
|
||||
layout(FRAME)
|
||||
await flushReady()
|
||||
|
||||
expect(measured).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('opens the engine before handing it over, because web-ready alone builds no terminal', async () => {
|
||||
const measured = vi.fn()
|
||||
let releaseReady: (() => void) | null = null
|
||||
engine.awaitReady.mockImplementation(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
releaseReady = resolve
|
||||
})
|
||||
)
|
||||
const { layout, webReady } = renderPrewarm(measured)
|
||||
|
||||
layout(FRAME)
|
||||
webReady()
|
||||
// Why: the WebView answers `measure` with null while it has no terminal, and the pane latches
|
||||
// once, so handing the engine over before init would spend the one measurement on nothing.
|
||||
expect(engine.init).toHaveBeenCalledOnce()
|
||||
expect(measured).not.toHaveBeenCalled()
|
||||
|
||||
releaseReady?.()
|
||||
await flushReady()
|
||||
expect(measured).toHaveBeenCalledOnce()
|
||||
expect(measured.mock.calls[0]?.[0]).toBe(engine)
|
||||
})
|
||||
|
||||
it('pre-warms at the text size the first pane will open with', () => {
|
||||
renderPrewarm(vi.fn())
|
||||
// Cell size is what the frame gets divided by, so a default-sized engine would measure a
|
||||
// different phone than the one the user is looking at.
|
||||
expect(engine.textScale).toBe(TEXT_SCALE)
|
||||
})
|
||||
|
||||
it('reports the frame the pane ended up with when a resize lands during engine start-up', async () => {
|
||||
const measured = vi.fn()
|
||||
let releaseReady: (() => void) | null = null
|
||||
engine.awaitReady.mockImplementation(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
releaseReady = resolve
|
||||
})
|
||||
)
|
||||
const { layout, webReady } = renderPrewarm(measured)
|
||||
|
||||
layout(FRAME)
|
||||
webReady()
|
||||
expect(measured).not.toHaveBeenCalled()
|
||||
|
||||
// A rotation or split-screen resize while the engine is still coming up. The latch has already
|
||||
// fired, so this is the last chance to correct the height the one measurement is taken against.
|
||||
const resized = { ...FRAME, width: 700, height: 360 }
|
||||
layout(resized)
|
||||
|
||||
releaseReady?.()
|
||||
await flushReady()
|
||||
|
||||
expect(measured).toHaveBeenCalledOnce()
|
||||
expect(measured.mock.calls[0]?.[1]).toBe(resized.height)
|
||||
})
|
||||
|
||||
it('drops the handoff when the pane unmounts before the engine is ready', async () => {
|
||||
const measured = vi.fn()
|
||||
let releaseReady: (() => void) | null = null
|
||||
engine.awaitReady.mockImplementation(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
releaseReady = resolve
|
||||
})
|
||||
)
|
||||
const { renderer, layout, webReady } = renderPrewarm(measured)
|
||||
layout(FRAME)
|
||||
webReady()
|
||||
|
||||
act(() => {
|
||||
renderer.unmount()
|
||||
})
|
||||
releaseReady?.()
|
||||
await flushReady()
|
||||
|
||||
// The frame this measurement was taken against is gone, so it describes nothing.
|
||||
expect(measured).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('is inert: no touches, no accessibility, and nothing sent to a terminal', async () => {
|
||||
const measured = vi.fn()
|
||||
const { renderer, layout, webReady } = renderPrewarm(measured)
|
||||
layout(FRAME)
|
||||
webReady()
|
||||
await flushReady()
|
||||
|
||||
const pane = renderer.root.findAllByType('View')[0]
|
||||
expect(pane?.props.pointerEvents).toBe('none')
|
||||
expect(pane?.props.accessibilityElementsHidden).toBe(true)
|
||||
expect(pane?.props.importantForAccessibility).toBe('no-hide-descendants')
|
||||
// The pane owns no handle, so it has no way to subscribe, send input, or resize a PTY.
|
||||
// Opening the engine is WebView-local; the measurement itself is the caller's to take.
|
||||
expect(engine.measureFitDimensions).not.toHaveBeenCalled()
|
||||
expect(measured.mock.calls[0]?.[0]).toBe(engine)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,111 @@
|
||||
import { useCallback, useRef } from 'react'
|
||||
import { StyleSheet, View, type LayoutChangeEvent } from 'react-native'
|
||||
import { TerminalWebView } from '../terminal/TerminalWebView'
|
||||
import type { TerminalWebViewHandle } from '../terminal/terminal-webview-contract'
|
||||
|
||||
// Diagnostics label for the measurement this pane contributes; it is not a PTY handle.
|
||||
export const TERMINAL_ENGINE_PREWARM_HANDLE = '(engine-prewarm)'
|
||||
|
||||
// Why: the WebView builds no xterm until it is told to, and `measure` answers null while `term`
|
||||
// is null, so the engine has to be opened before it can be asked anything. These are placeholder
|
||||
// dimensions for an empty buffer nobody reads; the measurement derives its own cols and rows from
|
||||
// the frame and the font's cell size, so nothing downstream inherits them.
|
||||
const PREWARM_INIT_COLS = 80
|
||||
const PREWARM_INIT_ROWS = 24
|
||||
|
||||
type Props = {
|
||||
// Height the tab bar will claim from the top of this frame once the session has a tab. The
|
||||
// loading state has no visible tab, so the bar is not mounted yet and the box the pane will
|
||||
// finally occupy is this much shorter. Reserving it keeps the measurement honest; measuring
|
||||
// the taller box would latch too many rows and send them to the host as the PTY size.
|
||||
reservedTabBarHeight: number
|
||||
// Why: the first pane opens at the user's saved text size, and cell size is what the
|
||||
// measurement divides the frame by. Pre-warming at a different size measures a different phone.
|
||||
textScale: number
|
||||
onEngineMeasured: (ref: TerminalWebViewHandle, frameHeight: number) => void
|
||||
}
|
||||
|
||||
// Why: a session still resolving its tabs already knows it is heading for a terminal, so load
|
||||
// the xterm engine alongside the startup RPCs instead of after terminal.list returns. This pane
|
||||
// owns no handle: it never subscribes, never sends input, and can never resize a PTY. Its only
|
||||
// output is the viewport measurement the first real pane would otherwise pay a round trip for.
|
||||
export function TerminalEnginePrewarm({
|
||||
reservedTabBarHeight,
|
||||
textScale,
|
||||
onEngineMeasured
|
||||
}: Props) {
|
||||
const engineRef = useRef<TerminalWebViewHandle | null>(null)
|
||||
const frameHeightRef = useRef(0)
|
||||
const webReadyRef = useRef(false)
|
||||
const measuredRef = useRef(false)
|
||||
|
||||
// Idempotent by construction: both triggers funnel here and the latch fires once per mount.
|
||||
const measureWhenSized = useCallback(() => {
|
||||
const engine = engineRef.current
|
||||
// Why: an unsized or unmounted WebView measures xterm's 80x24 default, and that number
|
||||
// rides the first subscribe to the host. Only a laid-out engine is allowed to answer.
|
||||
if (measuredRef.current || !webReadyRef.current || !engine || frameHeightRef.current <= 0) {
|
||||
return
|
||||
}
|
||||
measuredRef.current = true
|
||||
// `web-ready` only says the xterm bundle loaded. Opening the engine is what creates `term`,
|
||||
// and `awaitReady` is what lets its cell dimensions exist before anything reads them.
|
||||
engine.init(PREWARM_INIT_COLS, PREWARM_INIT_ROWS)
|
||||
void engine.awaitReady().then(() => {
|
||||
// React nulls the ref on unmount, so this proves the pane the frame belongs to is still up.
|
||||
if (engineRef.current !== engine) {
|
||||
return
|
||||
}
|
||||
// Why read the height here and not before the wait: a rotation or split-screen resize during
|
||||
// engine start-up re-lays out this pane, and the latch above already refused the second
|
||||
// handoff, so a height captured earlier would be the only one this pane ever reports.
|
||||
onEngineMeasured(engine, frameHeightRef.current)
|
||||
})
|
||||
}, [onEngineMeasured])
|
||||
|
||||
const handleLayout = useCallback(
|
||||
(event: LayoutChangeEvent) => {
|
||||
const { height, width } = event.nativeEvent.layout
|
||||
if (width <= 0 || height <= 0) {
|
||||
return
|
||||
}
|
||||
frameHeightRef.current = height
|
||||
measureWhenSized()
|
||||
},
|
||||
[measureWhenSized]
|
||||
)
|
||||
|
||||
const handleWebReady = useCallback(() => {
|
||||
webReadyRef.current = true
|
||||
measureWhenSized()
|
||||
}, [measureWhenSized])
|
||||
|
||||
return (
|
||||
<View
|
||||
// Why: sized like the real pane so the measurement matches, but invisible and inert so it
|
||||
// cannot paint over the loading state or steal a touch from the retry affordance above it.
|
||||
accessibilityElementsHidden
|
||||
importantForAccessibility="no-hide-descendants"
|
||||
pointerEvents="none"
|
||||
style={[styles.prewarmPane, { top: reservedTabBarHeight }]}
|
||||
onLayout={handleLayout}
|
||||
>
|
||||
<TerminalWebView
|
||||
ref={engineRef}
|
||||
style={styles.prewarmWebView}
|
||||
textScale={textScale}
|
||||
onWebReady={handleWebReady}
|
||||
/>
|
||||
</View>
|
||||
)
|
||||
}
|
||||
|
||||
const styles = StyleSheet.create({
|
||||
prewarmPane: {
|
||||
...StyleSheet.absoluteFillObject,
|
||||
opacity: 0
|
||||
},
|
||||
prewarmWebView: {
|
||||
flex: 1
|
||||
}
|
||||
})
|
||||
@@ -2,6 +2,20 @@ import { StyleSheet } from 'react-native'
|
||||
|
||||
import { colors, spacing, radii, typography } from '../theme/mobile-theme'
|
||||
|
||||
// Why one constant for the whole strip: the terminal frame is whatever the tab bar leaves behind,
|
||||
// and the engine pre-warm has to reserve exactly that much before the bar exists. Every row child
|
||||
// is pinned to this height so nothing can grow the bar without moving the reservation with it.
|
||||
//
|
||||
// The row deliberately has NO explicit height. React Native lays out border-box, so `height: 36`
|
||||
// with a 1 px top border would render a 36 px row over a 35 px content area and squeeze children
|
||||
// that are themselves 36 -- and it would leave this constant one pixel long, which is a whole row
|
||||
// of drift once a frame sits near a row boundary. Left to size itself the row takes its tallest
|
||||
// child and adds the border outside it, which is exactly the sum below.
|
||||
export const MOBILE_SESSION_TAB_BAR_CONTENT_HEIGHT = 36
|
||||
export const MOBILE_SESSION_TAB_BAR_BORDER_WIDTH = 1
|
||||
export const MOBILE_SESSION_TAB_BAR_HEIGHT =
|
||||
MOBILE_SESSION_TAB_BAR_CONTENT_HEIGHT + MOBILE_SESSION_TAB_BAR_BORDER_WIDTH
|
||||
|
||||
export const mobileSessionFrameStyles = StyleSheet.create({
|
||||
container: {
|
||||
flex: 1,
|
||||
@@ -80,12 +94,12 @@ export const mobileSessionFrameStyles = StyleSheet.create({
|
||||
tabBar: {
|
||||
flexDirection: 'row',
|
||||
alignItems: 'center',
|
||||
borderTopWidth: 1,
|
||||
borderTopWidth: MOBILE_SESSION_TAB_BAR_BORDER_WIDTH,
|
||||
borderTopColor: colors.borderSubtle
|
||||
},
|
||||
tabScroll: {
|
||||
flex: 1,
|
||||
maxHeight: 36
|
||||
maxHeight: MOBILE_SESSION_TAB_BAR_CONTENT_HEIGHT
|
||||
},
|
||||
tabContent: {
|
||||
paddingLeft: spacing.sm,
|
||||
@@ -94,7 +108,7 @@ export const mobileSessionFrameStyles = StyleSheet.create({
|
||||
tab: {
|
||||
width: 128,
|
||||
maxWidth: 128,
|
||||
minHeight: 36,
|
||||
minHeight: MOBILE_SESSION_TAB_BAR_CONTENT_HEIGHT,
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
paddingHorizontal: spacing.sm,
|
||||
@@ -102,6 +116,11 @@ export const mobileSessionFrameStyles = StyleSheet.create({
|
||||
borderBottomWidth: 2,
|
||||
borderBottomColor: 'transparent'
|
||||
},
|
||||
// Why: a cached row is inert until the reconnect lands, so it carries the same de-emphasis as
|
||||
// the disabled tab-bar buttons beside it rather than passing for a live tab.
|
||||
tabPreview: {
|
||||
opacity: 0.45
|
||||
},
|
||||
tabActive: {
|
||||
// Neutral grey underline, matching the desktop terminal tab's active
|
||||
// indicator (a muted foreground/card mix), not a blue accent.
|
||||
@@ -123,7 +142,7 @@ export const mobileSessionFrameStyles = StyleSheet.create({
|
||||
},
|
||||
newTerminalButton: {
|
||||
width: 40,
|
||||
height: 36,
|
||||
height: MOBILE_SESSION_TAB_BAR_CONTENT_HEIGHT,
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
borderBottomWidth: 2,
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { selectMobileSessionReconnectViewState } from './mobile-session-reconnect-view-state'
|
||||
import {
|
||||
getMobileSessionTabStripRows,
|
||||
toMobileSessionTabStripPreview,
|
||||
type MobileSessionTabStripPreview
|
||||
} from './mobile-session-tab-strip-entries'
|
||||
import type { MobileSessionTab } from './mobile-session-route-types'
|
||||
|
||||
function terminalTab(id: string, title: string, isActive = false): MobileSessionTab {
|
||||
return { type: 'terminal', id, title, terminal: `h-${id}`, isActive }
|
||||
}
|
||||
|
||||
const cachedPreview: MobileSessionTabStripPreview = {
|
||||
tabs: [
|
||||
{ id: 'tab-1', type: 'terminal', title: 'claude', agentId: 'claude' },
|
||||
{ id: 'tab-2', type: 'terminal', title: 'shell', agentId: null }
|
||||
],
|
||||
activeTabId: 'tab-1'
|
||||
}
|
||||
|
||||
const base = {
|
||||
connState: 'reconnecting',
|
||||
verdictKind: 'normal',
|
||||
terminalsLoaded: false,
|
||||
liveTabCount: 0,
|
||||
activeHandle: null,
|
||||
cachedPreview: null
|
||||
} as const
|
||||
|
||||
describe('selectMobileSessionReconnectViewState', () => {
|
||||
it('renders the cached strip with a progress label while reconnecting', () => {
|
||||
const state = selectMobileSessionReconnectViewState({ ...base, cachedPreview })
|
||||
|
||||
expect(state).toEqual({
|
||||
kind: 'reconnecting-with-cache',
|
||||
preview: cachedPreview,
|
||||
label: 'Reconnecting…'
|
||||
})
|
||||
})
|
||||
|
||||
it('labels the post-connect hydration gap as loading, not reconnecting', () => {
|
||||
const state = selectMobileSessionReconnectViewState({
|
||||
...base,
|
||||
connState: 'connected',
|
||||
cachedPreview
|
||||
})
|
||||
|
||||
expect(state.kind === 'reconnecting-with-cache' && state.label).toBe('Loading tabs…')
|
||||
})
|
||||
|
||||
it('blocks when nothing is cached for this workspace', () => {
|
||||
expect(selectMobileSessionReconnectViewState(base)).toEqual({ kind: 'blocking' })
|
||||
expect(
|
||||
selectMobileSessionReconnectViewState({
|
||||
...base,
|
||||
cachedPreview: { tabs: [], activeTabId: null }
|
||||
})
|
||||
).toEqual({ kind: 'blocking' })
|
||||
})
|
||||
|
||||
it('keeps mounted live content instead of swapping in its own cached snapshot', () => {
|
||||
expect(
|
||||
selectMobileSessionReconnectViewState({ ...base, liveTabCount: 2, cachedPreview })
|
||||
).toEqual({ kind: 'live' })
|
||||
expect(
|
||||
selectMobileSessionReconnectViewState({ ...base, activeHandle: 'h-1', cachedPreview })
|
||||
).toEqual({ kind: 'live' })
|
||||
})
|
||||
|
||||
it('treats a host-confirmed empty workspace as live', () => {
|
||||
expect(
|
||||
selectMobileSessionReconnectViewState({
|
||||
...base,
|
||||
connState: 'connected',
|
||||
terminalsLoaded: true,
|
||||
cachedPreview
|
||||
})
|
||||
).toEqual({ kind: 'live' })
|
||||
})
|
||||
|
||||
it('falls back to the offline state once the retry loop or the pairing has failed', () => {
|
||||
expect(
|
||||
selectMobileSessionReconnectViewState({ ...base, verdictKind: 'unreachable', cachedPreview })
|
||||
).toEqual({ kind: 'offline' })
|
||||
expect(
|
||||
selectMobileSessionReconnectViewState({ ...base, verdictKind: 'auth-failed', cachedPreview })
|
||||
).toEqual({ kind: 'offline' })
|
||||
})
|
||||
|
||||
it('keeps showing the cache through a transient warning verdict', () => {
|
||||
expect(
|
||||
selectMobileSessionReconnectViewState({ ...base, verdictKind: 'warning', cachedPreview }).kind
|
||||
).toBe('reconnecting-with-cache')
|
||||
})
|
||||
})
|
||||
|
||||
describe('getMobileSessionTabStripRows', () => {
|
||||
it('draws disabled preview rows while reconnecting, then the live tabs under the same keys', () => {
|
||||
const preview = selectMobileSessionReconnectViewState({ ...base, cachedPreview })
|
||||
const previewRows = getMobileSessionTabStripRows({
|
||||
liveTabs: [],
|
||||
activeSessionTabId: null,
|
||||
preview: preview.kind === 'reconnecting-with-cache' ? preview.preview : null
|
||||
})
|
||||
|
||||
expect(previewRows.map((row) => row.entry.id)).toEqual(['tab-1', 'tab-2'])
|
||||
expect(previewRows.map((row) => row.tab)).toEqual([null, null])
|
||||
expect(previewRows.map((row) => row.isActive)).toEqual([true, false])
|
||||
|
||||
const liveTabs = [terminalTab('tab-1', 'claude', true), terminalTab('tab-2', 'shell')]
|
||||
const liveRows = getMobileSessionTabStripRows({
|
||||
liveTabs,
|
||||
activeSessionTabId: 'tab-1',
|
||||
preview: null
|
||||
})
|
||||
|
||||
expect(liveRows.map((row) => row.entry.id)).toEqual(previewRows.map((row) => row.entry.id))
|
||||
expect(liveRows.map((row) => row.isActive)).toEqual(previewRows.map((row) => row.isActive))
|
||||
expect(liveRows.every((row) => row.tab !== null)).toBe(true)
|
||||
})
|
||||
|
||||
it('prefers live tabs over a preview that is still present', () => {
|
||||
const rows = getMobileSessionTabStripRows({
|
||||
liveTabs: [terminalTab('tab-9', 'fresh', true)],
|
||||
activeSessionTabId: 'tab-9',
|
||||
preview: cachedPreview
|
||||
})
|
||||
|
||||
expect(rows.map((row) => row.entry.id)).toEqual(['tab-9'])
|
||||
})
|
||||
|
||||
it('keeps only the drawn fields when projecting a preview to persist', () => {
|
||||
const preview = toMobileSessionTabStripPreview(
|
||||
[
|
||||
{
|
||||
type: 'terminal',
|
||||
id: 'tab-1',
|
||||
title: 'claude',
|
||||
terminal: 'h-1',
|
||||
launchAgent: 'claude',
|
||||
launchDraft: 'unsent secret prompt',
|
||||
isActive: true
|
||||
}
|
||||
],
|
||||
'tab-1'
|
||||
)
|
||||
|
||||
expect(preview).toEqual({
|
||||
tabs: [{ id: 'tab-1', type: 'terminal', title: 'claude', agentId: 'claude' }],
|
||||
activeTabId: 'tab-1'
|
||||
})
|
||||
expect(JSON.stringify(preview)).not.toContain('unsent secret prompt')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,61 @@
|
||||
import type { ConnectionVerdict } from '../transport/connection-health'
|
||||
import type { ConnectionState } from '../transport/types'
|
||||
import type { MobileSessionTabStripPreview } from './mobile-session-tab-strip-entries'
|
||||
|
||||
/**
|
||||
* What the session screen should draw while the phone is not yet serving live tabs.
|
||||
*
|
||||
* - `live`: real tabs are mounted (or the host has confirmed there are none). The existing
|
||||
* loading/empty/content branches own the screen.
|
||||
* - `reconnecting-with-cache`: nothing live yet, but this workspace's last strip is on the
|
||||
* device. Draw it, disabled, with a compact progress line instead of a bare spinner.
|
||||
* - `offline`: the retry loop has given up or the pairing is rejected. A stale strip would
|
||||
* imply a session we cannot reach, so fall back to the existing offline affordance.
|
||||
* - `blocking`: nothing live and nothing cached. Unchanged from before this state existed.
|
||||
*/
|
||||
export type MobileSessionReconnectViewState =
|
||||
| { kind: 'live' }
|
||||
| { kind: 'reconnecting-with-cache'; preview: MobileSessionTabStripPreview; label: string }
|
||||
| { kind: 'offline' }
|
||||
| { kind: 'blocking' }
|
||||
|
||||
export function selectMobileSessionReconnectViewState(args: {
|
||||
connState: ConnectionState
|
||||
verdictKind: ConnectionVerdict['kind']
|
||||
terminalsLoaded: boolean
|
||||
liveTabCount: number
|
||||
activeHandle: string | null
|
||||
cachedPreview: MobileSessionTabStripPreview | null
|
||||
}): MobileSessionReconnectViewState {
|
||||
const { connState, verdictKind, terminalsLoaded, liveTabCount, activeHandle, cachedPreview } =
|
||||
args
|
||||
// A mounted terminal or tab is the real thing; a mid-session drop must never trade it for a
|
||||
// snapshot of itself, however the connection is faring.
|
||||
if (liveTabCount > 0 || activeHandle !== null) {
|
||||
return { kind: 'live' }
|
||||
}
|
||||
// The host has answered and said this workspace is empty — that is live truth, not a gap.
|
||||
if (connState === 'connected' && terminalsLoaded) {
|
||||
return { kind: 'live' }
|
||||
}
|
||||
if (verdictKind === 'unreachable' || verdictKind === 'auth-failed') {
|
||||
return { kind: 'offline' }
|
||||
}
|
||||
if (cachedPreview && cachedPreview.tabs.length > 0) {
|
||||
return {
|
||||
kind: 'reconnecting-with-cache',
|
||||
preview: cachedPreview,
|
||||
label: reconnectProgressLabel(connState)
|
||||
}
|
||||
}
|
||||
return { kind: 'blocking' }
|
||||
}
|
||||
|
||||
function reconnectProgressLabel(connState: ConnectionState): string {
|
||||
if (connState === 'connected') {
|
||||
return 'Loading tabs…'
|
||||
}
|
||||
return connState === 'reconnecting' || connState === 'disconnected'
|
||||
? 'Reconnecting…'
|
||||
: 'Connecting…'
|
||||
}
|
||||
@@ -37,6 +37,7 @@ const LOGIC_EXPANSION_NAMES = new Set([
|
||||
'useMobileSessionContentCreateActions',
|
||||
'useMobileSessionCloseActions',
|
||||
'useMobileSessionBulkClose',
|
||||
'useMobileSessionTabStripCache',
|
||||
'useMobileSessionPresentation',
|
||||
'useMobileSessionPanelRouteActions'
|
||||
])
|
||||
@@ -62,32 +63,32 @@ const HOST_COMPONENT_NAMES = new Set([
|
||||
'View'
|
||||
])
|
||||
|
||||
const HEAD_MAIN_HOOK_SHA256 = '10071240ef9edafc2b9c8bed73be83dceaf7828e3b29f17dab55da020a7697a6'
|
||||
const HEAD_HOOK_BINDING_SHA256 = '1dadb8c3dc0573ea20659ce7251629669e618dd0effaeac3a4536b29c2e865a1'
|
||||
const HEAD_MAIN_HOOK_SHA256 = 'e22e7d3a1147ef19c747f0e216b778e794a73e027e96cf84fac2dde03b37b640'
|
||||
const HEAD_HOOK_BINDING_SHA256 = '531fe06cf2c261b1346bbc949c9ceba5aea8b8ace2dcb8a1898e9759745e013c'
|
||||
const HEAD_CALLBACK_IDENTITY_SHA256 =
|
||||
'2a9e4825df007f6ef53b81aa5004991d6318eee7507b44d625c07e630be432eb'
|
||||
const HEAD_CALLBACK_BODY_SHA256 = '22103ba85a86e3a3fcb80a7509c7a455d79863010cde3af02db6565b55e3ebe9'
|
||||
const HEAD_EFFECT_SHA256 = 'd9ebfaabc1e79773cdada7ab370b20459ed972f1f8edce1652199f4d0391cd13'
|
||||
'e5df1043256bcb0b3813bf89161d91f5e65c00749fbb6d98176bca82e878d061'
|
||||
const HEAD_CALLBACK_BODY_SHA256 = '6d9ed614ed139aef5cc911c33ea4220cc1fc5f888a1a564ef85e6910cc118bc3'
|
||||
const HEAD_EFFECT_SHA256 = 'a6d4d5cb573926f40faa7701cef7885a0f2c7e7c5cfaa91f4e480c29aba44d79'
|
||||
const HEAD_CONTENT_HOOK_SHA256 = '9c3b612fef3f370d66873aefdbe1d701f20cb64ded31fef5cc45fde6f8189581'
|
||||
const HEAD_NESTED_FUNCTION_SHA256 =
|
||||
'536c72b233c813bb0cea164b090bdce5406ceb965bbc5b83c1f89b89b46f3821'
|
||||
'0e553eb5ec7aeda8f8336b8da85ff87eb3657a21fa32d3c75c9cc32e36860244'
|
||||
const HEAD_NATIVE_REGISTRATION_SHA256 =
|
||||
'cab85e4e4a3f43289ba93ddea9ccce57aea83e0bf14fd1620a965aad0c1cb49e'
|
||||
const HEAD_NATIVE_REMOVAL_SHA256 =
|
||||
'4c994574675a2a0f9c607b3ea89ab7a2ed5a83f7c72fa42342ddcb5f00fc3f4f'
|
||||
const HEAD_TIMER_CREATION_SHA256 =
|
||||
'1a31b625e2174c3db77272249843196d2b6b06ab1e654a96d8f7858e3082e66b'
|
||||
const HEAD_TIMER_CLEANUP_SHA256 = 'c73f1d1c2cc89642f3d727d6f3b6b81860a9d6f34234541a2065ec3d1a8cd116'
|
||||
'36c3ccef371698e25cd2eb239df7a8dea6dcc674d9da43cc38cabfa3a8f64929'
|
||||
const HEAD_TIMER_CLEANUP_SHA256 = '2f41ddc30d0e9c1b6d1d6b5e09d96d1b3facd3133acae1ff7436bb40e4ef39dc'
|
||||
const HEAD_RUNTIME_STRING_SHA256 =
|
||||
'31951b0b83be01ebfa659c4b94df9ad7eaff6404df5338fbade89eb7473a3cb4'
|
||||
const HEAD_HOST_JSX_SHA256 = '390405926b1695fa3a33686f0bc192b432f5468d8576499d7cafbb4922defbb5'
|
||||
const HEAD_LEAF_JSX_SHA256 = '21dba981875e173f692590bf910d60964660c5f4cbb79f3a377c7e54f6a1f016'
|
||||
'694a22ed924ebc2a7d380089ff2cfd3e27f5d72d3c4d4b7b06aa3006db93c053'
|
||||
const HEAD_HOST_JSX_SHA256 = '0aca9fe4b6738228020fe20334fe2716471a2fdf57e4feea6a2a92cac1c04c58'
|
||||
const HEAD_LEAF_JSX_SHA256 = '2c38e19ffbcaae14f9df2fdb44751546d2b936f9a4b2c5e90727a5f74f3c2665'
|
||||
const HEAD_STYLE_REFERENCE_SHA256 =
|
||||
'295a3501c2c6d7bea7c8bbf38b3f3534f01344cd7e1b91bb8e07c040821d596a'
|
||||
'da81d6065c5c1ebafbbd721321023cddd0bfc1afa0325749f736bb97898f9556'
|
||||
const HEAD_IDENTITY_FIELD_SHA256 =
|
||||
'91146853930a34dd1f3d80e5c97fbacd7cf19fb93dd26fe8fc6f29169622f9d6'
|
||||
'a7444b7d0953edb34abc77180ba11d458b02081547b8499249571efd30ac0609'
|
||||
const HEAD_NAVIGATION_SHA256 = '9d96f5dad7de555d6553eac39c0fab00efad507470fd562cb9beaa32db16f512'
|
||||
const HEAD_CAPABILITY_SHA256 = 'ca219f7909a091717110b823d5b94a20770ad3ae51894e0fa765e8628309392d'
|
||||
const HEAD_CAPABILITY_SHA256 = '54c74cdb468d015c31517004e005187f6cff2ddb07e25fdb4a7060a2fac6b786'
|
||||
|
||||
type Definition = { declaration: ts.FunctionDeclaration; sourceFile: ts.SourceFile }
|
||||
type HookFacts = {
|
||||
@@ -456,8 +457,10 @@ function readCompatibilityFacts(definitions: ReadonlyMap<string, Definition>): {
|
||||
: ''
|
||||
const callText = canonical(node, sourceFile)
|
||||
if (
|
||||
['startRuntimeCapabilityProbe', 'supportsMobileQuickCommands'].includes(callName) ||
|
||||
(callName === 'includes' && callText.includes('capabilities.includes'))
|
||||
// hostCapabilities.* is included: the session route now reads the gate's shared status.get
|
||||
// answer instead of running its own probe, and those reads still have to stay ratcheted.
|
||||
['useHostProtocolGates', 'supportsMobileQuickCommands'].includes(callName) ||
|
||||
(callName === 'includes' && /[cC]apabilities\.includes/.test(callText))
|
||||
) {
|
||||
capabilities.push(callText)
|
||||
}
|
||||
@@ -472,18 +475,18 @@ describe('mobile session route extraction parity', () => {
|
||||
const contentBindings = CONTENT_COMPONENT_NAMES.flatMap(
|
||||
(name) => readHookFacts(name, definitions).bindings
|
||||
)
|
||||
expect(main.hooks).toHaveLength(266)
|
||||
expect(main.hooks).toHaveLength(272)
|
||||
expect(hash(main.hooks)).toBe(HEAD_MAIN_HOOK_SHA256)
|
||||
expect(hash(main.bindings)).toBe(HEAD_HOOK_BINDING_SHA256)
|
||||
expect(main.callbacks).toHaveLength(77)
|
||||
expect(main.callbacks).toHaveLength(78)
|
||||
expect(hash(main.callbacks)).toBe(HEAD_CALLBACK_IDENTITY_SHA256)
|
||||
expect(hash(main.callbackBodies)).toBe(HEAD_CALLBACK_BODY_SHA256)
|
||||
expect(main.effects).toHaveLength(24)
|
||||
expect(main.effects).toHaveLength(27)
|
||||
expect(hash(main.effects)).toBe(HEAD_EFFECT_SHA256)
|
||||
expect(contentBindings).toHaveLength(14)
|
||||
expect(hash(contentBindings)).toBe(HEAD_CONTENT_HOOK_SHA256)
|
||||
const nestedFunctions = readNestedFunctions(definitions)
|
||||
expect(nestedFunctions).toHaveLength(12)
|
||||
expect(nestedFunctions).toHaveLength(13)
|
||||
expect(hash(nestedFunctions)).toBe(HEAD_NESTED_FUNCTION_SHA256)
|
||||
})
|
||||
|
||||
@@ -494,20 +497,23 @@ describe('mobile session route extraction parity', () => {
|
||||
expect(hash(native.registrations)).toBe(HEAD_NATIVE_REGISTRATION_SHA256)
|
||||
expect(native.removals).toHaveLength(9)
|
||||
expect(hash(native.removals)).toBe(HEAD_NATIVE_REMOVAL_SHA256)
|
||||
expect(native.creations.filter((fact) => fact.startsWith('setTimeout'))).toHaveLength(7)
|
||||
expect(native.creations.filter((fact) => fact.startsWith('setTimeout'))).toHaveLength(8)
|
||||
expect(native.creations.filter((fact) => fact.startsWith('setInterval'))).toHaveLength(1)
|
||||
expect(
|
||||
native.creations.filter((fact) => fact.startsWith('requestAnimationFrame'))
|
||||
).toHaveLength(1)
|
||||
expect(hash(native.creations)).toBe(HEAD_TIMER_CREATION_SHA256)
|
||||
expect(native.cleanups.filter((fact) => fact.startsWith('clearTimeout'))).toHaveLength(11)
|
||||
expect(native.cleanups.filter((fact) => fact.startsWith('clearTimeout'))).toHaveLength(12)
|
||||
expect(native.cleanups.filter((fact) => fact.startsWith('clearInterval'))).toHaveLength(1)
|
||||
expect(native.cleanups.filter((fact) => fact.startsWith('cancelAnimationFrame'))).toHaveLength(
|
||||
1
|
||||
)
|
||||
expect(hash(native.cleanups)).toBe(HEAD_TIMER_CLEANUP_SHA256)
|
||||
const compatibility = readCompatibilityFacts(definitions)
|
||||
expect(compatibility.identityFields).toHaveLength(14)
|
||||
// 13, not 14: both worktree.activate call sites now share one payload builder, so the
|
||||
// literal `notifyClients: false` they used to repeat appears once. The guarantee itself is
|
||||
// pinned in mobile-session-startup-source.test.ts, which requires exactly one call site.
|
||||
expect(compatibility.identityFields).toHaveLength(13)
|
||||
expect(hash(compatibility.identityFields)).toBe(HEAD_IDENTITY_FIELD_SHA256)
|
||||
expect(compatibility.navigation).toHaveLength(6)
|
||||
expect(hash(compatibility.navigation)).toBe(HEAD_NAVIGATION_SHA256)
|
||||
@@ -517,14 +523,14 @@ describe('mobile session route extraction parity', () => {
|
||||
|
||||
it('preserves runtime strings, styles, and the expanded JSX tree', () => {
|
||||
const strings = readRuntimeStrings()
|
||||
expect(strings).toHaveLength(546)
|
||||
expect(strings).toHaveLength(545)
|
||||
expect(hash(strings)).toBe(HEAD_RUNTIME_STRING_SHA256)
|
||||
const jsx = readJsxFacts(readDefinitions())
|
||||
expect(jsx.host).toHaveLength(124)
|
||||
expect(jsx.host).toHaveLength(127)
|
||||
expect(hash(jsx.host)).toBe(HEAD_HOST_JSX_SHA256)
|
||||
expect(jsx.leaf).toHaveLength(61)
|
||||
expect(jsx.leaf).toHaveLength(62)
|
||||
expect(hash(jsx.leaf)).toBe(HEAD_LEAF_JSX_SHA256)
|
||||
expect(jsx.styleReferences).toHaveLength(172)
|
||||
expect(jsx.styleReferences).toHaveLength(176)
|
||||
expect(hash(jsx.styleReferences)).toBe(HEAD_STYLE_REFERENCE_SHA256)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -33,6 +33,7 @@ export const MOBILE_SESSION_ROUTE_SOURCE_FILES = [
|
||||
'./use-mobile-session-content-create-actions.ts',
|
||||
'./use-mobile-session-close-actions.ts',
|
||||
'./use-mobile-session-bulk-close.ts',
|
||||
'./use-mobile-session-tab-strip-cache.ts',
|
||||
'./use-mobile-session-presentation.ts',
|
||||
'./use-mobile-session-panel-route-actions.tsx',
|
||||
'./MobileSessionMarkdownReader.tsx',
|
||||
|
||||
@@ -0,0 +1,279 @@
|
||||
import { createElement, type ReactElement } from 'react'
|
||||
import { act, create, type ReactTestRenderer } from 'react-test-renderer'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { useMobileSessionStartup } from './use-mobile-session-startup'
|
||||
import type { MobileSessionKeyboardStateModel } from './use-mobile-session-keyboard-state'
|
||||
|
||||
type Deferred<T> = { promise: Promise<T>; resolve: (value: T) => void; reject: (e: Error) => void }
|
||||
|
||||
function defer<T>(): Deferred<T> {
|
||||
let resolve!: (value: T) => void
|
||||
let reject!: (error: Error) => void
|
||||
const promise = new Promise<T>((res, rej) => {
|
||||
resolve = res
|
||||
reject = rej
|
||||
})
|
||||
return { promise, resolve, reject }
|
||||
}
|
||||
|
||||
type StartupCall = { rpc: 'session.tabs.list' | 'terminal.list'; worktreeId: string }
|
||||
|
||||
// One session's worth of scope: only the fields useMobileSessionStartup actually reads, plus
|
||||
// the two reads under test wired to deferreds so a test controls exactly when they settle.
|
||||
function makeScope(worktreeId: string, calls: StartupCall[], protocolVerified = true) {
|
||||
const tabs = defer<void>()
|
||||
const terminals = defer<boolean>()
|
||||
const sendRequest = vi.fn().mockResolvedValue({ ok: true, result: {} })
|
||||
const scope = {
|
||||
hostId: 'host-1',
|
||||
worktreeId,
|
||||
created: '0',
|
||||
isFloatingWorkspaceRoute: false,
|
||||
connState: 'connected',
|
||||
client: { sendRequest },
|
||||
protocolVerified,
|
||||
setTerminals: vi.fn(),
|
||||
terminalsRef: { current: [] },
|
||||
setSessionTabs: vi.fn(),
|
||||
appliedSnapshotMarkerRef: { current: { epoch: null, version: -1 } },
|
||||
closedTabTombstonesRef: { current: new Map() },
|
||||
setTerminalsLoaded: vi.fn(),
|
||||
setActiveHandle: vi.fn(),
|
||||
setActiveSessionTabId: vi.fn(),
|
||||
setMarkdownDocs: vi.fn(),
|
||||
setFileDocs: vi.fn(),
|
||||
terminalGestureInputQueuesRef: { current: new Map() },
|
||||
terminalGestureInputInFlightRef: { current: new Set() },
|
||||
sessionTabActionSheetKeyboardHideSubRef: { current: null },
|
||||
sessionTabActionSheetRequestSeqRef: { current: 0 },
|
||||
initializedHandlesRef: { current: new Set<string>() },
|
||||
terminalDiagnosticsRef: { current: { resetRoute: vi.fn() } },
|
||||
activeHandleRef: { current: null },
|
||||
activeSessionTabTypeRef: { current: null },
|
||||
pendingActiveSessionTabIdRef: { current: null },
|
||||
selectedSessionTabIdRef: { current: null },
|
||||
pendingActiveTerminalHandleRef: { current: null },
|
||||
pendingBrowserFocusPageIdRef: { current: null },
|
||||
pendingTerminalActivationAttemptRef: { current: null },
|
||||
initialSessionAutoCreateRef: { current: null },
|
||||
bufferedTerminalDraftState: { resetDrafts: vi.fn(), clearPendingRestorations: vi.fn() },
|
||||
clearPendingLiveInputCommit: vi.fn(),
|
||||
clearDelayedActionTimers: vi.fn(),
|
||||
showToast: vi.fn(),
|
||||
clearTerminalCache: vi.fn(),
|
||||
fetchTerminals: vi.fn(() => {
|
||||
calls.push({ rpc: 'terminal.list', worktreeId })
|
||||
return terminals.promise
|
||||
}),
|
||||
ensureSessionTabs: vi.fn(() => {
|
||||
calls.push({ rpc: 'session.tabs.list', worktreeId })
|
||||
return tabs.promise
|
||||
})
|
||||
}
|
||||
return {
|
||||
scope: scope as unknown as MobileSessionKeyboardStateModel,
|
||||
tabs,
|
||||
terminals,
|
||||
sendRequest,
|
||||
activateCalls: () =>
|
||||
sendRequest.mock.calls.filter(([method]) => method === 'worktree.activate').length
|
||||
}
|
||||
}
|
||||
|
||||
function StartupHarness({
|
||||
scope
|
||||
}: {
|
||||
scope: MobileSessionKeyboardStateModel
|
||||
}): ReactElement | null {
|
||||
useMobileSessionStartup(scope)
|
||||
return null
|
||||
}
|
||||
|
||||
async function flush(): Promise<void> {
|
||||
await act(async () => {
|
||||
await Promise.resolve()
|
||||
await Promise.resolve()
|
||||
await Promise.resolve()
|
||||
})
|
||||
}
|
||||
|
||||
describe('mobile session startup parallelism', () => {
|
||||
let renderer: ReactTestRenderer | null = null
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers({ shouldAdvanceTime: true })
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
act(() => renderer?.unmount())
|
||||
renderer = null
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('puts session.tabs.list and terminal.list on the wire together', async () => {
|
||||
const calls: StartupCall[] = []
|
||||
const { scope } = makeScope('wt-1', calls)
|
||||
|
||||
await act(async () => {
|
||||
renderer = create(createElement(StartupHarness, { scope }))
|
||||
await Promise.resolve()
|
||||
})
|
||||
await flush()
|
||||
|
||||
// Neither deferred has settled, so both requests are in flight at the same moment. Under the
|
||||
// old chain the second call could not have been made until the first resolved.
|
||||
expect(calls).toEqual([
|
||||
{ rpc: 'session.tabs.list', worktreeId: 'wt-1' },
|
||||
{ rpc: 'terminal.list', worktreeId: 'wt-1' }
|
||||
])
|
||||
})
|
||||
|
||||
it('isolates each read so one rejection cannot strand the follow-up refreshes', async () => {
|
||||
const calls: StartupCall[] = []
|
||||
const { scope, tabs, terminals } = makeScope('wt-1', calls)
|
||||
|
||||
await act(async () => {
|
||||
renderer = create(createElement(StartupHarness, { scope }))
|
||||
await Promise.resolve()
|
||||
})
|
||||
await flush()
|
||||
|
||||
await act(async () => {
|
||||
tabs.reject(new Error('tabs rejected'))
|
||||
terminals.reject(new Error('terminals rejected'))
|
||||
await Promise.resolve()
|
||||
})
|
||||
await flush()
|
||||
|
||||
await act(async () => {
|
||||
vi.advanceTimersByTime(1600)
|
||||
await Promise.resolve()
|
||||
})
|
||||
// The 750 ms and 1500 ms follow-up refreshes still armed despite both rejections; an
|
||||
// unguarded await would have thrown out of the startup block and armed neither.
|
||||
expect(calls.filter((call) => call.rpc === 'terminal.list')).toHaveLength(3)
|
||||
})
|
||||
|
||||
it('drops results that land after the route moved to another session', async () => {
|
||||
const calls: StartupCall[] = []
|
||||
const first = makeScope('wt-1', calls)
|
||||
const second = makeScope('wt-2', calls)
|
||||
|
||||
await act(async () => {
|
||||
renderer = create(createElement(StartupHarness, { scope: first.scope }))
|
||||
await Promise.resolve()
|
||||
})
|
||||
await flush()
|
||||
|
||||
await act(async () => {
|
||||
renderer?.update(createElement(StartupHarness, { scope: second.scope }))
|
||||
await Promise.resolve()
|
||||
})
|
||||
await flush()
|
||||
|
||||
// The first session's reads land only now, after its effect was torn down.
|
||||
await act(async () => {
|
||||
first.tabs.resolve(undefined)
|
||||
first.terminals.resolve(true)
|
||||
await Promise.resolve()
|
||||
})
|
||||
await flush()
|
||||
await act(async () => {
|
||||
vi.advanceTimersByTime(1600)
|
||||
await Promise.resolve()
|
||||
})
|
||||
|
||||
// Why: a stale settlement must not schedule refreshes for a worktree the route has left.
|
||||
expect(calls.filter((call) => call.worktreeId === 'wt-1')).toHaveLength(2)
|
||||
})
|
||||
|
||||
it('withholds worktree.activate until the compatibility verdict lands', async () => {
|
||||
const calls: StartupCall[] = []
|
||||
const pending = makeScope('wt-1', calls, false)
|
||||
|
||||
await act(async () => {
|
||||
renderer = create(createElement(StartupHarness, { scope: pending.scope }))
|
||||
await Promise.resolve()
|
||||
})
|
||||
await flush()
|
||||
|
||||
// Why: a desktop that omits protocolVersion evaluates as version 0 and IS blocked, so the
|
||||
// routes that now mount pre-verdict must not mutate a host the gate is about to refuse.
|
||||
expect(pending.activateCalls()).toBe(0)
|
||||
// The reads are not held back with it; that is the whole point of mounting early.
|
||||
expect(calls).toHaveLength(2)
|
||||
})
|
||||
|
||||
it('activates once the verdict lands without re-issuing the reads', async () => {
|
||||
const calls: StartupCall[] = []
|
||||
const pending = makeScope('wt-1', calls, false)
|
||||
|
||||
await act(async () => {
|
||||
renderer = create(createElement(StartupHarness, { scope: pending.scope }))
|
||||
await Promise.resolve()
|
||||
})
|
||||
await flush()
|
||||
expect(pending.activateCalls()).toBe(0)
|
||||
|
||||
// Same session, verdict now proven: only the activation effect may re-run.
|
||||
const verified = {
|
||||
...(pending.scope as unknown as Record<string, unknown>),
|
||||
protocolVerified: true
|
||||
} as unknown as MobileSessionKeyboardStateModel
|
||||
await act(async () => {
|
||||
renderer?.update(createElement(StartupHarness, { scope: verified }))
|
||||
await Promise.resolve()
|
||||
})
|
||||
await flush()
|
||||
|
||||
expect(pending.activateCalls()).toBe(1)
|
||||
expect(pending.sendRequest).toHaveBeenCalledWith('worktree.activate', {
|
||||
worktree: 'id:wt-1',
|
||||
notifyClients: false,
|
||||
navigation: 'caller'
|
||||
})
|
||||
expect(calls).toHaveLength(2)
|
||||
})
|
||||
|
||||
it('discards both parallel results when the session changes mid-flight', async () => {
|
||||
const calls: StartupCall[] = []
|
||||
const first = makeScope('wt-1', calls)
|
||||
const second = makeScope('wt-2', calls)
|
||||
|
||||
await act(async () => {
|
||||
renderer = create(createElement(StartupHarness, { scope: first.scope }))
|
||||
await Promise.resolve()
|
||||
})
|
||||
await flush()
|
||||
expect(calls.filter((call) => call.worktreeId === 'wt-1')).toHaveLength(2)
|
||||
|
||||
await act(async () => {
|
||||
renderer?.update(createElement(StartupHarness, { scope: second.scope }))
|
||||
await Promise.resolve()
|
||||
})
|
||||
await flush()
|
||||
|
||||
// Tabs land late first, then terminals, so each is separately proven inert.
|
||||
await act(async () => {
|
||||
first.tabs.resolve(undefined)
|
||||
await Promise.resolve()
|
||||
})
|
||||
await flush()
|
||||
await act(async () => {
|
||||
vi.advanceTimersByTime(1600)
|
||||
await Promise.resolve()
|
||||
})
|
||||
expect(calls.filter((call) => call.worktreeId === 'wt-1')).toHaveLength(2)
|
||||
|
||||
await act(async () => {
|
||||
first.terminals.resolve(true)
|
||||
await Promise.resolve()
|
||||
})
|
||||
await flush()
|
||||
await act(async () => {
|
||||
vi.advanceTimersByTime(1600)
|
||||
await Promise.resolve()
|
||||
})
|
||||
expect(calls.filter((call) => call.worktreeId === 'wt-1')).toHaveLength(2)
|
||||
})
|
||||
})
|
||||
@@ -30,6 +30,10 @@ const autoCreateHookSource = readMobileSessionRouteSource(
|
||||
'./use-initial-session-terminal-autocreate.ts'
|
||||
)
|
||||
const foundationSource = readMobileSessionRouteSource('./use-mobile-session-foundation.ts')
|
||||
const activeContentSource = readMobileSessionRouteSource('./MobileSessionActiveContent.tsx')
|
||||
const subscriptionFoundationSource = readMobileSessionRouteSource(
|
||||
'./use-mobile-session-terminal-subscription-foundation.ts'
|
||||
)
|
||||
const terminalRuntimeSource = readMobileSessionRouteSource(
|
||||
'./use-mobile-session-terminal-runtime.ts'
|
||||
)
|
||||
@@ -147,35 +151,72 @@ describe('mobile session startup', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('loads session tabs without waiting for desktop activation', () => {
|
||||
const startupEffect = sliceBetween(
|
||||
// Was: one effect that awaited tabs, then terminals, and fired worktree.activate alongside them.
|
||||
// The reads are now concurrent and unblocked, while the activation moved to its own effect that
|
||||
// waits for the compatibility verdict, because it writes host state.
|
||||
it('loads session tabs and terminals concurrently, ahead of any desktop activation', () => {
|
||||
const readEffect = sliceBetween(
|
||||
'void (async () => {',
|
||||
'return () => {\n disposed = true',
|
||||
startupSource
|
||||
)
|
||||
|
||||
expect(startupEffect).toContain("void client\n .sendRequest('worktree.activate'")
|
||||
expect(startupEffect).toContain("if (client && created !== '1' && !isFloatingWorkspaceRoute)")
|
||||
expect(startupEffect).toContain("if (client && created === '1' && !isFloatingWorkspaceRoute)")
|
||||
expect(startupEffect).toContain('notifyClients: false')
|
||||
expect(startupEffect).toContain("navigation: 'caller'")
|
||||
expect(startupEffect).not.toContain("await client\n .sendRequest('worktree.activate'")
|
||||
expect(startupEffect.indexOf("sendRequest('worktree.activate'")).toBeLessThan(
|
||||
startupEffect.indexOf('await ensureSessionTabs()')
|
||||
expect(readEffect).toContain(
|
||||
'await Promise.all([\n ensureSessionTabs().catch(() => null),\n fetchTerminals({ allowEmptyLoaded: false }).catch(() => false)\n ])'
|
||||
)
|
||||
expect(startupEffect).toContain('headlessActivationNeedsHostRenderer(response.result)')
|
||||
expect(startupEffect).toContain("showToast('Open Orca on the host to wake sleeping agents.'")
|
||||
// The reads must not wait on the verdict; that is the point of mounting under the gate.
|
||||
expect(readEffect).not.toContain('protocolVerified')
|
||||
expect(readEffect).not.toContain('worktree.activate')
|
||||
expect(startupSource).toContain('}, [connState, fetchTerminals, ensureSessionTabs])')
|
||||
})
|
||||
|
||||
it('fails runtime capability gates closed before probing a replacement client', () => {
|
||||
it('holds worktree.activate until the compatibility verdict lands', () => {
|
||||
const activationEffect = sliceBetween(
|
||||
"if (connState !== 'connected' || !client || !protocolVerified || isFloatingWorkspaceRoute) {",
|
||||
'return () => {\n disposed = true',
|
||||
startupSource.slice(startupSource.indexOf('worktree.activate') - 2000)
|
||||
)
|
||||
|
||||
// Why: a desktop that omits protocolVersion reads as version 0 and IS blocked, so mounting
|
||||
// this route pre-verdict must not let it mutate a host the gate is about to refuse.
|
||||
expect(activationEffect).toContain("sendRequest('worktree.activate'")
|
||||
expect(activationEffect).toContain('notifyClients: false')
|
||||
expect(activationEffect).toContain("navigation: 'caller'")
|
||||
expect(activationEffect).toContain("if (created !== '1') {")
|
||||
expect(activationEffect).toContain('headlessActivationNeedsHostRenderer(response.result)')
|
||||
expect(activationEffect).toContain("showToast('Open Orca on the host to wake sleeping agents.'")
|
||||
// The only worktree.activate calls in the route are the two this gated effect owns.
|
||||
expect(startupSource.split("sendRequest('worktree.activate'")).toHaveLength(2)
|
||||
expect(startupSource).toContain(' protocolVerified,\n showToast,\n worktreeId\n ])')
|
||||
})
|
||||
|
||||
// Was: this route ran its own retrying status.get. The gate above every /h/ route already
|
||||
// holds that answer, so the second request is gone and the gates read it instead.
|
||||
it('fails runtime capability gates closed until the shared status.get is proven', () => {
|
||||
const capabilityEffect = sliceBetween(
|
||||
'const hostQueryReplyInputSupportedRef = useRef(false)',
|
||||
'return {\n consumeAcceptedSessionTabs',
|
||||
tabReconciliationSource
|
||||
)
|
||||
const probeStart = capabilityEffect.indexOf('startRuntimeCapabilityProbe(client,')
|
||||
|
||||
expect(probeStart).toBeGreaterThanOrEqual(0)
|
||||
expect(tabReconciliationSource).not.toContain('startRuntimeCapabilityProbe')
|
||||
expect(tabReconciliationSource).not.toContain('useHostProtocolGates')
|
||||
// One read of the gate for the whole route, taken in the foundation and passed down.
|
||||
expect(foundationSource).toContain(
|
||||
'const { compatVerdict, compatVerified, hostCapabilities, statusPending } = useHostProtocolGates()'
|
||||
)
|
||||
// Settled is not passing, and passing-by-fallback is not answered. The write gate reads all
|
||||
// three, so a host that never answered status.get cannot be mistaken for a verified one.
|
||||
expect(foundationSource).toContain(
|
||||
"const protocolVerified = !statusPending && compatVerified && compatVerdict.kind === 'ok'"
|
||||
)
|
||||
expect(capabilityEffect).toContain(
|
||||
"if (!client || connState !== 'connected' || !protocolVerified) {"
|
||||
)
|
||||
const readStart = capabilityEffect.indexOf(
|
||||
"setBrowserScreencastSupported(hostCapabilities.includes('browser.screencast.v1'))"
|
||||
)
|
||||
expect(readStart).toBeGreaterThanOrEqual(0)
|
||||
for (const reset of [
|
||||
'setBrowserScreencastSupported(null)',
|
||||
'setAgentSessionHistorySupported(null)',
|
||||
@@ -185,7 +226,7 @@ describe('mobile session startup', () => {
|
||||
]) {
|
||||
const resetIndex = capabilityEffect.lastIndexOf(reset)
|
||||
expect(resetIndex).toBeGreaterThanOrEqual(0)
|
||||
expect(resetIndex).toBeLessThan(probeStart)
|
||||
expect(resetIndex).toBeLessThan(readStart)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -290,4 +331,43 @@ describe('mobile session startup', () => {
|
||||
expect(source).toContain('onPendingTerminalRecoveryParked: setParkedPendingTerminalContext')
|
||||
expect(source).toContain('retryPendingTerminalRecovery()')
|
||||
})
|
||||
|
||||
it('boots the terminal engine while the startup reads are still in flight', () => {
|
||||
// Why: the loading and pending-terminal states are exactly the window in which the startup
|
||||
// RPCs are outstanding, so the engine loads there rather than after terminal.list answers.
|
||||
const loadingBranch = sliceBetween(
|
||||
"return reconnectViewState.kind === 'reconnecting-with-cache' || showLoadingState ? (",
|
||||
') : showEmptyState ? (',
|
||||
activeContentSource
|
||||
)
|
||||
const prewarmElement =
|
||||
'<TerminalEnginePrewarm\n reservedTabBarHeight={prewarmReservedTabBarHeight}\n textScale={terminalTextScale}\n onEngineMeasured={measurePrewarmViewport}\n />'
|
||||
expect(loadingBranch).toContain(prewarmElement)
|
||||
expect(loadingBranch).toContain('<View style={styles.terminalFrame}>')
|
||||
|
||||
const pendingBranch = sliceBetween(
|
||||
') : activePendingTerminalTab ? (',
|
||||
') : (\n <View\n style={styles.terminalFrame}',
|
||||
activeContentSource
|
||||
)
|
||||
expect(pendingBranch).toContain(prewarmElement)
|
||||
|
||||
// The pre-warm never reaches a terminal: the pane list is still the only attachment point.
|
||||
expect(activeContentSource).toContain('{terminals.map((terminal) => (')
|
||||
expect(activeContentSource.indexOf('<TerminalEnginePrewarm')).toBeLessThan(
|
||||
activeContentSource.indexOf('{terminals.map((terminal) => (')
|
||||
)
|
||||
})
|
||||
|
||||
it('refuses a pre-warm viewport measured before the frame had a height', () => {
|
||||
const measure = sliceBetween(
|
||||
'const measurePrewarmViewport = useCallback(',
|
||||
' return {\n getTerminalRef',
|
||||
subscriptionFoundationSource
|
||||
)
|
||||
expect(measure).toContain('if (viewportMeasuredRef.current || frameHeight <= 0) {')
|
||||
expect(measure).toContain('await engine.measureFitDimensions(frameHeight)')
|
||||
// Why: the latch is re-checked after the await so a real pane that measured first wins.
|
||||
expect(measure).toContain('if (dims && !viewportMeasuredRef.current) {')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
import { TUI_AGENT_DISPLAY_NAMES } from '../../../src/shared/tui-agent-display-names'
|
||||
import type { MobileSessionTab, MobileSessionTabType } from './mobile-session-route-types'
|
||||
import {
|
||||
getMobileSessionTabTitle,
|
||||
resolveMobileTerminalTabAgentId
|
||||
} from './mobile-terminal-tab-agent'
|
||||
|
||||
/**
|
||||
* The only session-tab fields the tab strip draws. Everything else the live tab carries (unsent
|
||||
* launch drafts, absolute file paths, browser URLs, agent session ids) stays on the wire.
|
||||
*/
|
||||
export type MobileSessionTabStripEntry = {
|
||||
id: string
|
||||
type: MobileSessionTabType
|
||||
title: string
|
||||
agentId: string | null
|
||||
}
|
||||
|
||||
export type MobileSessionTabStripPreview = {
|
||||
tabs: readonly MobileSessionTabStripEntry[]
|
||||
activeTabId: string | null
|
||||
}
|
||||
|
||||
export type MobileSessionTabStripRow = {
|
||||
entry: MobileSessionTabStripEntry
|
||||
isActive: boolean
|
||||
/** null on a preview row: switching to that tab needs a live connection. */
|
||||
tab: MobileSessionTab | null
|
||||
}
|
||||
|
||||
export function toMobileSessionTabStripEntry(tab: MobileSessionTab): MobileSessionTabStripEntry {
|
||||
return {
|
||||
id: tab.id,
|
||||
type: tab.type,
|
||||
title: getMobileSessionTabTitle(tab),
|
||||
agentId:
|
||||
tab.type === 'agent-session'
|
||||
? tab.agent
|
||||
: tab.type === 'terminal'
|
||||
? resolveMobileTerminalTabAgentId(tab)
|
||||
: null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Every tab type the strip knows how to draw. A stored entry naming anything else is dropped
|
||||
* rather than trusted, so a type added later fails closed: its rows go missing from the preview
|
||||
* instead of carrying an unreviewed title into storage.
|
||||
*/
|
||||
const drawableTabTypes = new Set<string>([
|
||||
'terminal',
|
||||
'markdown',
|
||||
'file',
|
||||
'browser',
|
||||
'agent-session'
|
||||
] satisfies readonly MobileSessionTabType[])
|
||||
|
||||
export function isDrawableTabStripType(type: string): type is MobileSessionTabType {
|
||||
return drawableTabTypes.has(type)
|
||||
}
|
||||
|
||||
const agentDisplayNames: Readonly<Record<string, string>> = TUI_AGENT_DISPLAY_NAMES
|
||||
|
||||
/**
|
||||
* The title a strip entry may be written to disk under.
|
||||
*
|
||||
* A terminal's title is whatever the shell last set, which is routinely the command line —
|
||||
* `psql postgres://user:password@host/db`, `curl -H "Authorization: Bearer ..."`. None of that
|
||||
* belongs in plaintext storage, and a browser tab's page title is no better. Both collapse to a
|
||||
* fixed label, so what survives is the shape of the strip, not its contents. A resolved agent
|
||||
* still names itself, because that lookup is a closed enum: an unrecognised id yields the
|
||||
* generic label rather than passing text through.
|
||||
*/
|
||||
export function getPersistableTabStripTitle(
|
||||
entry: Pick<MobileSessionTabStripEntry, 'type' | 'title' | 'agentId'>
|
||||
): string {
|
||||
if (entry.type === 'terminal') {
|
||||
const agentLabel = entry.agentId === null ? undefined : agentDisplayNames[entry.agentId]
|
||||
return agentLabel ?? 'Terminal'
|
||||
}
|
||||
if (entry.type === 'browser') {
|
||||
return 'Browser'
|
||||
}
|
||||
return entry.title
|
||||
}
|
||||
|
||||
export function toMobileSessionTabStripPreview(
|
||||
tabs: readonly MobileSessionTab[],
|
||||
activeTabId: string | null
|
||||
): MobileSessionTabStripPreview {
|
||||
return { tabs: tabs.map(toMobileSessionTabStripEntry), activeTabId }
|
||||
}
|
||||
|
||||
/**
|
||||
* Rows for the header strip. Live tabs always win; the preview only fills a strip that has no
|
||||
* live rows yet, and its ids are the live ids, so the swap reuses the same React keys.
|
||||
*/
|
||||
export function getMobileSessionTabStripRows(args: {
|
||||
liveTabs: readonly MobileSessionTab[]
|
||||
activeSessionTabId: string | null
|
||||
preview: MobileSessionTabStripPreview | null
|
||||
}): MobileSessionTabStripRow[] {
|
||||
const { liveTabs, activeSessionTabId, preview } = args
|
||||
if (liveTabs.length > 0 || !preview) {
|
||||
return liveTabs.map((tab) => ({
|
||||
entry: toMobileSessionTabStripEntry(tab),
|
||||
isActive: tab.id === activeSessionTabId,
|
||||
tab
|
||||
}))
|
||||
}
|
||||
return preview.tabs.map((entry) => ({
|
||||
entry,
|
||||
isActive: entry.id === preview.activeTabId,
|
||||
tab: null
|
||||
}))
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
import { createElement } from 'react'
|
||||
import { act, create, type ReactTestRenderer } from 'react-test-renderer'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { TerminalWebViewHandle } from '../terminal/terminal-webview-contract'
|
||||
import { readMobileSessionRouteSource } from './mobile-session-route-source-family.test-support'
|
||||
|
||||
type StyleLayer = { top?: number }
|
||||
|
||||
// The applied top offset, read off the rendered pane rather than assumed.
|
||||
function appliedTopOffset(style: unknown): number {
|
||||
const layers = (Array.isArray(style) ? style : [style]) as (StyleLayer | null | undefined)[]
|
||||
return layers.reduce<number>(
|
||||
(top, layer) => (typeof layer?.top === 'number' ? layer.top : top),
|
||||
0
|
||||
)
|
||||
}
|
||||
|
||||
const engine = vi.hoisted(() => ({
|
||||
init: vi.fn((_cols: number, _rows: number) => {}),
|
||||
awaitReady: vi.fn(async () => {}),
|
||||
measureFitDimensions: vi.fn(async (_containerHeight?: number) => ({ cols: 100, rows: 40 })),
|
||||
onWebReady: null as (() => void) | null
|
||||
}))
|
||||
|
||||
vi.mock('react-native', () => ({
|
||||
StyleSheet: {
|
||||
create: <T>(styles: T) => styles,
|
||||
absoluteFillObject: { position: 'absolute', top: 0, left: 0, right: 0, bottom: 0 }
|
||||
},
|
||||
View: 'View'
|
||||
}))
|
||||
|
||||
vi.mock('../terminal/TerminalWebView', async () => {
|
||||
const { forwardRef, useImperativeHandle } = await import('react')
|
||||
return {
|
||||
TerminalWebView: forwardRef<TerminalWebViewHandle, { onWebReady?: () => void }>(
|
||||
function MockTerminalWebView(props, ref) {
|
||||
engine.onWebReady = props.onWebReady ?? null
|
||||
useImperativeHandle(ref, () => engine as unknown as TerminalWebViewHandle, [])
|
||||
return createElement('MockTerminalWebView')
|
||||
}
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
import { TerminalEnginePrewarm } from './TerminalEnginePrewarm'
|
||||
import {
|
||||
MOBILE_SESSION_TAB_BAR_BORDER_WIDTH,
|
||||
MOBILE_SESSION_TAB_BAR_CONTENT_HEIGHT,
|
||||
MOBILE_SESSION_TAB_BAR_HEIGHT,
|
||||
mobileSessionFrameStyles
|
||||
} from './mobile-session-frame-styles'
|
||||
|
||||
// The box the session content row occupies. Both states below live in it, so it is the one
|
||||
// number the two frame heights are derived from.
|
||||
const CONTENT_ROW_HEIGHT = 700
|
||||
|
||||
// The bar's rendered height, derived from the styles the header actually mounts rather than from
|
||||
// the constant the pre-warm consumes — otherwise the comparison below would just restate itself.
|
||||
// React Native sizes a row with no explicit height to its tallest child and puts the border
|
||||
// outside that, so this is max(children) + border.
|
||||
function renderedTabBarHeight(): number {
|
||||
const row = mobileSessionFrameStyles.tabBar as { height?: number; borderTopWidth: number }
|
||||
// An explicit height here would be border-box and would shrink the row below its children.
|
||||
expect(row.height).toBeUndefined()
|
||||
const tallestChild = Math.max(
|
||||
mobileSessionFrameStyles.tabScroll.maxHeight,
|
||||
mobileSessionFrameStyles.tab.minHeight,
|
||||
mobileSessionFrameStyles.newTerminalButton.height,
|
||||
mobileSessionFrameStyles.tabActionDivider.height
|
||||
)
|
||||
return tallestChild + row.borderTopWidth
|
||||
}
|
||||
|
||||
// What the first real pane gets once its tab exists and the bar mounts above it.
|
||||
function firstPaneFrameHeight(): number {
|
||||
return CONTENT_ROW_HEIGHT - renderedTabBarHeight()
|
||||
}
|
||||
const headerSource = readMobileSessionRouteSource('./MobileSessionHeader.tsx')
|
||||
const activeContentSource = readMobileSessionRouteSource('./MobileSessionActiveContent.tsx')
|
||||
|
||||
// Reproduces React Native's absolute-fill layout: a box pinned to every edge of its parent with
|
||||
// a top offset gets exactly that much less height. The offset is read off the component, never
|
||||
// assumed, so a pre-warm that stopped reserving the bar would report the taller box here.
|
||||
function measuredPrewarmHeight(reservedTabBarHeight: number): number {
|
||||
let renderer: ReactTestRenderer | null = null
|
||||
act(() => {
|
||||
renderer = create(
|
||||
createElement(TerminalEnginePrewarm, { reservedTabBarHeight, onEngineMeasured: () => {} })
|
||||
)
|
||||
})
|
||||
const created = renderer as unknown as ReactTestRenderer
|
||||
const applied = appliedTopOffset(created.root.findAllByType('View')[0]?.props.style)
|
||||
act(() => created.unmount())
|
||||
return CONTENT_ROW_HEIGHT - applied
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
engine.measureFitDimensions.mockClear()
|
||||
engine.onWebReady = null
|
||||
})
|
||||
|
||||
describe('terminal pre-warm frame geometry', () => {
|
||||
it('states the height the bar actually renders at', () => {
|
||||
// The constant is what the pre-warm reserves, so it has to equal what the header mounts.
|
||||
// Deriving the latter from the styles catches the border-box trap: pinning an explicit
|
||||
// height on the row would render it a pixel short of this sum and drift a whole row.
|
||||
expect(renderedTabBarHeight()).toBe(MOBILE_SESSION_TAB_BAR_HEIGHT)
|
||||
expect(MOBILE_SESSION_TAB_BAR_HEIGHT).toBe(
|
||||
MOBILE_SESSION_TAB_BAR_CONTENT_HEIGHT + MOBILE_SESSION_TAB_BAR_BORDER_WIDTH
|
||||
)
|
||||
expect(mobileSessionFrameStyles.tabBar.borderTopWidth).toBe(MOBILE_SESSION_TAB_BAR_BORDER_WIDTH)
|
||||
// Every child is pinned to the content height, so nothing can grow the row unnoticed.
|
||||
expect(mobileSessionFrameStyles.tabScroll.maxHeight).toBe(MOBILE_SESSION_TAB_BAR_CONTENT_HEIGHT)
|
||||
expect(mobileSessionFrameStyles.tab.minHeight).toBe(MOBILE_SESSION_TAB_BAR_CONTENT_HEIGHT)
|
||||
expect(mobileSessionFrameStyles.newTerminalButton.height).toBe(
|
||||
MOBILE_SESSION_TAB_BAR_CONTENT_HEIGHT
|
||||
)
|
||||
})
|
||||
|
||||
it('mounts the tab bar only once a tab is visible, which is what shortens the pane', () => {
|
||||
expect(headerSource).toContain(
|
||||
'{tabStripRows.length > 0 && (\n <View style={styles.tabBar}>'
|
||||
)
|
||||
// So the reservation has to be the exact complement of that condition, read off the same rows
|
||||
// the header gates on (live tabs or the cached reconnect preview) rather than a proxy for it.
|
||||
expect(activeContentSource).toContain(
|
||||
'const prewarmReservedTabBarHeight = tabStripRows.length > 0 ? 0 : MOBILE_SESSION_TAB_BAR_HEIGHT'
|
||||
)
|
||||
})
|
||||
|
||||
it('measures the same frame height the first real pane will get', () => {
|
||||
// Loading: no visible tab, so no tab bar, so the content row is all the pre-warm's to fill,
|
||||
// minus whatever it reserves. Loaded: the first terminal produces a tab, the bar mounts, and
|
||||
// the pane gets what is left. The right side is derived from the header's own styles.
|
||||
expect(measuredPrewarmHeight(MOBILE_SESSION_TAB_BAR_HEIGHT)).toBe(firstPaneFrameHeight())
|
||||
})
|
||||
|
||||
it('would latch a taller frame than the pane if the bar were not reserved', () => {
|
||||
// Guards the fix rather than the code: without the reservation the pre-warm measures the
|
||||
// pre-tab-bar box, and every row of that difference is a row the host never had.
|
||||
const unreserved = measuredPrewarmHeight(0)
|
||||
expect(unreserved).toBe(CONTENT_ROW_HEIGHT)
|
||||
expect(unreserved - firstPaneFrameHeight()).toBe(renderedTabBarHeight())
|
||||
})
|
||||
|
||||
it('hands the engine the reserved height, so no refit is owed after the first subscribe', async () => {
|
||||
let measuredWith: number | null = null
|
||||
let renderer: ReactTestRenderer | null = null
|
||||
act(() => {
|
||||
renderer = create(
|
||||
createElement(TerminalEnginePrewarm, {
|
||||
reservedTabBarHeight: MOBILE_SESSION_TAB_BAR_HEIGHT,
|
||||
textScale: 1,
|
||||
onEngineMeasured: (_ref: unknown, frameHeight: number) => {
|
||||
measuredWith = frameHeight
|
||||
}
|
||||
})
|
||||
)
|
||||
})
|
||||
const created = renderer as unknown as ReactTestRenderer
|
||||
const pane = created.root.findAllByType('View')[0]
|
||||
const applied = appliedTopOffset(pane?.props.style)
|
||||
act(() => {
|
||||
pane?.props.onLayout({
|
||||
nativeEvent: { layout: { x: 0, y: 0, width: 390, height: CONTENT_ROW_HEIGHT - applied } }
|
||||
})
|
||||
})
|
||||
act(() => {
|
||||
engine.onWebReady?.()
|
||||
})
|
||||
// The handoff waits on the engine's ready promise, so let those microtasks land.
|
||||
await act(async () => {})
|
||||
|
||||
// The height the latched viewport is computed from equals the real pane's frame height, so
|
||||
// the frame-height refit re-measures the same cols/rows and returns before it would send
|
||||
// terminal.updateViewport (see the prev-dims guard in terminal-viewport-refit.ts).
|
||||
expect(measuredWith).toBe(firstPaneFrameHeight())
|
||||
act(() => created.unmount())
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,147 @@
|
||||
import { createElement, useRef, type ReactElement } from 'react'
|
||||
import { act, create, type ReactTestRenderer } from 'react-test-renderer'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { RpcClient } from '../transport/rpc-client'
|
||||
import type { TerminalWebViewHandle } from '../terminal/terminal-webview-contract'
|
||||
|
||||
vi.mock('react-native', () => ({
|
||||
AppState: { currentState: 'active', addEventListener: () => ({ remove: () => {} }) },
|
||||
Platform: { OS: 'android' },
|
||||
StyleSheet: {
|
||||
create: <T>(styles: T) => styles,
|
||||
absoluteFillObject: { position: 'absolute', top: 0, left: 0, right: 0, bottom: 0 },
|
||||
hairlineWidth: 1
|
||||
},
|
||||
useWindowDimensions: () => ({ width: 390, height: 844 }),
|
||||
View: 'View'
|
||||
}))
|
||||
|
||||
import { useTerminalViewportRefit } from '../terminal/terminal-viewport-refit'
|
||||
import { MOBILE_SESSION_TAB_BAR_HEIGHT } from './mobile-session-frame-styles'
|
||||
|
||||
const CONTENT_ROW_HEIGHT = 700
|
||||
const CELL_HEIGHT = 17
|
||||
const HANDLE = 'term-1'
|
||||
const REFIT_DEBOUNCE_MS = 150
|
||||
|
||||
// Stands in for the WebView's fit: the taller the box it is handed, the more rows it reports.
|
||||
// This is what turns a frame that is one tab bar too tall into a row count the host never had.
|
||||
function fitDimensions(containerHeight: number): { cols: number; rows: number } {
|
||||
return { cols: 100, rows: Math.floor(containerHeight / CELL_HEIGHT) }
|
||||
}
|
||||
|
||||
type ColdOpenResult = {
|
||||
updateViewportCalls: number
|
||||
resubscribes: number
|
||||
latchedRows: number
|
||||
}
|
||||
|
||||
// Replays a single-terminal cold open: the pre-warm measured `prewarmFrameHeight` and latched it,
|
||||
// the first pane subscribed with those dims, and only then does the real frame report its layout.
|
||||
async function runSingleTerminalColdOpen(prewarmFrameHeight: number): Promise<ColdOpenResult> {
|
||||
const firstPaneFrameHeight = CONTENT_ROW_HEIGHT - MOBILE_SESSION_TAB_BAR_HEIGHT
|
||||
const sendRequest = vi.fn(async () => ({ ok: true, result: { updated: true, applied: true } }))
|
||||
const client = {
|
||||
sendRequest,
|
||||
updateTerminalSubscriptionViewport: vi.fn()
|
||||
} as unknown as RpcClient
|
||||
const engine = {
|
||||
measureFitDimensions: vi.fn(async (containerHeight?: number) =>
|
||||
fitDimensions(containerHeight ?? 0)
|
||||
),
|
||||
reflow: vi.fn()
|
||||
} as unknown as TerminalWebViewHandle
|
||||
const subscribeToTerminal = vi.fn()
|
||||
const unsubscribeTerminal = vi.fn()
|
||||
const viewport = { current: fitDimensions(prewarmFrameHeight) as { cols: number; rows: number } }
|
||||
const viewportMeasured = { current: true }
|
||||
// The real pane's frame, reported by its onLayout once the tab bar has mounted.
|
||||
const frameHeight = { current: firstPaneFrameHeight }
|
||||
|
||||
let notify: ((height: number) => void) | null = null
|
||||
function RefitHarness(): ReactElement | null {
|
||||
const terminalRefs = useRef(new Map([[HANDLE, engine]]))
|
||||
const { notifyTerminalFrameHeight } = useTerminalViewportRefit({
|
||||
activeHandleRef: useRef<string | null>(HANDLE),
|
||||
terminalRefs,
|
||||
terminalFrameHeightRef: frameHeight,
|
||||
viewportRef: viewport,
|
||||
viewportMeasuredRef: viewportMeasured,
|
||||
nativeChatCoveredRef: useRef(false),
|
||||
clientRef: useRef<RpcClient | null>(client),
|
||||
deviceTokenRef: useRef<string | null>('device-1'),
|
||||
initializedHandlesRef: useRef(new Set([HANDLE])),
|
||||
connState: 'connected',
|
||||
// One terminal, so the tab-strip corrector is not armed — this is the case that used to
|
||||
// fall through to the frame-height reducer and pay for the mis-measurement.
|
||||
tabStripVisible: false,
|
||||
textScale: 1,
|
||||
terminalFrameWidth: 390,
|
||||
unsubscribeTerminal,
|
||||
subscribeToTerminal
|
||||
})
|
||||
notify = notifyTerminalFrameHeight
|
||||
return null
|
||||
}
|
||||
|
||||
let renderer: ReactTestRenderer | null = null
|
||||
await act(async () => {
|
||||
renderer = create(createElement(RefitHarness))
|
||||
})
|
||||
await act(async () => {
|
||||
notify?.(firstPaneFrameHeight)
|
||||
})
|
||||
// Why drain microtasks between ticks and before unmount: the refit measures and sends inside an
|
||||
// async block that bails once disposedRef flips, so tearing down early would fake a clean run.
|
||||
await act(async () => {
|
||||
vi.advanceTimersByTime(REFIT_DEBOUNCE_MS + 1)
|
||||
for (let i = 0; i < 10; i += 1) {
|
||||
await Promise.resolve()
|
||||
}
|
||||
})
|
||||
await act(async () => {
|
||||
vi.advanceTimersByTime(REFIT_DEBOUNCE_MS + 1)
|
||||
for (let i = 0; i < 10; i += 1) {
|
||||
await Promise.resolve()
|
||||
}
|
||||
})
|
||||
act(() => (renderer as unknown as ReactTestRenderer).unmount())
|
||||
|
||||
return {
|
||||
updateViewportCalls: sendRequest.mock.calls.filter(
|
||||
([method]) => method === 'terminal.updateViewport'
|
||||
).length,
|
||||
resubscribes: subscribeToTerminal.mock.calls.length,
|
||||
latchedRows: viewport.current.rows
|
||||
}
|
||||
}
|
||||
|
||||
describe('terminal pre-warm refit debt', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers({ shouldAdvanceTime: true })
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('owes the host nothing after the first subscribe when the pre-warm reserved the tab bar', async () => {
|
||||
const reserved = CONTENT_ROW_HEIGHT - MOBILE_SESSION_TAB_BAR_HEIGHT
|
||||
const result = await runSingleTerminalColdOpen(reserved)
|
||||
|
||||
expect(result.updateViewportCalls).toBe(0)
|
||||
expect(result.resubscribes).toBe(0)
|
||||
expect(result.latchedRows).toBe(fitDimensions(reserved).rows)
|
||||
})
|
||||
|
||||
it('pays a terminal.updateViewport round trip if the pre-warm measured the pre-tab-bar box', async () => {
|
||||
// Guards the fix, not the code: this is the frame the pre-warm saw before it reserved the bar.
|
||||
const result = await runSingleTerminalColdOpen(CONTENT_ROW_HEIGHT)
|
||||
|
||||
expect(result.updateViewportCalls).toBe(1)
|
||||
// And the rows it had to correct are rows the host was told about and never had.
|
||||
expect(fitDimensions(CONTENT_ROW_HEIGHT).rows).toBeGreaterThan(
|
||||
fitDimensions(CONTENT_ROW_HEIGHT - MOBILE_SESSION_TAB_BAR_HEIGHT).rows
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -27,6 +27,7 @@ import { useMobileSessionTerminalCreateActions } from './use-mobile-session-term
|
||||
import { useMobileSessionContentCreateActions } from './use-mobile-session-content-create-actions'
|
||||
import { useMobileSessionCloseActions } from './use-mobile-session-close-actions'
|
||||
import { useMobileSessionBulkClose } from './use-mobile-session-bulk-close'
|
||||
import { useMobileSessionTabStripCache } from './use-mobile-session-tab-strip-cache'
|
||||
import { useMobileSessionPresentation } from './use-mobile-session-presentation'
|
||||
import { useMobileSessionPanelRouteActions } from './use-mobile-session-panel-route-actions'
|
||||
|
||||
@@ -113,7 +114,8 @@ export function useMobileSessionController() {
|
||||
useMobileSessionCloseActions(contentCreateActions)
|
||||
)
|
||||
const bulkClose = Object.assign(closeActions, useMobileSessionBulkClose(closeActions))
|
||||
const presentation = Object.assign(bulkClose, useMobileSessionPresentation(bulkClose))
|
||||
const tabStripCache = Object.assign(bulkClose, useMobileSessionTabStripCache(bulkClose))
|
||||
const presentation = Object.assign(tabStripCache, useMobileSessionPresentation(tabStripCache))
|
||||
const panelRouteActions = Object.assign(
|
||||
presentation,
|
||||
useMobileSessionPanelRouteActions(presentation)
|
||||
|
||||
@@ -14,6 +14,7 @@ import { isFloatingWorkspaceWorktreeId } from './floating-workspace'
|
||||
import { useLiveWorktreeName } from './use-live-worktree-name'
|
||||
import { useMissingWorktreeBounce } from './use-missing-worktree-bounce'
|
||||
import { hostRouteWithNotice } from '../host-route-notice'
|
||||
import { useHostProtocolGates } from '../components/HostProtocolGate'
|
||||
|
||||
export function useMobileSessionFoundation() {
|
||||
const {
|
||||
@@ -36,6 +37,14 @@ export function useMobileSessionFoundation() {
|
||||
const insets = useSafeAreaInsets()
|
||||
// Why: shared client per host owned by RpcClientProvider (docs/mobile-shared-client-per-host.md).
|
||||
const { client, clientId, state: connState } = useHostClient(hostId)
|
||||
// Why: HostProtocolGate holds this connection's single status.get. Reading it here gives the
|
||||
// whole route one source for host capabilities and for whether the compatibility verdict has
|
||||
// landed — the routes now mount while it is still in flight, so "not yet known" is a real state.
|
||||
const { compatVerdict, compatVerified, hostCapabilities, statusPending } = useHostProtocolGates()
|
||||
// Why all three: a settled verdict is not necessarily a passing one, and a settled *passing*
|
||||
// verdict is not necessarily an answered one — a host that cannot answer status.get fails open
|
||||
// to `ok` so navigation still works. Writes read this flag, so they wait for a real reply.
|
||||
const protocolVerified = !statusPending && compatVerified && compatVerdict.kind === 'ok'
|
||||
const reconnectAttempts = useReconnectAttempt(hostId)
|
||||
const lastConnectedAt = useLastConnectedAt(hostId)
|
||||
const forceReconnectHost = useForceReconnect()
|
||||
@@ -98,6 +107,8 @@ export function useMobileSessionFoundation() {
|
||||
client,
|
||||
clientId,
|
||||
connState,
|
||||
hostCapabilities,
|
||||
protocolVerified,
|
||||
reconnectAttempts,
|
||||
lastConnectedAt,
|
||||
forceReconnectHost,
|
||||
|
||||
@@ -3,9 +3,11 @@ import { classifyConnection, verdictDisplayLabel } from '../transport/connection
|
||||
import { computeActiveTerminalKeyboardLift } from '../terminal/terminal-keyboard-avoidance-lift'
|
||||
import { useInitialSessionTerminalAutoCreate } from './use-initial-session-terminal-autocreate'
|
||||
import { MOBILE_SESSION_STATUS_LABELS } from './mobile-session-route-helpers'
|
||||
import type { MobileSessionBulkCloseModel } from './use-mobile-session-bulk-close'
|
||||
import { selectMobileSessionReconnectViewState } from './mobile-session-reconnect-view-state'
|
||||
import { getMobileSessionTabStripRows } from './mobile-session-tab-strip-entries'
|
||||
import type { MobileSessionTabStripCacheModel } from './use-mobile-session-tab-strip-cache'
|
||||
|
||||
export function useMobileSessionPresentation(scope: MobileSessionBulkCloseModel) {
|
||||
export function useMobileSessionPresentation(scope: MobileSessionTabStripCacheModel) {
|
||||
const {
|
||||
created,
|
||||
worktreeId,
|
||||
@@ -24,6 +26,8 @@ export function useMobileSessionPresentation(scope: MobileSessionBulkCloseModel)
|
||||
terminalKeyboardMetrics,
|
||||
toastOpacityRef,
|
||||
hostEndpoint,
|
||||
activeSessionTabId,
|
||||
cachedTabStrip,
|
||||
initialSessionAutoCreateRef,
|
||||
terminalFrameHeightRef,
|
||||
handleCreateTerminal,
|
||||
@@ -58,6 +62,23 @@ export function useMobileSessionPresentation(scope: MobileSessionBulkCloseModel)
|
||||
const showConnectionRetry =
|
||||
connectionVerdict.kind === 'warning' || connectionVerdict.kind === 'unreachable'
|
||||
|
||||
// Why: a reconnect to a workspace this phone has already drawn should re-draw it, not blank
|
||||
// the screen while the RPCs land. See mobile-session-reconnect-view-state.
|
||||
const reconnectViewState = selectMobileSessionReconnectViewState({
|
||||
connState,
|
||||
verdictKind: connectionVerdict.kind,
|
||||
terminalsLoaded,
|
||||
liveTabCount: visibleTabs.length,
|
||||
activeHandle,
|
||||
cachedPreview: cachedTabStrip
|
||||
})
|
||||
const tabStripRows = getMobileSessionTabStripRows({
|
||||
liveTabs: visibleTabs,
|
||||
activeSessionTabId,
|
||||
preview:
|
||||
reconnectViewState.kind === 'reconnecting-with-cache' ? reconnectViewState.preview : null
|
||||
})
|
||||
|
||||
const terminalSummary =
|
||||
connState === 'connected'
|
||||
? showLoadingState
|
||||
@@ -88,6 +109,8 @@ export function useMobileSessionPresentation(scope: MobileSessionBulkCloseModel)
|
||||
return {
|
||||
showLoadingState,
|
||||
showEmptyState,
|
||||
reconnectViewState,
|
||||
tabStripRows,
|
||||
connectionVerdict,
|
||||
showConnectionRetry,
|
||||
terminalSummary,
|
||||
@@ -97,5 +120,5 @@ export function useMobileSessionPresentation(scope: MobileSessionBulkCloseModel)
|
||||
}
|
||||
}
|
||||
|
||||
export type MobileSessionPresentationModel = MobileSessionBulkCloseModel &
|
||||
export type MobileSessionPresentationModel = MobileSessionTabStripCacheModel &
|
||||
ReturnType<typeof useMobileSessionPresentation>
|
||||
|
||||
@@ -12,6 +12,7 @@ export function useMobileSessionStartup(scope: MobileSessionKeyboardStateModel)
|
||||
isFloatingWorkspaceRoute,
|
||||
connState,
|
||||
client,
|
||||
protocolVerified,
|
||||
setTerminals,
|
||||
terminalsRef,
|
||||
setSessionTabs,
|
||||
@@ -95,6 +96,8 @@ export function useMobileSessionStartup(scope: MobileSessionKeyboardStateModel)
|
||||
worktreeId
|
||||
])
|
||||
|
||||
// Reads only. They carry no side effect on the host, so they do not wait on the compatibility
|
||||
// verdict — that is the whole point of mounting this route while status.get is still in flight.
|
||||
// Every setTimeout goes through addTimer into `timers`, which the returned cleanup clears.
|
||||
// react-doctor-disable-next-line react-doctor/effect-needs-cleanup
|
||||
useEffect(() => {
|
||||
@@ -116,58 +119,81 @@ export function useMobileSessionStartup(scope: MobileSessionKeyboardStateModel)
|
||||
timers.push(setTimeout(fn, ms))
|
||||
}
|
||||
void (async () => {
|
||||
const reportActivationOutcome = (response: RpcSuccess | null): void => {
|
||||
if (!disposed && response && headlessActivationNeedsHostRenderer(response.result)) {
|
||||
showToast('Open Orca on the host to wake sleeping agents.', 3000)
|
||||
}
|
||||
}
|
||||
if (client && created !== '1' && !isFloatingWorkspaceRoute) {
|
||||
// Why: hydrate host-owned tabs without pulling other paired clients (esp. desktop) into this worktree.
|
||||
void client
|
||||
.sendRequest('worktree.activate', {
|
||||
worktree: `id:${worktreeId}`,
|
||||
notifyClients: false,
|
||||
navigation: 'caller'
|
||||
})
|
||||
.then((response) => reportActivationOutcome(response.ok ? response : null))
|
||||
.catch(() => null)
|
||||
}
|
||||
if (disposed) {
|
||||
return
|
||||
}
|
||||
await ensureSessionTabs().catch(() => null)
|
||||
if (disposed) {
|
||||
return
|
||||
}
|
||||
await fetchTerminals({ allowEmptyLoaded: false })
|
||||
// Why: session.tabs.list and terminal.list are independent reads, so issue both now and
|
||||
// wait for the pair. Serialising them cost a full extra round trip before the first
|
||||
// terminal could paint, which on a far relay cell is seconds, not milliseconds. Each
|
||||
// call keeps its own catch so one rejection cannot strand the other's follow-up refreshes.
|
||||
await Promise.all([
|
||||
ensureSessionTabs().catch(() => null),
|
||||
fetchTerminals({ allowEmptyLoaded: false }).catch(() => false)
|
||||
])
|
||||
if (disposed) {
|
||||
return
|
||||
}
|
||||
addTimer(() => void fetchTerminals({ allowEmptyLoaded: false }), 750)
|
||||
addTimer(() => void fetchTerminals({ allowEmptyLoaded: true }), 1500)
|
||||
if (client && created === '1' && !isFloatingWorkspaceRoute) {
|
||||
addTimer(() => {
|
||||
if (activeHandleRef.current) {
|
||||
})()
|
||||
return () => {
|
||||
disposed = true
|
||||
for (const t of timers) {
|
||||
clearTimeout(t)
|
||||
}
|
||||
}
|
||||
// Why no client/worktreeId here: both reads are useCallbacks that already list them, so a
|
||||
// host or worktree change replaces their identity and re-runs this effect with them.
|
||||
}, [connState, fetchTerminals, ensureSessionTabs])
|
||||
|
||||
// worktree.activate writes host state, so unlike the reads above it waits for the compatibility
|
||||
// verdict. A missing protocolVersion reads as 0 and is blocked, so "pending" is not a formality:
|
||||
// mounting early must not let this route mutate a host the gate is about to refuse.
|
||||
// Every setTimeout goes through addTimer into `timers`, which the returned cleanup clears.
|
||||
// react-doctor-disable-next-line react-doctor/effect-needs-cleanup
|
||||
useEffect(() => {
|
||||
if (connState !== 'connected' || !client || !protocolVerified || isFloatingWorkspaceRoute) {
|
||||
return
|
||||
}
|
||||
let disposed = false
|
||||
const timers: ReturnType<typeof setTimeout>[] = []
|
||||
function addTimer(fn: () => void, ms: number) {
|
||||
if (disposed) {
|
||||
return
|
||||
}
|
||||
timers.push(setTimeout(fn, ms))
|
||||
}
|
||||
const activateWorktree = () =>
|
||||
client
|
||||
.sendRequest('worktree.activate', {
|
||||
worktree: `id:${worktreeId}`,
|
||||
notifyClients: false,
|
||||
navigation: 'caller'
|
||||
})
|
||||
.catch(() => null)
|
||||
const reportActivationOutcome = (response: RpcSuccess | null): void => {
|
||||
if (!disposed && response && headlessActivationNeedsHostRenderer(response.result)) {
|
||||
showToast('Open Orca on the host to wake sleeping agents.', 3000)
|
||||
}
|
||||
}
|
||||
if (created !== '1') {
|
||||
// Why: hydrate host-owned tabs without pulling other paired clients (esp. desktop) into this worktree.
|
||||
void activateWorktree().then((response) =>
|
||||
reportActivationOutcome(response?.ok ? response : null)
|
||||
)
|
||||
} else {
|
||||
addTimer(() => {
|
||||
if (activeHandleRef.current) {
|
||||
return
|
||||
}
|
||||
void (async () => {
|
||||
const activationResponse = await activateWorktree()
|
||||
reportActivationOutcome(activationResponse?.ok ? activationResponse : null)
|
||||
if (disposed) {
|
||||
return
|
||||
}
|
||||
void (async () => {
|
||||
const activationResponse = await client
|
||||
.sendRequest('worktree.activate', {
|
||||
worktree: `id:${worktreeId}`,
|
||||
notifyClients: false,
|
||||
navigation: 'caller'
|
||||
})
|
||||
.catch(() => null)
|
||||
reportActivationOutcome(activationResponse?.ok ? activationResponse : null)
|
||||
if (disposed) {
|
||||
return
|
||||
}
|
||||
await fetchTerminals({ allowEmptyLoaded: true })
|
||||
addTimer(() => void fetchTerminals({ allowEmptyLoaded: true }), 750)
|
||||
})()
|
||||
}, 1800)
|
||||
}
|
||||
})()
|
||||
await fetchTerminals({ allowEmptyLoaded: true })
|
||||
addTimer(() => void fetchTerminals({ allowEmptyLoaded: true }), 750)
|
||||
})()
|
||||
}, 1800)
|
||||
}
|
||||
return () => {
|
||||
disposed = true
|
||||
for (const t of timers) {
|
||||
@@ -179,8 +205,8 @@ export function useMobileSessionStartup(scope: MobileSessionKeyboardStateModel)
|
||||
connState,
|
||||
created,
|
||||
fetchTerminals,
|
||||
ensureSessionTabs,
|
||||
isFloatingWorkspaceRoute,
|
||||
protocolVerified,
|
||||
showToast,
|
||||
worktreeId
|
||||
])
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { useEffect, useRef, useCallback, useMemo, useState } from 'react'
|
||||
import { startRuntimeCapabilityProbe } from '../transport/runtime-capability-probe'
|
||||
import { supportsMobileQuickCommands } from '../terminal/quick-commands'
|
||||
import { MOBILE_AI_VAULT_CAPABILITY } from '../agent-history/agent-history-capability'
|
||||
import { TERMINAL_QUERY_REPLY_INPUT_RUNTIME_CAPABILITY } from '../../../src/shared/protocol-version'
|
||||
@@ -17,6 +16,8 @@ export function useMobileSessionTabReconciliation(scope: MobileSessionMarkdownAc
|
||||
worktreeId,
|
||||
client,
|
||||
connState,
|
||||
hostCapabilities,
|
||||
protocolVerified,
|
||||
sessionTabsRef,
|
||||
activeSessionTabIdRef,
|
||||
terminalsRef,
|
||||
@@ -144,8 +145,14 @@ export function useMobileSessionTabReconciliation(scope: MobileSessionMarkdownAc
|
||||
|
||||
const hostQueryReplyInputSupportedRef = useRef(false)
|
||||
|
||||
// Why: the gate above every /h/ route already holds this connection's status.get answer (and
|
||||
// retries it until one lands), so the route reads it through the foundation instead of issuing
|
||||
// a second one. It reports no capabilities until the verdict is proven, which keeps the
|
||||
// fail-closed reset below identical to the old pre-probe clear.
|
||||
useEffect(() => {
|
||||
if (!client || connState !== 'connected') {
|
||||
// Why: a client swap can keep the route connected while moving to an older
|
||||
// host; clear the prior capability before exposing host-specific actions.
|
||||
if (!client || connState !== 'connected' || !protocolVerified) {
|
||||
setBrowserScreencastSupported(null)
|
||||
setAgentSessionHistorySupported(null)
|
||||
setQuickCommandsSupported(null)
|
||||
@@ -153,26 +160,15 @@ export function useMobileSessionTabReconciliation(scope: MobileSessionMarkdownAc
|
||||
hostQueryReplyInputSupportedRef.current = false
|
||||
return
|
||||
}
|
||||
// Why: a client swap can keep the route connected while moving to an older
|
||||
// host; clear the prior capability before exposing host-specific actions.
|
||||
setBrowserScreencastSupported(null)
|
||||
setAgentSessionHistorySupported(null)
|
||||
setQuickCommandsSupported(null)
|
||||
setShowQuickCommands(false)
|
||||
hostQueryReplyInputSupportedRef.current = false
|
||||
// Why: the probe retries — a relay→direct cutover or request timeout rejects
|
||||
// status.get without changing connState, which used to latch these hidden.
|
||||
return startRuntimeCapabilityProbe(client, (capabilities) => {
|
||||
setBrowserScreencastSupported(capabilities.includes('browser.screencast.v1'))
|
||||
setAgentSessionHistorySupported(capabilities.includes(MOBILE_AI_VAULT_CAPABILITY))
|
||||
setQuickCommandsSupported(supportsMobileQuickCommands(capabilities))
|
||||
// Why: hosts without this capability strip inputKind from terminal.send,
|
||||
// so a forwarded xterm reply would become floor-stealing shell input.
|
||||
hostQueryReplyInputSupportedRef.current = capabilities.includes(
|
||||
TERMINAL_QUERY_REPLY_INPUT_RUNTIME_CAPABILITY
|
||||
)
|
||||
})
|
||||
}, [client, connState])
|
||||
setBrowserScreencastSupported(hostCapabilities.includes('browser.screencast.v1'))
|
||||
setAgentSessionHistorySupported(hostCapabilities.includes(MOBILE_AI_VAULT_CAPABILITY))
|
||||
setQuickCommandsSupported(supportsMobileQuickCommands(hostCapabilities))
|
||||
// Why: hosts without this capability strip inputKind from terminal.send,
|
||||
// so a forwarded xterm reply would become floor-stealing shell input.
|
||||
hostQueryReplyInputSupportedRef.current = hostCapabilities.includes(
|
||||
TERMINAL_QUERY_REPLY_INPUT_RUNTIME_CAPABILITY
|
||||
)
|
||||
}, [client, connState, hostCapabilities, protocolVerified])
|
||||
return {
|
||||
consumeAcceptedSessionTabs,
|
||||
hasSessionTabsRecoveryNeed,
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
import {
|
||||
getSessionTabStripCacheKey,
|
||||
loadCachedSessionTabStrip,
|
||||
readCachedSessionTabStrip,
|
||||
saveCachedSessionTabStrip
|
||||
} from '../cache/session-tab-strip-cache'
|
||||
import {
|
||||
toMobileSessionTabStripPreview,
|
||||
type MobileSessionTabStripPreview
|
||||
} from './mobile-session-tab-strip-entries'
|
||||
import type { MobileSessionBulkCloseModel } from './use-mobile-session-bulk-close'
|
||||
|
||||
/**
|
||||
* Keeps the last drawn tab strip for this workspace on the device, so a reconnect has something
|
||||
* to render before the first snapshot lands. See mobile-session-reconnect-view-state.
|
||||
*/
|
||||
export function useMobileSessionTabStripCache(scope: MobileSessionBulkCloseModel) {
|
||||
const { hostId, worktreeId, connState, terminalsLoaded } = scope
|
||||
const { visibleTabs, activeSessionTabId, activeHandle } = scope
|
||||
const cacheKey = getSessionTabStripCacheKey(hostId, worktreeId)
|
||||
// Why: state settles a commit behind the key it was read for, so carry the key with it —
|
||||
// otherwise the first render after a workspace switch draws the previous workspace's strip.
|
||||
const [loaded, setLoaded] = useState<{
|
||||
key: string | null
|
||||
preview: MobileSessionTabStripPreview | null
|
||||
}>(() => ({ key: cacheKey, preview: readCachedSessionTabStrip(cacheKey) }))
|
||||
|
||||
useEffect(() => {
|
||||
// Synchronous first, so an in-session revisit never blinks through the uncached branch.
|
||||
setLoaded({ key: cacheKey, preview: readCachedSessionTabStrip(cacheKey) })
|
||||
let disposed = false
|
||||
void loadCachedSessionTabStrip(cacheKey).then((preview) => {
|
||||
if (!disposed) {
|
||||
setLoaded({ key: cacheKey, preview })
|
||||
}
|
||||
})
|
||||
return () => {
|
||||
disposed = true
|
||||
}
|
||||
}, [cacheKey])
|
||||
const cachedTabStrip = loaded.key === cacheKey ? loaded.preview : null
|
||||
|
||||
// Only a host-confirmed strip is worth persisting, and an emptied workspace has to be written
|
||||
// too — skipping it would leave yesterday's tabs to be drawn over a session that no longer has
|
||||
// them. The one reading we do not trust is a live terminal with no tab record behind it, which
|
||||
// is the same case the empty state refuses to claim (use-mobile-session-presentation).
|
||||
// react-doctor-disable-next-line react-doctor/effect-needs-cleanup
|
||||
useEffect(() => {
|
||||
if (connState !== 'connected' || !terminalsLoaded) {
|
||||
return
|
||||
}
|
||||
if (visibleTabs.length === 0 && activeHandle !== null) {
|
||||
return
|
||||
}
|
||||
saveCachedSessionTabStrip(
|
||||
cacheKey,
|
||||
toMobileSessionTabStripPreview(visibleTabs, activeSessionTabId)
|
||||
)
|
||||
}, [activeHandle, activeSessionTabId, cacheKey, connState, terminalsLoaded, visibleTabs])
|
||||
|
||||
return { cachedTabStrip }
|
||||
}
|
||||
|
||||
export type MobileSessionTabStripCacheModel = MobileSessionBulkCloseModel &
|
||||
ReturnType<typeof useMobileSessionTabStripCache>
|
||||
@@ -1,4 +1,6 @@
|
||||
import { useRef, useCallback } from 'react'
|
||||
import type { TerminalWebViewHandle } from '../terminal/terminal-webview-contract'
|
||||
import { TERMINAL_ENGINE_PREWARM_HANDLE } from './TerminalEnginePrewarm'
|
||||
import type { MobileSessionNativeChatDictationModel } from './use-mobile-session-native-chat-dictation'
|
||||
|
||||
export function useMobileSessionTerminalSubscriptionFoundation(
|
||||
@@ -101,12 +103,34 @@ export function useMobileSessionTerminalSubscriptionFoundation(
|
||||
},
|
||||
[getTerminalRef]
|
||||
)
|
||||
// Why: the pre-warm engine occupies the frame the first pane will occupy, so let it satisfy
|
||||
// the one-shot measurement. It has no handle, so it is passed its own ref instead of looking
|
||||
// one up, and it must never latch a measurement taken before the frame has a real height.
|
||||
const measurePrewarmViewport = useCallback(
|
||||
async (engine: TerminalWebViewHandle, frameHeight: number) => {
|
||||
if (viewportMeasuredRef.current || frameHeight <= 0) {
|
||||
return
|
||||
}
|
||||
const dims = await engine.measureFitDimensions(frameHeight)
|
||||
terminalDiagnosticsRef.current.viewportMeasured(
|
||||
TERMINAL_ENGINE_PREWARM_HANDLE,
|
||||
dims,
|
||||
frameHeight
|
||||
)
|
||||
if (dims && !viewportMeasuredRef.current) {
|
||||
viewportRef.current = dims
|
||||
viewportMeasuredRef.current = true
|
||||
}
|
||||
},
|
||||
[]
|
||||
)
|
||||
return {
|
||||
getTerminalRef,
|
||||
unsubscribeTerminal,
|
||||
unsubscribeTerminalRef,
|
||||
clearTerminalCache,
|
||||
measureViewportOnce
|
||||
measureViewportOnce,
|
||||
measurePrewarmViewport
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -16,6 +16,29 @@ describe('connection log buffer', () => {
|
||||
expect(store.get('host-b').map((e) => e.id)).toEqual(['log-2'])
|
||||
})
|
||||
|
||||
it('retains phase timings through redaction and evicts them with the cap', () => {
|
||||
const store = createConnectionLogStore(2)
|
||||
store.append('host-a', {
|
||||
...entry(1),
|
||||
timing: { kind: 'connection-state', name: 'reconnecting', ms: 800, complete: true }
|
||||
})
|
||||
store.append('host-a', {
|
||||
...entry(2),
|
||||
detail: '4280ms in connecting; resumeToken=secret-resume-token',
|
||||
timing: { kind: 'connection-state', name: 'connecting', ms: 4_280, complete: true }
|
||||
})
|
||||
store.append('host-a', {
|
||||
...entry(3),
|
||||
timing: { kind: 'relay-dial-stage', name: 'awaiting-hello', ms: 9_100, complete: false }
|
||||
})
|
||||
|
||||
expect(store.get('host-a').map((e) => e.timing)).toEqual([
|
||||
{ kind: 'connection-state', name: 'connecting', ms: 4_280, complete: true },
|
||||
{ kind: 'relay-dial-stage', name: 'awaiting-hello', ms: 9_100, complete: false }
|
||||
])
|
||||
expect(store.get('host-a')[0]!.detail).toBe('4280ms in connecting; resumeToken=[redacted]')
|
||||
})
|
||||
|
||||
it('drops the oldest entries past the cap', () => {
|
||||
const store = createConnectionLogStore(3)
|
||||
for (let i = 1; i <= 5; i++) {
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { DirectConnectionLog } from './direct-connection-log'
|
||||
import { RpcClientConnectionState } from './rpc-client-connection-state'
|
||||
import type { ConnectionLogEntry, ConnectionState } from './types'
|
||||
|
||||
function openStateWithLog(sink?: (entry: ConnectionLogEntry) => void) {
|
||||
const entries: ConnectionLogEntry[] = []
|
||||
const log = new DirectConnectionLog(
|
||||
'ws://192.168.1.50:6768',
|
||||
sink ?? ((entry) => entries.push(entry))
|
||||
)
|
||||
let now = 0
|
||||
const state = new RpcClientConnectionState({
|
||||
endpoint: 'ws://192.168.1.50:6768',
|
||||
getReconnectAttempt: () => 0,
|
||||
isClosed: () => false,
|
||||
onStateDwell: log.stateDwell,
|
||||
now: () => now
|
||||
})
|
||||
const publishAfter = (elapsedMs: number, next: ConnectionState): void => {
|
||||
now += elapsedMs
|
||||
state.publish(next)
|
||||
}
|
||||
return { entries, state, publishAfter }
|
||||
}
|
||||
|
||||
describe('connection state dwell logging', () => {
|
||||
it('records the time spent in each state as a structured log entry', () => {
|
||||
const { entries, publishAfter } = openStateWithLog()
|
||||
|
||||
publishAfter(300, 'connecting')
|
||||
publishAfter(4_200, 'handshaking')
|
||||
publishAfter(250, 'connected')
|
||||
|
||||
expect(entries.map((entry) => entry.timing)).toEqual([
|
||||
{ kind: 'connection-state', name: 'disconnected', ms: 300, complete: true },
|
||||
{ kind: 'connection-state', name: 'connecting', ms: 4_200, complete: true },
|
||||
{ kind: 'connection-state', name: 'handshaking', ms: 250, complete: true }
|
||||
])
|
||||
expect(entries[1]!.message).toBe('Connection state connecting → handshaking')
|
||||
expect(entries[1]!.detail).toBe('4200ms in connecting')
|
||||
})
|
||||
|
||||
it('skips transitions too short to explain a slow connect', () => {
|
||||
const { entries, publishAfter } = openStateWithLog()
|
||||
|
||||
publishAfter(99, 'connecting')
|
||||
publishAfter(100, 'handshaking')
|
||||
|
||||
expect(entries.map((entry) => entry.timing?.name)).toEqual(['connecting'])
|
||||
})
|
||||
|
||||
it('does not log a dwell when the state does not change', () => {
|
||||
const { entries, publishAfter } = openStateWithLog()
|
||||
|
||||
publishAfter(500, 'connecting')
|
||||
publishAfter(500, 'connecting')
|
||||
|
||||
expect(entries).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('still publishes the state when the log sink throws', () => {
|
||||
const seen: ConnectionState[] = []
|
||||
const { state, publishAfter } = openStateWithLog(() => {
|
||||
throw new Error('sink exploded')
|
||||
})
|
||||
state.addListener((next) => seen.push(next))
|
||||
const connected = state.waitForConnected()
|
||||
|
||||
publishAfter(500, 'connecting')
|
||||
publishAfter(500, 'connected')
|
||||
|
||||
expect(seen).toEqual(['connecting', 'connected'])
|
||||
expect(state.get()).toBe('connected')
|
||||
return expect(connected).resolves.toBeUndefined()
|
||||
})
|
||||
})
|
||||
@@ -4,9 +4,15 @@ import type {
|
||||
ConnectionLogEntry,
|
||||
ConnectionLogLevel,
|
||||
ConnectionLogSink,
|
||||
ConnectionState,
|
||||
MobileConnectionDiagnosticPath
|
||||
} from './types'
|
||||
|
||||
// Why: every reconnect cycle walks four states, and the per-host buffer is capped.
|
||||
// Logging sub-100ms transitions would halve the history a report can show while
|
||||
// telling support nothing — those states are never where a slow connect spent time.
|
||||
const MIN_LOGGED_DWELL_MS = 100
|
||||
|
||||
export class DirectConnectionLog {
|
||||
private sequence = 0
|
||||
private readonly path: MobileConnectionDiagnosticPath
|
||||
@@ -22,7 +28,7 @@ export class DirectConnectionLog {
|
||||
level: ConnectionLogLevel,
|
||||
message: string,
|
||||
detail?: string,
|
||||
evidence?: Pick<ConnectionLogEntry, 'code' | 'path'>
|
||||
evidence?: Pick<ConnectionLogEntry, 'code' | 'path' | 'timing'>
|
||||
): void => {
|
||||
this.sink?.({
|
||||
id: `log-${++this.sequence}-${Date.now()}`,
|
||||
@@ -44,6 +50,26 @@ export class DirectConnectionLog {
|
||||
)
|
||||
}
|
||||
|
||||
// Why: how long the client sat in each ConnectionState used to go only to
|
||||
// console, so a shared diagnostics report could not show where a slow connect
|
||||
// spent its seconds.
|
||||
stateDwell = (previous: ConnectionState, next: ConnectionState, dweltMs: number): void => {
|
||||
if (dweltMs < MIN_LOGGED_DWELL_MS) {
|
||||
return
|
||||
}
|
||||
this.emit('info', `Connection state ${previous} → ${next}`, `${dweltMs}ms in ${previous}`, {
|
||||
timing: { kind: 'connection-state', name: previous, ms: dweltMs, complete: true }
|
||||
})
|
||||
}
|
||||
|
||||
retryScheduled = (message: string, detail?: string): void => {
|
||||
this.emit('info', message, detail, { code: 'retry-scheduled' })
|
||||
}
|
||||
|
||||
authenticationRejected = (message: string, detail?: string): void => {
|
||||
this.emit('warn', message, detail, { code: 'authentication-rejected' })
|
||||
}
|
||||
|
||||
connected = (): void => {
|
||||
this.emit('success', 'Authenticated', 'Channel ready for RPC', { code: 'direct-connected' })
|
||||
}
|
||||
|
||||
@@ -48,14 +48,14 @@ export class DirectRpcClient implements RpcClient {
|
||||
this.reconnect = new RpcClientReconnectSchedule({
|
||||
openConnection: () => this.openConnection(),
|
||||
rejectConnectWaiters: (reason) => this.connectionState.rejectWaiters(reason),
|
||||
emitLog: (message, detail) =>
|
||||
this.connectionLog.emit('info', message, detail, { code: 'retry-scheduled' })
|
||||
emitLog: this.connectionLog.retryScheduled
|
||||
})
|
||||
this.connectionState = new RpcClientConnectionState({
|
||||
endpoint,
|
||||
initialListener: options.onStateChange,
|
||||
getReconnectAttempt: () => this.reconnect.getAttempt(),
|
||||
isClosed: () => this.intentionallyClosed
|
||||
isClosed: () => this.intentionallyClosed,
|
||||
onStateDwell: this.connectionLog.stateDwell
|
||||
})
|
||||
this.streams = new RpcClientStreamRegistry({
|
||||
nextId: () => this.nextId(),
|
||||
@@ -102,8 +102,7 @@ export class DirectRpcClient implements RpcClient {
|
||||
this.authenticationRetry = new RpcClientAuthenticationRetry({
|
||||
endpoint,
|
||||
stopLiveness: () => this.stopLiveness(),
|
||||
emitWarning: (message, detail) =>
|
||||
this.connectionLog.emit('warn', message, detail, { code: 'authentication-rejected' }),
|
||||
emitWarning: this.connectionLog.authenticationRejected,
|
||||
retry: (reason) => this.retryAuthentication(reason),
|
||||
latchFailure: (reason) => this.latchAuthenticationFailure(reason)
|
||||
})
|
||||
|
||||
@@ -17,6 +17,12 @@ vi.mock('./host-store', () => ({
|
||||
}))
|
||||
|
||||
import { removeHostAndCloseClient } from './host-removal-lifecycle'
|
||||
import {
|
||||
getSessionTabStripCacheKey,
|
||||
readCachedSessionTabStrip,
|
||||
resetSessionTabStripCacheForTests,
|
||||
saveCachedSessionTabStrip
|
||||
} from '../cache/session-tab-strip-cache'
|
||||
import {
|
||||
getHostNotificationSession,
|
||||
resetHostNotificationSessionsForTests
|
||||
@@ -26,7 +32,9 @@ describe('host removal lifecycle', () => {
|
||||
beforeEach(() => {
|
||||
removeHostMock.mockReset()
|
||||
asyncStorage.removeItem.mockClear()
|
||||
asyncStorage.setItem.mockReset().mockResolvedValue(undefined)
|
||||
resetHostNotificationSessionsForTests()
|
||||
resetSessionTabStripCacheForTests()
|
||||
})
|
||||
|
||||
it('closes the client only after metadata removal commits', async () => {
|
||||
@@ -88,4 +96,40 @@ describe('host removal lifecycle', () => {
|
||||
|
||||
expect(asyncStorage.removeItem).toHaveBeenCalledWith('orca:mobileNotificationsWatermark:host-1')
|
||||
})
|
||||
|
||||
it('drops the removed host cached tab strip and keeps every other host', async () => {
|
||||
// Why: the strip is plaintext and nothing else in the app ever expires an entry, so a
|
||||
// forgotten host would keep its tab titles on disk and get them rewritten by the next
|
||||
// save for any surviving host.
|
||||
removeHostMock.mockResolvedValue(undefined)
|
||||
const removed = getSessionTabStripCacheKey('host-1', 'wt-1')
|
||||
const kept = getSessionTabStripCacheKey('host-2', 'wt-1')
|
||||
const strip = {
|
||||
tabs: [{ id: 'tab-1', type: 'terminal' as const, title: 'Terminal', agentId: null }],
|
||||
activeTabId: 'tab-1'
|
||||
}
|
||||
saveCachedSessionTabStrip(removed, strip)
|
||||
saveCachedSessionTabStrip(kept, strip)
|
||||
|
||||
await removeHostAndCloseClient('host-1', vi.fn())
|
||||
// Fire-and-forget, like clearWatermark above; let its microtasks land.
|
||||
await vi.waitFor(() => expect(readCachedSessionTabStrip(removed)).toBeNull())
|
||||
|
||||
expect(readCachedSessionTabStrip(kept)?.tabs).toHaveLength(1)
|
||||
})
|
||||
it('finishes the removal even when the cached tab strip write fails', async () => {
|
||||
// The metadata removal has already committed and the client is closed by this
|
||||
// point, so a cache write that fails must be reported, not thrown: surfacing it
|
||||
// as a failed removal would leave the user staring at a host that is really gone.
|
||||
removeHostMock.mockResolvedValue(undefined)
|
||||
asyncStorage.setItem.mockRejectedValue(new Error('storage full'))
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const closeHostClient = vi.fn()
|
||||
|
||||
await expect(removeHostAndCloseClient('host-1', closeHostClient)).resolves.toBeUndefined()
|
||||
|
||||
expect(closeHostClient).toHaveBeenCalledWith('host-1')
|
||||
expect(warn).toHaveBeenCalled()
|
||||
warn.mockRestore()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { deleteCachedSessionTabStripForHost } from '../cache/session-tab-strip-cache'
|
||||
import {
|
||||
clearWatermark,
|
||||
forgetHostNotificationSession
|
||||
@@ -17,4 +18,12 @@ export async function removeHostAndCloseClient(
|
||||
// re-pair of the same host would inherit a watermark for a counter it never saw.
|
||||
forgetHostNotificationSession(hostId)
|
||||
void clearWatermark(hostId)
|
||||
// Why: the cached tab strip is plaintext and host-scoped, so forgetting the host has to drop
|
||||
// it here too — nothing else in the app ever expires an entry. Awaited so a storage failure
|
||||
// is observed rather than swallowed, but never fatal: the metadata removal has already
|
||||
// committed and the client is closed, so failing here would report a finished removal as
|
||||
// failed. The cache refuses further saves for this host either way.
|
||||
await deleteCachedSessionTabStripForHost(hostId).catch((error: unknown) => {
|
||||
console.warn('[host-removal] cached tab strip delete failed', error)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
import type { RpcClient } from './rpc-client'
|
||||
import type { ConnectionState, RpcSuccess } from './types'
|
||||
import type { ConnectionState } from './types'
|
||||
import { readRuntimeCapabilities, startRuntimeStatusProbe } from './runtime-status-probe'
|
||||
import { evaluateCompat, type CompatVerdict } from './protocol-compat'
|
||||
import type { DesktopStatus } from '../worktree/host-worktree-rpc-types'
|
||||
import { normalizeHostAppVersion, recordHostAppVersion } from './host-app-version-store'
|
||||
@@ -10,6 +11,10 @@ export type HostStatusGates = {
|
||||
floatingWorkspaceEnabled: boolean
|
||||
desktopAppVersion: string | null
|
||||
compatVerdict: CompatVerdict
|
||||
// Why: `compatVerdict.kind === 'ok'` is not proof. A host that never answers status.get settles
|
||||
// the same `ok` so navigation is not trapped, and that fallback must not read as a passing
|
||||
// verdict. Only an evaluated status reply sets this, so writes to the host can gate on it.
|
||||
compatVerified: boolean
|
||||
statusPending: boolean
|
||||
}
|
||||
|
||||
@@ -21,8 +26,11 @@ type LoadedHostStatusGates = Omit<HostStatusGates, 'statusPending'> & {
|
||||
|
||||
const EMPTY_HOST_CAPABILITIES: string[] = []
|
||||
|
||||
// Reads status.get on connect for capabilities, protocol-compat verdict, and the
|
||||
// floating-workspace flag. Compat constants are wide-open today so this never blocks yet.
|
||||
// The route tree's single status.get: it reads capabilities, the protocol-compat verdict, and
|
||||
// the floating-workspace flag once per connection and publishes them through HostProtocolGate,
|
||||
// so no descendant issues its own. The verdict really can block — evaluateCompat reads a missing
|
||||
// protocolVersion as 0, below MIN_COMPATIBLE_DESKTOP_VERSION — so a pending verdict is a real
|
||||
// state, not a formality, and anything that writes to the host must wait for it.
|
||||
export function useHostStatusGates(args: {
|
||||
hostId: string | undefined
|
||||
client: RpcClient | null
|
||||
@@ -39,30 +47,33 @@ export function useHostStatusGates(args: {
|
||||
setUnverified(true)
|
||||
return
|
||||
}
|
||||
let cancelled = false
|
||||
const requestClient = client
|
||||
const settle = (gates: Omit<HostStatusGates, 'statusPending'>) => {
|
||||
setLoaded({ hostId, client: requestClient, ...gates })
|
||||
setUnverified(false)
|
||||
}
|
||||
void (async () => {
|
||||
try {
|
||||
const response = await requestClient.sendRequest('status.get')
|
||||
if (cancelled) {
|
||||
return
|
||||
}
|
||||
if (!response.ok) {
|
||||
settle({
|
||||
hostCapabilities: [],
|
||||
floatingWorkspaceEnabled: false,
|
||||
desktopAppVersion: null,
|
||||
compatVerdict: { kind: 'ok' }
|
||||
})
|
||||
return
|
||||
}
|
||||
const status = (response as RpcSuccess).result as DesktopStatus & {
|
||||
capabilities?: string[]
|
||||
}
|
||||
// Why: a transient status failure must not trap navigation, so the first miss settles
|
||||
// conservative gates and releases the pending overlay; the probe keeps retrying underneath
|
||||
// so a cutover or timeout no longer latches capability-gated UI hidden until a remount.
|
||||
// compatVerified stays false: this releases the UI, it proves nothing about the host.
|
||||
let failedOpen = false
|
||||
const failOpen = () => {
|
||||
if (failedOpen) {
|
||||
return
|
||||
}
|
||||
failedOpen = true
|
||||
settle({
|
||||
hostCapabilities: [],
|
||||
floatingWorkspaceEnabled: false,
|
||||
desktopAppVersion: null,
|
||||
compatVerdict: { kind: 'ok' },
|
||||
compatVerified: false
|
||||
})
|
||||
}
|
||||
return startRuntimeStatusProbe(requestClient, {
|
||||
onUnavailable: failOpen,
|
||||
onStatus: (result) => {
|
||||
const status = result as DesktopStatus & { capabilities?: string[] }
|
||||
const verdict = evaluateCompat({
|
||||
desktopProtocolVersion: status.protocolVersion,
|
||||
desktopMinCompatibleMobileVersion: status.minCompatibleMobileVersion
|
||||
@@ -72,10 +83,11 @@ export function useHostStatusGates(args: {
|
||||
void recordHostAppVersion(hostId, desktopAppVersion)
|
||||
}
|
||||
settle({
|
||||
hostCapabilities: status.capabilities ?? [],
|
||||
hostCapabilities: [...readRuntimeCapabilities(result)],
|
||||
floatingWorkspaceEnabled: status.floatingWorkspaceEnabled === true,
|
||||
desktopAppVersion,
|
||||
compatVerdict: verdict
|
||||
compatVerdict: verdict,
|
||||
compatVerified: true
|
||||
})
|
||||
if (verdict.kind === 'blocked') {
|
||||
// Why: support breadcrumb to confirm a block fired vs a render bug; no PII, just version ints.
|
||||
@@ -86,21 +98,8 @@ export function useHostStatusGates(args: {
|
||||
requiredDesktopVersion: verdict.requiredDesktopVersion
|
||||
})
|
||||
}
|
||||
} catch {
|
||||
// Why: a transient status failure must not trap navigation; conservative feature gates remain disabled.
|
||||
if (!cancelled) {
|
||||
settle({
|
||||
hostCapabilities: [],
|
||||
floatingWorkspaceEnabled: false,
|
||||
desktopAppVersion: null,
|
||||
compatVerdict: { kind: 'ok' }
|
||||
})
|
||||
}
|
||||
}
|
||||
})()
|
||||
return () => {
|
||||
cancelled = true
|
||||
}
|
||||
})
|
||||
}, [client, connState, hostId])
|
||||
|
||||
// Why: effects run after render, so key loaded gates by host and client to fail closed during route reuse.
|
||||
@@ -111,6 +110,7 @@ export function useHostStatusGates(args: {
|
||||
floatingWorkspaceEnabled: false,
|
||||
desktopAppVersion: null,
|
||||
compatVerdict: { kind: 'ok' },
|
||||
compatVerified: false,
|
||||
statusPending: connState === 'connected' && client !== null
|
||||
}
|
||||
}
|
||||
@@ -119,6 +119,7 @@ export function useHostStatusGates(args: {
|
||||
floatingWorkspaceEnabled: proven.floatingWorkspaceEnabled,
|
||||
desktopAppVersion: proven.desktopAppVersion,
|
||||
compatVerdict: proven.compatVerdict,
|
||||
compatVerified: proven.compatVerified,
|
||||
// Why (F10): unchanged pending timing — the reconnect refetch is still "unknown", it just no
|
||||
// longer blanks the capabilities this same host already proved.
|
||||
statusPending: connState === 'connected' && unverified
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
|
||||
import { DirectReturnProbe } from './mobile-direct-return-probe'
|
||||
import { MobileEndpointHysteresis } from './mobile-endpoint-hysteresis'
|
||||
import { FakeSession, host } from './mobile-endpoint-supervisor-test-fakes'
|
||||
|
||||
vi.mock('react-native', () => ({ Platform: { OS: 'ios' } }))
|
||||
|
||||
// A LAN that never answers: every dial sits open until the probe's own 12s budget.
|
||||
function fixture() {
|
||||
const opened: FakeSession[] = []
|
||||
const probe = new DirectReturnProbe(
|
||||
{
|
||||
now: Date.now,
|
||||
setTimer: setTimeout,
|
||||
clearTimer: clearTimeout,
|
||||
openDirect: () => {
|
||||
const candidate = new FakeSession('connecting')
|
||||
opened.push(candidate)
|
||||
return candidate
|
||||
}
|
||||
},
|
||||
{
|
||||
hysteresis: new MobileEndpointHysteresis(Date.now(), {
|
||||
directSuccessesRequired: 1,
|
||||
directObservationMs: 60_000,
|
||||
failureCooldownMs: 0,
|
||||
minimumDwellMs: 0
|
||||
}),
|
||||
host: () => host,
|
||||
canSchedule: () => true,
|
||||
canDial: () => true,
|
||||
canAttempt: () => true,
|
||||
// These cases model a live relay session, so hysteresis still arbitrates.
|
||||
adoptsOutright: () => false,
|
||||
beginOperation: () => {},
|
||||
migrate: async () => {},
|
||||
onDirectMigrated: async () => {},
|
||||
afterProbe: () => {}
|
||||
}
|
||||
)
|
||||
return { opened, probe }
|
||||
}
|
||||
|
||||
beforeEach(() => vi.useFakeTimers())
|
||||
afterEach(() => vi.useRealTimers())
|
||||
|
||||
it('never opens a second dial while one is still in flight', async () => {
|
||||
const { opened, probe } = fixture()
|
||||
probe.schedule(0)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(opened).toHaveLength(1)
|
||||
|
||||
// A relay drop and a foreground return both ask for an immediate probe while the
|
||||
// first dial is still awaiting authentication.
|
||||
probe.schedule(0)
|
||||
probe.schedule(0)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(opened).toHaveLength(1)
|
||||
|
||||
// Why this is the assertion that matters: a second probe would have overwritten
|
||||
// activeProbe, so stop() would abort only the newest dial and leave this socket
|
||||
// open for the rest of its 12s budget.
|
||||
probe.stop()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(opened[0]!.close).toHaveBeenCalledOnce()
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
})
|
||||
|
||||
it('honors an urgent reprobe asked for mid-dial instead of dropping it on the 15s floor', async () => {
|
||||
const { opened, probe } = fixture()
|
||||
probe.schedule(0)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
probe.schedule(0)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(opened).toHaveLength(1)
|
||||
|
||||
// The deferred ask survives the dial and runs at once when it settles, so holding
|
||||
// the slot does not cost the caller the 15s it was trying to skip.
|
||||
await vi.advanceTimersByTimeAsync(12_000)
|
||||
await vi.advanceTimersByTimeAsync(1)
|
||||
expect(opened).toHaveLength(2)
|
||||
probe.stop()
|
||||
})
|
||||
|
||||
it('falls back to the ordinary interval when nothing asked for a sooner probe', async () => {
|
||||
const { opened, probe } = fixture()
|
||||
probe.schedule(0)
|
||||
await vi.advanceTimersByTimeAsync(12_000)
|
||||
expect(opened).toHaveLength(1)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(14_999)
|
||||
expect(opened).toHaveLength(1)
|
||||
await vi.advanceTimersByTimeAsync(1)
|
||||
expect(opened).toHaveLength(2)
|
||||
probe.stop()
|
||||
})
|
||||
@@ -6,13 +6,18 @@ import type { MobileConnectionPath } from './stable-logical-rpc-client'
|
||||
|
||||
const DIRECT_PROBE_INTERVAL_MS = 15_000
|
||||
|
||||
// While the runtime channel rides the relay, periodically probe the direct
|
||||
// endpoint and migrate back once hysteresis proves it stable.
|
||||
// Re-acquires the direct endpoint while the runtime channel rides the relay.
|
||||
// Two adoption policies, because what is at stake differs:
|
||||
// - against a live relay, hysteresis must prove direct stable before the swap;
|
||||
// - during a reconnect nothing is live, so this dial races the relay dial from
|
||||
// t=0 and the first authenticated socket is adopted outright.
|
||||
export class DirectReturnProbe {
|
||||
private timer: ReturnType<typeof setTimeout> | null = null
|
||||
|
||||
private stopped = false
|
||||
private activeProbe: AbortController | null = null
|
||||
// Soonest delay a caller asked for while a dial was in flight.
|
||||
private deferredDelayMs: number | null = null
|
||||
|
||||
constructor(
|
||||
private readonly deps: {
|
||||
@@ -25,7 +30,14 @@ export class DirectReturnProbe {
|
||||
hysteresis: MobileEndpointHysteresis
|
||||
host: () => HostProfile
|
||||
canSchedule: () => boolean
|
||||
// A dial is a pure observation on its own socket, so it only needs a live
|
||||
// supervisor; the cutover is the part that needs the operation mutex.
|
||||
canDial: () => boolean
|
||||
canAttempt: () => boolean
|
||||
// True while no session is live: the reconnect is a race, so an
|
||||
// authenticated direct socket wins without consulting hysteresis.
|
||||
adoptsOutright: () => boolean
|
||||
// Takes the supervisor's operation mutex, now held for the cutover only.
|
||||
beginOperation: () => void
|
||||
migrate: (
|
||||
client: RpcClient,
|
||||
@@ -38,7 +50,18 @@ export class DirectReturnProbe {
|
||||
) {}
|
||||
|
||||
schedule(delayMs = DIRECT_PROBE_INTERVAL_MS): void {
|
||||
if (this.stopped || !this.hooks.canSchedule() || this.timer) {
|
||||
if (this.stopped || !this.hooks.canSchedule()) {
|
||||
return
|
||||
}
|
||||
// Why: the dial no longer holds the supervisor's mutex, so nothing else stops a
|
||||
// second probe from overwriting activeProbe — stop() would then reach only the
|
||||
// newest socket and leave the earlier one dialing for its full 12s budget. The
|
||||
// in-flight probe owns the next slot and re-arms it on the soonest ask.
|
||||
if (this.activeProbe) {
|
||||
this.deferredDelayMs = Math.min(this.deferredDelayMs ?? delayMs, delayMs)
|
||||
return
|
||||
}
|
||||
if (this.timer) {
|
||||
return
|
||||
}
|
||||
this.timer = this.deps.setTimer(() => {
|
||||
@@ -47,7 +70,18 @@ export class DirectReturnProbe {
|
||||
}, delayMs)
|
||||
}
|
||||
|
||||
// Why: a reconnect races both paths from t=0, and schedule(0) yields to a
|
||||
// pending 15s tick — that would hand the relay dial a head start by another name.
|
||||
probeNow(): void {
|
||||
if (this.stopped || this.activeProbe) {
|
||||
return
|
||||
}
|
||||
this.clear()
|
||||
this.schedule(0)
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.deferredDelayMs = null
|
||||
if (this.timer) {
|
||||
this.deps.clearTimer(this.timer)
|
||||
this.timer = null
|
||||
@@ -64,15 +98,23 @@ export class DirectReturnProbe {
|
||||
if (this.stopped) {
|
||||
return
|
||||
}
|
||||
if (!this.hooks.canAttempt() || !this.hooks.hysteresis.canProbe(this.deps.now())) {
|
||||
// Why: the failure cooldown exists to stop a healthy relay flapping onto a
|
||||
// marginal LAN. With nothing connected there is no session to protect, and
|
||||
// honouring it would leave the phone waiting on relay alone.
|
||||
const racing = this.hooks.adoptsOutright()
|
||||
if (!this.hooks.canDial() || (!racing && !this.hooks.hysteresis.canProbe(this.deps.now()))) {
|
||||
this.schedule()
|
||||
return
|
||||
}
|
||||
const controller = new AbortController()
|
||||
this.activeProbe = controller
|
||||
this.hooks.beginOperation()
|
||||
let owned = false
|
||||
let successful: Awaited<ReturnType<typeof openAuthenticatedDirectEndpoint>> = null
|
||||
try {
|
||||
// Why: the dial is a pure observation on its own socket — holding the
|
||||
// supervisor's mutex across its 12s budget stalled every relay recovery
|
||||
// that landed during a foreground return, and makes the reconnect race
|
||||
// unwinnable while a relay dial holds it.
|
||||
successful = await openAuthenticatedDirectEndpoint(
|
||||
this.hooks.host(),
|
||||
this.deps.openDirect,
|
||||
@@ -86,17 +128,42 @@ export class DirectReturnProbe {
|
||||
this.hooks.hysteresis.recordDirectFailure(this.deps.now())
|
||||
return
|
||||
}
|
||||
if (!this.hooks.hysteresis.recordDirectSuccess(this.deps.now())) {
|
||||
successful.client.close()
|
||||
// Both early returns leave the candidate to the finally, which owns it until
|
||||
// migration takes over — closing here too would double-close it.
|
||||
const outright = this.hooks.adoptsOutright()
|
||||
// Why: a socket that entered the race and lost books nothing and leaves the
|
||||
// promotion streak untouched — the winner is this reconnect's whole verdict.
|
||||
if (!outright && (racing || !this.hooks.hysteresis.recordDirectSuccess(this.deps.now()))) {
|
||||
return
|
||||
}
|
||||
const mutexFree = this.hooks.canAttempt()
|
||||
if (!mutexFree && !outright) {
|
||||
// A relay dial owns the mutex; the streak survives, so the next probe
|
||||
// promotes direct instead of this one.
|
||||
return
|
||||
}
|
||||
if (mutexFree) {
|
||||
this.hooks.beginOperation()
|
||||
owned = true
|
||||
}
|
||||
// Why: when a relay dial holds the mutex the race still cuts over — that
|
||||
// dial withdraws itself in migrateTo and books no failure against relay.
|
||||
const candidate = successful
|
||||
// Migration owns the candidate, including closing it if cutover is canceled.
|
||||
successful = null
|
||||
// Why: the relay dial can authenticate between this socket's authentication
|
||||
// and the swap. migrateTo re-checks after auth, so the loser withdraws.
|
||||
const abortCutover = outright
|
||||
? (): boolean => this.stopped || !this.hooks.adoptsOutright()
|
||||
: (): boolean => this.stopped
|
||||
try {
|
||||
await this.hooks.migrate(candidate.client, candidate.path, () => this.stopped)
|
||||
await this.hooks.migrate(candidate.client, candidate.path, abortCutover)
|
||||
} catch (error) {
|
||||
if (this.stopped) {
|
||||
// Why: a withdrawn cutover is the ordinary end of a lost race, and
|
||||
// migrateTo has already closed the candidate. Only the timer calls this
|
||||
// method, and it discards the promise, so rethrowing here would surface
|
||||
// a routine loss as an unhandled rejection.
|
||||
if (this.stopped || abortCutover()) {
|
||||
return
|
||||
}
|
||||
throw error
|
||||
@@ -109,10 +176,14 @@ export class DirectReturnProbe {
|
||||
} finally {
|
||||
this.activeProbe = null
|
||||
successful?.client.close()
|
||||
// Why: a relay drop or backoff timer can arrive while the probe owns the
|
||||
// Why: a relay drop or backoff timer can arrive while the cutover owns the
|
||||
// operation mutex; afterProbe releases it and replays deferred recovery.
|
||||
this.hooks.afterProbe()
|
||||
this.schedule()
|
||||
if (owned) {
|
||||
this.hooks.afterProbe()
|
||||
}
|
||||
const deferred = this.deferredDelayMs
|
||||
this.deferredDelayMs = null
|
||||
this.schedule(deferred ?? undefined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -86,7 +86,7 @@ function createSupervisor(
|
||||
): MobileEndpointSupervisor {
|
||||
return new MobileEndpointSupervisor(logical, host, {
|
||||
openDirect: (endpoint) => connect(endpoint, host.deviceToken, host.publicKeyB64, { onLog }),
|
||||
openRelay: (relay, credential, confirmReqId, onHostCloseReason) =>
|
||||
openRelay: (relay, credential, confirmReqId, onHostCloseReason, isForeground) =>
|
||||
connectMobileRelayRpcSession({
|
||||
relay,
|
||||
resumeToken: credential.token,
|
||||
@@ -94,6 +94,7 @@ function createSupervisor(
|
||||
resumeConfirmReqId: confirmReqId,
|
||||
deviceToken: host.deviceToken,
|
||||
desktopPublicKeyB64: host.publicKeyB64,
|
||||
isForeground,
|
||||
onHostCloseReason,
|
||||
onLog
|
||||
}),
|
||||
|
||||
@@ -0,0 +1,362 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { MobileEndpointHysteresis } from './mobile-endpoint-hysteresis'
|
||||
import { MobileEndpointSupervisor } from './mobile-endpoint-supervisor'
|
||||
import { RelayOuterError } from './mobile-relay-e2ee-link'
|
||||
import {
|
||||
dependencies,
|
||||
FakeLogicalClient,
|
||||
FakeRelaySession,
|
||||
FakeSession,
|
||||
host,
|
||||
unreachableDirect
|
||||
} from './mobile-endpoint-supervisor-test-fakes'
|
||||
|
||||
vi.mock('react-native', () => ({ Platform: { OS: 'ios' } }))
|
||||
vi.mock('expo-secure-store', () => ({ WHEN_UNLOCKED_THIS_DEVICE_ONLY: 'when-unlocked' }))
|
||||
vi.mock('expo-crypto', () => ({ getRandomBytes: (length: number) => new Uint8Array(length) }))
|
||||
|
||||
// Holds the relay cutover open so the direct path can authenticate mid-dial. The
|
||||
// fake's migrateTo otherwise settles inside the dial, which no real cell does.
|
||||
function holdRelayCutover(logical: FakeLogicalClient): () => void {
|
||||
const settle = logical.migrateTo.getMockImplementation()!
|
||||
let release!: () => void
|
||||
const held = new Promise<void>((resolve) => {
|
||||
release = resolve
|
||||
})
|
||||
logical.migrateTo.mockImplementationOnce(async (session, path, timeoutMs, shouldAbort) => {
|
||||
await held
|
||||
// Why: replays the real post-authentication checks, so a superseded dial
|
||||
// still withdraws instead of stealing the client from the winner.
|
||||
return await settle(session, path, timeoutMs, shouldAbort)
|
||||
})
|
||||
return release
|
||||
}
|
||||
|
||||
// One full lost race: the relay dial starts, direct returns mid-cutover and wins.
|
||||
async function loseOneRace(
|
||||
logical: FakeLogicalClient,
|
||||
openRelay: ReturnType<typeof vi.fn>
|
||||
): Promise<void> {
|
||||
const before = openRelay.mock.calls.length
|
||||
const release = holdRelayCutover(logical)
|
||||
logical.publishState('reconnecting')
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(openRelay.mock.calls.length).toBe(before + 1)
|
||||
logical.publishState('connected')
|
||||
release()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
}
|
||||
|
||||
function relaySessionsFrom(openRelay: ReturnType<typeof vi.fn>): FakeRelaySession[] {
|
||||
return openRelay.mock.results.map((result) => result.value as FakeRelaySession)
|
||||
}
|
||||
|
||||
describe('mobile endpoint reconnect race', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
vi.setSystemTime(new Date('2026-07-13T12:00:00Z'))
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('dials relay at t=0 while the direct dial is still connecting', async () => {
|
||||
const logical = new FakeLogicalClient('connecting', 'lan')
|
||||
const deps = dependencies({ openDirect: unreachableDirect() })
|
||||
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
|
||||
|
||||
// No timer advance at all: an unfinished direct dial buys no head start.
|
||||
await supervisor.start()
|
||||
|
||||
expect(deps.openRelay).toHaveBeenCalledOnce()
|
||||
expect(logical.getActivePath()).toBe('relay')
|
||||
expect(logical.migrateTo).toHaveBeenCalledWith(
|
||||
expect.any(FakeRelaySession),
|
||||
'relay',
|
||||
undefined,
|
||||
expect.any(Function)
|
||||
)
|
||||
supervisor.stop()
|
||||
})
|
||||
|
||||
it('adopts the direct dial and withdraws the slower relay dial without booking it', async () => {
|
||||
const logical = new FakeLogicalClient('connecting', 'lan')
|
||||
const openRelay = vi.fn(() => new FakeRelaySession('connected'))
|
||||
const deps = dependencies({ openRelay, openDirect: unreachableDirect() })
|
||||
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
|
||||
|
||||
const release = holdRelayCutover(logical)
|
||||
const starting = supervisor.start()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(openRelay).toHaveBeenCalledOnce()
|
||||
|
||||
// The direct dial authenticates while the cell is still cutting over.
|
||||
logical.publishState('connected')
|
||||
release()
|
||||
await starting
|
||||
|
||||
expect(logical.getActivePath()).toBe('lan')
|
||||
expect(relaySessionsFrom(openRelay)[0]!.close).toHaveBeenCalled()
|
||||
// A withdrawn dial is not a failure: no cooldown is armed, so no redial lands.
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
expect(openRelay).toHaveBeenCalledOnce()
|
||||
expect(logical.setRecoveryPath).toHaveBeenLastCalledWith(null)
|
||||
supervisor.stop()
|
||||
})
|
||||
|
||||
it('adopts a direct socket that wins a reconnect the relay path started', async () => {
|
||||
const recordMigration = vi.spyOn(MobileEndpointHysteresis.prototype, 'recordMigration')
|
||||
const logical = new FakeLogicalClient('connected', 'relay')
|
||||
const openRelay = vi.fn(() => new FakeRelaySession('connected'))
|
||||
const deps = dependencies({ openRelay })
|
||||
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
|
||||
await supervisor.start()
|
||||
|
||||
const release = holdRelayCutover(logical)
|
||||
logical.publishState('disconnected')
|
||||
// The direct dial runs while the relay dial is still in flight and wins it.
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(deps.openDirect).toHaveBeenCalledOnce()
|
||||
expect(logical.getActivePath()).toBe('lan')
|
||||
|
||||
release()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(relaySessionsFrom(openRelay)[0]!.close).toHaveBeenCalled()
|
||||
// Hysteresis stamps the dwell, and the losing relay dial books no backoff.
|
||||
expect(recordMigration).toHaveBeenCalled()
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
expect(openRelay).toHaveBeenCalledOnce()
|
||||
supervisor.stop()
|
||||
})
|
||||
|
||||
it('books one backoff, not two, when both paths lose the reconnect', async () => {
|
||||
const recordDirectFailure = vi.spyOn(MobileEndpointHysteresis.prototype, 'recordDirectFailure')
|
||||
const logical = new FakeLogicalClient('connected', 'relay')
|
||||
const openRelay = vi.fn(() => new FakeRelaySession('disconnected', new RelayOuterError(4408)))
|
||||
const deps = dependencies({
|
||||
openRelay,
|
||||
openDirect: unreachableDirect(),
|
||||
randomBytes: () => new Uint8Array([128, 0])
|
||||
})
|
||||
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
|
||||
await supervisor.start()
|
||||
|
||||
logical.publishState('disconnected')
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(openRelay).toHaveBeenCalledOnce()
|
||||
expect(deps.openDirect).toHaveBeenCalledOnce()
|
||||
expect(recordDirectFailure).toHaveBeenCalledOnce()
|
||||
|
||||
// One failure, so one 250ms step. A double-booked loss would redial at 500ms.
|
||||
await vi.advanceTimersByTimeAsync(249)
|
||||
expect(openRelay).toHaveBeenCalledOnce()
|
||||
await vi.advanceTimersByTimeAsync(1)
|
||||
expect(openRelay).toHaveBeenCalledTimes(2)
|
||||
supervisor.stop()
|
||||
})
|
||||
|
||||
it('leaves the promotion streak alone when the direct socket loses the race', async () => {
|
||||
const recordDirectSuccess = vi.spyOn(MobileEndpointHysteresis.prototype, 'recordDirectSuccess')
|
||||
const logical = new FakeLogicalClient('connected', 'relay')
|
||||
const direct = new FakeSession('connecting')
|
||||
const openRelay = vi.fn(() => new FakeRelaySession('connected'))
|
||||
const deps = dependencies({ openRelay, openDirect: vi.fn(() => direct) })
|
||||
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
|
||||
await supervisor.start()
|
||||
|
||||
const release = holdRelayCutover(logical)
|
||||
logical.publishState('disconnected')
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(deps.openDirect).toHaveBeenCalledOnce()
|
||||
|
||||
// Relay authenticates first, then the direct socket finally answers.
|
||||
release()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(logical.getActivePath()).toBe('relay')
|
||||
direct.publishState('connected')
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
|
||||
expect(direct.close).toHaveBeenCalled()
|
||||
expect(recordDirectSuccess).not.toHaveBeenCalled()
|
||||
expect(logical.getActivePath()).toBe('relay')
|
||||
supervisor.stop()
|
||||
})
|
||||
|
||||
it('ignores a loser that closes after the winner has been adopted', async () => {
|
||||
const logical = new FakeLogicalClient('connecting', 'lan')
|
||||
const openRelay = vi.fn(() => new FakeRelaySession('connected'))
|
||||
const deps = dependencies({ openRelay, openDirect: unreachableDirect() })
|
||||
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
|
||||
|
||||
const release = holdRelayCutover(logical)
|
||||
const starting = supervisor.start()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
logical.publishState('connected')
|
||||
release()
|
||||
await starting
|
||||
expect(logical.getActivePath()).toBe('lan')
|
||||
|
||||
// The withdrawn cell socket reports its close afterwards.
|
||||
relaySessionsFrom(openRelay)[0]!.publishState('disconnected')
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
|
||||
expect(logical.getState()).toBe('connected')
|
||||
expect(logical.getActivePath()).toBe('lan')
|
||||
expect(openRelay).toHaveBeenCalledOnce()
|
||||
supervisor.stop()
|
||||
})
|
||||
|
||||
it('withdraws the relay socket before it authenticates once direct wins', async () => {
|
||||
const logical = new FakeLogicalClient('connecting', 'lan')
|
||||
const relaySession = new FakeRelaySession('connecting')
|
||||
const openRelay = vi.fn(() => relaySession)
|
||||
const deps = dependencies({ openRelay, openDirect: unreachableDirect() })
|
||||
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
|
||||
|
||||
const release = holdRelayCutover(logical)
|
||||
const starting = supervisor.start()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(openRelay).toHaveBeenCalledOnce()
|
||||
expect(relaySession.close).not.toHaveBeenCalled()
|
||||
|
||||
// The direct dial authenticates while the cell socket is still pre-handshake.
|
||||
// migrateTo would not withdraw until after E2EE auth, so the cell would have
|
||||
// reserved a splice and the desktop would have finished a handshake for it.
|
||||
logical.publishState('connected')
|
||||
expect(relaySession.close).toHaveBeenCalled()
|
||||
expect(relaySession.getState()).not.toBe('connected')
|
||||
|
||||
release()
|
||||
await starting
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
expect(logical.getActivePath()).toBe('lan')
|
||||
expect(openRelay).toHaveBeenCalledOnce()
|
||||
supervisor.stop()
|
||||
})
|
||||
|
||||
it('damps the race after a loss so a flapping LAN opens one cell socket', async () => {
|
||||
const logical = new FakeLogicalClient('connecting', 'lan')
|
||||
const openRelay = vi.fn(() => new FakeRelaySession('connecting'))
|
||||
const deps = dependencies({ openRelay, openDirect: unreachableDirect() })
|
||||
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
|
||||
|
||||
// The first blip races, and the returning direct dial wins it.
|
||||
const release = holdRelayCutover(logical)
|
||||
const starting = supervisor.start()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
logical.publishState('connected')
|
||||
release()
|
||||
await starting
|
||||
expect(openRelay).toHaveBeenCalledOnce()
|
||||
|
||||
// Two more blips inside the damper window open no further cell socket.
|
||||
for (const _blip of [1, 2]) {
|
||||
logical.publishState('reconnecting')
|
||||
await vi.advanceTimersByTimeAsync(100)
|
||||
logical.publishState('connected')
|
||||
await vi.advanceTimersByTimeAsync(400)
|
||||
}
|
||||
expect(openRelay).toHaveBeenCalledOnce()
|
||||
|
||||
// The window lapses against a live direct path, so it still opens nothing.
|
||||
await vi.advanceTimersByTimeAsync(10_000)
|
||||
expect(openRelay).toHaveBeenCalledOnce()
|
||||
supervisor.stop()
|
||||
})
|
||||
|
||||
it('races at once when the LAN dies inside a damper window grown to the cap', async () => {
|
||||
const logical = new FakeLogicalClient('connecting', 'lan')
|
||||
const openRelay = vi.fn(() => new FakeRelaySession('connecting'))
|
||||
const deps = dependencies({ openRelay, openDirect: unreachableDirect() })
|
||||
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
|
||||
|
||||
const release = holdRelayCutover(logical)
|
||||
const starting = supervisor.start()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
logical.publishState('connected')
|
||||
release()
|
||||
await starting
|
||||
|
||||
// Four more losses, each once its window has run: 2s, 4s, 8s, 16s, then the
|
||||
// fifth earns the 30s cap.
|
||||
for (const window of [2_000, 4_000, 8_000, 16_000]) {
|
||||
await vi.advanceTimersByTimeAsync(window)
|
||||
await loseOneRace(logical, openRelay)
|
||||
}
|
||||
expect(openRelay).toHaveBeenCalledTimes(5)
|
||||
|
||||
// This time direct does not come back. Waiting out the window a blip earned
|
||||
// would strand the phone offline for 30s with nothing else scheduled.
|
||||
logical.publishState('reconnecting')
|
||||
await vi.advanceTimersByTimeAsync(249)
|
||||
expect(openRelay).toHaveBeenCalledTimes(5)
|
||||
await vi.advanceTimersByTimeAsync(1)
|
||||
expect(openRelay.mock.calls.length).toBeGreaterThan(5)
|
||||
supervisor.stop()
|
||||
})
|
||||
|
||||
it('lets a foreground resume race immediately inside a damper window', async () => {
|
||||
const logical = new FakeLogicalClient('connecting', 'lan')
|
||||
const openRelay = vi.fn(() => new FakeRelaySession('connecting'))
|
||||
const deps = dependencies({ openRelay, openDirect: unreachableDirect() })
|
||||
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
|
||||
|
||||
const release = holdRelayCutover(logical)
|
||||
const starting = supervisor.start()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
logical.publishState('connected')
|
||||
release()
|
||||
await starting
|
||||
|
||||
logical.publishState('reconnecting')
|
||||
await vi.advanceTimersByTimeAsync(100)
|
||||
expect(openRelay).toHaveBeenCalledOnce()
|
||||
|
||||
// A resume is the user waiting on the screen; it never serves out the window.
|
||||
supervisor.setForeground(false)
|
||||
supervisor.setForeground(true)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
|
||||
expect(openRelay.mock.calls.length).toBeGreaterThan(1)
|
||||
supervisor.stop()
|
||||
})
|
||||
|
||||
it('starts no dial in the background and races both paths on resume', async () => {
|
||||
const logical = new FakeLogicalClient('connecting', 'lan')
|
||||
const deps = dependencies({ openDirect: unreachableDirect() })
|
||||
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
|
||||
|
||||
supervisor.setForeground(false)
|
||||
await supervisor.start()
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
expect(deps.openRelay).not.toHaveBeenCalled()
|
||||
|
||||
supervisor.setForeground(true)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
|
||||
expect(deps.openRelay).toHaveBeenCalledOnce()
|
||||
expect(logical.getActivePath()).toBe('relay')
|
||||
supervisor.stop()
|
||||
})
|
||||
|
||||
it('runs the resume probe against a relay that survived the background grace', async () => {
|
||||
const logical = new FakeLogicalClient('connected', 'relay')
|
||||
const deps = dependencies()
|
||||
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
|
||||
await supervisor.start()
|
||||
|
||||
supervisor.setForeground(false)
|
||||
await vi.advanceTimersByTimeAsync(1_000)
|
||||
expect(deps.openDirect).not.toHaveBeenCalled()
|
||||
|
||||
// A live relay is not a reconnect: the resume probe dials direct, but the
|
||||
// promotion still has to earn its hysteresis streak.
|
||||
supervisor.setForeground(true)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(deps.openDirect).toHaveBeenCalledOnce()
|
||||
expect(logical.getActivePath()).toBe('relay')
|
||||
expect(deps.openRelay).not.toHaveBeenCalled()
|
||||
supervisor.stop()
|
||||
})
|
||||
})
|
||||
@@ -12,7 +12,9 @@ export type MobileEndpointSupervisorDependencies = {
|
||||
relay: MobileRelayEndpoint,
|
||||
credential: { token: string; version: number },
|
||||
confirmReqId: string,
|
||||
onHostCloseReason?: (reason: RelayHostCloseReason) => void
|
||||
onHostCloseReason?: (reason: RelayHostCloseReason) => void,
|
||||
// Gates the session's idle liveness sweep; a backgrounded app spends no probes.
|
||||
isForeground?: () => boolean
|
||||
) => MobileRelayRpcSession
|
||||
resolveRelay: typeof resolveMobileRelayEndpoint
|
||||
readBundle: (hostId: string) => Promise<MobileRelayCredentialBundle | null>
|
||||
|
||||
@@ -1,13 +1,26 @@
|
||||
import { beforeEach, afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { MobileEndpointSupervisor } from './mobile-endpoint-supervisor'
|
||||
import { MobileEndpointHysteresis } from './mobile-endpoint-hysteresis'
|
||||
import {
|
||||
dependencies,
|
||||
FakeLogicalClient,
|
||||
FakeRelaySession,
|
||||
FakeSession,
|
||||
host
|
||||
host,
|
||||
unreachableDirect
|
||||
} from './mobile-endpoint-supervisor-test-fakes'
|
||||
|
||||
// A cell that authenticates and then answers the confirm for a different relay host
|
||||
// — what a rehomed desktop produces. The session fails after the logical cutover.
|
||||
function confirmRejectingRelaySession(logical: FakeLogicalClient): FakeRelaySession {
|
||||
const session = new FakeRelaySession('connected', new Error('relay resume confirmation missing'))
|
||||
session.whenResumeConfirmed = async () => {
|
||||
session.publishState('disconnected')
|
||||
logical.publishState('disconnected')
|
||||
}
|
||||
return session
|
||||
}
|
||||
|
||||
vi.mock('react-native', () => ({ Platform: { OS: 'ios' } }))
|
||||
vi.mock('expo-secure-store', () => ({ WHEN_UNLOCKED_THIS_DEVICE_ONLY: 'when-unlocked' }))
|
||||
vi.mock('expo-crypto', () => ({ getRandomBytes: (length: number) => new Uint8Array(length) }))
|
||||
@@ -48,4 +61,103 @@ describe('mobile endpoint supervisor direct probe', () => {
|
||||
expect(logical.getActivePath()).toBe('relay')
|
||||
supervisor.stop()
|
||||
})
|
||||
|
||||
it('recovers the relay at once while the probe is still dialing direct', async () => {
|
||||
const logical = new FakeLogicalClient('connected', 'relay')
|
||||
// A black-holed LAN endpoint: the dial sits unanswered for its whole 12s budget.
|
||||
const direct = new FakeSession('connecting')
|
||||
const openRelay = vi.fn(() => new FakeRelaySession('connected'))
|
||||
const deps = dependencies({ openDirect: vi.fn(() => direct), openRelay })
|
||||
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
|
||||
await supervisor.start()
|
||||
|
||||
await vi.advanceTimersByTimeAsync(15_000)
|
||||
expect(deps.openDirect).toHaveBeenCalledOnce()
|
||||
logical.publishState('disconnected')
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
|
||||
// Why: the dial is a pure observation, so it no longer owns the operation
|
||||
// mutex — recovery does not wait out the probe's budget.
|
||||
expect(openRelay).toHaveBeenCalledOnce()
|
||||
expect(logical.getState()).toBe('connected')
|
||||
expect(logical.getActivePath()).toBe('relay')
|
||||
supervisor.stop()
|
||||
})
|
||||
|
||||
it('backs off a dial whose resume confirm fails after the cutover', async () => {
|
||||
const recordMigration = vi.spyOn(MobileEndpointHysteresis.prototype, 'recordMigration')
|
||||
const logical = new FakeLogicalClient('disconnected', 'lan')
|
||||
const openRelay = vi.fn(() => confirmRejectingRelaySession(logical))
|
||||
// No LAN to race: this is about the relay cadence after a confirm failure.
|
||||
const deps = dependencies({
|
||||
openRelay,
|
||||
openDirect: unreachableDirect(),
|
||||
randomBytes: () => new Uint8Array([128, 0])
|
||||
})
|
||||
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
|
||||
|
||||
await supervisor.start()
|
||||
// Two sockets per pass: a confirm mismatch reads as a stale cell assignment, so
|
||||
// the existing director fallback re-resolves and dials the authoritative target.
|
||||
expect(openRelay).toHaveBeenCalledTimes(2)
|
||||
expect(logical.migrateTo).toHaveBeenCalledTimes(2)
|
||||
|
||||
// Why: `connected` is published at authentication, so the cutover happens before
|
||||
// the confirm answers. A confirm that then fails must still book the shared
|
||||
// cooldown — reporting it as an established dial redials in a tight loop.
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(openRelay).toHaveBeenCalledTimes(2)
|
||||
|
||||
// 250ms, then 500ms, then 1000ms: the streak grows instead of resetting, which
|
||||
// it could not do if setActiveSession had run for this dying session.
|
||||
await vi.advanceTimersByTimeAsync(249)
|
||||
expect(openRelay).toHaveBeenCalledTimes(2)
|
||||
await vi.advanceTimersByTimeAsync(1)
|
||||
expect(openRelay).toHaveBeenCalledTimes(4)
|
||||
await vi.advanceTimersByTimeAsync(250)
|
||||
expect(openRelay).toHaveBeenCalledTimes(4)
|
||||
await vi.advanceTimersByTimeAsync(250)
|
||||
expect(openRelay).toHaveBeenCalledTimes(6)
|
||||
await vi.advanceTimersByTimeAsync(999)
|
||||
expect(openRelay).toHaveBeenCalledTimes(6)
|
||||
await vi.advanceTimersByTimeAsync(1)
|
||||
expect(openRelay).toHaveBeenCalledTimes(8)
|
||||
|
||||
// No session whose confirm failed is ever booked as a migration.
|
||||
expect(recordMigration).not.toHaveBeenCalled()
|
||||
supervisor.stop()
|
||||
})
|
||||
|
||||
it('replays a relay recovery that landed while the direct cutover owned the mutex', async () => {
|
||||
const logical = new FakeLogicalClient('connected', 'relay')
|
||||
const openRelay = vi.fn(() => new FakeRelaySession('connected'))
|
||||
const deps = dependencies({ openDirect: vi.fn(() => new FakeSession('connected')), openRelay })
|
||||
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
|
||||
await supervisor.start()
|
||||
|
||||
let release!: () => void
|
||||
const cutover = new Promise<void>((resolve) => {
|
||||
release = resolve
|
||||
})
|
||||
// The candidate loses the cutover, so the logical client stays on the relay path.
|
||||
logical.migrateTo.mockImplementationOnce(async (candidate) => {
|
||||
await cutover
|
||||
candidate.close()
|
||||
})
|
||||
// Three authenticated probes plus the observation and dwell windows.
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
expect(logical.migrateTo).toHaveBeenCalledOnce()
|
||||
|
||||
logical.publishState('disconnected')
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(openRelay).not.toHaveBeenCalled()
|
||||
|
||||
release()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
|
||||
// The queued request is replayed by afterProbe, never dropped.
|
||||
expect(openRelay).toHaveBeenCalledOnce()
|
||||
expect(logical.getState()).toBe('connected')
|
||||
supervisor.stop()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -65,6 +65,7 @@ export class FakeRelaySession extends FakeSession implements MobileRelayRpcSessi
|
||||
renewed: this.renewed,
|
||||
resumeExpiresAt: this.resumeExpiry
|
||||
})
|
||||
whenResumeConfirmed = () => Promise.resolve()
|
||||
getFailure = () => this.failure
|
||||
}
|
||||
|
||||
@@ -203,6 +204,15 @@ export const bundle: MobileRelayCredentialBundle = {
|
||||
}
|
||||
}
|
||||
|
||||
// Why: LAN unreachable. A throwing open beats a never-answering socket — the
|
||||
// direct dial resolves synchronously, so a relay-only test leaves no probe timer
|
||||
// behind and the reconnect race has exactly one runner.
|
||||
export function unreachableDirect(): MobileEndpointSupervisorDependencies['openDirect'] {
|
||||
return vi.fn(() => {
|
||||
throw new Error('direct endpoint unreachable')
|
||||
})
|
||||
}
|
||||
|
||||
export function dependencies(
|
||||
overrides: Partial<MobileEndpointSupervisorDependencies> = {}
|
||||
): MobileEndpointSupervisorDependencies {
|
||||
|
||||
@@ -10,7 +10,8 @@ import {
|
||||
FakeSession,
|
||||
host,
|
||||
mockCredentialRotation,
|
||||
relay
|
||||
relay,
|
||||
unreachableDirect
|
||||
} from './mobile-endpoint-supervisor-test-fakes'
|
||||
import { MobileEndpointSupervisor } from './mobile-endpoint-supervisor'
|
||||
|
||||
@@ -48,19 +49,17 @@ describe('mobile endpoint supervisor', () => {
|
||||
supervisor.stop()
|
||||
})
|
||||
|
||||
it('fails over when the direct retry loop publishes reconnecting', async () => {
|
||||
it('fails over while the direct retry loop is still dialing', async () => {
|
||||
const logical = new FakeLogicalClient('connecting', 'lan')
|
||||
const deps = dependencies()
|
||||
const deps = dependencies({ openDirect: unreachableDirect() })
|
||||
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
|
||||
await supervisor.start()
|
||||
|
||||
// An unfinished direct dial no longer holds relay back, so the failover has
|
||||
// already happened by the time the direct client gives up.
|
||||
logical.publishState('handshaking')
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(deps.openRelay).not.toHaveBeenCalled()
|
||||
|
||||
supervisor.setForeground(true)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(deps.openRelay).not.toHaveBeenCalled()
|
||||
expect(deps.openRelay).toHaveBeenCalledOnce()
|
||||
|
||||
logical.publishState('reconnecting')
|
||||
await vi.waitFor(() => expect(logical.getActivePath()).toBe('relay'))
|
||||
@@ -148,11 +147,12 @@ describe('mobile endpoint supervisor', () => {
|
||||
expect(logical.getPendingPath()).toBeNull()
|
||||
})
|
||||
|
||||
it('does not spend a queued relay retry while direct authentication is progressing', async () => {
|
||||
it('keeps retrying relay on its own cadence while a direct handshake drags on', async () => {
|
||||
const logical = new FakeLogicalClient('disconnected', 'lan')
|
||||
const openRelay = vi.fn(() => new FakeRelaySession('disconnected', new RelayOuterError(4408)))
|
||||
const deps = dependencies({
|
||||
openRelay,
|
||||
openDirect: unreachableDirect(),
|
||||
randomBytes: () => new Uint8Array([128, 0])
|
||||
})
|
||||
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
|
||||
@@ -160,12 +160,13 @@ describe('mobile endpoint supervisor', () => {
|
||||
await supervisor.start()
|
||||
expect(openRelay).toHaveBeenCalledOnce()
|
||||
|
||||
// A direct dial that reaches 'handshaking' and stays there used to park relay
|
||||
// recovery until it gave up; the retry now runs on the failure cadence alone.
|
||||
logical.publishState('handshaking')
|
||||
await vi.advanceTimersByTimeAsync(250)
|
||||
await vi.advanceTimersByTimeAsync(249)
|
||||
expect(openRelay).toHaveBeenCalledOnce()
|
||||
|
||||
logical.publishState('disconnected')
|
||||
await vi.waitFor(() => expect(openRelay).toHaveBeenCalledTimes(2))
|
||||
await vi.advanceTimersByTimeAsync(1)
|
||||
expect(openRelay).toHaveBeenCalledTimes(2)
|
||||
supervisor.stop()
|
||||
})
|
||||
|
||||
@@ -189,6 +190,7 @@ describe('mobile endpoint supervisor', () => {
|
||||
resolved,
|
||||
expect.any(Object),
|
||||
expect.any(String),
|
||||
expect.any(Function),
|
||||
expect.any(Function)
|
||||
)
|
||||
expect(deps.saveHost).toHaveBeenCalledWith(
|
||||
@@ -243,6 +245,7 @@ describe('mobile endpoint supervisor', () => {
|
||||
const deps = dependencies({
|
||||
openRelay,
|
||||
onLog,
|
||||
openDirect: unreachableDirect(),
|
||||
randomBytes: () => new Uint8Array([128, 0])
|
||||
})
|
||||
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
|
||||
@@ -287,6 +290,7 @@ describe('mobile endpoint supervisor', () => {
|
||||
const openRelay = vi.fn(() => new FakeRelaySession('connected', new RelayOuterError(4408)))
|
||||
const deps = dependencies({
|
||||
openRelay,
|
||||
openDirect: unreachableDirect(),
|
||||
randomBytes: () => new Uint8Array([128, 0])
|
||||
})
|
||||
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
|
||||
@@ -469,6 +473,7 @@ describe('mobile endpoint supervisor', () => {
|
||||
.mockImplementation(() => new FakeRelaySession('connected'))
|
||||
const deps = dependencies({
|
||||
openRelay,
|
||||
openDirect: unreachableDirect(),
|
||||
writeBundle: vi.fn(() => writePending),
|
||||
randomBytes: () => new Uint8Array([128, 0])
|
||||
})
|
||||
@@ -562,6 +567,7 @@ describe('mobile endpoint supervisor', () => {
|
||||
relay,
|
||||
expect.objectContaining({ version: 3 }),
|
||||
expect.any(String),
|
||||
expect.any(Function),
|
||||
expect.any(Function)
|
||||
)
|
||||
supervisor.stop()
|
||||
@@ -610,6 +616,7 @@ describe('mobile endpoint supervisor', () => {
|
||||
relay,
|
||||
expect.objectContaining({ version: 3 }),
|
||||
expect.any(String),
|
||||
expect.any(Function),
|
||||
expect.any(Function)
|
||||
)
|
||||
supervisor.stop()
|
||||
@@ -805,6 +812,7 @@ describe('mobile endpoint supervisor', () => {
|
||||
.mockImplementation(() => new FakeRelaySession('connected'))
|
||||
const deps = dependencies({
|
||||
openRelay,
|
||||
openDirect: unreachableDirect(),
|
||||
randomBytes: () => new Uint8Array([128, 0])
|
||||
})
|
||||
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
|
||||
@@ -841,43 +849,6 @@ describe('mobile endpoint supervisor', () => {
|
||||
supervisor.stop()
|
||||
})
|
||||
|
||||
it('races a relay dial when the direct dial stalls unauthenticated', async () => {
|
||||
const logical = new FakeLogicalClient('connecting', 'lan')
|
||||
const deps = dependencies()
|
||||
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
|
||||
|
||||
await supervisor.start()
|
||||
await vi.advanceTimersByTimeAsync(2_499)
|
||||
expect(deps.openRelay).not.toHaveBeenCalled()
|
||||
expect(logical.getState()).toBe('connecting')
|
||||
|
||||
// The direct dial never authenticates; the relay wins the race through migrateTo.
|
||||
await vi.advanceTimersByTimeAsync(1)
|
||||
await vi.waitFor(() => expect(logical.getActivePath()).toBe('relay'))
|
||||
expect(logical.migrateTo).toHaveBeenCalledWith(
|
||||
expect.any(FakeRelaySession),
|
||||
'relay',
|
||||
undefined,
|
||||
expect.any(Function)
|
||||
)
|
||||
supervisor.stop()
|
||||
})
|
||||
|
||||
it('cancels the grace race when the direct dial authenticates first', async () => {
|
||||
const logical = new FakeLogicalClient('connecting', 'lan')
|
||||
const deps = dependencies()
|
||||
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
|
||||
|
||||
await supervisor.start()
|
||||
logical.publishState('connected')
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5_000)
|
||||
expect(deps.openRelay).not.toHaveBeenCalled()
|
||||
expect(logical.getActivePath()).toBe('lan')
|
||||
supervisor.stop()
|
||||
})
|
||||
|
||||
it('never races a relay dial against a desktop with no relay endpoint', async () => {
|
||||
const logical = new FakeLogicalClient('connecting', 'lan')
|
||||
const deps = dependencies()
|
||||
@@ -890,39 +861,4 @@ describe('mobile endpoint supervisor', () => {
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
supervisor.stop()
|
||||
})
|
||||
|
||||
it('drops the pending grace race when the phone backgrounds', async () => {
|
||||
const logical = new FakeLogicalClient('connecting', 'lan')
|
||||
const deps = dependencies()
|
||||
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
|
||||
|
||||
await supervisor.start()
|
||||
supervisor.setForeground(false)
|
||||
await vi.advanceTimersByTimeAsync(5_000)
|
||||
|
||||
expect(deps.openRelay).not.toHaveBeenCalled()
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
supervisor.stop()
|
||||
})
|
||||
|
||||
it('books the shared cooldown when the grace race loses its dial', async () => {
|
||||
const logical = new FakeLogicalClient('connecting', 'lan')
|
||||
const openRelay = vi.fn(() => new FakeRelaySession('disconnected', new RelayOuterError(4408)))
|
||||
const deps = dependencies({ openRelay, randomBytes: () => new Uint8Array([128, 0]) })
|
||||
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
|
||||
|
||||
await supervisor.start()
|
||||
await vi.advanceTimersByTimeAsync(2_500)
|
||||
expect(openRelay).toHaveBeenCalledOnce()
|
||||
|
||||
// The armed retry runs unforced, so it yields to the still-progressing direct
|
||||
// dial: the race gets one attempt, never a socket-per-cooldown loop.
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
expect(openRelay).toHaveBeenCalledOnce()
|
||||
|
||||
// Direct finally gives up: ordinary recovery still owns the failure.
|
||||
logical.publishState('reconnecting')
|
||||
await vi.waitFor(() => expect(openRelay).toHaveBeenCalledTimes(2))
|
||||
supervisor.stop()
|
||||
})
|
||||
})
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user